diff --git a/arch/sw_64/Kbuild b/arch/sw_64/Kbuild new file mode 100644 index 0000000000000000000000000000000000000000..aa0bf0507406c9790b10d0b0c0b6dd57d22286ae --- /dev/null +++ b/arch/sw_64/Kbuild @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-y += kernel/ mm/ platform/ +obj-$(CONFIG_NET) += net/ +obj-$(CONFIG_KVM) += kvm/ +obj-$(CONFIG_MATHEMU) += math-emu/ + +obj-$(CONFIG_BUILTIN_DTB) += boot/dts/ diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 6067b0e47c9ca3a1227208d87221190773ede280..a28344f3316086c702d582eed493055823cb8277 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -105,9 +105,14 @@ config SW64 select NO_BOOTMEM select OF_EARLY_FLATTREE if OF select OLD_SIGSUSPEND - select PCI_MSI_ARCH_FALLBACKS + select PCI_MSI_ARCH_FALLBACKS if PCI_MSI + select PCI_SW64 if PCI select SET_FS select SPARSEMEM_EXTREME if SPARSEMEM + select SW64_IRQ_CPU + select SW64_IRQ_MSI if PCI_MSI + select SW64_IRQ_MSI_VT if PCI_MSI + select SW64_TIMER select SWIOTLB select THREAD_INFO_IN_TASK @@ -128,13 +133,6 @@ config PGTABLE_LEVELS config SYS_SUPPORTS_HUGETLBFS def_bool y -config RWSEM_GENERIC_SPINLOCK - bool - -config RWSEM_XCHGADD_ALGORITHM - bool - default y - config ARCH_ENABLE_MEMORY_HOTPLUG bool default y @@ -187,6 +185,9 @@ config AUDIT_ARCH config SYS_HAS_EARLY_PRINTK bool +config HAVE_CSRRW + bool + menu "System setup" menu "Machine Configuration" @@ -197,63 +198,48 @@ choice config SUBARCH_C3B bool "C3B" +config SUBARCH_C4 + bool "C4" + select HAVE_CSRRW + select GENERIC_SCHED_CLOCK endchoice choice - prompt "Chipset Family" + prompt "Uncore Configuration" -config SW64_CHIP3 - bool "Chip3" +config UNCORE_XUELANG + bool "Uncore for C3B" depends on SUBARCH_C3B -endchoice - -choice - prompt "Runtime System" - depends on SW64_CHIP3 - default SW64_ASIC - -config SW64_FPGA - bool "FPGA" - help - Support for chip3 FPGA. - -config SW64_SIM - bool "Hardware Simulator" help - Support for chip3 hardware simulator. + Sunway cpu uncore for C3B -config SW64_ASIC - bool "ASIC" +config UNCORE_JUNZHANG + bool "Uncore for C4" + depends on SUBARCH_C4 help - Support for chip3 asic. - + Sunway cpu uncore for C4 endchoice -config SW64_CHIP3_ASIC_DEBUG - bool "Debug Support for Chip3 Asic" - depends on SW64_ASIC - help - Used for debug - -config CPUFREQ_DEBUGFS - bool "CPU Frequency debugfs interface for Chip3 Asic" - depends on SW64_CHIP3 && DEBUG_FS - help - Turns on the DebugFS interface for CPU Frequency. - - If you don't know what to do here, say N. - choice prompt "Platform Type" config PLATFORM_XUELANG bool "Xuelang" - depends on SW64_CHIP3 + depends on UNCORE_XUELANG select SPARSE_IRQ select SYS_HAS_EARLY_PRINTK select SW64_INTC_V2 + select I2C_SUNWAY if I2C help - Sunway chip3 board chipset + Sunway board chipset for C3B + +config PLATFORM_JUNZHANG + bool "JunZhang" + depends on UNCORE_JUNZHANG + select SPARSE_IRQ + select SYS_HAS_EARLY_PRINTK + help + Sunway board chipset for C4 endchoice @@ -264,10 +250,7 @@ config MIGHT_HAVE_PC_SERIO endmenu -config LOCK_MEMB - bool "Insert mem barrier before lock instruction" - default y - +menu "CPU Power Management" source "drivers/cpufreq/Kconfig" config SW64_CPUAUTOPLUG @@ -277,6 +260,7 @@ config SW64_CPUAUTOPLUG help Turns on the interface for SW64_CPU CPUAUTOPLUG. +endmenu # clear all implied options (don't want default values for those): # Most of these machines have ISA slots; not exactly sure which don't, # and this doesn't activate hordes of code, so do it always. @@ -361,13 +345,6 @@ config GENERIC_HWEIGHT bool default y -config LOCK_FIXUP - bool "fix up the lock" - depends on SW64 - help - Add an instruction("memb\n") to ensure the correctness of the lock. - - config SMP bool "Symmetric multi-processing support" depends on SW64 @@ -402,11 +379,19 @@ config TRACE_IRQFLAGS_SUPPORT config ARCH_SUPPORTS_UPROBES def_bool y +config SCHED_SMT + bool "SMT scheduler support" + depends on SMP && SUBARCH_C4 + help + Improves the CPU scheduler's decision making when dealing with + MultiThreading at a cost of slightly increased overhead in some + places. If unsure say N here. + config NR_CPUS int "Maximum number of CPUs (2-256)" range 2 256 depends on SMP - default "64" if SW64_CHIP3 + default "64" if UNCORE_XUELANG help SW6 support can handle a maximum of 256 CPUs. @@ -519,16 +504,16 @@ config USE_OF help Include support for flattened device tree machine descriptions. -config SW64_BUILTIN_DTB +config BUILTIN_DTB bool "Embed DTB in kernel image" depends on OF default n help Embeds a device tree binary in the kernel image. -config SW64_BUILTIN_DTB_NAME +config BUILTIN_DTB_NAME string "Built in DTB" - depends on SW64_BUILTIN_DTB + depends on BUILTIN_DTB help Set the name of the DTB to embed, leave blank to pick one automatically based on kernel configuration. @@ -642,23 +627,6 @@ config ARCH_HIBERNATION_POSSIBLE depends on SW64 def_bool y -config SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE - depends on SUSPEND - bool "SW64 non bootcore suspend into deep sleep mode" - default n - -config SW64_SUSPEND_DEEPSLEEP_BOOTCORE - depends on SUSPEND - bool "SW64 bootcore suspend into deep sleep mode" - default n - -config SW64_SUPPORT_S3_SLEEPING_STATE - depends on SUSPEND - bool "SW64 support S3 sleeping state" - default n - help - Only SW831 support S3 sleep option and needs SROM, HMCode and BIOS support. - source "drivers/cpuidle/Kconfig" source "drivers/idle/Kconfig" diff --git a/arch/sw_64/Kconfig.debug b/arch/sw_64/Kconfig.debug index 2cd2036e0996cc15d0e02539251f5f41d367c31f..6cb3c2488b368e3acc38cd32b33fe658036ac8d6 100644 --- a/arch/sw_64/Kconfig.debug +++ b/arch/sw_64/Kconfig.debug @@ -44,3 +44,10 @@ config SW64_RRK help Duplicate kernel log to specific space. Do not enable it in a production kernel. + +config DEBUG_MATCH + bool "instruction-flow and data-flow match debugfs interface" + depends on DEBUG_FS + default n + help + Turns on the DebugFS interface for instruction-flow and data-flow match. diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index 7d86e80362f69b24cbce973da7da1179ded57f03..7b2ef781b1b1f18f1133e54c681b616b97e06756 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -26,24 +26,22 @@ endif CHECKFLAGS += -D__sw__ cflags-y := -pipe -ffixed-8 -mno-fp-regs #-msmall-data +ifeq ($(CONFIG_SUBARCH_C4),y) + cflags-y += -fsw-rev +endif cflags-y += $(call cc-option, -fno-jump-tables) cflags-y += $(cpuflags-y) KBUILD_CFLAGS += $(cflags-y) -KBUILD_DEFCONFIG = defconfig +KBUILD_DEFCONFIG = openeuler_defconfig head-y := arch/sw_64/kernel/head.o -core-y += arch/sw_64/kernel/ arch/sw_64/mm/ -core-y += arch/sw_64/platform/ -core-y += arch/sw_64/chip/ -core-$(CONFIG_MATHEMU) += arch/sw_64/math-emu/ +core-y += arch/sw_64/ drivers-$(CONFIG_OPROFILE) += arch/sw_64/oprofile/ +drivers-$(CONFIG_PCI) += arch/sw_64/pci/ libs-y += arch/sw_64/lib/ -core-$(CONFIG_KVM) += arch/sw_64/kvm/ -core-$(CONFIG_SW64_BUILTIN_DTB) += arch/sw_64/boot/dts/ -core-$(CONFIG_NET) += arch/sw_64/net/ # export what is needed by arch/sw_64/boot/Makefile LIBS_Y := $(patsubst %/, %/lib.a, $(libs-y)) diff --git a/arch/sw_64/boot/dts/Makefile b/arch/sw_64/boot/dts/Makefile index b9834c70be220fcecbb209ceff2f45a043d565a3..e32c159cab641fb82d09f0e9eaeb428c7770f006 100644 --- a/arch/sw_64/boot/dts/Makefile +++ b/arch/sw_64/boot/dts/Makefile @@ -1,11 +1,17 @@ # SPDX-License-Identifier: GPL-2.0 # Built-in dtb +ifeq ($(CONFIG_PLATFORM_XUELANG),y) builtindtb-y := chip3 +endif + +ifeq ($(CONFIG_PLATFORM_JUNZHANG),y) +builtindtb-y := empty +endif -ifeq ($(CONFIG_SW64_BUILTIN_DTB), y) -ifneq ($(CONFIG_SW64_BUILTIN_DTB_NAME),"") - builtindtb-y := $(patsubst "%",%,$(CONFIG_SW64_BUILTIN_DTB_NAME)) +ifeq ($(CONFIG_BUILTIN_DTB), y) +ifneq ($(CONFIG_BUILTIN_DTB_NAME),"") + builtindtb-y := $(patsubst "%",%,$(CONFIG_BUILTIN_DTB_NAME)) endif obj-y += $(builtindtb-y).dtb.o diff --git a/arch/sw_64/boot/dts/empty.dts b/arch/sw_64/boot/dts/empty.dts new file mode 100644 index 0000000000000000000000000000000000000000..f8fe34e29641f78dfdc1cc163b63d9becfcaeacd --- /dev/null +++ b/arch/sw_64/boot/dts/empty.dts @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Default device tree; + */ + +/dts-v1/; +/ { + compatible = "sunway,chip3"; + model = "chip3"; + #address-cells = <2>; + #size-cells = <2>; + + soc { + }; +}; diff --git a/arch/sw_64/chip/Makefile b/arch/sw_64/chip/Makefile deleted file mode 100644 index a64818cdf35bfcd252b6c7ef2af1ef7d6a5b7b37..0000000000000000000000000000000000000000 --- a/arch/sw_64/chip/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_SW64_CHIP3) := chip3/ diff --git a/arch/sw_64/chip/chip3/Makefile b/arch/sw_64/chip/chip3/Makefile deleted file mode 100644 index ba0ab3f67f98faef6232654ee5f5e90f37262e70..0000000000000000000000000000000000000000 --- a/arch/sw_64/chip/chip3/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -obj-y := chip.o i2c-lib.o - -obj-$(CONFIG_PCI) += pci-quirks.o -obj-$(CONFIG_PCI_MSI) += msi.o vt_msi.o -obj-$(CONFIG_CPUFREQ_DEBUGFS) += cpufreq_debugfs.o diff --git a/arch/sw_64/chip/chip3/pci-quirks.c b/arch/sw_64/chip/chip3/pci-quirks.c deleted file mode 100644 index 22887d269fe38c6d2328b2f72618b087ce63dad7..0000000000000000000000000000000000000000 --- a/arch/sw_64/chip/chip3/pci-quirks.c +++ /dev/null @@ -1,244 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -#include - -static int handshake(void __iomem *ptr, u32 mask, u32 done, - int wait_usec, int delay_usec) -{ - u32 result; - - do { - result = readl(ptr); - result &= mask; - if (result == done) - return 0; - udelay(delay_usec); - wait_usec -= delay_usec; - } while (wait_usec > 0); - return -ETIMEDOUT; -} - -#define XHCI_HCC_EXT_CAPS(p) (((p) >> 16) & 0xffff) -#define XHCI_EXT_CAPS_ID(p) (((p) >> 0) & 0xff) -#define XHCI_EXT_CAPS_NEXT(p) (((p) >> 8) & 0xff) -#define XHCI_HC_LENGTH(p) (((p) >> 0) & 0x00ff) -#define XHCI_CMD_OFFSET (0x00) -#define XHCI_STS_OFFSET (0x04) -#define XHCI_EXT_CAPS_LEGACY (1) -#define XHCI_HCC_PARAMS_OFFSET (0x10) -#define XHCI_LEGACY_CONTROL_OFFSET (0x04) -#define XHCI_LEGACY_DISABLE_SMI ((0x7 << 1) + (0xff << 5) + (0x7 << 17)) -#define XHCI_LEGACY_SMI_EVENTS (0x7 << 29) -#define XHCI_HC_BIOS_OWNED (1 << 16) -#define XHCI_HC_OS_OWNED (1 << 24) -#define XHCI_CMD_RUN (1 << 0) -#define XHCI_STS_HALT (1 << 0) -#define XHCI_MAX_HALT_USEC (16 * 1000) -#define XHCI_CMD_EIE (1 << 2) -#define XHCI_CMD_HSEIE (1 << 3) -#define XHCI_CMD_EWE (1 << 10) -#define XHCI_IRQS (XHCI_CMD_EIE | XHCI_CMD_HSEIE | XHCI_CMD_EWE) -#define XHCI_STS_CNR (1 << 11) -#define STS_FATAL (1 << 2) -#define STS_EINT (1 << 3) -#define STS_PORT (1 << 4) -#define STS_SRE (1 << 10) -#define STS_RW1C_BITS (STS_FATAL | STS_EINT | STS_PORT | STS_SRE) - -static inline int xhci_find_next_ext_cap(void __iomem *base, u32 start, int id) -{ - u32 val; - u32 next; - u32 offset; - - offset = start; - if (!start || start == XHCI_HCC_PARAMS_OFFSET) { - val = readl(base + XHCI_HCC_PARAMS_OFFSET); - if (val == ~0) - return 0; - offset = XHCI_HCC_EXT_CAPS(val) << 2; - if (!offset) - return 0; - } - do { - val = readl(base + offset); - if (val == ~0) - return 0; - if (offset != start && (id == 0 || XHCI_EXT_CAPS_ID(val) == id)) - return offset; - - next = XHCI_EXT_CAPS_NEXT(val); - offset += next << 2; - } while (next); - - return 0; -} - -extern void usb_enable_intel_xhci_ports(struct pci_dev *xhci_pdev); - -static void -fixup_usb_xhci_reset(struct pci_dev *dev) -{ - void __iomem *op_reg_base; - int timeout; - u32 xhci_command; - u32 tmp, val; - void __iomem *base; - struct pci_controller *hose = dev->sysdata; - unsigned long offset; - int ext_cap_offset; - int retries = 3; - - pci_read_config_dword(dev, PCI_COMMAND, &tmp); - tmp |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); - pci_write_config_dword(dev, PCI_COMMAND, tmp); - - pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, &tmp); - if (tmp & PCI_BASE_ADDRESS_MEM_TYPE_MASK) { - pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, &val); - offset = (unsigned long)(val) << 32 | (tmp & (~0xf)); - } else - offset = (unsigned long)(tmp & (~0xf)); - - if (offset == 0) - return; - - base = (void *)__va(SW64_PCI_IO_BASE(hose->node, hose->index) | offset); - - ext_cap_offset = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_LEGACY); - if (!ext_cap_offset) - goto hc_init; - - val = readl(base + ext_cap_offset); - - if ((dev->vendor == PCI_VENDOR_ID_TI && dev->device == 0x8241) || - (dev->vendor == PCI_VENDOR_ID_RENESAS - && dev->device == 0x0014)) { - val = (val | XHCI_HC_OS_OWNED) & ~XHCI_HC_BIOS_OWNED; - writel(val, base + ext_cap_offset); - } - - if (val & XHCI_HC_BIOS_OWNED) { - writel(val | XHCI_HC_OS_OWNED, base + ext_cap_offset); - - timeout = handshake(base + ext_cap_offset, XHCI_HC_BIOS_OWNED, - 0, 1000000, 10); - if (timeout) { - pr_err("xHCI BIOS handoff failed (BIOS bug ?) %08x\n", val); - writel(val & ~XHCI_HC_BIOS_OWNED, base + ext_cap_offset); - } - } - - val = readl(base + ext_cap_offset + XHCI_LEGACY_CONTROL_OFFSET); - val &= XHCI_LEGACY_DISABLE_SMI; - val |= XHCI_LEGACY_SMI_EVENTS; - writel(val, base + ext_cap_offset + XHCI_LEGACY_CONTROL_OFFSET); - -hc_init: - if (dev->vendor == PCI_VENDOR_ID_INTEL) - usb_enable_intel_xhci_ports(dev); - - op_reg_base = base + XHCI_HC_LENGTH(readl(base)); - - timeout = handshake(op_reg_base + XHCI_STS_OFFSET, XHCI_STS_CNR, 0, - 5000000, 10); - if (timeout) { - val = readl(op_reg_base + XHCI_STS_OFFSET); - pr_err("xHCI HW not ready after 5 sec (HC bug?) status = 0x%x\n", val); - } - - xhci_command = readl(op_reg_base + XHCI_CMD_OFFSET); - xhci_command |= 0x2; - writel(xhci_command, op_reg_base + XHCI_CMD_OFFSET); - - timeout = handshake(op_reg_base + XHCI_CMD_OFFSET, - 0x2, 0, 10 * 1000 * 1000, 125); - if (timeout) - pr_err("xHCI BIOS handoff time out\n"); - -retry: - val = readl(op_reg_base + XHCI_STS_OFFSET); - val |= STS_RW1C_BITS; - writel(val, op_reg_base + XHCI_STS_OFFSET); - val = readl(op_reg_base + XHCI_STS_OFFSET); - - if ((val & STS_RW1C_BITS) && retries--) { - pr_err("clear USB Status Register (status = %#x) failed, retry\n", val); - goto retry; - } - - val = readl(op_reg_base + XHCI_CMD_OFFSET); - val &= ~(XHCI_CMD_RUN | XHCI_IRQS); - writel(val, op_reg_base + XHCI_CMD_OFFSET); - timeout = handshake(op_reg_base + XHCI_STS_OFFSET, XHCI_STS_HALT, 1, - XHCI_MAX_HALT_USEC, 125); - if (timeout) { - val = readl(op_reg_base + XHCI_STS_OFFSET); - pr_err("xHCI HW did not halt within %d usec status = 0x%x\n", - XHCI_MAX_HALT_USEC, val); - } - - xhci_command = readl(op_reg_base + XHCI_CMD_OFFSET); - xhci_command |= 0x2; - writel(xhci_command, op_reg_base + XHCI_CMD_OFFSET); - - timeout = handshake(op_reg_base + XHCI_CMD_OFFSET, - 0x2, 0, 10 * 1000 * 1000, 125); - if (timeout) - pr_err("xHCI BIOS handoff time out\n"); - - pci_read_config_dword(dev, PCI_COMMAND, &tmp); - tmp &= ~(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); - pci_write_config_dword(dev, PCI_COMMAND, tmp); -} -DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_ANY_ID, PCI_ANY_ID, - PCI_CLASS_SERIAL_USB_XHCI, 0, fixup_usb_xhci_reset); - -#ifdef CONFIG_DCA -static void enable_sw_dca(struct pci_dev *dev) -{ - struct pci_controller *hose = (struct pci_controller *)dev->sysdata; - unsigned long node, rc_index, dca_ctl, dca_conf; - int i; - - if (dev->class >> 8 != PCI_CLASS_NETWORK_ETHERNET) - return; - node = hose->node; - rc_index = hose->index; - for (i = 0; i < 256; i++) { - dca_conf = read_piu_ior1(node, rc_index, DEVICEID0 + (i << 7)); - if (dca_conf >> 63) - continue; - else { - dca_conf = (1UL << 63) | (dev->bus->number << 8) | dev->devfn; - printk("dca device index %d, dca_conf = %#lx\n", i, dca_conf); - write_piu_ior1(node, rc_index, DEVICEID0 + (i << 7), dca_conf); - break; - } - } - dca_ctl = read_piu_ior1(node, rc_index, DCACONTROL); - if (dca_ctl & 0x1) { - dca_ctl = 0x2; - write_piu_ior1(node, rc_index, DCACONTROL, dca_ctl); - printk("Node %ld RC %ld enable DCA 1.0\n", node, rc_index); - } -} -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, enable_sw_dca); -#endif - -void __init reserve_mem_for_pci(void) -{ - int ret; - unsigned long base = PCI_32BIT_MEMIO; - - ret = add_memmap_region(base, PCI_32BIT_MEMIO_SIZE, memmap_pci); - if (ret) { - pr_err("reserved pages for pcie memory space failed\n"); - return; - } - - pr_info("reserved pages for pcie memory space %lx:%lx\n", base >> PAGE_SHIFT, - (base + PCI_32BIT_MEMIO_SIZE) >> PAGE_SHIFT); -} diff --git a/arch/sw_64/configs/junzhang_defconfig b/arch/sw_64/configs/junzhang_defconfig new file mode 100644 index 0000000000000000000000000000000000000000..2a9a7bb9d70f85b5db334a6498b23904b9357fa1 --- /dev/null +++ b/arch/sw_64/configs/junzhang_defconfig @@ -0,0 +1,697 @@ +CONFIG_LOCALVERSION="-junzhang" +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +# CONFIG_CROSS_MEMORY_ATTACH is not set +CONFIG_USELIB=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_IKHEADERS=y +CONFIG_LOG_BUF_SHIFT=18 +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_NAMESPACES=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_RELAY=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_EXPERT=y +CONFIG_KALLSYMS_ALL=y +CONFIG_BPF_SYSCALL=y +CONFIG_PERF_EVENTS=y +CONFIG_DEBUG_PERF_USE_VMALLOC=y +# CONFIG_COMPAT_BRK is not set +CONFIG_SUBARCH_C4=y +CONFIG_SMP=y +CONFIG_SCHED_SMT=y +CONFIG_NR_CPUS=64 +CONFIG_ARCH_SPARSEMEM_ENABLE=y +CONFIG_NUMA=y +CONFIG_HZ=100 +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_USE_OF=y +CONFIG_FIRMWARE_MEMMAP=y +CONFIG_DMI_SYSFS=m +CONFIG_ACPI_TAD=y +CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE=y +CONFIG_SW64_SUSPEND_DEEPSLEEP_BOOTCORE=y +# CONFIG_CPU_IDLE is not set +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_SCSI=m +CONFIG_VHOST_VSOCK=m +CONFIG_VHOST_CROSS_ENDIAN_LEGACY=y +CONFIG_KPROBES=y +CONFIG_JUMP_LABEL=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_OSF_PARTITION=y +CONFIG_BSD_DISKLABEL=y +CONFIG_MINIX_SUBPARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_LDM_PARTITION=y +CONFIG_SGI_PARTITION=y +CONFIG_ULTRIX_PARTITION=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_CMA_AREAS=7 +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=y +CONFIG_UNIX=y +CONFIG_UNIX_DIAG=y +CONFIG_TLS=m +CONFIG_TLS_DEVICE=y +CONFIG_XFRM_USER=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_SUB_POLICY=y +CONFIG_XFRM_STATISTICS=y +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_FIB_TRIE_STATS=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_IP_MROUTE=y +CONFIG_NET_IPVTI=m +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_UDP_DIAG=m +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_MD5SIG=y +CONFIG_IPV6=m +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_MIP6=m +CONFIG_IPV6_ILA=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_GRE=m +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_MROUTE=y +CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y +CONFIG_IPV6_PIMSM_V2=y +CONFIG_IPV6_SEG6_LWTUNNEL=y +CONFIG_IPV6_SEG6_HMAC=y +CONFIG_NETFILTER=y +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_LOG_NETDEV=m +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=m +CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_OBJREF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_OSF=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_L2TP=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_NF_TABLES_IPV4=y +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_FLOW_TABLE_IPV4=m +CONFIG_NF_LOG_ARP=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_TARGET_CLUSTERIP=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NF_LOG_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BRIDGE=m +CONFIG_VLAN_8021Q=m +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_CBQ=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_DSMARK=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_SKBPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_DEFAULT=y +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_TCINDEX=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_RSVP=m +CONFIG_NET_CLS_RSVP6=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=m +CONFIG_NETLINK_DIAG=m +CONFIG_CGROUP_NET_PRIO=y +CONFIG_BPF_JIT=y +# CONFIG_WIRELESS is not set +CONFIG_PCI=y +CONFIG_PCIEPORTBUS=y +CONFIG_PCIEAER=y +# CONFIG_PCIEASPM is not set +CONFIG_PCI_MSI=y +CONFIG_PCI_IOV=y +CONFIG_UEVENT_HELPER=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +# CONFIG_STANDALONE is not set +# CONFIG_PREVENT_FIRMWARE_BUILD is not set +CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_JEDECPROBE=y +CONFIG_MTD_CFI_ADV_OPTIONS=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_CFI_STAA=y +CONFIG_MTD_ROM=y +CONFIG_MTD_ABSENT=y +CONFIG_MTD_COMPLEX_MAPPINGS=y +CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_PHYSMAP_OF=y +CONFIG_MTD_PLATRAM=y +CONFIG_MTD_SPI_NOR=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=5000000 +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_FC=y +CONFIG_NVME_TARGET=y +CONFIG_NVME_TARGET_LOOP=y +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_FC=y +CONFIG_NVME_TARGET_FCLOOP=y +CONFIG_RAID_ATTRS=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_ST=y +CONFIG_BLK_DEV_SR=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_SCSI_SPI_ATTRS=y +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_LIBSAS=y +CONFIG_SCSI_SAS_ATA=y +CONFIG_SCSI_SRP_ATTRS=y +CONFIG_ISCSI_TCP=m +CONFIG_ISCSI_BOOT_SYSFS=y +CONFIG_SCSI_CXGB3_ISCSI=m +CONFIG_SCSI_CXGB4_ISCSI=m +CONFIG_SCSI_BNX2_ISCSI=m +CONFIG_MEGARAID_NEWGEN=y +CONFIG_MEGARAID_MM=m +CONFIG_MEGARAID_MAILBOX=m +CONFIG_MEGARAID_LEGACY=m +CONFIG_MEGARAID_SAS=m +CONFIG_SCSI_MPT3SAS=m +CONFIG_SCSI_DH=y +CONFIG_SCSI_DH_RDAC=y +CONFIG_SCSI_DH_HP_SW=y +CONFIG_SCSI_DH_EMC=y +CONFIG_SCSI_DH_ALUA=y +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +# CONFIG_ATA_SFF is not set +CONFIG_MD=y +CONFIG_MD_LINEAR=m +CONFIG_MD_MULTIPATH=m +CONFIG_MD_FAULTY=m +CONFIG_BCACHE=m +CONFIG_BCACHE_DEBUG=y +CONFIG_BCACHE_CLOSURES_DEBUG=y +CONFIG_BLK_DEV_DM=m +CONFIG_DM_DEBUG=y +CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING=y +CONFIG_DM_DEBUG_BLOCK_STACK_TRACING=y +CONFIG_DM_UNSTRIPED=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_WRITECACHE=m +CONFIG_DM_ERA=m +CONFIG_DM_MIRROR=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_RAID=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_DELAY=m +CONFIG_DM_UEVENT=y +CONFIG_DM_FLAKEY=m +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_SWITCH=m +CONFIG_DM_LOG_WRITES=m +CONFIG_DM_INTEGRITY=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_ISCSI_TARGET=m +CONFIG_NET_FC=y +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_VIRTIO_NET=y +# CONFIG_NET_VENDOR_3COM is not set +# CONFIG_NET_VENDOR_AMD is not set +# CONFIG_NET_VENDOR_ARC is not set +# CONFIG_NET_VENDOR_AURORA is not set +CONFIG_CAVIUM_PTP=y +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_EZCHIP is not set +CONFIG_E100=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_IGB=y +CONFIG_IGBVF=m +CONFIG_IXGB=m +CONFIG_IXGBE=m +CONFIG_IXGBEVF=m +CONFIG_I40E=y +CONFIG_I40EVF=y +# CONFIG_NET_VENDOR_MARVELL is not set +CONFIG_MLX4_EN=y +CONFIG_MLX5_CORE=m +CONFIG_MLX5_FPGA=y +CONFIG_MLX5_CORE_EN=y +CONFIG_MLXSW_CORE=y +CONFIG_MLXSW_PCI=y +CONFIG_MLXSW_I2C=y +CONFIG_MLXSW_MINIMAL=y +# CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_QUALCOMM is not set +# CONFIG_NET_VENDOR_RENESAS is not set +# CONFIG_NET_VENDOR_ROCKER is not set +# CONFIG_NET_VENDOR_SAMSUNG is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SMSC is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set +# CONFIG_NET_VENDOR_VIA is not set +# CONFIG_NET_VENDOR_WIZNET is not set +# CONFIG_WLAN is not set +CONFIG_INPUT_FF_MEMLESS=y +CONFIG_INPUT_POLLDEV=y +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_EVDEV=y +# CONFIG_KEYBOARD_ATKBD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO_SERPORT is not set +CONFIG_SERIO_LIBPS2=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +# CONFIG_SERIAL_8250_PCI is not set +CONFIG_SERIAL_8250_SUNWAY=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_VIRTIO_CONSOLE=y +# CONFIG_HW_RANDOM is not set +# CONFIG_DEVPORT is not set +# CONFIG_I2C_COMPAT is not set +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_MUX=y +CONFIG_SPI=y +CONFIG_SPI_SPIDEV=y +CONFIG_SENSORS_PVT=y +CONFIG_SENSORS_LM75=y +CONFIG_SSB=y +CONFIG_DRM=y +CONFIG_DRM_RADEON=y +CONFIG_DRM_AST=y +CONFIG_DRM_VIRTIO_GPU=y +CONFIG_FIRMWARE_EDID=y +CONFIG_LCD_CLASS_DEVICE=y +# CONFIG_VGA_CONSOLE is not set +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_LOGO=y +CONFIG_USB=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_STORAGE=y +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_INFINIBAND_MTHCA=m +# CONFIG_INFINIBAND_MTHCA_DEBUG is not set +CONFIG_MLX4_INFINIBAND=m +CONFIG_MLX5_INFINIBAND=m +CONFIG_INFINIBAND_IPOIB=m +CONFIG_INFINIBAND_IPOIB_CM=y +CONFIG_RTC_CLASS=y +# CONFIG_RTC_NVMEM is not set +# CONFIG_RTC_INTF_PROC is not set +CONFIG_RTC_DRV_PCF8523=y +CONFIG_UIO=y +CONFIG_UIO_PCI_GENERIC=m +CONFIG_VIRTIO_PCI=y +# CONFIG_VIRTIO_PCI_LEGACY is not set +CONFIG_VIRTIO_MMIO=y +CONFIG_STAGING=y +CONFIG_FB_SM750=y +CONFIG_IOMMU_DEFAULT_PASSTHROUGH=y +CONFIG_SUNWAY_IOMMU=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_EXT4_DEBUG=y +CONFIG_XFS_FS=y +CONFIG_GFS2_FS=y +CONFIG_FANOTIFY=y +CONFIG_QUOTA=y +CONFIG_AUTOFS4_FS=y +CONFIG_FUSE_FS=y +CONFIG_FSCACHE=y +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_FAT_DEFAULT_UTF8=y +CONFIG_NTFS_FS=y +CONFIG_NTFS_RW=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_CONFIGFS_FS=y +# CONFIG_MISC_FILESYSTEMS is not set +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_SWAP=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_NFS_V4_1_MIGRATION=y +CONFIG_ROOT_NFS=y +CONFIG_NFS_FSCACHE=y +CONFIG_NFS_USE_LEGACY_DNS=y +CONFIG_NFSD=m +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_SCSILAYOUT=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=y +CONFIG_NLS_CODEPAGE_950=y +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_INFINIBAND=y +CONFIG_SECURITY_PATH=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_CRYPTO_ECHAINIV=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_DEFLATE=y +CONFIG_CRYPTO_LZO=y +# CONFIG_CRYPTO_HW is not set +CONFIG_CONSOLE_LOGLEVEL_QUIET=7 +# CONFIG_ENABLE_MUST_CHECK is not set +# CONFIG_FRAME_POINTER is not set +CONFIG_SCHEDSTATS=y +# CONFIG_RCU_TRACE is not set diff --git a/arch/sw_64/configs/openeuler_defconfig b/arch/sw_64/configs/openeuler_defconfig index 78bf5d4e0523980e1eae5a6fec91daf7a1022942..92fc21b99b6070eebe6eee7301a4fad8beb46566 100644 --- a/arch/sw_64/configs/openeuler_defconfig +++ b/arch/sw_64/configs/openeuler_defconfig @@ -629,7 +629,7 @@ CONFIG_NFSD_V3_ACL=y CONFIG_NFSD_V4=y CONFIG_NFSD_SCSILAYOUT=y CONFIG_NFSD_V4_SECURITY_LABEL=y -CONFIG_NLS_CODEPAGE_437=m +CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_737=m CONFIG_NLS_CODEPAGE_775=m CONFIG_NLS_CODEPAGE_850=m @@ -653,7 +653,7 @@ CONFIG_NLS_ISO8859_8=m CONFIG_NLS_CODEPAGE_1250=m CONFIG_NLS_CODEPAGE_1251=m CONFIG_NLS_ASCII=m -CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_1=y CONFIG_NLS_ISO8859_2=m CONFIG_NLS_ISO8859_3=m CONFIG_NLS_ISO8859_4=m diff --git a/arch/sw_64/include/asm/acpi.h b/arch/sw_64/include/asm/acpi.h index 38615b969555db00a999eb700659efb4360365c9..a74db7667e09f397bdcac39b7cae2bc4e1ab4b76 100644 --- a/arch/sw_64/include/asm/acpi.h +++ b/arch/sw_64/include/asm/acpi.h @@ -43,6 +43,7 @@ extern int acpi_pci_disabled; ACPI_PDC_C_C2C3_FFH) #define ACPI_TABLE_UPGRADE_MAX_PHYS (max_low_pfn_mapped << PAGE_SHIFT) + static inline void disable_acpi(void) { acpi_disabled = 1; @@ -50,7 +51,18 @@ static inline void disable_acpi(void) acpi_noirq = 1; } -static inline void acpi_noirq_set(void) { acpi_noirq = 1; } +static inline void enable_acpi(void) +{ + acpi_disabled = 0; + acpi_pci_disabled = 0; + acpi_noirq = 0; +} + +static inline void acpi_noirq_set(void) +{ + acpi_noirq = 1; +} + static inline void acpi_disable_pci(void) { acpi_pci_disabled = 1; diff --git a/arch/sw_64/include/asm/asm-offsets.h b/arch/sw_64/include/asm/asm-offsets.h index 5ddfd96ccb793c8d1011bc1542968d29e9a68416..d370ee36a182ba510c28459f856b17f321bd57fc 100644 --- a/arch/sw_64/include/asm/asm-offsets.h +++ b/arch/sw_64/include/asm/asm-offsets.h @@ -1,7 +1 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_SW64_ASM_OFFSETS_H -#define _ASM_SW64_ASM_OFFSETS_H - #include - -#endif /* _ASM_SW64_ASM_OFFSETS_H */ diff --git a/arch/sw_64/include/asm/atomic.h b/arch/sw_64/include/asm/atomic.h index 66f4437be103766dbf81dd0542f85f5c91b44d40..1c662b077c316524603917dc03cf7cdd4cb5a05e 100644 --- a/arch/sw_64/include/asm/atomic.h +++ b/arch/sw_64/include/asm/atomic.h @@ -35,6 +35,7 @@ #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#ifdef CONFIG_SUBARCH_C3B /** * atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t @@ -50,18 +51,12 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) unsigned long addr; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %2\n" "1: lldw %0, 0(%3)\n" " cmpeq %0, %5, %4\n" " seleq %4, 1, $31, %4\n" " wr_f %4\n" " addw %0, %6, %1\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %1, 0(%3)\n" " rd_f %1\n" " beq %4, 2f\n" @@ -74,7 +69,6 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) : "Ir" (u), "Ir" (a), "m" (v->counter)); return old; } -#define atomic_fetch_add_unless atomic_fetch_add_unless /** * atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t @@ -90,18 +84,12 @@ static inline long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) unsigned long addr; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %2\n" "1: lldl %0, 0(%3)\n" " cmpeq %0, %5, %4\n" " seleq %4, 1, $31, %4\n" " wr_f %4\n" " addl %0, %6, %1\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %1, 0(%3)\n" " rd_f %1\n" " beq %4, 2f\n" @@ -114,7 +102,6 @@ static inline long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) : "Ir" (u), "Ir" (a), "m" (v->counter)); return old; } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless /* * atomic64_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic_t @@ -127,18 +114,12 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) unsigned long old, temp1, addr, temp2; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %2\n" "1: lldl %4, 0(%3)\n" " cmple %4, 0, %0\n" " seleq %0, 1, $31, %0\n" " wr_f %0\n" " subl %4, 1, %1\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %1, 0(%3)\n" " rd_f %1\n" " beq %0, 2f\n" @@ -153,33 +134,17 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) } -#define atomic64_dec_if_positive atomic64_dec_if_positive - -#ifdef CONFIG_LOCK_MEMB -#define LOCK_MEMB "memb\n" -#else -#define LOCK_MEMB -#endif - -#ifdef CONFIG_LOCK_FIXUP -#define LOCK_FIXUP "memb\n" -#else -#define LOCK_FIXUP -#endif - #define ATOMIC_OP(op, asm_op) \ static inline void atomic_##op(int i, atomic_t *v) \ { \ unsigned long temp1, temp2, addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldw %0, 0(%3)\n" \ " ldi %1, 1\n" \ " wr_f %1\n" \ " " #asm_op " %0, %4, %0\n" \ - LOCK_FIXUP \ " lstw %0, 0(%3)\n" \ " rd_f %0\n" \ " beq %0, 2f\n" \ @@ -197,14 +162,12 @@ static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ int temp1, temp2; \ unsigned long addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldw %0, 0(%3)\n" \ " ldi %1, 1\n" \ " wr_f %1\n" \ " " #asm_op " %0, %4, %1\n" \ " " #asm_op " %0, %4, %0\n" \ - LOCK_FIXUP \ " lstw %1, 0(%3)\n" \ " rd_f %1\n" \ " beq %1, 2f\n" \ @@ -224,13 +187,11 @@ static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ int temp1, temp2; \ unsigned long addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldw %0, 0(%3)\n" \ " ldi %1, 1\n" \ " wr_f %1\n" \ " " #asm_op " %0, %4, %1\n" \ - LOCK_FIXUP \ " lstw %1, 0(%3)\n" \ " rd_f %1\n" \ " beq %1, 2f\n" \ @@ -248,13 +209,11 @@ static inline void atomic64_##op(long i, atomic64_t *v) \ { \ unsigned long temp1, temp2, addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldl %0, 0(%3)\n" \ " ldi %1, 1\n" \ " wr_f %1\n" \ " " #asm_op " %0, %4, %0\n" \ - LOCK_FIXUP \ " lstl %0, 0(%3)\n" \ " rd_f %0\n" \ " beq %0, 2f\n" \ @@ -272,14 +231,12 @@ static inline long atomic64_##op##_return_relaxed(long i, atomic64_t *v)\ long temp1, temp2; \ unsigned long addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldl %0, 0(%3)\n" \ " ldi %1, 1\n" \ " wr_f %1\n" \ " " #asm_op " %0, %4, %1\n" \ " " #asm_op " %0, %4, %0\n" \ - LOCK_FIXUP \ " lstl %1, 0(%3)\n" \ " rd_f %1\n" \ " beq %1, 2f\n" \ @@ -297,13 +254,11 @@ static inline long atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ long temp1, temp2; \ unsigned long addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldl %0, 0(%3)\n" \ " ldi %1, 1\n" \ " wr_f %1\n" \ " " #asm_op " %0, %4, %1\n" \ - LOCK_FIXUP \ " lstl %1, 0(%3)\n" \ " rd_f %1\n" \ " beq %1, 2f\n" \ @@ -315,6 +270,219 @@ static inline long atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ return temp1; \ } \ +#else /* !CONFIG_SUBARCH_C3B */ + +/** + * atomic_fetch_add_unless - add unless the number is a given value + * @v: pointer of type atomic_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns the old value of @v. + */ +static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + int old, new, c; + unsigned long addr; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldw %0, 0(%3)\n" + " cmpeq %0, %5, %4\n" + " bne %4, 2f\n" + " addw %0, %6, %1\n" + " lstw %1, 0(%3)\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (old), "=&r" (new), "=m" (v->counter), "=&r" (addr), "=&r" (c) + : "Ir" (u), "Ir" (a), "m" (v->counter)); + return old; +} + +/** + * atomic64_fetch_add_unless - add unless the number is a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns the old value of @v. + */ +static inline long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) +{ + long old, new, c; + unsigned long addr; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldl %0, 0(%3)\n" + " cmpeq %0, %5, %4\n" + " bne %4, 2f\n" + " addl %0, %6, %1\n" + " lstl %1, 0(%3)\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (old), "=&r" (new), "=m" (v->counter), "=&r" (addr), "=&r" (c) + : "Ir" (u), "Ir" (a), "m" (v->counter)); + return old; +} + +/* + * atomic64_dec_if_positive - decrement by 1 if old value positive + * @v: pointer of type atomic_t + * + * The function returns the old value of *v minus 1, even if + * the atomic variable, v, was not decremented. + */ +static inline long atomic64_dec_if_positive(atomic64_t *v) +{ + unsigned long old, temp1, addr, temp2; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldl %4, 0(%3)\n" + " cmple %4, 0, %0\n" + " bne %0, 2f\n" + " subl %4, 1, %1\n" + " lstl %1, 0(%3)\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr), "=&r" (old) + : "m" (v->counter)); + return old - 1; +} + +#define ATOMIC_OP(op, asm_op) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + unsigned long temp1, addr; \ + __asm__ __volatile__( \ + " ldi %2, %1\n" \ + "1: lldw %0, 0(%2)\n" \ + " " #asm_op " %0, %3, %0\n" \ + " lstw %0, 0(%2)\n" \ + " beq %0, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ +} \ + + +#define ATOMIC_OP_RETURN(op, asm_op) \ +static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ +{ \ + int temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldw %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " " #asm_op " %0, %4, %0\n" \ + " lstw %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + +#define ATOMIC_FETCH_OP(op, asm_op) \ +static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ +{ \ + int temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldw %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " lstw %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + + +#define ATOMIC64_OP(op, asm_op) \ +static inline void atomic64_##op(long i, atomic64_t *v) \ +{ \ + unsigned long temp1, addr; \ + __asm__ __volatile__( \ + " ldi %2, %1\n" \ + "1: lldl %0, 0(%2)\n" \ + " " #asm_op " %0, %3, %0\n" \ + " lstl %0, 0(%2)\n" \ + " beq %0, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ +} \ + + +#define ATOMIC64_OP_RETURN(op, asm_op) \ +static inline long atomic64_##op##_return_relaxed(long i, atomic64_t *v)\ +{ \ + long temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldl %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " " #asm_op " %0, %4, %0\n" \ + " lstl %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} + +#define ATOMIC64_FETCH_OP(op, asm_op) \ +static inline long atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ +{ \ + long temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldl %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " lstl %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + +#endif /* CONFIG_SUBARCH_C3B */ + +#define atomic_fetch_add_unless atomic_fetch_add_unless +#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#define atomic64_dec_if_positive atomic64_dec_if_positive + #define ATOMIC_OPS(op) \ ATOMIC_OP(op, op##w) \ ATOMIC_OP_RETURN(op, op##w) \ diff --git a/arch/sw_64/include/asm/barrier.h b/arch/sw_64/include/asm/barrier.h index 5f4a03d700c68f667c62c17b45983f907779a3f7..bff199126c9fa49c44028fab3d4a10e0a18ce136 100644 --- a/arch/sw_64/include/asm/barrier.h +++ b/arch/sw_64/include/asm/barrier.h @@ -8,7 +8,13 @@ #define rmb() __asm__ __volatile__("memb" : : : "memory") +#if defined(CONFIG_SUBARCH_C3B) #define wmb() __asm__ __volatile__("memb" : : : "memory") +#elif defined(CONFIG_SUBARCH_C4) +#define wmb() __asm__ __volatile__("wmemb" : : : "memory") +#endif + +#define imemb() __asm__ __volatile__("imemb" : : : "memory") #ifdef CONFIG_SMP #define __ASM_SMP_MB "\tmemb\n" diff --git a/arch/sw_64/include/asm/bitops.h b/arch/sw_64/include/asm/bitops.h index 8c4844e8d0676057fabc971d9a4be78a7b55ec0c..46b242f4b4babb25863ed3597cdd74e690352995 100644 --- a/arch/sw_64/include/asm/bitops.h +++ b/arch/sw_64/include/asm/bitops.h @@ -9,10 +9,7 @@ #include #include -/* - * Copyright 1994, Linus Torvalds. - */ - +#ifdef CONFIG_SUBARCH_C3B /* * These have to be done with inline assembly: that way the bit-setting * is guaranteed to be atomic. All bit operations return 0 if the bit @@ -32,20 +29,14 @@ set_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %5\n" "1: lldw %0, 0(%3)\n" " ldi %1, 1\n" " wr_f %1\n" " bis %0, %4, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%3)\n" - " rd_f %1\n" - " beq %1, 2f\n" + " rd_f %0\n" + " beq %0, 2f\n" ".subsection 2\n" "2: br 1b\n" ".previous" @@ -53,19 +44,6 @@ set_bit(unsigned long nr, volatile void *addr) : "Ir" (1UL << (nr & 31)), "m" (*m)); } -/* - * WARNING: non atomic version. - */ -static inline void -__set_bit(unsigned long nr, volatile void *addr) -{ - int *m = ((int *) addr) + (nr >> 5); - - *m |= 1 << (nr & 31); -} - -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() static inline void clear_bit(unsigned long nr, volatile void *addr) @@ -74,20 +52,14 @@ clear_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %5\n" "1: lldw %0, 0(%3)\n" " ldi %1, 1\n" " wr_f %1\n" " bic %0, %4, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%3)\n" - " rd_f %1\n" - " beq %1, 2f\n" + " rd_f %0\n" + " beq %0, 2f\n" ".subsection 2\n" "2: br 1b\n" ".previous" @@ -95,31 +67,6 @@ clear_bit(unsigned long nr, volatile void *addr) : "Ir" (1UL << (nr & 31)), "m" (*m)); } -static inline void -clear_bit_unlock(unsigned long nr, volatile void *addr) -{ - smp_mb(); - clear_bit(nr, addr); -} - -/* - * WARNING: non atomic version. - */ -static inline void -__clear_bit(unsigned long nr, volatile void *addr) -{ - int *m = ((int *) addr) + (nr >> 5); - - *m &= ~(1 << (nr & 31)); -} - -static inline void -__clear_bit_unlock(unsigned long nr, volatile void *addr) -{ - smp_mb(); - __clear_bit(nr, addr); -} - static inline void change_bit(unsigned long nr, volatile void *addr) { @@ -127,20 +74,14 @@ change_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %5\n" "1: lldw %0, 0(%3)\n" " ldi %1, 1\n" " wr_f %1\n" " xor %0, %4, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%3)\n" - " rd_f %1\n" - " beq %1, 2f\n" + " rd_f %0\n" + " beq %0, 2f\n" ".subsection 2\n" "2: br 1b\n" ".previous" @@ -148,18 +89,6 @@ change_bit(unsigned long nr, volatile void *addr) : "Ir" (1UL << (nr & 31)), "m" (*m)); } -/* - * WARNING: non atomic version. - */ -static inline void -__change_bit(unsigned long nr, volatile void *addr) -{ - int *m = ((int *) addr) + (nr >> 5); - - *m ^= 1 << (nr & 31); -} - - static inline int test_and_set_bit(unsigned long nr, volatile void *addr) { @@ -168,18 +97,12 @@ test_and_set_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %4, %6\n" "1: lldw %0, 0(%4)\n" " and %0, %5, %3\n" " seleq %3, 1, $31, %1\n" " wr_f %1\n" " bis %0, %5, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%4)\n" " rd_f %0\n" " bne %3, 2f\n" // %3 is not zero, no need to set, return @@ -202,18 +125,12 @@ test_and_set_bit_lock(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %4, %6\n" "1: lldw %0, 0(%4)\n" " and %0, %5, %3\n" " seleq %3, 1, $31, %1\n" " wr_f %1\n" " bis %0, %5, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%4)\n" " rd_f %0\n" " bne %3, 2f\n" // %3 is not zero, no need to set, return @@ -228,20 +145,6 @@ test_and_set_bit_lock(unsigned long nr, volatile void *addr) return oldbit != 0; } -/* - * WARNING: non atomic version. - */ -static inline int -__test_and_set_bit(unsigned long nr, volatile void *addr) -{ - unsigned long mask = 1 << (nr & 0x1f); - int *m = ((int *) addr) + (nr >> 5); - int old = *m; - - *m = old | mask; - return (old & mask) != 0; -} - static inline int test_and_clear_bit(unsigned long nr, volatile void *addr) { @@ -250,18 +153,12 @@ test_and_clear_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %4, %6\n" "1: lldw %0, 0(%4)\n" " and %0, %5, %3\n" " selne %3, 1, $31, %1\n" //Note: here is SELNE!!! " wr_f %1\n" " bic %0, %5, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%4)\n" " rd_f %0\n" " beq %3, 2f\n" // %3 is zero, no need to set, return @@ -276,18 +173,163 @@ test_and_clear_bit(unsigned long nr, volatile void *addr) return oldbit != 0; } -/* - * WARNING: non atomic version. - */ static inline int -__test_and_clear_bit(unsigned long nr, volatile void *addr) +test_and_change_bit(unsigned long nr, volatile void *addr) { - unsigned long mask = 1 << (nr & 0x1f); + unsigned long oldbit; + unsigned long temp, base; int *m = ((int *) addr) + (nr >> 5); - int old = *m; - *m = old & ~mask; - return (old & mask) != 0; + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " ldi %2, 1\n" + " wr_f %2\n" + " and %0, %4, %2\n" + " xor %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " rd_f %0\n" + " beq %0, 3f\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (temp), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +#else /* !CONFIG_SUBARCH_C3B */ +static inline void +set_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %2, %4\n" + "1: lldw %0, 0(%2)\n" + " bis %0, %3, %0\n" + " lstw %0, 0(%2)\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline void +clear_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %2, %4\n" + "1: lldw %0, 0(%2)\n" + " bic %0, %3, %0\n" + " lstw %0, 0(%2)\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline void +change_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %2, %4\n" + "1: lldw %0, 0(%2)\n" + " xor %0, %3, %0\n" + " lstw %0, 0(%2)\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline int +test_and_set_bit(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " and %0, %4, %2\n" + " bne %2, 2f\n" // %2 is not zero, no need to set, return + " bis %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +static inline int +test_and_set_bit_lock(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " and %0, %4, %2\n" + " bne %2, 2f\n" // %2 is not zero, no need to set, return + " bis %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +static inline int +test_and_clear_bit(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " and %0, %4, %2\n" + " beq %2, 2f\n" // %2 is zero, no need to set, return + " bic %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; } static inline int @@ -298,23 +340,14 @@ test_and_change_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %5\n" "1: lldw %0, 0(%3)\n" - " ldi %2, 1\n" - " wr_f %2\n" " and %0, %4, %2\n" " xor %0, %4, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%3)\n" - " rd_f %0\n" " beq %0, 3f\n" ".subsection 2\n" - "3: br 1b\n" + "3: lbr 1b\n" ".previous" : "=&r" (temp), "=m" (*m), "=&r" (oldbit), "=&r" (base) : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); @@ -322,9 +355,75 @@ test_and_change_bit(unsigned long nr, volatile void *addr) return oldbit != 0; } + +#endif /* CONFIG_SUBARCH_C3B */ + /* * WARNING: non atomic version. */ +static inline void +__set_bit(unsigned long nr, volatile void *addr) +{ + int *m = ((int *) addr) + (nr >> 5); + + *m |= 1 << (nr & 31); +} + +#define smp_mb__before_clear_bit() smp_mb() +#define smp_mb__after_clear_bit() smp_mb() + +static inline void +clear_bit_unlock(unsigned long nr, volatile void *addr) +{ + smp_mb(); + clear_bit(nr, addr); +} + +static inline void +__clear_bit(unsigned long nr, volatile void *addr) +{ + int *m = ((int *) addr) + (nr >> 5); + + *m &= ~(1 << (nr & 31)); +} + +static inline void +__clear_bit_unlock(unsigned long nr, volatile void *addr) +{ + smp_mb(); + __clear_bit(nr, addr); +} + +static inline void +__change_bit(unsigned long nr, volatile void *addr) +{ + int *m = ((int *) addr) + (nr >> 5); + + *m ^= 1 << (nr & 31); +} + +static inline int +__test_and_set_bit(unsigned long nr, volatile void *addr) +{ + unsigned long mask = 1 << (nr & 0x1f); + int *m = ((int *) addr) + (nr >> 5); + int old = *m; + + *m = old | mask; + return (old & mask) != 0; +} + +static inline int +__test_and_clear_bit(unsigned long nr, volatile void *addr) +{ + unsigned long mask = 1 << (nr & 0x1f); + int *m = ((int *) addr) + (nr >> 5); + int old = *m; + + *m = old & ~mask; + return (old & mask) != 0; +} + static inline int __test_and_change_bit(unsigned long nr, volatile void *addr) { diff --git a/arch/sw_64/include/asm/cacheflush.h b/arch/sw_64/include/asm/cacheflush.h index 536b0b7b78bdbb7269c2e772818043fee937bbf4..0d49830b8493464a520276661cc49dca9f04a0a4 100644 --- a/arch/sw_64/include/asm/cacheflush.h +++ b/arch/sw_64/include/asm/cacheflush.h @@ -5,7 +5,7 @@ /* * DCache: PIPT * ICache: - * - C3A/B is VIVT with ICTAG, support coherence. + * - C3B is VIVT with ICTAG, support coherence. * - C4 is VIPT */ #include diff --git a/arch/sw_64/include/asm/core.h b/arch/sw_64/include/asm/core.h index e5e4cc138102791c5976634799afd60d646cc032..3f4df6d6c92deb6167212f3d18056956228982f0 100644 --- a/arch/sw_64/include/asm/core.h +++ b/arch/sw_64/include/asm/core.h @@ -2,17 +2,54 @@ #ifndef _ASM_SW64_CORE_H #define _ASM_SW64_CORE_H +#include + #define II_II0 0 #define II_II1 1 #define II_SLEEP 2 #define II_WAKE 3 #define II_NMII 6 -#ifdef CONFIG_SW64_CHIP3 #define II_RESET II_NMII -#define CORES_PER_NODE_SHIFT 5 + +#if defined(CONFIG_SUBARCH_C3B) + +#define DOMAIN_ID_BITS 2 +#define DOMAIN_ID_SHIFT 5 + +#define THREAD_ID_BITS 1 +#define THREAD_ID_SHIFT 31 + +#define CORE_ID_BITS 5 +#define CORE_ID_SHIFT 0 + +static inline bool core_is_ht(void) +{ + return 0; +} + +#elif defined(CONFIG_SUBARCH_C4) + +#define DOMAIN_ID_BITS 2 +#define DOMAIN_ID_SHIFT 12 + +#define THREAD_ID_BITS 1 +#define THREAD_ID_SHIFT 8 + +#define CORE_ID_BITS 6 +#define CORE_ID_SHIFT 0 + +static inline bool core_is_ht(void) +{ + return rdhtctl() == 0x3; +} + #endif -#define CORES_PER_NODE (1UL << CORES_PER_NODE_SHIFT) + +#define DOMAIN_ID_MASK (GENMASK(DOMAIN_ID_BITS - 1, 0) << DOMAIN_ID_SHIFT) +#define THREAD_ID_MASK (GENMASK(THREAD_ID_BITS - 1, 0) << THREAD_ID_SHIFT) +#define CORE_ID_MASK (GENMASK(CORE_ID_BITS - 1, 0) << CORE_ID_SHIFT) +#define MAX_CORES_PER_CPU (1 << CORE_ID_BITS) /* * 0x00 ~ 0xff for hardware mm fault @@ -35,6 +72,7 @@ #define MMCSR__DAV_MATCH 0x102 #define MMCSR__IA_MATCH 0x103 #define MMCSR__IDA_MATCH 0x104 +#define MMCSR__IV_MATCH 0x105 /* entry.S */ extern void entArith(void); diff --git a/arch/sw_64/include/asm/cpu.h b/arch/sw_64/include/asm/cpu.h index ea32a7d3cf1bc87963b259a9902042a7c33e41e4..4da30bb91d89602e48c55c01fdff08d3f495b45c 100644 --- a/arch/sw_64/include/asm/cpu.h +++ b/arch/sw_64/include/asm/cpu.h @@ -1 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CPU_H +#define _ASM_SW64_CPU_H + +#endif /* _ASM_SW64_CPU_H */ diff --git a/arch/sw_64/include/asm/clock.h b/arch/sw_64/include/asm/cpufreq.h similarity index 67% rename from arch/sw_64/include/asm/clock.h rename to arch/sw_64/include/asm/cpufreq.h index 8a6548aa0a0dd7b9cce5f5667e046ff2887025a0..bb3f2ada79600e6eb241ba1d014948c041cfde29 100644 --- a/arch/sw_64/include/asm/clock.h +++ b/arch/sw_64/include/asm/cpufreq.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_SW64_CLOCK_H -#define _ASM_SW64_CLOCK_H +#ifndef _ASM_SW64_CPUFREQ_H +#define _ASM_SW64_CPUFREQ_H #include #include @@ -40,8 +40,22 @@ struct clk { #define CLK_ALWAYS_ENABLED (1 << 0) #define CLK_RATE_PROPAGATES (1 << 1) -int clk_init(void); +#define CLK_PRT 0x1UL +#define CORE_CLK0_V (0x1UL << 1) +#define CORE_CLK0_R (0x1UL << 2) +#define CORE_CLK2_V (0x1UL << 15) +#define CORE_CLK2_R (0x1UL << 16) + +#define CLK_LV1_SEL_PRT 0x1UL +#define CLK_LV1_SEL_MUXA (0x1UL << 2) +#define CLK_LV1_SEL_MUXB (0x1UL << 3) + +#define CORE_PLL0_CFG_SHIFT 4 +#define CORE_PLL2_CFG_SHIFT 18 +extern struct cpufreq_frequency_table freq_table[]; + +int clk_init(void); void sw64_set_rate(unsigned int index); struct clk *sw64_clk_get(struct device *dev, const char *id); @@ -51,4 +65,4 @@ void sw64_update_clockevents(unsigned long cpu, u32 freq); void sw64_store_policy(struct cpufreq_policy *policy); unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy); -#endif /* _ASM_SW64_CLOCK_H */ +#endif /* _ASM_SW64_CPUFREQ_H */ diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h new file mode 100644 index 0000000000000000000000000000000000000000..0610384208a460b32703ea7988ebbfa29be821c9 --- /dev/null +++ b/arch/sw_64/include/asm/csr.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CSR_H +#define _ASM_SW64_CSR_H + +#include + +#define CSR_EXC_SUM 0xd +#define CSR_INT_EN 0x1a +#define CSR_INT_STAT 0x1b +#define CSR_PCIE_MSI0_INT 0x1d +#define CSR_PCIE_MSI1_INT 0x1e +#define CSR_PCIE_MSI2_INT 0x1f +#define CSR_PCIE_MSI3_INT 0x20 +#define CSR_INT_VEC 0x2d +#define CSR_PCIE_MSI0_INTEN 0x35 +#define CSR_PCIE_MSI1_INTEN 0x36 +#define CSR_PCIE_MSI2_INTEN 0x37 +#define CSR_PCIE_MSI3_INTEN 0x38 +#define CSR_EXC_GPA 0x3b +#define CSR_EXC_PC 0xe +#define CSR_AS_INFO 0x3c +#define CSR_DS_STAT 0x48 +#define CSR_SOFTCID 0xc9 +#define CSR_DVA 0x54 +#define CSR_PTBR_SYS 0x68 +#define CSR_PTBR_USR 0x69 +#define CSR_APTP 0x6a +#define CSR_CID 0xc4 +#define CSR_WR_FREGS 0xc8 +#define CSR_SHTCLOCK 0xca +#define CSR_SHTCLOCK_OFFSET 0xcb + +#ifdef CONFIG_SUBARCH_C4 +#define CSR_IA_VPNMATCH 0xa +#define CSR_UPCR 0x15 +#define CSR_VPCR 0x16 +#define CSR_IA_MATCH 0x17 +#define CSR_IA_MASK 0x18 +#define CSR_IV_MATCH 0x19 +#define CSR_IA_UPNMATCH 0x3a +#define CSR_DC_CTLP 0x4e +#define CSR_DA_MATCH 0x51 +#define CSR_DA_MASK 0x52 +#define CSR_DA_MATCH_MODE 0x53 +#define CSR_DV_MATCH 0x56 +#define CSR_DV_MASK 0x57 +#define CSR_IDA_MATCH 0xc5 +#define CSR_IDA_MASK 0xc6 + +#define DA_MATCH_EN_S 4 +#define DV_MATCH_EN_S 6 +#define DAV_MATCH_EN_S 7 +#define DPM_MATCH 8 +#define DPM_MATCH_EN_S 10 +#define IDA_MATCH_EN_S 53 +#define IV_PM_EN_S 61 +#define IV_MATCH_EN_S 62 +#define IA_MATCH_EN_S 63 + +#endif + + +#ifdef CONFIG_HAVE_CSRRW +#define read_csr(x) \ + ({ unsigned long __val; \ + __asm__ __volatile__("csrr %0,%1" : "=r"(__val) : "i"(x)); \ + __val; }) + +#define write_csr(x, y) \ + ({ __asm__ __volatile__("csrw %0,%1" ::"r"(x), "i"(y)); }) + +#define write_csr_imb(x, y) \ + ({ __asm__ __volatile__("csrw %0,%1; imemb" ::"r"(x), "i"(y)); }) + + +#ifndef __ASSEMBLY__ +#include +static inline void update_ptbr_sys(unsigned long ptbr) +{ + imemb(); + write_csr_imb(ptbr, CSR_PTBR_SYS); +} +#endif +#else +#define read_csr(x) (0) +#define write_csr(x, y) do { } while (0) +#define write_csr_imb(x, y) do { } while (0) + +#ifndef __ASSEMBLY__ +static inline void update_ptbr_sys(unsigned long ptbr) +{ + wrptbr(ptbr); +} +#endif + +#endif +#endif /* _ASM_SW64_CSR_H */ diff --git a/arch/sw_64/include/asm/device.h b/arch/sw_64/include/asm/device.h index bc1408c47dd3875bcb7b371411e876cb62ad93f2..d999207e07d1e3b4a9c5eb1507710cfed18c2fc7 100644 --- a/arch/sw_64/include/asm/device.h +++ b/arch/sw_64/include/asm/device.h @@ -3,7 +3,7 @@ #define _ASM_SW64_DEVICE_H struct dev_archdata { -#if defined(CONFIG_SUNWAY_IOMMU) +#if defined(CONFIG_SUNWAY_IOMMU) || defined(CONFIG_SUNWAY_IOMMU_V2) void *iommu; #endif }; diff --git a/arch/sw_64/include/asm/dma.h b/arch/sw_64/include/asm/dma.h index 1211b71f347e80a39e7792f506933275b07b1e34..cf6a9cf75233878e8238d2f13e2e50c41fbb9564 100644 --- a/arch/sw_64/include/asm/dma.h +++ b/arch/sw_64/include/asm/dma.h @@ -88,20 +88,14 @@ */ #define MAX_ISA_DMA_ADDRESS 0x100000000UL +#define MAX_DMA32_PFN (1UL << (32 - PAGE_SHIFT)) + /* * If we have the iommu, we don't have any address limitations on DMA. * Otherwise (Nautilus, RX164), we have to have 0-16 Mb DMA zone * like i386. */ -#ifdef CONFIG_IOMMU_SUPPORT -#ifndef CONFIG_SWICH_GPU -#define MAX_DMA_ADDRESS (PAGE_OFFSET + 0x100000000) -#else -#define MAX_DMA_ADDRESS (~0UL) -#endif /* CONFIG_IOMMU_SUPPORT */ -#else -#define MAX_DMA_ADDRESS (PAGE_OFFSET + 0x80000000) -#endif +#define MAX_DMA_ADDRESS ~0UL /* 8237 DMA controllers */ #define IO_DMA1_BASE 0x00 /* 8 bit slave DMA, channels 0..3 */ diff --git a/arch/sw_64/include/asm/elf.h b/arch/sw_64/include/asm/elf.h index 8c858cff5573955714ad9785ca7e9f983e046aaf..2e29943b54f3eccc247819d43425a8fa1cce29e2 100644 --- a/arch/sw_64/include/asm/elf.h +++ b/arch/sw_64/include/asm/elf.h @@ -102,7 +102,7 @@ typedef struct user_fpsimd_state elf_fpregset_t; * such function. */ -#define ELF_PLAT_INIT(_r, load_addr) (_r->r0 = 0) +#define ELF_PLAT_INIT(_r, load_addr) (_r->regs[0] = 0) /* * The registers are laid out in pt_regs for HMCODE and syscall diff --git a/arch/sw_64/include/asm/extable.h b/arch/sw_64/include/asm/extable.h index 3680b4a918a6676e8ce6b31fba2ef487dccfe9f4..42f872ce6c3bb8531c67fb8c7a1f64b6bfc0b77f 100644 --- a/arch/sw_64/include/asm/extable.h +++ b/arch/sw_64/include/asm/extable.h @@ -34,26 +34,14 @@ struct exception_table_entry { } fixup; }; -/* Returns the new pc */ -#define fixup_exception(map_reg, _fixup, pc) \ -({ \ - if ((_fixup)->fixup.bits.valreg != 31) \ - map_reg((_fixup)->fixup.bits.valreg) = 0; \ - if ((_fixup)->fixup.bits.errreg != 31) \ - map_reg((_fixup)->fixup.bits.errreg) = -EFAULT; \ - (pc) + (_fixup)->fixup.bits.nextinsn; \ -}) - #define ARCH_HAS_RELATIVE_EXTABLE +extern int fixup_exception(struct pt_regs *regs, unsigned long pc); + #define swap_ex_entry_fixup(a, b, tmp, delta) \ do { \ (a)->fixup.unit = (b)->fixup.unit; \ (b)->fixup.unit = (tmp).fixup.unit; \ } while (0) -/* Macro for exception fixup code to access integer registers. */ -extern short regoffsets[]; -#define map_regs(r) (*(unsigned long *)((char *)regs + regoffsets[r])) - #endif /* _ASM_SW64_EXTABLE_H */ diff --git a/arch/sw_64/include/asm/futex.h b/arch/sw_64/include/asm/futex.h index 50324470173f5e586744ad1d9a4917755bebcd3c..78379981398053e3c7c29f1dddab934a8e44a003 100644 --- a/arch/sw_64/include/asm/futex.h +++ b/arch/sw_64/include/asm/futex.h @@ -9,33 +9,17 @@ #include #include -#ifndef LOCK_MEMB -#ifdef CONFIG_LOCK_MEMB -#define LOCK_MEMB "memb\n" -#else -#define LOCK_MEMB -#endif -#endif - -#ifndef LOCK_FIXUP -#ifdef CONFIG_LOCK_FIXUP -#define LOCK_FIXUP "memb\n" -#else -#define LOCK_FIXUP -#endif -#endif +#ifdef CONFIG_SUBARCH_C3B #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg, tmp) \ __asm__ __volatile__( \ - LOCK_MEMB \ "1: lldw %0, 0(%3)\n" \ " ldi %2, 1\n" \ " wr_f %2\n" \ insn \ - LOCK_FIXUP \ "2: lstw %1, 0(%3)\n" \ - " rd_f %2\n" \ - " beq %2, 4f\n" \ + " rd_f %1\n" \ + " beq %1, 4f\n" \ " bis $31, $31, %1\n" \ "3: .subsection 2\n" \ "4: br 1b\n" \ @@ -50,41 +34,63 @@ : "r" (uaddr), "r"(oparg) \ : "memory") -static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, - u32 __user *uaddr) +static inline int +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) { - int oldval = 0, ret; - unsigned long tmp; - - pagefault_disable(); - - switch (op) { - case FUTEX_OP_SET: - __futex_atomic_op("mov %4, %1\n", ret, oldval, uaddr, oparg, tmp); - break; - case FUTEX_OP_ADD: - __futex_atomic_op("addw %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); - break; - case FUTEX_OP_OR: - __futex_atomic_op("or %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); - break; - case FUTEX_OP_ANDN: - __futex_atomic_op("andnot %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); - break; - case FUTEX_OP_XOR: - __futex_atomic_op("xor %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); - break; - default: - ret = -ENOSYS; - } + int ret = 0, cmp; + u32 prev, tmp; - pagefault_enable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; - if (!ret) - *oval = oldval; + __asm__ __volatile__ ( + "1: lldw %1, 0(%4)\n" + " cmpeq %1, %5, %2\n" + " wr_f %2\n" + " bis $31, %6, %3\n" + "2: lstw %3, 0(%4)\n" + " rd_f %3\n" + " beq %2, 3f\n" + " beq %3, 4f\n" + "3: .subsection 2\n" + "4: br 1b\n" + " .previous\n" + " .section __ex_table, \"a\"\n" + " .long 1b-.\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b-.\n" + " ldi $31, 3b-2b(%0)\n" + " .previous\n" + : "+r"(ret), "=&r"(prev), "=&r"(cmp), "=&r"(tmp) + : "r"(uaddr), "r"((long)(int)oldval), "r"(newval) + : "memory"); + *uval = prev; return ret; } +#else /* !CONFIG_SUBARCH_C3B */ + +#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg, tmp) \ + __asm__ __volatile__( \ + "1: lldw %0, 0(%3)\n" \ + insn \ + "2: lstw %1, 0(%3)\n" \ + " beq %1, 4f\n" \ + " bis $31, $31, %1\n" \ + "3: .subsection 2\n" \ + "4: lbr 1b\n" \ + " .previous\n" \ + " .section __ex_table, \"a\"\n" \ + " .long 1b-.\n" \ + " ldi $31, 3b-1b(%1)\n" \ + " .long 2b-.\n" \ + " ldi $31, 3b-2b(%1)\n" \ + " .previous\n" \ + : "=&r" (oldval), "=&r"(ret), "=&r"(tmp) \ + : "r" (uaddr), "r"(oparg) \ + : "memory") + static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, @@ -97,22 +103,14 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, return -EFAULT; __asm__ __volatile__ ( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif "1: lldw %1, 0(%4)\n" " cmpeq %1, %5, %2\n" - " wr_f %2\n" + " beq %2, 3f\n" " bis $31, %6, %3\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif "2: lstw %3, 0(%4)\n" - " rd_f %3\n" - " beq %2, 3f\n" " beq %3, 4f\n" "3: .subsection 2\n" - "4: br 1b\n" + "4: lbr 1b\n" " .previous\n" " .section __ex_table, \"a\"\n" " .long 1b-.\n" @@ -127,6 +125,43 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, *uval = prev; return ret; } +#endif /* CONFIG_SUBARCH_C3B */ + +static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, + u32 __user *uaddr) +{ + int oldval = 0, ret; + unsigned long tmp; + + pagefault_disable(); + + switch (op) { + case FUTEX_OP_SET: + __futex_atomic_op("mov %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_ADD: + __futex_atomic_op("addw %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_OR: + __futex_atomic_op("or %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_ANDN: + __futex_atomic_op("andnot %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_XOR: + __futex_atomic_op("xor %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + default: + ret = -ENOSYS; + } + + pagefault_enable(); + + if (!ret) + *oval = oldval; + + return ret; +} #endif /* __KERNEL__ */ diff --git a/arch/sw_64/include/asm/hmcall.h b/arch/sw_64/include/asm/hmcall.h index 8bd5f5357bc0c32541c6afad2fc13520d7f9a4a3..65e644bd6a23ac2439c3dbd3a12fac6fa4818bbb 100644 --- a/arch/sw_64/include/asm/hmcall.h +++ b/arch/sw_64/include/asm/hmcall.h @@ -17,6 +17,7 @@ #define HMC_wrktp 0x0A #define HMC_rdptbr 0x0B #define HMC_wrptbr 0x0C +#define HMC_rdhtctl 0x0D #define HMC_wrksp 0x0E #define HMC_mtinten 0x0F #define HMC_load_mm 0x11 @@ -164,6 +165,7 @@ __CALL_HMC_W1(wrusp, unsigned long); __CALL_HMC_R0(rdksp, unsigned long); __CALL_HMC_W1(wrksp, unsigned long); +__CALL_HMC_R0(rdhtctl, unsigned long); /* * Load a mm context. This is needed when we change the page diff --git a/arch/sw_64/include/asm/hugetlb.h b/arch/sw_64/include/asm/hugetlb.h index 11565a8f86cbbb53f9370be2b6c86f844f97e255..0b6f16f913b23c080f333abd43fbd9430c2d0298 100644 --- a/arch/sw_64/include/asm/hugetlb.h +++ b/arch/sw_64/include/asm/hugetlb.h @@ -3,23 +3,41 @@ #define _ASM_SW64_HUGETLB_H #include -#include -static inline void hugetlb_prefault_arch_hook(struct mm_struct *mm) -{ -} +#ifdef CONFIG_SUBARCH_C4 +#define __HAVE_ARCH_HUGE_PTE_CLEAR +extern void huge_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long sz); + +#define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT +extern void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte); + +#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR +extern pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep); + +#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH +extern void huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, + pte_t *ptep); -void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr, - unsigned long end, unsigned long floor, - unsigned long ceiling); +#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT +extern void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep); -static inline int arch_prepare_hugepage(struct page *page) -{ - return 0; -} +#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS +extern int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, pte_t pte, int dirty); -static inline void arch_release_hugepage(struct page *page) -{ -} +#define arch_make_huge_pte arch_make_huge_pte +extern pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, + struct page *page, int writable); + +#define set_huge_swap_pte_at set_huge_swap_pte_at +extern void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned long sz); +#endif + +#include #endif /* _ASM_SW64_HUGETLB_H */ diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h index 1426892c83b4bf189bbf610a9a86e574953ff451..84dd2ee018878c5f353e84c201e8b62eba307915 100644 --- a/arch/sw_64/include/asm/hw_init.h +++ b/arch/sw_64/include/asm/hw_init.h @@ -83,11 +83,9 @@ static inline unsigned long get_cpu_freq(void) return cpu_desc.frequency; } -static inline void update_cpu_freq(unsigned long freq) +static inline void update_cpu_freq(unsigned long khz) { - freq = freq * 1000000; - if (cpu_desc.frequency != freq) - cpu_desc.frequency = freq; + cpu_desc.frequency = khz * 1000; } #define EMUL_FLAG (0x1UL << 63) @@ -104,6 +102,7 @@ DECLARE_STATIC_KEY_FALSE(run_mode_emul_key); #define CPU_SW3231 0x31 #define CPU_SW831 0x32 +#define CPU_SW8A 0x41 #define GET_TABLE_ENTRY 1 #define GET_VENDOR_ID 2 diff --git a/arch/sw_64/include/asm/hw_irq.h b/arch/sw_64/include/asm/hw_irq.h index ad5aed26efb7f9ca58bd3b5881e0858d401780c0..3cfc725f7517dcacadeffb3f47a675aef142288e 100644 --- a/arch/sw_64/include/asm/hw_irq.h +++ b/arch/sw_64/include/asm/hw_irq.h @@ -2,7 +2,7 @@ #ifndef _ASM_SW64_HW_IRQ_H #define _ASM_SW64_HW_IRQ_H -#include +#include extern volatile unsigned long irq_err_count; DECLARE_PER_CPU(unsigned long, irq_pmi_count); diff --git a/arch/sw_64/include/asm/idle.h b/arch/sw_64/include/asm/idle.h index 7e88af18ba39f20f891703ef424dca3af02de7f9..95e145f25306ada8722f28ad9976601076658afb 100644 --- a/arch/sw_64/include/asm/idle.h +++ b/arch/sw_64/include/asm/idle.h @@ -4,4 +4,4 @@ extern void arch_cpu_idle(void); -#endif /* _ASM_SW64_IDLE_H */ +#endif /* _ASM_SW64_IDLE_H */ diff --git a/arch/sw_64/include/asm/kexec.h b/arch/sw_64/include/asm/kexec.h index a99aba9638e6cf11ca2b263eb51873c36e0e31f3..25e0d8da84f8dbe98908179bb061ea5f4759aa6e 100644 --- a/arch/sw_64/include/asm/kexec.h +++ b/arch/sw_64/include/asm/kexec.h @@ -34,25 +34,37 @@ static inline void crash_setup_regs(struct pt_regs *newregs, if (oldregs) { memcpy(newregs, oldregs, sizeof(*newregs)); } else { - __asm__ __volatile__ ("stl $0, %0" : "=m" (newregs->r0)); - __asm__ __volatile__ ("stl $1, %0" : "=m" (newregs->r1)); - __asm__ __volatile__ ("stl $2, %0" : "=m" (newregs->r2)); - __asm__ __volatile__ ("stl $3, %0" : "=m" (newregs->r3)); - __asm__ __volatile__ ("stl $4, %0" : "=m" (newregs->r4)); - __asm__ __volatile__ ("stl $5, %0" : "=m" (newregs->r5)); - __asm__ __volatile__ ("stl $6, %0" : "=m" (newregs->r6)); - __asm__ __volatile__ ("stl $7, %0" : "=m" (newregs->r7)); - __asm__ __volatile__ ("stl $8, %0" : "=m" (newregs->r8)); - __asm__ __volatile__ ("stl $19, %0" : "=m" (newregs->r19)); - __asm__ __volatile__ ("stl $20, %0" : "=m" (newregs->r20)); - __asm__ __volatile__ ("stl $21, %0" : "=m" (newregs->r21)); - __asm__ __volatile__ ("stl $22, %0" : "=m" (newregs->r22)); - __asm__ __volatile__ ("stl $23, %0" : "=m" (newregs->r23)); - __asm__ __volatile__ ("stl $24, %0" : "=m" (newregs->r24)); - __asm__ __volatile__ ("stl $25, %0" : "=m" (newregs->r25)); - __asm__ __volatile__ ("stl $26, %0" : "=m" (newregs->r26)); - __asm__ __volatile__ ("stl $27, %0" : "=m" (newregs->r27)); - __asm__ __volatile__ ("stl $28, %0" : "=m" (newregs->r28)); + __asm__ __volatile__ ("stl $0, %0" : "=m" (newregs->regs[0])); + __asm__ __volatile__ ("stl $1, %0" : "=m" (newregs->regs[1])); + __asm__ __volatile__ ("stl $2, %0" : "=m" (newregs->regs[2])); + __asm__ __volatile__ ("stl $3, %0" : "=m" (newregs->regs[3])); + __asm__ __volatile__ ("stl $4, %0" : "=m" (newregs->regs[4])); + __asm__ __volatile__ ("stl $5, %0" : "=m" (newregs->regs[5])); + __asm__ __volatile__ ("stl $6, %0" : "=m" (newregs->regs[6])); + __asm__ __volatile__ ("stl $7, %0" : "=m" (newregs->regs[7])); + __asm__ __volatile__ ("stl $8, %0" : "=m" (newregs->regs[8])); + __asm__ __volatile__ ("stl $9, %0" : "=m" (newregs->regs[9])); + __asm__ __volatile__ ("stl $10, %0" : "=m" (newregs->regs[10])); + __asm__ __volatile__ ("stl $11, %0" : "=m" (newregs->regs[11])); + __asm__ __volatile__ ("stl $12, %0" : "=m" (newregs->regs[12])); + __asm__ __volatile__ ("stl $13, %0" : "=m" (newregs->regs[13])); + __asm__ __volatile__ ("stl $14, %0" : "=m" (newregs->regs[14])); + __asm__ __volatile__ ("stl $15, %0" : "=m" (newregs->regs[15])); + __asm__ __volatile__ ("stl $16, %0" : "=m" (newregs->regs[16])); + __asm__ __volatile__ ("stl $17, %0" : "=m" (newregs->regs[17])); + __asm__ __volatile__ ("stl $18, %0" : "=m" (newregs->regs[18])); + __asm__ __volatile__ ("stl $19, %0" : "=m" (newregs->regs[19])); + __asm__ __volatile__ ("stl $20, %0" : "=m" (newregs->regs[20])); + __asm__ __volatile__ ("stl $21, %0" : "=m" (newregs->regs[21])); + __asm__ __volatile__ ("stl $22, %0" : "=m" (newregs->regs[22])); + __asm__ __volatile__ ("stl $23, %0" : "=m" (newregs->regs[23])); + __asm__ __volatile__ ("stl $24, %0" : "=m" (newregs->regs[24])); + __asm__ __volatile__ ("stl $25, %0" : "=m" (newregs->regs[25])); + __asm__ __volatile__ ("stl $26, %0" : "=m" (newregs->regs[26])); + __asm__ __volatile__ ("stl $27, %0" : "=m" (newregs->regs[27])); + __asm__ __volatile__ ("stl $28, %0" : "=m" (newregs->regs[28])); + __asm__ __volatile__ ("stl $29, %0" : "=m" (newregs->regs[29])); + __asm__ __volatile__ ("stl $30, %0" : "=m" (newregs->regs[30])); newregs->pc = (unsigned long)current_text_addr(); } } diff --git a/arch/sw_64/include/asm/kvm_asm.h b/arch/sw_64/include/asm/kvm_asm.h index 67b4ff594074b78ecd3b4e49232329ce41ded714..fd1b25018fc8c37f97a176201f0b8e444bd2ed3e 100644 --- a/arch/sw_64/include/asm/kvm_asm.h +++ b/arch/sw_64/include/asm/kvm_asm.h @@ -12,12 +12,11 @@ #define SW64_KVM_EXIT_IPI 14 #define SW64_KVM_EXIT_STOP 16 #define SW64_KVM_EXIT_RESTART 17 +#define SW64_KVM_EXIT_APT_FAULT 18 #define SW64_KVM_EXIT_FATAL_ERROR 22 +#define SW64_KVM_EXIT_MEMHOTPLUG 23 #define SW64_KVM_EXIT_DEBUG 24 -#ifdef CONFIG_KVM_MEMHOTPLUG -#define SW64_KVM_EXIT_MEMHOTPLUG 23 -#endif #define kvm_sw64_exception_type \ {0, "HOST_INTR" }, \ @@ -28,8 +27,12 @@ {14, "IPI" }, \ {16, "STOP" }, \ {17, "RESTART" }, \ + {18, "APT_FAULT" }, \ {22, "FATAL_ERROR" }, \ {23, "MEMHOTPLUG" }, \ {24, "DEBUG" } + +#include + #endif /* _ASM_SW64_KVM_ASM_H */ diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index 5433a3b21b871b7014099b27d18b574fad18d363..eb90ca72ec3879a94cd38363517a93157c4ab29a 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -28,8 +28,24 @@ #include +#define last_vpn(cpu) (cpu_data[cpu].last_vpn) + +#ifdef CONFIG_SUBARCH_C3B +#define VPN_BITS 8 +#define GUEST_RESET_PC 0xffffffff80011100 +#endif + +#ifdef CONFIG_SUBARCH_C4 +#define VPN_BITS 10 +#define GUEST_RESET_PC 0xfff0000000011002 +#endif + +#define VPN_FIRST_VERSION (1UL << VPN_BITS) +#define VPN_MASK ((1UL << VPN_BITS) - 1) +#define VPN_SHIFT (64 - VPN_BITS) + #define KVM_MAX_VCPUS 64 -#define KVM_USER_MEM_SLOTS 64 +#define KVM_USER_MEM_SLOTS 512 #define KVM_HALT_POLL_NS_DEFAULT 0 #define KVM_IRQCHIP_NUM_PINS 256 @@ -41,6 +57,13 @@ #define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) #define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) +/* + * The architecture supports 48-bit GPA as input to the addtional stage translations. + */ +#define KVM_PHYS_SHIFT (48) +#define KVM_PHYS_SIZE (_AC(1, ULL) << KVM_PHYS_SHIFT) +#define KVM_PHYS_MASK (KVM_PHYS_SIZE - _AC(1, ULL)) + struct kvm_arch_memory_slot { unsigned long host_phys_addr; bool valid; @@ -52,8 +75,22 @@ struct kvm_arch { /* segment table */ unsigned long *seg_pgd; + + struct swvm_mem mem; + /* Addtional stage page table*/ + pgd_t *pgd; }; +#define KVM_NR_MEM_OBJS 40 + +/* + * We don't want allocation failures within the mmu code, so we preallocate + * enough memory for a single page fault in a cache. + */ +struct kvm_mmu_memory_cache { + int nobjs; + void *objects[KVM_NR_MEM_OBJS]; +}; struct kvm_vcpu_arch { struct kvm_regs regs __attribute__((__aligned__(32))); @@ -65,6 +102,7 @@ struct kvm_vcpu_arch { struct hrtimer hrt; unsigned long timer_next_event; unsigned long vtimer_freq; + int first_run; int halted; int stopped; @@ -74,6 +112,9 @@ struct kvm_vcpu_arch { DECLARE_BITMAP(irqs_pending, SWVM_IRQS); unsigned long vpnc[NR_CPUS]; + /* Detect first run of a vcpu */ + bool has_run_once; + /* WAIT executed */ int wait; @@ -84,6 +125,13 @@ struct kvm_vcpu_arch { bool pause; struct kvm_decode mmio_decode; + + /* Cache some mmu pages needed inside spinlock regions */ + struct kvm_mmu_memory_cache mmu_page_cache; + + /* guest live migration */ + unsigned long migration_mark; + unsigned long shtclock; }; struct vmem_info { @@ -130,9 +178,24 @@ struct kvm_vcpu_stat { #ifdef CONFIG_KVM_MEMHOTPLUG void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr); #endif +#ifdef CONFIG_SUBARCH_C4 +#define KVM_ARCH_WANT_MMU_NOTIFIER +#endif +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); +int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable); +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); + +void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat); +void check_vcpu_requests(struct kvm_vcpu *vcpu); +void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu); +int vmem_init(void); +void vmem_exit(void); +int __sw64_vcpu_run(unsigned long vcb_pa, struct kvm_regs *regs, + struct hcall_args *args); int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, int exception_index, struct hcall_args *hargs); -void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid); +void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type); static inline void kvm_arch_hardware_disable(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {} @@ -140,14 +203,22 @@ static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} -static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} -static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); + int kvm_sw64_perf_init(void); int kvm_sw64_perf_teardown(void); - +void kvm_flush_tlb_all(void); +void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn); +int kvm_sw64_init_vm(struct kvm *kvm); +void kvm_sw64_destroy_vm(struct kvm *kvm); +int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu); +long kvm_sw64_set_vcb(struct file *filp, unsigned long arg); +long kvm_sw64_get_vcb(struct file *filp, unsigned long arg); + +void update_aptp(unsigned long); +void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu); #endif /* _ASM_SW64_KVM_HOST_H */ diff --git a/arch/sw_64/include/asm/kvm_mmu.h b/arch/sw_64/include/asm/kvm_mmu.h new file mode 100644 index 0000000000000000000000000000000000000000..f4493de934bab402a088d0deec64fe86b86a7b70 --- /dev/null +++ b/arch/sw_64/include/asm/kvm_mmu.h @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_MMU_H +#define _ASM_SW64_KVM_MMU_H + +#define AF_ACCESS_TYPE_SHIFT 55 +#define AF_INV_LEVEL_SHIFT 53 +#define AF_FAULT_STATUS_SHIFT 48 + +#define AF_ACCESS_TYPE_MASK 0x3 +#define AF_INV_LEVEL_MASK 0x3 +#define AF_FAULT_STATUS_MASK 0x1f +#define AF_ENTRY_ADDR_MASK ((0x1UL << AF_FAULT_STATUS_SHIFT) - 1) + +/* access type defination */ +#define AF_READ_ACCESS_TYPE 0x1 +#define AF_WRITE_ACCESS_TYPE 0x2 +#define AF_EXEC_ACCESS_TYPE 0x3 + +/* invalid page level */ +#define AF_INV_LEVEL_1 0 +#define AF_INV_LEVEL_2 1 +#define AF_INV_LEVEL_3 2 +#define AF_INV_LEVEL_4 3 + +/* fault status */ +#define AF_STATUS_MISCONFIG 0x1 +#define AF_STATUS_FOR 0x2 +#define AF_STATUS_FOW 0x4 +#define AF_STATUS_FOE 0x8 +#define AF_STATUS_INV 0x10 + +#define KVM_MMU_CACHE_MIN_PAGES 2 + +static inline void kvm_set_aptpte_readonly(pte_t *pte) +{ + pte_val(*pte) |= _PAGE_FOW; +} + +static inline bool kvm_aptpte_readonly(pte_t *pte) +{ + return (pte_val(*pte) & _PAGE_FOW) == _PAGE_FOW; +} + +static inline void kvm_set_aptpmd_readonly(pmd_t *pmd) +{ + pmd_val(*pmd) |= _PAGE_FOW; +} + +static inline bool kvm_aptpmd_readonly(pmd_t *pmd) +{ + return (pmd_val(*pmd) & _PAGE_FOW) == _PAGE_FOW; +} + +static inline void kvm_set_aptpud_readonly(pud_t *pud) +{ + pud_val(*pud) |= _PAGE_FOW; +} + +static inline bool kvm_aptpud_readonly(pud_t *pud) +{ + return (pud_val(*pud) & _PAGE_FOW) == _PAGE_FOW; +} + +static inline pte_t kvm_pte_mkwrite(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_FOW; + return pte; +} + +static inline pte_t kvm_pte_mkexec(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_FOE; + return pte; +} + +static inline bool kvm_pte_exec(pte_t *pte) +{ + return !(pte_val(*pte) & _PAGE_FOE); +} + +static inline pmd_t kvm_pmd_mkwrite(pmd_t pmd) +{ + pmd_val(pmd) &= ~_PAGE_FOW; + return pmd; +} + +static inline pmd_t kvm_pmd_mkexec(pmd_t pmd) +{ + pmd_val(pmd) &= ~_PAGE_FOE; + return pmd; +} + +static inline bool kvm_pmd_exec(pmd_t *pmd) +{ + return !(pmd_val(*pmd) & _PAGE_FOE); +} + +static inline pud_t kvm_pud_mkwrite(pud_t pud) +{ + pud_val(pud) &= ~_PAGE_FOW; + return pud; +} + +static inline pud_t kvm_pud_mkexec(pud_t pud) +{ + pud_val(pud) &= ~_PAGE_FOE; + return pud; +} + +static inline bool kvm_pud_exec(pud_t *pud) +{ + return !(pud_val(*pud) & _PAGE_FOE); +} + +void kvm_core4_commit_memory_region(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + const struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change); +void kvm_core4_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot); +void kvm_core4_flush_shadow_all(struct kvm *kvm); +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); +void kvm_handle_apt_fault(struct kvm_vcpu *vcpu); +int kvm_alloc_addtional_stage_pgd(struct kvm *kvm); +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot); +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run); +void apt_unmap_vm(struct kvm *kvm); +#endif /* _ASM_SW64_KVM_MMU_H */ diff --git a/arch/sw_64/include/asm/livepatch.h b/arch/sw_64/include/asm/livepatch.h index f93c2131113bfc13547c25f1c5ff9c61e5e1e607..1feec0f6be76ddad2c1e65e0bfacf3d511510af0 100644 --- a/arch/sw_64/include/asm/livepatch.h +++ b/arch/sw_64/include/asm/livepatch.h @@ -15,8 +15,8 @@ static inline int klp_check_compiler_support(void) static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip) { - regs->r27 = ip; - regs->r28 = ip; + regs->regs[27] = ip; + regs->regs[28] = ip; } #endif /* _ASM_SW64_LIVEPATCH_H */ diff --git a/arch/sw_64/include/asm/mmu_context.h b/arch/sw_64/include/asm/mmu_context.h index 5ae9d4616937f7f3f22bfeb79fe394a9409f6ef2..c69de5e7a2aa1e0fbabbb825493a146e7ba5e7be 100644 --- a/arch/sw_64/include/asm/mmu_context.h +++ b/arch/sw_64/include/asm/mmu_context.h @@ -11,7 +11,7 @@ * The maximum ASID's the processor supports. */ -#ifdef CONFIG_SUBARCH_C3B +#if defined(CONFIG_SUBARCH_C3B) || defined(CONFIG_SUBARCH_C4) #define ASID_BITS 10 #endif diff --git a/arch/sw_64/include/asm/msi.h b/arch/sw_64/include/asm/msi.h index f27a1fcf6a207bd8c83e80ef938155122023abf8..e378326b67545fda031c19a8b67ee7f2149cce7b 100644 --- a/arch/sw_64/include/asm/msi.h +++ b/arch/sw_64/include/asm/msi.h @@ -2,6 +2,8 @@ #ifndef _ASM_SW64_MSI_H #define _ASM_SW64_MSI_H +#include + #define NR_VECTORS NR_IRQS #define NR_IRQ_VECTORS NR_IRQS @@ -15,8 +17,6 @@ #define PERCPU_MSI_IRQS 256 -#define MSIX_MSG_ADDR (0x91abc0UL) - #define VT_MSIX_MSG_ADDR (0x8000fee00000UL) #define VT_MSIX_ADDR_DEST_ID_SHIFT 12 #define VT_MSIX_ADDR_DEST_ID_MASK (0xff << VT_MSIX_ADDR_DEST_ID_SHIFT) @@ -32,14 +32,23 @@ extern bool find_free_cpu_vector(const struct cpumask *search_mask, extern int msi_compose_msg(unsigned int irq, struct msi_msg *msg); extern void sw64_irq_noop(struct irq_data *d); extern struct irq_chip sw64_irq_chip; +extern void handle_pci_msi_interrupt(unsigned long type, + unsigned long vector, + unsigned long pci_msi1_addr); #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN #define MSI_ADDR_BASE_HI 0 #define MSI_ADDR_BASE_LO 0x91abc0 +#define MSI_ADDR_SHIFT 20 +#define MSI_ADDR_DEST_ID_SHIFT 10 + struct sw64_msi_chip_data { spinlock_t cdata_lock; - unsigned long msi_config; + union { + unsigned long msi_config; + unsigned long msiaddr; + }; unsigned long rc_node; unsigned long rc_index; unsigned int msi_config_index; @@ -47,9 +56,21 @@ struct sw64_msi_chip_data { unsigned int vector; unsigned int prev_cpu; unsigned int prev_vector; + unsigned int multi_msi; bool move_in_progress; }; +static inline int rcid_to_msicid(int rcid) +{ + int msicid = 0; + + msicid |= (rcid_to_domain_id(rcid) << 7); + msicid |= (rcid_to_thread_id(rcid) << 6); + msicid |= (rcid_to_core_id(rcid) << 0); + + return msicid; +} + extern void arch_init_msi_domain(struct irq_domain *domain); enum irq_alloc_type { IRQ_ALLOC_TYPE_MSI, @@ -64,5 +85,11 @@ struct irq_alloc_info { }; typedef struct irq_alloc_info msi_alloc_info_t; #endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ +#else /* !CONFIG_PCI_MSI */ +static inline void handle_pci_msi_interrupt(unsigned long type, + unsigned long vector, unsigned long pci_msi1_addr) +{ + pr_warn("SW arch disable CONFIG_PCI_MSI option.\n"); +} #endif /* CONFIG_PCI_MSI */ #endif /* _ASM_SW64_MSI_H */ diff --git a/arch/sw_64/include/asm/page.h b/arch/sw_64/include/asm/page.h index 1ba34f4f507787773f77c0e88fe2166abd87d24c..5a32d1298c3ba3f922e6c12744232de947879cbf 100644 --- a/arch/sw_64/include/asm/page.h +++ b/arch/sw_64/include/asm/page.h @@ -33,20 +33,26 @@ extern void copy_page(void *_to, void *_from); typedef struct page *pgtable_t; extern unsigned long __phys_addr(unsigned long); +#ifdef CONFIG_SUBARCH_C3B +extern unsigned long __boot_phys_addr(unsigned long); +#else +#define __boot_phys_addr(x) __phys_addr(x) +#endif + #endif /* !__ASSEMBLY__ */ #define KERNEL_IMAGE_SIZE (512 * 1024 * 1024) #include -#if defined(CONFIG_SW64_LEGACY_KTEXT_ADDRESS) #define __START_KERNEL_map PAGE_OFFSET -#else -#define __START_KERNEL_map 0xffffffff80000000 -#endif #define __pa(x) __phys_addr((unsigned long)(x)) #define __va(x) ((void *)((unsigned long) (x) | PAGE_OFFSET)) + +#define __boot_pa(x) __boot_phys_addr((unsigned long)(x)) +#define __boot_va(x) __va(x) + #define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) #define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index de175b3c1043f86d4e81be7e78d92554de62b944..37d82d84449dbe62198ef7887087a3421b59f921 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -74,7 +74,7 @@ struct pci_controller { * bus numbers. */ -#define pcibios_assign_all_busses() 1 +#define pcibios_assign_all_busses() (pci_has_flag(PCI_REASSIGN_ALL_BUS)) #define PCIBIOS_MIN_IO 0 #define PCIBIOS_MIN_MEM 0 @@ -89,8 +89,13 @@ extern void __init sw64_init_arch(void); extern struct pci_ops sw64_pci_ops; extern int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); extern struct pci_controller *hose_head; +#ifdef CONFIG_PCI_SW64 +extern void __init setup_chip_pci_ops(void); +#else +#define setup_chip_pci_ops() do { } while (0) +#endif -#ifdef CONFIG_SUNWAY_IOMMU +#if defined(CONFIG_SUNWAY_IOMMU) || defined(CONFIG_SUNWAY_IOMMU_V2) extern struct syscore_ops iommu_cpu_syscore_ops; #endif @@ -147,7 +152,7 @@ extern void __init reserve_mem_for_pci(void); extern int chip_pcie_configure(struct pci_controller *hose); #define PCI_VENDOR_ID_JN 0x5656 -#define PCI_DEVICE_ID_CHIP3 0x3231 +#define PCI_DEVICE_ID_SW64_ROOT_BRIDGE 0x3231 #define PCI_DEVICE_ID_JN_PCIESW 0x1000 #define PCI_DEVICE_ID_JN_PCIEUSIP 0x1200 #define PCI_DEVICE_ID_JN_PCIE2PCI 0x1314 diff --git a/arch/sw_64/include/asm/perf_event.h b/arch/sw_64/include/asm/perf_event.h index 382a74e8501124b2f646a956940a6866581b694c..dc55a361babd015aa92fbf7b0387f1e2beeecc40 100644 --- a/arch/sw_64/include/asm/perf_event.h +++ b/arch/sw_64/include/asm/perf_event.h @@ -2,7 +2,7 @@ #ifndef _ASM_SW64_PERF_EVENT_H #define _ASM_SW64_PERF_EVENT_H -#include +#include #include #ifdef CONFIG_PERF_EVENTS diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 6610e62f4123ec8ebf2aeb40e88e4d99004d5d28..270baa15a8294ff12100aa4e031af9636c2ddc0e 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -15,6 +15,7 @@ #include #include +#include #include #include /* For TASK_SIZE */ #include @@ -22,21 +23,6 @@ struct mm_struct; struct vm_area_struct; -/* Certain architectures need to do special things when PTEs - * within a page table are directly modified. Thus, the following - * hook is made available. - */ -static inline void set_pte(pte_t *ptep, pte_t pteval) -{ - *ptep = pteval; -} - -static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pteval) -{ - set_pte(ptep, pteval); -} - static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; @@ -72,6 +58,11 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE - 1)) +#define CONT_PMD_SHIFT 6 +#define CONT_PMDS (1 << CONT_PMD_SHIFT) +#define CONT_PMD_SIZE (CONT_PMDS * PMD_SIZE) +#define CONT_PMD_MASK (~(CONT_PMD_SIZE - 1)) + /* * Entries per page directory level: the sw64 is three-level, with * all levels having a one-page page table. @@ -92,20 +83,23 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define VMALLOC_END (-PGDIR_SIZE) #else #define VMEMMAP_END (-PGDIR_SIZE) -#define vmemmap ((struct page *)VMEMMAP_END - (1UL << (3 * (PAGE_SHIFT - 3)))) +#define vmemmap ((struct page *)VMEMMAP_END - (1UL << (MAX_PHYSMEM_BITS - PAGE_SHIFT))) #define VMALLOC_END ((unsigned long)vmemmap) #endif /* * HMcode-imposed page table bits */ +#if defined(CONFIG_SUBARCH_C3B) + #define _PAGE_VALID 0x0001 +#define _PAGE_PRESENT _PAGE_VALID #define _PAGE_FOR 0x0002 /* used for page protection (fault on read) */ #define _PAGE_FOW 0x0004 /* used for page protection (fault on write) */ #define _PAGE_FOE 0x0008 /* used for page protection (fault on exec) */ #define _PAGE_ASM 0x0010 -#define _PAGE_PHU 0x0020 /* used for 256M page size bit */ -#define _PAGE_PSE 0x0040 /* used for 8M page size bit */ +#define _PAGE_CONT 0x0020 /* used for 256M page size bit */ +#define _PAGE_LEAF 0x0040 /* used for 8M page size bit */ #define _PAGE_PROTNONE 0x0080 /* used for numa page balancing */ #define _PAGE_SPECIAL 0x0100 #define _PAGE_KRE 0x0400 /* xxx - see below on the "accessed" bit */ @@ -117,12 +111,13 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define _PAGE_DIRTY 0x20000 #define _PAGE_ACCESSED 0x40000 -#define _PAGE_BIT_ACCESSED 18 /* bit of _PAGE_ACCESSED */ -#define _PAGE_BIT_FOW 2 /* bit of _PAGE_FOW */ #define _PAGE_SPLITTING 0x200000 /* For Transparent Huge Page */ +#define _PAGE_DEVMAP 0x400000 /* For ZONE DEVICE page */ + +#define _PAGE_BIT_FOW 2 /* bit of _PAGE_FOW */ +#define _PAGE_BIT_ACCESSED 18 /* bit of _PAGE_ACCESSED */ #define _PAGE_BIT_SPLITTING 21 /* bit of _PAGE_SPLITTING */ #define _PAGE_BIT_DEVMAP 22 /* bit of _PAGE_DEVMAP */ - /* * NOTE! The "accessed" bit isn't necessarily exact: it can be kept exactly * by software (use the KRE/URE/KWE/UWE bits appropriately), but I'll fake it. @@ -137,13 +132,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define __DIRTY_BITS (_PAGE_DIRTY | _PAGE_KWE | _PAGE_UWE) #define __ACCESS_BITS (_PAGE_ACCESSED | _PAGE_KRE | _PAGE_URE) - #define _PFN_SHIFT 28 -#define _PFN_MASK ((-1UL) << _PFN_SHIFT) - -#define _PAGE_TABLE (_PAGE_VALID | __DIRTY_BITS | __ACCESS_BITS) -#define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL) -#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PHU) /* * All the normal masks have the "page accessed" bits on, as any time they are used, @@ -156,6 +145,59 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define PAGE_KERNEL __pgprot(_PAGE_VALID | _PAGE_ASM | _PAGE_KRE | _PAGE_KWE) #define _PAGE_NORMAL(x) __pgprot(_PAGE_VALID | __ACCESS_BITS | (x)) +#define page_valid_kern(x) (0) + +#elif defined(CONFIG_SUBARCH_C4) + +#define _PAGE_VALID 0x0001 +#define _PAGE_PRESENT _PAGE_VALID +#define _PAGE_FOR 0x0002 /* used for page protection (fault on read) */ +#define _PAGE_FOW 0x0004 /* used for page protection (fault on write) */ +#define _PAGE_FOE 0x0008 /* used for page protection (fault on exec) */ +#define _PAGE_FIXED 0x0010 +#define _PAGE_CONT 0x0020 /* used for 512M page size bit*/ +#define _PAGE_LEAF 0x0040 /* used for huge page bit */ +#define _PAGE_PCD 0x0080 /* used for page cache disabled */ + +/* and these are sw definition */ +#define _PAGE_WCD 0x0100 +#define _PAGE_ACCESSED 0x0200 +#define _PAGE_SPLITTING 0x0400 /* For Transparent Huge Page */ +#define _PAGE_SPECIAL 0x0800 +#define _PAGE_DEVMAP 0x1000 /* For ZONE DEVICE page */ +#define _PAGE_KERN 0x2000 +#define _PAGE_DIRTY _BITUL(62) +#define _PAGE_PROTNONE _BITUL(63) +#define _PAGE_BIT_FOW 2 /* bit of _PAGE_FOW */ +#define _PAGE_BIT_ACCESSED 9 /* bit of _PAGE_ACCESSED */ +#define _PAGE_BIT_SPLITTING 10 /* bit of _PAGE_SPLITTING */ +#define _PAGE_BIT_DEVMAP 12 /* bit of _PAGE_DEVMAP */ + +#define __DIRTY_BITS _PAGE_DIRTY +#define __ACCESS_BITS _PAGE_ACCESSED + +#define _PFN_SHIFT 24 + +/* + * All the normal masks have the "page accessed" bits on, as any time they are used, + * the page is accessed. They are cleared only by the page-out routines + */ +#define PAGE_NONE __pgprot(__ACCESS_BITS | _PAGE_FOR | _PAGE_FOW | _PAGE_FOE | _PAGE_LEAF | _PAGE_PROTNONE) +#define PAGE_SHARED __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_LEAF) +#define PAGE_COPY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW | _PAGE_LEAF) +#define PAGE_READONLY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW | _PAGE_LEAF) +#define PAGE_KERNEL __pgprot(_PAGE_VALID | _PAGE_KERN | _PAGE_LEAF) +#define _PAGE_NORMAL(x) __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_LEAF | (x)) + +#define page_valid_kern(x) ((x & (_PAGE_VALID | _PAGE_KERN)) == (_PAGE_VALID | _PAGE_KERN)) +#endif + +#define _PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) +#define _PFN_MASK (GENMASK(_PFN_BITS - 1, 0) << _PFN_SHIFT) + +#define _PAGE_TABLE (_PAGE_VALID | __DIRTY_BITS | __ACCESS_BITS) +#define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL | _PAGE_LEAF | _PAGE_CONT) + #define _PAGE_P(x) _PAGE_NORMAL((x) | _PAGE_FOW) #define _PAGE_S(x) _PAGE_NORMAL(x) @@ -195,9 +237,25 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. */ +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; +#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) -extern struct page *empty_zero_page; -#define ZERO_PAGE(vaddr) (empty_zero_page) +static inline void set_pte(pte_t *ptep, pte_t pteval) +{ + *ptep = pteval; + + if (page_valid_kern(pte_val(pteval))) { + mb(); + if ((pte_val(pteval) & _PAGE_FOE) == 0) + imemb(); + } +} + +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pteval) +{ + set_pte(ptep, pteval); +} static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) { @@ -214,6 +272,13 @@ static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) pmd_val(pmd) = (pfn << _PFN_SHIFT) | pgprot_val(prot); return pmd; } +static inline pud_t pfn_pud(unsigned long pfn, pgprot_t pgprot) +{ + pud_t pud; + + pud_val(pud) = (pfn << _PFN_SHIFT) | pgprot_val(pgprot); + return pud; +} static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { @@ -223,40 +288,41 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { - pmd_val(pmd) = (pmd_val(pmd) & _HPAGE_CHG_MASK) | pgprot_val(newprot); + pmd_val(pmd) = (pmd_val(pmd) & _PAGE_CHG_MASK) | pgprot_val(newprot); return pmd; } -static inline unsigned long pmd_page_vaddr(pmd_t pmd) -{ - return (unsigned long)pfn_to_virt(pmd_val(pmd) >> _PFN_SHIFT); -} - /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. */ #define page_to_pa(page) (page_to_pfn(page) << PAGE_SHIFT) -#define pud_pfn(pud) (pud_val(pud) >> _PFN_SHIFT) -#define pmd_pfn(pmd) (pmd_val(pmd) >> _PFN_SHIFT) -#define pte_pfn(pte) (pte_val(pte) >> _PFN_SHIFT) +#define p4d_pfn(p4d) ((p4d_val(p4d) & _PFN_MASK) >> _PFN_SHIFT) +#define pud_pfn(pud) ((pud_val(pud) & _PFN_MASK) >> _PFN_SHIFT) +#define pmd_pfn(pmd) ((pmd_val(pmd) & _PFN_MASK) >> _PFN_SHIFT) +#define pte_pfn(pte) ((pte_val(pte) & _PFN_MASK) >> _PFN_SHIFT) +#define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d)) +#define pud_page(pud) pfn_to_page(pud_pfn(pud)) +#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd)) #define pte_page(pte) pfn_to_page(pte_pfn(pte)) -#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) -#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> _PFN_SHIFT)) -#define pud_page(pud) (pfn_to_page(pud_val(pud) >> _PFN_SHIFT)) -#define p4d_page(p4d) (pfn_to_page(p4d_val(p4d) >> _PFN_SHIFT)) +#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) static inline pud_t *p4d_pgtable(p4d_t p4d) { - return (pud_t *)pfn_to_virt(p4d_val(p4d) >> _PFN_SHIFT); + return (pud_t *)pfn_to_virt(p4d_pfn(p4d)); } static inline pmd_t *pud_pgtable(pud_t pud) { - return (pmd_t *)pfn_to_virt(pud_val(pud) >> _PFN_SHIFT); + return (pmd_t *)pfn_to_virt(pud_pfn(pud)); +} + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)pfn_to_virt(pmd_pfn(pmd)); } static inline int pte_none(pte_t pte) @@ -271,7 +337,7 @@ static inline int pte_present(pte_t pte) static inline int pte_huge(pte_t pte) { - return pte_val(pte) & _PAGE_PSE; + return pte_val(pte) & _PAGE_LEAF; } static inline void pte_clear(struct mm_struct *mm, @@ -306,12 +372,12 @@ static inline int pmd_bad(pmd_t pmd) static inline int pmd_present(pmd_t pmd) { /* - * Checking for _PAGE_PSE is needed too because + * Checking for _PAGE_LEAF is needed too because * split_huge_page will temporarily clear the valid bit (but - * the _PAGE_PSE flag will remain set at all times while the + * the _PAGE_LEAF flag will remain set at all times while the * _PAGE_VALID bit is clear). */ - return pmd_val(pmd) & (_PAGE_VALID | _PAGE_PROTNONE | _PAGE_PSE); + return pmd_val(pmd) & (_PAGE_VALID | _PAGE_PROTNONE | _PAGE_LEAF); } static inline void pmd_clear(pmd_t *pmdp) @@ -373,6 +439,12 @@ static inline pmd_t pmd_mkdirty(pmd_t pmd) return pmd; } +static inline pmd_t pmd_mkdevmap(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_DEVMAP; + return pmd; +} + static inline pmd_t pmd_mkyoung(pmd_t pmd) { pmd_val(pmd) |= __ACCESS_BITS; @@ -381,7 +453,13 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) static inline pmd_t pmd_mkhuge(pmd_t pmd) { - pmd_val(pmd) |= _PAGE_PSE; + pmd_val(pmd) |= _PAGE_LEAF; + return pmd; +} + +static inline pmd_t pmd_mkcont(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_CONT; return pmd; } @@ -405,6 +483,12 @@ static inline void pud_clear(pud_t *pudp) pud_val(*pudp) = 0; } +static inline pud_t pud_mkhuge(pud_t pud) +{ + pud_val(pud) |= _PAGE_LEAF; + return pud; +} + static inline int p4d_none(p4d_t p4d) { return !p4d_val(p4d); @@ -425,6 +509,16 @@ static inline void p4d_clear(p4d_t *p4dp) p4d_val(*p4dp) = 0; } +static inline pte_t pmd_pte(pmd_t pmd) +{ + return __pte(pmd_val(pmd)); +} + +static inline pmd_t pte_pmd(pte_t pte) +{ + return __pmd(pte_val(pte)); +} + /* * The following only work if pte_present() is true. * Undefined behaviour if not.. @@ -449,6 +543,11 @@ static inline int pte_special(pte_t pte) return pte_val(pte) & _PAGE_SPECIAL; } +static inline int pte_cont(pte_t pte) +{ + return pte_val(pte) & _PAGE_CONT; +} + static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) |= _PAGE_FOW; @@ -488,7 +587,7 @@ static inline pte_t pte_mkyoung(pte_t pte) static inline pte_t pte_mkhuge(pte_t pte) { - pte_val(pte) |= _PAGE_PSE; + pte_val(pte) |= _PAGE_LEAF; return pte; } @@ -521,6 +620,12 @@ static inline int pmd_protnone(pmd_t pmd) } #endif +#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP +static inline int pte_devmap(pte_t a) +{ + return (pte_val(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP; +} +#endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -532,9 +637,14 @@ static inline int pmd_trans_splitting(pmd_t pmd) return pmd_val(pmd) & _PAGE_SPLITTING; } +static inline int pmd_trans_cont(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_CONT; +} + static inline int pmd_trans_huge(pmd_t pmd) { - return pmd_val(pmd) & _PAGE_PSE; + return pmd_val(pmd) & _PAGE_LEAF; } static inline int has_transparent_hugepage(void) @@ -543,12 +653,6 @@ static inline int has_transparent_hugepage(void) } #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP -#define _PAGE_DEVMAP (_AT(u64, 1) << _PAGE_BIT_DEVMAP) -static inline int pte_devmap(pte_t a) -{ - return (pte_val(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP; -} - static inline int pmd_devmap(pmd_t pmd) { return !!(pmd_val(pmd) & _PAGE_DEVMAP); @@ -573,12 +677,6 @@ static inline int pgd_devmap(pgd_t pgd) #endif #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static inline pmd_t pmd_mkdevmap(pmd_t pmd) -{ - pmd_val(pmd) |= _PAGE_DEVMAP; - return pmd; -} - #define __HAVE_ARCH_PMDP_GET_AND_CLEAR static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) @@ -610,7 +708,6 @@ extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, extern int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); - #define __HAVE_ARCH_PMDP_SPLITTING_FLUSH extern void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp); @@ -624,18 +721,37 @@ extern pgd_t swapper_pg_dir[1024]; #define update_mmu_cache(vma, address, ptep) do { } while (0) #define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) +#if defined(CONFIG_SUBARCH_C3B) + /* * Encode and decode a swap entry: * * Format of swap PTE: * bit 0: _PAGE_VALID (must be zero) - * bit 6: _PAGE_PSE (must be zero) + * bit 6: _PAGE_LEAF (must be zero) * bit 7: _PAGE_PROTNONE (must be zero) * bits 8-15: swap type * bits 16-63: swap offset */ #define __SWP_TYPE_SHIFT 8 #define __SWP_TYPE_BITS 8 + +#elif defined(CONFIG_SUBARCH_C4) + +/* + * Encode and decode a swap entry: + * + * Format of swap PTE: + * bit 0: _PAGE_VALID (must be zero) + * bits 6-10: swap type + * bits 11-58: swap offset + * bit 63: _PAGE_PROTNONE (must be zero) + */ +#define __SWP_TYPE_SHIFT 6 +#define __SWP_TYPE_BITS 5 + +#endif + #define __SWP_OFFSET_BITS 48 #define __SWP_TYPE_MASK ((1UL << __SWP_TYPE_BITS) - 1) #define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h index 03f098bb4cadb2198a9ddd154f9c9abfa68166d7..ad54cdc772e13e8a5f9d8430406d11064dfb7ccb 100644 --- a/arch/sw_64/include/asm/platform.h +++ b/arch/sw_64/include/asm/platform.h @@ -3,13 +3,11 @@ #define _ASM_SW64_PLATFORM_H #include -#include - -struct sw64_platform_ops { - void __iomem *(*ioportmap)(unsigned long); - void (*register_platform_devices)(void); - void (*ops_fixup)(void); -}; +#if defined(CONFIG_UNCORE_XUELANG) +#include +#elif defined(CONFIG_UNCORE_JUNZHANG) +#include +#endif #ifdef CONFIG_EFI #define BIOS_VERSION_GUID EFI_GUID(0xc47a23c3, 0xcebb, 0x4cc9, 0xa5, 0xe2, 0xde, 0xd0, 0x8f, 0xe4, 0x20, 0xb5) @@ -20,9 +18,6 @@ extern unsigned long bios_version; #endif -extern struct sw64_platform_ops *sw64_platform; - -extern struct sw64_platform_ops xuelang_ops; extern struct boot_params *sunway_boot_params; extern void sw64_halt(void); diff --git a/arch/sw_64/include/asm/wrperfmon.h b/arch/sw_64/include/asm/pmc.h similarity index 41% rename from arch/sw_64/include/asm/wrperfmon.h rename to arch/sw_64/include/asm/pmc.h index c06a05121a68be93e265928f0651bfda2ad43ac8..d5672dd940a791c62e0edbe1e5e2356183cdd131 100644 --- a/arch/sw_64/include/asm/wrperfmon.h +++ b/arch/sw_64/include/asm/pmc.h @@ -1,44 +1,35 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Definitions for use with the sw64 wrperfmon HMCODE call. + * Definitions for use with the sw64 PMC interface. */ -#ifndef _ASM_SW64_WRPERFMON_H -#define _ASM_SW64_WRPERFMON_H +#ifndef _ASM_SW64_PMC_H +#define _ASM_SW64_PMC_H -#define PERFMON_PC0 0 -#define PERFMON_PC1 1 +#define PMC_PC0 0 +#define PMC_PC1 1 /* Following commands are implemented on all CPUs */ -#define PERFMON_CMD_DISABLE 0 -#define PERFMON_CMD_ENABLE 1 -#define PERFMON_CMD_EVENT_PC0 2 -#define PERFMON_CMD_EVENT_PC1 3 -#define PERFMON_CMD_PM 4 -#define PERFMON_CMD_READ 5 -#define PERFMON_CMD_READ_CLEAR 6 -#define PERFMON_CMD_WRITE_PC0 7 -#define PERFMON_CMD_WRITE_PC1 8 +#define PMC_CMD_DISABLE 0 +#define PMC_CMD_ENABLE 1 +#define PMC_CMD_EVENT_BASE 2 +#define PMC_CMD_PM 4 +#define PMC_CMD_READ 5 +#define PMC_CMD_READ_CLEAR 6 +#define PMC_CMD_WRITE_BASE 7 -#define PERFMON_DISABLE_ARGS_PC0 1 -#define PERFMON_DISABLE_ARGS_PC1 2 -#define PERFMON_DISABLE_ARGS_PC 3 +#define PMC_DISABLE_BASE 1 -#define PERFMON_ENABLE_ARGS_PC0 1 -#define PERFMON_ENABLE_ARGS_PC1 2 -#define PERFMON_ENABLE_ARGS_PC 3 +#define PMC_ENABLE_BASE 1 -#define PERFMON_READ_PC0 0 -#define PERFMON_READ_PC1 1 +#define PC0_RAW_BASE 0x0 +#define PC1_RAW_BASE 0x100 +#define PC0_MAX 0xF +#define PC1_MAX 0x3D -#define PC0_RAW_BASE 0x0 -#define PC1_RAW_BASE 0x100 -#define PC0_MAX 0xF -#define PC1_MAX 0x3D - -#define SW64_PERFCTRL_KM 2 -#define SW64_PERFCTRL_UM 3 -#define SW64_PERFCTRL_AM 4 +#define SW64_PERFCTRL_KM 2 +#define SW64_PERFCTRL_UM 3 +#define SW64_PERFCTRL_AM 4 /* pc0 events */ #define PC0_INSTRUCTIONS 0x0 @@ -61,4 +52,4 @@ #define MAX_HWEVENTS 2 #define PMC_COUNT_MASK ((1UL << 58) - 1) -#endif /* _ASM_SW64_WRPERFMON_H */ +#endif /* _ASM_SW64_PMC_H */ diff --git a/arch/sw_64/include/asm/processor.h b/arch/sw_64/include/asm/processor.h index 4c1065b61af22d514224c7beba7a68394187af2f..26394854071f1897ba859fc802bf4ce6873b8bff 100644 --- a/arch/sw_64/include/asm/processor.h +++ b/arch/sw_64/include/asm/processor.h @@ -65,8 +65,7 @@ unsigned long get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc) -#define KSTK_ESP(tsk) \ - ((tsk) == current ? rdusp() : task_thread_info(tsk)->pcb.usp) +#define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[30]) #define cpu_relax() barrier() diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h index 2c60bc6730ad406d04a9ba44193380236fd19227..62fc43696316bba09b47338fa2c8aa96dce74a0b 100644 --- a/arch/sw_64/include/asm/ptrace.h +++ b/arch/sw_64/include/asm/ptrace.h @@ -6,6 +6,11 @@ #include #include +#define NO_SYSCALL (-1) + +#ifdef __KERNEL__ +#ifndef __ASSEMBLY__ + /* * This struct defines the way the registers are stored on the * kernel stack during a system call or other kernel entry @@ -15,41 +20,13 @@ struct pt_regs { union { struct user_pt_regs user_regs; struct { - unsigned long r0; - unsigned long r1; - unsigned long r2; - unsigned long r3; - unsigned long r4; - unsigned long r5; - unsigned long r6; - unsigned long r7; - unsigned long r8; - unsigned long r9; - unsigned long r10; - unsigned long r11; - unsigned long r12; - unsigned long r13; - unsigned long r14; - unsigned long r15; - unsigned long r16; - unsigned long r17; - unsigned long r18; - unsigned long r19; - unsigned long r20; - unsigned long r21; - unsigned long r22; - unsigned long r23; - unsigned long r24; - unsigned long r25; - unsigned long r26; - unsigned long r27; - unsigned long r28; - unsigned long gp; - unsigned long sp; + unsigned long regs[31]; unsigned long pc; unsigned long ps; }; }; + unsigned long orig_r0; + unsigned long orig_r19; /* These are saved by HMcode: */ unsigned long hm_ps; unsigned long hm_pc; @@ -63,17 +40,19 @@ struct pt_regs { #define user_mode(regs) (((regs)->ps & 8) != 0) #define instruction_pointer(regs) ((regs)->pc) #define profile_pc(regs) instruction_pointer(regs) -#define current_user_stack_pointer() rdusp() -#define user_stack_pointer(regs) rdusp() +#define user_stack_pointer(pt_regs) ((pt_regs)->regs[30]) #define kernel_stack_pointer(regs) ((unsigned long)((regs) + 1)) #define instruction_pointer_set(regs, val) ((regs)->pc = val) -#define force_successful_syscall_return() (current_pt_regs()->r0 = 0) +#define force_successful_syscall_return() (current_pt_regs()->orig_r0 = NO_SYSCALL) -#define MAX_REG_OFFSET (offsetof(struct pt_regs, ps)) +#define MAX_REG_OFFSET (offsetof(struct pt_regs, orig_r0)) extern short regoffsets[]; +extern unsigned long syscall_trace_enter(void); +extern void syscall_trace_leave(void); + /** * regs_get_register() - get register value from its offset * @regs: pt_regs from which register value is gotten @@ -94,8 +73,20 @@ extern int regs_query_register_offset(const char *name); extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n); -static inline unsigned long regs_return_value(struct pt_regs *regs) +static inline int is_syscall_success(struct pt_regs *regs) +{ + return !regs->regs[19]; +} + +static inline long regs_return_value(struct pt_regs *regs) { - return regs->r0; + if ((regs->orig_r0 == NO_SYSCALL) || is_syscall_success(regs)) + return regs->regs[0]; + else + return -regs->regs[0]; } + +#endif /* !__ASSEMBLY__ */ +#endif /* __KERNEL__ */ + #endif /* _ASM_SW64_PTRACE_H */ diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h index fed9d682f1d745862bd0ea20a6ef02abd2db2505..3a2fcf62b30cabe5512a85946a31852d339e62b4 100644 --- a/arch/sw_64/include/asm/smp.h +++ b/arch/sw_64/include/asm/smp.h @@ -16,18 +16,6 @@ extern cpumask_t core_start; -static inline unsigned char -__hard_smp_processor_id(void) -{ - register unsigned char __r0 __asm__("$0"); - __asm__ __volatile__( - "sys_call %1 #whami" - : "=r"(__r0) - : "i" (HMC_whami) - : "$1", "$22", "$23", "$24", "$25"); - return __r0; -} - static inline unsigned long read_vpcr(void) { @@ -56,7 +44,6 @@ struct smp_rcb_struct { #define INIT_SMP_RCB ((struct smp_rcb_struct *) __va(0x820000UL)) -#define hard_smp_processor_id() __hard_smp_processor_id() #ifdef GENERATING_ASM_OFFSETS #define raw_smp_processor_id() (0) @@ -64,17 +51,11 @@ struct smp_rcb_struct { #include #define raw_smp_processor_id() (*((unsigned int *)((void *)current + TASK_CPU))) #endif +#define hard_smp_processor_id() cpu_to_rcid(raw_smp_processor_id()) /* The map from sequential logical cpu number to hard cid. */ extern int __cpu_to_rcid[NR_CPUS]; #define cpu_to_rcid(cpu) __cpu_to_rcid[cpu] - -/* - * Map from hard cid to sequential logical cpu number. This will only - * not be idempotent when cpus failed to come on-line. - */ -extern int __rcid_to_cpu[NR_CPUS]; -#define rcid_to_cpu(cpu) __rcid_to_cpu[cpu] #define cpu_physical_id(cpu) __cpu_to_rcid[cpu] extern unsigned long tidle_pcb[NR_CPUS]; @@ -89,8 +70,10 @@ void __cpu_die(unsigned int cpu); #else /* CONFIG_SMP */ #define hard_smp_processor_id() 0 #define smp_call_function_on_cpu(func, info, wait, cpu) ({ 0; }) -#define cpu_to_rcid(cpu) ((int)whami()) -#define rcid_to_cpu(rcid) 0 +/* The map from sequential logical cpu number to hard cid. */ +extern int __cpu_to_rcid[NR_CPUS]; +#define cpu_to_rcid(cpu) __cpu_to_rcid[0] +#define cpu_physical_id(cpu) __cpu_to_rcid[0] #endif /* CONFIG_SMP */ #define NO_PROC_ID (-1) diff --git a/arch/sw_64/include/asm/suspend.h b/arch/sw_64/include/asm/suspend.h index 7b6d7bfc95952197e91421b8567f0e0b696fd2a0..833e27f9d5e14a729a285406234e90fc03afbdfe 100644 --- a/arch/sw_64/include/asm/suspend.h +++ b/arch/sw_64/include/asm/suspend.h @@ -46,4 +46,5 @@ struct processor_state { }; extern void sw64_suspend_deep_sleep(struct processor_state *state); +extern const struct platform_suspend_ops native_suspend_ops; #endif /* _ASM_SW64_SUSPEND_H */ diff --git a/arch/sw_64/include/asm/sw64_init.h b/arch/sw_64/include/asm/sw64_init.h index 8b437ef73cde74b94cf52fe2c7f6d9661e7c4070..86ddd2cb65f839bfa9cc609aa6878740b4c484b3 100644 --- a/arch/sw_64/include/asm/sw64_init.h +++ b/arch/sw_64/include/asm/sw64_init.h @@ -8,7 +8,7 @@ #include struct sw64_early_init_ops { - void (*setup_core_start)(struct cpumask *cpumask); + void (*setup_core_map)(struct cpumask *cpumask); unsigned long (*get_node_mem)(int nodeid); void (*get_smp_info)(void); }; @@ -38,9 +38,12 @@ struct sw64_chip_ops { }; extern void sw64_init_noop(void); -extern void sw64_setup_chip_ops(void); +extern void setup_chip_ops(void); extern struct sw64_chip_ops *sw64_chip; extern struct sw64_chip_init_ops *sw64_chip_init; +#ifdef CONFIG_PM +extern struct syscore_ops io_syscore_ops; +#endif DECLARE_PER_CPU(unsigned long, hard_node_id); diff --git a/arch/sw_64/include/asm/sw64io.h b/arch/sw_64/include/asm/sw64io.h index 0892356b8e6b7ad6cca0d81906afbcd48a24df33..d52cd8cc86bf24aa3a7b45356bcbbf2651cf7b01 100644 --- a/arch/sw_64/include/asm/sw64io.h +++ b/arch/sw_64/include/asm/sw64io.h @@ -5,10 +5,12 @@ #include #include -extern void setup_chip_clocksource(void); +#if defined(CONFIG_UNCORE_XUELANG) +#include +#endif -#if defined(CONFIG_SW64_CHIP3) -#include +#if defined(CONFIG_UNCORE_JUNZHANG) +#include #endif #define MK_RC_CFG(nid, idx) \ @@ -95,4 +97,13 @@ sw64_io_write(unsigned long node, unsigned long reg, unsigned long data) addr = __va(SW64_IO_BASE(node) | reg); writeq(data, addr); } + +#if defined(CONFIG_UNCORE_XUELANG) +#include +#endif + +#if defined(CONFIG_UNCORE_JUNZHANG) +#include +#endif + #endif /* _ASM_SW64_SW64IO_H */ diff --git a/arch/sw_64/include/asm/switch_to.h b/arch/sw_64/include/asm/switch_to.h index e5596a735b2dbb4ef3fc98b86953db3c2666cd28..f4af18c5710c5b7ac6f3f241214f922cdaaa8946 100644 --- a/arch/sw_64/include/asm/switch_to.h +++ b/arch/sw_64/include/asm/switch_to.h @@ -2,7 +2,7 @@ #ifndef _ASM_SW64_SWITCH_TO_H #define _ASM_SW64_SWITCH_TO_H -#include +#include extern void __fpstate_save(struct task_struct *save_to); extern void __fpstate_restore(struct task_struct *restore_from); @@ -16,7 +16,6 @@ static inline void aux_save(struct task_struct *task) if (likely(!(task->flags & PF_KTHREAD))) { pcb = &task_thread_info(task)->pcb; - pcb->usp = rdusp(); pcb->tp = rtid(); __fpstate_save(task); } @@ -28,7 +27,6 @@ static inline void aux_restore(struct task_struct *task) if (likely(!(task->flags & PF_KTHREAD))) { pcb = &task_thread_info(task)->pcb; - wrusp(pcb->usp); wrtp(pcb->tp); __fpstate_restore(task); } diff --git a/arch/sw_64/include/asm/syscall.h b/arch/sw_64/include/asm/syscall.h index 0b1556a460b463ab5962ce7368768fdfcf8a510e..a821bf68be1643d7f457ad738d9cde9cc1930d77 100644 --- a/arch/sw_64/include/asm/syscall.h +++ b/arch/sw_64/include/asm/syscall.h @@ -4,23 +4,28 @@ #include -extern void *sys_call_table[]; +#ifndef __ASSEMBLY__ + +typedef long (*syscall_fn_t)(ulong, ulong, ulong, ulong, ulong, ulong); + +extern syscall_fn_t sys_call_table[]; + static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) { - return regs->r0; + return regs->regs[0]; } static inline long syscall_get_error(struct task_struct *task, struct pt_regs *regs) { - return regs->r19 ? -regs->r0 : 0; + return regs->regs[19] ? -regs->regs[0] : 0; } static inline long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) { - return regs->r0; + return regs->regs[0]; } static inline void syscall_set_return_value(struct task_struct *task, @@ -28,43 +33,43 @@ static inline void syscall_set_return_value(struct task_struct *task, int error, long val) { if (error) { - regs->r0 = -error; - regs->r19 = -1; + regs->regs[0] = -error; + regs->regs[19] = 1; } else { - regs->r0 = val; - regs->r19 = 0; + regs->regs[0] = val; + regs->regs[19] = 0; } } - static inline void syscall_rollback(struct task_struct *task, struct pt_regs *regs) { - /* Do nothing */ + regs->regs[0] = regs->orig_r0; + regs->regs[19] = regs->orig_r19; } static inline void syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, unsigned long *args) { - *args++ = regs->r16; - *args++ = regs->r17; - *args++ = regs->r18; - *args++ = regs->r19; - *args++ = regs->r20; - *args = regs->r21; + *args++ = regs->regs[16]; + *args++ = regs->regs[17]; + *args++ = regs->regs[18]; + *args++ = regs->regs[19]; + *args++ = regs->regs[20]; + *args = regs->regs[21]; } static inline void syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, const unsigned long *args) { - regs->r16 = *args++; - regs->r17 = *args++; - regs->r18 = *args++; - regs->r19 = *args++; - regs->r20 = *args++; - regs->r21 = *args; + regs->regs[16] = *args++; + regs->regs[17] = *args++; + regs->regs[18] = *args++; + regs->regs[19] = *args++; + regs->regs[20] = *args++; + regs->regs[21] = *args; } static inline int syscall_get_arch(struct task_struct *task) @@ -72,4 +77,6 @@ static inline int syscall_get_arch(struct task_struct *task) return AUDIT_ARCH_SW64; } +#endif /* !__ASSEMBLY__ */ + #endif /* _ASM_SW64_SYSCALL_H */ diff --git a/arch/sw_64/include/asm/thread_info.h b/arch/sw_64/include/asm/thread_info.h index a8341d64ad43045dec13bcb574da61b96cb41786..38b06413121c874279bb663cc17d76b7d950752b 100644 --- a/arch/sw_64/include/asm/thread_info.h +++ b/arch/sw_64/include/asm/thread_info.h @@ -14,11 +14,16 @@ typedef struct { struct pcb_struct { - unsigned long usp; unsigned long tp; unsigned long da_match, da_mask; unsigned long dv_match, dv_mask; - unsigned long dc_ctl; + union { + unsigned long dc_ctl; + unsigned long match_ctl; + }; + unsigned long ia_match, ia_mask; + unsigned long iv_match; + unsigned long ida_match, ida_mask; }; struct thread_info { @@ -82,6 +87,7 @@ static __always_inline u64 rtid(void) #define TIF_SYSCALL_AUDIT 4 /* syscall audit active */ #define TIF_UPROBE 5 /* uprobe breakpoint or singlestep */ #define TIF_PATCH_PENDING 6 /* pending live patching update */ +#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ #define TIF_DIE_IF_KERNEL 9 /* dik recursion lock */ #define TIF_SYSCALL_TRACEPOINT 10 #define TIF_SECCOMP 11 /* secure computing */ @@ -92,15 +98,18 @@ static __always_inline u64 rtid(void) #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_PATCH_PENDING (1 << TIF_PATCH_PENDING) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) #define _TIF_SECCOMP (1 << TIF_SECCOMP) +#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) #define _TIF_UPROBE (1 << TIF_UPROBE) /* Work to do on interrupt/exception return. */ #define _TIF_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ - _TIF_NOTIFY_RESUME | _TIF_UPROBE) + _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ + _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL) #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP) diff --git a/arch/sw_64/include/asm/timer.h b/arch/sw_64/include/asm/timer.h new file mode 100644 index 0000000000000000000000000000000000000000..9ea9e0a538d01f84360f0d032450062835e1b1d0 --- /dev/null +++ b/arch/sw_64/include/asm/timer.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_TIMER_H +#define _ASM_SW64_TIMER_H + +extern void sw64_setup_clocksource(void); + +extern void sw64_setup_timer(void); + +extern void __init setup_sched_clock(void); + +#endif /* _ASM_SW64_TIMER_H */ diff --git a/arch/sw_64/include/asm/topology.h b/arch/sw_64/include/asm/topology.h index c39ca5b4beb78f1dc65628695e3a0432547a77b3..3ec906df276c61789fd07b316e336402c49cd028 100644 --- a/arch/sw_64/include/asm/topology.h +++ b/arch/sw_64/include/asm/topology.h @@ -8,10 +8,6 @@ #include #include -#define THREAD_ID_SHIFT 5 /* thread_id is removed from rcid */ -#define THREAD_ID_MASK 0 /* set mask to 0 */ -#define CORE_ID_MASK ((1 << THREAD_ID_SHIFT) - 1) - extern struct cpu_topology cpu_topology[NR_CPUS]; #define topology_physical_package_id(cpu) (cpu_topology[cpu].package_id) @@ -25,9 +21,19 @@ void store_cpu_topology(int cpuid); void remove_cpu_topology(int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); -static inline int rcid_to_package(int rcid) +static inline int rcid_to_thread_id(int rcid) { - return rcid >> CORES_PER_NODE_SHIFT; + return (rcid & THREAD_ID_MASK) >> THREAD_ID_SHIFT; +} + +static inline int rcid_to_core_id(int rcid) +{ + return (rcid & CORE_ID_MASK) >> CORE_ID_SHIFT; +} + +static inline int rcid_to_domain_id(int rcid) +{ + return (rcid & DOMAIN_ID_MASK) >> DOMAIN_ID_SHIFT; } #ifdef CONFIG_NUMA @@ -54,6 +60,9 @@ static inline void numa_add_cpu(unsigned int cpu) { } static inline void numa_remove_cpu(unsigned int cpu) { } static inline void numa_store_cpu_info(unsigned int cpu) { } #endif /* CONFIG_NUMA */ + +extern void get_vt_smp_info(void); + #include static inline void arch_fix_phys_package_id(int num, u32 slot) { } diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h new file mode 100644 index 0000000000000000000000000000000000000000..37cfe1fd68070c3ec194c181e4f416a790713947 --- /dev/null +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNCORE_IO_JUNZHANG_H +#define _ASM_SW64_UNCORE_IO_JUNZHANG_H + +#include + +#define IO_BASE (0x1UL << 47) +#define PCI_BASE (0x1UL << 43) +#define PCI_IOR0_BASE (0x2UL << 32) +#define PCI_IOR1_BASE (0x3UL << 32) + +#define PCI_RC_CFG (0x5UL << 32) + +#define PCI_EP_CFG (0x3UL << 33) +#define PCI_LEGACY_IO (0x1UL << 32) +#define PCI_LEGACY_IO_SIZE (0x100000000UL) +#define PCI_MEM_UNPRE 0x0UL +#define PCI_32BIT_VT_MEMIO (0xc0000000UL) +#define PCI_32BIT_MEMIO (0xe0000000UL) +#define PCI_32BIT_MEMIO_SIZE (0x20000000UL) +#define PCI_64BIT_MEMIO (0x1UL << 39) +#define PCI_64BIT_MEMIO_SIZE (0x8000000000UL) + +#define IO_RC_SHIFT 40 +#define IO_NODE_SHIFT 44 +#define IO_MARK_BIT 47 + +#define VT_MAX_CPUS_SHIFT 0 +#define VT_MAX_CPUS_MASK 0x3ff +#define VT_CORES_SHIFT 10 +#define VT_CORES_MASK 0x3ff +#define VT_THREADS_SHIFT 20 +#define VT_THREADS_MASK 0xfff + +#define QEMU_PRINTF_BUFF_BASE (IO_BASE | SPBU_BASE | 0x40000UL) + +/* MSIConfig */ +#define MSICONFIG_VALID (0x1UL << 63) +#define MSICONFIG_EN (0x1UL << 62) +#define MSICONFIG_VECTOR_SHIFT 10 + +#define MSIX_MSG_ADDR (0xfff00000UL) + +#define SW64_PCI_IO_BASE(m, n) \ + (IO_BASE | ((m) << IO_NODE_SHIFT) | PCI_BASE | ((n) << IO_RC_SHIFT)) +#define SW64_IO_BASE(x) (IO_BASE | ((x) << IO_NODE_SHIFT)) + +#define SW64_PCI0_BUS 0 +#define PCI0_BUS SW64_PCI0_BUS + +#define MAX_NR_NODES 0x2 +#define MAX_NR_RCS 0x6 + +#define SPBU_BASE (0x3UL << 36) +#define INTPU_BASE (0x3aUL << 32) +#define IIC0_BASE (0x31UL << 32) +#define SPI_BASE (0x32UL << 32) +#define UART_BASE (0x33UL << 32) +#define IIC1_BASE (0x34UL << 32) +#define IIC2_BASE (0x35UL << 32) +#define GPIO_BASE (0x36UL << 32) +#define LPC_BASE (0x37UL << 32) +#define LPC_LEGACY_IO (0x1UL << 28 | IO_BASE | LPC_BASE) +#define LPC_MEM_IO (0x2UL << 28 | IO_BASE | LPC_BASE) +#define LPC_FIRMWARE_IO (0x3UL << 28 | IO_BASE | LPC_BASE) +#define PCI_VT_LEGACY_IO (IO_BASE | PCI_BASE | PCI_LEGACY_IO) + +#define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) +#define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) + +#define PIUCONFIG0_INIT_VAL 0x38016 + +/*-----------------------addr-----------------------*/ +/* INTPU REG */ +enum { + DEVINT_MISS = INTPU_BASE | 0x100UL, + MT_INT_CONFIG = INTPU_BASE | 0x300UL, + DEV_INT_CONFIG = INTPU_BASE | 0x480UL, + FMT_ERR = INTPU_BASE | 0x700UL, + FAULT_INT_CONFIG = INTPU_BASE | 0x780UL, + SERR_CNTTH = INTPU_BASE | 0x880UL, + SPBUSERR_CNT = INTPU_BASE | 0x900UL, + IRUSERR_CNT = INTPU_BASE | 0xa80UL, + ERRRPT_EN = INTPU_BASE | 0xb00UL, + IINT_MISS_VECTOR0 = INTPU_BASE | 0x1080UL, + IINT_MISS_VECTOR1 = INTPU_BASE | 0x1100UL, + IINT_MISS = INTPU_BASE | 0x1180UL, + IINT_MISS_RPTEN = INTPU_BASE | 0x1200UL, + DEVINT_MISS_RPTEN = INTPU_BASE | 0x1280UL, + ECCSERR = INTPU_BASE | 0x1300UL, + ECCSERR_RPTEN = INTPU_BASE | 0x1380UL, + ECCMERR = INTPU_BASE | 0x1400UL, + ECCMERR_RPTEN = INTPU_BASE | 0x1480UL, + DEVINT_WKEN = INTPU_BASE | 0x1500UL, + ADR_INT_CONFIG = INTPU_BASE | 0x1580UL, + DEVINTWK_INTEN = INTPU_BASE | 0x1600UL, +}; + +/* SPBU CSR */ +enum { + SMP_INFO = SPBU_BASE | 0x80UL, + INIT_CTL = SPBU_BASE | 0x680UL, + CORE_ONLINE = SPBU_BASE | 0x780UL, + DLI_RLTD_FAULT = SPBU_BASE | 0x980UL, + DLI_RLTD_FAULT_EN = SPBU_BASE | 0xa00UL, + DLI_RLTD_FAULT_INTEN = SPBU_BASE | 0xa80UL, + CFG_INFO = SPBU_BASE | 0x1100UL, + IO_START = SPBU_BASE | 0x1300UL, + I2C0_SRST_L = SPBU_BASE | 0x1900UL, + I2C1_SRST_L = SPBU_BASE | 0x1980UL, + I2C2_SRST_L = SPBU_BASE | 0x1a00UL, + MCU_DVC_INT = SPBU_BASE | 0x3000UL, + MCU_DVC_INT_EN = SPBU_BASE | 0x3080UL, + SI_FAULT_STAT = SPBU_BASE | 0x3100UL, + SI_FAULT_STAT_EN = SPBU_BASE | 0x3180UL, + SI_FAULT_INT_EN = SPBU_BASE | 0x3200UL, + ADR_CTL = SPBU_BASE | 0x3600UL, + MC_ONLINE = SPBU_BASE | 0x3780UL, + PIU_TOP0_CONFIG = SPBU_BASE | 0x4c80UL, + PIU_TOP1_CONFIG = SPBU_BASE | 0x4d00UL, + SOFT_INFO0 = SPBU_BASE | 0xa000UL, +}; + +/*--------------------------offset-----------------------------------*/ +/* PIU IOR0 */ +enum { + PIUCONFIG0 = 0x0UL, + EPDMABAR = 0x80UL, + IOMMUSEGITEM0 = 0x100UL, + IOMMUEXCPT_CTRL = 0x2100UL, + MSIADDR = 0x2180UL, + MSICONFIG0 = 0x2200UL, + INTACONFIG = 0xa200UL, + INTBCONFIG = 0xa280UL, + INTCCONFIG = 0xa300UL, + INTDCONFIG = 0xa380UL, + AERERRINTCONFIG = 0xa400UL, + AERERRMSICONFIG = 0xa480UL, + PMEINTCONFIG = 0xa500UL, + PMEMSICONFIG = 0xa580UL, + HPINTCONFIG = 0xa600UL, + HPMSICONFIG = 0xa680UL, + DTBASEADDR = 0xb000UL, + DTLB_FLUSHALL = 0xb080UL, + DTLB_FLUSHDEV = 0xb100UL, + PTLB_FLUSHALL = 0xb180UL, + PTLB_FLUSHDEV = 0xb200UL, + PTLB_FLUSHVADDR = 0xb280UL, + PCACHE_FLUSHALL = 0xb300UL, + PCACHE_FLUSHDEV = 0xb380UL, + PCACHE_FLUSHPADDR = 0xb400UL, + TIMEOUT_CONFIG = 0xb480UL, + IOMMUEXCPT_STATUS = 0xb500UL, + IOMMUPAGE_PADDR1 = 0xb580UL, + IOMMUPAGE_PADDR2 = 0xb600UL, + IOMMUPAGE_PADDR3 = 0xb680UL, + PTLB_ACCESS = 0xb700UL, + PTLB_ITEM_TAG = 0xb780UL, + PTLB_ITEM_DATA = 0xb800UL, + PCACHE_ACCESS = 0xb880UL, + PCACHE_ITEM_TAG = 0xb900UL, + PCACHE_ITEM_DATA0 = 0xb980UL, +}; + +/* PIU IOR1 */ +enum { + PIUCONFIG1 = 0x0UL, + ERRENABLE = 0x880UL, + RCDEBUGINF1 = 0xc80UL, + DCACONTROL = 0x1a00UL, + DEVICEID0 = 0x1a80UL, +}; + +/* RC */ +enum { + RC_VENDOR_ID = 0x0UL, + RC_COMMAND = 0x80UL, + RC_REVISION_ID = 0x100UL, + RC_PRIMARY_BUS = 0x300UL, + RC_MSI_CONTROL = 0xa00UL, + RC_EXP_DEVCAP = 0xe80UL, + RC_EXP_DEVCTL = 0xf00UL, + RC_SLOT_CTRL = 0x1100UL, + RC_LINK_STAT = 0x1000UL, + RC_CONTROL = 0X1180UL, + RC_STATUS = 0X1200UL, + RC_EXP_DEVCTL2 = 0x1300UL, + RC_PORT_LINK_CTL = 0xe200UL, + RC_ORDER_RULE_CTL = 0x11680UL, + RC_MISC_CONTROL_1 = 0x11780UL, + RC_PHY_INT_REG = 0x80000UL, + RC_PHY_EXT_GEN1 = 0x82400UL, + RC_PHY_EXT_GEN2 = 0x82480UL, +}; +/* GPIO */ +enum { + GPIO_SWPORTA_DR = GPIO_BASE | 0x0UL, + GPIO_SWPORTA_DDR = GPIO_BASE | 0x200UL, +}; +/*--------------------------------------------------------------------------*/ +#endif /* _ASM_SW64_UNCORE_IO_JUNZHANG_H */ diff --git a/arch/sw_64/include/asm/uncore_io_ops_junzhang.h b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h new file mode 100644 index 0000000000000000000000000000000000000000..95a3b5c8053192398cc48764ff790c4d032e7cbd --- /dev/null +++ b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H +#define _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H + +static inline int __get_cpu_nums(void) +{ + int cpus; + unsigned long cfg_info; + + cfg_info = sw64_io_read(0, CFG_INFO); + cfg_info = (cfg_info >> 33) & 0x3; + cpus = 1 << cfg_info; + + return cpus; +} + +static inline unsigned long __get_node_mem(int node) +{ + unsigned long node_mem; + unsigned long total_mem; + + total_mem = sw64_io_read(node, CFG_INFO) >> 3; + total_mem = (total_mem & 0xffff) << 28; + node_mem = total_mem / __get_cpu_nums(); + + return node_mem; +} + +#define __io_read_longtime(node) (0UL) +#define __io_write_longtime(node, data) do { } while (0) +#define __io_write_longtime_start_en(node, data) do { } while (0) + +static inline void +__io_write_fault_int_en(int node, unsigned long data) +{ + sw64_io_write(node, FAULT_INT_CONFIG, data); +} + +#endif /* _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H */ diff --git a/arch/sw_64/include/asm/uncore_io_ops_xuelang.h b/arch/sw_64/include/asm/uncore_io_ops_xuelang.h new file mode 100644 index 0000000000000000000000000000000000000000..9336e473211d7ff131d54df51a3f820a4eaca4e7 --- /dev/null +++ b/arch/sw_64/include/asm/uncore_io_ops_xuelang.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNCORE_IO_OPS_XUELANG_H +#define _ASM_SW64_UNCORE_IO_OPS_XUELANG_H + +static inline int __get_cpu_nums(void) +{ + int cpus; + unsigned long trkmode; + + trkmode = sw64_io_read(0, TRKMODE); + trkmode = (trkmode >> 6) & 0x3; + cpus = 1 << trkmode; + + return cpus; +} + +static inline unsigned long __get_node_mem(int node) +{ + unsigned long node_mem; + unsigned long mc_config; + unsigned long mc_online; + unsigned long mc_cap; + unsigned long mc_num; + + mc_config = sw64_io_read(node, MC_CAP_CFG) & 0xf; + mc_cap = (1UL << mc_config) << 28; + mc_online = sw64_io_read(node, MC_ONLINE) & 0xff; + mc_num = __kernel_ctpop(mc_online); + node_mem = mc_cap * mc_num; + + return node_mem; +} + +static inline unsigned long +__io_read_longtime(int node) +{ + return sw64_io_read(node, LONG_TIME); +} + +static inline void +__io_write_longtime(int node, unsigned long data) +{ + sw64_io_write(node, LONG_TIME, data); +} + +static inline void +__io_write_longtime_start_en(int node, unsigned long data) +{ + sw64_io_write(node, LONG_TIME_START_EN, data); +} + +static inline void +__io_write_fault_int_en(int node, unsigned long data) +{ + sw64_io_write(node, DUAL_CG0_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG1_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG2_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG3_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG4_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG5_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG6_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG7_FAULT_INTEN, data); +} + +#endif /* _ASM_SW64_UNCORE_IO_OPS_XUELANG_H */ diff --git a/arch/sw_64/include/asm/chip3_io.h b/arch/sw_64/include/asm/uncore_io_xuelang.h similarity index 96% rename from arch/sw_64/include/asm/chip3_io.h rename to arch/sw_64/include/asm/uncore_io_xuelang.h index 7f64e8816ed315fd31965ce334f387be237b08af..aeaadec5be16378e0944490319f6225b57d418be 100644 --- a/arch/sw_64/include/asm/chip3_io.h +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_SW64_CHIP3_IO_H -#define _ASM_SW64_CHIP3_IO_H +#ifndef _ASM_SW64_UNCORE_IO_XUELANG_H +#define _ASM_SW64_UNCORE_IO_XUELANG_H #include @@ -9,11 +9,7 @@ #define PCI_IOR0_BASE (0x2UL << 32) #define PCI_IOR1_BASE (0x3UL << 32) -#ifdef CONFIG_SW64_FPGA -#define PCI_RC_CFG (0x4UL << 32) -#else #define PCI_RC_CFG (0x5UL << 32) -#endif #define PCI_EP_CFG (0x3UL << 33) #define PCI_LEGACY_IO (0x1UL << 32) @@ -28,9 +24,6 @@ #define IO_NODE_SHIFT 44 #define IO_MARK_BIT 47 -extern int topo_nr_threads; -extern int topo_nr_cores; -extern int topo_nr_maxcpus; #define VT_MAX_CPUS_SHIFT 0 #define VT_MAX_CPUS_MASK 0x3ff #define VT_CORES_SHIFT 10 @@ -38,11 +31,15 @@ extern int topo_nr_maxcpus; #define VT_THREADS_SHIFT 20 #define VT_THREADS_MASK 0xfff +#define QEMU_PRINTF_BUFF_BASE (IO_BASE | MCU_BASE | 0x40000UL) + /* MSIConfig */ #define MSICONFIG_VALID (0x1UL << 63) #define MSICONFIG_EN (0x1UL << 62) #define MSICONFIG_VECTOR_SHIFT 10 +#define MSIX_MSG_ADDR (0x91abc0UL) + #define SW64_PCI_IO_BASE(m, n) \ (IO_BASE | ((m) << IO_NODE_SHIFT) | PCI_BASE | ((n) << IO_RC_SHIFT)) #define SW64_IO_BASE(x) (IO_BASE | ((x) << IO_NODE_SHIFT)) @@ -53,13 +50,6 @@ extern int topo_nr_maxcpus; #define MAX_NR_NODES 0x2 #define MAX_NR_RCS 0x6 -#define SW64_PCI_DEBUG 0 -#if SW64_PCI_DEBUG -#define PCIINFO(fmt, args...) printk(fmt, ##args) -#else -#define PCIINFO(fmt, args...) -#endif - #define MCU_BASE (0x3UL << 36) #define CAB0_BASE (0x10UL << 32) #define INTPU_BASE (0x2aUL << 32) @@ -82,6 +72,8 @@ extern int topo_nr_maxcpus; #define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x1UL << 10) #define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x1UL << 10) +#define PIUCONFIG0_INIT_VAL 0x38056 + /*-----------------------addr-----------------------*/ /* CAB0 REG */ enum { @@ -328,4 +320,4 @@ enum { GPIO_SWPORTA_DDR = GPIO_BASE | 0x200UL, }; /*--------------------------------------------------------------------------*/ -#endif /* _ASM_SW64_CHIP3_IO_H */ +#endif /* _ASM_SW64_UNCORE_IO_XUELANG_H */ diff --git a/arch/sw_64/include/asm/uprobes.h b/arch/sw_64/include/asm/uprobes.h index 2a5b268cb88f41a050a9b871670ad935651d0f6c..fcd2026c3622e20a781107c70d414f075d1bf588 100644 --- a/arch/sw_64/include/asm/uprobes.h +++ b/arch/sw_64/include/asm/uprobes.h @@ -35,6 +35,11 @@ struct arch_uprobe_task { unsigned long saved_trap_nr; }; -extern void sw64_fix_uretprobe(struct pt_regs *regs); +#ifdef CONFIG_UPROBES +void sw64_fix_uretprobe(struct pt_regs *regs, unsigned long exc_pc); +#else +static inline void +sw64_fix_uretprobe(struct pt_regs *regs, unsigned long exc_pc) {} +#endif #endif /* _ASM_SW64_UPROBES_H */ diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h index c43ebe72e3a1e513b4e8f4e718e1190cc34a8d8e..c4e3caacbc70c7409a3539317d781d8997eb5d6c 100644 --- a/arch/sw_64/include/asm/vcpu.h +++ b/arch/sw_64/include/asm/vcpu.h @@ -4,6 +4,8 @@ #ifndef __ASSEMBLY__ +#ifdef CONFIG_SUBARCH_C3B + struct vcpucb { unsigned long go_flag; unsigned long pcbb; @@ -20,19 +22,19 @@ struct vcpucb { unsigned long new_a0; unsigned long new_a1; unsigned long new_a2; - unsigned long whami; + unsigned long soft_cid; unsigned long csr_save; unsigned long wakeup_magic; unsigned long host_vcpucb; unsigned long upcr; unsigned long vpcr; - unsigned long dtb_pcr; + unsigned long dtb_vpcr; unsigned long guest_ksp; unsigned long guest_usp; unsigned long vcpu_irq_disabled; unsigned long vcpu_irq; unsigned long ptbr; - unsigned long tid; + unsigned long soft_tid; unsigned long int_stat1; unsigned long int_stat2; unsigned long int_stat3; @@ -53,5 +55,52 @@ struct vcpucb { unsigned long reserved[3]; }; +#else + +struct vcpucb { + unsigned long ktp; + unsigned long pcbb; + unsigned long ksp; + unsigned long usp; + unsigned long kgp; + unsigned long ent_arith; + unsigned long ent_if; + unsigned long ent_int; + unsigned long ent_mm; + unsigned long ent_sys; + unsigned long ent_una; + unsigned long stack_pc; + unsigned long new_a0; + unsigned long new_a1; + unsigned long new_a2; + unsigned long soft_cid; + unsigned long csr_save; + unsigned long wakeup_magic; + unsigned long host_vcpucb; + unsigned long upcr; + unsigned long vpcr; + unsigned long dtb_vpcr; + unsigned long dtb_upcr; + unsigned long guest_ksp; + unsigned long guest_usp; + unsigned long vcpu_irq_disabled; + unsigned long vcpu_irq; + unsigned long ptbr_usr; + unsigned long ptbr_sys; + unsigned long soft_tid; + unsigned long int_stat0; + unsigned long int_stat1; + unsigned long int_stat2; + unsigned long int_stat3; + unsigned long reset_entry; + unsigned long pvcpu; + unsigned long exit_reason; + unsigned long ipaddr; + unsigned long vcpu_pc_save; + unsigned long shtclock_offset; + unsigned long reserved[8]; +}; +#endif + #endif /* __ASSEMBLY__ */ #endif /* _ASM_SW64_VCPU_H */ diff --git a/arch/sw_64/include/asm/xchg.h b/arch/sw_64/include/asm/xchg.h index ba4e6d1a27ad064aeef83ec4e4a2b2c89181093a..16d3f3e4401cbb328dc7786ae2ea85c29e0fe22f 100644 --- a/arch/sw_64/include/asm/xchg.h +++ b/arch/sw_64/include/asm/xchg.h @@ -11,6 +11,7 @@ * So this file is included twice from asm/cmpxchg.h. */ +#if defined(CONFIG_SUBARCH_C3B) /* * Atomic exchange. * Since it can be used to implement critical sections @@ -23,9 +24,6 @@ ____xchg(_u8, volatile char *m, unsigned long val) unsigned long ret, tmp, addr64; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " andnot %4, 7, %3\n" " inslb %1, %4, %1\n" @@ -35,9 +33,6 @@ ____xchg(_u8, volatile char *m, unsigned long val) " extlb %2, %4, %0\n" " masklb %2, %4, %2\n" " or %1, %2, %2\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %2, 0(%3)\n" " rd_f %2\n" " beq %2, 2f\n" @@ -56,9 +51,6 @@ ____xchg(_u16, volatile short *m, unsigned long val) unsigned long ret, tmp, addr64; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " andnot %4, 7, %3\n" " inslh %1, %4, %1\n" "1: lldl %2, 0(%3)\n" @@ -67,9 +59,6 @@ ____xchg(_u16, volatile short *m, unsigned long val) " extlh %2, %4, %0\n" " masklh %2, %4, %2\n" " or %1, %2, %2\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %2, 0(%3)\n" " rd_f %2\n" " beq %2, 2f\n" @@ -88,22 +77,16 @@ ____xchg(_u32, volatile int *m, unsigned long val) unsigned long dummy, addr; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif - " ldi %3, %5\n" - "1: lldw %0, 0(%3)\n" - " ldi %1, 1\n" - " wr_f %1\n" - " bis $31, %4, %1\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif - " lstw %1, 0(%3)\n" - " rd_f %1\n" - " beq %1, 2f\n" + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " ldi %1, 1\n" + " wr_f %1\n" + " bis $31, %4, %1\n" + " lstw %1, 0(%3)\n" + " rd_f %1\n" + " beq %1, 2f\n" ".subsection 2\n" - "2: br 1b\n" + "2: br 1b\n" ".previous" : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) : "rI" (val), "m" (*m) : "memory"); @@ -117,22 +100,16 @@ ____xchg(_u64, volatile long *m, unsigned long val) unsigned long dummy, addr; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif - " ldi %3, %5\n" - "1: lldl %0, 0(%3)\n" - " ldi %1, 1\n" - " wr_f %1\n" - " bis $31, %4, %1\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif - " lstl %1, 0(%3)\n" - " rd_f %1\n" - " beq %1, 2f\n" + " ldi %3, %5\n" + "1: lldl %0, 0(%3)\n" + " ldi %1, 1\n" + " wr_f %1\n" + " bis $31, %4, %1\n" + " lstl %1, 0(%3)\n" + " rd_f %1\n" + " beq %1, 2f\n" ".subsection 2\n" - "2: br 1b\n" + "2: br 1b\n" ".previous" : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) : "rI" (val), "m" (*m) : "memory"); @@ -140,29 +117,6 @@ ____xchg(_u64, volatile long *m, unsigned long val) return val; } -/* - * This function doesn't exist, so you'll get a linker error - * if something tries to do an invalid xchg(). - */ -extern void __xchg_called_with_bad_pointer(void); - -static __always_inline unsigned long -____xchg(, volatile void *ptr, unsigned long x, int size) -{ - switch (size) { - case 1: - return ____xchg(_u8, ptr, x); - case 2: - return ____xchg(_u16, ptr, x); - case 4: - return ____xchg(_u32, ptr, x); - case 8: - return ____xchg(_u64, ptr, x); - } - __xchg_called_with_bad_pointer(); - return x; -} - /* * Atomic compare and exchange. Compare OLD with MEM, if identical, * store NEW in MEM. Return the initial value in MEM. Success is @@ -180,9 +134,6 @@ ____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new) unsigned long prev, tmp, cmp, addr64; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " andnot %5, 7, %4\n" " inslb %1, %5, %1\n" "1: lldl %2, 0(%4)\n" @@ -191,9 +142,6 @@ ____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new) " wr_f %3\n" " masklb %2, %5, %2\n" " or %1, %2, %2\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %2, 0(%4)\n" " rd_f %2\n" " beq %3, 2f\n" @@ -214,9 +162,6 @@ ____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new) unsigned long prev, tmp, cmp, addr64; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " andnot %5, 7, %4\n" " inslh %1, %5, %1\n" "1: lldl %2, 0(%4)\n" @@ -225,9 +170,6 @@ ____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new) " wr_f %3\n" " masklh %2, %5, %2\n" " or %1, %2, %2\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %2, 0(%4)\n" " rd_f %2\n" " beq %3, 2f\n" @@ -248,24 +190,18 @@ ____cmpxchg(_u32, volatile int *m, int old, int new) unsigned long prev, cmp, addr, tmp; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif - " ldi %3, %7\n" - "1: lldw %0, 0(%3)\n" - " cmpeq %0, %5, %1\n" - " wr_f %1\n" - " bis $31, %6, %4\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif - " lstw %4, 0(%3)\n" - " rd_f %4\n" - " beq %1, 2f\n" - " beq %4, 3f\n" + " ldi %3, %7\n" + "1: lldw %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " wr_f %1\n" + " bis $31, %6, %4\n" + " lstw %4, 0(%3)\n" + " rd_f %4\n" + " beq %1, 2f\n" + " beq %4, 3f\n" "2:\n" ".subsection 2\n" - "3: br 1b\n" + "3: br 1b\n" ".previous" : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) : "r"((long) old), "r"(new), "m"(*m) : "memory"); @@ -279,24 +215,219 @@ ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) unsigned long prev, cmp, addr, tmp; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif - " ldi %3, %7\n" - "1: lldl %0, 0(%3)\n" - " cmpeq %0, %5, %1\n" - " wr_f %1\n" - " bis $31, %6, %4\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif - " lstl %4, 0(%3)\n" - " rd_f %4\n" - " beq %1, 2f\n" - " beq %4, 3f\n" + " ldi %3, %7\n" + "1: lldl %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " wr_f %1\n" + " bis $31, %6, %4\n" + " lstl %4, 0(%3)\n" + " rd_f %4\n" + " beq %1, 2f\n" + " beq %4, 3f\n" + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) + : "r"((long) old), "r"(new), "m"(*m) : "memory"); + + return prev; +} + +#elif defined(CONFIG_SUBARCH_C4) +/* + * Atomic exchange. + * Since it can be used to implement critical sections + * it must clobber "memory" (also for interrupts in UP). + */ + +static inline unsigned long +____xchg(_u8, volatile char *m, unsigned long val) +{ + unsigned long ret, tmp, addr64; + + __asm__ __volatile__( + " andnot %4, 7, %3\n" + " inslb %1, %4, %1\n" + "1: lldl %2, 0(%3)\n" + " extlb %2, %4, %0\n" + " masklb %2, %4, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%3)\n" + " beq %2, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) + : "r" ((long)m), "1" (val) : "memory"); + + return ret; +} + +static inline unsigned long +____xchg(_u16, volatile short *m, unsigned long val) +{ + unsigned long ret, tmp, addr64; + + __asm__ __volatile__( + " andnot %4, 7, %3\n" + " inslh %1, %4, %1\n" + "1: lldl %2, 0(%3)\n" + " extlh %2, %4, %0\n" + " masklh %2, %4, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%3)\n" + " beq %2, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) + : "r" ((long)m), "1" (val) : "memory"); + + return ret; +} + +static inline unsigned long +____xchg(_u32, volatile int *m, unsigned long val) +{ + unsigned long dummy, addr; + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " bis $31, %4, %1\n" + " lstw %1, 0(%3)\n" + " beq %1, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) + : "rI" (val), "m" (*m) : "memory"); + + return val; +} + +static inline unsigned long +____xchg(_u64, volatile long *m, unsigned long val) +{ + unsigned long dummy, addr; + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldl %0, 0(%3)\n" + " bis $31, %4, %1\n" + " lstl %1, 0(%3)\n" + " beq %1, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) + : "rI" (val), "m" (*m) : "memory"); + + return val; +} + +/* + * Atomic compare and exchange. Compare OLD with MEM, if identical, + * store NEW in MEM. Return the initial value in MEM. Success is + * indicated by comparing RETURN with OLD. + * + * The memory barrier should be placed in SMP only when we actually + * make the change. If we don't change anything (so if the returned + * prev is equal to old) then we aren't acquiring anything new and + * we don't need any memory barrier as far I can tell. + */ +static inline unsigned long +____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new) +{ + unsigned long prev, tmp, cmp, addr64; + + __asm__ __volatile__( + " andnot %5, 7, %4\n" + " inslb %1, %5, %1\n" + "1: lldl %2, 0(%4)\n" + " extlb %2, %5, %0\n" + " cmpeq %0, %6, %3\n" + " beq %3, 2f\n" + " masklb %2, %5, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%4)\n" + " beq %2, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) + : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new) +{ + unsigned long prev, tmp, cmp, addr64; + + __asm__ __volatile__( + " andnot %5, 7, %4\n" + " inslh %1, %5, %1\n" + "1: lldl %2, 0(%4)\n" + " extlh %2, %5, %0\n" + " cmpeq %0, %6, %3\n" + " beq %3, 2f\n" + " masklh %2, %5, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%4)\n" + " beq %2, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) + : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u32, volatile int *m, int old, int new) +{ + unsigned long prev, cmp, addr, tmp; + + __asm__ __volatile__( + " ldi %3, %7\n" + "1: lldw %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " beq %1, 2f\n" + " bis $31, %6, %4\n" + " lstw %4, 0(%3)\n" + " beq %4, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) + : "r"((long) old), "r"(new), "m"(*m) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) +{ + unsigned long prev, cmp, addr, tmp; + + __asm__ __volatile__( + " ldi %3, %7\n" + "1: lldl %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " beq %1, 2f\n" + " bis $31, %6, %4\n" + " lstl %4, 0(%3)\n" + " beq %4, 3f\n" "2:\n" ".subsection 2\n" - "3: br 1b\n" + "3: lbr 1b\n" ".previous" : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) : "r"((long) old), "r"(new), "m"(*m) : "memory"); @@ -304,6 +435,30 @@ ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) return prev; } +#endif + +/* This function doesn't exist, so you'll get a linker error + * if something tries to do an invalid xchg(). + */ +extern void __xchg_called_with_bad_pointer(void); + +static __always_inline unsigned long +____xchg(, volatile void *ptr, unsigned long x, int size) +{ + switch (size) { + case 1: + return ____xchg(_u8, ptr, x); + case 2: + return ____xchg(_u16, ptr, x); + case 4: + return ____xchg(_u32, ptr, x); + case 8: + return ____xchg(_u64, ptr, x); + } + __xchg_called_with_bad_pointer(); + return x; +} + /* * This function doesn't exist, so you'll get a linker error * if something tries to do an invalid cmpxchg(). diff --git a/arch/sw_64/include/uapi/asm/bpf_perf_event.h b/arch/sw_64/include/uapi/asm/bpf_perf_event.h index b551b741653d251d6155a214023b2215ae02e871..52f6f1e555f162ef7668965386cc758125726224 100644 --- a/arch/sw_64/include/uapi/asm/bpf_perf_event.h +++ b/arch/sw_64/include/uapi/asm/bpf_perf_event.h @@ -1,9 +1,9 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__ -#define _UAPI__ASM_BPF_PERF_EVENT_H__ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_BPF_PERF_EVENT_H +#define _UAPI_ASM_SW64_BPF_PERF_EVENT_H #include typedef struct user_pt_regs bpf_user_pt_regs_t; -#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */ +#endif /* _UAPI_ASM_SW64_BPF_PERF_EVENT_H */ diff --git a/arch/sw_64/include/uapi/asm/fpu.h b/arch/sw_64/include/uapi/asm/fpu.h index 035ca65b1ba38a711348e5bb85c1b22f5d699eef..ca95ecbb6765c0510c09f956fca04613fea834a7 100644 --- a/arch/sw_64/include/uapi/asm/fpu.h +++ b/arch/sw_64/include/uapi/asm/fpu.h @@ -6,7 +6,11 @@ * SW-64 floating-point control register defines: */ #define FPCR_DNOD (1UL << 47) /* denorm INV trap disable */ +#ifdef CONFIG_SUBARCH_C3B #define FPCR_DNZ (1UL << 48) /* denorms to zero */ +#else +#define FPCR_DNOE (1UL << 48) /* hardware denormal support */ +#endif #define FPCR_INVD (1UL << 49) /* invalid op disable (opt.) */ #define FPCR_DZED (1UL << 50) /* division by zero disable (opt.) */ #define FPCR_OVFD (1UL << 51) /* overflow disable (optional) */ @@ -30,6 +34,12 @@ #define FPCR_MASK 0xffff800000000000L +#ifdef CONFIG_SUBARCH_C3B +#define FPCR_INIT FPCR_DYN_NORMAL +#else +#define FPCR_INIT (FPCR_DYN_NORMAL | FPCR_DNOE) +#endif + /* status bit coming from hardware fpcr . definde by fire3 */ #define FPCR_STATUS_INV0 (1UL << 52) #define FPCR_STATUS_DZE0 (1UL << 53) diff --git a/arch/sw_64/include/uapi/asm/kvm.h b/arch/sw_64/include/uapi/asm/kvm.h index 0ca8c10b855034cc809a1d0e98547ae131c40595..2253475deaa5a12ca3c14637573f79ea4baf02c7 100644 --- a/arch/sw_64/include/uapi/asm/kvm.h +++ b/arch/sw_64/include/uapi/asm/kvm.h @@ -117,4 +117,15 @@ struct kvm_sync_regs { struct kvm_sregs { }; +struct swvm_mem_bank { + unsigned long guest_phys_addr; + unsigned long host_phys_addr; + unsigned long host_addr; + unsigned long size; +}; + +struct swvm_mem { + struct swvm_mem_bank membank[SWVM_NUM_NUMA_MEMBANKS]; +}; + #endif /* _UAPI_ASM_SW64_KVM_H */ diff --git a/arch/sw_64/include/uapi/asm/ptrace.h b/arch/sw_64/include/uapi/asm/ptrace.h index 5cf3ca1d3dd843b0dc18b7c7452e9b7220d9359b..3fd53450e418bcedc8faf45f37cb0e0bc8dca8ad 100644 --- a/arch/sw_64/include/uapi/asm/ptrace.h +++ b/arch/sw_64/include/uapi/asm/ptrace.h @@ -29,23 +29,28 @@ struct user_fpsimd_state { /* PTRACE_ATTACH is 16 */ /* PTRACE_DETACH is 17 */ -#define REG_BASE 0 -#define REG_END 29 -#define USP 30 -#define FPREG_BASE 32 -#define FPREG_END 62 -#define FPCR 63 -#define PC 64 -#define TP 65 -#define UNIQUE TP -#define VECREG_BASE 67 -#define VECREG_END 161 -#define F31_V1 98 -#define F31_V2 130 -#define DA_MATCH 163 -#define DA_MASK 164 -#define DV_MATCH 165 -#define DV_MASK 166 -#define DC_CTL 167 +#define PT_REG_BASE 0 +#define PT_REG_END 30 +#define PT_FPREG_BASE 32 +#define PT_FPREG_END 62 +#define PT_FPCR 63 +#define PT_PC 64 +#define PT_TP 65 +#define PT_UNIQUE PT_TP +#define PT_VECREG_BASE 67 +#define PT_VECREG_END 161 +#define PT_F31_V1 98 +#define PT_F31_V2 130 +#define PT_DA_MATCH 163 +#define PT_DA_MASK 164 +#define PT_DV_MATCH 165 +#define PT_DV_MASK 166 +#define PT_DC_CTL 167 +#define PT_MATCH_CTL 167 +#define PT_IA_MATCH 168 +#define PT_IA_MASK 169 +#define PT_IV_MATCH 170 +#define PT_IDA_MATCH 171 +#define PT_IDA_MASK 172 #endif /* _UAPI_ASM_SW64_PTRACE_H */ diff --git a/arch/sw_64/include/uapi/asm/resource.h b/arch/sw_64/include/uapi/asm/resource.h index fecca2214849d99ad460c547ff93a8c5d2f4e1c9..2e1ce8f6ee64cdf82e04eed3a5a571489040bb97 100644 --- a/arch/sw_64/include/uapi/asm/resource.h +++ b/arch/sw_64/include/uapi/asm/resource.h @@ -11,13 +11,6 @@ #define RLIMIT_NPROC 8 /* max number of processes */ #define RLIMIT_MEMLOCK 9 /* max locked-in-memory address space */ -/* - * SuS says limits have to be unsigned. Fine, it's unsigned, but - * we retain the old value for compatibility, especially with DU. - * When you run into the 2^63 barrier, you call me. - */ -#define RLIM_INFINITY 0x7ffffffffffffffful - #include #endif /* _UAPI_ASM_SW64_RESOURCE_H */ diff --git a/arch/sw_64/include/uapi/asm/siginfo.h b/arch/sw_64/include/uapi/asm/siginfo.h index 28c656c283132e1771179391d0c5a95144b470e4..f47fb917c9b2858ec3fbc9fd01517daf9edf1864 100644 --- a/arch/sw_64/include/uapi/asm/siginfo.h +++ b/arch/sw_64/include/uapi/asm/siginfo.h @@ -3,7 +3,6 @@ #define _UAPI_ASM_SW64_SIGINFO_H #define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) -#define __ARCH_SI_TRAPNO #include diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 667e06039987a7d304d8cafc4faa0c3cde14e528..b71be5783812b3f77fb3ce88674a44b03e026cbb 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -15,38 +15,35 @@ endif obj-y := entry.o fpu.o traps.o process.o sys_sw64.o irq.o \ irq_sw64.o signal.o setup.o ptrace.o time.o \ - systbls.o dup_print.o tc.o timer.o \ + systbls.o dup_print.o chip_setup.o \ insn.o early_init.o topology.o cacheinfo.o \ vdso.o vdso/ hmcall.o stacktrace.o idle.o reset.o +obj-$(CONFIG_SUBARCH_C3B) += tc.o obj-$(CONFIG_ACPI) += acpi.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_PCI) += pci.o pci-sysfs.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_PCI_MSI) += msi.o +obj-$(CONFIG_PM) += pm.o obj-$(CONFIG_SUSPEND) += suspend_asm.o suspend.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_HIBERNATION) += hibernate_asm.o hibernate.o obj-$(CONFIG_AUDIT) += audit.o -obj-$(CONFIG_PCI) += pci_common.o obj-$(CONFIG_RELOCATABLE) += relocate.o obj-$(CONFIG_DEBUG_FS) += segvdbg.o unaligned.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o - -ifeq ($(CONFIG_DEBUG_FS)$(CONFIG_NUMA),yy) -obj-y += bindvcpu.o -endif +obj-$(CONFIG_DEBUG_MATCH) += match.o ifndef CONFIG_PCI obj-y += pci-noop.o endif ifdef CONFIG_KVM -obj-y += kvm_cma.o +obj-$(CONFIG_SUBARCH_C3B) += kvm_cma.o endif # Core logic support -obj-$(CONFIG_SW64_CPUFREQ) += platform.o clock.o obj-$(CONFIG_SW64_CPUAUTOPLUG) += cpuautoplug.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 61f2948f17818ef4d5fee71d3bef177792436c03..71c28b0fc73505d2c794fb282f73861b0fdf4ef7 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -8,9 +8,14 @@ int acpi_disabled = 1; EXPORT_SYMBOL(acpi_disabled); -int acpi_noirq; /* skip ACPI IRQ initialization */ -int acpi_pci_disabled; /* skip ACPI PCI scan and IRQ initialization */ + +int acpi_noirq = 1; /* skip ACPI IRQ initialization */ +int acpi_pci_disabled = 1; /* skip ACPI PCI scan and IRQ initialization */ EXPORT_SYMBOL(acpi_pci_disabled); + +static bool param_acpi_on __initdata; +static bool param_acpi_off __initdata; + int acpi_strict; u64 arch_acpi_wakeup_start; u64 acpi_saved_sp_s3; @@ -115,13 +120,14 @@ static int __init parse_acpi(char *arg) if (!arg) return -EINVAL; - /* "acpi=off" disables both ACPI table parsing and interpreter */ - if (strcmp(arg, "off") == 0) { - disable_acpi(); - } else { - /* Core will printk when we return error. */ - return -EINVAL; - } + /* disable both ACPI table parsing and interpreter */ + if (strcmp(arg, "off") == 0) + param_acpi_off = true; + else if (strcmp(arg, "on") == 0) /* prefer ACPI over device tree */ + param_acpi_on = true; + else + return -EINVAL; /* Core will printk when we return error. */ + return 0; } early_param("acpi", parse_acpi); @@ -364,16 +370,25 @@ EXPORT_SYMBOL(acpi_unmap_cpu); #endif /* CONFIG_ACPI_HOTPLUG_CPU */ void __init acpi_boot_table_init(void) - { + /** + * ACPI is disabled by default. + * ACPI is only enabled when firmware passes ACPI table + * and sets boot parameter "acpi=on". + */ + if (param_acpi_on) + enable_acpi(); + /* * If acpi_disabled, bail out */ if (!acpi_disabled) { + pr_warn("Currently, ACPI is an experimental feature!\n"); if (acpi_table_init()) { pr_err("Failed to init ACPI tables\n"); disable_acpi(); - } - pr_info("Enable ACPI support\n"); + } else + pr_info("Successfully parsed ACPI table\n"); } } + diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c index fe46b42bc94349022b424d6021cbe1107466afcb..41310a8a7af12a6f1a4fabb8ec52a396a7ef6403 100644 --- a/arch/sw_64/kernel/asm-offsets.c +++ b/arch/sw_64/kernel/asm-offsets.c @@ -15,7 +15,7 @@ #include #include "traps.c" - +#include "signal.c" void foo(void) { @@ -68,39 +68,41 @@ void foo(void) BLANK(); DEFINE(PT_REGS_SIZE, sizeof(struct pt_regs)); - DEFINE(PT_REGS_R0, offsetof(struct pt_regs, r0)); - DEFINE(PT_REGS_R1, offsetof(struct pt_regs, r1)); - DEFINE(PT_REGS_R2, offsetof(struct pt_regs, r2)); - DEFINE(PT_REGS_R3, offsetof(struct pt_regs, r3)); - DEFINE(PT_REGS_R4, offsetof(struct pt_regs, r4)); - DEFINE(PT_REGS_R5, offsetof(struct pt_regs, r5)); - DEFINE(PT_REGS_R6, offsetof(struct pt_regs, r6)); - DEFINE(PT_REGS_R7, offsetof(struct pt_regs, r7)); - DEFINE(PT_REGS_R8, offsetof(struct pt_regs, r8)); - DEFINE(PT_REGS_R9, offsetof(struct pt_regs, r9)); - DEFINE(PT_REGS_R10, offsetof(struct pt_regs, r10)); - DEFINE(PT_REGS_R11, offsetof(struct pt_regs, r11)); - DEFINE(PT_REGS_R12, offsetof(struct pt_regs, r12)); - DEFINE(PT_REGS_R13, offsetof(struct pt_regs, r13)); - DEFINE(PT_REGS_R14, offsetof(struct pt_regs, r14)); - DEFINE(PT_REGS_R15, offsetof(struct pt_regs, r15)); - DEFINE(PT_REGS_R16, offsetof(struct pt_regs, r16)); - DEFINE(PT_REGS_R17, offsetof(struct pt_regs, r17)); - DEFINE(PT_REGS_R18, offsetof(struct pt_regs, r18)); - DEFINE(PT_REGS_R19, offsetof(struct pt_regs, r19)); - DEFINE(PT_REGS_R20, offsetof(struct pt_regs, r20)); - DEFINE(PT_REGS_R21, offsetof(struct pt_regs, r21)); - DEFINE(PT_REGS_R22, offsetof(struct pt_regs, r22)); - DEFINE(PT_REGS_R23, offsetof(struct pt_regs, r23)); - DEFINE(PT_REGS_R24, offsetof(struct pt_regs, r24)); - DEFINE(PT_REGS_R25, offsetof(struct pt_regs, r25)); - DEFINE(PT_REGS_R26, offsetof(struct pt_regs, r26)); - DEFINE(PT_REGS_R27, offsetof(struct pt_regs, r27)); - DEFINE(PT_REGS_R28, offsetof(struct pt_regs, r28)); - DEFINE(PT_REGS_GP, offsetof(struct pt_regs, gp)); - DEFINE(PT_REGS_SP, offsetof(struct pt_regs, sp)); + DEFINE(PT_REGS_R0, offsetof(struct pt_regs, regs[0])); + DEFINE(PT_REGS_R1, offsetof(struct pt_regs, regs[1])); + DEFINE(PT_REGS_R2, offsetof(struct pt_regs, regs[2])); + DEFINE(PT_REGS_R3, offsetof(struct pt_regs, regs[3])); + DEFINE(PT_REGS_R4, offsetof(struct pt_regs, regs[4])); + DEFINE(PT_REGS_R5, offsetof(struct pt_regs, regs[5])); + DEFINE(PT_REGS_R6, offsetof(struct pt_regs, regs[6])); + DEFINE(PT_REGS_R7, offsetof(struct pt_regs, regs[7])); + DEFINE(PT_REGS_R8, offsetof(struct pt_regs, regs[8])); + DEFINE(PT_REGS_R9, offsetof(struct pt_regs, regs[9])); + DEFINE(PT_REGS_R10, offsetof(struct pt_regs, regs[10])); + DEFINE(PT_REGS_R11, offsetof(struct pt_regs, regs[11])); + DEFINE(PT_REGS_R12, offsetof(struct pt_regs, regs[12])); + DEFINE(PT_REGS_R13, offsetof(struct pt_regs, regs[13])); + DEFINE(PT_REGS_R14, offsetof(struct pt_regs, regs[14])); + DEFINE(PT_REGS_R15, offsetof(struct pt_regs, regs[15])); + DEFINE(PT_REGS_R16, offsetof(struct pt_regs, regs[16])); + DEFINE(PT_REGS_R17, offsetof(struct pt_regs, regs[17])); + DEFINE(PT_REGS_R18, offsetof(struct pt_regs, regs[18])); + DEFINE(PT_REGS_R19, offsetof(struct pt_regs, regs[19])); + DEFINE(PT_REGS_R20, offsetof(struct pt_regs, regs[20])); + DEFINE(PT_REGS_R21, offsetof(struct pt_regs, regs[21])); + DEFINE(PT_REGS_R22, offsetof(struct pt_regs, regs[22])); + DEFINE(PT_REGS_R23, offsetof(struct pt_regs, regs[23])); + DEFINE(PT_REGS_R24, offsetof(struct pt_regs, regs[24])); + DEFINE(PT_REGS_R25, offsetof(struct pt_regs, regs[25])); + DEFINE(PT_REGS_R26, offsetof(struct pt_regs, regs[26])); + DEFINE(PT_REGS_R27, offsetof(struct pt_regs, regs[27])); + DEFINE(PT_REGS_R28, offsetof(struct pt_regs, regs[28])); + DEFINE(PT_REGS_GP, offsetof(struct pt_regs, regs[29])); + DEFINE(PT_REGS_SP, offsetof(struct pt_regs, regs[30])); DEFINE(PT_REGS_PC, offsetof(struct pt_regs, pc)); DEFINE(PT_REGS_PS, offsetof(struct pt_regs, ps)); + DEFINE(PT_REGS_ORIG_R0, offsetof(struct pt_regs, orig_r0)); + DEFINE(PT_REGS_ORIG_R19, offsetof(struct pt_regs, orig_r19)); DEFINE(PT_REGS_HM_PS, offsetof(struct pt_regs, hm_ps)); DEFINE(PT_REGS_HM_PC, offsetof(struct pt_regs, hm_pc)); DEFINE(PT_REGS_HM_GP, offsetof(struct pt_regs, hm_gp)); @@ -232,4 +234,7 @@ void foo(void) OFFSET(TASK_THREAD_S6, task_struct, thread.s[6]); BLANK(); DEFINE(ASM_THREAD_SIZE, THREAD_SIZE); + BLANK(); + DEFINE(RT_SIGFRAME_SIZE, sizeof(struct rt_sigframe)); + OFFSET(RT_SIGFRAME_MCTX, rt_sigframe, uc.uc_mcontext); } diff --git a/arch/sw_64/kernel/bindvcpu.c b/arch/sw_64/kernel/bindvcpu.c deleted file mode 100644 index 46617eb68b7a3a09e0d264167e91bf615dfcd5c4..0000000000000000000000000000000000000000 --- a/arch/sw_64/kernel/bindvcpu.c +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2022 Wang Yuanheng - * Author: Wang Yuanheng - * - */ - -#include -#include -#include -#include -#include - -__read_mostly bool bind_vcpu_enabled; -EXPORT_SYMBOL(bind_vcpu_enabled); - -static int __init bind_vcpu_init(void) -{ - struct dentry *bindvcpu; - - if (!sw64_debugfs_dir) - return -ENODEV; - - bindvcpu = debugfs_create_bool("bind_vcpu", 0644, - sw64_debugfs_dir, &bind_vcpu_enabled); - if (!bindvcpu) - return -ENOMEM; - return 0; -} -late_initcall(bind_vcpu_init); diff --git a/arch/sw_64/kernel/cacheinfo.c b/arch/sw_64/kernel/cacheinfo.c index 87d3f4bcd10f12efdea915910fbeb2ab9f25c50d..e340c53690a9e486269465512e9ddf494afd1fe0 100644 --- a/arch/sw_64/kernel/cacheinfo.c +++ b/arch/sw_64/kernel/cacheinfo.c @@ -89,7 +89,7 @@ int populate_cache_leaves(unsigned int cpu) } if (c->tcache.size) { - cpumask_copy(&this_leaf->shared_cpu_map, cpu_online_mask); + cpumask_copy(&this_leaf->shared_cpu_map, topology_llc_cpumask(cpu)); populate_cache(tcache, this_leaf, 3, CACHE_TYPE_UNIFIED, topo->package_id); } diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c new file mode 100644 index 0000000000000000000000000000000000000000..b8c359db2ef67b1272600987621860cbefbfc47f --- /dev/null +++ b/arch/sw_64/kernel/chip_setup.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include + +struct sw64_chip_ops *sw64_chip; +struct sw64_chip_init_ops *sw64_chip_init; + +static int get_cpu_nums(void) +{ + if (is_guest_or_emul()) + return 1; + + return __get_cpu_nums(); +} + +static unsigned long __init get_node_mem(int nodeid) +{ + + if (is_guest_or_emul()) + return *(unsigned long *)MMSIZE & MMSIZE_MASK; + + return __get_node_mem(nodeid); +} + +static void __init setup_core_map(struct cpumask *cpumask) +{ + int i, j, cpu_num, cpuid, max_cores_per_cpu; + unsigned long coreonline; + + cpu_num = get_cpu_nums(); + cpuid = 0; + for (i = 0; i < cpu_num; i++) { + coreonline = sw64_io_read(i, CORE_ONLINE); + max_cores_per_cpu = MAX_CORES_PER_CPU; + + if (is_guest_or_emul()) + max_cores_per_cpu = 64; + + for (j = 0; j < max_cores_per_cpu; j++) { + if (coreonline & (1UL << j)) { + __cpu_to_rcid[cpuid] = (i << DOMAIN_ID_SHIFT) | (j << CORE_ID_SHIFT); + cpuid++; + } + } + } + + if (is_in_host() && core_is_ht()) { + for (i = 0; i < cpuid; i++) + __cpu_to_rcid[cpuid + i] = __cpu_to_rcid[i] | (1 << THREAD_ID_SHIFT); + + cpuid = cpuid + i; + } + + while (cpuid < NR_CPUS) { + __cpu_to_rcid[cpuid] = -1; + cpuid++; + } +} + +#ifdef CONFIG_PM +static void i2c_srst(void) +{ + sw64_io_write(0, I2C0_SRST_L, 0x0); + sw64_io_write(0, I2C0_SRST_L, 0x1); + + sw64_io_write(0, I2C1_SRST_L, 0x0); + sw64_io_write(0, I2C1_SRST_L, 0x1); + + sw64_io_write(0, I2C2_SRST_L, 0x0); + sw64_io_write(0, I2C2_SRST_L, 0x1); +} + +static void pcie_save(void) +{ + struct pci_controller *hose; + struct piu_saved *piu_save; + unsigned long node, index; + unsigned long i; + + for (hose = hose_head; hose; hose = hose->next) { + piu_save = kzalloc(sizeof(*piu_save), GFP_KERNEL); + + node = hose->node; + index = hose->index; + hose->sysdata = piu_save; + + piu_save->piuconfig0 = read_piu_ior0(node, index, PIUCONFIG0); + piu_save->piuconfig1 = read_piu_ior1(node, index, PIUCONFIG1); + piu_save->epdmabar = read_piu_ior0(node, index, EPDMABAR); + piu_save->msiaddr = read_piu_ior0(node, index, MSIADDR); + + if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { + for (i = 0; i < 256; i++) { + piu_save->msiconfig[i] = read_piu_ior0(node, index, + MSICONFIG0 + (i << 7)); + } + } + + piu_save->iommuexcpt_ctrl = read_piu_ior0(node, index, IOMMUEXCPT_CTRL); + piu_save->dtbaseaddr = read_piu_ior0(node, index, DTBASEADDR); + + piu_save->intaconfig = read_piu_ior0(node, index, INTACONFIG); + piu_save->intbconfig = read_piu_ior0(node, index, INTBCONFIG); + piu_save->intcconfig = read_piu_ior0(node, index, INTCCONFIG); + piu_save->intdconfig = read_piu_ior0(node, index, INTDCONFIG); + piu_save->pmeintconfig = read_piu_ior0(node, index, PMEINTCONFIG); + piu_save->aererrintconfig = read_piu_ior0(node, index, AERERRINTCONFIG); + piu_save->hpintconfig = read_piu_ior0(node, index, HPINTCONFIG); + + } +} + +static void pcie_restore(void) +{ + struct pci_controller *hose; + struct piu_saved *piu_save; + unsigned long node, index; + u32 rc_misc_ctrl; + unsigned int value; + unsigned long i; + + for (hose = hose_head; hose; hose = hose->next) { + node = hose->node; + index = hose->index; + piu_save = hose->sysdata; + + write_piu_ior0(node, index, PIUCONFIG0, piu_save->piuconfig0); + write_piu_ior1(node, index, PIUCONFIG1, piu_save->piuconfig1); + write_piu_ior0(node, index, EPDMABAR, piu_save->epdmabar); + write_piu_ior0(node, index, MSIADDR, piu_save->msiaddr); + + if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { + for (i = 0; i < 256; i++) { + write_piu_ior0(node, index, MSICONFIG0 + (i << 7), + piu_save->msiconfig[i]); + } + } + + write_piu_ior0(node, index, IOMMUEXCPT_CTRL, piu_save->iommuexcpt_ctrl); + write_piu_ior0(node, index, DTBASEADDR, piu_save->dtbaseaddr); + + write_piu_ior0(node, index, INTACONFIG, piu_save->intaconfig); + write_piu_ior0(node, index, INTBCONFIG, piu_save->intbconfig); + write_piu_ior0(node, index, INTCCONFIG, piu_save->intcconfig); + write_piu_ior0(node, index, INTDCONFIG, piu_save->intdconfig); + write_piu_ior0(node, index, PMEINTCONFIG, piu_save->pmeintconfig); + write_piu_ior0(node, index, AERERRINTCONFIG, piu_save->aererrintconfig); + write_piu_ior0(node, index, HPINTCONFIG, piu_save->hpintconfig); + + /* Enable DBI_RO_WR_EN */ + rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); + write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); + + /* Fix up DEVICE_ID_VENDOR_ID register */ + value = (PCI_DEVICE_ID_SW64_ROOT_BRIDGE << 16) | PCI_VENDOR_ID_JN; + write_rc_conf(node, index, RC_VENDOR_ID, value); + + /* Set PCI-E root class code */ + value = read_rc_conf(node, index, RC_REVISION_ID); + write_rc_conf(node, index, RC_REVISION_ID, (PCI_CLASS_BRIDGE_HOST << 16) | value); + + /* Disable DBI_RO_WR_EN */ + write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); + } + +} + +static unsigned long saved_dvc_int, saved_long_time; + +static inline void intpu_save(void) +{ + switch (cpu_desc.model) { + case CPU_SW831: + saved_long_time = __io_read_longtime(0); + default: + break; + } +} + +static inline void intpu_restore(void) +{ + switch (cpu_desc.model) { + case CPU_SW831: + __io_write_longtime(0, saved_long_time); + __io_write_longtime_start_en(0, 0x1); + break; + default: + pr_info("long time start is disable!"); + break; + } +} + +static inline void spbu_save(void) +{ + saved_dvc_int = sw64_io_read(0, MCU_DVC_INT_EN); +} + +static inline void spbu_restore(void) +{ + i2c_srst(); + sw64_io_write(0, MCU_DVC_INT_EN, saved_dvc_int); +} + +static int io_suspend(void) +{ + spbu_save(); + intpu_save(); + pcie_save(); + + return 0; +} + +static void io_resume(void) +{ + pcie_restore(); + intpu_restore(); + spbu_restore(); +} +#endif /* CONFIG_PM */ + +static struct sw64_chip_init_ops chip_init_ops = { + .early_init = { + .setup_core_map = setup_core_map, + .get_node_mem = get_node_mem, + }, +}; + +static struct sw64_chip_ops chip_ops = { + .get_cpu_num = get_cpu_nums, +}; + +void __init setup_chip_ops(void) +{ + sw64_chip_init = &chip_init_ops; + sw64_chip = &chip_ops; + setup_chip_pci_ops(); +#ifdef CONFIG_PM + io_syscore_ops.suspend = io_suspend; + io_syscore_ops.resume = io_resume; +#endif +} diff --git a/arch/sw_64/kernel/cpuautoplug.c b/arch/sw_64/kernel/cpuautoplug.c index de6f77086185abdc164ffc7cdac79fa89f5f28ef..16017b6d12548fd433e49928c4b6ff78b67665ae 100644 --- a/arch/sw_64/kernel/cpuautoplug.c +++ b/arch/sw_64/kernel/cpuautoplug.c @@ -12,7 +12,7 @@ #include #include -#include +#include #include #include @@ -257,18 +257,17 @@ static inline cputime64_t sw64_get_idle_time(cputime64_t *wall) static cputime64_t get_min_busy_time(cputime64_t arr[], int size) { - int loop, min_idx; + int i, min_cpu_idx; cputime64_t min_time = arr[0]; - for (loop = 1; loop < size; loop++) { - if (arr[loop] > 0) { - if (arr[loop] < min_time) { - min_time = arr[loop]; - min_idx = loop; - } + for (i = 0; i < size; i++) { + if (arr[i] > 0 && arr[i] < min_time) { + min_time = arr[i]; + min_cpu_idx = i; } } - return min_idx; + + return min_cpu_idx; } static int find_min_busy_cpu(void) @@ -276,7 +275,7 @@ static int find_min_busy_cpu(void) int nr_all_cpus = num_possible_cpus(); unsigned int cpus, target_cpu; cputime64_t busy_time; - cputime64_t b_time[nr_all_cpus]; + cputime64_t b_time[NR_CPUS]; memset(b_time, 0, sizeof(b_time)); for_each_online_cpu(cpus) { @@ -284,19 +283,19 @@ static int find_min_busy_cpu(void) b_time[cpus] = busy_time; } target_cpu = get_min_busy_time(b_time, nr_all_cpus); - pr_info("The target_cpu is %d, the cpu_num is %d\n", - target_cpu, num_online_cpus() - 1); return target_cpu; } static void increase_cores(int cur_cpus) { + struct device *dev; + if (cur_cpus == ap_info.maxcpus) return; cur_cpus = cpumask_next_zero(0, cpu_online_mask); - struct device *dev = get_cpu_device(cur_cpus); + dev = get_cpu_device(cur_cpus); per_cpu(cpu_adjusting, dev->id) = 1; lock_device_hotplug(); @@ -310,17 +309,21 @@ static void increase_cores(int cur_cpus) static void decrease_cores(int cur_cpus) { + struct device *dev; + if (cur_cpus == ap_info.mincpus) return; cur_cpus = find_min_busy_cpu(); - struct device *dev = get_cpu_device(cur_cpus); + dev = get_cpu_device(cur_cpus); if (dev->id > 0) { per_cpu(cpu_adjusting, dev->id) = -1; lock_device_hotplug(); cpu_device_down(dev); + pr_info("The target_cpu is %d. After cpu_down, the cpu_num is %d\n", + cur_cpus, num_online_cpus()); get_cpu_device(dev->id)->offline = true; unlock_device_hotplug(); per_cpu(cpu_adjusting, dev->id) = 0; @@ -452,7 +455,7 @@ static int __init cpuautoplug_init(void) ap_info.maxcpus = setup_max_cpus > nr_cpu_ids ? nr_cpu_ids : setup_max_cpus; - ap_info.mincpus = 16; + ap_info.mincpus = ap_info.maxcpus / 4; ap_info.dec_reqs = 0; ap_info.inc_reqs = 0; ap_info.sampling_rate = 720; /* 720ms */ diff --git a/arch/sw_64/kernel/dup_print.c b/arch/sw_64/kernel/dup_print.c index e28e0053239cad1d59b2f568029a4052c9e6368f..439ac75feb01f23ed201eb4e4dc3ba23da8ff232 100644 --- a/arch/sw_64/kernel/dup_print.c +++ b/arch/sw_64/kernel/dup_print.c @@ -15,12 +15,6 @@ static DEFINE_SPINLOCK(printk_lock); unsigned long sw64_printk_offset; #define PRINTK_SIZE 0x100000UL -/* - * For output the kernel message on the console - * with full-system emulator. - */ -#define QEMU_PRINTF_BUFF_BASE (IO_BASE | MCU_BASE | 0x40000UL) - int sw64_printk(const char *fmt, va_list args) { char *sw64_printk_buf; @@ -52,7 +46,7 @@ int sw64_printk(const char *fmt, va_list args) #endif #ifdef CONFIG_SW64_RRU -#include +#include static DEFINE_SPINLOCK(printf_lock); #define USER_PRINT_BUFF_BASE (0x600000UL + __START_KERNEL_map) diff --git a/arch/sw_64/kernel/early_init.c b/arch/sw_64/kernel/early_init.c index bcd458a9bdad4e01cfc745451b275406aaae9e02..2ec7a3e994436034ec5d507858d9588d0f3ed6c0 100644 --- a/arch/sw_64/kernel/early_init.c +++ b/arch/sw_64/kernel/early_init.c @@ -3,30 +3,9 @@ #include -void sw64_init_noop(void) { } -struct sw64_platform_ops *sw64_platform; -EXPORT_SYMBOL(sw64_platform); -struct sw64_chip_ops *sw64_chip; -struct sw64_chip_init_ops *sw64_chip_init; - -static void __init sw64_setup_platform_ops(void) -{ - /* - * FIXME: set platform operation depending on CONFIG now. - * SMBIOS will help use to determin actual board. - */ -#ifdef CONFIG_PLATFORM_XUELANG - sw64_platform = &xuelang_ops; -#endif -} - - asmlinkage __visible void __init sw64_start_kernel(void) { fixup_hmcall(); save_ktp(); - sw64_setup_chip_ops(); - sw64_setup_platform_ops(); - sw64_platform->ops_fixup(); start_kernel(); } diff --git a/arch/sw_64/kernel/entry.S b/arch/sw_64/kernel/entry.S index 013656e78d08532eefe5d7fed16cc9dad2c1d29c..59c2ff4eb91504efc802dd8994854947aea6315f 100644 --- a/arch/sw_64/kernel/entry.S +++ b/arch/sw_64/kernel/entry.S @@ -8,6 +8,7 @@ #include #include #include +#include .text .set noat @@ -63,14 +64,20 @@ and $6, 0x8, $7 beq $7, 1f sys_call HMC_rdusp - br $31, 2f + br 2f 1: ldi $0, PT_REGS_SIZE($sp) 2: stl $0, PT_REGS_SP($sp) + ldi $1, NO_SYSCALL + stl $1, PT_REGS_ORIG_R0($sp) sys_call HMC_rdktp .endm .macro RESTORE_ALL - ldl $1, PT_REGS_R16($sp) + ldl $16, PT_REGS_SP($sp) + /* skip wrusp if returning to kernel */ + blt $16, 1f + sys_call HMC_wrusp +1: ldl $1, PT_REGS_R16($sp) ldl $2, PT_REGS_R17($sp) ldl $3, PT_REGS_R18($sp) ldl $4, PT_REGS_GP($sp) @@ -120,9 +127,9 @@ .ent entInt entInt: SAVE_ALL - ldi $26, ret_from_sys_call mov $sp, $19 - call $31, do_entInt + call $26, do_entInt + br ret_from_sys_call .end entInt .align 4 @@ -130,9 +137,9 @@ entInt: .ent entArith entArith: SAVE_ALL - ldi $26, ret_from_sys_call mov $sp, $18 - call $31, do_entArith + call $26, do_entArith + br ret_from_sys_call .end entArith .align 4 @@ -140,9 +147,9 @@ entArith: .ent entMM entMM: SAVE_ALL - ldi $26, ret_from_sys_call mov $sp, $19 - call $31, do_page_fault + call $26, do_page_fault + br ret_from_sys_call .end entMM .align 4 @@ -150,9 +157,9 @@ entMM: .ent entIF entIF: SAVE_ALL - ldi $26, ret_from_sys_call mov $sp, $18 - call $31, do_entIF + call $26, do_entIF + br ret_from_sys_call .end entIF /* @@ -170,8 +177,8 @@ entUna: ldl $0, PT_REGS_PS($sp) and $0, 8, $0 /* user mode ? */ beq $0, 1f - ldi $26, ret_from_sys_call - call $31, do_entUnaUser /* return to ret_from_syscall */ + call $26, do_entUnaUser /* return to ret_from_syscall */ + br ret_from_sys_call 1: ldl $9, PT_REGS_GP($sp) call $26, do_entUna stl $9, PT_REGS_GP($sp) @@ -192,197 +199,44 @@ entUna: .align 4 .globl entSys - .globl ret_from_sys_call .ent entSys entSys: - SAVE_ALL - ldl $0, PT_REGS_R0($sp) - ldi $4, NR_SYSCALLS($31) stl $16, PT_REGS_R16($sp) - ldi $5, sys_call_table - ldi $27, sys_ni_syscall - cmpult $0, $4, $4 - ldw $3, TI_FLAGS($8) stl $17, PT_REGS_R17($sp) - s8addl $0, $5, $5 stl $18, PT_REGS_R18($sp) - ldi $6, _TIF_SYSCALL_WORK - and $3, $6, $3 - bne $3, strace - - beq $4, 1f - ldl $27, 0($5) -1: call $26, ($27), ni_syscall - ldgp $gp, 0($26) - blt $0, $syscall_error /* the call failed */ - stl $0, PT_REGS_R0($sp) - stl $31, PT_REGS_R19($sp) /* a3=0 => no error */ + mov $sp, $16 + call $26, do_entSys + br ret_from_sys_call + .end entSys .align 4 + .globl ret_from_sys_call + .ent ret_from_sys_call ret_from_sys_call: #ifdef CONFIG_SUBARCH_C3B fillcs 0($sp) /* prefetch */ fillcs 128($sp) /* prefetch */ #endif - selne $26, 0, $18, $18 /* $18 = 0 => non-restartable */ - ldl $0, PT_REGS_PS($sp) - and $0, 8, $0 - beq $0, ret_to_kernel -ret_to_user: -#ifdef CONFIG_DEBUG_RSEQ - mov $sp, $16 - call $26, rseq_syscall -#endif + br $27, 1f +1: ldgp $29, 0($27) /* Make sure need_resched and sigpending don't change between sampling and the rti. */ ldi $16, 7 sys_call HMC_swpipl + ldl $0, PT_REGS_PS($sp) + and $0, 8, $0 + beq $0, restore_all +ret_to_user: ldw $17, TI_FLAGS($8) and $17, _TIF_WORK_MASK, $2 - bne $2, work_pending + beq $2, restore_all + mov $sp, $16 + call $26, do_notify_resume restore_all: RESTORE_ALL sys_call HMC_rti - -ret_to_kernel: - ldi $16, 7 - sys_call HMC_swpipl - br restore_all - - - .align 3 -$syscall_error: - /* - * Some system calls (e.g., ptrace) can return arbitrary - * values which might normally be mistaken as error numbers. - * Those functions must zero $0 (v0) directly in the stack - * frame to indicate that a negative return value wasn't an - * error number.. - */ - ldl $18, PT_REGS_R0($sp) /* old syscall nr (zero if success) */ - beq $18, $ret_success - - ldl $19, PT_REGS_R19($sp) /* .. and this a3 */ - subl $31, $0, $0 /* with error in v0 */ - addl $31, 1, $1 /* set a3 for errno return */ - stl $0, PT_REGS_R0($sp) - mov $31, $26 /* tell "ret_from_sys_call" we can restart */ - stl $1, PT_REGS_R19($sp) /* a3 for return */ - br ret_from_sys_call - - -$ret_success: - stl $0, PT_REGS_R0($sp) - stl $31, PT_REGS_R19($sp) /* a3=0 => no error */ - br ret_from_sys_call - .end entSys - -/* - * Do all cleanup when returning from all interrupts and system calls. - * - * Arguments: - * $8: current. - * $17: TI_FLAGS. - * $18: The old syscall number, or zero if this is not a return - * from a syscall that errored and is possibly restartable. - * $19: The old a3 value - */ - - .align 4 - .ent work_pending -work_pending: - and $17, _TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_UPROBE, $2 - bne $2, $work_notifysig - -$work_resched: - /* - * We can get here only if we returned from syscall without SIGPENDING - * or got through work_notifysig already. Either case means no syscall - * restarts for us, so let $18 and $19 burn. - */ - call $26, schedule - mov 0, $18 - br ret_to_user - -$work_notifysig: - mov $sp, $16 - call $26, do_work_pending - br restore_all - .end work_pending - - - -/* - * PTRACE syscall handler - */ - - .align 4 - .ent strace -strace: - /* set up signal stack, call syscall_trace */ - mov $0, $9 - mov $19, $10 - call $26, syscall_trace_enter - blt $0, $syscall_trace_failed - - /* get the system call number and the arguments back.. */ - ldl $16, PT_REGS_R16($sp) - ldl $17, PT_REGS_R17($sp) - ldl $18, PT_REGS_R18($sp) - ldl $19, PT_REGS_R19($sp) - ldl $20, PT_REGS_R20($sp) - ldl $21, PT_REGS_R21($sp) - - /* get the system call pointer.. */ - ldi $1, NR_SYSCALLS($31) - ldi $2, sys_call_table - ldi $27, ni_syscall - - cmpult $0, $1, $1 - s8addl $0, $2, $2 - beq $1, 1f - ldl $27, 0($2) -1: call $26, ($27), sys_gettimeofday -ret_from_straced: - ldgp $gp, 0($26) - - /* check return.. */ - blt $0, $strace_error /* the call failed */ - stl $31, PT_REGS_R19($sp) /* a3=0 => no error */ -$strace_success: - stl $0, PT_REGS_R0($sp) /* save return value */ - call $26, syscall_trace_leave - br $31, ret_from_sys_call - - .align 3 -$strace_error: - ldl $18, PT_REGS_R0($sp) /* old syscall nr (zero if success) */ - - beq $18, $strace_success - ldl $19, PT_REGS_R19($sp) /* .. and this a3 */ - - subl $31, $0, $0 /* with error in v0 */ - addl $31, 1, $1 /* set a3 for errno return */ - stl $0, PT_REGS_R0($sp) - stl $1, PT_REGS_R19($sp) /* a3 for return */ - - mov $18, $9 /* save old syscall number */ - mov $19, $10 /* save old a3 */ - call $26, syscall_trace_leave - mov $9, $18 - mov $10, $19 - - mov $31, $26 /* tell "ret_from_sys_call" we can restart */ - br ret_from_sys_call - -$syscall_trace_failed: - call $26, syscall_trace_leave - mov $9, $18 - mov $10, $19 - mov $31, $26 /* tell "ret_from_sys_call" we can restart */ - br ret_from_sys_call - .end strace + .end ret_from_sys_call /* * Integer register context switch @@ -433,8 +287,8 @@ __switch_to: .align 4 .ent ret_from_fork ret_from_fork: - ldi $26, ret_from_sys_call - call $31, schedule_tail + call $26, schedule_tail + br ret_from_sys_call .end ret_from_fork /* @@ -448,47 +302,5 @@ ret_from_kernel_thread: mov $9, $27 mov $10, $16 call $26, ($9) - mov $31, $19 /* to disable syscall restarts */ - br $31, ret_to_user + br ret_to_user .end ret_from_kernel_thread - - .align 4 - .globl sys_sigreturn - .ent sys_sigreturn -sys_sigreturn: - .prologue 0 - ldi $9, ret_from_straced - cmpult $26, $9, $9 - call $26, do_sigreturn - bne $9, 1f - call $26, syscall_trace_leave -1: br ret_from_sys_call - .end sys_sigreturn - - .align 4 - .globl sys_rt_sigreturn - .ent sys_rt_sigreturn -sys_rt_sigreturn: - .prologue 0 - ldi $9, ret_from_straced - cmpult $26, $9, $9 - call $26, do_rt_sigreturn - bne $9, 1f - call $26, syscall_trace_leave -1: br ret_from_sys_call - .end sys_rt_sigreturn - - .align 4 - .globl ni_syscall - .ent ni_syscall -ni_syscall: - .prologue 0 - /* Special because it also implements overflow handling via - * syscall number 0. And if you recall, zero is a special - * trigger for "not an error". Store large non-zero there. - */ - ldi $0, -ENOSYS - unop - stl $0, PT_REGS_R0($sp) - ret - .end ni_syscall diff --git a/arch/sw_64/kernel/fpu.S b/arch/sw_64/kernel/fpu.S index 3cb3bfab08e8270e9b0fe60f08e17b0700fc0486..ddc988681fdd01986c6d2ed068f6b37c75e811d3 100644 --- a/arch/sw_64/kernel/fpu.S +++ b/arch/sw_64/kernel/fpu.S @@ -3,11 +3,16 @@ #include #include #include +#include .text .set noat ENTRY(__fpstate_save) /* a0: prev task */ +#ifdef CONFIG_SUBARCH_C4 + csrr $1, CSR_WR_FREGS + beq $1, out +#endif vstd $f0, TASK_THREAD_F0(a0) vstd $f1, TASK_THREAD_F1(a0) vstd $f2, TASK_THREAD_F2(a0) @@ -42,6 +47,7 @@ ENTRY(__fpstate_save) vstd $f30, TASK_THREAD_F30(a0) fstd $f0, TASK_THREAD_FPCR(a0) vldd $f0, TASK_THREAD_F0(a0) +out: ret END(__fpstate_save) @@ -98,5 +104,8 @@ $setfpec_over: vldd $f28, TASK_THREAD_F28(a0) vldd $f29, TASK_THREAD_F29(a0) vldd $f30, TASK_THREAD_F30(a0) +#ifdef CONFIG_SUBARCH_C4 + csrw $31, CSR_WR_FREGS +#endif ret END(__fpstate_restore) diff --git a/arch/sw_64/kernel/head.S b/arch/sw_64/kernel/head.S index c855d31de7154097d6aa059f2a9e0bb688adf6bc..fd0fbfbcf5b6418b561d3d56265d567d36ed437b 100644 --- a/arch/sw_64/kernel/head.S +++ b/arch/sw_64/kernel/head.S @@ -66,11 +66,23 @@ __smp_callin: bis $31, $31, $16 # invalidate all TLB with current VPN sys_call HMC_tbi +#if defined(CONFIG_SUBARCH_C3B) sys_call HMC_whami # Get hard cid - - ldi $1, __rcid_to_cpu - s4addl $0, $1, $1 - ldw $0, 0($1) # Get logical cpu number + ldi $1, __cpu_to_rcid + ldi $2, 0($31) + ldi $4, CONFIG_NR_CPUS +3: ldw $3, 0($1) + cmpeq $3, $0, $3 + bne $3, 4f + addl $1, 4, $1 + addl $2, 1, $2 + cmpeq $2, $4, $5 + bne $5, 5f + br $31, 3b +4: ldi $0, 0($2) +#else + rcid $0 +#endif ldi $2, idle_task_pointer s8addl $0, $2, $2 @@ -81,6 +93,7 @@ __smp_callin: ldi $30, ASM_THREAD_SIZE($30) call $26, smp_callin +5: sys_call HMC_halt .end __smp_callin #endif /* CONFIG_SMP */ diff --git a/arch/sw_64/kernel/hibernate.c b/arch/sw_64/kernel/hibernate.c index 0e7e860c507e7576b1bdb895d94a374aa584918e..644ea85043136066c1129b059735d3feb7dc9f71 100644 --- a/arch/sw_64/kernel/hibernate.c +++ b/arch/sw_64/kernel/hibernate.c @@ -14,7 +14,7 @@ void save_processor_state(void) vcb->ksp = rdksp(); vcb->usp = rdusp(); - vcb->tid = rtid(); + vcb->soft_tid = rtid(); vcb->ptbr = rdptbr(); } @@ -24,7 +24,7 @@ void restore_processor_state(void) wrksp(vcb->ksp); wrusp(vcb->usp); - wrtp(vcb->tid); + wrtp(vcb->soft_tid); wrptbr(vcb->ptbr); sflush(); tbiv(); diff --git a/arch/sw_64/kernel/hibernate_asm.S b/arch/sw_64/kernel/hibernate_asm.S index 1e9abcf77beebf3327ff57252d54ba14de3bee9b..ff997cd76c5aef4bb9fa2eaaced2f57c21a0c631 100644 --- a/arch/sw_64/kernel/hibernate_asm.S +++ b/arch/sw_64/kernel/hibernate_asm.S @@ -114,7 +114,7 @@ $hibernate_setfpec_over: ldl sp, PSTATE_SP($16) ldl $8, PSTATE_KTP($16) - + sys_call HMC_wrktp ldi $0, 0($31) diff --git a/arch/sw_64/kernel/hmcall.c b/arch/sw_64/kernel/hmcall.c index e2be9f618e57690e095f47fcf86059c158b5e067..d2054a930bd72f648c363bee4282ddddd0f36572 100644 --- a/arch/sw_64/kernel/hmcall.c +++ b/arch/sw_64/kernel/hmcall.c @@ -92,6 +92,22 @@ static inline void fixup_wrktp(void) entry[1] = 0x1ee00000; /* pri_ret $23 */ } +static inline void fixup_rdusp(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(rdusp)); + + entry[0] = 0x94161018; /* pri_ldl/p $0, VC__USP(vcpucb) */ + entry[1] = 0x1ee00000; /* pri_ret $23 */ +} + +static inline void fixup_wrusp(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(wrusp)); + + entry[0] = 0xb6161018; /* pri_stl/p $16, VC__USP(vcpucb) */ + entry[1] = 0x1ee00000; /* pri_ret $23 */ +} + void __init fixup_hmcall(void) { #if defined(CONFIG_SUBARCH_C3B) @@ -101,6 +117,9 @@ void __init fixup_hmcall(void) fixup_wrasid(); fixup_rdktp(); fixup_wrktp(); + fixup_rdusp(); + fixup_wrusp(); + imemb(); #endif } diff --git a/arch/sw_64/kernel/idle.c b/arch/sw_64/kernel/idle.c index 8193a7093b570cb79838928d4320d86133608971..eb4738dedac5fec49f7defb0ed6ed7492d702125 100644 --- a/arch/sw_64/kernel/idle.c +++ b/arch/sw_64/kernel/idle.c @@ -9,7 +9,7 @@ #include #include -void cpu_idle(void) +void arch_cpu_idle(void) { local_irq_enable(); cpu_relax(); @@ -32,8 +32,3 @@ void cpu_idle(void) : "$1"); } } - -void arch_cpu_idle(void) -{ - cpu_idle(); -} diff --git a/arch/sw_64/kernel/irq_sw64.c b/arch/sw_64/kernel/irq_sw64.c index 88809fa531dd0de199abe83dc48f0b8f4524efc5..989d55ee1b1b83afd6aad586d0d29b28e48f7284 100644 --- a/arch/sw_64/kernel/irq_sw64.c +++ b/arch/sw_64/kernel/irq_sw64.c @@ -16,6 +16,13 @@ init_IRQ(void) * Just in case the platform init_irq() causes interrupts/mchecks * (as is the case with RAWHIDE, at least). */ + if (is_in_host()) { + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); + } + wrent(entInt, 0); sw64_init_irq(); diff --git a/arch/sw_64/kernel/kgdb.c b/arch/sw_64/kernel/kgdb.c index 95970b293de0773234137d29b9ca263a15bac8d1..833f72a1577ca8f2d2f01113c0443739a9a9c025 100644 --- a/arch/sw_64/kernel/kgdb.c +++ b/arch/sw_64/kernel/kgdb.c @@ -24,40 +24,40 @@ #include struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { - { "r0", 8, offsetof(struct pt_regs, r0)}, - { "r1", 8, offsetof(struct pt_regs, r1)}, - { "r2", 8, offsetof(struct pt_regs, r2)}, - { "r3", 8, offsetof(struct pt_regs, r3)}, - { "r4", 8, offsetof(struct pt_regs, r4)}, - { "r5", 8, offsetof(struct pt_regs, r5)}, - { "r6", 8, offsetof(struct pt_regs, r6)}, - { "r7", 8, offsetof(struct pt_regs, r7)}, - { "r8", 8, offsetof(struct pt_regs, r8)}, - - { "r9", 8, offsetof(struct pt_regs, r9)}, - { "r10", 8, offsetof(struct pt_regs, r10)}, - { "r11", 8, offsetof(struct pt_regs, r11)}, - { "r12", 8, offsetof(struct pt_regs, r12)}, - { "r13", 8, offsetof(struct pt_regs, r13)}, - { "r14", 8, offsetof(struct pt_regs, r14)}, - { "r15", 8, offsetof(struct pt_regs, r15)}, - - { "r16", 8, offsetof(struct pt_regs, r16)}, - { "r17", 8, offsetof(struct pt_regs, r17)}, - { "r18", 8, offsetof(struct pt_regs, r18)}, - - { "r19", 8, offsetof(struct pt_regs, r19)}, - { "r20", 8, offsetof(struct pt_regs, r20)}, - { "r21", 8, offsetof(struct pt_regs, r21)}, - { "r22", 8, offsetof(struct pt_regs, r22)}, - { "r23", 8, offsetof(struct pt_regs, r23)}, - { "r24", 8, offsetof(struct pt_regs, r24)}, - { "r25", 8, offsetof(struct pt_regs, r25)}, - { "r26", 8, offsetof(struct pt_regs, r26)}, - { "r27", 8, offsetof(struct pt_regs, r27)}, - { "at", 8, offsetof(struct pt_regs, r28)}, - { "gp", 8, offsetof(struct pt_regs, gp)}, - { "sp", 8, -1 }, + { "r0", 8, offsetof(struct pt_regs, regs[0])}, + { "r1", 8, offsetof(struct pt_regs, regs[1])}, + { "r2", 8, offsetof(struct pt_regs, regs[2])}, + { "r3", 8, offsetof(struct pt_regs, regs[3])}, + { "r4", 8, offsetof(struct pt_regs, regs[4])}, + { "r5", 8, offsetof(struct pt_regs, regs[5])}, + { "r6", 8, offsetof(struct pt_regs, regs[6])}, + { "r7", 8, offsetof(struct pt_regs, regs[7])}, + { "r8", 8, offsetof(struct pt_regs, regs[8])}, + + { "r9", 8, offsetof(struct pt_regs, regs[9])}, + { "r10", 8, offsetof(struct pt_regs, regs[10])}, + { "r11", 8, offsetof(struct pt_regs, regs[11])}, + { "r12", 8, offsetof(struct pt_regs, regs[12])}, + { "r13", 8, offsetof(struct pt_regs, regs[13])}, + { "r14", 8, offsetof(struct pt_regs, regs[14])}, + { "r15", 8, offsetof(struct pt_regs, regs[15])}, + + { "r16", 8, offsetof(struct pt_regs, regs[16])}, + { "r17", 8, offsetof(struct pt_regs, regs[17])}, + { "r18", 8, offsetof(struct pt_regs, regs[18])}, + + { "r19", 8, offsetof(struct pt_regs, regs[19])}, + { "r20", 8, offsetof(struct pt_regs, regs[20])}, + { "r21", 8, offsetof(struct pt_regs, regs[21])}, + { "r22", 8, offsetof(struct pt_regs, regs[22])}, + { "r23", 8, offsetof(struct pt_regs, regs[23])}, + { "r24", 8, offsetof(struct pt_regs, regs[24])}, + { "r25", 8, offsetof(struct pt_regs, regs[25])}, + { "r26", 8, offsetof(struct pt_regs, regs[26])}, + { "r27", 8, offsetof(struct pt_regs, regs[27])}, + { "at", 8, offsetof(struct pt_regs, regs[28])}, + { "gp", 8, offsetof(struct pt_regs, regs[29])}, + { "sp", 8, offsetof(struct pt_regs, regs[30])}, { "zero", 8, -1 }, { "f0", 8, -1 }, diff --git a/arch/sw_64/kernel/kprobes/decode-insn.c b/arch/sw_64/kernel/kprobes/decode-insn.c index d376a7e2bee41a167270f4dd54b1c0a4755d462d..411dcc89fc4f2af4056951c2b489657f06994222 100644 --- a/arch/sw_64/kernel/kprobes/decode-insn.c +++ b/arch/sw_64/kernel/kprobes/decode-insn.c @@ -90,10 +90,12 @@ bool __kprobes sw64_insn_can_kprobe(kprobe_opcode_t *addr) printk("addr can't steppable\n"); return false; } +#ifdef CONFIG_SUBARCH_C3B if (!is_probed_between_atomic(addr)) { printk("addr between atomic cant probe\n"); return false; } +#endif return true; } #endif diff --git a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c index 69fd38135cad990d4ef1f579b30194cbb3d8c2ba..89d7dba9dc25c7938019f8362ef461ab2607b4c6 100644 --- a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c +++ b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c @@ -24,12 +24,12 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, if (kprobe_running()) { kprobes_inc_nmissed_count(p); } else { - regs->r28 -= MCOUNT_INSN_SIZE; + regs->regs[28] -= MCOUNT_INSN_SIZE; __this_cpu_write(current_kprobe, p); kcb->kprobe_status = KPROBE_HIT_ACTIVE; if (!p->pre_handler || !p->pre_handler(p, regs)) { - regs->r28 += MCOUNT_INSN_SIZE; + regs->regs[28] += MCOUNT_INSN_SIZE; if (unlikely(p->post_handler)) { kcb->kprobe_status = KPROBE_HIT_SSDONE; p->post_handler(p, regs, 0); diff --git a/arch/sw_64/kernel/kprobes/kprobes.c b/arch/sw_64/kernel/kprobes/kprobes.c index 7080c892a24d1e1fe8acb794548c9a74499baede..3bacfb3b3a4c58592e5e030923f2daf8d0d8f8a3 100644 --- a/arch/sw_64/kernel/kprobes/kprobes.c +++ b/arch/sw_64/kernel/kprobes/kprobes.c @@ -267,11 +267,11 @@ void kretprobe_trampoline(void); void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { - ri->ret_addr = (kprobe_opcode_t *) regs->r26; + ri->ret_addr = (kprobe_opcode_t *) regs->regs[26]; ri->fp = NULL; /* Replace the return addr with trampoline addr */ - regs->r26 = (unsigned long)kretprobe_trampoline; + regs->regs[26] = (unsigned long)kretprobe_trampoline; } /* @@ -284,7 +284,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, orig_ret_address = __kretprobe_trampoline_handler(regs, kretprobe_trampoline, NULL); instruction_pointer(regs) = orig_ret_address; - regs->r26 = orig_ret_address; + regs->regs[26] = orig_ret_address; /* * By returning a non-zero value, we are telling diff --git a/arch/sw_64/kernel/match.c b/arch/sw_64/kernel/match.c new file mode 100644 index 0000000000000000000000000000000000000000..3926391270daa6d3e03ce7e34dd9b04eba111a95 --- /dev/null +++ b/arch/sw_64/kernel/match.c @@ -0,0 +1,551 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#include +#include +#include + + +char da_match_buf[1024], dv_match_buf[1024], dav_match_buf[1024]; +char ia_match_buf[1024], iv_match_buf[1024], ida_match_buf[1024]; + +unsigned long da_match_cf1, da_match_cf2, da_match_cf3; +unsigned long dv_match_cf1, dv_match_cf2, dv_match_cf3; +unsigned long dav_match_cf1, dav_match_cf2, dav_match_cf3, + dav_match_cf4, dav_match_cf5; +unsigned long ia_match_cf1, ia_match_cf2, ia_match_cf3, ia_match_cf4; +unsigned long iv_match_cf1, iv_match_cf2; +unsigned long ida_match_cf1, ida_match_cf2; + +static int da_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", da_match_buf); + return 0; +} + +static int dv_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", dv_match_buf); + return 0; +} + +static int dav_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", dav_match_buf); + return 0; +} + +static int ia_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", ia_match_buf); + return 0; +} + +static int iv_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", iv_match_buf); + return 0; +} + +static int ida_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", ida_match_buf); + return 0; +} + +static int da_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, da_match_show, NULL); +} + +static int dv_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, dv_match_show, NULL); +} + +static int dav_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, dav_match_show, NULL); +} + +static int ia_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, ia_match_show, NULL); +} + +static int iv_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, iv_match_show, NULL); +} + +static int ida_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, ida_match_show, NULL); +} + +static void +write_da_match(void *i) +{ + unsigned long dc_ctl; + + write_csr(da_match_cf1, CSR_DA_MATCH); + write_csr(da_match_cf2, CSR_DA_MASK); + dc_ctl = read_csr(CSR_DC_CTLP); + dc_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) + | (0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) + | (0x3UL << DPM_MATCH)); + dc_ctl |= da_match_cf3; + write_csr(dc_ctl, CSR_DC_CTLP); +} + +static void +write_dv_match(void *i) +{ + unsigned long dc_ctl; + + write_csr(dv_match_cf1, CSR_DV_MATCH); + write_csr(dv_match_cf2, CSR_DV_MASK); + dc_ctl = read_csr(CSR_DC_CTLP); + dc_ctl &= ~((0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) + | (0x3UL << DPM_MATCH)); + dc_ctl |= ((0x1UL << DV_MATCH_EN_S) | dv_match_cf3); + write_csr(dc_ctl, CSR_DC_CTLP); +} + +static void +write_dav_match(void *i) +{ + unsigned long dc_ctl; + + write_csr(dav_match_cf1, CSR_DA_MATCH); + write_csr(dav_match_cf2, CSR_DA_MASK); + write_csr(dav_match_cf3, CSR_DV_MATCH); + write_csr(dav_match_cf4, CSR_DV_MASK); + dc_ctl = read_csr(CSR_DC_CTLP); + dc_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) + | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH)); + dc_ctl |= ((0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S) + | dav_match_cf5); + write_csr(dc_ctl, CSR_DC_CTLP); +} + +static void +write_ia_match(void *i) +{ + ia_match_cf1 |= (0x1UL << IA_MATCH_EN_S); + write_csr_imb(ia_match_cf1, CSR_IA_MATCH); + write_csr_imb(ia_match_cf2, CSR_IA_MASK); + write_csr(((0x3ffUL << 18) | ia_match_cf3), CSR_IA_VPNMATCH); + write_csr(((0x3ffUL << 18) | ia_match_cf4), CSR_IA_UPNMATCH); +} + +static void +write_iv_match(void *i) +{ + unsigned long ia_match_tmp; + + ia_match_tmp = read_csr(CSR_IA_MATCH); + ia_match_tmp &= ~(0x1UL << IV_PM_EN_S); + ia_match_tmp |= ((((iv_match_cf2 >> IV_PM_EN_S) & 0x1) << IV_PM_EN_S) + | (iv_match_cf2 & 0x3) | (0x1UL << IV_MATCH_EN_S)); + write_csr_imb(iv_match_cf1, CSR_IV_MATCH); + write_csr_imb(ia_match_tmp, CSR_IA_MATCH); +} + +static void +write_ida_match(void *i) +{ + + ida_match_cf1 |= (0x1UL << IDA_MATCH_EN_S); + write_csr(ida_match_cf1, CSR_IDA_MATCH); + write_csr(ida_match_cf2, CSR_IDA_MASK); +} + +static ssize_t da_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(da_match_buf) - 1, len); + if (copy_from_user(da_match_buf, user_buf, size)) + return -EFAULT; + + da_match_buf[size] = '\0'; + strcpy(tmp, da_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &da_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &da_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &da_match_cf3); + if (err) + return err; + + if (on_each_cpu(write_da_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + + return len; +} + +static ssize_t dv_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(dv_match_buf) - 1, len); + if (copy_from_user(dv_match_buf, user_buf, size)) + return -EFAULT; + + dv_match_buf[size] = '\0'; + strcpy(tmp, dv_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &dv_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &dv_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &dv_match_cf3); + if (err) + return err; + + if (on_each_cpu(write_dv_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + + return len; +} + +static ssize_t dav_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[500]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[500]; + int err; + char *ret = NULL; + + size = min(sizeof(dav_match_buf) - 1, len); + if (copy_from_user(dav_match_buf, user_buf, size)) + return -EFAULT; + + dav_match_buf[size] = '\0'; + strcpy(tmp, dav_match_buf); + p = tmp; + + for (i = 0 ; i < 5; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[500] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &dav_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &dav_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &dav_match_cf3); + if (err) + return err; + + err = kstrtoul(&tmp1[300], 0, &dav_match_cf4); + if (err) + return err; + + err = kstrtoul(&tmp1[400], 0, &dav_match_cf5); + if (err) + return err; + + + if (on_each_cpu(write_dav_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + return len; +} + +static ssize_t ia_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(ia_match_buf) - 1, len); + if (copy_from_user(ia_match_buf, user_buf, size)) + return -EFAULT; + + ia_match_buf[size] = '\0'; + strcpy(tmp, ia_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &ia_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &ia_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &ia_match_cf3); + if (err) + return err; + + err = kstrtoul(&tmp1[300], 0, &ia_match_cf4); + if (err) + return err; + + if (on_each_cpu(write_ia_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + return len; +} + +static ssize_t iv_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(ia_match_buf) - 1, len); + if (copy_from_user(ia_match_buf, user_buf, size)) + return -EFAULT; + + ia_match_buf[size] = '\0'; + strcpy(tmp, ia_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &iv_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &iv_match_cf2); + if (err) + return err; + + if (on_each_cpu(write_iv_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + return len; +} + + +static ssize_t ida_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(ida_match_buf) - 1, len); + if (copy_from_user(ida_match_buf, user_buf, size)) + return -EFAULT; + + ida_match_buf[size] = '\0'; + strcpy(tmp, ida_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &ida_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &ida_match_cf2); + if (err) + return err; + + if (on_each_cpu(write_ida_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + + return len; +} + +static const struct file_operations set_da_match_fops = { + .open = da_match_open, + .read = seq_read, + .write = da_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_dv_match_fops = { + .open = dv_match_open, + .read = seq_read, + .write = dv_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_dav_match_fops = { + .open = dav_match_open, + .read = seq_read, + .write = dav_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_ia_match_fops = { + .open = ia_match_open, + .read = seq_read, + .write = ia_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_iv_match_fops = { + .open = iv_match_open, + .read = seq_read, + .write = iv_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + + +static const struct file_operations set_ida_match_fops = { + .open = ida_match_open, + .read = seq_read, + .write = ida_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __init match_debugfs_init(void) +{ + struct dentry *match_entry; + + if (!sw64_debugfs_dir) + return -ENODEV; + + match_entry = debugfs_create_file("da_match", 0600, + sw64_debugfs_dir, NULL, + &set_da_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("dv_match", 0600, + sw64_debugfs_dir, NULL, + &set_dv_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("dav_match", 0600, + sw64_debugfs_dir, NULL, + &set_dav_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("ia_match", 0600, + sw64_debugfs_dir, NULL, + &set_ia_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("iv_match", 0600, + sw64_debugfs_dir, NULL, + &set_iv_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("ida_match", 0600, + sw64_debugfs_dir, NULL, + &set_ida_match_fops); + if (!match_entry) + return -ENOMEM; + + return 0; +} +late_initcall(match_debugfs_init); diff --git a/arch/sw_64/kernel/msi.c b/arch/sw_64/kernel/msi.c index ee1bda3c644741915f6a6cc376d8cf0fd0f41f38..eeb01e63e636c1111e9b7b80a0dc07dec4c392a2 100644 --- a/arch/sw_64/kernel/msi.c +++ b/arch/sw_64/kernel/msi.c @@ -2,6 +2,7 @@ #include #include #include +#include int msi_compose_msg(unsigned int irq, struct msi_msg *msg) { diff --git a/arch/sw_64/kernel/pci-noop.c b/arch/sw_64/kernel/pci-noop.c index a0aa2e5bb675d2c181d6711a3973778f61cbf5d7..abfba92fa6a9c388542859b69884b208a424c9c3 100644 --- a/arch/sw_64/kernel/pci-noop.c +++ b/arch/sw_64/kernel/pci-noop.c @@ -8,6 +8,7 @@ #include #include #include +#include /* * The PCI controller list. diff --git a/arch/sw_64/kernel/pci.c b/arch/sw_64/kernel/pci.c index 6cc872ba9ca54786c9bd44db3bd4a7f9665afde0..6e635c05bbacbc6c60e259104ea6898e27ef63fd 100644 --- a/arch/sw_64/kernel/pci.c +++ b/arch/sw_64/kernel/pci.c @@ -57,6 +57,39 @@ static void quirk_isa_bridge(struct pci_dev *dev) } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82378, quirk_isa_bridge); +/* + * Early fix up the Root Complex settings + */ +static void fixup_root_complex(struct pci_dev *dev) +{ + int i; + struct pci_bus *bus = dev->bus; + struct pci_controller *hose = bus->sysdata; + + hose->self_busno = hose->busn_space->start; + + if (likely(bus->number == hose->self_busno)) { + if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) { + /* Check Root Complex port again */ + dev->is_hotplug_bridge = 0; + dev->current_state = PCI_D0; + } + + dev->class &= 0xff; + dev->class |= PCI_CLASS_BRIDGE_PCI << 8; + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + dev->resource[i].start = 0; + dev->resource[i].end = 0; + dev->resource[i].flags = IORESOURCE_PCI_FIXED; + } + } + atomic_inc(&dev->enable_cnt); + + dev->no_msi = 1; +} + +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, fixup_root_complex); + /* Just declaring that the power-of-ten prefixes are actually the * power-of-two ones doesn't make it true :) */ @@ -136,7 +169,8 @@ resource_size_t pcibios_align_resource(void *data, const struct resource *res, static int __init pcibios_init(void) { - sw64_init_pci(); + if (acpi_disabled) + sw64_init_pci(); return 0; } subsys_initcall(pcibios_init); @@ -601,6 +635,7 @@ sw64_init_host(unsigned long node, unsigned long index) } void __weak set_devint_wken(int node) {} +void __weak set_adr_int(int node) {} void __init sw64_init_arch(void) { @@ -610,12 +645,20 @@ void __init sw64_init_arch(void) char id[8], msg[64]; int i; - pr_info("SW arch PCI initialize!\n"); cpu_num = sw64_chip->get_cpu_num(); for (node = 0; node < cpu_num; node++) { - if (is_in_host()) + if (is_in_host()) { set_devint_wken(node); + set_adr_int(node); + } + } + + if (!acpi_disabled) + return; + + pr_info("SW arch PCI initialize!\n"); + for (node = 0; node < cpu_num; node++) { rc_enable = sw64_chip_init->pci_init.get_rc_enable(node); if (rc_enable == 0) { printk("PCIe is disabled on node %ld\n", node); @@ -690,7 +733,24 @@ void __init sw64_init_irq(void) void __init sw64_init_pci(void) { + pci_add_flags(PCI_REASSIGN_ALL_BUS); common_init_pci(); + pci_clear_flags(PCI_REASSIGN_ALL_BUS); +} + +void __init reserve_mem_for_pci(void) +{ + int ret; + unsigned long base = PCI_32BIT_MEMIO; + + ret = add_memmap_region(base, PCI_32BIT_MEMIO_SIZE, memmap_pci); + if (ret) { + pr_err("reserved pages for pcie memory space failed\n"); + return; + } + + pr_info("reserved pages for pcie memory space %lx:%lx\n", base >> PAGE_SHIFT, + (base + PCI_32BIT_MEMIO_SIZE) >> PAGE_SHIFT); } static int setup_bus_dma_cb(struct pci_dev *pdev, void *data) @@ -705,3 +765,41 @@ static void fix_bus_dma_limit(struct pci_dev *dev) pr_info("Set zx200 bus_dma_limit to 32-bit\n"); } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ZHAOXIN, 0x071f, fix_bus_dma_limit); + +const struct dma_map_ops *dma_ops; +EXPORT_SYMBOL(dma_ops); + +#ifdef CONFIG_DCA +static void enable_sw_dca(struct pci_dev *dev) +{ + struct pci_controller *hose = (struct pci_controller *)dev->sysdata; + unsigned long node, rc_index, dca_ctl, dca_conf; + int i; + + if (dev->class >> 8 != PCI_CLASS_NETWORK_ETHERNET) + return; + + node = hose->node; + rc_index = hose->index; + + for (i = 0; i < 256; i++) { + dca_conf = read_piu_ior1(node, rc_index, DEVICEID0 + (i << 7)); + if (dca_conf >> 63) + continue; + else { + dca_conf = (1UL << 63) | (dev->bus->number << 8) | dev->devfn; + pr_info("dca device index %d, dca_conf = %#lx\n", i, dca_conf); + write_piu_ior1(node, rc_index, DEVICEID0 + (i << 7), dca_conf); + break; + } + } + + dca_ctl = read_piu_ior1(node, rc_index, DCACONTROL); + if (dca_ctl & 0x1) { + dca_ctl = 0x2; + write_piu_ior1(node, rc_index, DCACONTROL, dca_ctl); + pr_info("Node %ld RC %ld enable DCA 1.0\n", node, rc_index); + } +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, enable_sw_dca); +#endif diff --git a/arch/sw_64/kernel/pci_common.c b/arch/sw_64/kernel/pci_common.c deleted file mode 100644 index f996baca9d935a7881a510c1aaf190a2f0975a66..0000000000000000000000000000000000000000 --- a/arch/sw_64/kernel/pci_common.c +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/arch/sw_64/kernel/pci_iommu.c - */ - -#include -#include -#include -#include -#include - -static dma_addr_t sw64_direct_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - dma_addr_t dma_addr = page_to_phys(page) + offset; - - if (unlikely(swiotlb_force == SWIOTLB_FORCE)) - return swiotlb_map(dev, dma_addr, size, dir, attrs); - - if (unlikely(!dma_capable(dev, dma_addr, size, true))) { - if (swiotlb_force != SWIOTLB_NO_FORCE) - return swiotlb_map(dev, dma_addr, size, dir, attrs); - - dev_WARN_ONCE(dev, 1, - "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n", - &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); - return DMA_MAPPING_ERROR; - } - - return dma_addr; -} - -static inline void sw64_direct_unmap_page(struct device *dev, dma_addr_t addr, - size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - if (unlikely(is_swiotlb_buffer(addr))) - swiotlb_tbl_unmap_single(dev, addr, size, size, dir, attrs); -} - -static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) -{ - return phys + size - 1 <= - min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); -} - -static void *sw64_direct_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_addrp, gfp_t gfp, - unsigned long attrs) -{ - struct page *page; - void *ret; - u64 dma_limit; - - size = PAGE_ALIGN(size); - if (attrs & DMA_ATTR_NO_WARN) - gfp |= __GFP_NOWARN; - - dma_limit = min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); - if (dma_limit <= DMA_BIT_MASK(32)) - gfp |= GFP_DMA32; - - /* we always manually zero the memory once we are done */ - gfp &= ~__GFP_ZERO; -again: - page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size)); - if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { - dma_free_contiguous(dev, page, size); - page = NULL; - - if (IS_ENABLED(CONFIG_ZONE_DMA32) && - dma_limit < DMA_BIT_MASK(64) && - !(gfp & (GFP_DMA32 | GFP_DMA))) { - gfp |= GFP_DMA32; - goto again; - } - } - - if (!page) - return NULL; - - ret = page_address(page); - memset(ret, 0, size); - *dma_addrp = page_to_phys(page); - - return ret; -} - -static void sw64_direct_free_coherent(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_addr, - unsigned long attrs) -{ - if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { - /* cpu_addr is a struct page cookie, not a kernel address */ - dma_free_contiguous(dev, cpu_addr, size); - return; - } - - free_pages((unsigned long)cpu_addr, get_order(size)); -} - -static void sw64_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - struct scatterlist *sg; - int i; - - for_each_sg(sgl, sg, nents, i) - sw64_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir, - attrs); -} - -static int sw64_direct_map_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - int i; - struct scatterlist *sg; - - for_each_sg(sgl, sg, nents, i) { - sg_dma_address(sg) = sw64_direct_map_page(dev, sg_page(sg), - sg->offset, sg->length, dir, attrs); - if (sg->dma_address == DMA_MAPPING_ERROR) - goto out_unmap; - sg_dma_len(sg) = sg->length; - } - return nents; - -out_unmap: - sw64_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); - return 0; -} - -const struct dma_map_ops sw64_dma_direct_ops = { - .alloc = sw64_direct_alloc_coherent, - .free = sw64_direct_free_coherent, - .map_page = sw64_direct_map_page, - .unmap_page = sw64_direct_unmap_page, - .map_sg = sw64_direct_map_sg, - .unmap_sg = sw64_direct_unmap_sg, - .dma_supported = dma_direct_supported, -}; - -const struct dma_map_ops *dma_ops = &sw64_dma_direct_ops; -EXPORT_SYMBOL(dma_ops); diff --git a/arch/sw_64/kernel/pci_iommu.c b/arch/sw_64/kernel/pci_iommu.c deleted file mode 100644 index 79760c4ac6fcb086bfc55b15733f6a10e978f3e4..0000000000000000000000000000000000000000 --- a/arch/sw_64/kernel/pci_iommu.c +++ /dev/null @@ -1,772 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* iommu.c: Generic sw_64 IOMMU support for 3231 - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "proto.h" -#include "pci_impl.h" -#include "sw_pci_impl.h" - -#define DEBUG_ALLOC 0 -#if DEBUG_ALLOC > 0 -# define DBGA(args...) printk(KERN_DEBUG args) -#else -# define DBGA(args...) -#endif -#if DEBUG_ALLOC > 1 -# define DBGA2(args...) printk(KERN_DEBUG args) -#else -# define DBGA2(args...) -#endif - -unsigned long iommu_cmd; - -static void sw_iommu_create_new(struct pci_controller *hose, unsigned int error_bus_number, - unsigned int error_devfn, unsigned int error_da) -{ - unsigned long dtbr; - u64 *paddr; - u32 ofs; - unsigned long dtbbaseaddr, dtbbasecond; - - sw_read_piu_ior0(hose->node, hose->index, DTBASEADDR, &dtbr); - dtbr += PAGE_OFFSET; - ofs = error_da >> PAGE_SHIFT; - - dtbbaseaddr = dtbr + (error_bus_number << 3); - dtbbasecond = (*(u64 *)(dtbbaseaddr)) & (~(SW_IOMMU_ENTRY_VALID)) & PAGE_MASK; - dtbbasecond += (error_devfn << 3) + PAGE_OFFSET; - - paddr = (u64 *)get_zeroed_page(GFP_DMA); - sw_iommu_map(__pa(paddr), ofs, dtbbasecond, hose, NULL); -} - -irqreturn_t iommu_interrupt(int irq, void *dev) -{ - struct pci_controller *hose = (struct pci_controller *)dev; - unsigned long iommu_status; - unsigned int type, bus_number; - unsigned int devfn, error_da; - - sw_read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS, &iommu_status); - if (!(iommu_status >> 63)) - return IRQ_NONE; - - type = (iommu_status >> 59) & 0x7; - bus_number = (iommu_status >> 45) & 0xff; - devfn = (iommu_status >> 37) & 0xff; - error_da = iommu_status & 0xffffffff; - - if (type == 0x3) { - iommu_status &= ~(1UL << 62); - iommu_status = iommu_status | (1UL << 63); - sw_write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS, iommu_status); - return IRQ_HANDLED; - } - - if (type == 0x2) - sw_iommu_create_new(hose, bus_number, devfn, error_da); - - udelay(100); - sw_write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); - - iommu_status = iommu_status | (3UL << 62); - sw_write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS, iommu_status); - - return IRQ_HANDLED; -} - -struct irqaction iommu_irqaction = { - .handler = iommu_interrupt, - .flags = IRQF_SHARED | IRQF_NO_THREAD, - .name = "sw_iommu", -}; - -void sw_enable_iommu_func(struct pci_controller *hose) -{ - struct irqaction *action; - unsigned int iommu_irq; - unsigned long iommu_conf, iommu_ctrl; - - iommu_irq = hose->int_irq; - action = &iommu_irqaction; - action->dev_id = hose; - request_irq(iommu_irq, action.iommu_interrupt, action.flags, "sw_iommu", action->dev_id); - iommu_ctrl = (1UL << 63) | (0x100UL << 10); - sw_write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_CTRL, iommu_ctrl); - sw_read_piu_ior0(hose->node, hose->index, PIUCONFIG0, &iommu_conf); - iommu_conf = iommu_conf | (0x3 << 7); - sw_write_piu_ior0(hose->node, hose->index, PIUCONFIG0, iommu_conf); - sw_write_piu_ior0(hose->node, hose->index, TIMEOUT_CONFIG, 0xf); - sw_read_piu_ior0(hose->node, hose->index, PIUCONFIG0, &iommu_conf); - pr_info("SW arch configure node %ld hose-%ld iommu_conf = %#lx\n", - hose->node, hose->index, iommu_conf); -} - -struct sw_iommu_dev *pci_to_iommu(struct pci_dev *pdev) -{ - struct sw_iommu *iommu; - struct pci_controller *hose = (struct pci_controller *)pdev->sysdata; - struct sw_iommu_dev *sw_dev; - int busnumber, devid; - - iommu = hose->pci_iommu; - - list_for_each_entry(sw_dev, &iommu->dev_list, list) { - busnumber = sw_dev->dev_id >> 8; - devid = sw_dev->dev_id & 0xff; - if ((busnumber == pdev->bus->number) && (devid == pdev->devfn)) - return sw_dev; - } - - return NULL; -} - -struct sw_iommu_dev *create_sw_iommu_dev(struct pci_dev *dev, unsigned long *pte, struct sw_iommu *iommu) -{ - struct sw_iommu_dev *sw_dev = kzalloc(sizeof(struct sw_iommu_dev), GFP_KERNEL); - - sw_dev->dev_id = (dev->bus->number << 8) + dev->devfn; - sw_dev->io_page_base = pte; - sw_dev->iommu = iommu; - - list_add_tail(&sw_dev->list, &iommu->dev_list); - return sw_dev; -} - -void __sw_pci_iommu_dte_alloc(struct pci_bus *bus, struct sw_iommu *iommu) -{ - struct pci_dev *dev; - struct sw_iommu_dev *iommu_dev; - unsigned long *pte; - u64 *dte; - u64 dtebaseaddr; - u64 dtentry; - u64 dtebaseaddr2, ptentry; - - dtebaseaddr = (unsigned long)iommu->iommu_dtbr + (bus->number << 3); - dte = (u64 *)get_zeroed_page(GFP_KERNEL); - dtentry = (__pa(dte) & PAGE_MASK) | (1UL << 63); - *(u64 *)dtebaseaddr = dtentry; - - list_for_each_entry(dev, &bus->devices, bus_list) { - if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL) { - pte = (unsigned long *)get_zeroed_page(GFP_KERNEL); - dtebaseaddr2 = ((unsigned long)dte & PAGE_MASK) + ((dev->devfn) << 3); - iommu_dev = create_sw_iommu_dev(dev, pte, iommu); - ptentry = (__pa(pte) & PAGE_MASK) | (1UL << 63); - iommu_dev->iommu_bypass = 0; - *(u64 *)dtebaseaddr2 = ptentry; - /* legacy VGA frame buffer has occupied 0xA0000-0xBFFFF memory segment */ - iommu_dev->iommu_area = sw_iommu_area_new(iommu_dev, 0x100000UL); - } else if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { - struct pci_bus *b = dev->subordinate; - - if (b) - __sw_pci_iommu_dte_alloc(b, iommu); - } - } -} - -static int iommu_cpu_suspend(void) -{ - return 0; -} - -static void iommu_cpu_resume(void) -{ -} - -struct syscore_ops iommu_cpu_syscore_ops = { - .suspend = iommu_cpu_suspend, - .resume = iommu_cpu_resume, -}; - -int sw_iommu_init(struct pci_controller *hose) -{ - struct sw_iommu *iommu; - unsigned long base; - unsigned long rc_mask = 0x1; - - rc_mask <<= (8 * hose->node + hose->index); - if (!(iommu_cmd & rc_mask)) - return 0; - sw_write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); - sw_write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); - hose->pci_iommu = kzalloc(sizeof(struct sw_iommu), GFP_KERNEL); - if (!hose->pci_iommu) { - printk("Can't alloc memory for pci_iommu!\n"); - return 0; - } - iommu = hose->pci_iommu; - spin_lock_init(&iommu->dt_lock); - iommu->index = hose->index; - iommu->enabled = true; - iommu->iommu_dtbr = (unsigned long *)get_zeroed_page(GFP_KERNEL); - base = __pa(iommu->iommu_dtbr) & PAGE_MASK; - sw_write_piu_ior0(hose->node, hose->index, DTBASEADDR, base); - INIT_LIST_HEAD(&iommu->dev_list); - __sw_pci_iommu_dte_alloc(hose->bus, iommu); - sw_write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); - sw_write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); - sw_enable_iommu_func(hose); - hose->iommu_enable = true; - - return 0; -} - -struct sw_pci_dev_iommu_area *sw_iommu_area_new(struct sw_iommu_dev *iommu_dev, dma_addr_t base) -{ - struct sw_pci_dev_iommu_area *iommu_area = kzalloc(sizeof(struct sw_pci_dev_iommu_area), GFP_KERNEL); - - if (!iommu_area) { - pr_err("SW arch could not allocate pci iommu dma_area.\n"); - return NULL; - } - - spin_lock_init(&iommu_area->lock); - iommu_area->iommu = iommu_dev->iommu; - iommu_area->dma_base = base; - iommu_area->bitmap = (void *)__get_free_pages(GFP_KERNEL, 3); - if (!iommu_area->bitmap) { - free_pages((unsigned long)iommu_area->bitmap, 3); - pr_err("SW arch could not allocate dma_area->bitmap.\n"); - return NULL; - } - memset(iommu_area->bitmap, 0, 8*PAGE_SIZE); - iommu_area->next_address = 0; - return iommu_area; -} - -/** - * sw_iommu_map - - * @paddr: buffer of the indicated size for PCI DMA - * @dma_ofs: virtual DMA buffer page frame number allocated from pdev private DMA zone - * @dtbaddr: Device Table Base Addr for Level 2 - * @index: PCIe host index - */ -int sw_iommu_map(unsigned long paddr, long dma_ofs, unsigned long dtbaddr, - struct pci_controller *hose, struct pci_dev *pdev) -{ - unsigned long pde, pte; /*pde means Page Table Base Addr for Level 2 pte means Page Table Entry*/ - unsigned long pdebaseaddr; - u64 *ptebasesecond, ptebaseaddr; /*ptebasesecond means Page Table Pointer for Level 2*/ - unsigned long pcache_flush_addr; - - pdebaseaddr = ((dma_ofs >> 10) & SW_IOMMU_LEVEL1_OFFSET) << 3; /* Offset of Page Table Entry for Level 1 */ - pdebaseaddr += ((*(volatile u64 *)dtbaddr) & (~(SW_IOMMU_ENTRY_VALID)) & (PAGE_MASK)) + PAGE_OFFSET; - pte = (paddr & PAGE_MASK) | SW_IOMMU_ENTRY_VALID | SW_IOMMU_GRN | SW_IOMMU_ENABLE; - - /* If pde exists, no need to allocate a new page */ - if ((*(volatile u64 *)pdebaseaddr) & SW_IOMMU_ENTRY_VALID) { - ptebaseaddr = ((*(volatile u64 *)pdebaseaddr) & (~(SW_IOMMU_ENTRY_VALID)) & (PAGE_MASK)) + PAGE_OFFSET; - ptebaseaddr += (dma_ofs & SW_IOMMU_LEVEL2_OFFSET) << 3; - - pcache_flush_addr = __pa(ptebaseaddr) & 0xffffffff80; - - *(volatile u64 *)ptebaseaddr = pte; - mb(); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, pcache_flush_addr); - } else { - ptebasesecond = (u64 *)get_zeroed_page(GFP_ATOMIC); - - if (!ptebasesecond) { - printk("allocating pages fails.\n"); - free_page((unsigned long)ptebasesecond); - return -1; - } - pde = (__pa(ptebasesecond) & PAGE_MASK) | SW_IOMMU_ENTRY_VALID; - - pcache_flush_addr = __pa(pdebaseaddr) & 0xffffffff80; - - *(volatile u64 *)pdebaseaddr = pde; - mb(); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, pcache_flush_addr); - - ptebaseaddr = (unsigned long)ptebasesecond + ((dma_ofs & SW_IOMMU_LEVEL2_OFFSET) << 3); - - pcache_flush_addr = __pa(ptebaseaddr) & 0xffffffff80; - - *(volatile u64 *)ptebaseaddr = pte; - mb(); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, pcache_flush_addr); - } - - return 0; -} - -static unsigned long -sw_iommu_area_alloc(struct sw_pci_dev_iommu_area *area, unsigned int pages, - unsigned long start) -{ - unsigned long next_bit = start >> PAGE_SHIFT; - unsigned long address = -1; - unsigned long boundary_size = ((4UL << 30)) >> PAGE_SHIFT; - unsigned long limit = boundary_size - (1UL << 17) - (area->dma_base >> PAGE_SHIFT); - - address = iommu_area_alloc(area->bitmap, limit, next_bit, pages, 0, boundary_size, 0); - if (address != -1) { - address = address << PAGE_SHIFT; - area->next_address = address + (pages << PAGE_SHIFT); - } - - return address; -} - -static long -sw_iommu_alloc_da(struct sw_pci_dev_iommu_area *area, long n) -{ - unsigned long address; - - address = sw_iommu_area_alloc(area, n, area->next_address); - if (address == -1) { - area->next_address = 0; - address = sw_iommu_area_alloc(area, n, area->next_address); - if (address == -1) - pr_err("SW arch failed to allocate device address.\n"); - } - - return address; -} - -static void sw_iommu_free_da(unsigned long *map, long dma_ofs, long n) -{ - bitmap_clear(map, dma_ofs, n); -} - -static void sw_iommu_unmap(struct pci_dev *pdev, long ofs) -{ - unsigned long dtbbaseaddr, dtbbasecond; /* dtbbaseaddr means Device Table Base Addr for Level 1 */ - /* dtbbasecond means Device Table Base Addr for Level 2 */ - unsigned long pde, pte; /* pde means Page Table Base Addr for Level 2 */ - /* pte means Page Table Entry */ - unsigned long tlb_flush_addr, pcache_flush_addr; - unsigned long addr; - unsigned long pdebaseaddr; /* ptebasefirst means Page Table Pointer for Level 1 */ - unsigned long ptebaseaddr; /* ptebasesecond means Page Table Pointer for Level 2 */ - unsigned long ptebaseaddr_full; /* ptebasesecond means Page Table Pointer for Level 2 */ - unsigned long ptebaseaddr_offset; - struct pci_controller *hose = (struct pci_controller *)pdev->sysdata; - int i; - u64 per_pte; - struct sw_iommu *sw_pci_iommu = hose->pci_iommu; - - addr = (unsigned long)sw_pci_iommu->iommu_dtbr; - dtbbaseaddr = addr + (pdev->bus->number << 3); - - dtbbasecond = (*(volatile u64 *)dtbbaseaddr) & (~(SW_IOMMU_ENTRY_VALID)) & PAGE_MASK; - dtbbasecond += (pdev->devfn << 3) + PAGE_OFFSET; - - pdebaseaddr = ((*(volatile u64 *)dtbbasecond) & (~(SW_IOMMU_ENTRY_VALID)) & (PAGE_MASK)) + PAGE_OFFSET; - pdebaseaddr += ((ofs >> 10) & SW_IOMMU_LEVEL1_OFFSET) << 3; - - pde = *(volatile u64 *)(pdebaseaddr); - ptebaseaddr = (pde & (~(SW_IOMMU_ENTRY_VALID)) & PAGE_MASK) + PAGE_OFFSET; - ptebaseaddr_offset = ptebaseaddr + ((ofs & SW_IOMMU_LEVEL2_OFFSET) << 3); - - tlb_flush_addr = (pdev->bus->number << 8) | pdev->devfn | (ofs << 16); - sw_write_piu_ior0(hose->node, hose->index, PTLB_FLUSHVADDR, tlb_flush_addr); /* TLB FLUSH*/ - - pte = *(volatile u64 *)(ptebaseaddr_offset); - pte &= ~(SW_IOMMU_ENTRY_VALID); /*disable Page Table Entry*/ - pcache_flush_addr = __pa(ptebaseaddr_offset) & 0xffffffff80; - - *(volatile u64 *)(ptebaseaddr_offset) = pte; - mb(); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, pcache_flush_addr); - - ptebaseaddr_full = ptebaseaddr + 0x1ff8; - if (ptebaseaddr_offset == ptebaseaddr_full) { - for (i = 0; i < 1024; i++) { - per_pte = *(volatile u64 *)(ptebaseaddr + i * 8); - if (per_pte & SW_IOMMU_ENTRY_VALID) - break; - } - if (i == 1024) { - free_page(ptebaseaddr); - pde &= ~(SW_IOMMU_ENTRY_VALID); - - pcache_flush_addr = __pa(pdebaseaddr) & 0xffffffff80; - - *(volatile u64 *)(pdebaseaddr) = pde; - mb(); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, pcache_flush_addr); - } - } -} - -static dma_addr_t __sw_map_single(struct pci_dev *pdev, unsigned long paddr, - struct sw_pci_dev_iommu_area *iommu_area, size_t size) -{ - long npages, dma_ofs, i, ofs; - unsigned long dtbbaseaddr; /* dtbbaseaddr means Device Table Base Addr for Level 1 */ - unsigned long dtbbasecond; /* dtbbasecond means Device Table Base Addr for Level 2 */ - unsigned long addr; - dma_addr_t ret = -1; - struct pci_controller *hose = pdev->sysdata; - struct sw_iommu *sw_pci_iommu = hose->pci_iommu; - unsigned long flags; - - if (hose == NULL) { - pr_err("%s: hose does not exist!\n", __func__); - return 0; - } - - addr = (unsigned long)sw_pci_iommu->iommu_dtbr; - dtbbaseaddr = addr + (pdev->bus->number << 3); - - dtbbasecond = (*(volatile u64 *)dtbbaseaddr) & ~(SW_IOMMU_ENTRY_VALID) & PAGE_MASK; - dtbbasecond += (pdev->devfn << 3) + PAGE_OFFSET; - npages = iommu_num_pages(paddr, size, PAGE_SIZE); - - if (hose->iommu_enable) { - spin_lock_irqsave(&iommu_area->lock, flags); - - dma_ofs = sw_iommu_alloc_da(iommu_area, npages); - if (dma_ofs == -1) { - pr_warn("%s %s failed: could not allocate dma page tables\n", - pci_name(pdev), __func__); - spin_unlock_irqrestore(&iommu_area->lock, flags); - return 0; - } - - ret = iommu_area->dma_base + dma_ofs; - - for (i = 0; i < npages; ++i, paddr += PAGE_SIZE) { - ofs = (ret >> PAGE_SHIFT) + i; - sw_iommu_map(paddr, ofs, dtbbasecond, hose, pdev); - } - - spin_unlock_irqrestore(&iommu_area->lock, flags); - - ret += paddr & ~PAGE_MASK; - } - - return ret; -} - -/* - * Map a single buffer of the indicated size for PCI DMA in streaming - * mode. The 32-bit PCI bus mastering address to use is returned. - * Once the device is given the dma address, the device owns this memory - * until either pci_unmap_single or pci_dma_sync_single is performed. - */ - -static dma_addr_t -pci_iommu_map_single(struct pci_dev *pdev, void *cpu_addr, size_t size) -{ - struct pci_controller *hose = pdev->sysdata; - unsigned long paddr; - - if (hose == NULL) { - pr_err("%s: hose does not exist!\n", __func__); - return 0; - } - - if (!hose->iommu_enable) { - unsigned long dma_offset; - - sw_read_piu_ior0(hose->node, hose->index, EPDMABAR, &dma_offset); - paddr = __pa(cpu_addr) + dma_offset; - } else { - struct sw_pci_dev_iommu_area *iommu_area; - struct sw_iommu_dev *sw_dev = pci_to_iommu(pdev); - - paddr = __pa(cpu_addr); - iommu_area = sw_dev->iommu_area; - if (!iommu_area) { - pr_err("SW arch get iommu_area error!\n"); - return 0; - } - - paddr = __sw_map_single(pdev, paddr, iommu_area, size); - } - - return paddr; -} - -static dma_addr_t sw_iommu_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - struct pci_dev *pdev = sw_gendev_to_pci(dev); - - if (dir == PCI_DMA_NONE) - BUG(); - - return pci_iommu_map_single(pdev, (char *)page_address(page) + offset, size); -} - -/* - * Unmap a single streaming mode DMA translation. The DMA_ADDR and - * SIZE must match what was provided for in a previous pci_map_single - * call. All other usages are undefined. After this call, reads by - * the cpu to the buffer are guaranteed to see whatever the device - * wrote there. - */ - -static void sw_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - struct pci_dev *pdev = sw_gendev_to_pci(dev); - struct pci_controller *hose = pdev->sysdata; - struct sw_iommu_dev *sw_dev = pci_to_iommu(pdev); - struct sw_pci_dev_iommu_area *iommu_area; - long dma_ofs, npages, ofs; - unsigned long flags; - int i; - - if (hose == NULL) { - pr_err("%s: hose does not exist!\n", __func__); - return 0; - } - - if (!hose->iommu_enable) - return; - - iommu_area = sw_dev->iommu_area; - dma_ofs = (dma_addr - iommu_area->dma_base) >> PAGE_SHIFT; - npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); - - spin_lock_irqsave(&iommu_area->lock, flags); - - for (i = 0; i < npages; ++i) { - ofs = (dma_addr >> PAGE_SHIFT) + i; - sw_iommu_unmap(pdev, ofs); - } - - sw_iommu_free_da(iommu_area->bitmap, dma_ofs, npages); - spin_unlock_irqrestore(&iommu_area->lock, flags); -} - -/* - * Allocate and map kernel buffer using consistent mode DMA for PCI - * device. Returns non-NULL cpu-view pointer to the buffer if - * successful and sets *DMA_ADDRP to the pci side dma address as well, - * else DMA_ADDRP is undefined. - */ -static void *sw_iommu_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_addrp, gfp_t gfp, - unsigned long attrs) -{ - struct pci_dev *pdev = sw_gendev_to_pci(dev); - void *cpu_addr; - long order = get_order(size); - - gfp &= ~GFP_DMA; - -try_again: - cpu_addr = (void *)__get_free_pages(gfp | __GFP_ZERO, order); - if (!cpu_addr) { - pr_info("pci_alloc_consistent: get_free_pages failed from %ps\n", - __builtin_return_address(0)); - /* ??? Really atomic allocation? Otherwise we could play - * with vmalloc and sg if we can't find contiguous memory. - */ - return NULL; - } - memset(cpu_addr, 0, size); - - *dma_addrp = pci_iommu_map_single(pdev, cpu_addr, size); - if (*dma_addrp == 0) { - free_pages((unsigned long)cpu_addr, order); - if (gfp & GFP_DMA) - return NULL; - /* The address doesn't fit required mask and we - * do not have iommu. Try again with GFP_DMA. - */ - gfp |= GFP_DMA; - goto try_again; - } - - DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %ps\n", - size, cpu_addr, *dma_addrp, __builtin_return_address(0)); - - return cpu_addr; -} - -/* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must - * be values that were returned from pci_alloc_consistent. SIZE must - * be the same as what as passed into pci_alloc_consistent. - * References to the memory and mappings associated with CPU_ADDR or - * DMA_ADDR past this call are illegal. - */ - -static void sw_iommu_free_coherent(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_addr, - unsigned long attrs) -{ - struct pci_dev *pdev = sw_gendev_to_pci(dev); - - pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); - free_pages((unsigned long)cpu_addr, get_order(size)); - - DBGA2("pci_free_consistent: [%llx,%zx] from %ps\n", - dma_addr, size, __builtin_return_address(0)); -} - -#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) -#define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG)) - -static int sw_iommu_map_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, - unsigned long attrs) -{ - int i; - struct scatterlist *sg; - struct pci_dev *pdev = sw_gendev_to_pci(dev); - int out_nents = 0; - - if (dir == PCI_DMA_NONE) - BUG(); - - for_each_sg(sgl, sg, nents, i) { - BUG_ON(!sg_page(sg)); - - sg_dma_address(sg) = pci_iommu_map_single(pdev, SG_ENT_VIRT_ADDRESS(sg), sg->length); - if (sg_dma_address(sg) == 0) - goto error; - sg_dma_len(sg) = sg->length; - out_nents++; - } - - return nents; - -error: - pr_warn("pci_map_sg failed: could not allocate dma page tables\n"); - - /* Some allocation failed while mapping the scatterlist - * entries. Unmap them now. - */ - if (out_nents) - pci_unmap_sg(pdev, sgl, out_nents, dir); - return 0; -} - -/* - * Unmap a set of streaming mode DMA translations. Again, cpu read - * rules concerning calls here are the same as for pci_unmap_single() - * above. - */ -static void sw_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, - unsigned long attrs) -{ - struct pci_dev *pdev = sw_gendev_to_pci(dev); - struct pci_controller *hose = pdev->sysdata; - struct scatterlist *sg; - int i, j; - dma_addr_t dma_addr; - struct sw_pci_dev_iommu_area *iommu_area; - struct sw_iommu_dev *sw_dev = pci_to_iommu(pdev); - long dma_ofs, npages, ofs, size; - unsigned long flags; - - if (hose == NULL) { - pr_err("%s: hose does not exist!\n", __func__); - return 0; - } - - if (!hose->iommu_enable) - return; - - iommu_area = sw_dev->iommu_area; - for_each_sg(sgl, sg, nents, j) { - BUG_ON(!sg_page(sg)); - dma_addr = sg->dma_address; - size = sg->dma_length; - if (!size) - break; - npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); - dma_ofs = (dma_addr - iommu_area->dma_base) >> PAGE_SHIFT; - - spin_lock_irqsave(&iommu_area->lock, flags); - for (i = 0; i < npages; ++i) { - ofs = (dma_addr >> PAGE_SHIFT) + i; - sw_iommu_unmap(pdev, ofs); - } - - sw_iommu_free_da(iommu_area->bitmap, dma_ofs, npages); - - spin_unlock_irqrestore(&iommu_area->lock, flags); - } -} - -/* Return whether the given PCI device DMA address mask can be - * supported properly. - */ - -static int sw_iommu_supported(struct device *dev, u64 mask) -{ - /* As last resort try ZONE_DMA. */ - if (MAX_DMA_ADDRESS - PAGE_OFFSET - 1 <= mask) - return 1; - - return 0; -} - -static int sw_iommu_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - return dma_addr == 0; -} - -static int iommu_get_option(char **str, unsigned long *pint) -{ - char *cur = *str; - - if (!cur || !(*cur)) - return 0; - *pint = kstrtol(cur, str, 16); - - return 1; -} - -static int __init iommu_enable_setup(char *s) -{ - unsigned long rc_bitmap = 0; - - iommu_get_option(&s, &rc_bitmap); - iommu_cmd = rc_bitmap; - - return 1; -} -__setup("iommu_enable=", iommu_enable_setup); - -const struct dma_map_ops sw_iommu_dma_ops = { - .alloc = sw_iommu_alloc_coherent, - .free = sw_iommu_free_coherent, - .map_page = sw_iommu_map_page, - .unmap_page = sw_iommu_unmap_page, - .map_sg = sw_iommu_map_sg, - .unmap_sg = sw_iommu_unmap_sg, - .mapping_error = sw_iommu_mapping_error, - .dma_supported = sw_iommu_supported, -}; - -const struct dma_map_ops *dma_ops = &sw_iommu_dma_ops; -EXPORT_SYMBOL(dma_ops); diff --git a/arch/sw_64/kernel/perf_event.c b/arch/sw_64/kernel/perf_event.c index a30817c47e66ea33a974d5b319b2086ab2b4faca..63922113de64ef40552d977d12d229b1b3958082 100644 --- a/arch/sw_64/kernel/perf_event.c +++ b/arch/sw_64/kernel/perf_event.c @@ -91,12 +91,12 @@ static const struct sw64_pmu_t *sw64_pmu; /* Mapping of the hw event types to the perf tool interface */ static const struct sw64_perf_event core3_hw_event_map[] = { - [PERF_COUNT_HW_CPU_CYCLES] = {PERFMON_PC0, PC0_CPU_CYCLES}, - [PERF_COUNT_HW_INSTRUCTIONS] = {PERFMON_PC0, PC0_INSTRUCTIONS}, - [PERF_COUNT_HW_CACHE_REFERENCES] = {PERFMON_PC0, PC0_SCACHE_REFERENCES}, - [PERF_COUNT_HW_CACHE_MISSES] = {PERFMON_PC1, PC1_SCACHE_MISSES}, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = {PERFMON_PC0, PC0_BRANCH_INSTRUCTIONS}, - [PERF_COUNT_HW_BRANCH_MISSES] = {PERFMON_PC1, PC1_BRANCH_MISSES}, + [PERF_COUNT_HW_CPU_CYCLES] = {PMC_PC0, PC0_CPU_CYCLES}, + [PERF_COUNT_HW_INSTRUCTIONS] = {PMC_PC0, PC0_INSTRUCTIONS}, + [PERF_COUNT_HW_CACHE_REFERENCES] = {PMC_PC0, PC0_SCACHE_REFERENCES}, + [PERF_COUNT_HW_CACHE_MISSES] = {PMC_PC1, PC1_SCACHE_MISSES}, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = {PMC_PC0, PC0_BRANCH_INSTRUCTIONS}, + [PERF_COUNT_HW_BRANCH_MISSES] = {PMC_PC1, PC1_BRANCH_MISSES}, }; /* Mapping of the hw cache event types to the perf tool interface */ @@ -107,8 +107,8 @@ static const struct sw64_perf_event core3_cache_event_map [PERF_COUNT_HW_CACHE_RESULT_MAX] = { [C(L1D)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = {PERFMON_PC0, PC0_DCACHE_READ}, - [C(RESULT_MISS)] = {PERFMON_PC1, PC1_DCACHE_MISSES} + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_DCACHE_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_DCACHE_MISSES} }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = SW64_OP_UNSUP, @@ -121,8 +121,8 @@ static const struct sw64_perf_event core3_cache_event_map }, [C(L1I)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = {PERFMON_PC0, PC0_ICACHE_READ}, - [C(RESULT_MISS)] = {PERFMON_PC1, PC1_ICACHE_READ_MISSES}, + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_ICACHE_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_ICACHE_READ_MISSES}, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = SW64_OP_UNSUP, @@ -149,8 +149,8 @@ static const struct sw64_perf_event core3_cache_event_map }, [C(DTLB)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = {PERFMON_PC0, PC0_DTB_READ}, - [C(RESULT_MISS)] = {PERFMON_PC1, PC1_DTB_SINGLE_MISSES}, + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_DTB_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_DTB_SINGLE_MISSES}, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = SW64_OP_UNSUP, @@ -163,8 +163,8 @@ static const struct sw64_perf_event core3_cache_event_map }, [C(ITLB)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = {PERFMON_PC0, PC0_ITB_READ}, - [C(RESULT_MISS)] = {PERFMON_PC1, PC1_ITB_MISSES}, + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_ITB_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_ITB_MISSES}, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = SW64_OP_UNSUP, @@ -267,21 +267,12 @@ static const struct sw64_pmu_t core3_pmu = { */ static void sw64_write_pmc(int idx, unsigned long val) { - if (idx == PERFMON_PC0) - wrperfmon(PERFMON_CMD_WRITE_PC0, val); - else - wrperfmon(PERFMON_CMD_WRITE_PC1, val); + wrperfmon(PMC_CMD_WRITE_BASE + idx, val); } static unsigned long sw64_read_pmc(int idx) { - unsigned long val; - - if (idx == PERFMON_PC0) - val = wrperfmon(PERFMON_CMD_READ, PERFMON_READ_PC0); - else - val = wrperfmon(PERFMON_CMD_READ, PERFMON_READ_PC1); - return val; + return wrperfmon(PMC_CMD_READ, idx); } /* Set a new period to sample over */ @@ -386,14 +377,9 @@ static void sw64_pmu_start(struct perf_event *event, int flags) hwc->state = 0; /* counting in selected modes, for both counters */ - wrperfmon(PERFMON_CMD_PM, hwc->config_base); - if (hwc->idx == PERFMON_PC0) { - wrperfmon(PERFMON_CMD_EVENT_PC0, hwc->event_base); - wrperfmon(PERFMON_CMD_ENABLE, PERFMON_ENABLE_ARGS_PC0); - } else { - wrperfmon(PERFMON_CMD_EVENT_PC1, hwc->event_base); - wrperfmon(PERFMON_CMD_ENABLE, PERFMON_ENABLE_ARGS_PC1); - } + wrperfmon(PMC_CMD_PM, hwc->config_base); + wrperfmon(PMC_CMD_EVENT_BASE + hwc->idx, hwc->event_base); + wrperfmon(PMC_CMD_ENABLE, PMC_ENABLE_BASE + hwc->idx); } /* @@ -404,9 +390,7 @@ static void sw64_pmu_stop(struct perf_event *event, int flags) struct hw_perf_event *hwc = &event->hw; if (!(hwc->state & PERF_HES_STOPPED)) { - wrperfmon(PERFMON_CMD_DISABLE, hwc->idx == 0 ? - PERFMON_DISABLE_ARGS_PC0 : - PERFMON_DISABLE_ARGS_PC1); + wrperfmon(PMC_CMD_DISABLE, PMC_DISABLE_BASE + hwc->idx); hwc->state |= PERF_HES_STOPPED; barrier(); } @@ -610,28 +594,25 @@ void perf_event_print_debug(void) cpu = smp_processor_id(); - pcr0 = wrperfmon(PERFMON_CMD_READ, PERFMON_READ_PC0); - pcr1 = wrperfmon(PERFMON_CMD_READ, PERFMON_READ_PC1); + pcr0 = wrperfmon(PMC_CMD_READ, PMC_PC0); + pcr1 = wrperfmon(PMC_CMD_READ, PMC_PC1); pr_info("CPU#%d: PCTR0[%lx] PCTR1[%lx]\n", cpu, pcr0, pcr1); local_irq_restore(flags); } -static void sw64_perf_event_irq_handler(unsigned long perfmon_num, +static void sw64_perf_event_irq_handler(unsigned long idx, struct pt_regs *regs) { struct cpu_hw_events *cpuc; struct perf_sample_data data; struct perf_event *event; struct hw_perf_event *hwc; - int idx; __this_cpu_inc(irq_pmi_count); cpuc = this_cpu_ptr(&cpu_hw_events); - idx = perfmon_num; - event = cpuc->event[idx]; if (unlikely(!event)) { @@ -683,7 +664,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, perf_callchain_store(entry, regs->pc); - fp = (unsigned long __user *)regs->r15; + fp = (unsigned long __user *)regs->regs[15]; while (entry->nr < entry->max_stack && (unsigned long)fp < current->mm->start_stack) { if (!access_ok(fp, sizeof(frame))) @@ -705,7 +686,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - unsigned long usp = current_user_stack_pointer(); + unsigned long usp = rdusp(); unsigned long user_addr; int err; diff --git a/arch/sw_64/kernel/perf_regs.c b/arch/sw_64/kernel/perf_regs.c index 8fc3597d9e41209dc4ba01da3babeebaf00f7e90..b036f213936bc6d79214c9b7bdf1ab9a82a40b69 100644 --- a/arch/sw_64/kernel/perf_regs.c +++ b/arch/sw_64/kernel/perf_regs.c @@ -8,24 +8,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) if (WARN_ON_ONCE((u32)idx >= PERF_REG_SW64_MAX)) return 0; - switch (idx) { - case PERF_REG_SW64_R16: - return regs->r16; - case PERF_REG_SW64_R17: - return regs->r17; - case PERF_REG_SW64_R18: - return regs->r18; - case PERF_REG_SW64_R19 ... PERF_REG_SW64_R28: - return ((unsigned long *)regs)[idx - 3]; - case PERF_REG_SW64_GP: - return regs->gp; - case PERF_REG_SW64_SP: - return (user_mode(regs) ? rdusp() : (u64)(regs + 1)); - case PERF_REG_SW64_PC: - return regs->pc; - default: - return ((unsigned long *)regs)[idx]; - } + return ((unsigned long *)regs)[idx]; } #define REG_RESERVED (~((1ULL << PERF_REG_SW64_MAX) - 1)) diff --git a/arch/sw_64/kernel/platform.c b/arch/sw_64/kernel/platform.c deleted file mode 100644 index f4c880acaa40da8c366675b1b31047d980e3a932..0000000000000000000000000000000000000000 --- a/arch/sw_64/kernel/platform.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/arch/sw/kernel/setup.c - * - * Copyright (C) 1995 Linus Torvalds - */ - -#include - -static struct platform_device sw64_cpufreq_device = { - .name = "sw64_cpufreq", - .id = -1, -}; - -static int __init sw64_cpufreq_init(void) -{ - return platform_device_register(&sw64_cpufreq_device); -} - -arch_initcall(sw64_cpufreq_init); diff --git a/arch/sw_64/kernel/pm.c b/arch/sw_64/kernel/pm.c new file mode 100644 index 0000000000000000000000000000000000000000..f0a35e5d0486167340b44f3bac1c80104f25649e --- /dev/null +++ b/arch/sw_64/kernel/pm.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include + +struct syscore_ops io_syscore_ops; + +static int __init sw64_pm_init(void) +{ +#ifdef CONFIG_SUSPEND + suspend_set_ops(&native_suspend_ops); +#endif + register_syscore_ops(&io_syscore_ops); + + return 0; +} +device_initcall(sw64_pm_init); diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c index 4ffa724897c3d0bfbdc36ba8cd9d5d57e76a1fae..ad54129a48e539ef1c862c40f469981310094fa8 100644 --- a/arch/sw_64/kernel/process.c +++ b/arch/sw_64/kernel/process.c @@ -22,7 +22,7 @@ start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) { regs->pc = pc; regs->ps = 8; - wrusp(sp); + regs->regs[30] = sp; } EXPORT_SYMBOL(start_thread); @@ -34,7 +34,7 @@ flush_thread(void) * with respect to the FPU. This is all exceptions disabled. */ current_thread_info()->ieee_state = 0; - wrfpcr(FPCR_DYN_NORMAL | ieee_swcr_to_fpcr(0)); + wrfpcr(FPCR_INIT | ieee_swcr_to_fpcr(0)); /* Clean slate for TLS. */ current_thread_info()->pcb.tp = 0; @@ -80,7 +80,6 @@ copy_thread(unsigned long clone_flags, unsigned long usp, p->thread.ra = (unsigned long) ret_from_kernel_thread; p->thread.s[0] = usp; /* function */ p->thread.s[1] = kthread_arg; - childti->pcb.usp = 0; return 0; } @@ -92,14 +91,14 @@ copy_thread(unsigned long clone_flags, unsigned long usp, * application calling fork. */ if (clone_flags & CLONE_SETTLS) - childti->pcb.tp = regs->r20; + childti->pcb.tp = regs->regs[20]; else - regs->r20 = 0; - if (usp) - childti->pcb.usp = usp; + regs->regs[20] = 0; *childregs = *regs; - childregs->r0 = 0; - childregs->r19 = 0; + if (usp) + childregs->regs[30] = usp; + childregs->regs[0] = 0; + childregs->regs[19] = 0; p->thread.ra = (unsigned long) ret_from_fork; return 0; } @@ -115,9 +114,8 @@ void sw64_elf_core_copy_regs(elf_greg_t *dest, struct pt_regs *regs) ti = (void *)((__u64)regs & ~(THREAD_SIZE - 1)); - for (i = 0; i < 30; i++) - dest[i] = *(__u64 *)((void *)regs + regoffsets[i]); - dest[30] = ti == current_thread_info() ? rdusp() : ti->pcb.usp; + for (i = 0; i < 31; i++) + dest[i] = regs->regs[i]; dest[31] = regs->pc; dest[32] = ti->pcb.tp; } diff --git a/arch/sw_64/kernel/proto.h b/arch/sw_64/kernel/proto.h index 7b2564f47a9df4612d4cf53174b9cf8f01afca81..d7222334d1b99ffab3e0345bebcb1ec15177290c 100644 --- a/arch/sw_64/kernel/proto.h +++ b/arch/sw_64/kernel/proto.h @@ -15,12 +15,4 @@ extern int ptrace_cancel_bpt(struct task_struct *child); extern void show_regs(struct pt_regs *regs); extern void die(char *str, struct pt_regs *regs, long err); -/* timer.c */ -extern void setup_timer(void); - -extern void __init setup_sched_clock(void); -#ifdef CONFIG_GENERIC_SCHED_CLOCK -extern void __init sw64_sched_clock_init(void); -#endif - #endif /* _SW64_KERNEL_PROTO_H */ diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index 2ca8d5804537242fb8a50aad3c391a9146cdd3d4..1911d5bd058b8bf605729709f1bad400c81e26c6 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -15,6 +15,7 @@ #include #include "proto.h" +#include #define CREATE_TRACE_POINTS #include @@ -46,28 +47,21 @@ * zero have no stack-slot and need to be treated specially (see * get_reg/put_reg below). */ -#define R(x) ((size_t) &((struct pt_regs *)0)->x) - -short regoffsets[32] = { - R(r0), R(r1), R(r2), R(r3), R(r4), R(r5), R(r6), R(r7), R(r8), - R(r9), R(r10), R(r11), R(r12), R(r13), R(r14), R(r15), - R(r16), R(r17), R(r18), - R(r19), R(r20), R(r21), R(r22), R(r23), R(r24), R(r25), R(r26), - R(r27), R(r28), R(gp), 0, 0 -}; - -#undef R - #define PCB_OFF(var) offsetof(struct pcb_struct, var) static int pcboff[] = { - [USP] = PCB_OFF(usp), - [TP] = PCB_OFF(tp), - [DA_MATCH] = PCB_OFF(da_match), - [DA_MASK] = PCB_OFF(da_mask), - [DV_MATCH] = PCB_OFF(dv_match), - [DV_MASK] = PCB_OFF(dv_mask), - [DC_CTL] = PCB_OFF(dc_ctl) + [PT_TP] = PCB_OFF(tp), + [PT_DA_MATCH] = PCB_OFF(da_match), + [PT_DA_MASK] = PCB_OFF(da_mask), + [PT_DV_MATCH] = PCB_OFF(dv_match), + [PT_DV_MASK] = PCB_OFF(dv_mask), + [PT_DC_CTL] = PCB_OFF(dc_ctl), + [PT_MATCH_CTL] = PCB_OFF(match_ctl), + [PT_IA_MATCH] = PCB_OFF(ia_match), + [PT_IA_MASK] = PCB_OFF(ia_mask), + [PT_IV_MATCH] = PCB_OFF(iv_match), + [PT_IDA_MATCH] = PCB_OFF(ida_match), + [PT_IDA_MASK] = PCB_OFF(ida_mask) }; static unsigned long zero; @@ -83,39 +77,43 @@ get_reg_addr(struct task_struct *task, unsigned long regno) int fno, vno; switch (regno) { - case USP: - case UNIQUE: - case DA_MATCH: - case DA_MASK: - case DV_MATCH: - case DV_MASK: - case DC_CTL: + case PT_UNIQUE: + case PT_DA_MATCH: + case PT_DA_MASK: + case PT_DV_MATCH: + case PT_DV_MASK: + case PT_MATCH_CTL: + case PT_IA_MATCH: + case PT_IA_MASK: + case PT_IV_MATCH: + case PT_IDA_MATCH: + case PT_IDA_MASK: addr = (void *)task_thread_info(task) + pcboff[regno]; break; - case REG_BASE ... REG_END: - addr = (void *)task_pt_regs(task) + regoffsets[regno]; + case PT_REG_BASE ... PT_REG_END: + addr = &task_pt_regs(task)->regs[regno]; break; - case FPREG_BASE ... FPREG_END: - fno = regno - FPREG_BASE; + case PT_FPREG_BASE ... PT_FPREG_END: + fno = regno - PT_FPREG_BASE; addr = &task->thread.fpstate.fp[fno].v[0]; break; - case VECREG_BASE ... VECREG_END: + case PT_VECREG_BASE ... PT_VECREG_END: /* * return addr for zero value if we catch vectors of f31 * v0 and v3 of f31 are not in this range so ignore them */ - if (regno == F31_V1 || regno == F31_V2) { + if (regno == PT_F31_V1 || regno == PT_F31_V2) { addr = &zero; break; } - fno = (regno - VECREG_BASE) & 0x1f; - vno = 1 + ((regno - VECREG_BASE) >> 5); + fno = (regno - PT_VECREG_BASE) & 0x1f; + vno = 1 + ((regno - PT_VECREG_BASE) >> 5); addr = &task->thread.fpstate.fp[fno].v[vno]; break; - case FPCR: + case PT_FPCR: addr = &task->thread.fpstate.fpcr; break; - case PC: + case PT_PC: addr = (void *)task_pt_regs(task) + PT_REGS_PC; break; default: @@ -170,7 +168,7 @@ ptrace_set_bpt(struct task_struct *child) unsigned int insn, op_code; unsigned long pc; - pc = get_reg(child, PC); + pc = get_reg(child, PT_PC); res = read_int(child, pc, (int *)&insn); if (res < 0) return res; @@ -263,21 +261,7 @@ static int gpr_get(struct task_struct *target, const struct user_regset *regset, struct membuf to) { - struct pt_regs *regs; - struct user_pt_regs uregs; - int i, ret; - - regs = task_pt_regs(target); - for (i = 0; i < 30; i++) - uregs.regs[i] = *(__u64 *)((void *)regs + regoffsets[i]); - - uregs.regs[30] = task_thread_info(target)->pcb.usp; - uregs.pc = regs->pc; - uregs.pstate = regs->ps; - - ret = membuf_write(&to, &uregs, sizeof(uregs)); - - return ret; + return membuf_write(&to, task_pt_regs(target), sizeof(struct user_pt_regs)); } static int gpr_set(struct task_struct *target, @@ -285,24 +269,8 @@ static int gpr_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct pt_regs *regs; - struct user_pt_regs uregs; - int i, ret; - - ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, - &uregs, 0, sizeof(uregs)); - if (ret) - return ret; - - regs = task_pt_regs(target); - for (i = 0; i < 30; i++) - *(__u64 *)((void *)regs + regoffsets[i]) = uregs.regs[i]; - - task_thread_info(target)->pcb.usp = uregs.regs[30]; - regs->pc = uregs.pc; - regs->ps = uregs.pstate; - - return 0; + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, + task_pt_regs(target), 0, sizeof(struct user_pt_regs)); } static int fpr_get(struct task_struct *target, @@ -406,19 +374,19 @@ asmlinkage unsigned long syscall_trace_enter(void) struct pt_regs *regs = current_pt_regs(); if (test_thread_flag(TIF_SYSCALL_TRACE) && - tracehook_report_syscall_entry(current_pt_regs())) - ret = -1UL; + tracehook_report_syscall_entry(regs)) + return NO_SYSCALL; #ifdef CONFIG_SECCOMP /* Do seccomp after ptrace, to catch any tracer changes. */ if (secure_computing() == -1) - return -1; + return NO_SYSCALL; #endif if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) - trace_sys_enter(regs, regs->r0); - audit_syscall_entry(regs->r0, regs->r16, regs->r17, regs->r18, regs->r19); - return ret ?: current_pt_regs()->r0; + trace_sys_enter(regs, regs->regs[0]); + audit_syscall_entry(regs->regs[0], regs->regs[16], regs->regs[17], regs->regs[18], regs->regs[19]); + return ret ?: regs->regs[0]; } asmlinkage void @@ -426,13 +394,14 @@ syscall_trace_leave(void) { struct pt_regs *regs = current_pt_regs(); - audit_syscall_exit(current_pt_regs()); + audit_syscall_exit(regs); if (test_thread_flag(TIF_SYSCALL_TRACE)) - tracehook_report_syscall_exit(current_pt_regs(), 0); + tracehook_report_syscall_exit(regs, 0); if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) trace_sys_exit(regs, regs_return_value(regs)); } +#ifdef CONFIG_SUBARCH_C3B static long rwcsr(int rw, unsigned long csr, unsigned long value) { register unsigned long __r0 __asm__("$0"); @@ -522,7 +491,7 @@ int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_r task_thread_info(current)->pcb.dv_match = 0; task_thread_info(current)->pcb.dc_ctl = 0; printk("do_page_fault: want to send SIGTRAP, pid = %d\n", current->pid); - force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void *) address, 0); + force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void *) address); return 1; case MMCSR__IA_MATCH: @@ -556,11 +525,250 @@ void restore_da_match_after_sched(void) } } +#elif defined(CONFIG_SUBARCH_C4) +int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_regs *regs) +{ + kernel_siginfo_t info; + unsigned long match_ctl, ia_match; + sigval_t sw64_value; + + printk("%s: pid %d, name = %s, cause = %#lx, mmcsr = %#lx, address = %#lx, pc %#lx\n", + __func__, current->pid, current->comm, cause, mmcsr, address, regs->pc); + + switch (mmcsr) { + case MMCSR__DA_MATCH: + case MMCSR__DV_MATCH: + case MMCSR__DAV_MATCH: + case MMCSR__IA_MATCH: + case MMCSR__IDA_MATCH: + case MMCSR__IV_MATCH: + show_regs(regs); + + if (!(current->ptrace & PT_PTRACED)) { + printk(" pid %d %s not be ptraced, return\n", current->pid, current->comm); + if (mmcsr == MMCSR__DA_MATCH) { + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x3UL << DA_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + task_thread_info(current)->pcb.match_ctl &= ~0x1; + task_thread_info(current)->pcb.da_match = 0; + } + if (mmcsr == MMCSR__DV_MATCH) { + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x1UL << DV_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 1); + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__DAV_MATCH) { + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 | (0x1 << 1) | (0x1 << 2)); + task_thread_info(current)->pcb.da_match = 0; + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__IA_MATCH) { + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IA_MATCH_EN_S) | (0x7ffffffffffffUL << 2)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 3); + task_thread_info(current)->pcb.ia_match = 0; + } + if (mmcsr == MMCSR__IV_MATCH) { + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + write_csr(0, CSR_IV_MATCH); // clear iv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 4); + task_thread_info(current)->pcb.ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + task_thread_info(current)->pcb.iv_match = 0; + } + if (mmcsr == MMCSR__IDA_MATCH) { + write_csr(0, CSR_IDA_MATCH); // clear ida_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 5); + task_thread_info(current)->pcb.ida_match = 0; + } + return 1; + } + + info.si_signo = SIGTRAP; + info.si_addr = (void *) address; + sw64_value.sival_ptr = (void *)(regs->pc); + info.si_value = sw64_value; + info.si_code = TRAP_HWBKPT; + + if (mmcsr == MMCSR__DA_MATCH) { + info.si_errno = 1; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x3UL << DA_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + task_thread_info(current)->pcb.match_ctl &= ~0x1; + task_thread_info(current)->pcb.da_match = 0; + } + if (mmcsr == MMCSR__DV_MATCH) { + info.si_errno = 2; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x1UL << DV_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 1); + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__DAV_MATCH) { + info.si_errno = 3; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 | (0x1 << 1) | (0x1 << 2)); + task_thread_info(current)->pcb.da_match = 0; + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__IA_MATCH) { + info.si_errno = 4; + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IA_MATCH_EN_S) | (0x7ffffffffffffUL << 2)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 3); + task_thread_info(current)->pcb.ia_match = 0; + } + if (mmcsr == MMCSR__IV_MATCH) { + info.si_errno = 5; + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + write_csr(0, CSR_IV_MATCH); // clear iv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 4); + task_thread_info(current)->pcb.ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + task_thread_info(current)->pcb.iv_match = 0; + } + if (mmcsr == MMCSR__IDA_MATCH) { + info.si_errno = 6; + write_csr(0, CSR_IDA_MATCH); // clear ida_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 5); + task_thread_info(current)->pcb.ida_match = 0; + } + printk("do_page_fault: want to send SIGTRAP, pid = %d\n", current->pid); + force_sig_info(&info); + return 1; + } + + return 0; +} + +/* + *pcb->match_ctl: + * [0] DA_MATCH + * [1] DV_MATCH + * [2] DAV_MATCH + * [3] IA_MATCH + * [4] IV_MATCH + * [5] IDA_MATCH + * [8:9] match_ctl_mode + * + */ +#define DA_MATCH 0x1 +#define DV_MATCH 0x2 +#define DAV_MATCH 0x4 +#define IA_MATCH 0x8 +#define IV_MATCH 0x10 +#define IDA_MATCH 0x20 + +void restore_da_match_after_sched(void) +{ + unsigned long match_ctl_mode; + unsigned long match_ctl; + struct pcb_struct *pcb = &task_thread_info(current)->pcb; + unsigned long vpn, upn; + + if (!pcb->match_ctl) + return; + pr_info("Restroe MATCH status, pid: %d\n", current->pid); + + if (pcb->match_ctl & DA_MATCH) { + write_csr(pcb->da_match, CSR_DA_MATCH); + write_csr(pcb->da_mask, CSR_DA_MASK); + match_ctl_mode = (pcb->match_ctl >> 8) & 0x3; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + match_ctl |= (match_ctl_mode << DA_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH); + write_csr(match_ctl, CSR_DC_CTLP); + pr_info("da_match:%#lx da_mask:%#lx match_ctl:%#lx\n", pcb->da_match, pcb->da_mask, match_ctl); + } + + if (pcb->match_ctl & DV_MATCH) { + write_csr(pcb->dv_match, CSR_DV_MATCH); + write_csr(pcb->dv_mask, CSR_DV_MASK); + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + match_ctl |= (0x1UL << DV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH); + write_csr(match_ctl, CSR_DC_CTLP); + pr_info("dv_match:%#lx dv_mask:%#lx match_ctl:%#lx\n", pcb->dv_match, pcb->dv_mask, match_ctl); + } + + if (pcb->match_ctl & DAV_MATCH) { + write_csr(pcb->da_match, CSR_DA_MATCH); + write_csr(pcb->da_mask, CSR_DA_MASK); + write_csr(pcb->dv_match, CSR_DV_MATCH); + write_csr(pcb->dv_mask, CSR_DV_MASK); + write_csr(0xfffffffff, CSR_DA_MATCH_MODE); + match_ctl_mode = (pcb->match_ctl >> 8) & 0x3; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + match_ctl |= (match_ctl_mode << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) + | (0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) + | (0x3UL << DPM_MATCH); + write_csr(match_ctl, CSR_DC_CTLP); + pr_info("da_match:%#lx da_mask:%#lx dv_match:%#lx dv_mask:%#lx match_ctl:%#lx\n", + pcb->da_match, pcb->da_mask, pcb->dv_match, pcb->dv_mask, match_ctl); + } + + if (pcb->match_ctl & IA_MATCH) { + pcb->ia_match |= (0x1UL << IA_MATCH_EN_S) | 0x3; + pcb->ia_mask |= 0x3; + write_csr(pcb->ia_match, CSR_IA_MATCH); + write_csr(pcb->ia_mask, CSR_IA_MASK); + vpn = read_csr(CSR_VPCR) >> 44; + vpn &= 0x3ff; + upn = read_csr(CSR_UPCR); + upn &= 0x3ff; + write_csr(((0x3ff << 18) | vpn), CSR_IA_VPNMATCH); + write_csr(((0x3ff << 18) | upn), CSR_IA_UPNMATCH); + pr_info("ia_match:%#lx ia_mask:%#lx\n", pcb->ia_match, pcb->ia_mask); + } + if (pcb->match_ctl & IV_MATCH) { + pcb->ia_match |= (0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S) | 0x3; + write_csr(pcb->ia_match, CSR_IA_MATCH); + write_csr(pcb->iv_match, CSR_IV_MATCH); + pr_info("ia_match:%#lx iv_match:%#lx\n", pcb->ia_match, pcb->iv_match); + } + if (pcb->match_ctl & IDA_MATCH) { + pcb->ida_match |= (0x1UL << IDA_MATCH_EN_S) | 0x3; + pcb->ida_mask |= 0x3; + write_csr(pcb->ida_match, CSR_IDA_MATCH); + write_csr(pcb->ida_mask, CSR_IDA_MASK); + pr_info("ida_match:%#lx ida_mask:%#lx\n", pcb->ida_match, pcb->ida_mask); + } +} +#endif + struct pt_regs_offset { const char *name; int offset; }; +#define GPR_OFFSET_NAME(r) { \ + .name = "r" #r, \ + .offset = offsetof(struct pt_regs, regs[r]) \ +} + #define REG_OFFSET_NAME(r) { \ .name = #r, \ .offset = offsetof(struct pt_regs, r) \ @@ -572,38 +780,39 @@ struct pt_regs_offset { } static const struct pt_regs_offset regoffset_table[] = { - REG_OFFSET_NAME(r0), - REG_OFFSET_NAME(r1), - REG_OFFSET_NAME(r2), - REG_OFFSET_NAME(r3), - REG_OFFSET_NAME(r4), - REG_OFFSET_NAME(r5), - REG_OFFSET_NAME(r6), - REG_OFFSET_NAME(r7), - REG_OFFSET_NAME(r8), - REG_OFFSET_NAME(r9), - REG_OFFSET_NAME(r10), - REG_OFFSET_NAME(r11), - REG_OFFSET_NAME(r12), - REG_OFFSET_NAME(r13), - REG_OFFSET_NAME(r14), - REG_OFFSET_NAME(r15), - REG_OFFSET_NAME(r19), - REG_OFFSET_NAME(r20), - REG_OFFSET_NAME(r21), - REG_OFFSET_NAME(r22), - REG_OFFSET_NAME(r23), - REG_OFFSET_NAME(r24), - REG_OFFSET_NAME(r25), - REG_OFFSET_NAME(r26), - REG_OFFSET_NAME(r27), - REG_OFFSET_NAME(r28), - REG_OFFSET_NAME(ps), + GPR_OFFSET_NAME(0), + GPR_OFFSET_NAME(1), + GPR_OFFSET_NAME(2), + GPR_OFFSET_NAME(3), + GPR_OFFSET_NAME(4), + GPR_OFFSET_NAME(5), + GPR_OFFSET_NAME(6), + GPR_OFFSET_NAME(7), + GPR_OFFSET_NAME(8), + GPR_OFFSET_NAME(9), + GPR_OFFSET_NAME(10), + GPR_OFFSET_NAME(11), + GPR_OFFSET_NAME(12), + GPR_OFFSET_NAME(13), + GPR_OFFSET_NAME(14), + GPR_OFFSET_NAME(15), + GPR_OFFSET_NAME(16), + GPR_OFFSET_NAME(17), + GPR_OFFSET_NAME(18), + GPR_OFFSET_NAME(19), + GPR_OFFSET_NAME(20), + GPR_OFFSET_NAME(21), + GPR_OFFSET_NAME(22), + GPR_OFFSET_NAME(23), + GPR_OFFSET_NAME(24), + GPR_OFFSET_NAME(25), + GPR_OFFSET_NAME(26), + GPR_OFFSET_NAME(27), + GPR_OFFSET_NAME(28), + GPR_OFFSET_NAME(29), + GPR_OFFSET_NAME(30), REG_OFFSET_NAME(pc), - REG_OFFSET_NAME(gp), - REG_OFFSET_NAME(r16), - REG_OFFSET_NAME(r17), - REG_OFFSET_NAME(r18), + REG_OFFSET_NAME(ps), REG_OFFSET_END, }; diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 16c13646f3c7312729a610965785c7921fc6520b..9118c440bd16dcf4064bb7a74c801b5e85f30f5e 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -27,11 +27,13 @@ #include #include #include +#include #include #include #include #include +#include #include "proto.h" #include "pci_impl.h" @@ -43,9 +45,12 @@ #define DBGDCONT(args...) #endif +int __cpu_to_rcid[NR_CPUS]; /* Map logical to physical */ +EXPORT_SYMBOL(__cpu_to_rcid); DEFINE_PER_CPU(unsigned long, hard_node_id) = { 0 }; +#ifdef CONFIG_SUBARCH_C3B #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) struct cma *sw64_kvm_cma; EXPORT_SYMBOL(sw64_kvm_cma); @@ -56,6 +61,7 @@ static phys_addr_t kvm_mem_base; struct gen_pool *sw64_kvm_pool; EXPORT_SYMBOL(sw64_kvm_pool); #endif +#endif static inline int phys_addr_valid(unsigned long addr) { @@ -361,7 +367,7 @@ void __init process_memmap(void) static int i; // Make it static so we won't start over again every time. int ret; phys_addr_t base, size; - unsigned long dma_end __maybe_unused = virt_to_phys((void *)MAX_DMA_ADDRESS); + unsigned long dma_end __maybe_unused = (MAX_DMA32_PFN << PAGE_SHIFT); if (!memblock_initialized) return; @@ -577,7 +583,7 @@ static void __init setup_machine_fdt(void) const char *name; /* Give a chance to select kernel builtin DTB firstly */ - if (IS_ENABLED(CONFIG_SW64_BUILTIN_DTB)) + if (IS_ENABLED(CONFIG_BUILTIN_DTB)) dt_virt = (void *)__dtb_start; else { dt_virt = (void *)sunway_boot_params->dtb_start; @@ -632,25 +638,6 @@ static void __init setup_cpu_info(void) cpu_desc.pa_bits = CPUID_PA_BITS(val); cpu_desc.va_bits = CPUID_VA_BITS(val); - if (*(unsigned long *)MMSIZE) { - static_branch_disable(&run_mode_host_key); - if (*(unsigned long *)MMSIZE & EMUL_FLAG) { - pr_info("run mode: emul\n"); - static_branch_disable(&run_mode_guest_key); - static_branch_enable(&run_mode_emul_key); - - } else { - pr_info("run mode: guest\n"); - static_branch_enable(&run_mode_guest_key); - static_branch_disable(&run_mode_emul_key); - } - } else { - pr_info("run mode: host\n"); - static_branch_enable(&run_mode_host_key); - static_branch_disable(&run_mode_guest_key); - static_branch_disable(&run_mode_emul_key); - } - for (i = 0; i < VENDOR_ID_MAX; i++) { val = cpuid(GET_VENDOR_ID, i); memcpy(cpu_desc.vendor_id + (i * 8), &val, 8); @@ -694,6 +681,28 @@ static void __init setup_cpu_info(void) } } +static void __init setup_run_mode(void) +{ + if (*(unsigned long *)MMSIZE) { + static_branch_disable(&run_mode_host_key); + if (*(unsigned long *)MMSIZE & EMUL_FLAG) { + pr_info("run mode: emul\n"); + static_branch_disable(&run_mode_guest_key); + static_branch_enable(&run_mode_emul_key); + + } else { + pr_info("run mode: guest\n"); + static_branch_enable(&run_mode_guest_key); + static_branch_disable(&run_mode_emul_key); + } + } else { + pr_info("run mode: host\n"); + static_branch_enable(&run_mode_host_key); + static_branch_disable(&run_mode_guest_key); + static_branch_disable(&run_mode_emul_key); + } +} + static void __init setup_socket_info(void) { int i; @@ -728,6 +737,7 @@ static void __init reserve_mem_for_initrd(void) } #endif /* CONFIG_BLK_DEV_INITRD */ +#ifdef CONFIG_SUBARCH_C3B #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) static int __init early_kvm_reserved_mem(char *p) { @@ -750,24 +760,28 @@ void __init sw64_kvm_reserve(void) PAGE_SIZE, 0, "sw64_kvm_cma", &sw64_kvm_cma); } #endif +#endif void __init setup_arch(char **cmdline_p) { + /** + * Work around the unaligned access exception to parse ACPI + * tables in the following function acpi_boot_table_init(). + */ + trap_init(); + jump_label_init(); setup_cpu_info(); - sw64_chip->fixup(); - sw64_chip_init->fixup(); + setup_run_mode(); + setup_chip_ops(); setup_socket_info(); show_socket_mem_layout(); - sw64_chip_init->early_init.setup_core_start(&core_start); + sw64_chip_init->early_init.setup_core_map(&core_start); if (is_guest_or_emul()) - sw64_chip_init->early_init.get_smp_info(); + get_vt_smp_info(); setup_sched_clock(); -#ifdef CONFIG_GENERIC_SCHED_CLOCK - sw64_sched_clock_init(); -#endif setup_machine_fdt(); @@ -795,17 +809,6 @@ setup_arch(char **cmdline_p) } #endif /* CMDLINE_EXTEND */ #endif - if (IS_ENABLED(CONFIG_SW64_CHIP3_ASIC_DEBUG) && - IS_ENABLED(CONFIG_SW64_CHIP3)) { - unsigned long bmc, cpu_online, node; - - bmc = *(unsigned long *)__va(0x800000); - pr_info("bmc = %ld\n", bmc); - cpu_online = sw64_chip->get_cpu_num(); - for (node = 0; node < cpu_online; node++) - sw64_io_write(node, SI_FAULT_INT_EN, 0); - sprintf(boot_command_line, "root=/dev/sda2 ip=172.16.137.%ld::172.16.137.254:255.255.255.0::eth0:off", 180+bmc); - } strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE); *cmdline_p = command_line; @@ -831,8 +834,10 @@ setup_arch(char **cmdline_p) reserve_crashkernel(); /* Reserve large chunks of memory for use by CMA for KVM. */ +#ifdef CONFIG_SUBARCH_C3B #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) sw64_kvm_reserve(); +#endif #endif sw64_numa_init(); @@ -897,7 +902,7 @@ show_cpuinfo(struct seq_file *f, void *slot) int i; unsigned long cpu_freq; - cpu_freq = get_cpu_freq() / 1000 / 1000; + cpu_freq = cpuid(GET_CPU_FREQ, 0); for_each_online_cpu(i) { /* @@ -921,7 +926,7 @@ show_cpuinfo(struct seq_file *f, void *slot) "cache size\t: %u KB\n" "physical id\t: %d\n" "bogomips\t: %lu.%02lu\n", - cpu_freq, cpu_data[i].tcache.size >> 10, + get_cpu_freq() / 1000 / 1000, cpu_data[i].tcache.size >> 10, cpu_topology[i].package_id, loops_per_jiffy / (500000/HZ), (loops_per_jiffy / (5000/HZ)) % 100); @@ -989,6 +994,8 @@ device_initcall(add_pcspkr); #ifdef CONFIG_DEBUG_FS struct dentry *sw64_debugfs_dir; +EXPORT_SYMBOL(sw64_debugfs_dir); + static int __init debugfs_sw64(void) { struct dentry *d; @@ -1011,6 +1018,7 @@ static int __init sw64_of_init(void) core_initcall(sw64_of_init); #endif +#ifdef CONFIG_SUBARCH_C3B #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) static int __init sw64_kvm_pool_init(void) { @@ -1055,3 +1063,4 @@ static int __init sw64_kvm_pool_init(void) } core_initcall_sync(sw64_kvm_pool_init); #endif +#endif diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 6414654a0f595833e2ad16c75f4ea9e7e705978a..9ba151bb82d3993d6908d8ded38799bd0adb94cc 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -11,10 +11,13 @@ #include #include #include +#include #include +#include #include #include +#include #include "proto.h" @@ -92,43 +95,9 @@ extern char compile_time_assert static long restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) { - unsigned long usp; long err = __get_user(regs->pc, &sc->sc_pc); - current->restart_block.fn = do_no_restart_syscall; - - err |= __get_user(regs->r0, sc->sc_regs+0); - err |= __get_user(regs->r1, sc->sc_regs+1); - err |= __get_user(regs->r2, sc->sc_regs+2); - err |= __get_user(regs->r3, sc->sc_regs+3); - err |= __get_user(regs->r4, sc->sc_regs+4); - err |= __get_user(regs->r5, sc->sc_regs+5); - err |= __get_user(regs->r6, sc->sc_regs+6); - err |= __get_user(regs->r7, sc->sc_regs+7); - err |= __get_user(regs->r8, sc->sc_regs+8); - err |= __get_user(regs->r9, sc->sc_regs+9); - err |= __get_user(regs->r10, sc->sc_regs+10); - err |= __get_user(regs->r11, sc->sc_regs+11); - err |= __get_user(regs->r12, sc->sc_regs+12); - err |= __get_user(regs->r13, sc->sc_regs+13); - err |= __get_user(regs->r14, sc->sc_regs+14); - err |= __get_user(regs->r15, sc->sc_regs+15); - err |= __get_user(regs->r16, sc->sc_regs+16); - err |= __get_user(regs->r17, sc->sc_regs+17); - err |= __get_user(regs->r18, sc->sc_regs+18); - err |= __get_user(regs->r19, sc->sc_regs+19); - err |= __get_user(regs->r20, sc->sc_regs+20); - err |= __get_user(regs->r21, sc->sc_regs+21); - err |= __get_user(regs->r22, sc->sc_regs+22); - err |= __get_user(regs->r23, sc->sc_regs+23); - err |= __get_user(regs->r24, sc->sc_regs+24); - err |= __get_user(regs->r25, sc->sc_regs+25); - err |= __get_user(regs->r26, sc->sc_regs+26); - err |= __get_user(regs->r27, sc->sc_regs+27); - err |= __get_user(regs->r28, sc->sc_regs+28); - err |= __get_user(regs->gp, sc->sc_regs+29); - err |= __get_user(usp, sc->sc_regs+30); - wrusp(usp); + err |= __copy_from_user(regs, sc->sc_regs, sizeof_field(struct pt_regs, regs)); /* simd-fp */ err |= __copy_from_user(¤t->thread.fpstate, &sc->sc_fpregs, offsetof(struct user_fpsimd_state, fpcr)); @@ -146,12 +115,16 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) * registers and transfer control from userland. */ -asmlinkage void -do_sigreturn(struct sigcontext __user *sc) +SYSCALL_DEFINE1(sigreturn, struct sigcontext __user *, sc) { struct pt_regs *regs = current_pt_regs(); sigset_t set; + force_successful_syscall_return(); + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + /* Verify that it's a good sigcontext before using it */ if (!access_ok(sc, sizeof(*sc))) goto give_sigsegv; @@ -166,20 +139,25 @@ do_sigreturn(struct sigcontext __user *sc) /* Send SIGTRAP if we're single-stepping: */ if (ptrace_cancel_bpt(current)) { force_sig_fault(SIGTRAP, TRAP_BRKPT, - (void __user *)regs->pc, 0); + (void __user *)regs->pc); } - return; + return regs->regs[0]; give_sigsegv: force_sig(SIGSEGV); + return 0; } -asmlinkage void -do_rt_sigreturn(struct rt_sigframe __user *frame) +SYSCALL_DEFINE1(rt_sigreturn, struct rt_sigframe __user *, frame) { struct pt_regs *regs = current_pt_regs(); sigset_t set; + force_successful_syscall_return(); + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + /* Verify that it's a good ucontext_t before using it */ if (!access_ok(&frame->uc, sizeof(frame->uc))) goto give_sigsegv; @@ -197,12 +175,13 @@ do_rt_sigreturn(struct rt_sigframe __user *frame) /* Send SIGTRAP if we're single-stepping: */ if (ptrace_cancel_bpt(current)) { force_sig_fault(SIGTRAP, TRAP_BRKPT, - (void __user *)regs->pc, 0); + (void __user *)regs->pc); } - return; + return regs->regs[0]; give_sigsegv: force_sig(SIGSEGV); + return 0; } @@ -218,7 +197,7 @@ get_sigframe(struct ksignal *ksig, unsigned long sp, size_t frame_size) static long setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, - unsigned long mask, unsigned long sp) + unsigned long mask) { long err = 0; @@ -227,37 +206,7 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, err |= __put_user(regs->pc, &sc->sc_pc); err |= __put_user(8, &sc->sc_ps); - err |= __put_user(regs->r0, sc->sc_regs+0); - err |= __put_user(regs->r1, sc->sc_regs+1); - err |= __put_user(regs->r2, sc->sc_regs+2); - err |= __put_user(regs->r3, sc->sc_regs+3); - err |= __put_user(regs->r4, sc->sc_regs+4); - err |= __put_user(regs->r5, sc->sc_regs+5); - err |= __put_user(regs->r6, sc->sc_regs+6); - err |= __put_user(regs->r7, sc->sc_regs+7); - err |= __put_user(regs->r8, sc->sc_regs+8); - err |= __put_user(regs->r9, sc->sc_regs+9); - err |= __put_user(regs->r10, sc->sc_regs+10); - err |= __put_user(regs->r11, sc->sc_regs+11); - err |= __put_user(regs->r12, sc->sc_regs+12); - err |= __put_user(regs->r13, sc->sc_regs+13); - err |= __put_user(regs->r14, sc->sc_regs+14); - err |= __put_user(regs->r15, sc->sc_regs+15); - err |= __put_user(regs->r16, sc->sc_regs+16); - err |= __put_user(regs->r17, sc->sc_regs+17); - err |= __put_user(regs->r18, sc->sc_regs+18); - err |= __put_user(regs->r19, sc->sc_regs+19); - err |= __put_user(regs->r20, sc->sc_regs+20); - err |= __put_user(regs->r21, sc->sc_regs+21); - err |= __put_user(regs->r22, sc->sc_regs+22); - err |= __put_user(regs->r23, sc->sc_regs+23); - err |= __put_user(regs->r24, sc->sc_regs+24); - err |= __put_user(regs->r25, sc->sc_regs+25); - err |= __put_user(regs->r26, sc->sc_regs+26); - err |= __put_user(regs->r27, sc->sc_regs+27); - err |= __put_user(regs->r28, sc->sc_regs+28); - err |= __put_user(regs->gp, sc->sc_regs+29); - err |= __put_user(sp, sc->sc_regs+30); + err |= __copy_to_user(sc->sc_regs, regs, sizeof_field(struct pt_regs, regs)); err |= __put_user(0, sc->sc_regs+31); /* simd-fp */ __fpstate_save(current); @@ -271,11 +220,10 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { - unsigned long oldsp, err = 0; + unsigned long err = 0; struct rt_sigframe __user *frame; - oldsp = rdusp(); - frame = get_sigframe(ksig, oldsp, sizeof(*frame)); + frame = get_sigframe(ksig, regs->regs[30], sizeof(*frame)); if (!access_ok(frame, sizeof(*frame))) return -EFAULT; @@ -286,31 +234,30 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) err |= __put_user(0, &frame->uc.uc_flags); err |= __put_user(0, &frame->uc.uc_link); err |= __put_user(set->sig[0], &frame->uc.uc_old_sigmask); - err |= __save_altstack(&frame->uc.uc_stack, oldsp); - err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, - set->sig[0], oldsp); + err |= __save_altstack(&frame->uc.uc_stack, regs->regs[30]); + err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, set->sig[0]); err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); if (err) return -EFAULT; /* "Return" to the handler */ - regs->r26 = VDSO_SYMBOL(current->mm->context.vdso, rt_sigreturn); - regs->r27 = regs->pc = (unsigned long) ksig->ka.sa.sa_handler; - regs->r16 = ksig->sig; /* a0: signal number */ + regs->regs[26] = VDSO_SYMBOL(current->mm->context.vdso, rt_sigreturn); + regs->regs[27] = regs->pc = (unsigned long) ksig->ka.sa.sa_handler; + regs->regs[16] = ksig->sig; /* a0: signal number */ if (ksig->ka.sa.sa_flags & SA_SIGINFO) { /* a1: siginfo pointer, a2: ucontext pointer */ - regs->r17 = (unsigned long) &frame->info; - regs->r18 = (unsigned long) &frame->uc; + regs->regs[17] = (unsigned long) &frame->info; + regs->regs[18] = (unsigned long) &frame->uc; } else { /* a1: exception code, a2: sigcontext pointer */ - regs->r17 = 0; - regs->r18 = (unsigned long) &frame->uc.uc_mcontext; + regs->regs[17] = 0; + regs->regs[18] = (unsigned long) &frame->uc.uc_mcontext; } - wrusp((unsigned long) frame); + regs->regs[30] = (unsigned long) frame; #if DEBUG_SIG printk("SIG deliver (%s:%d): sp=%p pc=%p ra=%p\n", - current->comm, current->pid, frame, regs->pc, regs->r26); + current->comm, current->pid, frame, regs->pc, regs->regs[26]); #endif return 0; @@ -332,32 +279,6 @@ handle_signal(struct ksignal *ksig, struct pt_regs *regs) signal_setup_done(ret, ksig, 0); } -static inline void -syscall_restart(unsigned long r0, unsigned long r19, - struct pt_regs *regs, struct k_sigaction *ka) -{ - switch (regs->r0) { - case ERESTARTSYS: - if (!(ka->sa.sa_flags & SA_RESTART)) { - regs->r0 = EINTR; - break; - } - /* else: fallthrough */ - case ERESTARTNOINTR: - regs->r0 = r0; /* reset v0 and a3 and replay syscall */ - regs->r19 = r19; - regs->pc -= 4; - break; - case ERESTART_RESTARTBLOCK: - regs->r0 = EINTR; - break; - case ERESTARTNOHAND: - regs->r0 = EINTR; - break; - } -} - - /* * Note that 'init' is a special process: it doesn't get signals it doesn't * want to handle. Thus you cannot kill init even with a SIGKILL even by @@ -366,13 +287,9 @@ syscall_restart(unsigned long r0, unsigned long r19, * Note that we go through the signals twice: once to check the signals that * the kernel can handle, and then we build all the user-level signal handling * stack-frames in one go after that. - * - * "r0" and "r19" are the registers we need to restore for system call - * restart. "r0" is also used as an indicator whether we can restart at - * all (if we get here from anything but a syscall return, it will be 0) */ static void -do_signal(struct pt_regs *regs, unsigned long r0, unsigned long r19) +do_signal(struct pt_regs *regs) { unsigned long single_stepping = ptrace_cancel_bpt(current); struct ksignal ksig; @@ -382,27 +299,47 @@ do_signal(struct pt_regs *regs, unsigned long r0, unsigned long r19) /* ... so re-check the single stepping. */ single_stepping |= ptrace_cancel_bpt(current); /* Whee! Actually deliver the signal. */ - if (r0) - syscall_restart(r0, r19, regs, &ksig.ka); + if (regs->orig_r0 != NO_SYSCALL) { + switch (syscall_get_error(current, regs)) { + case -ERESTARTSYS: + if (!(ksig.ka.sa.sa_flags & SA_RESTART)) { + regs->regs[0] = EINTR; + break; + } + fallthrough; + case -ERESTARTNOINTR: + /* reset v0 and a3 and replay syscall */ + regs->regs[0] = regs->orig_r0; + regs->regs[19] = regs->orig_r19; + regs->pc -= 4; + break; + case -ERESTARTNOHAND: + case -ERESTART_RESTARTBLOCK: + regs->regs[0] = EINTR; + break; + } + regs->orig_r0 = NO_SYSCALL; + } handle_signal(&ksig, regs); } else { single_stepping |= ptrace_cancel_bpt(current); - if (r0) { - switch (regs->r0) { - case ERESTARTNOHAND: - case ERESTARTSYS: - case ERESTARTNOINTR: + if (regs->orig_r0 != NO_SYSCALL) { + switch (syscall_get_error(current, regs)) { + case -ERESTARTSYS: + case -ERESTARTNOINTR: + case -ERESTARTNOHAND: /* Reset v0 and a3 and replay syscall. */ - regs->r0 = r0; - regs->r19 = r19; + regs->regs[0] = regs->orig_r0; + regs->regs[19] = regs->orig_r19; regs->pc -= 4; break; - case ERESTART_RESTARTBLOCK: + case -ERESTART_RESTARTBLOCK: /* Set v0 to the restart_syscall and replay */ - regs->r0 = __NR_restart_syscall; + regs->regs[0] = __NR_restart_syscall; regs->pc -= 4; break; } + regs->orig_r0 = NO_SYSCALL; } restore_saved_sigmask(); } @@ -410,29 +347,35 @@ do_signal(struct pt_regs *regs, unsigned long r0, unsigned long r19) ptrace_set_bpt(current); /* re-set breakpoint */ } -void -do_work_pending(struct pt_regs *regs, unsigned long thread_flags, - unsigned long r0, unsigned long r19) +asmlinkage void +do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) { do { - if (thread_flags & _TIF_NEED_RESCHED) { + local_irq_enable(); + + if (thread_flags & _TIF_NEED_RESCHED) schedule(); - } else { - local_irq_enable(); - - if (thread_flags & _TIF_UPROBE) - uprobe_notify_resume(regs); - - if (thread_flags & _TIF_SIGPENDING) { - do_signal(regs, r0, r19); - r0 = 0; - } else { - clear_thread_flag(TIF_NOTIFY_RESUME); - tracehook_notify_resume(regs); - rseq_handle_notify_resume(NULL, regs); - } + + if (thread_flags & _TIF_UPROBE) { + unsigned long pc = regs->pc; + + uprobe_notify_resume(regs); + sw64_fix_uretprobe(regs, pc - 4); + } + + if (thread_flags & _TIF_PATCH_PENDING) + klp_update_patch_state(current); + + if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs); + + if (thread_flags & _TIF_NOTIFY_RESUME) { + clear_thread_flag(TIF_NOTIFY_RESUME); + tracehook_notify_resume(regs); + rseq_handle_notify_resume(NULL, regs); } + local_irq_disable(); - thread_flags = current_thread_info()->flags; + thread_flags = READ_ONCE(current_thread_info()->flags); } while (thread_flags & _TIF_WORK_MASK); } diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 003fee7f0ea19859fc430004a2b488c81491c34e..5b71852ab8722a02d0b400d68893c7d562bd6a4b 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "proto.h" @@ -28,12 +29,6 @@ int smp_booted; #define DBGS(fmt, arg...) \ do { if (smp_debug) printk("SMP: " fmt, ## arg); } while (0) -int __cpu_to_rcid[NR_CPUS]; /* Map logical to physical */ -EXPORT_SYMBOL(__cpu_to_rcid); - -int __rcid_to_cpu[NR_CPUS]; /* Map physical to logical */ -EXPORT_SYMBOL(__rcid_to_cpu); - void *idle_task_pointer[NR_CPUS]; /* State of each CPU */ @@ -56,8 +51,6 @@ EXPORT_SYMBOL(smp_num_cpus); #define send_sleep_interrupt(cpu) send_ipi((cpu), II_SLEEP) #define send_wakeup_interrupt(cpu) send_ipi((cpu), II_WAKE) -void __weak enable_chip_int(void) { } - /* * Where secondaries begin a life of C. */ @@ -67,8 +60,6 @@ void smp_callin(void) local_irq_disable(); - enable_chip_int(); - if (cpu_online(cpuid)) { printk("??, cpu 0x%x already present??\n", cpuid); BUG(); @@ -83,22 +74,28 @@ void smp_callin(void) trap_init(); /* Set interrupt vector. */ + if (is_in_host()) { + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); + } wrent(entInt, 0); /* Get our local ticker going. */ - setup_timer(); + sw64_setup_timer(); /* All kernel threads share the same mm context. */ mmgrab(&init_mm); current->active_mm = &init_mm; /* update csr:ptbr */ - wrptbr(virt_to_phys(init_mm.pgd)); + update_ptbr_sys(virt_to_phys(init_mm.pgd)); /* inform the notifiers about the new cpu */ notify_cpu_starting(cpuid); per_cpu(cpu_state, cpuid) = CPU_ONLINE; - per_cpu(hard_node_id, cpuid) = cpu_to_rcid(cpuid) >> CORES_PER_NODE_SHIFT; + per_cpu(hard_node_id, cpuid) = rcid_to_domain_id(cpu_to_rcid(cpuid)); /* Must have completely accurate bogos. */ local_irq_enable(); @@ -195,29 +192,19 @@ void __init smp_rcb_init(void) */ void __init setup_smp(void) { - int i = 0, num = 0; /* i: physical id, num: logical id */ + int i = 0, num = 0; init_cpu_possible(cpu_none_mask); /* For unified kernel, NR_CPUS is the maximum possible value */ for (; i < NR_CPUS; i++) { - if (cpumask_test_cpu(i, &core_start)) { - __cpu_to_rcid[num] = i; - __rcid_to_cpu[i] = num; + if (cpu_to_rcid(i) != -1) { set_cpu_possible(num, true); store_cpu_data(num); if (!cpumask_test_cpu(i, &cpu_offline)) set_cpu_present(num, true); num++; - } else - __rcid_to_cpu[i] = -1; - } - /* for sw64, the BSP must be logical core 0 */ - BUG_ON(cpu_to_rcid(0) != hard_smp_processor_id()); - - while (num < NR_CPUS) { - __cpu_to_rcid[num] = -1; - num++; + } } process_nr_cpu_ids(); @@ -268,12 +255,19 @@ int vt_cpu_up(unsigned int cpu, struct task_struct *tidle) wmb(); smp_rcb->ready = 0; + if (smp_booted) { + /* irq must be disabled before reset vCPU */ + reset_cpu(cpu); + } smp_boot_one_cpu(cpu, tidle); return cpu_online(cpu) ? 0 : -ENOSYS; } +#ifdef CONFIG_SUBARCH_C3B DECLARE_STATIC_KEY_FALSE(use_tc_as_sched_clock); +#endif + int __cpu_up(unsigned int cpu, struct task_struct *tidle) { if (is_in_guest()) @@ -282,10 +276,8 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) wmb(); smp_rcb->ready = 0; -#ifdef CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE /* send wake up signal */ send_wakeup_interrupt(cpu); -#endif /* send reset signal */ if (smp_booted) { if (is_in_host()) { @@ -298,7 +290,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) } smp_boot_one_cpu(cpu, tidle); -#ifdef CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE +#ifdef CONFIG_SUBARCH_C3B if (static_branch_likely(&use_tc_as_sched_clock)) { if (smp_booted) { tc_sync_clear(); @@ -373,6 +365,8 @@ void handle_ipi(struct pt_regs *regs) case IPI_CPU_STOP: ipi_cpu_stop(cpu); + break; + default: pr_crit("Unknown IPI on CPU %d: %lu\n", cpu, which); break; @@ -583,19 +577,10 @@ void arch_cpu_idle_dead(void) } #ifdef CONFIG_SUSPEND - -#ifdef CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE sleepen(); send_sleep_interrupt(smp_processor_id()); while (1) asm("nop"); -#else - asm volatile("halt"); - while (1) - asm("nop"); -#endif /* SW64_SUSPEND_DEEPSLEEP */ - - #else asm volatile("memb"); asm volatile("halt"); diff --git a/arch/sw_64/kernel/stacktrace.c b/arch/sw_64/kernel/stacktrace.c index 71c0e5b7b3e00bfccc184430a0845f043e23ca6f..1a67a8bb854b09aec50587f6a36cb3d106abc1ce 100644 --- a/arch/sw_64/kernel/stacktrace.c +++ b/arch/sw_64/kernel/stacktrace.c @@ -71,7 +71,7 @@ void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, if (regs) { unsigned long offset; pc = regs->pc; - fp = regs->r15; + fp = regs->regs[15]; if (kallsyms_lookup_size_offset(pc, NULL, &offset) && offset < 16) { /* call stack has not been setup @@ -79,7 +79,7 @@ void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, */ if (fn(pc, data)) return; - pc = regs->r26; + pc = regs->regs[26]; } } else if (tsk == current || tsk == NULL) { fp = (unsigned long)__builtin_frame_address(0); @@ -130,7 +130,7 @@ void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, while (!kstack_end(ksp)) { if (__kernel_text_address(pc) && fn(pc, data)) break; - pc = (*ksp++) - 0x4; + pc = *ksp++; } } EXPORT_SYMBOL_GPL(walk_stackframe); @@ -174,6 +174,19 @@ int save_trace(unsigned long pc, void *d) return (trace->nr_entries >= trace->max_entries); } +void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) +{ + struct stack_trace_data data; + + data.trace = trace; + data.nosched = 0; + + walk_stackframe(current, regs, save_trace, &data); + + if (trace->nr_entries < trace->max_entries) + trace->entries[trace->nr_entries++] = ULONG_MAX; +} + static void __save_stack_trace(struct task_struct *tsk, struct stack_trace *trace, unsigned int nosched) { diff --git a/arch/sw_64/kernel/suspend.c b/arch/sw_64/kernel/suspend.c index b9798baa246745c68bcd21ce98f2d4a8fdaac66e..27a240e6614955835f7abe8c21558b956898da43 100644 --- a/arch/sw_64/kernel/suspend.c +++ b/arch/sw_64/kernel/suspend.c @@ -33,46 +33,25 @@ void sw64_suspend_enter(void) /* boot processor will go to deep sleep mode from here * After wake up boot processor, pc will go here */ - -#ifdef CONFIG_SW64_SUPPORT_S3_SLEEPING_STATE - if (sw64_chip->suspend) - sw64_chip->suspend(false); -#endif - disable_local_timer(); current_thread_info()->pcb.tp = rtid(); -#ifdef CONFIG_SW64_SUSPEND_DEEPSLEEP_BOOTCORE sw64_suspend_deep_sleep(&suspend_state); -#else - mtinten(); - asm("halt"); -#endif wrtp(current_thread_info()->pcb.tp); -#ifdef CONFIG_SW64_SUPPORT_S3_SLEEPING_STATE - if (sw64_chip->suspend) - sw64_chip->suspend(true); -#endif - disable_local_timer(); } static int native_suspend_enter(suspend_state_t state) { + if (is_in_guest()) + return 0; /* processor specific suspend */ sw64_suspend_enter(); return 0; } -static const struct platform_suspend_ops native_suspend_ops = { +const struct platform_suspend_ops native_suspend_ops = { .valid = native_suspend_state_valid, .enter = native_suspend_enter, }; - -static int __init sw64_pm_init(void) -{ - suspend_set_ops(&native_suspend_ops); - return 0; -} -arch_initcall(sw64_pm_init); diff --git a/arch/sw_64/kernel/sys_sw64.c b/arch/sw_64/kernel/sys_sw64.c index 470c74853d477041e181d73a1f91991bc384fe19..d0198aef554d78a60dd4cb67acc78324a49d1b33 100644 --- a/arch/sw_64/kernel/sys_sw64.c +++ b/arch/sw_64/kernel/sys_sw64.c @@ -93,7 +93,7 @@ SYSCALL_DEFINE5(setsysinfo, unsigned long, op, void __user *, buffer, if (fex & IEEE_TRAP_ENABLE_INV) si_code = FPE_FLTINV; - send_sig_fault(SIGFPE, si_code, (void __user *)NULL, 0, current); + send_sig_fault(SIGFPE, si_code, (void __user *)NULL, current); } return 0; } @@ -121,19 +121,19 @@ SYSCALL_DEFINE2(odd_getpriority, int, which, int, who) SYSCALL_DEFINE0(getxuid) { - current_pt_regs()->r20 = sys_geteuid(); + current_pt_regs()->regs[20] = sys_geteuid(); return sys_getuid(); } SYSCALL_DEFINE0(getxgid) { - current_pt_regs()->r20 = sys_getegid(); + current_pt_regs()->regs[20] = sys_getegid(); return sys_getgid(); } SYSCALL_DEFINE0(getxpid) { - current_pt_regs()->r20 = sys_getppid(); + current_pt_regs()->regs[20] = sys_getppid(); return sys_getpid(); } @@ -144,7 +144,7 @@ SYSCALL_DEFINE0(sw64_pipe) if (!res) { /* The return values are in $0 and $20. */ - current_pt_regs()->r20 = fd[1]; + current_pt_regs()->regs[20] = fd[1]; res = fd[0]; } return res; diff --git a/arch/sw_64/kernel/time.c b/arch/sw_64/kernel/time.c index 8be7a11df51d8c4f61a36d9d3c95114aab14148f..533a6a14c20051da05469ed5bd1bf0d824c8c85d 100644 --- a/arch/sw_64/kernel/time.c +++ b/arch/sw_64/kernel/time.c @@ -4,19 +4,15 @@ #include #include #include -#ifndef CONFIG_SMP -#include -#endif #include +#include #include "proto.h" DEFINE_SPINLOCK(rtc_lock); EXPORT_SYMBOL(rtc_lock); -DECLARE_PER_CPU(u64, tc_offset); - #define TICK_SIZE (tick_nsec / 1000) /* @@ -28,25 +24,6 @@ DECLARE_PER_CPU(u64, tc_offset); unsigned long est_cycle_freq; -static u64 sc_start; -static u64 sc_shift; -static u64 sc_multi; - -DEFINE_STATIC_KEY_FALSE(use_tc_as_sched_clock); -static int __init sched_clock_setup(char *opt) -{ - if (!opt) - return -EINVAL; - - if (!strncmp(opt, "on", 2)) { - static_branch_enable(&use_tc_as_sched_clock); - pr_info("Using TC instead of jiffies as source of sched_clock()\n"); - } - - return 0; -} -early_param("tc_sched_clock", sched_clock_setup); - #ifdef CONFIG_IRQ_WORK DEFINE_PER_CPU(u8, irq_work_pending); @@ -67,36 +44,6 @@ void arch_irq_work_raise(void) #endif /* CONFIG_IRQ_WORK */ -#ifndef CONFIG_SMP -static u64 read_tc(struct clocksource *cs) -{ - return rdtc(); -} - -static struct clocksource clocksource_tc = { - .name = "tc", - .rating = 300, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .mask = CLOCKSOURCE_MASK(64), - .shift = 22, - .mult = 0, /* To be filled in */ - .read = read_tc, -}; - -void setup_clocksource(void) -{ - clocksource_register_hz(&clocksource_tc, get_cpu_freq()); - pr_info("Setup clocksource TC, mult = %d\n", clocksource_tc.mult); -} - -#else /* !CONFIG_SMP */ -void setup_clocksource(void) -{ - setup_chip_clocksource(); -} -#endif /* !CONFIG_SMP */ - - void __init time_init(void) { @@ -107,115 +54,10 @@ time_init(void) pr_info("CPU Cycle frequency = %ld Hz\n", cycle_freq); /* Register clocksource */ - setup_clocksource(); + sw64_setup_clocksource(); of_clk_init(NULL); /* Startup the timer source. */ - setup_timer(); + sw64_setup_timer(); /* Calibrate the delay loop directly */ lpj_fine = cycle_freq / HZ; } - -static void __init calibrate_sched_clock(void) -{ - sc_start = rdtc(); -} - -void __init setup_sched_clock(void) -{ - unsigned long step; - - sc_shift = 7; - step = 1UL << sc_shift; - sc_multi = step * NSEC_PER_SEC / get_cpu_freq(); - calibrate_sched_clock(); - - pr_info("sched_clock: sc_multi=%llu, sc_shift=%llu\n", sc_multi, sc_shift); -} - -#ifdef CONFIG_GENERIC_SCHED_CLOCK -static u64 notrace sched_clock_read(void) -{ - return (rdtc() - sc_start) >> sc_shift; -} - -void __init sw64_sched_clock_init(void) -{ - sched_clock_register(sched_clock_read, BITS_PER_LONG, get_cpu_freq() >> sc_shift); -} -#else -unsigned long long notrace sched_clock(void) -{ - if (static_branch_likely(&use_tc_as_sched_clock)) - return ((rdtc() - sc_start + __this_cpu_read(tc_offset)) >> sc_shift) * sc_multi; - else - return (jiffies - INITIAL_JIFFIES) * (NSEC_PER_SEC / HZ); -} - -#ifdef CONFIG_DEBUG_FS -static ssize_t sched_clock_status_read(struct file *file, char __user *user_buf, - size_t count, loff_t *ppos) -{ - char buf[2]; - - if (static_key_enabled(&use_tc_as_sched_clock)) - buf[0] = 'Y'; - else - buf[0] = 'N'; - buf[1] = '\n'; - return simple_read_from_buffer(user_buf, count, ppos, buf, 2); -} - -static ssize_t sched_clock_status_write(struct file *file, const char __user *user_buf, - size_t count, loff_t *ppos) -{ - int r; - bool bv; - bool val = static_key_enabled(&use_tc_as_sched_clock); - - r = kstrtobool_from_user(user_buf, count, &bv); - if (!r) { - if (val != bv) { - if (bv) { - static_branch_enable(&use_tc_as_sched_clock); - pr_info("source of sched_clock() switched from jiffies to TC\n"); - } else { - static_branch_disable(&use_tc_as_sched_clock); - pr_info("source of sched_clock() switched from TC to jiffies\n"); - } - } else { - if (val) - pr_info("source of sched_clock() unchanged (using TC)\n"); - else - pr_info("source of sched_clock() unchanged (using jiffies)\n"); - } - } - - return count; -} - -static const struct file_operations sched_clock_status_fops = { - .read = sched_clock_status_read, - .write = sched_clock_status_write, - .open = nonseekable_open, - .llseek = no_llseek, -}; - -static int __init sched_clock_debug_init(void) -{ - struct dentry *sched_clock_status; - - if (!sw64_debugfs_dir) - return -ENODEV; - - sched_clock_status = debugfs_create_file("tc_sched_clock", - 0644, sw64_debugfs_dir, NULL, - &sched_clock_status_fops); - - if (!sched_clock_status) - return -ENOMEM; - - return 0; -} -late_initcall(sched_clock_debug_init); -#endif /* CONFIG_DEBUG_FS */ -#endif /* CONFIG_GENERIC_SCHED_CLOCK */ diff --git a/arch/sw_64/kernel/topology.c b/arch/sw_64/kernel/topology.c index d1037e33480e1fd36bd0e59575712ae696fc8525..2db9f5aec048ccd09dc2d0bfb0ed73137b1a2262 100644 --- a/arch/sw_64/kernel/topology.c +++ b/arch/sw_64/kernel/topology.c @@ -3,6 +3,7 @@ #include #include #include +#include #include static int __init parse_dt_topology(void) @@ -23,6 +24,18 @@ static int topo_threads[NR_CPUS]; static int topo_cores[NR_CPUS]; static int topo_packages[NR_CPUS]; +void __init get_vt_smp_info(void) +{ + unsigned long smp_info; + + smp_info = sw64_io_read(0, SMP_INFO); + if (smp_info == -1UL) + smp_info = 0; + topo_nr_threads = (smp_info >> VT_THREADS_SHIFT) & VT_THREADS_MASK; + topo_nr_cores = (smp_info >> VT_CORES_SHIFT) & VT_CORES_MASK; + topo_nr_maxcpus = (smp_info >> VT_MAX_CPUS_SHIFT) & VT_MAX_CPUS_MASK; +} + static void __init init_topo_threads(void) { int i, j; @@ -120,10 +133,10 @@ void store_cpu_topology(int cpu) goto topology_populated; } - cpu_topo->package_id = rcid_to_package(cpu_to_rcid(cpu)); - cpu_topo->core_id = cpu_to_rcid(cpu) & CORE_ID_MASK; - cpu_topo->thread_id = (cpu_to_rcid(cpu) >> THREAD_ID_SHIFT) & THREAD_ID_MASK; - cpu_topo->llc_id = rcid_to_package(cpu_to_rcid(cpu)); + cpu_topo->package_id = rcid_to_domain_id(cpu_to_rcid(cpu)); + cpu_topo->core_id = rcid_to_core_id(cpu_to_rcid(cpu)); + cpu_topo->thread_id = rcid_to_thread_id(cpu_to_rcid(cpu)); + cpu_topo->llc_id = rcid_to_domain_id(cpu_to_rcid(cpu)); pr_debug("CPU%u: socket %d core %d thread %d llc %d\n", cpu, cpu_topo->package_id, cpu_topo->core_id, diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index fda9ef61a2e449db0439abc1d6476236d61f60d2..cc14d914c7d45b408c5a8bfa566af79cb4fd8598 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -30,6 +31,8 @@ #include #include #include +#include +#include #include "proto.h" @@ -47,33 +50,32 @@ void show_regs(struct pt_regs *regs) show_regs_print_info(KERN_DEFAULT); printk("pc = [<%016lx>] ra = [<%016lx>] ps = %04lx %s\n", - regs->pc, regs->r26, regs->ps, print_tainted()); + regs->pc, regs->regs[26], regs->ps, print_tainted()); printk("pc is at %pSR\n", (void *)regs->pc); - printk("ra is at %pSR\n", (void *)regs->r26); + printk("ra is at %pSR\n", (void *)regs->regs[26]); printk("v0 = %016lx t0 = %016lx t1 = %016lx\n", - regs->r0, regs->r1, regs->r2); + regs->regs[0], regs->regs[1], regs->regs[2]); printk("t2 = %016lx t3 = %016lx t4 = %016lx\n", - regs->r3, regs->r4, regs->r5); + regs->regs[3], regs->regs[4], regs->regs[5]); printk("t5 = %016lx t6 = %016lx t7 = %016lx\n", - regs->r6, regs->r7, regs->r8); + regs->regs[6], regs->regs[7], regs->regs[8]); printk("s0 = %016lx s1 = %016lx s2 = %016lx\n", - regs->r9, regs->r10, regs->r11); + regs->regs[9], regs->regs[10], regs->regs[11]); printk("s3 = %016lx s4 = %016lx s5 = %016lx\n", - regs->r12, regs->r13, regs->r14); + regs->regs[12], regs->regs[13], regs->regs[14]); printk("s6 = %016lx\n", - regs->r15); + regs->regs[15]); printk("a0 = %016lx a1 = %016lx a2 = %016lx\n", - regs->r16, regs->r17, regs->r18); + regs->regs[16], regs->regs[17], regs->regs[18]); printk("a3 = %016lx a4 = %016lx a5 = %016lx\n", - regs->r19, regs->r20, regs->r21); + regs->regs[19], regs->regs[20], regs->regs[21]); printk("t8 = %016lx t9 = %016lx t10 = %016lx\n", - regs->r22, regs->r23, regs->r24); + regs->regs[22], regs->regs[23], regs->regs[24]); printk("t11= %016lx pv = %016lx at = %016lx\n", - regs->r25, regs->r27, regs->r28); - printk("gp = %016lx sp = %px\n", regs->gp, - user_mode(regs) ? (void *)rdusp() : (regs + 1)); + regs->regs[25], regs->regs[27], regs->regs[28]); + printk("gp = %016lx sp = %016lx\n", regs->regs[29], regs->regs[30]); } static void show_code(unsigned int *pc) @@ -164,7 +166,11 @@ do_entArith(unsigned long summary, unsigned long write_mask, if (!user_mode(regs)) die("Arithmetic fault", regs, 0); - force_sig_fault(SIGFPE, si_code, (void __user *)regs->pc, 0); + /*summary<39> means integer divide by zero in C4.*/ + if ((summary >> 39) & 1) + si_code = FPE_INTDIV; + + force_sig_fault(SIGFPE, si_code, (void __user *)regs->pc); } void simd_emulate(unsigned int inst, unsigned long va) @@ -217,12 +223,12 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) switch (type) { case IF_BREAKPOINT: /* gdb do pc-4 for sigtrap */ - force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc, 0); + force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc); return; case IF_GENTRAP: regs->pc -= 4; - switch ((long)regs->r16) { + switch ((long)regs->regs[16]) { case GEN_INTOVF: signo = SIGFPE; code = FPE_INTOVF; @@ -280,7 +286,7 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) break; } - force_sig_fault(signo, code, (void __user *)regs->pc, regs->r16); + force_sig_fault(signo, code, (void __user *)regs->pc); return; case IF_FEN: @@ -293,17 +299,20 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) case BREAK_KPROBE: if (notify_die(DIE_BREAK, "kprobe", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) return; + break; case BREAK_KPROBE_SS: if (notify_die(DIE_SSTEPBP, "single_step", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) return; + break; #endif #ifdef CONFIG_UPROBES case UPROBE_BRK_UPROBE: if (notify_die(DIE_UPROBE, "uprobe", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) - return sw64_fix_uretprobe(regs); + return; + break; case UPROBE_BRK_UPROBE_XOL: if (notify_die(DIE_UPROBE_XOL, "uprobe_xol", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) - return sw64_fix_uretprobe(regs); + return; #endif } @@ -318,7 +327,7 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) break; } - force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->pc, 0); + force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->pc); } asmlinkage void @@ -328,7 +337,6 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, long error; unsigned long tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; unsigned long pc = regs->pc - 4; - const struct exception_table_entry *fixup; /* * We don't want to use the generic get/put unaligned macros as @@ -355,7 +363,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, if (error) goto got_exception; - map_regs(reg) = tmp1 | tmp2; + regs->regs[reg] = tmp1 | tmp2; return; case 0x22: @@ -376,7 +384,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, if (error) goto got_exception; - map_regs(reg) = (int)(tmp1 | tmp2); + regs->regs[reg] = (int)(tmp1 | tmp2); return; case 0x23: /* ldl */ @@ -397,7 +405,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, if (error) goto got_exception; - map_regs(reg) = tmp1 | tmp2; + regs->regs[reg] = tmp1 | tmp2; return; case 0x29: /* sth */ @@ -415,7 +423,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(map_regs(reg)), "0"(0)); + : "r"(va), "r"(regs->regs[reg]), "0"(0)); if (error) goto got_exception; @@ -447,7 +455,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(map_regs(reg)), "0"(0)); + : "r"(va), "r"(regs->regs[reg]), "0"(0)); if (error) goto got_exception; @@ -499,7 +507,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(map_regs(reg)), "0"(0)); + : "r"(va), "r"(regs->regs[reg]), "0"(0)); if (error) goto got_exception; @@ -514,15 +522,9 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, /* Ok, we caught the exception, but we don't want it. Is there * someone to pass it along to? */ - fixup = search_exception_tables(pc); - if (fixup != 0) { - unsigned long newpc; - - newpc = fixup_exception(map_regs, fixup, pc); + if (fixup_exception(regs, pc)) { printk("Forwarding unaligned exception at %lx (%lx)\n", - pc, newpc); - - regs->pc = newpc; + pc, regs->pc); return; } @@ -619,12 +621,8 @@ do_entUnaUser(void __user *va, unsigned long opcode, if ((1L << opcode) & OP_INT_MASK) { /* it's an integer load/store */ - if (reg < 30) { - reg_addr = (unsigned long *) - ((char *)regs + regoffsets[reg]); - } else if (reg == 30) { - /* usp in HMCODE regs */ - fake_reg = rdusp(); + if (reg < 31) { + reg_addr = ®s->regs[reg]; } else { /* zero "register" */ fake_reg = 0; @@ -1457,9 +1455,6 @@ do_entUnaUser(void __user *va, unsigned long opcode, goto give_sigbus; } - /* Only integer loads should get here; everyone else returns early. */ - if (reg == 30) - wrusp(fake_reg); return; give_sigsegv: @@ -1481,12 +1476,51 @@ do_entUnaUser(void __user *va, unsigned long opcode, si_code = SEGV_MAPERR; up_read(&mm->mmap_lock); } - force_sig_fault(SIGSEGV, si_code, va, 0); + force_sig_fault(SIGSEGV, si_code, va); return; give_sigbus: regs->pc -= 4; - force_sig_fault(SIGBUS, BUS_ADRALN, va, 0); + force_sig_fault(SIGBUS, BUS_ADRALN, va); +} + +asmlinkage void do_entSys(struct pt_regs *regs) +{ + long ret = -ENOSYS; + unsigned long nr; + unsigned long ti_flags = current_thread_info()->flags; + + regs->orig_r0 = regs->regs[0]; + regs->orig_r19 = regs->regs[19]; + nr = regs->regs[0]; + + if (ti_flags & _TIF_SYSCALL_WORK) { + nr = syscall_trace_enter(); + if (nr == NO_SYSCALL) + goto syscall_out; + regs->orig_r0 = regs->regs[0]; + regs->orig_r19 = regs->regs[19]; + } + + if (nr < __NR_syscalls) { + syscall_fn_t syscall_fn = sys_call_table[nr]; + + ret = syscall_fn(regs->regs[16], regs->regs[17], regs->regs[18], + regs->regs[19], regs->regs[20], regs->regs[21]); + } + + if ((nr != __NR_sigreturn) && (nr != __NR_rt_sigreturn)) { + if (likely((ret >= 0) || regs->orig_r0 == NO_SYSCALL)) + syscall_set_return_value(current, regs, 0, ret); + else + syscall_set_return_value(current, regs, ret, 0); + } + +syscall_out: + rseq_syscall(regs); + + if (ti_flags & _TIF_SYSCALL_WORK) + syscall_trace_leave(); } void @@ -1502,6 +1536,7 @@ trap_init(void) wrent(entUna, 4); wrent(entSys, 5); #ifdef CONFIG_EFI - wrent((void *)entSuspend, 6); + if (smp_processor_id() == 0) + wrent((void *)entSuspend, 6); #endif } diff --git a/arch/sw_64/kernel/uprobes.c b/arch/sw_64/kernel/uprobes.c index e25793f4a058e1763facd76f90584f62a95f1a56..25da4cee7d5ea4c52bf8829ebbef5a8b0db1374d 100644 --- a/arch/sw_64/kernel/uprobes.c +++ b/arch/sw_64/kernel/uprobes.c @@ -127,10 +127,10 @@ unsigned long arch_uretprobe_hijack_return_addr( { unsigned long ra; - ra = regs->r26; + ra = regs->regs[26]; /* Replace the return address with the trampoline address */ - regs->r26 = trampoline_vaddr; + regs->regs[26] = trampoline_vaddr; return ra; } @@ -172,14 +172,11 @@ static unsigned long get_trampoline_vaddr(void) return trampoline_vaddr; } -void sw64_fix_uretprobe(struct pt_regs *regs) +void sw64_fix_uretprobe(struct pt_regs *regs, unsigned long exc_pc) { - unsigned long bp_vaddr; - - bp_vaddr = uprobe_get_swbp_addr(regs); /* * regs->pc has been changed to orig_ret_vaddr in handle_trampoline(). */ - if (bp_vaddr == get_trampoline_vaddr()) - regs->r26 = regs->pc; + if (exc_pc == get_trampoline_vaddr()) + regs->regs[26] = regs->pc; } diff --git a/arch/sw_64/kernel/vdso/vgettimeofday.c b/arch/sw_64/kernel/vdso/vgettimeofday.c index 49bb4e2e66ed40526f6154ae4649a4c5db824235..0aa16e988e88efaeda40178f46583ee6647e9f59 100644 --- a/arch/sw_64/kernel/vdso/vgettimeofday.c +++ b/arch/sw_64/kernel/vdso/vgettimeofday.c @@ -16,6 +16,7 @@ #include #include +#include #include static __always_inline int syscall_fallback(clockid_t clkid, struct timespec64 *ts) @@ -74,6 +75,7 @@ static __always_inline int do_monotonic_coarse(struct timespec64 *ts, return 0; } +#if defined(CONFIG_SUBARCH_C3B) static __always_inline u64 read_longtime(void) { register unsigned long __r0 __asm__("$0"); @@ -83,6 +85,12 @@ static __always_inline u64 read_longtime(void) return __r0; } +#elif defined(CONFIG_SUBARCH_C4) +static __always_inline u64 read_longtime(void) +{ + return read_csr(CSR_SHTCLOCK); +} +#endif static __always_inline u64 get_ns(const struct vdso_data *data) { diff --git a/arch/sw_64/kernel/vdso/vrt_sigreturn.S b/arch/sw_64/kernel/vdso/vrt_sigreturn.S index d2d7295ffa7ac13d2c0fea96cb56b50eb72af6b3..f1946c2bf4765791b21f334b856b7d0a89afefed 100644 --- a/arch/sw_64/kernel/vdso/vrt_sigreturn.S +++ b/arch/sw_64/kernel/vdso/vrt_sigreturn.S @@ -20,9 +20,7 @@ #include #include #include - -#define RT_SIGFRAME_SIZE 1600 -#define RT_SIGFRAME_MCTX 176 +#include .text diff --git a/arch/sw_64/kernel/vmlinux.lds.S b/arch/sw_64/kernel/vmlinux.lds.S index 07bc3d8ee7e47bcc98c2c3de0635ab1f152b3662..acdcf3c1ab1f32814c7b5ed20f859253c51232b2 100644 --- a/arch/sw_64/kernel/vmlinux.lds.S +++ b/arch/sw_64/kernel/vmlinux.lds.S @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#define RUNTIME_DISCARD_EXIT #define EMITS_PT_NOTE #define RO_EXCEPTION_TABLE_ALIGN 16 @@ -39,6 +40,15 @@ SECTIONS __init_begin = ALIGN(PAGE_SIZE); INIT_TEXT_SECTION(PAGE_SIZE) INIT_DATA_SECTION(16) + /* we have to discard exit text and such at runtime, not link time */ + .exit.text : + { + EXIT_TEXT + } + .exit.data : + { + EXIT_DATA + } PERCPU_SECTION(L1_CACHE_BYTES) /* diff --git a/arch/sw_64/kvm/Kconfig b/arch/sw_64/kvm/Kconfig index 8077ea4527654fe6e0329cd0a8942da9490f49f5..b7e43d0bae510ebe8297d60f0ff35fe30d97cbea 100644 --- a/arch/sw_64/kvm/Kconfig +++ b/arch/sw_64/kvm/Kconfig @@ -17,7 +17,6 @@ if VIRTUALIZATION config KVM tristate "Kernel-based Virtual Machine (KVM) support" - select KVM_SW64_HOST select PREEMPT_NOTIFIERS select CMA depends on NET @@ -27,6 +26,8 @@ config KVM select HAVE_KVM_IRQFD select HAVE_KVM_MSI select KVM_VFIO + select MMU_NOTIFIER + select KVM_GENERIC_DIRTYLOG_READ_PROTECT select TUN select GENERIC_ALLOCATOR select KVM_GENERIC_DIRTYLOG_READ_PROTECT @@ -36,16 +37,9 @@ config KVM If unsure, say N. -config KVM_SW64_HOST - tristate "KVM for SW64 processors support" - depends on KVM - help - Provides host support for SW64 processors. - To compile this as a module, choose M here. - config KVM_MEMHOTPLUG bool "Memory hotplug support for guest" - depends on KVM && MEMORY_HOTPLUG + depends on KVM && MEMORY_HOTPLUG && SUBARCH_C3B help Provides memory hotplug support for SW64 guest. diff --git a/arch/sw_64/kvm/Makefile b/arch/sw_64/kvm/Makefile index 43cea19215ffa91a6006631553cee9ce98098be9..5e24b82ce5b8d24fa54565b7b7bbbaef7ded127e 100644 --- a/arch/sw_64/kvm/Makefile +++ b/arch/sw_64/kvm/Makefile @@ -7,7 +7,10 @@ KVM := ../../../virt/kvm ccflags-y += -Ivirt/kvm -Iarch/sw_64/kvm -kvm-$(CONFIG_KVM_SW64_HOST) += $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o -kvm-$(CONFIG_KVM_SW64_HOST) += kvm-sw64.o entry.o emulate.o mmio.o kvm_timer.o handle_exit.o perf.o +obj-$(CONFIG_KVM) += kvm.o -obj-$(CONFIG_KVM_SW64_HOST) += kvm.o +kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o \ + sw64.o entry.o emulate.o mmio.o kvm_timer.o handle_exit.o perf.o + +kvm-$(CONFIG_SUBARCH_C3B) += kvm_core3.o +kvm-$(CONFIG_SUBARCH_C4) += kvm_core4.o mmu.o diff --git a/arch/sw_64/kvm/emulate.c b/arch/sw_64/kvm/emulate.c index bcc06c0dd618757c844a54cfc86c0bc8c9beab58..160c5c29c4d0d5c906397dc0a989d59ec0ea9537 100644 --- a/arch/sw_64/kvm/emulate.c +++ b/arch/sw_64/kvm/emulate.c @@ -5,6 +5,7 @@ * linhn */ #include +#include #include #include @@ -12,8 +13,18 @@ void sw64_decode(struct kvm_vcpu *vcpu, unsigned int insn, struct kvm_run *run) { int opc, ra; +#ifdef CONFIG_SUBARCH_C3B opc = (insn >> 26) & 0x3f; ra = (insn >> 21) & 0x1f; +#elif defined(CONFIG_SUBARCH_C4) + unsigned long ds_stat, exc_sum; + + ds_stat = read_csr(CSR_DS_STAT); + exc_sum = read_csr(CSR_EXC_SUM); + + opc = (ds_stat >> 4) & 0x3f; + ra = (exc_sum >> 8) & 0x1f; +#endif switch (opc) { case 0x20: /* LDBU */ diff --git a/arch/sw_64/kvm/handle_exit.c b/arch/sw_64/kvm/handle_exit.c index 5d14f2a22f1fadea8b0879d491ff75e9eaf03e2b..08c7f283c14ea7c9f1184d45a2103444703a0ecb 100644 --- a/arch/sw_64/kvm/handle_exit.c +++ b/arch/sw_64/kvm/handle_exit.c @@ -8,12 +8,14 @@ #include #include #include +#include #include +#include int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, int exception_index, struct hcall_args *hargs) { - gfn_t gfn; + gfn_t gfn __maybe_unused; switch (exception_index) { case SW64_KVM_EXIT_IO: @@ -54,7 +56,7 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, return 1; case SW64_KVM_EXIT_IPI: vcpu->stat.ipi_exits++; - vcpu_send_ipi(vcpu, hargs->arg0); + vcpu_send_ipi(vcpu, hargs->arg0, hargs->arg1); return 1; case SW64_KVM_EXIT_DEBUG: vcpu->stat.debug_exits++; @@ -66,6 +68,10 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, vcpu->stat.memhotplug_exits++; vcpu_mem_hotplug(vcpu, hargs->arg0); return 1; +#endif +#ifdef CONFIG_SUBARCH_C4 + case SW64_KVM_EXIT_APT_FAULT: + return kvm_handle_guest_abort(vcpu, run); #endif case SW64_KVM_EXIT_FATAL_ERROR: vcpu->stat.fatal_error_exits++; diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c new file mode 100644 index 0000000000000000000000000000000000000000..0fab957bfd7b4763b6b088e6aa830d458afea2b4 --- /dev/null +++ b/arch/sw_64/kvm/kvm_core3.c @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 yangzh + * linhn + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include "trace.h" + +#include "../kernel/pci_impl.h" +#include "vmem.c" + + +__read_mostly bool bind_vcpu_enabled; + +#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) +struct dentry *bindvcpu; + +static int __init bind_vcpu_init(void) +{ + if (!sw64_debugfs_dir) + return -ENODEV; + bindvcpu = debugfs_create_bool("bind_vcpu", 0644, + sw64_debugfs_dir, &bind_vcpu_enabled); + if (IS_ERR(bindvcpu)) + return PTR_ERR(bindvcpu); + return 0; +} + +static void bind_vcpu_exit(void) +{ + bind_vcpu_enabled = false; + debugfs_remove(bindvcpu); +} +#else +static int __init bind_vcpu_init(void) +{ + return 0; +} + +static void bind_vcpu_exit(void) { } + +#endif + +static unsigned long longtime_offset; + +#ifdef CONFIG_KVM_MEMHOTPLUG +static unsigned long get_vpcr(struct kvm_vcpu *vcpu, u64 vpn) +{ + unsigned long base; + + base = virt_to_phys(vcpu->kvm->arch.seg_pgd); + return base | ((vpn & VPN_MASK) << 44); +} +#else +static unsigned long get_vpcr(struct kvm_vcpu *vcpu, u64 vpn) +{ + unsigned long base, size; + + base = vcpu->kvm->arch.host_phys_addr; + size = vcpu->kvm->arch.size; + return (base >> 23) | ((size >> 23) << 16) | ((vpn & VPN_MASK) << 44); +} +#endif + +void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.vcb.vpcr == 0) { + vcpu->arch.vcb.vpcr = get_vpcr(vcpu, 0); +#ifndef CONFIG_KVM_MEMHOTPLUG + if (unlikely(bind_vcpu_enabled)) { + int nid; + unsigned long end; + + end = vcpu->kvm->arch.host_phys_addr + vcpu->kvm->arch.size; + nid = pfn_to_nid(PHYS_PFN(vcpu->kvm->arch.host_phys_addr)); + if (pfn_to_nid(PHYS_PFN(end)) == nid) + set_cpus_allowed_ptr(vcpu->arch.tsk, cpumask_of_node(nid)); + } +#endif + vcpu->arch.vcb.upcr = 0x7; + } +} + +void kvm_flush_tlb_all(void) +{ + tbia(); +} + +void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) +{ + vcpu->arch.vcb.vpcr = ((vcpu->arch.vcb.vpcr) & (~(VPN_MASK << 44))) | (vpn << 44); + vcpu->arch.vcb.dtb_vpcr = ((vcpu->arch.vcb.dtb_vpcr) & (~(VPN_MASK << VPN_SHIFT))) | (vpn << VPN_SHIFT); +} + +int kvm_sw64_init_vm(struct kvm *kvm) +{ +#ifdef CONFIG_KVM_MEMHOTPLUG + unsigned long *seg_pgd; + + if (kvm->arch.seg_pgd != NULL) { + kvm_err("kvm_arch already initialized?\n"); + return -EINVAL; + } + + seg_pgd = alloc_pages_exact(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO); + if (!seg_pgd) + return -ENOMEM; + + kvm->arch.seg_pgd = seg_pgd; + #endif + return 0; +} + +void kvm_sw64_destroy_vm(struct kvm *kvm) +{ + int i; + #ifdef CONFIG_KVM_MEMHOTPLUG + void *seg_pgd = NULL; + + if (kvm->arch.seg_pgd) { + seg_pgd = READ_ONCE(kvm->arch.seg_pgd); + kvm->arch.seg_pgd = NULL; + } + + if (seg_pgd) + free_pages_exact(seg_pgd, PAGE_SIZE); + #endif + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + if (kvm->vcpus[i]) + kvm_vcpu_destroy(kvm->vcpus[i]); + } + atomic_set(&kvm->online_vcpus, 0); +} + +#ifdef CONFIG_KVM_MEMHOTPLUG +static void setup_segment_table(struct kvm *kvm, + struct kvm_memory_slot *memslot, unsigned long addr, size_t size) +{ + unsigned long *seg_pgd = kvm->arch.seg_pgd; + unsigned long num_of_entry; + unsigned long base_hpa = addr; + unsigned long i; + + num_of_entry = round_up(size, 1 << 30) >> 30; + + for (i = 0; i < num_of_entry; i++) { + *seg_pgd = base_hpa + (i << 30); + seg_pgd++; + } +} +#endif + +int kvm_arch_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *memslot, + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) +{ + unsigned long addr; + struct file *vm_file; + struct vm_area_struct *vma; + struct vmem_info *info; + unsigned long ret; + size_t size; + + if (change == KVM_MR_FLAGS_ONLY || change == KVM_MR_DELETE) + return 0; + + if (test_bit(IO_MARK_BIT, &(mem->guest_phys_addr))) + return 0; + + if (test_bit(IO_MARK_BIT + 1, &(mem->guest_phys_addr))) + return 0; + +#ifndef CONFIG_KVM_MEMHOTPLUG + if (mem->guest_phys_addr) { + pr_info("%s, No KVM MEMHOTPLUG support!\n", __func__); + return 0; + } +#endif + if (!sw64_kvm_pool) + return -ENOMEM; + + pr_info("%s: %#llx %#llx, user addr: %#llx\n", __func__, + mem->guest_phys_addr, mem->memory_size, mem->userspace_addr); + + vma = find_vma(current->mm, mem->userspace_addr); + if (!vma) + return -ENOMEM; + vm_file = vma->vm_file; + + if (!vm_file) { + info = kzalloc(sizeof(struct vmem_info), GFP_KERNEL); + + size = round_up(mem->memory_size, 8<<20); + addr = gen_pool_alloc(sw64_kvm_pool, size); + if (!addr) + return -ENOMEM; + vm_munmap(mem->userspace_addr, mem->memory_size); + ret = vm_mmap(vm_file, mem->userspace_addr, mem->memory_size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, 0); + if ((long)ret < 0) + return ret; + + vma = find_vma(current->mm, mem->userspace_addr); + if (!vma) + return -ENOMEM; + +#ifdef CONFIG_KVM_MEMHOTPLUG + if (memslot->base_gfn == 0x0UL) { + setup_segment_table(kvm, memslot, addr, size); + kvm->arch.host_phys_addr = (u64)addr; + memslot->arch.host_phys_addr = addr; + } else { + /* used for memory hotplug */ + memslot->arch.host_phys_addr = addr; + memslot->arch.valid = false; + } +#endif + info->start = addr; + info->size = size; + vma->vm_private_data = (void *) info; + + vma->vm_ops = &vmem_vm_ops; + vma->vm_ops->open(vma); + + ret = vmem_vm_insert_page(vma); + if ((int)ret < 0) + return ret; + } else { + info = vm_file->private_data; + addr = info->start; + } + + pr_info("guest phys addr = %#lx, size = %#lx\n", + addr, vma->vm_end - vma->vm_start); +#ifndef CONFIG_KVM_MEMHOTPLUG + kvm->arch.host_phys_addr = (u64)addr; + kvm->arch.size = round_up(mem->memory_size, 8<<20); +#endif + memset(__va(addr), 0, 0x2000000); + + return 0; +} + +/* + * kvm_mark_migration write the mark on every vcpucbs of the kvm, which tells + * the system to do migration while the mark is on, and flush all vcpu's tlbs + * at the beginning of the migration. + */ +void kvm_mark_migration(struct kvm *kvm, int mark) +{ + struct kvm_vcpu *vcpu; + int cpu; + + kvm_for_each_vcpu(cpu, vcpu, kvm) + vcpu->arch.vcb.migration_mark = mark << 2; + + kvm_flush_remote_tlbs(kvm); +} + +void kvm_arch_commit_memory_region(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + /* + * At this point memslot has been committed and there is an + * allocated dirty_bitmap[], dirty pages will be tracked while the + * memory slot is write protected. + */ + + /* If dirty logging has been stopped, do nothing for now. */ + if ((change != KVM_MR_DELETE) && (old->flags & KVM_MEM_LOG_DIRTY_PAGES) + && (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))) { + kvm_mark_migration(kvm, 0); + return; + } + + /* If it's the first time dirty logging, flush all vcpu tlbs. */ + if ((change == KVM_MR_FLAGS_ONLY) && (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES)) + && (new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + kvm_mark_migration(kvm, 1); +} + +int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) +{ + unsigned long addr = vcpu->kvm->arch.host_phys_addr; + + hrtimer_cancel(&vcpu->arch.hrt); + vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; + vcpu->arch.vcb.vcpu_irq_disabled = 1; + vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ + vcpu->arch.power_off = 0; + memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); + + if (vcpu->vcpu_id == 0) + memset(__va(addr), 0, 0x2000000); + + return 0; +} + +long kvm_sw64_get_vcb(struct file *filp, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + + if (vcpu->arch.vcb.migration_mark) { + unsigned long result = sw64_io_read(0, LONG_TIME) + + vcpu->arch.vcb.guest_longtime_offset; + vcpu->arch.vcb.guest_longtime = result; + vcpu->arch.vcb.guest_irqs_pending = vcpu->arch.irqs_pending[0]; + } + + if (copy_to_user((void __user *)arg, &(vcpu->arch.vcb), sizeof(struct vcpucb))) + return -EINVAL; + + return 0; +} + +long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) +{ + unsigned long result; + struct kvm_vcpu *vcpu = filp->private_data; + struct vcpucb *kvm_vcb; + + kvm_vcb = memdup_user((void __user *)arg, sizeof(*kvm_vcb)); + memcpy(&(vcpu->arch.vcb), kvm_vcb, sizeof(struct vcpucb)); + + if (vcpu->arch.vcb.migration_mark) { + /* updated vpcr needed by destination vm */ + vcpu->arch.vcb.vpcr = get_vpcr(vcpu, 0); + /* synchronize the longtime of source and destination */ + if (vcpu->arch.vcb.soft_cid == 0) { + result = sw64_io_read(0, LONG_TIME); + vcpu->arch.vcb.guest_longtime_offset = vcpu->arch.vcb.guest_longtime - result; + longtime_offset = vcpu->arch.vcb.guest_longtime_offset; + } else + vcpu->arch.vcb.guest_longtime_offset = longtime_offset; + + set_timer(vcpu, 200000000); + vcpu->arch.vcb.migration_mark = 0; + } + + return 0; +} + +#ifdef CONFIG_KVM_MEMHOTPLUG +void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr) +{ + struct kvm *kvm = vcpu->kvm; + struct kvm_memory_slot *slot; + unsigned long start_pfn = start_addr >> PAGE_SHIFT; + + kvm_for_each_memslot(slot, kvm_memslots(kvm)) { + if (start_pfn == slot->base_gfn) { + unsigned long *seg_pgd; + unsigned long num_of_entry = slot->npages >> 17; + unsigned long base_hpa = slot->arch.host_phys_addr; + unsigned long i; + + seg_pgd = kvm->arch.seg_pgd + (start_pfn >> 17); + for (i = 0; i < num_of_entry; i++) { + *seg_pgd = base_hpa + (i << 30); + seg_pgd++; + } + } + } +} +#endif + +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) +{ +} + +void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, gfn_t gfn_offset, + unsigned long mask) +{ +} + +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) +{ +} + +void kvm_arch_flush_shadow_all(struct kvm *kvm) +{ +} + +void update_aptp(unsigned long pgd) +{ +} + +static int __init kvm_core3_init(void) +{ + int i, ret; + + bind_vcpu_init(); + + ret = vmem_init(); + if (unlikely(ret)) + goto out; + + for (i = 0; i < NR_CPUS; i++) + last_vpn(i) = VPN_FIRST_VERSION; + + ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); + + if (likely(!ret)) + return 0; + + vmem_exit(); +out: + bind_vcpu_exit(); + return ret; +} + +static void __exit kvm_core3_exit(void) +{ + kvm_exit(); + vmem_exit(); + bind_vcpu_exit(); +} + +module_init(kvm_core3_init); +module_exit(kvm_core3_exit); diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c new file mode 100644 index 0000000000000000000000000000000000000000..c1bc9c8f859510fd88905c4e20a05edca75f9328 --- /dev/null +++ b/arch/sw_64/kvm/kvm_core4.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 yangzh + * linhn + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "trace.h" + +#include "../kernel/pci_impl.h" + +static unsigned long shtclock_offset; + +void update_aptp(unsigned long pgd) +{ + imemb(); + write_csr_imb(pgd, CSR_APTP); +} + +void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) +{ + vcpu->arch.vcb.vpcr = vpn << 44; + vcpu->arch.vcb.dtb_vpcr = vpn; +} + +void kvm_flush_tlb_all(void) +{ + tbivpn(-1, 0, 0); +} + +int kvm_sw64_init_vm(struct kvm *kvm) +{ + return kvm_alloc_addtional_stage_pgd(kvm); +} + +void kvm_sw64_destroy_vm(struct kvm *kvm) +{ + int i; + + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + if (kvm->vcpus[i]) { + kvm_arch_vcpu_free(kvm->vcpus[i]); + kvm->vcpus[i] = NULL; + } + } + atomic_set(&kvm->online_vcpus, 0); +} + +int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.has_run_once) + apt_unmap_vm(vcpu->kvm); + + hrtimer_cancel(&vcpu->arch.hrt); + vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; + vcpu->arch.vcb.vcpu_irq_disabled = 1; + vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ + vcpu->arch.power_off = 0; + memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); + + return 0; +} + +long kvm_sw64_get_vcb(struct file *filp, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + + if (vcpu->arch.migration_mark) + vcpu->arch.shtclock = read_csr(CSR_SHTCLOCK) + + vcpu->arch.vcb.shtclock_offset; + if (copy_to_user((void __user *)arg, &(vcpu->arch.vcb), sizeof(struct vcpucb))) + return -EINVAL; + + return 0; +} + +long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + struct vcpucb *kvm_vcb; + + kvm_vcb = memdup_user((void __user *)arg, sizeof(*kvm_vcb)); + memcpy(&(vcpu->arch.vcb), kvm_vcb, sizeof(struct vcpucb)); + + if (vcpu->arch.migration_mark) { + /* synchronize the longtime of source and destination */ + if (vcpu->arch.vcb.soft_cid == 0) + shtclock_offset = vcpu->arch.shtclock - read_csr(CSR_SHTCLOCK); + vcpu->arch.vcb.shtclock_offset = shtclock_offset; + set_timer(vcpu, 200000000); + vcpu->arch.migration_mark = 0; + } + return 0; +} + +int kvm_arch_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *memslot, + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) +{ + return 0; +} + +void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu) +{ + return; +} + +static int __init kvm_core4_init(void) +{ + int i, ret; + + for (i = 0; i < NR_CPUS; i++) + last_vpn(i) = VPN_FIRST_VERSION; + + ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); + + if (ret) + return ret; + + return 0; +} + +static void __exit kvm_core4_exit(void) +{ + kvm_exit(); +} + +module_init(kvm_core4_init); +module_exit(kvm_core4_exit); diff --git a/arch/sw_64/kvm/mmio.c b/arch/sw_64/kvm/mmio.c index fe6ae6f5ed5c497b6d7daef7cbe39fc250498fe8..21ad89722f9ae12f2246f88625966a718058c250 100644 --- a/arch/sw_64/kvm/mmio.c +++ b/arch/sw_64/kvm/mmio.c @@ -7,6 +7,7 @@ #include #include #include +#include static unsigned long mmio_read_buf(char *buf, unsigned int len) { @@ -63,8 +64,13 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, { int ret; +#ifdef CONFIG_SUBARCH_C3B run->mmio.phys_addr = hargs->arg1 & 0xfffffffffffffUL; sw64_decode(vcpu, hargs->arg2, run); +#elif defined(CONFIG_SUBARCH_C4) + run->mmio.phys_addr = read_csr(CSR_DVA) & 0xfffffffffffffUL; + sw64_decode(vcpu, 0, run); +#endif if (run->mmio.is_write) ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, run->mmio.len, run->mmio.data); diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c new file mode 100644 index 0000000000000000000000000000000000000000..a0b1db9244404c6a7c804922cd8acb994229a0a8 --- /dev/null +++ b/arch/sw_64/kvm/mmu.c @@ -0,0 +1,1561 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 - os kernal + * Author: lff + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#define KVM_APT_FLAG_LOGGING_ACTIVE (1UL << 1) + +static bool memslot_is_logging(struct kvm_memory_slot *memslot) +{ + return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); +} + +/* + * Return values of kvm_handle_mmio_page_fault and mmu.page_fault: + * RET_AF_RETRY: let CPU fault again on the address. + * RET_AF_EMULATE: mmio page fault, emulate the instruction directly. + * + * For kvm_handle_mmio_page_fault only: + * RET_AF_INVALID: the spte is invalid, let the real page fault path update it. + */ +enum { + RET_AF_RETRY = 0, + RET_AF_EMULATE = 1, + RET_AF_INVALID = 2, +}; + +/** + * apt_dissolve_pmd() - clear and flush huge PMD entry + * @kvm: pointer to kvm structure. + * @addr: IPA + * @pmd: pmd pointer for IPA + * + * Function clears a PMD entry, flushes TLBs. + */ +static void apt_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd) +{ + int i; + + if (!pmd_trans_huge(*pmd)) + return; + + if (pmd_trans_cont(*pmd)) { + for (i = 0; i < CONT_PMDS; i++, pmd++) + pmd_clear(pmd); + } else + pmd_clear(pmd); + + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pmd)); +} + +/** + * apt_dissolve_pud() - clear and flush huge PUD entry + * @kvm: pointer to kvm structure. + * @addr: IPA + * @pud: pud pointer for IPA + * + * Function clears a PUD entry, flushes TLBs. + */ +static void apt_dissolve_pud(struct kvm *kvm, phys_addr_t addr, pud_t *pudp) +{ + if (!pud_huge(*pudp)) + return; + + pud_clear(pudp); + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pudp)); +} + +static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, + int min, int max) +{ + void *page; + + BUG_ON(max > KVM_NR_MEM_OBJS); + if (cache->nobjs >= min) + return 0; + while (cache->nobjs < max) { + page = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return -ENOMEM; + cache->objects[cache->nobjs++] = page; + } + return 0; +} + +static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) +{ + while (mc->nobjs) + free_page((unsigned long)mc->objects[--mc->nobjs]); +} + +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) +{ + mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); +} + +static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) +{ + void *p; + + BUG_ON(!mc || !mc->nobjs); + p = mc->objects[--mc->nobjs]; + return p; +} + +static void unmap_apt_ptes(struct kvm *kvm, pmd_t *pmd, + phys_addr_t addr, phys_addr_t end) +{ + pte_t *pte, *start_pte; + struct page *ptr_page; + + start_pte = pte = pte_offset_kernel(pmd, addr); + do { + if (!pte_none(*pte)) { + /* Do we need WRITE_ONCE(pte, 0)? */ + set_pte(pte, __pte(0)); + put_page(virt_to_page(pte)); + } + } while (pte++, addr += PAGE_SIZE, addr != end); + + ptr_page = virt_to_page(start_pte); + if (page_count(ptr_page) == 1) { + pte_t *pte_table = pte_offset_kernel(pmd, 0); + + pmd_clear(pmd); + free_page((unsigned long)pte_table); + put_page(virt_to_page(pmd)); + } +} + +static void unmap_apt_pmds(struct kvm *kvm, pud_t *pud, + phys_addr_t addr, phys_addr_t end) +{ + phys_addr_t next; + pmd_t *pmd, *start_pmd; + struct page *ptr_page; + int i; + + start_pmd = pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (!pmd_none(*pmd)) { + if (pmd_trans_huge(*pmd)) { + if (pmd_trans_cont(*pmd)) { + for (i = 0; i < CONT_PMDS; i++, pmd++) + pmd_clear(pmd); + } else + pmd_clear(pmd); + /* Do we need flush tlb???? edited by lff */ + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pmd)); + } else { + unmap_apt_ptes(kvm, pmd, addr, next); + } + } + } while (pmd++, addr = next, addr != end); + + ptr_page = virt_to_page(start_pmd); + if (page_count(ptr_page) == 1) { + pmd_t *pmd_table __maybe_unused = pmd_offset(pud, 0UL); + pud_clear(pud); + free_page((unsigned long)pmd_table); + put_page(virt_to_page(pud)); + } +} + +static void unmap_apt_puds(struct kvm *kvm, p4d_t *p4d, + phys_addr_t addr, phys_addr_t end) +{ + phys_addr_t next; + pud_t *pud, *start_pud; + struct page *ptr_page; + + start_pud = pud = pud_offset(p4d, addr); + do { + next = pud_addr_end(addr, end); + if (!pud_none(*pud)) { + if (pud_huge(*pud)) { + pud_clear(pud); + /* Do we need flush tlb???? edited by lff */ + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pud)); + } else { + unmap_apt_pmds(kvm, pud, addr, next); + } + } + } while (pud++, addr = next, addr != end); + + ptr_page = virt_to_page(start_pud); + if (page_count(ptr_page) == 1) { + pud_t *pud_table __maybe_unused = pud_offset(p4d, 0UL); + p4d_clear(p4d); + kvm_flush_remote_tlbs(kvm); + free_page((unsigned long)pud_table); + put_page(virt_to_page(p4d)); + } +} + +/** + * unmap_apt_range -- Clear addtional page table entries to unmap a range + * @kvm: The VM pointer + * @start: The intermediate physical base address of the range to unmap + * @size: The size of the area to unmap + * + * Clear a range of apt mappings, lowering the various ref-counts. Must + * be called while holding mmu_lock (unless for freeing the apt pgd before + * destroying the VM), otherwise another faulting VCPU may come in and mess + * with things behind our backs. + */ +static void unmap_apt_range(struct kvm *kvm, phys_addr_t start, u64 size) +{ + pgd_t *pgd; + p4d_t *p4d; + phys_addr_t addr = start, end = start + size; + phys_addr_t next; + + assert_spin_locked(&kvm->mmu_lock); + WARN_ON(size & ~PAGE_MASK); + + pgd = kvm->arch.pgd + pgd_index(addr); + p4d = p4d_offset(pgd, addr); + do { + /* + * Make sure the page table is still active, as another thread + * could have possibly freed the page table, while we released + * the lock. + */ + if (!READ_ONCE(kvm->arch.pgd)) + break; + next = p4d_addr_end(addr, end); + if (!p4d_none(*p4d)) + unmap_apt_puds(kvm, p4d, addr, next); + /* + * If the range is too large, release the kvm->mmu_lock + * to prevent starvation and lockup detector warnings. + */ + if (next != end) + cond_resched_lock(&kvm->mmu_lock); + } while (pgd++, addr = next, addr != end); +} + +static void apt_unmap_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) +{ + hva_t hva = memslot->userspace_addr; + phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT; + phys_addr_t size = PAGE_SIZE * memslot->npages; + hva_t reg_end = hva + size; + + /* + * A memory region could potentially cover multiple VMAs, and any holes + * between them, so iterate over all of them to find out if we should + * unmap any of them. + * + * +--------------------------------------------+ + * +---------------+----------------+ +----------------+ + * | : VMA 1 | VMA 2 | | VMA 3 : | + * +---------------+----------------+ +----------------+ + * | memory region | + * +--------------------------------------------+ + */ + do { + struct vm_area_struct *vma = find_vma(current->mm, hva); + hva_t vm_start, vm_end; + + if (!vma || vma->vm_start >= reg_end) + break; + + /* + * Take the intersection of this VMA with the memory region + */ + vm_start = max(hva, vma->vm_start); + vm_end = min(reg_end, vma->vm_end); + + if (!(vma->vm_flags & VM_PFNMAP)) { + gpa_t gpa = addr + (vm_start - memslot->userspace_addr); + + unmap_apt_range(kvm, gpa, vm_end - vm_start); + } + hva = vm_end; + } while (hva < reg_end); +} + +/** + * apt_unmap_vm - Unmap Additional Stage RAM mappings + * @kvm: The struct kvm pointer + * + * Go through the memregions and unmap any reguler RAM + * backing memory already mapped to the VM. + */ +void apt_unmap_vm(struct kvm *kvm) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *memslot; + int idx; + + idx = srcu_read_lock(&kvm->srcu); + down_read(¤t->mm->mmap_lock); + spin_lock(&kvm->mmu_lock); + + slots = kvm_memslots(kvm); + kvm_for_each_memslot(memslot, slots) + apt_unmap_memslot(kvm, memslot); + spin_unlock(&kvm->mmu_lock); + up_read(¤t->mm->mmap_lock); + srcu_read_unlock(&kvm->srcu, idx); +} + +static pud_t *apt_get_pud(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr) +{ + p4d_t *p4d; + pud_t *pud; + + pgd += pgd_index(addr); + if (pgd_none(*pgd)) { + /* Not used on SW64 yet */ + VM_BUG_ON(pgd); + return NULL; + } + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) { + if (!cache) + return NULL; + pud = mmu_memory_cache_alloc(cache); + p4d_populate(NULL, p4d, pud); + get_page(virt_to_page(p4d)); + } + return pud_offset(p4d, addr); +} + +static pmd_t *apt_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, unsigned long sz) +{ + pud_t *pud; + pmd_t *pmd; + + pud = apt_get_pud(kvm->arch.pgd, cache, addr); + if (!pud || pud_huge(*pud)) + return NULL; + + if (pud_none(*pud)) { + if (!cache) + return NULL; + pmd = mmu_memory_cache_alloc(cache); + pud_populate(NULL, pud, pmd); + get_page(virt_to_page(pud)); + } + if (sz == CONT_PMD_SIZE) + addr &= CONT_PMD_MASK; + + return pmd_offset(pud, addr); +} + +static bool kvm_is_write_fault(unsigned long access_type) +{ + if (access_type == AF_WRITE_ACCESS_TYPE) + return true; + + return false; +} + +static bool kvm_is_exec_fault(unsigned long access_type) +{ + if (access_type == AF_EXEC_ACCESS_TYPE) + return true; + + return false; +} +/** + * apt_wp_ptes - write protect PMD range + * @pmd: pointer to pmd entry + * @addr: range start address + * @end: range end address + */ +static void apt_wp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end) +{ + pte_t *pte; + + pte = pte_offset_kernel(pmd, addr); + do { + if (!pte_none(*pte)) { + if (!kvm_aptpte_readonly(pte)) + kvm_set_aptpte_readonly(pte); + } + } while (pte++, addr += PAGE_SIZE, addr != end); +} + +/** + * apt_wp_pmds - write protect PUD range + * @pud: pointer to pud entry + * @addr: range start address + * @end: range end address + */ +static void apt_wp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end) +{ + pmd_t *pmd; + phys_addr_t next; + + pmd = pmd_offset(pud, addr); + + do { + next = pmd_addr_end(addr, end); + if (!pmd_none(*pmd)) { + if (pmd_trans_huge(*pmd)) { + if (!kvm_aptpmd_readonly(pmd)) + kvm_set_aptpmd_readonly(pmd); + } else { + apt_wp_ptes(pmd, addr, next); + } + } + } while (pmd++, addr = next, addr != end); +} + +/** + * apt_wp_puds - write protect PGD range + * @pgd: pointer to pgd entry + * @addr: range start address + * @end: range end address + * + * Process PUD entries, for a huge PUD we cause a panic. + */ +static void apt_wp_puds(p4d_t *p4d, phys_addr_t addr, phys_addr_t end) +{ + pud_t *pud; + phys_addr_t next; + + pud = pud_offset(p4d, addr); + do { + next = pud_addr_end(addr, end); + if (!pud_none(*pud)) { + if (pud_huge(*pud)) { + if (!kvm_aptpud_readonly(pud)) + kvm_set_aptpud_readonly(pud); + } else { + /* TODO:PUD not supported, revisit later if supported */ +// BUG_ON(pud_trans_huge(*pud)); + apt_wp_pmds(pud, addr, next); + } + } + } while (pud++, addr = next, addr != end); +} + +/** + * apt_wp_range() - write protect apt memory region range + * @kvm: The KVM pointer + * @addr: Start address of range + * @end: End address of range + */ +static void apt_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) +{ + pgd_t *pgd; + p4d_t *p4d; + phys_addr_t next; + + pgd = kvm->arch.pgd + pgd_index(addr); + p4d = p4d_offset(pgd, addr); + + do { + /* + * Release kvm_mmu_lock periodically if the memory region is + * large. Otherwise, we may see kernel panics with + * CONFIG_DETECT_HUNG_TASK, CONFIG_LOCKUP_DETECTOR, + * CONFIG_LOCKDEP. Additionally, holding the lock too long + * will also starve other vCPUs. We have to also make sure + * that the page tables are not freed while we released + * the lock. + */ + cond_resched_lock(&kvm->mmu_lock); + if (!READ_ONCE(kvm->arch.pgd)) + break; + next = p4d_addr_end(addr, end); + if (p4d_present(*p4d)) + apt_wp_puds(p4d, addr, next); + } while (p4d++, addr = next, addr != end); +} + +/** + * kvm_mmu_wp_memory_region() - write protect apt entries for memory slot + * @kvm: The KVM pointer + * @slot: The memory slot to write protect + * + * Called to start logging dirty pages after memory region + * KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns + * all present PMD and PTEs are write protected in the memory region. + * Afterwards read of dirty page log can be called. + * + * Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired, + * serializing operations for VM memory regions. + */ +void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot) +{ + struct kvm_memslots *slots = kvm_memslots(kvm); + struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); + phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; + phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; + + spin_lock(&kvm->mmu_lock); + apt_wp_range(kvm, start, end); + spin_unlock(&kvm->mmu_lock); + kvm_flush_remote_tlbs(kvm); // 需要通知其他vcpu进行tlb刷新，利用request机制 +} + +void kvm_mark_migration(struct kvm *kvm, int mark) +{ + struct kvm_vcpu *vcpu; + int cpu; + + kvm_for_each_vcpu(cpu, vcpu, kvm) + vcpu->arch.migration_mark = mark; +} + +void kvm_arch_commit_memory_region(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + /* + * There is no need to do this in any of the following cases: + * CREATE: No dirty mappings will already exist. + * MOVE/DELETE: The old mappings will already have been cleaned up by + * kvm_arch_flush_shadow_memslot() + */ + if (change == KVM_MR_FLAGS_ONLY && (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES) && + new->flags & KVM_MEM_LOG_DIRTY_PAGES)) { + kvm_mark_migration(kvm, 1); + kvm_mmu_wp_memory_region(kvm, mem->slot); + } + /* If dirty logging has been stopped, do nothing for now. */ + if ((change != KVM_MR_DELETE) + && (old->flags & KVM_MEM_LOG_DIRTY_PAGES) + && (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))) { + kvm_mark_migration(kvm, 0); + return; + } +} + +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) +{ + gpa_t gpa = slot->base_gfn << PAGE_SHIFT; + phys_addr_t size = slot->npages << PAGE_SHIFT; + + spin_lock(&kvm->mmu_lock); +// flush_apt_tlbs(kvm); + unmap_apt_range(kvm, gpa, size); + spin_unlock(&kvm->mmu_lock); +} + +/** + * kvm_alloc_addtional_stage_pgd - allocate level-1 table for addtional stage translation. + * @kvm: The KVM struct pointer for the VM. + * + * Allocates only the addtional stage HW PGD level table(s) (can support full + * 48-bit input addresses). Clears the allocated pages. + * + * Note we don't need locking here as this is only called when the VM is + * created, which can only be done once. + */ +int kvm_alloc_addtional_stage_pgd(struct kvm *kvm) +{ + pgd_t *pgd; + + if (kvm->arch.pgd != NULL) { + kvm_err("kvm_arch already initialized?\n"); + return -EINVAL; + } + + /* Allocate the HW PGD, making sure that each page gets its own refcount */ + pgd = alloc_pages_exact(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO); + if (!pgd) + return -ENOMEM; + + kvm->arch.pgd = pgd; + return 0; +} + +/** + * kvm_free_apt_pgd - free all apt tables + * @kvm: The KVM struct pointer for the VM. + * + * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all + * underlying level-2 and level-3 tables before freeing the actual level-1 table + * and setting the struct pointer to NULL. + */ +void kvm_free_apt_pgd(struct kvm *kvm) +{ + void *pgd = NULL; + + spin_lock(&kvm->mmu_lock); + if (kvm->arch.pgd) { + unmap_apt_range(kvm, 0, KVM_PHYS_SIZE); + pgd = READ_ONCE(kvm->arch.pgd); + kvm->arch.pgd = NULL; + } + spin_unlock(&kvm->mmu_lock); + + /* Free the HW pgd, one page at a time */ + if (pgd) + free_pages_exact(pgd, PAGE_SIZE); +} + +void kvm_arch_flush_shadow_all(struct kvm *kvm) +{ + kvm_free_apt_pgd(kvm); +} + +static void kvm_send_hwpoison_signal(unsigned long address, + struct vm_area_struct *vma) +{ + kernel_siginfo_t info; + + clear_siginfo(&info); + info.si_signo = SIGBUS; + info.si_errno = 0; + info.si_code = BUS_MCEERR_AR; + info.si_addr = (void __user *)address; + + if (is_vm_hugetlb_page(vma)) + info.si_addr_lsb = huge_page_shift(hstate_vma(vma)); + else + info.si_addr_lsb = PAGE_SHIFT; + + send_sig_info(SIGBUS, &info, current); +} + +static bool fault_supports_apt_huge_mapping(struct kvm_memory_slot *memslot, + unsigned long hva, + unsigned long map_size) +{ + gpa_t gpa_start; + hva_t uaddr_start, uaddr_end; + size_t size; + + /* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */ + if (map_size == PAGE_SIZE) + return true; + + size = memslot->npages * PAGE_SIZE; + + gpa_start = memslot->base_gfn << PAGE_SHIFT; + + uaddr_start = memslot->userspace_addr; + uaddr_end = uaddr_start + size; + + /* + * Pages belonging to memslots that don't have the same alignment + * within a PMD/PUD for userspace and IPA cannot be mapped with stage-2 + * PMD/PUD entries, because we'll end up mapping the wrong pages. + * + * Consider a layout like the following: + * + * memslot->userspace_addr: + * +-----+--------------------+--------------------+---+ + * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz| + * +-----+--------------------+--------------------+---+ + * + * memslot->base_gfn << PAGE_SHIFT: + * +---+--------------------+--------------------+-----+ + * |abc|def Stage-2 block | Stage-2 block |tvxyz| + * +---+--------------------+--------------------+-----+ + * + * If we create those stage-2 blocks, we'll end up with this incorrect + * mapping: + * d -> f + * e -> g + * f -> h + */ + if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1))) + return false; + + /* + * Next, let's make sure we're not trying to map anything not covered + * by the memslot. This means we have to prohibit block size mappings + * for the beginning and end of a non-block aligned and non-block sized + * memory slot (illustrated by the head and tail parts of the + * userspace view above containing pages 'abcde' and 'xyz', + * respectively). + * + * Note that it doesn't matter if we do the check using the + * userspace_addr or the base_gfn, as both are equally aligned (per + * the check above) and equally sized. + */ + return (hva & ~(map_size - 1)) >= uaddr_start && + (hva & ~(map_size - 1)) + map_size <= uaddr_end; +} + +/* + * apt_get_leaf_entry - walk the stage2 VM page tables and return + * true if a valid and present leaf-entry is found. A pointer to the + * leaf-entry is returned in the appropriate level variable - pudpp, + * pmdpp, ptepp. + */ +static bool apt_get_leaf_entry(struct kvm *kvm, phys_addr_t addr, + pud_t **pudpp, pmd_t **pmdpp, pte_t **ptepp) +{ + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + + *pudpp = NULL; + *pmdpp = NULL; + *ptepp = NULL; + + pudp = apt_get_pud(kvm->arch.pgd, NULL, addr); + if (!pudp || pud_none(*pudp) || !pud_present(*pudp)) + return false; + + if (pud_huge(*pudp)) { + *pudpp = pudp; + return true; + } + + pmdp = pmd_offset(pudp, addr); + if (!pmdp || pmd_none(*pmdp) || !pmd_present(*pmdp)) + return false; + + if (pmd_trans_huge(*pmdp)) { + *pmdpp = pmdp; + return true; + } + + ptep = pte_offset_kernel(pmdp, addr); + if (!ptep || pte_none(*ptep) || !pte_present(*ptep)) + return false; + + *ptepp = ptep; + return true; +} + +static bool apt_is_exec(struct kvm *kvm, phys_addr_t addr) +{ + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + bool found; + + found = apt_get_leaf_entry(kvm, addr, &pudp, &pmdp, &ptep); + if (!found) + return false; + + if (pudp) + return kvm_pud_exec(pudp); + else if (pmdp) + return kvm_pmd_exec(pmdp); + else + return kvm_pte_exec(ptep); +} + +static int apt_set_pte_fast(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, const pte_t *new_pte, + unsigned long flags) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte, old_pte; + bool logging_active = flags & KVM_APT_FLAG_LOGGING_ACTIVE; + int inv_level = ((read_csr(CSR_AS_INFO)) >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; + unsigned long inv_hpa = read_csr(CSR_AS_INFO) & AF_ENTRY_ADDR_MASK; + + VM_BUG_ON(logging_active && !cache); + + if (inv_level == 1) { + pud = (pud_t *)(inv_hpa | PAGE_OFFSET); + goto find_pud; + } else if (inv_level == 2) { + pmd = (pmd_t *)(inv_hpa | PAGE_OFFSET); + goto find_pmd; + } else if (inv_level == 3) { + pte = (pte_t *)(inv_hpa | PAGE_OFFSET); + goto find_pte; + } + + /* Create addtional page table mapping - Levels 0 and 1 */ + pud = apt_get_pud(kvm->arch.pgd, cache, addr); + if (!pud) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PUD, then continue + * on to allocate page. + */ + if (logging_active) + apt_dissolve_pud(kvm, addr, pud); + +find_pud: + if (pud_none(*pud)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pmd = mmu_memory_cache_alloc(cache); + pud_populate(NULL, pud, pmd); + get_page(virt_to_page(pud)); + } + + pmd = pmd_offset(pud, addr); + if (!pmd) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PMD, then continue on to + * allocate page. + */ + if (logging_active) + apt_dissolve_pmd(kvm, addr, pmd); + +find_pmd: + /* Create stage-2 page mappings - Level 2 */ + if (pmd_none(*pmd)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pte = mmu_memory_cache_alloc(cache); + pmd_populate_kernel(NULL, pmd, pte); + get_page(virt_to_page(pmd)); + } + + pte = pte_offset_kernel(pmd, addr); + +find_pte: + /* Create 2nd stage page table mapping - Level 3 */ + old_pte = *pte; + + /* new pte should be readonly? */ +// *new_pte = pte_wrprotect(*new_pte); + + if (pte_present(old_pte)) { + /* Skip page table update if there is no change */ + if (pte_val(old_pte) == pte_val(*new_pte)) + return 0; + + /* Do we need WRITE_ONCE(pte, 0)? */ + set_pte(pte, __pte(0)); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pte)); + } + + /* Do we need WRITE_ONCE(pte, new_pte)? */ + set_pte(pte, *new_pte); + return 0; +} + +static int apt_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, const pte_t *new_pte, + unsigned long flags) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte, old_pte; + bool logging_active = flags & KVM_APT_FLAG_LOGGING_ACTIVE; + + VM_BUG_ON(logging_active && !cache); + + /* Create addtional page table mapping - Levels 0 and 1 */ + pud = apt_get_pud(kvm->arch.pgd, cache, addr); + if (!pud) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PUD, then continue + * on to allocate page. + */ + if (logging_active) + apt_dissolve_pud(kvm, addr, pud); + + if (pud_none(*pud)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pmd = mmu_memory_cache_alloc(cache); + pud_populate(NULL, pud, pmd); + get_page(virt_to_page(pud)); + } + + pmd = pmd_offset(pud, addr); + if (!pmd) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PMD, then continue on to + * allocate page. + */ + if (logging_active) + apt_dissolve_pmd(kvm, addr, pmd); + + /* Create stage-2 page mappings - Level 2 */ + if (pmd_none(*pmd)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pte = mmu_memory_cache_alloc(cache); + pmd_populate_kernel(NULL, pmd, pte); + get_page(virt_to_page(pmd)); + } + + pte = pte_offset_kernel(pmd, addr); + + /* Create 2nd stage page table mapping - Level 3 */ + old_pte = *pte; + + /* new pte should be readonly? */ +// *new_pte = pte_wrprotect(*new_pte); + + if (pte_present(old_pte)) { + /* Skip page table update if there is no change */ + if (pte_val(old_pte) == pte_val(*new_pte)) + return 0; + + /* Do we need WRITE_ONCE(pte, 0)? */ + set_pte(pte, __pte(0)); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pte)); + } + + /* Do we need WRITE_ONCE(pte, new_pte)? */ + set_pte(pte, *new_pte); + return 0; +} + + + +static int apt_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache + *cache, phys_addr_t addr, const pmd_t *new_pmd, unsigned long sz) +{ + pmd_t *pmd, old_pmd, *ori_pmd; + int i; +retry: + pmd = apt_get_pmd(kvm, cache, addr, sz); + VM_BUG_ON(!pmd); + ori_pmd = pmd; + old_pmd = *pmd; + if (pmd_present(old_pmd)) { + /* + * If we already have PTE level mapping for this block, + * we must unmap it to avoid inconsistent TLB state and + * leaking the table page. We could end up in this situation + * if the memory slot was marked for dirty logging and was + * reverted, leaving PTE level mappings for the pages accessed + * during the period. So, unmap the PTE level mapping for this + * block and retry, as we could have released the upper level + * table in the process. + * + * Normal THP split/merge follows mmu_notifier callbacks and do + * get handled accordingly. + */ + if (!pmd_trans_huge(old_pmd)) { + unmap_apt_range(kvm, addr & PMD_MASK, PMD_SIZE); + goto retry; + } + /* + * Multiple vcpus faulting on the same PMD entry, can + * lead to them sequentially updating the PMD with the + * same value. Following the break-before-make + * (pmd_clear() followed by tlb_flush()) process can + * hinder forward progress due to refaults generated + * on missing translations. + * + * Skip updating the page table if the entry is + * unchanged. + */ + if (pmd_val(old_pmd) == pmd_val(*new_pmd)) + return 0; + + /* + * Mapping in huge pages should only happen through a + * fault. If a page is merged into a transparent huge + * page, the individual subpages of that huge page + * should be unmapped through MMU notifiers before we + * get here. + * + * Merging of CompoundPages is not supported; they + * should become splitting first, unmapped, merged, + * and mapped back in on-demand. + */ + VM_BUG_ON(pmd_pfn(old_pmd) != pmd_pfn(*new_pmd)); + + if (sz == CONT_PMD_SIZE) { + for (i = 0; i < CONT_PMDS; i++, pmd++) + pmd_clear(pmd); + } else + pmd_clear(pmd); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pmd)); + } + + /* Do we need WRITE_ONCE(pmd, new_pmd)? */ + if (sz == CONT_PMD_SIZE) { + for (i = 0; i < CONT_PMDS; i++, ori_pmd++) + set_pmd(ori_pmd, *new_pmd); + } else + set_pmd(pmd, *new_pmd); + return 0; +} + +static int apt_set_pud_huge(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, const pud_t *new_pudp) +{ + pud_t *pudp, old_pud; + +retry: + pudp = apt_get_pud(kvm->arch.pgd, cache, addr); + VM_BUG_ON(!pudp); + + old_pud = *pudp; + + /* + * A large number of vcpus faulting on the same stage 2 entry, + * can lead to a refault due to the stage2_pud_clear()/tlb_flush(). + * Skip updating the page tables if there is no change. + */ + if (pud_val(old_pud) == pud_val(*new_pudp)) + return 0; + + if (pud_present(old_pud)) { + /* + * If we already have table level mapping for this block, unmap + * the range for this block and retry. + */ + if (!pud_huge(old_pud)) { + unmap_apt_range(kvm, addr & PUD_MASK, PUD_SIZE); + goto retry; + } + +// WARN_ON_ONCE(kvm_pud_pfn(old_pud) != kvm_pud_pfn(*new_pudp)); + pud_clear(pudp); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pudp)); + } + + set_pud(pudp, *new_pudp); + return 0; +} + +static unsigned long +transparent_hugepage_adjust(struct kvm_memory_slot *memslot, + unsigned long hva, kvm_pfn_t *pfnp, + phys_addr_t *gpap) +{ + kvm_pfn_t pfn = *pfnp; + struct page *page = pfn_to_page(pfn); + + /* + * Make sure the adjustment is done only for THP pages. Also make + * sure that the HVA and IPA are sufficiently aligned and that the + * block map is contained within the memslot. + */ + if (!PageHuge(page) && PageTransCompoundMap(page) && + fault_supports_apt_huge_mapping(memslot, hva, PMD_SIZE)) { + /* + * The address we faulted on is backed by a transparent huge + * page. However, because we map the compound huge page and + * not the individual tail page, we need to transfer the + * refcount to the head page. We have to be careful that the + * THP doesn't start to split while we are adjusting the + * refcounts. + * + * We are sure this doesn't happen, because mmu_notifier_retry + * was successful and we are holding the mmu_lock, so if this + * THP is trying to split, it will be blocked in the mmu + * notifier before touching any of the pages, specifically + * before being able to call __split_huge_page_refcount(). + * + * We can therefore safely transfer the refcount from PG_tail + * to PG_head and switch the pfn from a tail page to the head + * page accordingly. + */ + *gpap &= PMD_MASK; + kvm_release_pfn_clean(pfn); + pfn &= ~(PTRS_PER_PMD - 1); + kvm_get_pfn(pfn); + *pfnp = pfn; + return PMD_SIZE; + } + + return PAGE_SIZE; +} + +static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_gpa, + struct kvm_memory_slot *memslot, unsigned long hva, + unsigned long fault_status) +{ + int ret; + bool write_fault, exec_fault, writable, force_pte = false; + unsigned long mmu_seq; + gfn_t gfn = fault_gpa >> PAGE_SHIFT; + struct kvm *kvm = vcpu->kvm; + struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; + struct vm_area_struct *vma; + kvm_pfn_t pfn; + pgprot_t mem_type = PAGE_READONLY; + bool logging_active = memslot_is_logging(memslot); + unsigned long vma_pagesize, flags = 0; + unsigned long as_info, access_type; + unsigned int vma_shift; + + as_info = read_csr(CSR_AS_INFO); + access_type = (as_info >> AF_ACCESS_TYPE_SHIFT) & AF_ACCESS_TYPE_MASK; + write_fault = kvm_is_write_fault(access_type); + exec_fault = kvm_is_exec_fault(access_type); + VM_BUG_ON(write_fault && exec_fault); + + if (fault_status == AF_STATUS_FOR) { + kvm_err("Unexpected APT read permission error\n"); + return -EFAULT; + } + + /* Let's check if we will get back a huge page backed by hugetlbfs */ + down_read(¤t->mm->mmap_lock); + vma = find_vma_intersection(current->mm, hva, hva + 1); + if (unlikely(!vma)) { + kvm_err("Failed to find VMA for hva 0x%lx\n", hva); + up_read(¤t->mm->mmap_lock); + return -EFAULT; + } + + if (is_vm_hugetlb_page(vma)) + vma_shift = huge_page_shift(hstate_vma(vma)); + else + vma_shift = PAGE_SHIFT; + + vma_pagesize = 1ULL << vma_shift; + if (logging_active || (vma->vm_flags & VM_PFNMAP) || + !fault_supports_apt_huge_mapping(memslot, hva, vma_pagesize)) { + force_pte = true; + vma_pagesize = PAGE_SIZE; + } + + if (vma_pagesize == PMD_SIZE || vma_pagesize == CONT_PMD_SIZE || vma_pagesize == PUD_SIZE) + gfn = (fault_gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; + up_read(¤t->mm->mmap_lock); + /* We need minimum second+third level pages */ + ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES, + KVM_NR_MEM_OBJS); + if (ret) + return ret; + + mmu_seq = vcpu->kvm->mmu_notifier_seq; + /* + * Ensure the read of mmu_notifier_seq happens before we call + * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk + * the page we just got a reference to gets unmapped before we have a + * chance to grab the mmu_lock, which ensure that if the page gets + * unmapped afterwards, the call to kvm_unmap_hva will take it away + * from us again properly. This smp_rmb() interacts with the smp_wmb() + * in kvm_mmu_notifier_invalidate_. + */ + smp_rmb(); + + pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable); + if (pfn == KVM_PFN_ERR_HWPOISON) { + kvm_send_hwpoison_signal(hva, vma); + return 0; + } + if (is_error_noslot_pfn(pfn)) + return -EFAULT; + + if (logging_active) { + /* + * Faults on pages in a memslot with logging enabled + * should not be mapped with huge pages (it introduces churn + * and performance degradation), so force a pte mapping. + */ + flags |= KVM_APT_FLAG_LOGGING_ACTIVE; + + /* + * Only actually map the page as writable if this was a write + * fault. + */ + if (!write_fault) + writable = false; + } + + spin_lock(&kvm->mmu_lock); + if (mmu_notifier_retry(kvm, mmu_seq)) + goto out_unlock; + + /* + * If we are not forced to use page mapping, check if we are + * backed by a THP and thus use block mapping if possible. + */ + if (vma_pagesize == PAGE_SIZE && !force_pte) { + vma_pagesize = transparent_hugepage_adjust(memslot, hva, + &pfn, &fault_gpa); + } + + if (vma_pagesize == PUD_SIZE) { + pud_t new_pud = pfn_pud(pfn, mem_type); + + new_pud = pud_mkhuge(new_pud); + + if (writable) { + new_pud = kvm_pud_mkwrite(new_pud); + kvm_set_pfn_dirty(pfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pud = kvm_pud_mkexec(new_pud); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pud = kvm_pud_mkexec(new_pud); + } + + ret = apt_set_pud_huge(kvm, memcache, fault_gpa, &new_pud); + } else if (vma_pagesize == CONT_PMD_SIZE) { + pmd_t new_pmd = pfn_pmd(pfn, mem_type); + + new_pmd = pmd_mkhuge(new_pmd); + new_pmd = pmd_mkcont(new_pmd); + + if (writable) { + new_pmd = kvm_pmd_mkwrite(new_pmd); + kvm_set_pfn_dirty(pfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pmd = kvm_pmd_mkexec(new_pmd); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pmd = kvm_pmd_mkexec(new_pmd); + } + + ret = apt_set_pmd_huge(kvm, memcache, fault_gpa, &new_pmd, vma_pagesize); + } else if (vma_pagesize == PMD_SIZE) { + pmd_t new_pmd = pfn_pmd(pfn, mem_type); + + new_pmd = pmd_mkhuge(new_pmd); + + if (writable) { + new_pmd = kvm_pmd_mkwrite(new_pmd); + kvm_set_pfn_dirty(pfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pmd = kvm_pmd_mkexec(new_pmd); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pmd = kvm_pmd_mkexec(new_pmd); + } + + ret = apt_set_pmd_huge(kvm, memcache, fault_gpa, &new_pmd, vma_pagesize); + } else { + pte_t new_pte = pfn_pte(pfn, mem_type); + + if (writable) { + new_pte = kvm_pte_mkwrite(new_pte); + kvm_set_pfn_dirty(pfn); + mark_page_dirty(kvm, gfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pte = kvm_pte_mkexec(new_pte); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pte = kvm_pte_mkexec(new_pte); + } + + ret = apt_set_pte_fast(kvm, memcache, fault_gpa, &new_pte, flags); + if (!ret) + goto out_unlock; + } + +out_unlock: + spin_unlock(&kvm->mmu_lock); + kvm_set_pfn_accessed(pfn); + kvm_release_pfn_clean(pfn); + return ret; +} + +/** + * kvm_handle_guest_abort - handles all 2nd stage aborts + * @vcpu: the VCPU pointer + * @run: the kvm_run structure + * + * Any abort that gets to the host is almost guaranteed to be caused by a + * missing second stage translation table entry, which can mean that either the + * guest simply needs more memory and we must allocate an appropriate page or it + * can mean that the guest tried to access I/O memory, which is emulated by user + * space. The distinction is based on the IPA causing the fault and whether this + * memory region has been registered as standard RAM by user space. + */ +#ifdef CONFIG_SUBARCH_C4 +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + unsigned long as_info; /* the value of CSR: AS_INFO */ + unsigned int access_type, inv_level; + unsigned int fault_status; + unsigned long fault_entry_addr; + phys_addr_t fault_gpa; + struct kvm_memory_slot *memslot; + unsigned long hva; + bool write_fault, writable; + gfn_t gfn; + + int ret, idx; + + as_info = read_csr(CSR_AS_INFO); + access_type = (as_info >> AF_ACCESS_TYPE_SHIFT) & AF_ACCESS_TYPE_MASK; + inv_level = (as_info >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; + fault_status = (as_info >> AF_FAULT_STATUS_SHIFT) & AF_FAULT_STATUS_MASK; + fault_entry_addr = (as_info & AF_ENTRY_ADDR_MASK) >> 3; + + fault_gpa = read_csr(CSR_EXC_GPA); + idx = srcu_read_lock(&vcpu->kvm->srcu); + + gfn = fault_gpa >> PAGE_SHIFT; + memslot = gfn_to_memslot(vcpu->kvm, gfn); + hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); + + write_fault = kvm_is_write_fault(access_type); + + /* The memory slot for IO doesn't register in memory region + * with kvm, if hva == KVM_HVA_ERR_BAD, the gpa used for MMIO + * needs emulation. + */ + + if (hva == KVM_HVA_ERR_BAD) { + ret = io_mem_abort(vcpu, run, NULL); + goto out_unlock; + } + /* Userspace should not be able to register out-of-bounds IPAs */ + VM_BUG_ON(fault_gpa >= KVM_PHYS_SIZE); + + ret = user_mem_abort(vcpu, fault_gpa, memslot, hva, fault_status); + if (ret == 0) + ret = 1; +out_unlock: + srcu_read_unlock(&vcpu->kvm->srcu, idx); + return ret; +} +#endif +static int handle_hva_to_gpa(struct kvm *kvm, unsigned long start, unsigned long end, + int (*handler)(struct kvm *kvm, gpa_t gpa, u64 size, void *data), + void *data) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *memslot; + int ret = 0; + + slots = kvm_memslots(kvm); + + /* we only care about the pages that the guest sees */ + kvm_for_each_memslot(memslot, slots) { + unsigned long hva_start, hva_end; + gfn_t gpa; + + hva_start = max(start, memslot->userspace_addr); + hva_end = min(end, memslot->userspace_addr + + (memslot->npages << PAGE_SHIFT)); + if (hva_start >= hva_end) + continue; + + gpa = hva_to_gfn_memslot(hva_start, memslot) << PAGE_SHIFT; + ret |= handler(kvm, gpa, (u64)(hva_end - hva_start), data); + } + + return ret; +} + +static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + unmap_apt_range(kvm, gpa, size); + return 0; +} + +int kvm_unmap_hva_range(struct kvm *kvm, + unsigned long start, unsigned long end, bool blockable) +{ + if (!kvm->arch.pgd) + return 0; + + handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL); + return 1; +} + +static int apt_ptep_test_and_clear_young(pte_t *pte) +{ + if (pte_young(*pte)) { + *pte = pte_mkold(*pte); + return 1; + } + return 0; +} + +static int apt_pmdp_test_and_clear_young(pmd_t *pmd) +{ + return apt_ptep_test_and_clear_young((pte_t *)pmd); +} + +static int apt_pudp_test_and_clear_young(pud_t *pud) +{ + return apt_ptep_test_and_clear_young((pte_t *)pud); +} + +static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); + if (!apt_get_leaf_entry(kvm, gpa, &pud, &pmd, &pte)) + return 0; + + if (pud) + return apt_pudp_test_and_clear_young(pud); + else if (pmd) + return apt_pmdp_test_and_clear_young(pmd); + else + return apt_ptep_test_and_clear_young(pte); +} + +static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); + if (!apt_get_leaf_entry(kvm, gpa, &pud, &pmd, &pte)) + return 0; + + if (pud) + return apt_pudp_test_and_clear_young(pud); + else if (pmd) + return apt_pmdp_test_and_clear_young(pmd); + else + return apt_ptep_test_and_clear_young(pte); +} + +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) +{ + if (!kvm->arch.pgd) + return 0; + + return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL); +} + +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) +{ + if (!kvm->arch.pgd) + return 0; + return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL); +} + +static int kvm_set_apte_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + pte_t *pte = (pte_t *)data; + + WARN_ON(size != PAGE_SIZE); + + apt_set_pte(kvm, NULL, gpa, pte, 0); + return 0; +} + +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) +{ + unsigned long end = hva + PAGE_SIZE; + pte_t apt_pte; + + if (!kvm->arch.pgd) + return 0; + + apt_pte = pte_wrprotect(pte); + handle_hva_to_gpa(kvm, hva, end, &kvm_set_apte_handler, &apt_pte); + return 0; +} + +/** + * kvm_mmu_write_protect_pt_masked() - write protect dirty pages + * @kvm: The KVM pointer + * @slot: The memory slot associated with mask + * @gfn_offset: The gfn offset in memory slot + * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory + * slot to be write protected + * + * Walks bits set in mask write protects the associated pte's. Caller must + * acquire kvm_mmu_lock. + */ +static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask) +{ + phys_addr_t base_gfn = slot->base_gfn + gfn_offset; + phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; + phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; + + apt_wp_range(kvm, start, end); +} + +/* + * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected + * dirty pages. + * + * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to + * enable dirty logging for them. + */ +void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask) +{ + kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); +} diff --git a/arch/sw_64/kvm/kvm-sw64.c b/arch/sw_64/kvm/sw64.c similarity index 53% rename from arch/sw_64/kvm/kvm-sw64.c rename to arch/sw_64/kvm/sw64.c index 122fb02957ce709267a171f9134e9384d5746ae3..9bb880f0d5fc514c38bc237ccce321089fbfb7b2 100644 --- a/arch/sw_64/kvm/kvm-sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -1,9 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2018 - os kernal - * Author: fire3 yangzh - * linhn - */ #include #include @@ -16,41 +11,31 @@ #include #include #include +#include +#include +#include #define CREATE_TRACE_POINTS #include "trace.h" #include "../kernel/pci_impl.h" -#include "vmem.c" bool set_msi_flag; -static unsigned long longtime_offset; - -#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) -extern bool bind_vcpu_enabled; -#endif -#define last_vpn(cpu) (cpu_data[cpu].last_vpn) - -#ifdef CONFIG_SUBARCH_C3B -#define WIDTH_HARDWARE_VPN 8 -#endif -#define VPN_FIRST_VERSION (1UL << WIDTH_HARDWARE_VPN) -#define HARDWARE_VPN_MASK ((1UL << WIDTH_HARDWARE_VPN) - 1) -#define VPN_SHIFT (64 - WIDTH_HARDWARE_VPN) - -#define VCPU_STAT(n, x, ...) \ - { n, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ } -#define VM_STAT(n, x, ...) \ - { n, offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ } #define DFX_STAT(n, x, ...) \ { n, offsetof(struct kvm_vcpu_stat, x), DFX_STAT_U64, ## __VA_ARGS__ } -static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu); - -static void kvm_set_running_vcpu(struct kvm_vcpu *vcpu) +static unsigned long get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) { - __this_cpu_write(kvm_running_vcpu, vcpu); + unsigned long vpn = last_vpn(cpu); + unsigned long next = vpn + 1; + + if ((vpn & VPN_MASK) >= VPN_MASK) { + kvm_flush_tlb_all(); + next = (vpn & ~VPN_MASK) + VPN_FIRST_VERSION + 1; /* bypass 0 */ + } + last_vpn(cpu) = next; + return next; } int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) @@ -60,6 +45,11 @@ int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) return 0; } +int kvm_arch_check_processor_compat(void *opaque) +{ + return 0; +} + int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, bool line_status) { @@ -78,36 +68,7 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq return vcpu_interrupt_line(vcpu, irq, true); } -extern int __sw64_vcpu_run(unsigned long vcb_pa, - struct kvm_regs *regs, struct hcall_args *args); - -#ifdef CONFIG_KVM_MEMHOTPLUG -static u64 get_vpcr_memhp(u64 seg_base, u64 vpn) -{ - return seg_base | ((vpn & HARDWARE_VPN_MASK) << 44); -} -#else -static u64 get_vpcr(u64 hpa_base, u64 mem_size, u64 vpn) -{ - return (hpa_base >> 23) | ((mem_size >> 23) << 16) - | ((vpn & HARDWARE_VPN_MASK) << 44); -} -#endif - -static unsigned long __get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) -{ - unsigned long vpn = last_vpn(cpu); - unsigned long next = vpn + 1; - - if ((vpn & HARDWARE_VPN_MASK) >= HARDWARE_VPN_MASK) { - tbia(); - next = (vpn & ~HARDWARE_VPN_MASK) + VPN_FIRST_VERSION + 1; /* bypass 0 */ - } - last_vpn(cpu) = next; - return next; -} - -static void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) +void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) { unsigned long vpn; unsigned long vpnc; @@ -116,18 +77,17 @@ static void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) vpn = last_vpn(cpu); vpnc = vcpu->arch.vpnc[cpu]; - if ((vpnc ^ vpn) & ~HARDWARE_VPN_MASK) { + if ((vpnc ^ vpn) & ~VPN_MASK) { /* vpnc and cpu vpn not in the same version, get new vpnc and vpn */ - vpnc = __get_new_vpn_context(vcpu, cpu); + vpnc = get_new_vpn_context(vcpu, cpu); vcpu->arch.vpnc[cpu] = vpnc; } - vpn = vpnc & HARDWARE_VPN_MASK; + vpn = vpnc & VPN_MASK; /* Always update vpn */ /* Just setup vcb, hardware CSR will be changed later in HMcode */ - vcpu->arch.vcb.vpcr = ((vcpu->arch.vcb.vpcr) & (~(HARDWARE_VPN_MASK << 44))) | (vpn << 44); - vcpu->arch.vcb.dtb_pcr = ((vcpu->arch.vcb.dtb_pcr) & (~(HARDWARE_VPN_MASK << VPN_SHIFT))) | (vpn << VPN_SHIFT); + kvm_sw64_update_vpn(vcpu, vpn); /* * If vcpu migrate to a new physical cpu, the new physical cpu may keep @@ -145,14 +105,14 @@ static void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) } } -static void check_vcpu_requests(struct kvm_vcpu *vcpu) +void check_vcpu_requests(struct kvm_vcpu *vcpu) { unsigned long vpn; long cpu = smp_processor_id(); if (kvm_request_pending(vcpu)) { if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { - vpn = vcpu->arch.vpnc[cpu] & HARDWARE_VPN_MASK; + vpn = vcpu->arch.vpnc[cpu] & VPN_MASK; tbivpn(0, 0, vpn); } } @@ -214,11 +174,6 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) && !vcpu->arch.power_off); } -int kvm_arch_check_processor_compat(void *opaque) -{ - return 0; -} - int kvm_arch_hardware_enable(void) { return 0; @@ -248,49 +203,6 @@ int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; } -/* - * kvm_mark_migration write the mark on every vcpucbs of the kvm, which tells - * the system to do migration while the mark is on, and flush all vcpu's tlbs - * at the beginning of the migration. - */ -void kvm_mark_migration(struct kvm *kvm, int mark) -{ - struct kvm_vcpu *vcpu; - int cpu; - - kvm_for_each_vcpu(cpu, vcpu, kvm) - vcpu->arch.vcb.migration_mark = mark << 2; - - kvm_flush_remote_tlbs(kvm); -} - -void kvm_arch_commit_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, - struct kvm_memory_slot *old, - const struct kvm_memory_slot *new, - enum kvm_mr_change change) -{ - /* - * At this point memslot has been committed and there is an - * allocated dirty_bitmap[], dirty pages will be be tracked while the - * memory slot is write protected. - */ - - /* If dirty logging has been stopped, do nothing for now. */ - if ((change != KVM_MR_DELETE) - && (old->flags & KVM_MEM_LOG_DIRTY_PAGES) - && (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))) { - kvm_mark_migration(kvm, 0); - return; - } - - /* If it's the first time dirty logging, flush all vcpu tlbs. */ - if ((change == KVM_MR_FLAGS_ONLY) - && (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES)) - && (new->flags & KVM_MEM_LOG_DIRTY_PAGES)) - kvm_mark_migration(kvm, 1); -} - int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) { int r = 0; @@ -299,7 +211,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_IRQCHIP: case KVM_CAP_IOEVENTFD: case KVM_CAP_SYNC_MMU: - case KVM_CAP_IMMEDIATE_EXIT: r = 1; break; case KVM_CAP_NR_VCPUS: @@ -313,20 +224,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) return r; } -void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, - struct kvm_memory_slot *slot, gfn_t gfn_offset, - unsigned long mask) -{ -} - -int kvm_sw64_pending_timer(struct kvm_vcpu *vcpu) +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { - return test_bit(SW64_KVM_IRQ_TIMER, &vcpu->arch.irqs_pending); } int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) { - return kvm_sw64_pending_timer(vcpu); + return test_bit(SW64_KVM_IRQ_TIMER, &vcpu->arch.irqs_pending); } int kvm_arch_hardware_setup(void *opaque) @@ -334,54 +238,17 @@ int kvm_arch_hardware_setup(void *opaque) return 0; } -void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) -{ - hrtimer_cancel(&vcpu->arch.hrt); -} - int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) { -#ifdef CONFIG_KVM_MEMHOTPLUG - unsigned long *seg_pgd; - - if (kvm->arch.seg_pgd != NULL) { - kvm_err("kvm_arch already initialized?\n"); + if (type) return -EINVAL; - } - seg_pgd = alloc_pages_exact(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO); - if (!seg_pgd) - return -ENOMEM; - - kvm->arch.seg_pgd = seg_pgd; -#endif - - return 0; + return kvm_sw64_init_vm(kvm); } void kvm_arch_destroy_vm(struct kvm *kvm) { - int i; -#ifdef CONFIG_KVM_MEMHOTPLUG - void *seg_pgd = NULL; - - if (kvm->arch.seg_pgd) { - seg_pgd = READ_ONCE(kvm->arch.seg_pgd); - kvm->arch.seg_pgd = NULL; - } - - if (seg_pgd) - free_pages_exact(seg_pgd, PAGE_SIZE); -#endif - - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - if (kvm->vcpus[i]) { - kvm_vcpu_destroy(kvm->vcpus[i]); - kvm->vcpus[i] = NULL; - } - } - - atomic_set(&kvm->online_vcpus, 0); + return kvm_sw64_destroy_vm(kvm); } long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) @@ -395,117 +262,16 @@ int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, return 0; } -#ifdef CONFIG_KVM_MEMHOTPLUG -static void setup_segment_table(struct kvm *kvm, - struct kvm_memory_slot *memslot, unsigned long addr, size_t size) +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) { - unsigned long *seg_pgd = kvm->arch.seg_pgd; - unsigned int num_of_entry = size >> 30; - unsigned long base_hpa = addr >> 30; - int i; - - for (i = 0; i < num_of_entry; i++) { - *seg_pgd = base_hpa + i; - seg_pgd++; - } + kvm_mmu_free_memory_caches(vcpu); + hrtimer_cancel(&vcpu->arch.hrt); + kfree(vcpu); } -#endif -int kvm_arch_prepare_memory_region(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, - enum kvm_mr_change change) +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) { - unsigned long addr; - struct file *vm_file; - struct vm_area_struct *vma; - struct vmem_info *info; - unsigned long ret; - size_t size; - - if (change == KVM_MR_FLAGS_ONLY || change == KVM_MR_DELETE) - return 0; - - if (test_bit(IO_MARK_BIT, &(mem->guest_phys_addr))) - return 0; - - if (test_bit(IO_MARK_BIT + 1, &(mem->guest_phys_addr))) - return 0; - -#ifndef CONFIG_KVM_MEMHOTPLUG - if (mem->guest_phys_addr) { - pr_info("%s, No KVM MEMHOTPLUG support!\n", __func__); - return 0; - } -#endif - - if (!sw64_kvm_pool) - return -ENOMEM; - - pr_info("%s: %#llx %#llx, user addr: %#llx\n", __func__, - mem->guest_phys_addr, mem->memory_size, mem->userspace_addr); - - vma = find_vma(current->mm, mem->userspace_addr); - if (!vma) - return -ENOMEM; - vm_file = vma->vm_file; - - if (!vm_file) { - info = kzalloc(sizeof(struct vmem_info), GFP_KERNEL); - - size = round_up(mem->memory_size, 8 << 20); - addr = gen_pool_alloc(sw64_kvm_pool, size); - if (!addr) - return -ENOMEM; - vm_munmap(mem->userspace_addr, mem->memory_size); - ret = vm_mmap(vm_file, mem->userspace_addr, mem->memory_size, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, 0); - if ((long)ret < 0) - return ret; - - vma = find_vma(current->mm, mem->userspace_addr); - if (!vma) - return -ENOMEM; - -#ifdef CONFIG_KVM_MEMHOTPLUG - if (memslot->base_gfn == 0x0UL) { - setup_segment_table(kvm, memslot, addr, size); - kvm->arch.host_phys_addr = (u64)addr; - memslot->arch.host_phys_addr = addr; - } else { - /* used for memory hotplug */ - memslot->arch.host_phys_addr = addr; - memslot->arch.valid = false; - } -#endif - - info->start = addr; - info->size = size; - vma->vm_private_data = (void *) info; - - vma->vm_ops = &vmem_vm_ops; - vma->vm_ops->open(vma); - - ret = vmem_vm_insert_page(vma); - if ((int)ret < 0) - return ret; - } else { - info = vm_file->private_data; - addr = info->start; - } - - pr_info("guest phys addr = %#lx, size = %#lx\n", - addr, vma->vm_end - vma->vm_start); - -#ifndef CONFIG_KVM_MEMHOTPLUG - kvm->arch.host_phys_addr = (u64)addr; - kvm->arch.size = round_up(mem->memory_size, 8 << 20); -#endif - - memset(__va(addr), 0, 0x2000000); - - return 0; + kvm_arch_vcpu_free(vcpu); } int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) @@ -518,28 +284,15 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->arch.hrt.function = clockdev_fn; vcpu->arch.tsk = current; - /* For guest kernel "sys_call HMC_whami", indicate virtual cpu id */ - vcpu->arch.vcb.whami = vcpu->vcpu_id; + vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; vcpu->arch.vcb.vcpu_irq_disabled = 1; vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ return 0; } -int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) +int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) { - unsigned long addr = vcpu->kvm->arch.host_phys_addr; - - hrtimer_cancel(&vcpu->arch.hrt); - vcpu->arch.vcb.whami = vcpu->vcpu_id; - vcpu->arch.vcb.vcpu_irq_disabled = 1; - vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ - vcpu->arch.power_off = 0; - memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); - - if (vcpu->vcpu_id == 0) - memset(__va(addr), 0, 0x2000000); - return 0; } @@ -597,7 +350,6 @@ static void update_steal_time(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { vcpu->cpu = cpu; - kvm_set_running_vcpu(vcpu); update_steal_time(vcpu); } @@ -609,7 +361,6 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) * optimized make_all_cpus_request path. */ vcpu->cpu = -1; - kvm_set_running_vcpu(NULL); } int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, @@ -642,25 +393,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, return 0; } -void _debug_printk_vcpu(struct kvm_vcpu *vcpu) -{ - unsigned long pc = vcpu->arch.regs.pc; - unsigned long offset = vcpu->kvm->arch.host_phys_addr; - unsigned int *pc_phys = __va((pc & 0x7fffffffUL) + offset); - unsigned int insn; - int opc, ra, disp16; - - insn = *pc_phys; - opc = (insn >> 26) & 0x3f; - ra = (insn >> 21) & 0x1f; - disp16 = insn & 0xffff; - - if (opc == 0x06 && disp16 == 0x1000) /* RD_F */ - pr_info("vcpu exit: pc = %#lx (%px), insn[%x] : rd_f r%d [%#lx]\n", - pc, pc_phys, insn, ra, vcpu_get_reg(vcpu, ra)); -} - -static void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat) +void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat) { vcpu_stat->utime = current->utime; vcpu_stat->stime = current->stime; @@ -673,43 +406,16 @@ static void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat) */ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { - int ret; struct kvm_run *run = vcpu->run; struct vcpucb *vcb = &(vcpu->arch.vcb); struct hcall_args hargs; - int irq; + int irq, ret; bool more; sigset_t sigsaved; - if (run->immediate_exit) - return -EINTR; - /* Set guest vcb */ /* vpn will update later when vcpu is running */ - if (vcpu->arch.vcb.vpcr == 0) { -#ifndef CONFIG_KVM_MEMHOTPLUG - vcpu->arch.vcb.vpcr - = get_vpcr(vcpu->kvm->arch.host_phys_addr, vcpu->kvm->arch.size, 0); - -#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) - if (unlikely(bind_vcpu_enabled)) { - int nid; - unsigned long end; - - end = vcpu->kvm->arch.host_phys_addr + vcpu->kvm->arch.size; - nid = pfn_to_nid(PHYS_PFN(vcpu->kvm->arch.host_phys_addr)); - if (pfn_to_nid(PHYS_PFN(end)) == nid) - set_cpus_allowed_ptr(vcpu->arch.tsk, cpumask_of_node(nid)); - } -#endif -#else /* !CONFIG_KVM_MEMHOTPLUG */ - unsigned long seg_base = virt_to_phys(vcpu->kvm->arch.seg_pgd); - - vcpu->arch.vcb.vpcr = get_vpcr_memhp(seg_base, 0); -#endif /* CONFIG_KVM_MEMHOTPLUG */ - vcpu->arch.vcb.upcr = 0x7; - } - + vcpu_set_numa_affinity(vcpu); #ifdef CONFIG_PERF_EVENTS vcpu_load(vcpu); #endif @@ -721,7 +427,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) run->exit_reason = KVM_EXIT_UNKNOWN; ret = 1; - while (ret > 0) { /* Check conditions before entering the guest */ cond_resched(); @@ -744,6 +449,13 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) memset(&hargs, 0, sizeof(hargs)); clear_vcpu_irq(vcpu); + + if (vcpu->arch.restart == 1) { + /* handle reset vCPU */ + vcpu->arch.regs.pc = GUEST_RESET_PC; + vcpu->arch.restart = 0; + } + irq = interrupt_pending(vcpu, &more); if (irq < SWVM_IRQS) try_deliver_interrupt(vcpu, irq, more); @@ -754,6 +466,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) check_vcpu_requests(vcpu); guest_enter_irqoff(); + /* update aptp before the guest runs */ + update_aptp((unsigned long)vcpu->kvm->arch.pgd); + /* Enter the guest */ trace_kvm_sw64_entry(vcpu->vcpu_id, vcpu->arch.regs.pc); vcpu->mode = IN_GUEST_MODE; @@ -782,55 +497,31 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) #ifdef CONFIG_PERF_EVENTS vcpu_put(vcpu); #endif + return ret; } long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { - unsigned long result; struct kvm_vcpu *vcpu = filp->private_data; - struct vcpucb *kvm_vcb; + int r; switch (ioctl) { case KVM_SW64_VCPU_INIT: - return kvm_arch_vcpu_reset(vcpu); + r = kvm_sw64_vcpu_reset(vcpu); + break; case KVM_SW64_GET_VCB: - if (vcpu->arch.vcb.migration_mark) { - result = sw64_io_read(0, LONG_TIME) - + vcpu->arch.vcb.guest_longtime_offset; - vcpu->arch.vcb.guest_longtime = result; - vcpu->arch.vcb.guest_irqs_pending = vcpu->arch.irqs_pending[0]; - } - - if (copy_to_user((void __user *)arg, &(vcpu->arch.vcb), sizeof(struct vcpucb))) - return -EINVAL; + r = kvm_sw64_get_vcb(filp, arg); break; case KVM_SW64_SET_VCB: - kvm_vcb = memdup_user((void __user *)arg, sizeof(*kvm_vcb)); - memcpy(&(vcpu->arch.vcb), kvm_vcb, sizeof(struct vcpucb)); - - if (vcpu->arch.vcb.migration_mark) { - /* updated vpcr needed by destination vm */ - vcpu->arch.vcb.vpcr - = get_vpcr(vcpu->kvm->arch.host_phys_addr, vcpu->kvm->arch.size, 0); - - /* synchronize the longtime of source and destination */ - if (vcpu->arch.vcb.whami == 0) { - result = sw64_io_read(0, LONG_TIME); - vcpu->arch.vcb.guest_longtime_offset = vcpu->arch.vcb.guest_longtime - result; - longtime_offset = vcpu->arch.vcb.guest_longtime_offset; - } else - vcpu->arch.vcb.guest_longtime_offset = longtime_offset; - - set_timer(vcpu, 200000000); - vcpu->arch.vcb.migration_mark = 0; - } + r = kvm_sw64_set_vcb(filp, arg); break; default: - return -EINVAL; + r = -EINVAL; } - return 0; + + return r; } long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) @@ -864,21 +555,6 @@ void kvm_arch_exit(void) kvm_sw64_perf_teardown(); } -void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) -{ -} - -void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, - struct kvm_memory_slot *memslot) -{ - kvm_flush_remote_tlbs(kvm); -} - -int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) -{ - return 0; -} - int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) { @@ -910,6 +586,13 @@ vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) return VM_FAULT_SIGBUS; } +void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) +{ + /* Let implementation handle TLB/GVA invalidation */ + kvm_arch_flush_shadow_memslot(kvm, memslot); +} + int kvm_dev_ioctl_check_extension(long ext) { int r; @@ -929,34 +612,13 @@ int kvm_dev_ioctl_check_extension(long ext) return r; } -#ifdef CONFIG_KVM_MEMHOTPLUG -void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr) -{ - struct kvm *kvm = vcpu->kvm; - struct kvm_memory_slot *slot; - unsigned long start_pfn = start_addr >> PAGE_SHIFT; - - kvm_for_each_memslot(slot, kvm_memslots(kvm)) { - if (start_pfn == slot->base_gfn) { - unsigned long *seg_pgd; - unsigned long num_of_entry = slot->npages >> 17; - unsigned long base_hpa = slot->arch.host_phys_addr; - int i; - - seg_pgd = kvm->arch.seg_pgd + (start_pfn >> 17); - for (i = 0; i < num_of_entry; i++) { - *seg_pgd = (base_hpa >> 30) + i; - seg_pgd++; - } - } - } -} -#endif - -void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid) +void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type) { struct kvm_vcpu *target_vcpu = kvm_get_vcpu(vcpu->kvm, target_vcpuid); + if (type == II_RESET) + target_vcpu->arch.restart = 1; + if (target_vcpu != NULL) vcpu_interrupt_line(target_vcpu, 1, 1); } @@ -978,30 +640,3 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, return vcpu_interrupt_line(vcpu, irq_num, level); } -static int __init kvm_sw64_init(void) -{ - int i, ret; - - ret = vmem_init(); - if (ret) - return ret; - - for (i = 0; i < NR_CPUS; i++) - last_vpn(i) = VPN_FIRST_VERSION; - - ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); - if (ret) { - vmem_exit(); - return ret; - } - return 0; -} - -static void __exit kvm_sw64_exit(void) -{ - kvm_exit(); - vmem_exit(); -} - -module_init(kvm_sw64_init); -module_exit(kvm_sw64_exit); diff --git a/arch/sw_64/kvm/vmem.c b/arch/sw_64/kvm/vmem.c index c6f9d6cdf03b120b32d3e39ac017e412838f9efb..fa893f5e26a739ca6aa76e12220cfd098f8ed9a1 100644 --- a/arch/sw_64/kvm/vmem.c +++ b/arch/sw_64/kvm/vmem.c @@ -72,7 +72,7 @@ static void vmem_vm_close(struct vm_area_struct *vma) info = vma->vm_private_data; addr = info->start; - size = info->size; + size = round_up(info->size, 8 << 20); if (atomic_dec_and_test(&info->refcnt)) { if (sw64_kvm_pool && addr_in_pool(sw64_kvm_pool, addr, size)) { @@ -123,7 +123,7 @@ static int vmem_mmap(struct file *flip, struct vm_area_struct *vma) return -ENOMEM; if (flip->private_data == NULL) { - addr = gen_pool_alloc(sw64_kvm_pool, size); + addr = gen_pool_alloc(sw64_kvm_pool, round_up(size, 8 << 20)); if (!addr) return -ENOMEM; @@ -165,7 +165,7 @@ static struct miscdevice vmem_dev = { .fops = &vmem_fops, }; -static int __init vmem_init(void) +int __init vmem_init(void) { int err; @@ -177,7 +177,7 @@ static int __init vmem_init(void) return 0; } -static void vmem_exit(void) +void vmem_exit(void) { misc_deregister(&vmem_dev); } diff --git a/arch/sw_64/lib/deep-clear_user.S b/arch/sw_64/lib/deep-clear_user.S index 521586a7189fe433e62b8ced61974b2659324f39..c81418ed99a26b7bfc197863bb0ecc911b43a62c 100644 --- a/arch/sw_64/lib/deep-clear_user.S +++ b/arch/sw_64/lib/deep-clear_user.S @@ -34,7 +34,11 @@ __clear_user: bis $31, $31, $7 mov $17, $18 bis $31, $31, $17 +#if defined(CONFIG_SUBARCH_C3B) #include "deep-set_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-set_template_c4.S" +#endif $out: bis $31, $18, $0 beq $7, $return diff --git a/arch/sw_64/lib/deep-copy_page.S b/arch/sw_64/lib/deep-copy_page.S index 5061837e537bfa6b51912a72d8e58f33fbec59da..a9b9d97f318af18a02bc7307d8d7d1b0cc1f0088 100644 --- a/arch/sw_64/lib/deep-copy_page.S +++ b/arch/sw_64/lib/deep-copy_page.S @@ -5,6 +5,7 @@ * Copy an entire page. */ #include +#include .text .align 4 @@ -19,6 +20,9 @@ copy_page: stl $4, 0($sp) bic $4, 0x1f, $4 vstd $f16, 0($4) +#ifdef CONFIG_SUBARCH_C4 + csrr $5, CSR_WR_FREGS +#endif /* Optimize by GUOY from SOC 2013-06-04 */ 1: @@ -46,6 +50,9 @@ copy_page: ldi $4, 0x40($sp) bic $4, 0x1f, $4 vldd $f16, 0($4) +#ifdef CONFIG_SUBARCH_C4 + csrw $5, CSR_WR_FREGS +#endif addl $sp, 0x60, $sp ret diff --git a/arch/sw_64/lib/deep-copy_template_c4.S b/arch/sw_64/lib/deep-copy_template_c4.S new file mode 100644 index 0000000000000000000000000000000000000000..e0740874dfa32722b903793a9eaa797ba5007f1d --- /dev/null +++ b/arch/sw_64/lib/deep-copy_template_c4.S @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * template for memcpy and copy_user with SIMD + * + * $7: SIMD status + * 0: not in simd loop + * 1: in simd and simd_u loop + * $16: latest dest, clobbered + * $17: latest src, clobbered + * $18: bytes left to copy + * + */ + +#define SAVE_SIMD_REGS \ + ldi $sp, -0x60($sp); \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vstd $f1, 0($23); \ + vstd $f2, 0x20($23); \ + ldi $7, 1 + +#define RESTORE_SIMD_REGS \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vldd $f1, 0($23); \ + vldd $f2, 0x20($23); \ + ldi $sp, 0x60($sp); \ + bis $31, $31, $7 + + + ble $18, $out + + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + cmplt $18, 16, $1 + bne $1, $quad_loop_end + cmplt $18, 32, $1 + bne $1, $simd_end + +$prep_simd_loop: + SAVE_SIMD_REGS + cmplt $18, 64, $1 + bne $1, $simd_loop_end + + .align 4 +$simd_loop: + FIXUP_LDST( vldd $f1, 0($17) ) + FIXUP_LDST( vldd $f2, 32($17) ) + FIXUP_LDST( vstd $f1, 0($16) ) + FIXUP_LDST( vstd $f2, 32($16) ) + subl $18, 64, $18 + addl $17, 64, $17 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_loop + +$simd_loop_end: + cmplt $18, 32, $1 + bne $1, $no_more_simd + FIXUP_LDST( vldd $f1, 0($17) ) + FIXUP_LDST( vstd $f1, 0($16) ) + subl $18, 32, $18 + addl $17, 32, $17 + addl $16, 32, $16 + +$no_more_simd: + RESTORE_SIMD_REGS + +$simd_end: + ble $18, $out + cmplt $18, 16, $1 + bne $1, $quad_loop_end + + .align 4 +$quad_loop_tail: + FIXUP_LDST( ldl $2, 0($17) ) + FIXUP_LDST( ldl $3, 8($17) ) + FIXUP_LDST( stl $2, 0($16) ) + FIXUP_LDST( stl $3, 8($16) ) + subl $18, 16, $18 + addl $17, 16, $17 + addl $16, 16, $16 + cmplt $18, 16, $1 + beq $1, $quad_loop_tail + +$quad_loop_end: + ble $18, $out + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + +$move_one_quad: + FIXUP_LDST( ldl $2, 0($17) ) + FIXUP_LDST( stl $2, 0($16) ) + subl $18, 8, $18 + addl $17, 8, $17 + addl $16, 8, $16 + ble $18, $out + + .align 3 +$byte_loop_tail: + FIXUP_LDST( ldbu $2, 0($17) ) + FIXUP_LDST( stb $2, 0($16) ) + subl $18, 1, $18 + addl $17, 1, $17 + addl $16, 1, $16 + bgt $18, $byte_loop_tail + br $31, $out diff --git a/arch/sw_64/lib/deep-copy_user.S b/arch/sw_64/lib/deep-copy_user.S index 327cab322765ab2f4758812cd178d437726eb44d..b79f8f3f0f4ac1e85dd5d2f130e88f90c1e062c0 100644 --- a/arch/sw_64/lib/deep-copy_user.S +++ b/arch/sw_64/lib/deep-copy_user.S @@ -11,10 +11,13 @@ .previous /* - * $7: SIMD status + * $7: SIMD status for C3B * 0: not in simd loop * 1: in simd loop * 2: in simd_u loop + * $7: SIMD status for C4 + * 0: not in simd loop + * 1: in simd and simd_u loop * $18: bytes left to copy * */ @@ -22,17 +25,24 @@ .ent __copy_user __copy_user: .prologue 0 + .set noreorder bis $31, $31, $7 +#if defined(CONFIG_SUBARCH_C3B) #include "deep-copy_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-copy_template_c4.S" +#endif $out: bis $31, $18, $0 beq $7, $return subl $7, 1, $7 beq $7, $restore_simd +#if defined(CONFIG_SUBARCH_C3B) $restore_simd_u: RESTORE_SIMD_U_REGS br $31, $return +#endif $restore_simd: RESTORE_SIMD_REGS diff --git a/arch/sw_64/lib/deep-memcpy.S b/arch/sw_64/lib/deep-memcpy.S index c4b5bf3d26dfd55be9e701d26a68b9abe6361e93..78a6bd85cf016a49948c973ef3a8652a869bb8db 100644 --- a/arch/sw_64/lib/deep-memcpy.S +++ b/arch/sw_64/lib/deep-memcpy.S @@ -11,7 +11,11 @@ memcpy: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 +#if defined(CONFIG_SUBARCH_C3B) #include "deep-copy_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-copy_template_c4.S" +#endif $out: ret .end memcpy diff --git a/arch/sw_64/lib/deep-memset.S b/arch/sw_64/lib/deep-memset.S index 5d9beb1e2f539261e7ec72d725dc06cc0aa42098..c6b5355beec64d3d5557ae7b26c80cbb7c8b114a 100644 --- a/arch/sw_64/lib/deep-memset.S +++ b/arch/sw_64/lib/deep-memset.S @@ -26,6 +26,7 @@ */ #include +#include #define FIXUP_LDST(x, y) \ x, y @@ -44,6 +45,9 @@ ___memset: .frame $30, 0, $26, 0 .prologue 0 +#ifdef CONFIG_SUBARCH_C4 + csrr $6, CSR_WR_FREGS +#endif /* expand 1 byte data to 8 bytes */ and $17, 0xff, $17 sll $17, 8, $4 @@ -56,8 +60,15 @@ ___memset: __constant_c_memset: bis $31, $31, $7 bis $31, $16, $0 +#if defined(CONFIG_SUBARCH_C3B) #include "deep-set_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-set_template_c4.S" +#endif $out: +#ifdef CONFIG_SUBARCH_C4 + csrw $6, CSR_WR_FREGS +#endif ret .end ___memset diff --git a/arch/sw_64/lib/deep-set_template_c4.S b/arch/sw_64/lib/deep-set_template_c4.S new file mode 100644 index 0000000000000000000000000000000000000000..2b1bcab8fec96d9495f85477ca4ab87abfbf8308 --- /dev/null +++ b/arch/sw_64/lib/deep-set_template_c4.S @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * template for memset and clear_user with SIMD + * + * $7: SIMD status + * 0: not in simd loop + * 1: in simd loop + * $16: latest dest, clobbered + * $17: 8-byte data to set + * $18: bytes left to copy + * + */ + +#define SAVE_SIMD_REGS \ + ldi $sp, -0x40($sp); \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vstd $f1, 0($23); \ + ldi $7, 1 + +#define RESTORE_SIMD_REGS \ + vldd $f1, 0($23); \ + ldi $sp, 0x40($sp); \ + bis $31, $31, $7 + + ble $18, $out + + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + cmplt $18, 16, $1 + bne $1, $quad_loop_end + cmplt $18, 32, $1 + bne $1, $simd_end + +$prep_simd_loop: + SAVE_SIMD_REGS + ifmovd $17, $f1 + vcpyf $f1, $f1 + cmplt $18, 64, $1 + bne $1, $simd_loop_end + + .align 3 +$simd_loop: + FIXUP_LDST( vstd $f1, 0($16) ) + FIXUP_LDST( vstd $f1, 32($16) ) + subl $18, 64, $18 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_loop + +$simd_loop_end: + cmplt $18, 32, $1 + bne $1, $no_more_simd + FIXUP_LDST( vstd $f1, 0($16) ) + subl $18, 32, $18 + addl $16, 32, $16 + +$no_more_simd: + RESTORE_SIMD_REGS + +$simd_end: + ble $18, $out + cmplt $18, 16, $1 + bne $1, $quad_loop_end + + .align 3 +$quad_loop_tail: + FIXUP_LDST( stl $17, 0($16) ) + FIXUP_LDST( stl $17, 8($16) ) + subl $18, 16, $18 + addl $16, 16, $16 + cmplt $18, 16, $1 + beq $1, $quad_loop_tail + +$quad_loop_end: + ble $18, $out + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + +$move_one_quad: + FIXUP_LDST( stl $17, 0($16) ) + subl $18, 8, $18 + addl $16, 8, $16 + ble $18, $out + + .align 3 +$byte_loop_tail: + FIXUP_LDST( stb $17, 0($16) ) + subl $18, 1, $18 + addl $16, 1, $16 + bgt $18, $byte_loop_tail + br $31, $out diff --git a/arch/sw_64/lib/iomap.c b/arch/sw_64/lib/iomap.c index 3a8d879ef070e27bc27a5d386ba5708480a3d063..34cdedcb54de2388fe92e1be6bdc50f423279215 100644 --- a/arch/sw_64/lib/iomap.c +++ b/arch/sw_64/lib/iomap.c @@ -460,7 +460,14 @@ EXPORT_SYMBOL(_memset_c_io); void __iomem *ioport_map(unsigned long port, unsigned int size) { - return sw64_platform->ioportmap(port); + unsigned long io_offset; + + if (port < 0x100000) { + io_offset = is_in_host() ? LPC_LEGACY_IO : PCI_VT_LEGACY_IO; + port = port | io_offset; + } + + return __va(port); } EXPORT_SYMBOL(ioport_map); diff --git a/arch/sw_64/mm/Makefile b/arch/sw_64/mm/Makefile index 92be882cc82b31ab06d94b43a7e909ca6da9949a..8b9d6e4d2ebfb31bda1ee41cb00c163cd70ce577 100644 --- a/arch/sw_64/mm/Makefile +++ b/arch/sw_64/mm/Makefile @@ -5,8 +5,12 @@ #ccflags-y := -Werror -obj-y := init.o fault.o physaddr.o mmap.o +obj-y := init.o fault.o physaddr.o mmap.o extable.o obj-$(CONFIG_NUMA) += numa.o +ifeq ($(CONFIG_SUBARCH_C4),y) +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage_c4.o +else obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +endif obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += thp.o diff --git a/arch/sw_64/mm/extable.c b/arch/sw_64/mm/extable.c new file mode 100644 index 0000000000000000000000000000000000000000..d2678e12a1b1cb85ab257d6260fa33f0a7c36179 --- /dev/null +++ b/arch/sw_64/mm/extable.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +int fixup_exception(struct pt_regs *regs, unsigned long pc) +{ + const struct exception_table_entry *fixup; + + fixup = search_exception_tables(pc); + if (fixup) { + unsigned int valreg = fixup->fixup.bits.valreg; + unsigned int errreg = fixup->fixup.bits.errreg; + + if (valreg != 31) + regs->regs[valreg] = 0; + if (errreg != 31) + regs->regs[errreg] = -EFAULT; + pc += fixup->fixup.bits.nextinsn; + regs->pc = pc; + + return 1; + } + return 0; +} diff --git a/arch/sw_64/mm/fault.c b/arch/sw_64/mm/fault.c index 574fe7930aacd97d3830d6e3347770f9dab4eac6..5c9a42c77fb29776aad083adde73e4e0b2db3a00 100644 --- a/arch/sw_64/mm/fault.c +++ b/arch/sw_64/mm/fault.c @@ -121,7 +121,7 @@ unsigned long show_va_to_pa(struct mm_struct *mm, unsigned long addr) } pte = pte_offset_map(pmd, addr); if (pte_present(*pte)) { - ret = (unsigned long)pfn_to_virt(pte_val(*pte) >> _PFN_SHIFT); + ret = (unsigned long)pfn_to_virt(pte_pfn(*pte)); pr_debug("addr = %#lx, pgd = %#lx, pud = %#lx, pmd = %#lx, pte = %#lx, ret = %#lx\n", addr, *(unsigned long *)pgd, *(unsigned long *)pud, *(unsigned long *)pmd, *(unsigned long *)pte, ret); @@ -138,7 +138,6 @@ do_page_fault(unsigned long address, unsigned long mmcsr, { struct vm_area_struct *vma; struct mm_struct *mm = current->mm; - const struct exception_table_entry *fixup; int si_code = SEGV_MAPERR; vm_fault_t fault; unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; @@ -261,14 +260,8 @@ do_page_fault(unsigned long address, unsigned long mmcsr, no_context: /* Are we prepared to handle this fault as an exception? */ - fixup = search_exception_tables(regs->pc); - if (fixup != 0) { - unsigned long newpc; - - newpc = fixup_exception(map_regs, fixup, regs->pc); - regs->pc = newpc; + if (fixup_exception(regs, regs->pc)) return; - } /* * Oops. The kernel tried to access some bad page. We'll have to @@ -296,13 +289,13 @@ do_page_fault(unsigned long address, unsigned long mmcsr, * Send a sigbus, regardless of whether we were in kernel * or user mode. */ - force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *) address, 0); + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *) address); if (!user_mode(regs)) goto no_context; return; do_sigsegv: - force_sig_fault(SIGSEGV, si_code, (void __user *) address, 0); + force_sig_fault(SIGSEGV, si_code, (void __user *) address); if (unlikely(segv_debug_enabled)) { pr_info("fault: want to send_segv: pid %d, cause = %#lx, mmcsr = %#lx, address = %#lx, pc %#lx\n", diff --git a/arch/sw_64/mm/hugetlbpage.c b/arch/sw_64/mm/hugetlbpage.c index 2a40225af4d810361068bc5a6d8dcbad9b846e6e..f385afcb09a393775e9d39bfc9b85c60c532959e 100644 --- a/arch/sw_64/mm/hugetlbpage.c +++ b/arch/sw_64/mm/hugetlbpage.c @@ -19,7 +19,7 @@ int pmd_huge(pmd_t pmd) { return !pmd_none(pmd) && - (pmd_val(pmd) & (_PAGE_VALID | _PAGE_PSE)) != _PAGE_VALID; + (pmd_val(pmd) & (_PAGE_VALID | _PAGE_LEAF)) != _PAGE_VALID; } int pud_huge(pud_t pud) @@ -49,7 +49,7 @@ pte_t *sw64_256m_hugepte_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a return NULL; page = virt_to_page(pte); - pmd_val(*pmd) = pmd_val(*pmd) | _PAGE_PSE | _PAGE_PHU; + pmd_val(*pmd) = pmd_val(*pmd) | _PAGE_LEAF | _PAGE_CONT; for (i = 1; i < 32; i++) pmd_val(*(pmd+i)) = pmd_val(*pmd); return pte; @@ -101,7 +101,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, pmd = pmd_offset(pud, addr); if (!pmd_present(*pmd)) return NULL; - if (pmd_val(*pmd) & _PAGE_PHU) + if (pmd_val(*pmd) & _PAGE_CONT) pte = pte_offset_map(pmd, addr); else pte = (pte_t *) pmd; @@ -114,7 +114,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, static inline int sw64_huge_pmd_bad(pmd_t pmd) { return !(((pmd_val(pmd) & ~_PFN_MASK) == _PAGE_TABLE) || - ((pmd_val(pmd) & _PAGE_PHU) == _PAGE_PHU)); + ((pmd_val(pmd) & _PAGE_CONT) == _PAGE_CONT)); } static inline int sw64_huge_pmd_none_or_clear_bad(pmd_t *pmd) @@ -132,7 +132,7 @@ static void sw64_huge_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, unsigned long addr) { if ((((unsigned long)pmd & 0xffUL) == 0) && - ((pmd_val(*pmd) & _PAGE_PHU) == _PAGE_PHU)) { + ((pmd_val(*pmd) & _PAGE_CONT) == _PAGE_CONT)) { pgtable_t token = pmd_pgtable(*pmd); pmd_clear(pmd); @@ -306,19 +306,13 @@ arch_initcall(sw64_256m_hugetlb_init); #endif #endif /* CONFIG_HUGETLB_PAGE */ -static __init int setup_hugepagesz(char *opt) +bool __init arch_hugetlb_valid_size(unsigned long size) { - unsigned long ps = memparse(opt, &opt); - - if (ps == PMD_SIZE) { - hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); - } else if (ps == (PMD_SIZE << 5)) { - hugetlb_add_hstate(PMD_SHIFT + 5 - PAGE_SHIFT); - } else { - printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n", - ps >> 20); - return 0; + switch (size) { + case PMD_SIZE: + case (PMD_SIZE<<5): + return true; } - return 1; + + return false; } -__setup("hugepagesz=", setup_hugepagesz); diff --git a/arch/sw_64/mm/hugetlbpage_c4.c b/arch/sw_64/mm/hugetlbpage_c4.c new file mode 100644 index 0000000000000000000000000000000000000000..8c33643e6b87c54fd9a2e12d36e4bb04f8e04587 --- /dev/null +++ b/arch/sw_64/mm/hugetlbpage_c4.c @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SW_64 Huge TLB Page Support for Kernel. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/* + * pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal + * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry. + * Otherwise, returns 0. + */ +int pmd_huge(pmd_t pmd) +{ + return !pmd_none(pmd) && + (pmd_val(pmd) & (_PAGE_PRESENT|_PAGE_LEAF)) != _PAGE_PRESENT; +} + +int pud_huge(pud_t pud) +{ + return !pud_none(pud) && + (pud_val(pud) & (_PAGE_PRESENT|_PAGE_LEAF)) != _PAGE_PRESENT; +} +EXPORT_SYMBOL(pud_huge); + +#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE +#define want_pmd_share() (1) +#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ +#define want_pmd_share() (0) +#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ + +/* + * Select all bits except the pfn + */ +static inline pgprot_t pte_pgprot(pte_t pte) +{ + unsigned long pfn = pte_pfn(pte); + + return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); +} + +static inline int num_contig_ptes(unsigned long size, size_t *pgsize) +{ + int contig_ptes = 0; + + *pgsize = size; + + switch (size) { + case PUD_SIZE: + case PMD_SIZE: + contig_ptes = 1; + break; + case CONT_PMD_SIZE: + *pgsize = PMD_SIZE; + contig_ptes = CONT_PMDS; + break; + default: + break; + } + + return contig_ptes; +} + +static pte_t get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + unsigned long pgsize, unsigned long ncontig) +{ + pte_t orig_pte = huge_ptep_get(ptep); + unsigned long i; + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { + pte_t pte = ptep_get_and_clear(mm, addr, ptep); + + if (pte_dirty(pte)) + orig_pte = pte_mkdirty(orig_pte); + + if (pte_young(pte)) + orig_pte = pte_mkyoung(orig_pte); + } + + return orig_pte; +} + +static void clear_flush(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long pgsize, + unsigned long ncontig) +{ + struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0); + unsigned long i, saddr = addr; + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) + pte_clear(mm, addr, ptep); + + flush_tlb_range(&vma, saddr, addr); +} + +pte_t *huge_pte_alloc(struct mm_struct *mm, + unsigned long addr, unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pte_t *pte = NULL; + + pgd = pgd_offset(mm, addr); + p4d = p4d_alloc(mm, pgd, addr); + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return NULL; + + if (sz == PUD_SIZE) { + pte = (pte_t *)pud; + } else if (sz == PMD_SIZE) { + if (want_pmd_share() && pud_none(*pud)) + pte = huge_pmd_share(mm, addr, pud); + else + pte = (pte_t *)pmd_alloc(mm, pud, addr); + } else if (sz == (PMD_SIZE * CONT_PMDS)) { + pte = (pte_t *)pmd_alloc(mm, pud, addr); + WARN_ON(addr & (sz - 1)); + } + + WARN_ON(pte && !pte_none(*pte) && !pte_huge(*pte)); + return pte; +} + +pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, + unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd = NULL; + + pgd = pgd_offset(mm, addr); + if (!pgd_present(*pgd)) + return NULL; + + p4d = p4d_offset(pgd, addr); + if (!p4d_present(*p4d)) + return NULL; + + pud = pud_offset(p4d, addr); + + if (sz != PUD_SIZE && pud_none(*pud)) + return NULL; + /* hugepage or swap? */ + if (pud_huge(*pud) || !pud_present(*pud)) + return (pte_t *)pud; + /* table; check the next level */ + + if (sz == CONT_PMD_SIZE) + addr &= CONT_PMD_MASK; + + pmd = pmd_offset(pud, addr); + if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) && + pmd_none(*pmd)) + return NULL; + if (pmd_huge(*pmd) || !pmd_present(*pmd)) + return (pte_t *)pmd; + + return NULL; +} + +pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, + struct page *page, int writable) +{ + size_t pagesize = huge_page_size(hstate_vma(vma)); + + if (pagesize == CONT_PMD_SIZE) { + entry = pmd_pte(pmd_mkcont(pte_pmd(entry))); + } else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) { + pr_warn("%s: unrecognized huge page size 0x%lx\n", + __func__, pagesize); + } + return entry; +} + +void huge_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long sz) +{ + int i, ncontig; + size_t pgsize; + + ncontig = num_contig_ptes(sz, &pgsize); + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) + pte_clear(mm, addr, ptep); +} + +void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + size_t pgsize; + int i; + int ncontig; + unsigned long pfn; + pgprot_t hugeprot; + + /* + * Code needs to be expanded to handle huge swap and migration + * entries. Needed for HUGETLB and MEMORY_FAILURE. + */ + WARN_ON(!pte_present(pte)); + + if (!pte_cont(pte)) { + set_pte_at(mm, addr, ptep, pte); + return; + } + + ncontig = CONT_PMDS; + pfn = pte_pfn(pte); + hugeprot = pte_pgprot(pte); + + get_and_clear(mm, addr, ptep, pgsize, ncontig); + + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); +} + +void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned long sz) +{ + int i, ncontig; + size_t pgsize; + + ncontig = num_contig_ptes(sz, &pgsize); + + for (i = 0; i < ncontig; i++, ptep++) + set_pte(ptep, pte); +} + +void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + unsigned long pfn; + pgprot_t hugeprot; + int ncontig, i; + size_t pgsize; + pte_t pte; + + if (!pte_cont(READ_ONCE(*ptep))) { + ptep_set_wrprotect(mm, addr, ptep); + return; + } + + ncontig = CONT_PMDS; + + pte = get_and_clear(mm, addr, ptep, pgsize, ncontig); + pte = pte_wrprotect(pte); + + hugeprot = pte_pgprot(pte); + pfn = pte_pfn(pte); + + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); +} + +pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + int ncontig; + size_t pgsize; + pte_t orig_pte = huge_ptep_get(ptep); + + if (!pte_cont(orig_pte)) + return ptep_get_and_clear(mm, addr, ptep); + + ncontig = CONT_PMDS; + + return get_and_clear(mm, addr, ptep, pgsize, ncontig); +} + +void huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + size_t pgsize; + int ncontig; + + if (!pte_cont(READ_ONCE(*ptep))) { + ptep_clear_flush(vma, addr, ptep); + return; + } + + ncontig = CONT_PMDS; + clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig); +} + +static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig) +{ + int i; + + if (pte_write(pte) != pte_write(huge_ptep_get(ptep))) + return 1; + + for (i = 0; i < ncontig; i++) { + pte_t orig_pte = huge_ptep_get(ptep + i); + + if (pte_dirty(pte) != pte_dirty(orig_pte)) + return 1; + + if (pte_young(pte) != pte_young(orig_pte)) + return 1; + } + + return 0; +} + +int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, int dirty) +{ + int ncontig, i; + size_t pgsize = 0; + unsigned long pfn = pte_pfn(pte); + pgprot_t hugeprot; + pte_t orig_pte; + + if (!pte_cont(pte)) + return ptep_set_access_flags(vma, addr, ptep, pte, dirty); + + ncontig = CONT_PMDS; + + if (!__cont_access_flags_changed(ptep, pte, ncontig)) + return 0; + + orig_pte = get_and_clear(vma->vm_mm, addr, ptep, pgsize, ncontig); + flush_tlb_fix_spurious_fault(vma, addr); + + /* Make sure we don't lose the dirty or young state */ + if (pte_dirty(orig_pte)) + pte = pte_mkdirty(pte); + + if (pte_young(orig_pte)) + pte = pte_mkyoung(pte); + + hugeprot = pte_pgprot(pte); + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot)); + + return 1; +} + +#ifdef CONFIG_HUGETLB_PAGE +static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct vm_unmapped_area_info info; + + info.flags = 0; + info.length = len; + info.low_limit = current->mm->mmap_legacy_base; + info.high_limit = TASK_SIZE; + info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.align_offset = 0; + return vm_unmapped_area(&info); +} + +static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, + unsigned long addr0, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct vm_unmapped_area_info info; + unsigned long addr; + + info.flags = VM_UNMAPPED_AREA_TOPDOWN; + info.length = len; + info.low_limit = PAGE_SIZE; + info.high_limit = current->mm->mmap_base; + info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.align_offset = 0; + addr = vm_unmapped_area(&info); + + /* + * A failed mmap() very likely causes application failure, + * so fall back to the bottom-up function here. This scenario + * can happen with large stack limits and large mmap() + * allocations. + */ + if (addr & ~PAGE_MASK) { + VM_BUG_ON(addr != -ENOMEM); + info.flags = 0; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = TASK_SIZE; + addr = vm_unmapped_area(&info); + } + + return addr; +} + + unsigned long +hugetlb_get_unmapped_area(struct file *file, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + + if (len & ~huge_page_mask(h)) + return -EINVAL; + if (len > TASK_SIZE) + return -ENOMEM; + + if (flags & MAP_FIXED) { + if (prepare_hugepage_range(file, addr, len)) + return -EINVAL; + return addr; + } + + if (addr) { + addr = ALIGN(addr, huge_page_size(h)); + vma = find_vma(mm, addr); + if (TASK_SIZE - len >= addr && + (!vma || addr + len <= vma->vm_start)) + return addr; + } + if (mm->get_unmapped_area == arch_get_unmapped_area) + return hugetlb_get_unmapped_area_bottomup(file, addr, len, + pgoff, flags); + else + return hugetlb_get_unmapped_area_topdown(file, addr, len, + pgoff, flags); +} +#endif /* CONFIG_HUGETLB_PAGE */ + +static __init int setup_hugepagesz(char *opt) +{ + unsigned long ps = memparse(opt, &opt); + + switch (ps) { + case PUD_SIZE: + case PMD_SIZE * CONT_PMDS: + case PMD_SIZE: + hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT); + return 1; + } + + pr_err("hugepagesz: Unsupported page size %lu M\n", + ps >> 20); + return 0; +} +__setup("hugepagesz=", setup_hugepagesz); diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index 76104c09730907f0bd0fd511c1abad5157b9786e..f16bb4e1d2064302033bcf11d37c00b0a3883b59 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -24,7 +24,7 @@ struct numa_node_desc_t numa_nodes_desc[1]; * empty_zero_page is a special page that is used for * zero-initialized data and COW. */ -struct page *empty_zero_page; +unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; EXPORT_SYMBOL(empty_zero_page); pg_data_t *node_data[MAX_NUMNODES] __read_mostly; EXPORT_SYMBOL(node_data); @@ -65,6 +65,7 @@ static int __init setup_mem_size(char *p) } early_param("mem", setup_mem_size); +#if defined(CONFIG_SUBARCH_C3B) pgd_t * pgd_alloc(struct mm_struct *mm) { @@ -77,6 +78,17 @@ pgd_alloc(struct mm_struct *mm) return ret; } +#elif defined(CONFIG_SUBARCH_C4) +pgd_t * +pgd_alloc(struct mm_struct *mm) +{ + pgd_t *ret; + + ret = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + + return ret; +} +#endif /* Set up initial PCB, VPTB, and other such nicities. */ @@ -84,7 +96,7 @@ static inline void switch_to_system_map(void) { memset(swapper_pg_dir, 0, PAGE_SIZE); - wrptbr(virt_to_phys(swapper_pg_dir)); + update_ptbr_sys(virt_to_phys(swapper_pg_dir)); tbiv(); } @@ -104,14 +116,11 @@ void __init callback_init(void) void __init zone_sizes_init(void) { unsigned long max_zone_pfns[MAX_NR_ZONES]; - unsigned long dma_pfn; memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); - dma_pfn = PFN_DOWN(virt_to_phys((void *)MAX_DMA_ADDRESS)); - #ifdef CONFIG_ZONE_DMA32 - max_zone_pfns[ZONE_DMA32] = min(dma_pfn, max_low_pfn); + max_zone_pfns[ZONE_DMA32] = min(MAX_DMA32_PFN, max_low_pfn); #endif max_zone_pfns[ZONE_NORMAL] = max_low_pfn; @@ -123,12 +132,6 @@ void __init zone_sizes_init(void) */ void __init paging_init(void) { - void *zero_page; - - zero_page = __va(memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE)); - pr_info("zero page start: %p\n", zero_page); - memset(zero_page, 0, PAGE_SIZE); - empty_zero_page = virt_to_page(zero_page); } void __init mem_detect(void) @@ -172,7 +175,7 @@ void __init sw64_memblock_init(void) /* Make sure initrd is in memory range. */ if (sunway_boot_params->initrd_start) { - phys_addr_t base = __pa(sunway_boot_params->initrd_start); + phys_addr_t base = __boot_pa(sunway_boot_params->initrd_start); phys_addr_t size = sunway_boot_params->initrd_size; memblock_add(base, size); diff --git a/arch/sw_64/mm/numa.c b/arch/sw_64/mm/numa.c index 7cb13587e465d821f8fb56a6ea8e782091921d11..86b52bb140afec8a43ce319aca5a2a152b2dc601 100644 --- a/arch/sw_64/mm/numa.c +++ b/arch/sw_64/mm/numa.c @@ -378,7 +378,7 @@ void cpu_set_node(void) rr = first_node(node_online_map); for (i = 0; i < nr_cpu_ids; i++) { cid = cpu_to_rcid(i); - default_node = cid >> CORES_PER_NODE_SHIFT; + default_node = rcid_to_domain_id(cid); if (node_online(default_node)) { cpu_to_node_map[i] = default_node; } else { diff --git a/arch/sw_64/mm/physaddr.c b/arch/sw_64/mm/physaddr.c index 17840f4ef40bea3c26011d239d7c8aa1d5e020fc..c123dcad6aa6141561675b894f16fa0666ff42d2 100644 --- a/arch/sw_64/mm/physaddr.c +++ b/arch/sw_64/mm/physaddr.c @@ -6,30 +6,34 @@ unsigned long __phys_addr(unsigned long x) { - if (x >= __START_KERNEL_map) { - x -= __START_KERNEL_map; - VIRTUAL_BUG_ON(x >= KERNEL_IMAGE_SIZE); - } else { - VIRTUAL_BUG_ON(x < PAGE_OFFSET); - x -= PAGE_OFFSET; - VIRTUAL_BUG_ON(!phys_addr_valid(x)); - } + VIRTUAL_BUG_ON(x < PAGE_OFFSET); + x &= ~PAGE_OFFSET; + VIRTUAL_BUG_ON(!phys_addr_valid(x)); return x; } EXPORT_SYMBOL(__phys_addr); bool __virt_addr_valid(unsigned long x) { - if (x >= __START_KERNEL_map) { - x -= __START_KERNEL_map; - if (x >= KERNEL_IMAGE_SIZE) - return false; - } else { - if (x < PAGE_OFFSET) - return false; - x -= PAGE_OFFSET; - } - + if (x < PAGE_OFFSET) + return false; + x &= ~PAGE_OFFSET; return pfn_valid(x >> PAGE_SHIFT); } EXPORT_SYMBOL(__virt_addr_valid); + +#ifdef CONFIG_SUBARCH_C3B +#define LEGACY_BOOT_VA 0xffffffff80000000 +unsigned long __boot_phys_addr(unsigned long x) +{ + if (x >= LEGACY_BOOT_VA) { + x &= ~LEGACY_BOOT_VA; + VIRTUAL_BUG_ON(x >= KERNEL_IMAGE_SIZE); + } else { + VIRTUAL_BUG_ON(x < PAGE_OFFSET); + x &= ~PAGE_OFFSET; + VIRTUAL_BUG_ON(!phys_addr_valid(x)); + } + return x; +} +#endif diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index 43d548f7f755815444b7958458170f2e1f43306c..91bcb266f126dc4389e2a52446fe4649be0d7cd3 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -727,6 +727,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) break; case BPF_ALU | BPF_RSH | BPF_X: emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + fallthrough; case BPF_ALU64 | BPF_RSH | BPF_X: emit(SW64_BPF_SRL_REG(dst, src, dst), ctx); break; @@ -1010,6 +1011,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) src = tmp1; emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, dst, tmp2), ctx); dst = tmp2; + fallthrough; case BPF_JMP | BPF_JEQ | BPF_X: case BPF_JMP | BPF_JGT | BPF_X: case BPF_JMP | BPF_JLT | BPF_X: @@ -1080,6 +1082,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) case BPF_JMP32 | BPF_JSET | BPF_K: emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, dst, tmp2), ctx); dst = tmp2; + fallthrough; case BPF_JMP | BPF_JEQ | BPF_K: case BPF_JMP | BPF_JGT | BPF_K: case BPF_JMP | BPF_JLT | BPF_K: diff --git a/arch/sw_64/pci/Makefile b/arch/sw_64/pci/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..abc6553fd2c7906e655d576690bcf06772a7c3a9 --- /dev/null +++ b/arch/sw_64/pci/Makefile @@ -0,0 +1,8 @@ +PDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux kernel. +# + +obj-y += pci.o +obj-$(CONFIG_ACPI) += acpi.o + diff --git a/arch/sw_64/pci/acpi.c b/arch/sw_64/pci/acpi.c new file mode 100644 index 0000000000000000000000000000000000000000..c8666b9d82ae6a313805e7e24838c65567e769c8 --- /dev/null +++ b/arch/sw_64/pci/acpi.c @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c new file mode 100644 index 0000000000000000000000000000000000000000..510f3e72de75cb78e2572a2a3427029871781eee --- /dev/null +++ b/arch/sw_64/pci/pci.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#include +#include + diff --git a/arch/sw_64/platform/Makefile b/arch/sw_64/platform/Makefile index a972b931dea2374a00fa4880bd3787ff476c5e0d..4c0edceb4a2c1f4f7c8a5ee16617e80161b771a1 100644 --- a/arch/sw_64/platform/Makefile +++ b/arch/sw_64/platform/Makefile @@ -1,2 +1,2 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_PLATFORM_XUELANG) += platform_xuelang.o +obj-$(CONFIG_PLATFORM_XUELANG) += cpufreq_xuelang.o diff --git a/arch/sw_64/kernel/clock.c b/arch/sw_64/platform/cpufreq_xuelang.c similarity index 55% rename from arch/sw_64/kernel/clock.c rename to arch/sw_64/platform/cpufreq_xuelang.c index 8f2bec5fd7825dcfd455d766c6472f6bcac43a68..04b412c5f755298416b4eea092ea37b44a970d28 100644 --- a/arch/sw_64/kernel/clock.c +++ b/arch/sw_64/platform/cpufreq_xuelang.c @@ -1,29 +1,74 @@ // SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include +#include +#include +#include #include -#include -#include -#include -#include -#define CLK_PRT 0x1UL -#define CORE_CLK0_V (0x1UL << 1) -#define CORE_CLK0_R (0x1UL << 2) -#define CORE_CLK2_V (0x1UL << 15) -#define CORE_CLK2_R (0x1UL << 16) +/* Minimum CLK support */ +enum { + DC_0, DC_1, DC_2, DC_3, DC_4, DC_5, DC_6, DC_7, DC_8, + DC_9, DC_10, DC_11, DC_12, DC_13, DC_14, DC_15, DC_RESV +}; + +struct cpufreq_frequency_table freq_table[] = { + {0, 200, CPUFREQ_ENTRY_INVALID}, + {0, DC_1, CPUFREQ_ENTRY_INVALID}, + {0, DC_2, 0}, + {0, DC_3, 0}, + {0, DC_4, 0}, + {0, DC_5, 0}, + {0, DC_6, 0}, + {0, DC_7, 0}, + {0, DC_8, 0}, + {0, DC_9, 0}, + {0, DC_10, 0}, + {0, DC_11, 0}, + {0, DC_12, 0}, + {0, DC_13, 0}, + {0, DC_14, 0}, + {0, DC_15, 0}, + {-1, DC_RESV, CPUFREQ_TABLE_END}, +}; + -#define CLK_LV1_SEL_PRT 0x1UL -#define CLK_LV1_SEL_MUXA (0x1UL << 2) -#define CLK_LV1_SEL_MUXB (0x1UL << 3) +static struct platform_device sw64_cpufreq_device = { + .name = "sw64_cpufreq", + .id = -1, +}; + +static int __init sw64_cpufreq_init(void) +{ + int i; + unsigned char external_clk; + unsigned long max_rate, freq_off; -#define CORE_PLL0_CFG_SHIFT 4 -#define CORE_PLL2_CFG_SHIFT 18 + max_rate = get_cpu_freq() / 1000; + + external_clk = *((unsigned char *)__va(0x908011)); + + if (external_clk == 240) + freq_off = 60000; + else + freq_off = 50000; + + /* clock table init */ + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (i == 1) + freq_table[i].driver_data = freq_off * 24; + if (i == 2) + freq_table[i].frequency = freq_off * 36; + if (i > 2) + freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); + + if (freq_table[i].frequency == max_rate) + freq_table[i + 1].frequency = CPUFREQ_TABLE_END; + } + + return platform_device_register(&sw64_cpufreq_device); +} +arch_initcall(sw64_cpufreq_init); char curruent_policy[CPUFREQ_NAME_LEN]; @@ -48,7 +93,7 @@ unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy) val = sw64_io_read(0, CLK_CTL) >> CORE_PLL2_CFG_SHIFT; for (i = 0; ft[i].frequency != CPUFREQ_TABLE_END; i++) { - if (val == ft[i].driver_data) + if (val == i) return ft[i].frequency; } return 0; diff --git a/arch/sw_64/platform/platform_xuelang.c b/arch/sw_64/platform/platform_xuelang.c deleted file mode 100644 index 28c30cddcff2aa4ad08fee2e1987fcc2d75c997c..0000000000000000000000000000000000000000 --- a/arch/sw_64/platform/platform_xuelang.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -static inline void __iomem *xuelang_ioportmap(unsigned long addr) -{ - unsigned long io_offset; - - if (addr < 0x100000) { - io_offset = is_in_host() ? LPC_LEGACY_IO : PCI_VT_LEGACY_IO; - addr = addr | io_offset; - } - - return __va(addr); -} - -struct sw64_platform_ops xuelang_ops = { - .ioportmap = xuelang_ioportmap, - .ops_fixup = sw64_init_noop, -}; diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index a0c6e88bebe084bfc20fcdcd61a1e72992775abd..dc3ebb974eabd11a3f8118d7d68471792b12b8b6 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -740,4 +740,7 @@ config MICROCHIP_PIT64B modes and high resolution. It is used as a clocksource and a clockevent. +config SW64_TIMER + bool + endmenu diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index 1c444cc3bb4474016e7d45cc2e1794ea127f8824..70a9355f23a41c471422ccca6f28861e4f611e42 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -94,3 +94,4 @@ obj-$(CONFIG_CSKY_MP_TIMER) += timer-mp-csky.o obj-$(CONFIG_GX6605S_TIMER) += timer-gx6605s.o obj-$(CONFIG_HYPERV_TIMER) += hyperv_timer.o obj-$(CONFIG_MICROCHIP_PIT64B) += timer-microchip-pit64b.o +obj-$(CONFIG_SW64_TIMER) += timer-sw64.o diff --git a/arch/sw_64/kernel/timer.c b/drivers/clocksource/timer-sw64.c similarity index 31% rename from arch/sw_64/kernel/timer.c rename to drivers/clocksource/timer-sw64.c index f4745c1c6a7880cd6f9e1d9e42c7bcbf745c2561..10fab75e449a3fe0f968c1775d9a0cff87453448 100644 --- a/arch/sw_64/kernel/timer.c +++ b/drivers/clocksource/timer-sw64.c @@ -1,14 +1,276 @@ // SPDX-License-Identifier: GPL-2.0 -/* - * Filename: timer.c - * Description: percpu local timer, based on arch/x86/kernel/apic/apic.c - */ -#include #include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include +#include + +#define SHTCLK_RATE_KHZ 25000 +#define SHTCLK_RATE (SHTCLK_RATE_KHZ * 1000) + +#if defined(CONFIG_SUBARCH_C4) +static u64 read_longtime(struct clocksource *cs) +{ + return read_csr(CSR_SHTCLOCK); +} + +static struct clocksource clocksource_longtime = { + .name = "longtime", + .rating = 100, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_longtime, +}; + +static u64 notrace read_sched_clock(void) +{ + return read_csr(CSR_SHTCLOCK); +} + +void __init sw64_setup_clocksource(void) +{ + clocksource_register_khz(&clocksource_longtime, SHTCLK_RATE_KHZ); + sched_clock_register(read_sched_clock, BITS_PER_LONG, SHTCLK_RATE); +} + +void __init setup_sched_clock(void) { } +#elif defined(CONFIG_SUBARCH_C3B) +#ifdef CONFIG_SMP +static u64 read_longtime(struct clocksource *cs) +{ + unsigned long node; + + node = __this_cpu_read(hard_node_id); + return __io_read_longtime(node); +} + +static int longtime_enable(struct clocksource *cs) +{ + switch (cpu_desc.model) { + case CPU_SW3231: + sw64_io_write(0, GPIO_SWPORTA_DR, 0); + sw64_io_write(0, GPIO_SWPORTA_DDR, 0xff); + break; + case CPU_SW831: + __io_write_longtime_start_en(0, 0x1); + break; + default: + break; + } + + return 0; +} + +static struct clocksource clocksource_longtime = { + .name = "longtime", + .rating = 100, + .enable = longtime_enable, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_longtime, +}; + +static u64 read_vtime(struct clocksource *cs) +{ + unsigned long vtime_addr; + + vtime_addr = IO_BASE | LONG_TIME; + return rdio64(vtime_addr); +} + +static int vtime_enable(struct clocksource *cs) +{ + return 0; +} + +static struct clocksource clocksource_vtime = { + .name = "vtime", + .rating = 100, + .enable = vtime_enable, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_vtime, +}; +#else /* !SMP */ +static u64 read_tc(struct clocksource *cs) +{ + return rdtc(); +} + +static struct clocksource clocksource_tc = { + .name = "tc", + .rating = 300, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 22, + .mult = 0, /* To be filled in */ + .read = read_tc, +}; +#endif /* SMP */ + +#define DEFAULT_MCLK 25 /* Mhz */ + +void __init sw64_setup_clocksource(void) +{ + unsigned int mclk = *((unsigned char *)__va(0x908001)); + + if (!mclk) + mclk = DEFAULT_MCLK; + +#ifdef CONFIG_SMP + if (is_in_host()) + clocksource_register_khz(&clocksource_longtime, mclk * 1000); + else + clocksource_register_khz(&clocksource_vtime, DEFAULT_MCLK * 1000); +#else + clocksource_register_hz(&clocksource_tc, get_cpu_freq()); + pr_info("Setup clocksource TC, mult = %d\n", clocksource_tc.mult); +#endif +} + +DECLARE_PER_CPU(u64, tc_offset); +static u64 sc_start, sc_shift, sc_multi; +DEFINE_STATIC_KEY_FALSE(use_tc_as_sched_clock); + +static int __init sched_clock_setup(char *opt) +{ + if (!opt) + return -EINVAL; + + if (!strncmp(opt, "on", 2)) { + static_branch_enable(&use_tc_as_sched_clock); + pr_info("Using TC instead of jiffies as source of sched_clock()\n"); + } + + return 0; +} +early_param("tc_sched_clock", sched_clock_setup); + +static void __init calibrate_sched_clock(void) +{ + sc_start = rdtc(); +} + +void __init setup_sched_clock(void) +{ + unsigned long step; + + sc_shift = 7; + step = 1UL << sc_shift; + sc_multi = step * NSEC_PER_SEC / get_cpu_freq(); + calibrate_sched_clock(); + + pr_info("sched_clock: sc_multi=%llu, sc_shift=%llu\n", sc_multi, sc_shift); +} + +#ifdef CONFIG_GENERIC_SCHED_CLOCK +static u64 notrace read_sched_clock(void) +{ + return (rdtc() - sc_start) >> sc_shift; +} + +void __init sw64_sched_clock_init(void) +{ + sched_clock_register(sched_clock_read, BITS_PER_LONG, get_cpu_freq() >> sc_shift); +} +#else /* !CONFIG_GENERIC_SCHED_CLOCK */ +/* + * scheduler clock - returns current time in nanoseconds. + */ +unsigned long long notrace sched_clock(void) +{ + if (static_branch_likely(&use_tc_as_sched_clock)) + return ((rdtc() - sc_start + __this_cpu_read(tc_offset)) >> sc_shift) * sc_multi; + else + return (jiffies - INITIAL_JIFFIES) * (NSEC_PER_SEC / HZ); +} + +#ifdef CONFIG_DEBUG_FS +static ssize_t sched_clock_status_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + char buf[2]; + + if (static_key_enabled(&use_tc_as_sched_clock)) + buf[0] = 'Y'; + else + buf[0] = 'N'; + buf[1] = '\n'; + return simple_read_from_buffer(user_buf, count, ppos, buf, 2); +} + +static ssize_t sched_clock_status_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + int r; + bool bv; + bool val = static_key_enabled(&use_tc_as_sched_clock); + + r = kstrtobool_from_user(user_buf, count, &bv); + if (!r) { + if (val != bv) { + if (bv) { + static_branch_enable(&use_tc_as_sched_clock); + pr_info("source of sched_clock() switched from jiffies to TC\n"); + } else { + static_branch_disable(&use_tc_as_sched_clock); + pr_info("source of sched_clock() switched from TC to jiffies\n"); + } + } else { + if (val) + pr_info("source of sched_clock() unchanged (using TC)\n"); + else + pr_info("source of sched_clock() unchanged (using jiffies)\n"); + } + } + + return count; +} + +static const struct file_operations sched_clock_status_fops = { + .read = sched_clock_status_read, + .write = sched_clock_status_write, + .open = nonseekable_open, + .llseek = no_llseek, +}; + +static int __init sched_clock_debug_init(void) +{ + struct dentry *sched_clock_status; + + if (!sw64_debugfs_dir) + return -ENODEV; + + sched_clock_status = debugfs_create_file("tc_sched_clock", + 0644, sw64_debugfs_dir, NULL, + &sched_clock_status_fops); + + if (!sched_clock_status) + return -ENOMEM; + + return 0; +} +late_initcall(sched_clock_debug_init); +#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_GENERIC_SCHED_CLOCK */ + +#endif + + static int timer_next_event(unsigned long delta, struct clock_event_device *evt); @@ -93,17 +355,26 @@ void sw64_update_clockevents(unsigned long cpu, u32 freq) if (cpu == smp_processor_id()) clockevents_update_freq(swevt, freq); + else { + clockevents_calc_mult_shift(swevt, freq, 4); + swevt->min_delta_ns = clockevent_delta2ns(swevt->min_delta_ticks, swevt); + swevt->max_delta_ns = clockevent_delta2ns(swevt->max_delta_ticks, swevt); + } } /* - * Setup the local timer for this CPU. Copy the initilized values + * Setup the local timer for this CPU. Copy the initialized values * of the boot CPU and register the clock event in the framework. */ -void setup_timer(void) +void sw64_setup_timer(void) { + unsigned long min_delta; int cpu = smp_processor_id(); struct clock_event_device *swevt = &per_cpu(timer_events, cpu); + /* min_delta ticks => 100ns */ + min_delta = get_cpu_freq()/1000/1000/10; + if (is_in_guest()) { memcpy(swevt, &vtimer_clockevent, sizeof(*swevt)); /* @@ -111,19 +382,13 @@ void setup_timer(void) * If it's too small, the timer will timeout when the IER * haven't been opened. */ - swevt->min_delta_ns = 400; + min_delta *= 4; } else { memcpy(swevt, &timer_clockevent, sizeof(*swevt)); - swevt->min_delta_ns = 100; } - swevt->cpumask = cpumask_of(cpu); - swevt->mult = div_sc(get_cpu_freq(), NSEC_PER_SEC, swevt->shift); - swevt->max_delta_ns = clockevent_delta2ns(0xFFFFFFFFFFFFFFFF, swevt); - swevt->set_state_shutdown(swevt); - - clockevents_register_device(swevt); + clockevents_config_and_register(swevt, get_cpu_freq(), min_delta, ULONG_MAX); } void sw64_timer_interrupt(void) diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index ffa99a9d1299d6fadd531bcb51a46501398c88d8..e85a4a54356bd340d7f604477b6a5d1ed59706e2 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -356,16 +356,25 @@ endif if SW64 config SW64_CPUFREQ - bool "sw64 CPU Frequency interface for Chip3 Asic" - depends on SW64_CHIP3 + bool "SW64 CPU Frequency interface" + depends on UNCORE_XUELANG default y help - This adds the CPUFreq driver for SW6B processor which supports + This adds the CPUFreq driver for SW64 processor which supports software configurable cpu frequency. For details, take a look at . If unsure, say N. + +config SW64_CPUFREQ_DEBUGFS + bool "SW64 CPU Frequency debugfs interface" + depends on SW64_CPUFREQ && DEBUG_FS + default y + help + Turns on the DebugFS interface for CPU Frequency. + + If you don't know what to do here, say N. endif config QORIQ_CPUFREQ diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index a02e0e518a050d751c388d25f59c7bfb124620b3..d6c180ca39a7085c155c3291d35db012b134eaaa 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -112,3 +112,4 @@ obj-$(CONFIG_SH_CPU_FREQ) += sh-cpufreq.o obj-$(CONFIG_SPARC_US2E_CPUFREQ) += sparc-us2e-cpufreq.o obj-$(CONFIG_SPARC_US3_CPUFREQ) += sparc-us3-cpufreq.o obj-$(CONFIG_SW64_CPUFREQ) += sw64_cpufreq.o +obj-$(CONFIG_SW64_CPUFREQ_DEBUGFS) += sw64_cpufreq_debugfs.o diff --git a/drivers/cpufreq/sw64_cpufreq.c b/drivers/cpufreq/sw64_cpufreq.c index 9259753c8f061e110cf4e81ae39fb3b384db9f29..0cc10670e22df484ea615242287c5f876a42f946 100644 --- a/drivers/cpufreq/sw64_cpufreq.c +++ b/drivers/cpufreq/sw64_cpufreq.c @@ -20,40 +20,13 @@ #include #include -#include +#include #include -#define CRYSTAL_BIT (1UL << 34) - static uint nowait; static struct clk *cpuclk; -/* Minimum CLK support */ -enum { - DC_0, DC_1, DC_2, DC_3, DC_4, DC_5, DC_6, DC_7, DC_8, - DC_9, DC_10, DC_11, DC_12, DC_13, DC_14, DC_15, DC_RESV -}; - -static struct cpufreq_frequency_table freq_table[] = { - {0, DC_0, CPUFREQ_ENTRY_INVALID}, - {0, DC_1, CPUFREQ_ENTRY_INVALID}, - {0, DC_2, 0}, - {0, DC_3, 0}, - {0, DC_4, 0}, - {0, DC_5, 0}, - {0, DC_6, 0}, - {0, DC_7, 0}, - {0, DC_8, 0}, - {0, DC_9, 0}, - {0, DC_10, 0}, - {0, DC_11, 0}, - {0, DC_12, 0}, - {0, DC_13, 0}, - {0, DC_14, 0}, - {0, DC_15, 0}, - {-1, DC_RESV, CPUFREQ_TABLE_END}, -}; static int sw64_cpu_freq_notifier(struct notifier_block *nb, unsigned long val, void *data); @@ -69,7 +42,7 @@ static int sw64_cpu_freq_notifier(struct notifier_block *nb, unsigned long cpu = freqs->policy->cpu; if (val == CPUFREQ_POSTCHANGE) - sw64_update_clockevents(cpu, freqs->new * 1000000); + sw64_update_clockevents(cpu, freqs->new * 1000); return 0; } @@ -102,41 +75,23 @@ static int sw64_cpufreq_target(struct cpufreq_policy *policy, /* setting the cpu frequency */ sw64_set_rate(index); + update_cpu_freq(freq_table[index].frequency); return 0; } static int sw64_cpufreq_cpu_init(struct cpufreq_policy *policy) { - int i; - unsigned long max_rate, freq_off; - cpuclk = sw64_clk_get(NULL, "cpu_clk"); if (IS_ERR(cpuclk)) { pr_err("couldn't get CPU clk\n"); return PTR_ERR(cpuclk); } - max_rate = get_cpu_freq() / 1000000; - - if (sw64_io_read(0, INIT_CTL) & CRYSTAL_BIT) - freq_off = 50; - else - freq_off = 60; - - /* clock table init */ - for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { - if (i == 2) - freq_table[i].frequency = freq_off * 36; - if (i > 2) - freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); - - if (freq_table[i].frequency == max_rate) - freq_table[i + 1].frequency = CPUFREQ_TABLE_END; - } - policy->clk = cpuclk; + sw64_store_policy(policy); + cpufreq_generic_init(policy, freq_table, 0); return 0; @@ -187,6 +142,11 @@ static int __init cpufreq_init(void) { int ret; + if (is_in_guest()) { + pr_warn("Now sw_64 CPUFreq does not support virtual machines\n"); + return -ENODEV; + } + /* Register platform stuff */ ret = platform_driver_register(&platform_driver); if (ret) diff --git a/arch/sw_64/chip/chip3/cpufreq_debugfs.c b/drivers/cpufreq/sw64_cpufreq_debugfs.c similarity index 39% rename from arch/sw_64/chip/chip3/cpufreq_debugfs.c rename to drivers/cpufreq/sw64_cpufreq_debugfs.c index c58f1cee3907bd33b00a0c9ec49c664a5ff9e250..bb4ae26bc22b4a561039433f4e291c9a87d78a32 100644 --- a/arch/sw_64/chip/chip3/cpufreq_debugfs.c +++ b/drivers/cpufreq/sw64_cpufreq_debugfs.c @@ -6,40 +6,27 @@ #include #include - -#define CLK_PRT 0x1UL -#define CORE_CLK0_V (0x1UL << 1) -#define CORE_CLK0_R (0x1UL << 2) -#define CORE_CLK2_V (0x1UL << 15) -#define CORE_CLK2_R (0x1UL << 16) - -#define CLK_LV1_SEL_PRT 0x1UL -#define CLK_LV1_SEL_MUXA (0x1UL << 2) -#define CLK_LV1_SEL_MUXB (0x1UL << 3) - -#define CORE_PLL0_CFG_SHIFT 4 -#define CORE_PLL2_CFG_SHIFT 18 - -static int cpu_freq[16] = { - 200, 1200, 1800, 1900, - 1950, 2000, 2050, 2100, - 2150, 2200, 2250, 2300, - 2350, 2400, 2450, 2500 -}; +#include static int cpufreq_show(struct seq_file *m, void *v) { int i; u64 val; + int freq; val = sw64_io_read(0, CLK_CTL); val = val >> CORE_PLL2_CFG_SHIFT; - for (i = 0; i < sizeof(cpu_freq)/sizeof(int); i++) { - if (cpu_freq[val] == cpu_freq[i]) - seq_printf(m, "[%d] ", cpu_freq[i]); + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (freq_table[i].frequency != CPUFREQ_ENTRY_INVALID) + freq = freq_table[i].frequency; else - seq_printf(m, "%d ", cpu_freq[i]); + freq = freq_table[i].driver_data; + + if (val == i) + seq_printf(m, "[%d] ", freq); + else + seq_printf(m, "%d ", freq); } seq_puts(m, "\n"); @@ -56,8 +43,7 @@ static ssize_t cpufreq_set(struct file *file, const char __user *user_buf, { char buf[5]; size_t size; - int cf, i, err, index; - u64 val; + int cf, i, err, index, freq; size = min(sizeof(buf) - 1, len); if (copy_from_user(buf, user_buf, size)) @@ -69,8 +55,13 @@ static ssize_t cpufreq_set(struct file *file, const char __user *user_buf, return err; index = -1; - for (i = 0; i < sizeof(cpu_freq)/sizeof(int); i++) { - if (cf == cpu_freq[i]) { + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (freq_table[i].frequency != CPUFREQ_ENTRY_INVALID) + freq = freq_table[i].frequency; + else + freq = freq_table[i].driver_data; + + if (cf == freq) { index = i; break; } @@ -79,46 +70,8 @@ static ssize_t cpufreq_set(struct file *file, const char __user *user_buf, if (index < 0) return -EINVAL; - /* Set CLK_CTL PLL2 */ - sw64_io_write(0, CLK_CTL, CORE_CLK2_R | CORE_CLK2_V | CLK_PRT); - sw64_io_write(1, CLK_CTL, CORE_CLK2_R | CORE_CLK2_V | CLK_PRT); - val = sw64_io_read(0, CLK_CTL); - - sw64_io_write(0, CLK_CTL, val | index << CORE_PLL2_CFG_SHIFT); - sw64_io_write(1, CLK_CTL, val | index << CORE_PLL2_CFG_SHIFT); - - udelay(1); - - sw64_io_write(0, CLK_CTL, CORE_CLK2_V | CLK_PRT - | index << CORE_PLL2_CFG_SHIFT); - sw64_io_write(1, CLK_CTL, CORE_CLK2_V | CLK_PRT - | index << CORE_PLL2_CFG_SHIFT); - val = sw64_io_read(0, CLK_CTL); - - /* LV1 select PLL1/PLL2 */ - sw64_io_write(0, CLU_LV1_SEL, CLK_LV1_SEL_MUXA | CLK_LV1_SEL_PRT); - sw64_io_write(1, CLU_LV1_SEL, CLK_LV1_SEL_MUXA | CLK_LV1_SEL_PRT); - - /* Set CLK_CTL PLL0 */ - sw64_io_write(0, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V); - sw64_io_write(1, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V); - - sw64_io_write(0, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V - | index << CORE_PLL0_CFG_SHIFT); - sw64_io_write(1, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V - | index << CORE_PLL0_CFG_SHIFT); - - udelay(1); - - sw64_io_write(0, CLK_CTL, val | CORE_CLK0_V - | index << CORE_PLL0_CFG_SHIFT); - sw64_io_write(1, CLK_CTL, val | CORE_CLK0_V - | index << CORE_PLL0_CFG_SHIFT); - - /* LV1 select PLL0/PLL1 */ - sw64_io_write(0, CLU_LV1_SEL, CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PRT); - sw64_io_write(1, CLU_LV1_SEL, CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PRT); - + sw64_set_rate(index); + update_cpu_freq(freq); return len; } diff --git a/drivers/firmware/efi/sunway-init.c b/drivers/firmware/efi/sunway-init.c index d3af23feb8bb602359888827c34e7d498c72cbea..870abc2f5afea3ab7c4b9674fc0c4c7a4b93b1e9 100644 --- a/drivers/firmware/efi/sunway-init.c +++ b/drivers/firmware/efi/sunway-init.c @@ -36,9 +36,10 @@ static int __init is_memory(efi_memory_desc_t *md) return 0; } static efi_config_table_type_t arch_tables[] __initdata = { - {SMBIOS3_TABLE_GUID, NULL, NULL}, + {SMBIOS3_TABLE_GUID, NULL, ""}, {SLEEP_ENTRY_GUID, &entSuspend, "SLEEP ENTRY"}, - {BIOS_VERSION_GUID, &bios_version, "BIOS VERSION"} + {BIOS_VERSION_GUID, &bios_version, "BIOS VERSION"}, + {}, }; static int __init uefi_init(u64 efi_system_table) diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index f745cd6d440f3dc30bd01af71acd7e8ed9bc5c61..a5f18dfa896ef3b8d8469b40e30db7bfc1d83ce2 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -323,6 +323,17 @@ config I2C_VIAPRO This driver can also be built as a module. If so, the module will be called i2c-viapro. +config I2C_SUNWAY + tristate "Sunway i2c lib" + depends on UNCORE_XUELANG + help + If you say yes to this option, support will be included for the + Sunway Soc I2C interface on SW64 platfrom. + + This driver can also be built as a module. If so, the module + will be called i2c-sunway. + + if ACPI comment "ACPI drivers" @@ -554,15 +565,6 @@ config I2C_DESIGNWARE_PLATFORM This driver can also be built as a module. If so, the module will be called i2c-designware-platform. -config I2C_SUNWAY_SW6 - tristate "SUNWAY SW6 I2C Controller " - depends on SW64_CHIP3 - default y if PLATFORM_XUELANG - help - If you say yes to this option, support will be included for the - Synopsys DesignWare I2C adapter in MCU of CHIP3. Only master mode - is supported. - config I2C_DESIGNWARE_AMDPSP bool "AMD PSP I2C semaphore support" depends on X86_MSR diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index 1271a75117507b1a814675663c629fbafec071ea..e2a0df0fd858317d4c855eb56d75090086ff2c2c 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_I2C_SIS630) += i2c-sis630.o obj-$(CONFIG_I2C_SIS96X) += i2c-sis96x.o obj-$(CONFIG_I2C_VIA) += i2c-via.o obj-$(CONFIG_I2C_VIAPRO) += i2c-viapro.o +obj-$(CONFIG_I2C_SUNWAY) += i2c-sunway.o # Mac SMBus host controller drivers obj-$(CONFIG_I2C_HYDRA) += i2c-hydra.o @@ -51,10 +52,6 @@ obj-$(CONFIG_I2C_DAVINCI) += i2c-davinci.o obj-$(CONFIG_I2C_DESIGNWARE_CORE) += i2c-designware-core.o i2c-designware-core-y := i2c-designware-common.o i2c-designware-core-y += i2c-designware-master.o -obj-$(CONFIG_I2C_SUNWAY_SW6) += i2c-sunway-sw6.o -i2c-sunway-sw6-y := i2c-sunway-sw6-common.o -i2c-sunway-sw6-y += i2c-sunway-sw6-master.o -i2c-sunway-sw6-y += i2c-sunway-sw6-platdrv.o i2c-designware-core-$(CONFIG_I2C_DESIGNWARE_SLAVE) += i2c-designware-slave.o obj-$(CONFIG_I2C_DESIGNWARE_PLATFORM) += i2c-designware-platform.o i2c-designware-platform-y := i2c-designware-platdrv.o diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c index c9f7783ac7cb1e8b88a51e549302828c1b3a9878..87f36f676b4e98fa9014b9b9e0cb4f9399a8717a 100644 --- a/drivers/i2c/busses/i2c-designware-common.c +++ b/drivers/i2c/busses/i2c-designware-common.c @@ -63,6 +63,9 @@ static int dw_reg_read(void *context, unsigned int reg, unsigned int *val) { struct dw_i2c_dev *dev = context; + if ((dev->flags & MODEL_MASK) == MODEL_SUNWAY) + reg = reg << 7; + *val = readl(dev->base + reg); return 0; @@ -72,6 +75,9 @@ static int dw_reg_write(void *context, unsigned int reg, unsigned int val) { struct dw_i2c_dev *dev = context; + if ((dev->flags & MODEL_MASK) == MODEL_SUNWAY) + reg = reg << 7; + writel(val, dev->base + reg); return 0; @@ -149,6 +155,8 @@ int i2c_dw_init_regmap(struct dw_i2c_dev *dev) return ret; reg = readl(dev->base + DW_IC_COMP_TYPE); + if ((dev->flags & MODEL_MASK) == MODEL_SUNWAY) + reg = readl(dev->base + (DW_IC_COMP_TYPE << 7)); i2c_dw_release_lock(dev); if (reg == swab32(DW_IC_COMP_TYPE_VALUE)) { diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index 9b51f3300d478f74c298b5b421007d0f18aa6baf..cccddd3b63bea8aa200b30eca53669b0c71330f6 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -297,6 +297,7 @@ struct dw_i2c_dev { #define MODEL_MSCC_OCELOT 0x00000100 #define MODEL_BAIKAL_BT1 0x00000200 +#define MODEL_SUNWAY 0x00000800 #define MODEL_MASK 0x00000f00 struct i2c_dw_semaphore_callbacks { diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index c0d4f2b990c58054e0b123e437822d57bc90cc6a..d88a083fd82cd7114374472f564287d60b0011a5 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -151,9 +151,14 @@ static int dw_i2c_of_configure(struct platform_device *pdev) } static const struct of_device_id dw_i2c_of_match[] = { +#ifdef CONFIG_SW64 + { .compatible = "snps,designware-i2c", .data = (void *)MODEL_SUNWAY }, +#else { .compatible = "snps,designware-i2c", }, +#endif { .compatible = "mscc,ocelot-i2c", .data = (void *)MODEL_MSCC_OCELOT }, { .compatible = "baikal,bt1-sys-i2c", .data = (void *)MODEL_BAIKAL_BT1 }, + { .compatible = "sunway,suntai-i2c", .data = (void *)MODEL_SUNWAY }, {}, }; MODULE_DEVICE_TABLE(of, dw_i2c_of_match); diff --git a/drivers/i2c/busses/i2c-sunway-sw6-common.c b/drivers/i2c/busses/i2c-sunway-sw6-common.c deleted file mode 100644 index 80a59d199f7da947b5d1276c07f4120891d425f5..0000000000000000000000000000000000000000 --- a/drivers/i2c/busses/i2c-sunway-sw6-common.c +++ /dev/null @@ -1,362 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Synopsys DesignWare I2C adapter driver. - * - * Based on the TI DAVINCI I2C adapter driver. - * - * Copyright (C) 2006 Texas Instruments. - * Copyright (C) 2007 MontaVista Software Inc. - * Copyright (C) 2009 Provigent Ltd. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "i2c-sunway-sw6-core.h" - -static char *abort_sources[] = { - [ABRT_7B_ADDR_NOACK] = - "slave address not acknowledged (7bit mode)", - [ABRT_10ADDR1_NOACK] = - "first address byte not acknowledged (10bit mode)", - [ABRT_10ADDR2_NOACK] = - "second address byte not acknowledged (10bit mode)", - [ABRT_TXDATA_NOACK] = - "data not acknowledged", - [ABRT_GCALL_NOACK] = - "no acknowledgement for a general call", - [ABRT_GCALL_READ] = - "read after general call", - [ABRT_SBYTE_ACKDET] = - "start byte acknowledged", - [ABRT_SBYTE_NORSTRT] = - "trying to send start byte when restart is disabled", - [ABRT_10B_RD_NORSTRT] = - "trying to read when restart is disabled (10bit mode)", - [ABRT_MASTER_DIS] = - "trying to use disabled adapter", - [ARB_LOST] = - "lost arbitration", - [ABRT_SLAVE_FLUSH_TXFIFO] = - "read command so flush old data in the TX FIFO", - [ABRT_SLAVE_ARBLOST] = - "slave lost the bus while transmitting data to a remote master", - [ABRT_SLAVE_RD_INTX] = - "incorrect slave-transmitter mode configuration", -}; - -u32 dw_readl(struct dw_i2c_dev *dev, int offset) -{ - u32 value; - - offset = offset << 7; - if (dev->flags & ACCESS_16BIT) - value = readw_relaxed(dev->base + offset) | - (readw_relaxed(dev->base + offset + 2) << 16); - else - value = readl_relaxed(dev->base + offset); - - if (dev->flags & ACCESS_SWAP) - return swab32(value); - else - return value; -} - -void dw_writel(struct dw_i2c_dev *dev, u32 b, int offset) -{ - offset = offset << 7; - if (dev->flags & ACCESS_SWAP) - b = swab32(b); - - if (dev->flags & ACCESS_16BIT) { - writew_relaxed((u16)b, dev->base + offset); - writew_relaxed((u16)(b >> 16), dev->base + offset + 2); - } else { - writel_relaxed(b, dev->base + offset); - } -} - -/** - * i2c_dw_set_reg_access() - Set register access flags - * @dev: device private data - * - * Autodetects needed register access mode and sets access flags accordingly. - * This must be called before doing any other register access. - */ -int i2c_dw_set_reg_access(struct dw_i2c_dev *dev) -{ - u32 reg; - int ret; - - ret = i2c_dw_acquire_lock(dev); - if (ret) - return ret; - - reg = dw_readl(dev, DW_IC_COMP_TYPE); - i2c_dw_release_lock(dev); - - if (reg == swab32(DW_IC_COMP_TYPE_VALUE)) { - /* Configure register endianness access */ - dev->flags |= ACCESS_SWAP; - } else if (reg == (DW_IC_COMP_TYPE_VALUE & 0x0000ffff)) { - /* Configure register access mode 16bit */ - dev->flags |= ACCESS_16BIT; - } else if (reg != DW_IC_COMP_TYPE_VALUE) { - dev_err(dev->dev, - "Unknown Synopsys component type: 0x%08x\n", reg); - return -ENODEV; - } - - return 0; -} - -u32 i2c_dw_scl_hcnt(u32 ic_clk, u32 tSYMBOL, u32 tf, int cond, int offset) -{ - /* - * DesignWare I2C core doesn't seem to have solid strategy to meet - * the tHD;STA timing spec. Configuring _HCNT based on tHIGH spec - * will result in violation of the tHD;STA spec. - */ - if (cond) - /* - * Conditional expression: - * - * IC_[FS]S_SCL_HCNT + (1+4+3) >= IC_CLK * tHIGH - * - * This is based on the DW manuals, and represents an ideal - * configuration. The resulting I2C bus speed will be - * faster than any of the others. - * - * If your hardware is free from tHD;STA issue, try this one. - */ - return (ic_clk * tSYMBOL + 500000) / 1000000 - 8 + offset; - else - /* - * Conditional expression: - * - * IC_[FS]S_SCL_HCNT + 3 >= IC_CLK * (tHD;STA + tf) - * - * This is just experimental rule; the tHD;STA period turned - * out to be proportinal to (_HCNT + 3). With this setting, - * we could meet both tHIGH and tHD;STA timing specs. - * - * If unsure, you'd better to take this alternative. - * - * The reason why we need to take into account "tf" here, - * is the same as described in i2c_dw_scl_lcnt(). - */ - return (ic_clk * (tSYMBOL + tf) + 500000) / 1000000 - - 3 + offset; -} - -u32 i2c_dw_scl_lcnt(u32 ic_clk, u32 tLOW, u32 tf, int offset) -{ - /* - * Conditional expression: - * - * IC_[FS]S_SCL_LCNT + 1 >= IC_CLK * (tLOW + tf) - * - * DW I2C core starts counting the SCL CNTs for the LOW period - * of the SCL clock (tLOW) as soon as it pulls the SCL line. - * In order to meet the tLOW timing spec, we need to take into - * account the fall time of SCL signal (tf). Default tf value - * should be 0.3 us, for safety. - */ - return ((ic_clk * (tLOW + tf) + 500000) / 1000000) - 1 + offset; -} - -int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev) -{ - u32 reg; - int ret; - - ret = i2c_dw_acquire_lock(dev); - if (ret) - return ret; - - /* Configure SDA Hold Time if required */ - reg = dw_readl(dev, DW_IC_COMP_VERSION); - if (reg >= DW_IC_SDA_HOLD_MIN_VERS) { - if (!dev->sda_hold_time) { - /* Keep previous hold time setting if no one set it */ - dev->sda_hold_time = dw_readl(dev, DW_IC_SDA_HOLD); - } - - /* - * Workaround for avoiding TX arbitration lost in case I2C - * slave pulls SDA down "too quickly" after falling egde of - * SCL by enabling non-zero SDA RX hold. Specification says it - * extends incoming SDA low to high transition while SCL is - * high but it apprears to help also above issue. - */ - if (!(dev->sda_hold_time & DW_IC_SDA_HOLD_RX_MASK)) - dev->sda_hold_time |= 1 << DW_IC_SDA_HOLD_RX_SHIFT; - - dev_dbg(dev->dev, "SDA Hold Time TX:RX = %d:%d\n", - dev->sda_hold_time & ~(u32)DW_IC_SDA_HOLD_RX_MASK, - dev->sda_hold_time >> DW_IC_SDA_HOLD_RX_SHIFT); - } else if (dev->sda_hold_time) { - dev_warn(dev->dev, - "Hardware too old to adjust SDA hold time.\n"); - dev->sda_hold_time = 0; - } - - i2c_dw_release_lock(dev); - - return 0; -} - -void __i2c_dw_disable(struct dw_i2c_dev *dev) -{ - int timeout = 100; - - do { - __i2c_dw_disable_nowait(dev); - /* - * The enable status register may be unimplemented, but - * in that case this test reads zero and exits the loop. - */ - if ((dw_readl(dev, DW_IC_ENABLE_STATUS) & 1) == 0) - return; - - /* - * Wait 10 times the signaling period of the highest I2C - * transfer supported by the driver (for 400KHz this is - * 25us) as described in the DesignWare I2C databook. - */ - usleep_range(25, 250); - } while (timeout--); - - dev_warn(dev->dev, "timeout in disabling adapter\n"); -} - -unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev) -{ - /* - * Clock is not necessary if we got LCNT/HCNT values directly from - * the platform code. - */ - if (WARN_ON_ONCE(!dev->get_clk_rate_khz)) - return 0; - return dev->get_clk_rate_khz(dev); -} - -int i2c_dw_prepare_clk(struct dw_i2c_dev *dev, bool prepare) -{ - if (IS_ERR(dev->clk)) - return PTR_ERR(dev->clk); - - if (prepare) - return clk_prepare_enable(dev->clk); - - clk_disable_unprepare(dev->clk); - return 0; -} -EXPORT_SYMBOL_GPL(i2c_dw_prepare_clk); - -int i2c_dw_acquire_lock(struct dw_i2c_dev *dev) -{ - int ret; - - if (!dev->acquire_lock) - return 0; - - ret = dev->acquire_lock(dev); - if (!ret) - return 0; - - dev_err(dev->dev, "couldn't acquire bus ownership\n"); - - return ret; -} - -void i2c_dw_release_lock(struct dw_i2c_dev *dev) -{ - if (dev->release_lock) - dev->release_lock(dev); -} - -/* - * Waiting for bus not busy - */ -int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev) -{ - int timeout = TIMEOUT; - - while (dw_readl(dev, DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY) { - if (timeout <= 0) { - dev_warn(dev->dev, "timeout waiting for bus ready\n"); - i2c_recover_bus(&dev->adapter); - - if (dw_readl(dev, DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY) - return -ETIMEDOUT; - return 0; - } - timeout--; - usleep_range(1000, 1100); - } - - return 0; -} - -int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev) -{ - unsigned long abort_source = dev->abort_source; - int i; - - if (abort_source & DW_IC_TX_ABRT_NOACK) { - for_each_set_bit(i, &abort_source, ARRAY_SIZE(abort_sources)) - dev_dbg(dev->dev, - "%s: %s\n", __func__, abort_sources[i]); - return -EREMOTEIO; - } - - for_each_set_bit(i, &abort_source, ARRAY_SIZE(abort_sources)) - dev_err(dev->dev, "%s: %s\n", __func__, abort_sources[i]); - - if (abort_source & DW_IC_TX_ARB_LOST) - return -EAGAIN; - else if (abort_source & DW_IC_TX_ABRT_GCALL_READ) - return -EINVAL; /* wrong msgs[] data */ - else - return -EIO; -} - -u32 i2c_dw_func(struct i2c_adapter *adap) -{ - struct dw_i2c_dev *dev = i2c_get_adapdata(adap); - - return dev->functionality; -} - -void i2c_dw_disable(struct dw_i2c_dev *dev) -{ - /* Disable controller */ - __i2c_dw_disable(dev); - - /* Disable all interrupts */ - dw_writel(dev, 0, DW_IC_INTR_MASK); - dw_readl(dev, DW_IC_CLR_INTR); -} - -void i2c_dw_disable_int(struct dw_i2c_dev *dev) -{ - dw_writel(dev, 0, DW_IC_INTR_MASK); -} - -u32 i2c_dw_read_comp_param(struct dw_i2c_dev *dev) -{ - return dw_readl(dev, DW_IC_COMP_PARAM_1); -} -EXPORT_SYMBOL_GPL(i2c_dw_read_comp_param); - -MODULE_DESCRIPTION("Synopsys DesignWare I2C bus adapter core"); -MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-sunway-sw6-core.h b/drivers/i2c/busses/i2c-sunway-sw6-core.h deleted file mode 100644 index 8fd0a0b4a5c93b3695934536a7d2394691d11093..0000000000000000000000000000000000000000 --- a/drivers/i2c/busses/i2c-sunway-sw6-core.h +++ /dev/null @@ -1,325 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Synopsys DesignWare I2C adapter driver. - * - * Based on the TI DAVINCI I2C adapter driver. - * - * Copyright (C) 2006 Texas Instruments. - * Copyright (C) 2007 MontaVista Software Inc. - * Copyright (C) 2009 Provigent Ltd. - */ - -#include -#include - -#define DW_IC_DEFAULT_FUNCTIONALITY (I2C_FUNC_I2C | \ - I2C_FUNC_SMBUS_BYTE | \ - I2C_FUNC_SMBUS_BYTE_DATA | \ - I2C_FUNC_SMBUS_WORD_DATA | \ - I2C_FUNC_SMBUS_BLOCK_DATA | \ - I2C_FUNC_SMBUS_I2C_BLOCK) - -#define DW_IC_CON_MASTER 0x1 -#define DW_IC_CON_SPEED_STD 0x2 -#define DW_IC_CON_SPEED_FAST 0x4 -#define DW_IC_CON_SPEED_HIGH 0x6 -#define DW_IC_CON_SPEED_MASK 0x6 -#define DW_IC_CON_10BITADDR_SLAVE 0x8 -#define DW_IC_CON_10BITADDR_MASTER 0x10 -#define DW_IC_CON_RESTART_EN 0x20 -#define DW_IC_CON_SLAVE_DISABLE 0x40 -#define DW_IC_CON_STOP_DET_IFADDRESSED 0x80 -#define DW_IC_CON_TX_EMPTY_CTRL 0x100 -#define DW_IC_CON_RX_FIFO_FULL_HLD_CTRL 0x200 - -/* - * Registers offset - */ -#define DW_IC_CON 0x0 -#define DW_IC_TAR 0x4 -#define DW_IC_SAR 0x8 -#define DW_IC_DATA_CMD 0x10 -#define DW_IC_SS_SCL_HCNT 0x14 -#define DW_IC_SS_SCL_LCNT 0x18 -#define DW_IC_FS_SCL_HCNT 0x1c -#define DW_IC_FS_SCL_LCNT 0x20 -#define DW_IC_HS_SCL_HCNT 0x24 -#define DW_IC_HS_SCL_LCNT 0x28 -#define DW_IC_INTR_STAT 0x2c -#define DW_IC_INTR_MASK 0x30 -#define DW_IC_RAW_INTR_STAT 0x34 -#define DW_IC_RX_TL 0x38 -#define DW_IC_TX_TL 0x3c -#define DW_IC_CLR_INTR 0x40 -#define DW_IC_CLR_RX_UNDER 0x44 -#define DW_IC_CLR_RX_OVER 0x48 -#define DW_IC_CLR_TX_OVER 0x4c -#define DW_IC_CLR_RD_REQ 0x50 -#define DW_IC_CLR_TX_ABRT 0x54 -#define DW_IC_CLR_RX_DONE 0x58 -#define DW_IC_CLR_ACTIVITY 0x5c -#define DW_IC_CLR_STOP_DET 0x60 -#define DW_IC_CLR_START_DET 0x64 -#define DW_IC_CLR_GEN_CALL 0x68 -#define DW_IC_ENABLE 0x6c -#define DW_IC_STATUS 0x70 -#define DW_IC_TXFLR 0x74 -#define DW_IC_RXFLR 0x78 -#define DW_IC_SDA_HOLD 0x7c -#define DW_IC_TX_ABRT_SOURCE 0x80 -#define DW_IC_ENABLE_STATUS 0x9c -#define DW_IC_CLR_RESTART_DET 0xa8 -#define DW_IC_COMP_PARAM_1 0xf4 -#define DW_IC_COMP_VERSION 0xf8 -#define DW_IC_SDA_HOLD_MIN_VERS 0x3131312A -#define DW_IC_COMP_TYPE 0xfc -#define DW_IC_COMP_TYPE_VALUE 0x44570140 - -#define DW_IC_INTR_RX_UNDER 0x001 -#define DW_IC_INTR_RX_OVER 0x002 -#define DW_IC_INTR_RX_FULL 0x004 -#define DW_IC_INTR_TX_OVER 0x008 -#define DW_IC_INTR_TX_EMPTY 0x010 -#define DW_IC_INTR_RD_REQ 0x020 -#define DW_IC_INTR_TX_ABRT 0x040 -#define DW_IC_INTR_RX_DONE 0x080 -#define DW_IC_INTR_ACTIVITY 0x100 -#define DW_IC_INTR_STOP_DET 0x200 -#define DW_IC_INTR_START_DET 0x400 -#define DW_IC_INTR_GEN_CALL 0x800 -#define DW_IC_INTR_RESTART_DET 0x1000 - -#define DW_IC_INTR_DEFAULT_MASK (DW_IC_INTR_RX_FULL | \ - DW_IC_INTR_TX_ABRT | \ - DW_IC_INTR_STOP_DET) -#define DW_IC_INTR_MASTER_MASK (DW_IC_INTR_DEFAULT_MASK | \ - DW_IC_INTR_TX_EMPTY) -#define DW_IC_INTR_SLAVE_MASK (DW_IC_INTR_DEFAULT_MASK | \ - DW_IC_INTR_RX_DONE | \ - DW_IC_INTR_RX_UNDER | \ - DW_IC_INTR_RD_REQ) - -#define DW_IC_STATUS_ACTIVITY 0x1 -#define DW_IC_STATUS_TFE BIT(2) -#define DW_IC_STATUS_MASTER_ACTIVITY BIT(5) -#define DW_IC_STATUS_SLAVE_ACTIVITY BIT(6) - -#define DW_IC_SDA_HOLD_RX_SHIFT 16 -#define DW_IC_SDA_HOLD_RX_MASK GENMASK(23, DW_IC_SDA_HOLD_RX_SHIFT) - -#define DW_IC_ERR_TX_ABRT 0x1 - -#define DW_IC_TAR_10BITADDR_MASTER BIT(12) - -#define DW_IC_COMP_PARAM_1_SPEED_MODE_HIGH (BIT(2) | BIT(3)) -#define DW_IC_COMP_PARAM_1_SPEED_MODE_MASK GENMASK(3, 2) - -/* - * status codes - */ -#define STATUS_IDLE 0x0 -#define STATUS_WRITE_IN_PROGRESS 0x1 -#define STATUS_READ_IN_PROGRESS 0x2 - -#define TIMEOUT 20 /* ms */ - -/* - * operation modes - */ -#define DW_IC_MASTER 0 -#define DW_IC_SLAVE 1 - -/* - * Hardware abort codes from the DW_IC_TX_ABRT_SOURCE register - * - * Only expected abort codes are listed here - * refer to the datasheet for the full list - */ -#define ABRT_7B_ADDR_NOACK 0 -#define ABRT_10ADDR1_NOACK 1 -#define ABRT_10ADDR2_NOACK 2 -#define ABRT_TXDATA_NOACK 3 -#define ABRT_GCALL_NOACK 4 -#define ABRT_GCALL_READ 5 -#define ABRT_SBYTE_ACKDET 7 -#define ABRT_SBYTE_NORSTRT 9 -#define ABRT_10B_RD_NORSTRT 10 -#define ABRT_MASTER_DIS 11 -#define ARB_LOST 12 -#define ABRT_SLAVE_FLUSH_TXFIFO 13 -#define ABRT_SLAVE_ARBLOST 14 -#define ABRT_SLAVE_RD_INTX 15 - -#define DW_IC_TX_ABRT_7B_ADDR_NOACK (1UL << ABRT_7B_ADDR_NOACK) -#define DW_IC_TX_ABRT_10ADDR1_NOACK (1UL << ABRT_10ADDR1_NOACK) -#define DW_IC_TX_ABRT_10ADDR2_NOACK (1UL << ABRT_10ADDR2_NOACK) -#define DW_IC_TX_ABRT_TXDATA_NOACK (1UL << ABRT_TXDATA_NOACK) -#define DW_IC_TX_ABRT_GCALL_NOACK (1UL << ABRT_GCALL_NOACK) -#define DW_IC_TX_ABRT_GCALL_READ (1UL << ABRT_GCALL_READ) -#define DW_IC_TX_ABRT_SBYTE_ACKDET (1UL << ABRT_SBYTE_ACKDET) -#define DW_IC_TX_ABRT_SBYTE_NORSTRT (1UL << ABRT_SBYTE_NORSTRT) -#define DW_IC_TX_ABRT_10B_RD_NORSTRT (1UL << ABRT_10B_RD_NORSTRT) -#define DW_IC_TX_ABRT_MASTER_DIS (1UL << ABRT_MASTER_DIS) -#define DW_IC_TX_ARB_LOST (1UL << ARB_LOST) -#define DW_IC_RX_ABRT_SLAVE_RD_INTX (1UL << ABRT_SLAVE_RD_INTX) -#define DW_IC_RX_ABRT_SLAVE_ARBLOST (1UL << ABRT_SLAVE_ARBLOST) -#define DW_IC_RX_ABRT_SLAVE_FLUSH_TXFIFO (1UL << ABRT_SLAVE_FLUSH_TXFIFO) - -#define DW_IC_TX_ABRT_NOACK (DW_IC_TX_ABRT_7B_ADDR_NOACK | \ - DW_IC_TX_ABRT_10ADDR1_NOACK | \ - DW_IC_TX_ABRT_10ADDR2_NOACK | \ - DW_IC_TX_ABRT_TXDATA_NOACK | \ - DW_IC_TX_ABRT_GCALL_NOACK) - - -/** - * struct dw_i2c_dev - private i2c-designware data - * @dev: driver model device node - * @base: IO registers pointer - * @cmd_complete: tx completion indicator - * @clk: input reference clock - * @slave: represent an I2C slave device - * @cmd_err: run time hadware error code - * @msgs: points to an array of messages currently being transferred - * @msgs_num: the number of elements in msgs - * @msg_write_idx: the element index of the current tx message in the msgs - * array - * @tx_buf_len: the length of the current tx buffer - * @tx_buf: the current tx buffer - * @msg_read_idx: the element index of the current rx message in the msgs - * array - * @rx_buf_len: the length of the current rx buffer - * @rx_buf: the current rx buffer - * @msg_err: error status of the current transfer - * @status: i2c master status, one of STATUS_* - * @abort_source: copy of the TX_ABRT_SOURCE register - * @irq: interrupt number for the i2c master - * @adapter: i2c subsystem adapter node - * @slave_cfg: configuration for the slave device - * @tx_fifo_depth: depth of the hardware tx fifo - * @rx_fifo_depth: depth of the hardware rx fifo - * @rx_outstanding: current master-rx elements in tx fifo - * @timings: bus clock frequency, SDA hold and other timings - * @sda_hold_time: SDA hold value - * @ss_hcnt: standard speed HCNT value - * @ss_lcnt: standard speed LCNT value - * @fs_hcnt: fast speed HCNT value - * @fs_lcnt: fast speed LCNT value - * @fp_hcnt: fast plus HCNT value - * @fp_lcnt: fast plus LCNT value - * @hs_hcnt: high speed HCNT value - * @hs_lcnt: high speed LCNT value - * @pm_qos: pm_qos_request used while holding a hardware lock on the bus - * @acquire_lock: function to acquire a hardware lock on the bus - * @release_lock: function to release a hardware lock on the bus - * @pm_disabled: true if power-management should be disabled for this i2c-bus - * @disable: function to disable the controller - * @disable_int: function to disable all interrupts - * @init: function to initialize the I2C hardware - * @mode: operation mode - DW_IC_MASTER or DW_IC_SLAVE - * - * HCNT and LCNT parameters can be used if the platform knows more accurate - * values than the one computed based only on the input clock frequency. - * Leave them to be %0 if not used. - */ -struct dw_i2c_dev { - struct device *dev; - void __iomem *base; - struct completion cmd_complete; - struct clk *clk; - struct reset_control *rst; - struct i2c_client *slave; - u32 (*get_clk_rate_khz)(struct dw_i2c_dev *dev); - struct dw_pci_controller *controller; - int cmd_err; - struct i2c_msg *msgs; - int msgs_num; - int msg_write_idx; - u32 tx_buf_len; - u8 *tx_buf; - int msg_read_idx; - u32 rx_buf_len; - u8 *rx_buf; - int msg_err; - unsigned int status; - u32 abort_source; - int irq; - u32 flags; - struct i2c_adapter adapter; - u32 functionality; - u32 master_cfg; - u32 slave_cfg; - unsigned int tx_fifo_depth; - unsigned int rx_fifo_depth; - int rx_outstanding; - struct i2c_timings timings; - u32 sda_hold_time; - u32 intr_mask_value; - u16 ss_hcnt; - u16 ss_lcnt; - u16 fs_hcnt; - u16 fs_lcnt; - u16 fp_hcnt; - u16 fp_lcnt; - u16 hs_hcnt; - u16 hs_lcnt; - struct pm_qos_request pm_qos; - int (*acquire_lock)(struct dw_i2c_dev *dev); - void (*release_lock)(struct dw_i2c_dev *dev); - bool pm_disabled; - void (*disable)(struct dw_i2c_dev *dev); - void (*disable_int)(struct dw_i2c_dev *dev); - int (*init)(struct dw_i2c_dev *dev); - int mode; - struct i2c_bus_recovery_info rinfo; -}; - -#define ACCESS_SWAP 0x00000001 -#define ACCESS_16BIT 0x00000002 -#define ACCESS_INTR_MASK 0x00000004 - -#define MODEL_CHERRYTRAIL 0x00000100 - -u32 dw_readl(struct dw_i2c_dev *dev, int offset); -void dw_writel(struct dw_i2c_dev *dev, u32 b, int offset); -int i2c_dw_set_reg_access(struct dw_i2c_dev *dev); -u32 i2c_dw_scl_hcnt(u32 ic_clk, u32 tSYMBOL, u32 tf, int cond, int offset); -u32 i2c_dw_scl_lcnt(u32 ic_clk, u32 tLOW, u32 tf, int offset); -int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev); -unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev); -int i2c_dw_prepare_clk(struct dw_i2c_dev *dev, bool prepare); -int i2c_dw_acquire_lock(struct dw_i2c_dev *dev); -void i2c_dw_release_lock(struct dw_i2c_dev *dev); -int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev); -int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev); -u32 i2c_dw_func(struct i2c_adapter *adap); -void i2c_dw_disable(struct dw_i2c_dev *dev); -void i2c_dw_disable_int(struct dw_i2c_dev *dev); - -static inline void __i2c_dw_enable(struct dw_i2c_dev *dev) -{ - dw_writel(dev, 1, DW_IC_ENABLE); -} - -static inline void __i2c_dw_disable_nowait(struct dw_i2c_dev *dev) -{ - dw_writel(dev, 0, DW_IC_ENABLE); -} - -void __i2c_dw_disable(struct dw_i2c_dev *dev); - -extern u32 i2c_dw_read_comp_param(struct dw_i2c_dev *dev); -extern int i2c_dw_probe(struct dw_i2c_dev *dev); -#if IS_ENABLED(CONFIG_I2C_DESIGNWARE_SLAVE) -extern int i2c_dw_probe_slave(struct dw_i2c_dev *dev); -#else -static inline int i2c_dw_probe_slave(struct dw_i2c_dev *dev) { return -EINVAL; } -#endif - -#if IS_ENABLED(CONFIG_I2C_DESIGNWARE_BAYTRAIL) -extern int i2c_dw_probe_lock_support(struct dw_i2c_dev *dev); -extern void i2c_dw_remove_lock_support(struct dw_i2c_dev *dev); -#else -static inline int i2c_dw_probe_lock_support(struct dw_i2c_dev *dev) { return 0; } -static inline void i2c_dw_remove_lock_support(struct dw_i2c_dev *dev) {} -#endif diff --git a/drivers/i2c/busses/i2c-sunway-sw6-master.c b/drivers/i2c/busses/i2c-sunway-sw6-master.c deleted file mode 100644 index eccfd338ab362089bd43f3d987733a6e988a8c93..0000000000000000000000000000000000000000 --- a/drivers/i2c/busses/i2c-sunway-sw6-master.c +++ /dev/null @@ -1,755 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Synopsys DesignWare I2C adapter driver (master only). - * - * Based on the TI DAVINCI I2C adapter driver. - * - * Copyright (C) 2006 Texas Instruments. - * Copyright (C) 2007 MontaVista Software Inc. - * Copyright (C) 2009 Provigent Ltd. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "i2c-sunway-sw6-core.h" - -static void i2c_dw_configure_fifo_master(struct dw_i2c_dev *dev) -{ - /* Configure Tx/Rx FIFO threshold levels */ - dw_writel(dev, dev->tx_fifo_depth / 2, DW_IC_TX_TL); - dw_writel(dev, 0, DW_IC_RX_TL); - - /* Configure the I2C master */ - dw_writel(dev, dev->master_cfg, DW_IC_CON); -} - -static int i2c_dw_set_timings_master(struct dw_i2c_dev *dev) -{ - const char *mode_str, *fp_str = ""; - u32 comp_param1; - u32 sda_falling_time, scl_falling_time; - struct i2c_timings *t = &dev->timings; - u32 ic_clk; - int ret; - - ret = i2c_dw_acquire_lock(dev); - if (ret) - return ret; - comp_param1 = dw_readl(dev, DW_IC_COMP_PARAM_1); - i2c_dw_release_lock(dev); - - /* Set standard and fast speed dividers for high/low periods */ - sda_falling_time = t->sda_fall_ns ?: 300; /* ns */ - scl_falling_time = t->scl_fall_ns ?: 300; /* ns */ - - /* Calculate SCL timing parameters for standard mode if not set */ - if (!dev->ss_hcnt || !dev->ss_lcnt) { - ic_clk = i2c_dw_clk_rate(dev); - dev->ss_hcnt = - i2c_dw_scl_hcnt(ic_clk, - 4000, /* tHD;STA = tHIGH = 4.0 us */ - sda_falling_time, - 0, /* 0: DW default, 1: Ideal */ - 0); /* No offset */ - dev->ss_lcnt = - i2c_dw_scl_lcnt(ic_clk, - 4700, /* tLOW = 4.7 us */ - scl_falling_time, - 0); /* No offset */ - } - dev_dbg(dev->dev, "Standard Mode HCNT:LCNT = %d:%d\n", - dev->ss_hcnt, dev->ss_lcnt); - - /* - * Set SCL timing parameters for fast mode or fast mode plus. Only - * difference is the timing parameter values since the registers are - * the same. - */ - if (t->bus_freq_hz == 1000000) { - /* - * Check are fast mode plus parameters available and use - * fast mode if not. - */ - if (dev->fp_hcnt && dev->fp_lcnt) { - dev->fs_hcnt = dev->fp_hcnt; - dev->fs_lcnt = dev->fp_lcnt; - fp_str = " Plus"; - } - } - /* - * Calculate SCL timing parameters for fast mode if not set. They are - * needed also in high speed mode. - */ - if (!dev->fs_hcnt || !dev->fs_lcnt) { - ic_clk = i2c_dw_clk_rate(dev); - dev->fs_hcnt = - i2c_dw_scl_hcnt(ic_clk, - 600, /* tHD;STA = tHIGH = 0.6 us */ - sda_falling_time, - 0, /* 0: DW default, 1: Ideal */ - 0); /* No offset */ - dev->fs_lcnt = - i2c_dw_scl_lcnt(ic_clk, - 1300, /* tLOW = 1.3 us */ - scl_falling_time, - 0); /* No offset */ - } - dev_dbg(dev->dev, "Fast Mode%s HCNT:LCNT = %d:%d\n", - fp_str, dev->fs_hcnt, dev->fs_lcnt); - - /* Check is high speed possible and fall back to fast mode if not */ - if ((dev->master_cfg & DW_IC_CON_SPEED_MASK) == - DW_IC_CON_SPEED_HIGH) { - if ((comp_param1 & DW_IC_COMP_PARAM_1_SPEED_MODE_MASK) - != DW_IC_COMP_PARAM_1_SPEED_MODE_HIGH) { - dev_err(dev->dev, "High Speed not supported!\n"); - dev->master_cfg &= ~DW_IC_CON_SPEED_MASK; - dev->master_cfg |= DW_IC_CON_SPEED_FAST; - dev->hs_hcnt = 0; - dev->hs_lcnt = 0; - } else if (dev->hs_hcnt && dev->hs_lcnt) { - dev_dbg(dev->dev, "High Speed Mode HCNT:LCNT = %d:%d\n", - dev->hs_hcnt, dev->hs_lcnt); - } - } - - ret = i2c_dw_set_sda_hold(dev); - if (ret) - goto out; - - switch (dev->master_cfg & DW_IC_CON_SPEED_MASK) { - case DW_IC_CON_SPEED_STD: - mode_str = "Standard Mode"; - break; - case DW_IC_CON_SPEED_HIGH: - mode_str = "High Speed Mode"; - break; - default: - mode_str = "Fast Mode"; - } - dev_dbg(dev->dev, "Bus speed: %s%s\n", mode_str, fp_str); - -out: - return ret; -} - -/** - * i2c_dw_init() - Initialize the designware I2C master hardware - * @dev: device private data - * - * This functions configures and enables the I2C master. - * This function is called during I2C init function, and in case of timeout at - * run time. - */ -static int i2c_dw_init_master(struct dw_i2c_dev *dev) -{ - int ret; - - ret = i2c_dw_acquire_lock(dev); - if (ret) - return ret; - - /* Disable the adapter */ - __i2c_dw_disable(dev); - - /* Write standard speed timing parameters */ - dw_writel(dev, dev->ss_hcnt, DW_IC_SS_SCL_HCNT); - dw_writel(dev, dev->ss_lcnt, DW_IC_SS_SCL_LCNT); - - /* Write fast mode/fast mode plus timing parameters */ - dw_writel(dev, dev->fs_hcnt, DW_IC_FS_SCL_HCNT); - dw_writel(dev, dev->fs_lcnt, DW_IC_FS_SCL_LCNT); - - /* Write high speed timing parameters if supported */ - if (dev->hs_hcnt && dev->hs_lcnt) { - dw_writel(dev, dev->hs_hcnt, DW_IC_HS_SCL_HCNT); - dw_writel(dev, dev->hs_lcnt, DW_IC_HS_SCL_LCNT); - } - - /* Write SDA hold time if supported */ - if (dev->sda_hold_time) - dw_writel(dev, dev->sda_hold_time, DW_IC_SDA_HOLD); - - i2c_dw_configure_fifo_master(dev); - i2c_dw_release_lock(dev); - - return 0; -} - -static void i2c_dw_xfer_init(struct dw_i2c_dev *dev) -{ - struct i2c_msg *msgs = dev->msgs; - u32 ic_con, ic_tar = 0; - - /* Disable the adapter */ - __i2c_dw_disable(dev); - - /* If the slave address is ten bit address, enable 10BITADDR */ - ic_con = dw_readl(dev, DW_IC_CON); - if (msgs[dev->msg_write_idx].flags & I2C_M_TEN) { - ic_con |= DW_IC_CON_10BITADDR_MASTER; - /* - * If I2C_DYNAMIC_TAR_UPDATE is set, the 10-bit addressing - * mode has to be enabled via bit 12 of IC_TAR register. - * We set it always as I2C_DYNAMIC_TAR_UPDATE can't be - * detected from registers. - */ - ic_tar = DW_IC_TAR_10BITADDR_MASTER; - } else { - ic_con &= ~DW_IC_CON_10BITADDR_MASTER; - } - - dw_writel(dev, ic_con, DW_IC_CON); - - /* - * Set the slave (target) address and enable 10-bit addressing mode - * if applicable. - */ - dw_writel(dev, msgs[dev->msg_write_idx].addr | ic_tar, DW_IC_TAR); - - /* Enforce disabled interrupts (due to HW issues) */ - i2c_dw_disable_int(dev); - - /* Enable the adapter */ - __i2c_dw_enable(dev); - - /* Dummy read to avoid the register getting stuck on Bay Trail */ - dw_readl(dev, DW_IC_ENABLE_STATUS); - - /* Clear and enable interrupts */ - dw_readl(dev, DW_IC_CLR_INTR); - dev->intr_mask_value = DW_IC_INTR_DEFAULT_MASK; - dw_writel(dev, DW_IC_INTR_MASTER_MASK, DW_IC_INTR_MASK); -} - -/* - * Initiate (and continue) low level master read/write transaction. - * This function is only called from i2c_dw_isr, and pumping i2c_msg - * messages into the tx buffer. Even if the size of i2c_msg data is - * longer than the size of the tx buffer, it handles everything. - */ -static void -i2c_dw_xfer_msg(struct dw_i2c_dev *dev) -{ - struct i2c_msg *msgs = dev->msgs; - u32 intr_mask; - int tx_limit, rx_limit; - u32 addr = msgs[dev->msg_write_idx].addr; - u32 buf_len = dev->tx_buf_len; - u8 *buf = dev->tx_buf; - bool need_restart = false; - - intr_mask = DW_IC_INTR_MASTER_MASK; - - for (; dev->msg_write_idx < dev->msgs_num; dev->msg_write_idx++) { - u32 flags = msgs[dev->msg_write_idx].flags; - - /* - * If target address has changed, we need to - * reprogram the target address in the I2C - * adapter when we are done with this transfer. - */ - if (msgs[dev->msg_write_idx].addr != addr) { - dev_err(dev->dev, - "%s: invalid target address\n", __func__); - dev->msg_err = -EINVAL; - break; - } - - if (!(dev->status & STATUS_WRITE_IN_PROGRESS)) { - /* new i2c_msg */ - buf = msgs[dev->msg_write_idx].buf; - buf_len = msgs[dev->msg_write_idx].len; - - /* If both IC_EMPTYFIFO_HOLD_MASTER_EN and - * IC_RESTART_EN are set, we must manually - * set restart bit between messages. - */ - if ((dev->master_cfg & DW_IC_CON_RESTART_EN) && - (dev->msg_write_idx > 0)) - need_restart = true; - } - - tx_limit = dev->tx_fifo_depth - dw_readl(dev, DW_IC_TXFLR); - rx_limit = dev->rx_fifo_depth - dw_readl(dev, DW_IC_RXFLR); - - while (buf_len > 0 && tx_limit > 0 && rx_limit > 0) { - u32 cmd = 0; - - /* - * If IC_EMPTYFIFO_HOLD_MASTER_EN is set we must - * manually set the stop bit. However, it cannot be - * detected from the registers so we set it always - * when writing/reading the last byte. - */ - - /* - * i2c-core always sets the buffer length of - * I2C_FUNC_SMBUS_BLOCK_DATA to 1. The length will - * be adjusted when receiving the first byte. - * Thus we can't stop the transaction here. - */ - if (dev->msg_write_idx == dev->msgs_num - 1 && - buf_len == 1 && !(flags & I2C_M_RECV_LEN)) - cmd |= BIT(9); - - if (need_restart) { - cmd |= BIT(10); - need_restart = false; - } - - if (msgs[dev->msg_write_idx].flags & I2C_M_RD) { - - /* Avoid rx buffer overrun */ - if (dev->rx_outstanding >= dev->rx_fifo_depth) - break; - - dw_writel(dev, cmd | 0x100, DW_IC_DATA_CMD); - rx_limit--; - dev->rx_outstanding++; - } else - dw_writel(dev, cmd | *buf++, DW_IC_DATA_CMD); - tx_limit--; buf_len--; - } - - dev->tx_buf = buf; - dev->tx_buf_len = buf_len; - - /* - * Because we don't know the buffer length in the - * I2C_FUNC_SMBUS_BLOCK_DATA case, we can't stop - * the transaction here. - */ - if (buf_len > 0 || flags & I2C_M_RECV_LEN) { - /* more bytes to be written */ - dev->status |= STATUS_WRITE_IN_PROGRESS; - break; - } else - dev->status &= ~STATUS_WRITE_IN_PROGRESS; - } - - /* - * If i2c_msg index search is completed, we don't need TX_EMPTY - * interrupt any more. - */ - if (dev->msg_write_idx == dev->msgs_num) - intr_mask &= ~DW_IC_INTR_TX_EMPTY; - - if (dev->msg_err) - intr_mask = 0; - - dev->intr_mask_value = intr_mask; -} - -static u8 -i2c_dw_recv_len(struct dw_i2c_dev *dev, u8 len) -{ - struct i2c_msg *msgs = dev->msgs; - u32 flags = msgs[dev->msg_read_idx].flags; - - /* - * Adjust the buffer length and mask the flag - * after receiving the first byte. - */ - len += (flags & I2C_CLIENT_PEC) ? 2 : 1; - dev->tx_buf_len = len - min_t(u8, len, dev->rx_outstanding); - msgs[dev->msg_read_idx].len = len; - msgs[dev->msg_read_idx].flags &= ~I2C_M_RECV_LEN; - - return len; -} - -static void -i2c_dw_read(struct dw_i2c_dev *dev) -{ - struct i2c_msg *msgs = dev->msgs; - int rx_valid; - - for (; dev->msg_read_idx < dev->msgs_num; dev->msg_read_idx++) { - u32 len; - u8 *buf; - - if (!(msgs[dev->msg_read_idx].flags & I2C_M_RD)) - continue; - - if (!(dev->status & STATUS_READ_IN_PROGRESS)) { - len = msgs[dev->msg_read_idx].len; - buf = msgs[dev->msg_read_idx].buf; - } else { - len = dev->rx_buf_len; - buf = dev->rx_buf; - } - - rx_valid = dw_readl(dev, DW_IC_RXFLR); - - for (; len > 0 && rx_valid > 0; len--, rx_valid--) { - u32 flags = msgs[dev->msg_read_idx].flags; - - *buf = dw_readl(dev, DW_IC_DATA_CMD); - /* Ensure length byte is a valid value */ - if (flags & I2C_M_RECV_LEN && - *buf <= I2C_SMBUS_BLOCK_MAX && *buf > 0) { - len = i2c_dw_recv_len(dev, *buf); - } - buf++; - dev->rx_outstanding--; - } - - if (len > 0) { - dev->status |= STATUS_READ_IN_PROGRESS; - dev->rx_buf_len = len; - dev->rx_buf = buf; - return; - } else - dev->status &= ~STATUS_READ_IN_PROGRESS; - } -} - -/* - * Prepare controller for a transaction and call i2c_dw_xfer_msg. - */ -static int -i2c_dw_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) -{ - struct dw_i2c_dev *dev = i2c_get_adapdata(adap); - int ret; - - dev_dbg(dev->dev, "%s: msgs: %d\n", __func__, num); - - pm_runtime_get_sync(dev->dev); - - reinit_completion(&dev->cmd_complete); - dev->msgs = msgs; - dev->msgs_num = num; - dev->cmd_err = 0; - dev->msg_write_idx = 0; - dev->msg_read_idx = 0; - dev->msg_err = 0; - dev->status = STATUS_IDLE; - dev->abort_source = 0; - dev->rx_outstanding = 0; - - ret = i2c_dw_acquire_lock(dev); - if (ret) - goto done_nolock; - - ret = i2c_dw_wait_bus_not_busy(dev); - if (ret < 0) - goto done; - - /* Start the transfers */ - i2c_dw_xfer_init(dev); - - /* Wait for tx to complete */ - if (!wait_for_completion_timeout(&dev->cmd_complete, adap->timeout)) { - dev_err(dev->dev, "controller timed out\n"); - /* i2c_dw_init implicitly disables the adapter */ - i2c_recover_bus(&dev->adapter); - i2c_dw_init_master(dev); - ret = -ETIMEDOUT; - goto done; - } - - /* - * We must disable the adapter before returning and signaling the end - * of the current transfer. Otherwise the hardware might continue - * generating interrupts which in turn causes a race condition with - * the following transfer. Needs some more investigation if the - * additional interrupts are a hardware bug or this driver doesn't - * handle them correctly yet. - */ - __i2c_dw_disable_nowait(dev); - - if (dev->msg_err) { - ret = dev->msg_err; - goto done; - } - - /* No error */ - if (likely(!dev->cmd_err && !dev->status)) { - ret = num; - goto done; - } - - /* We have an error */ - if (dev->cmd_err == DW_IC_ERR_TX_ABRT) { - ret = i2c_dw_handle_tx_abort(dev); - goto done; - } - - if (dev->status) - dev_err(dev->dev, - "transfer terminated early - interrupt latency too high?\n"); - - ret = -EIO; - -done: - i2c_dw_release_lock(dev); - -done_nolock: - pm_runtime_mark_last_busy(dev->dev); - pm_runtime_put_autosuspend(dev->dev); - - return ret; -} - -static const struct i2c_algorithm i2c_dw_algo = { - .master_xfer = i2c_dw_xfer, - .functionality = i2c_dw_func, -}; - -static const struct i2c_adapter_quirks i2c_dw_quirks = { - .flags = I2C_AQ_NO_ZERO_LEN, -}; - -static u32 i2c_dw_read_clear_intrbits(struct dw_i2c_dev *dev) -{ - u32 stat; - - /* - * The IC_INTR_STAT register just indicates "enabled" interrupts. - * Ths unmasked raw version of interrupt status bits are available - * in the IC_RAW_INTR_STAT register. - * - * That is, - * stat = dw_readl(IC_INTR_STAT); - * equals to, - * stat = dw_readl(IC_RAW_INTR_STAT) & dw_readl(IC_INTR_MASK); - * - * The raw version might be useful for debugging purposes. - */ - stat = dw_readl(dev, DW_IC_INTR_STAT); - /* Disable all interrupts */ - dw_writel(dev, 0, DW_IC_INTR_MASK); - stat |= dw_readl(dev, DW_IC_RAW_INTR_STAT) & dev->intr_mask_value; /* Avoid to miss interrupt! */ - - - /* - * Do not use the IC_CLR_INTR register to clear interrupts, or - * you'll miss some interrupts, triggered during the period from - * dw_readl(IC_INTR_STAT) to dw_readl(IC_CLR_INTR). - * - * Instead, use the separately-prepared IC_CLR_* registers. - */ - if (stat & DW_IC_INTR_RX_UNDER) - dw_readl(dev, DW_IC_CLR_RX_UNDER); - if (stat & DW_IC_INTR_RX_OVER) - dw_readl(dev, DW_IC_CLR_RX_OVER); - if (stat & DW_IC_INTR_TX_OVER) - dw_readl(dev, DW_IC_CLR_TX_OVER); - if (stat & DW_IC_INTR_RD_REQ) - dw_readl(dev, DW_IC_CLR_RD_REQ); - if (stat & DW_IC_INTR_TX_ABRT) { - /* - * The IC_TX_ABRT_SOURCE register is cleared whenever - * the IC_CLR_TX_ABRT is read. Preserve it beforehand. - */ - dev->abort_source = dw_readl(dev, DW_IC_TX_ABRT_SOURCE); - dw_readl(dev, DW_IC_CLR_TX_ABRT); - } - if (stat & DW_IC_INTR_RX_DONE) - dw_readl(dev, DW_IC_CLR_RX_DONE); - if (stat & DW_IC_INTR_ACTIVITY) - dw_readl(dev, DW_IC_CLR_ACTIVITY); - if (stat & DW_IC_INTR_STOP_DET) - dw_readl(dev, DW_IC_CLR_STOP_DET); - if (stat & DW_IC_INTR_START_DET) - dw_readl(dev, DW_IC_CLR_START_DET); - if (stat & DW_IC_INTR_GEN_CALL) - dw_readl(dev, DW_IC_CLR_GEN_CALL); - - return stat; -} - -/* - * Interrupt service routine. This gets called whenever an I2C master interrupt - * occurs. - */ -static int i2c_dw_irq_handler_master(struct dw_i2c_dev *dev) -{ - u32 stat; - - stat = i2c_dw_read_clear_intrbits(dev); - if (stat & DW_IC_INTR_TX_ABRT) { - dev->cmd_err |= DW_IC_ERR_TX_ABRT; - dev->status = STATUS_IDLE; - - /* - * Anytime TX_ABRT is set, the contents of the tx/rx - * buffers are flushed. Make sure to skip them. - */ - dev->intr_mask_value = 0; - goto tx_aborted; - } - - if (stat & DW_IC_INTR_RX_FULL) - i2c_dw_read(dev); - - if (stat & DW_IC_INTR_TX_EMPTY) - i2c_dw_xfer_msg(dev); - - /* - * No need to modify or disable the interrupt mask here. - * i2c_dw_xfer_msg() will take care of it according to - * the current transmit status. - */ - -tx_aborted: - if ((stat & (DW_IC_INTR_TX_ABRT | DW_IC_INTR_STOP_DET)) || dev->msg_err) - complete(&dev->cmd_complete); - else if (unlikely(dev->flags & ACCESS_INTR_MASK)) { - /* Workaround to trigger pending interrupt */ - stat = dw_readl(dev, DW_IC_INTR_MASK); - i2c_dw_disable_int(dev); - dw_writel(dev, stat, DW_IC_INTR_MASK); - return 0; - } - - dw_writel(dev, dev->intr_mask_value, DW_IC_INTR_MASK); - return 0; - -} - -static irqreturn_t i2c_dw_isr(int this_irq, void *dev_id) -{ - struct dw_i2c_dev *dev = dev_id; - u32 stat, enabled; - - enabled = dw_readl(dev, DW_IC_ENABLE); - stat = dw_readl(dev, DW_IC_RAW_INTR_STAT); - dev_dbg(dev->dev, "enabled=%#x stat=%#x\n", enabled, stat); - if (!enabled || !(stat & ~DW_IC_INTR_ACTIVITY)) - return IRQ_NONE; - - i2c_dw_irq_handler_master(dev); - - return IRQ_HANDLED; -} - -static void i2c_dw_prepare_recovery(struct i2c_adapter *adap) -{ - struct dw_i2c_dev *dev = i2c_get_adapdata(adap); - - i2c_dw_disable(dev); - reset_control_assert(dev->rst); - i2c_dw_prepare_clk(dev, false); -} - -static void i2c_dw_unprepare_recovery(struct i2c_adapter *adap) -{ - struct dw_i2c_dev *dev = i2c_get_adapdata(adap); - - i2c_dw_prepare_clk(dev, true); - reset_control_deassert(dev->rst); - i2c_dw_init_master(dev); -} - -static int i2c_dw_init_recovery_info(struct dw_i2c_dev *dev) -{ - struct i2c_bus_recovery_info *rinfo = &dev->rinfo; - struct i2c_adapter *adap = &dev->adapter; - struct gpio_desc *gpio; - int r; - - gpio = devm_gpiod_get(dev->dev, "scl", GPIOD_OUT_HIGH); - if (IS_ERR(gpio)) { - r = PTR_ERR(gpio); - if (r == -ENOENT || r == -ENOSYS) - return 0; - return r; - } - rinfo->scl_gpiod = gpio; - - gpio = devm_gpiod_get_optional(dev->dev, "sda", GPIOD_IN); - if (IS_ERR(gpio)) - return PTR_ERR(gpio); - rinfo->sda_gpiod = gpio; - - rinfo->recover_bus = i2c_generic_scl_recovery; - rinfo->prepare_recovery = i2c_dw_prepare_recovery; - rinfo->unprepare_recovery = i2c_dw_unprepare_recovery; - adap->bus_recovery_info = rinfo; - - dev_info(dev->dev, "running with gpio recovery mode! scl%s", - rinfo->sda_gpiod ? ",sda" : ""); - - return 0; -} - -int i2c_dw_probe(struct dw_i2c_dev *dev) -{ - struct i2c_adapter *adap = &dev->adapter; - unsigned long irq_flags; - int ret; - - init_completion(&dev->cmd_complete); - - dev->init = i2c_dw_init_master; - dev->disable = i2c_dw_disable; - dev->disable_int = i2c_dw_disable_int; - - ret = i2c_dw_set_reg_access(dev); - if (ret) - return ret; - - ret = i2c_dw_set_timings_master(dev); - if (ret) - return ret; - - ret = dev->init(dev); - if (ret) - return ret; - - snprintf(adap->name, sizeof(adap->name), - "Synopsys DesignWare I2C adapter"); - adap->retries = 3; - adap->algo = &i2c_dw_algo; - adap->quirks = &i2c_dw_quirks; - adap->dev.parent = dev->dev; - i2c_set_adapdata(adap, dev); - - if (dev->pm_disabled) - irq_flags = IRQF_NO_SUSPEND; - else - irq_flags = IRQF_SHARED | IRQF_COND_SUSPEND; - - i2c_dw_disable_int(dev); - ret = devm_request_irq(dev->dev, dev->irq, i2c_dw_isr, irq_flags, - dev_name(dev->dev), dev); - if (ret) { - dev_err(dev->dev, "failure requesting irq %i: %d\n", - dev->irq, ret); - return ret; - } - - ret = i2c_dw_init_recovery_info(dev); - if (ret) - return ret; - - /* - * Increment PM usage count during adapter registration in order to - * avoid possible spurious runtime suspend when adapter device is - * registered to the device core and immediate resume in case bus has - * registered I2C slaves that do I2C transfers in their probe. - */ - pm_runtime_get_noresume(dev->dev); - ret = i2c_add_numbered_adapter(adap); - if (ret) - dev_err(dev->dev, "failure adding adapter: %d\n", ret); - pm_runtime_put_noidle(dev->dev); - - return ret; -} -EXPORT_SYMBOL_GPL(i2c_dw_probe); - -MODULE_DESCRIPTION("Synopsys DesignWare I2C bus master adapter"); -MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-sunway-sw6-platdrv.c b/drivers/i2c/busses/i2c-sunway-sw6-platdrv.c deleted file mode 100644 index fd757e2439331074367bbf0335e42b94acff204a..0000000000000000000000000000000000000000 --- a/drivers/i2c/busses/i2c-sunway-sw6-platdrv.c +++ /dev/null @@ -1,498 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Synopsys DesignWare I2C adapter driver. - * - * Based on the TI DAVINCI I2C adapter driver. - * - * Copyright (C) 2006 Texas Instruments. - * Copyright (C) 2007 MontaVista Software Inc. - * Copyright (C) 2009 Provigent Ltd. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "i2c-sunway-sw6-core.h" - -static u32 i2c_dw_get_clk_rate_khz(struct dw_i2c_dev *dev) -{ - return clk_get_rate(dev->clk)/1000; -} - -#ifdef CONFIG_ACPI -/* - * The HCNT/LCNT information coming from ACPI should be the most accurate - * for given platform. However, some systems get it wrong. On such systems - * we get better results by calculating those based on the input clock. - */ -static const struct dmi_system_id dw_i2c_no_acpi_params[] = { - { - .ident = "Dell Inspiron 7348", - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 7348"), - }, - }, - { } -}; - -static void dw_i2c_acpi_params(struct platform_device *pdev, char method[], - u16 *hcnt, u16 *lcnt, u32 *sda_hold) -{ - struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER }; - acpi_handle handle = ACPI_HANDLE(&pdev->dev); - union acpi_object *obj; - - if (dmi_check_system(dw_i2c_no_acpi_params)) - return; - - if (ACPI_FAILURE(acpi_evaluate_object(handle, method, NULL, &buf))) - return; - - obj = (union acpi_object *)buf.pointer; - if (obj->type == ACPI_TYPE_PACKAGE && obj->package.count == 3) { - const union acpi_object *objs = obj->package.elements; - - *hcnt = (u16)objs[0].integer.value; - *lcnt = (u16)objs[1].integer.value; - *sda_hold = (u32)objs[2].integer.value; - } - - kfree(buf.pointer); -} - -static int dw_i2c_acpi_configure(struct platform_device *pdev) -{ - struct dw_i2c_dev *dev = platform_get_drvdata(pdev); - struct i2c_timings *t = &dev->timings; - u32 ss_ht = 0, fp_ht = 0, hs_ht = 0, fs_ht = 0; - acpi_handle handle = ACPI_HANDLE(&pdev->dev); - const struct acpi_device_id *id; - struct acpi_device *adev; - const char *uid; - - dev->adapter.nr = -1; - dev->tx_fifo_depth = 32; - dev->rx_fifo_depth = 32; - - /* - * Try to get SDA hold time and *CNT values from an ACPI method for - * selected speed modes. - */ - dw_i2c_acpi_params(pdev, "SSCN", &dev->ss_hcnt, &dev->ss_lcnt, &ss_ht); - dw_i2c_acpi_params(pdev, "FPCN", &dev->fp_hcnt, &dev->fp_lcnt, &fp_ht); - dw_i2c_acpi_params(pdev, "HSCN", &dev->hs_hcnt, &dev->hs_lcnt, &hs_ht); - dw_i2c_acpi_params(pdev, "FMCN", &dev->fs_hcnt, &dev->fs_lcnt, &fs_ht); - - switch (t->bus_freq_hz) { - case 100000: - dev->sda_hold_time = ss_ht; - break; - case 1000000: - dev->sda_hold_time = fp_ht; - break; - case 3400000: - dev->sda_hold_time = hs_ht; - break; - case 400000: - default: - dev->sda_hold_time = fs_ht; - break; - } - - id = acpi_match_device(pdev->dev.driver->acpi_match_table, &pdev->dev); - if (id && id->driver_data) - dev->flags |= (u32)id->driver_data; - - if (acpi_bus_get_device(handle, &adev)) - return -ENODEV; - - /* - * Cherrytrail I2C7 gets used for the PMIC which gets accessed - * through ACPI opregions during late suspend / early resume - * disable pm for it. - */ - uid = adev->pnp.unique_id; - if ((dev->flags & MODEL_CHERRYTRAIL) && !strcmp(uid, "7")) - dev->pm_disabled = true; - - return 0; -} - -static const struct acpi_device_id dw_i2c_acpi_match[] = { - { "INT33C2", 0 }, - { "INT33C3", 0 }, - { "INT3432", 0 }, - { "INT3433", 0 }, - { "80860F41", 0 }, - { "808622C1", MODEL_CHERRYTRAIL }, - { "AMD0010", ACCESS_INTR_MASK }, - { "AMDI0010", ACCESS_INTR_MASK }, - { "AMDI0510", 0 }, - { "APMC0D0F", 0 }, - { "HISI02A1", 0 }, - { "HISI02A2", 0 }, - { } -}; -MODULE_DEVICE_TABLE(acpi, dw_i2c_acpi_match); -#else -static inline int dw_i2c_acpi_configure(struct platform_device *pdev) -{ - return -ENODEV; -} -#endif - -static void i2c_dw_configure_master(struct dw_i2c_dev *dev) -{ - struct i2c_timings *t = &dev->timings; - - dev->functionality = I2C_FUNC_10BIT_ADDR | DW_IC_DEFAULT_FUNCTIONALITY; - - dev->master_cfg = DW_IC_CON_MASTER | DW_IC_CON_SLAVE_DISABLE | - DW_IC_CON_RESTART_EN; - - dev->mode = DW_IC_MASTER; - - switch (t->bus_freq_hz) { - case 100000: - dev->master_cfg |= DW_IC_CON_SPEED_STD; - break; - case 3400000: - dev->master_cfg |= DW_IC_CON_SPEED_HIGH; - break; - default: - dev->master_cfg |= DW_IC_CON_SPEED_FAST; - } -} - -static void i2c_dw_configure_slave(struct dw_i2c_dev *dev) -{ - dev->functionality = I2C_FUNC_SLAVE | DW_IC_DEFAULT_FUNCTIONALITY; - - dev->slave_cfg = DW_IC_CON_RX_FIFO_FULL_HLD_CTRL | - DW_IC_CON_RESTART_EN | DW_IC_CON_STOP_DET_IFADDRESSED; - - dev->mode = DW_IC_SLAVE; -} - -static void dw_i2c_set_fifo_size(struct dw_i2c_dev *dev, int id) -{ - u32 param, tx_fifo_depth, rx_fifo_depth; - - /* - * Try to detect the FIFO depth if not set by interface driver, - * the depth could be from 2 to 256 from HW spec. - */ - param = i2c_dw_read_comp_param(dev); - tx_fifo_depth = ((param >> 16) & 0xff) + 1; - rx_fifo_depth = ((param >> 8) & 0xff) + 1; - if (!dev->tx_fifo_depth) { - dev->tx_fifo_depth = tx_fifo_depth; - dev->rx_fifo_depth = rx_fifo_depth; - dev->adapter.nr = id; - } else if (tx_fifo_depth >= 2) { - dev->tx_fifo_depth = min_t(u32, dev->tx_fifo_depth, - tx_fifo_depth); - dev->rx_fifo_depth = min_t(u32, dev->rx_fifo_depth, - rx_fifo_depth); - } -} - -static void dw_i2c_plat_pm_cleanup(struct dw_i2c_dev *dev) -{ - pm_runtime_disable(dev->dev); - - if (dev->pm_disabled) - pm_runtime_put_noidle(dev->dev); -} - -static int dw_i2c_plat_probe(struct platform_device *pdev) -{ - struct dw_i2c_platform_data *pdata = dev_get_platdata(&pdev->dev); - struct i2c_adapter *adap; - struct dw_i2c_dev *dev; - struct i2c_timings *t; - u32 acpi_speed; - struct resource *mem; - int i, irq, ret; - static const int supported_speeds[] = { - 0, 100000, 400000, 1000000, 3400000 - }; - - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; - - dev = devm_kzalloc(&pdev->dev, sizeof(struct dw_i2c_dev), GFP_KERNEL); - if (!dev) - return -ENOMEM; - - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dev->base = devm_ioremap_resource(&pdev->dev, mem); - if (IS_ERR(dev->base)) - return PTR_ERR(dev->base); - - dev->dev = &pdev->dev; - dev->irq = irq; - platform_set_drvdata(pdev, dev); - - dev->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL); - if (IS_ERR(dev->rst)) { - if (PTR_ERR(dev->rst) == -EPROBE_DEFER) - return -EPROBE_DEFER; - } else { - reset_control_deassert(dev->rst); - } - - t = &dev->timings; - if (pdata) - t->bus_freq_hz = pdata->i2c_scl_freq; - else - i2c_parse_fw_timings(&pdev->dev, t, false); - - acpi_speed = i2c_acpi_find_bus_speed(&pdev->dev); - /* - * Some DSTDs use a non standard speed, round down to the lowest - * standard speed. - */ - for (i = 1; i < ARRAY_SIZE(supported_speeds); i++) { - if (acpi_speed < supported_speeds[i]) - break; - } - acpi_speed = supported_speeds[i - 1]; - - /* - * Find bus speed from the "clock-frequency" device property, ACPI - * or by using fast mode if neither is set. - */ - if (acpi_speed && t->bus_freq_hz) - t->bus_freq_hz = min(t->bus_freq_hz, acpi_speed); - else if (acpi_speed || t->bus_freq_hz) - t->bus_freq_hz = max(t->bus_freq_hz, acpi_speed); - else - t->bus_freq_hz = 400000; - - if (has_acpi_companion(&pdev->dev)) - dw_i2c_acpi_configure(pdev); - - /* - * Only standard mode at 100kHz, fast mode at 400kHz, - * fast mode plus at 1MHz and high speed mode at 3.4MHz are supported. - */ - if (t->bus_freq_hz != 100000 && t->bus_freq_hz != 400000 && - t->bus_freq_hz != 1000000 && t->bus_freq_hz != 3400000) { - dev_err(&pdev->dev, - "%d Hz is unsupported, only 100kHz, 400kHz, 1MHz and 3.4MHz are supported\n", - t->bus_freq_hz); - ret = -EINVAL; - goto exit_reset; - } - - ret = i2c_dw_probe_lock_support(dev); - if (ret) - goto exit_reset; - - if (i2c_detect_slave_mode(&pdev->dev)) - i2c_dw_configure_slave(dev); - else - i2c_dw_configure_master(dev); - - dev->clk = devm_clk_get(&pdev->dev, NULL); - if (!i2c_dw_prepare_clk(dev, true)) { - u64 clk_khz; - - dev->get_clk_rate_khz = i2c_dw_get_clk_rate_khz; - clk_khz = dev->get_clk_rate_khz(dev); - - if (!dev->sda_hold_time && t->sda_hold_ns) - dev->sda_hold_time = - div_u64(clk_khz * t->sda_hold_ns + 500000, 1000000); - } - - dw_i2c_set_fifo_size(dev, pdev->id); - - adap = &dev->adapter; - adap->owner = THIS_MODULE; - adap->class = 0;/*lm_sensors, ...*/ - ACPI_COMPANION_SET(&adap->dev, ACPI_COMPANION(&pdev->dev)); - adap->dev.of_node = pdev->dev.of_node; - - dev_pm_set_driver_flags(&pdev->dev, - DPM_FLAG_SMART_PREPARE | - DPM_FLAG_SMART_SUSPEND | - DPM_FLAG_MAY_SKIP_RESUME); - - /* The code below assumes runtime PM to be disabled. */ - WARN_ON(pm_runtime_enabled(&pdev->dev)); - - pm_runtime_set_autosuspend_delay(&pdev->dev, 1000); - pm_runtime_use_autosuspend(&pdev->dev); - pm_runtime_set_active(&pdev->dev); - - if (dev->pm_disabled) - pm_runtime_get_noresume(&pdev->dev); - - pm_runtime_enable(&pdev->dev); - - if (dev->mode == DW_IC_SLAVE) - ret = i2c_dw_probe_slave(dev); - else - ret = i2c_dw_probe(dev); - - if (ret) - goto exit_probe; - - return ret; - -exit_probe: - dw_i2c_plat_pm_cleanup(dev); -exit_reset: - if (!IS_ERR_OR_NULL(dev->rst)) - reset_control_assert(dev->rst); - return ret; -} - -static int dw_i2c_plat_remove(struct platform_device *pdev) -{ - struct dw_i2c_dev *dev = platform_get_drvdata(pdev); - - pm_runtime_get_sync(&pdev->dev); - - i2c_del_adapter(&dev->adapter); - - dev->disable(dev); - - pm_runtime_dont_use_autosuspend(&pdev->dev); - pm_runtime_put_sync(&pdev->dev); - dw_i2c_plat_pm_cleanup(dev); - - if (!IS_ERR_OR_NULL(dev->rst)) - reset_control_assert(dev->rst); - - i2c_dw_remove_lock_support(dev); - - return 0; -} - -#ifdef CONFIG_OF -static const struct of_device_id dw_i2c_of_match[] = { - { .compatible = "snps,designware-i2c", }, - {}, -}; -MODULE_DEVICE_TABLE(of, dw_i2c_of_match); -#endif - -#ifdef CONFIG_PM_SLEEP -static int dw_i2c_plat_prepare(struct device *dev) -{ - /* - * If the ACPI companion device object is present for this device, it - * may be accessed during suspend and resume of other devices via I2C - * operation regions, so tell the PM core and middle layers to avoid - * skipping system suspend/resume callbacks for it in that case. - */ - return !has_acpi_companion(dev); -} - -static void dw_i2c_plat_complete(struct device *dev) -{ - /* - * The device can only be in runtime suspend at this point if it has not - * been resumed throughout the ending system suspend/resume cycle, so if - * the platform firmware might mess up with it, request the runtime PM - * framework to resume it. - */ - if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) - pm_request_resume(dev); -} -#else -#define dw_i2c_plat_prepare NULL -#define dw_i2c_plat_complete NULL -#endif - -#ifdef CONFIG_PM -static int dw_i2c_plat_suspend(struct device *dev) -{ - struct dw_i2c_dev *i_dev = dev_get_drvdata(dev); - - if (i_dev->pm_disabled) - return 0; - - i_dev->disable(i_dev); - i2c_dw_prepare_clk(i_dev, false); - - return 0; -} - -static int dw_i2c_plat_resume(struct device *dev) -{ - struct dw_i2c_dev *i_dev = dev_get_drvdata(dev); - - if (!i_dev->pm_disabled) - i2c_dw_prepare_clk(i_dev, true); - - i_dev->init(i_dev); - - return 0; -} - -static const struct dev_pm_ops dw_i2c_dev_pm_ops = { - .prepare = dw_i2c_plat_prepare, - .complete = dw_i2c_plat_complete, - SET_LATE_SYSTEM_SLEEP_PM_OPS(dw_i2c_plat_suspend, dw_i2c_plat_resume) - SET_RUNTIME_PM_OPS(dw_i2c_plat_suspend, dw_i2c_plat_resume, NULL) -}; - -#define DW_I2C_DEV_PMOPS (&dw_i2c_dev_pm_ops) -#else -#define DW_I2C_DEV_PMOPS NULL -#endif - -/* Work with hotplug and coldplug */ -MODULE_ALIAS("platform:i2c_designware"); - -static struct platform_driver dw_i2c_driver = { - .probe = dw_i2c_plat_probe, - .remove = dw_i2c_plat_remove, - .driver = { - .name = "i2c_designware", - .of_match_table = of_match_ptr(dw_i2c_of_match), - .acpi_match_table = ACPI_PTR(dw_i2c_acpi_match), - .pm = DW_I2C_DEV_PMOPS, - }, -}; - -static int __init dw_i2c_init_driver(void) -{ - return platform_driver_register(&dw_i2c_driver); -} -subsys_initcall(dw_i2c_init_driver); - -static void __exit dw_i2c_exit_driver(void) -{ - platform_driver_unregister(&dw_i2c_driver); -} -module_exit(dw_i2c_exit_driver); - -MODULE_AUTHOR("Baruch Siach "); -MODULE_DESCRIPTION("Synopsys DesignWare I2C bus adapter"); -MODULE_LICENSE("GPL"); diff --git a/arch/sw_64/chip/chip3/i2c-lib.c b/drivers/i2c/busses/i2c-sunway.c similarity index 99% rename from arch/sw_64/chip/chip3/i2c-lib.c rename to drivers/i2c/busses/i2c-sunway.c index e70f0f0c9a56af9e507db4637e75f83d2b36b8ad..f99955ac7ae3ab1eed351c935efc305e611a794c 100644 --- a/arch/sw_64/chip/chip3/i2c-lib.c +++ b/drivers/i2c/busses/i2c-sunway.c @@ -1,7 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 /* - * arch/sw_64/chip/chip3/i2c-lib.c - * * Copyright (C) 2020 Platform Software * * This program is free software; you can redistribute it and/or diff --git a/drivers/iommu/sw64/Kconfig b/drivers/iommu/sw64/Kconfig index a313c6e2d11b0b9f2197ed98e2bd8f71d5e9bea0..bffc464f2de8d8d88254ab942ecc98f46c52c82d 100644 --- a/drivers/iommu/sw64/Kconfig +++ b/drivers/iommu/sw64/Kconfig @@ -1,9 +1,20 @@ # SPDX-License-Identifier: GPL-2.0-only # SW64 IOMMU SUPPORT config SUNWAY_IOMMU - bool "Sunway IOMMU Support" - select IOMMU_API - select IOMMU_IOVA - depends on SW64 && PCI - help - Support for IOMMU on SW64 platform. + bool "Sunway IOMMU Support" + select IOMMU_API + select IOMMU_IOVA + depends on SW64 && PCI && SUBARCH_C3B + help + Support for IOMMU on SW64 platform. It can enable or bypass specific device by + adding boot param "iommu_enable" and "iommu.passthrough". + +# SW64 IOMMU V2 SUPPORT +config SUNWAY_IOMMU_V2 + bool "Sunway IOMMU V2 Support" + select IOMMU_API + select IOMMU_IOVA + depends on SW64 && PCI && SUBARCH_C4 + help + Support for IOMMU V2 on SW64 platform. It can enable or bypass specific device by + adding boot param "iommu_enable" and "iommu.passthrough". diff --git a/drivers/iommu/sw64/Makefile b/drivers/iommu/sw64/Makefile index e23dbd40a74d379e65267cecb68ff370e570d626..da7a604f253c9162934e9fec792ffae90ff28963 100644 --- a/drivers/iommu/sw64/Makefile +++ b/drivers/iommu/sw64/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_SUNWAY_IOMMU) += sunway_iommu.o +obj-$(CONFIG_SUNWAY_IOMMU_V2) += sunway_iommu_v2.o diff --git a/drivers/iommu/sw64/sunway_iommu.c b/drivers/iommu/sw64/sunway_iommu.c index 86920f88faacdf97495c7774c0af041a7afd65f5..77780c608a005fca09fa5e3b3ae134ded7ac10b7 100644 --- a/drivers/iommu/sw64/sunway_iommu.c +++ b/drivers/iommu/sw64/sunway_iommu.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,11 @@ #define SW64_DMA_LIMIT (0xe0000000 - 1) #define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) +#define SW64_IOMMU_LEVEL1_OFFSET 0x1ff +#define SW64_IOMMU_LEVEL2_OFFSET 0x3ff + +#define SW64_IOMMU_GRN_8K ((0UL) << 4) /* page size as 8KB */ +#define SW64_IOMMU_GRN_8M ((0x2UL) << 4) /* page size as 8MB */ #define SW64_IOMMU_PGSIZES (((1ULL) << PAGE_SHIFT) | ((1ULL) << PAGE_8M_SHIFT)) #define IDENTMAP_ALL ((1U) << 0) @@ -1293,14 +1299,6 @@ sunway_unmap_sg(struct device *dev, struct scatterlist *sgl, } } -static int sunway_supported(struct device *dev, u64 mask) -{ - if (MAX_DMA_ADDRESS - PAGE_OFFSET - 1 <= mask) - return 1; - - return 0; -} - static const struct dma_map_ops sunway_dma_ops = { .alloc = sunway_alloc_coherent, .free = sunway_free_coherent, @@ -1308,7 +1306,7 @@ static const struct dma_map_ops sunway_dma_ops = { .unmap_sg = sunway_unmap_sg, .map_page = sunway_map_page, .unmap_page = sunway_unmap_page, - .dma_supported = sunway_supported, + .dma_supported = dma_direct_supported, }; /********************************************************************** @@ -1466,7 +1464,7 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); unsigned long paddr, grn; - if (iova > SW64_BAR_ADDRESS) + if (iova >= SW64_BAR_ADDRESS) return iova; paddr = fetch_pte(sdomain, iova, PTE_LEVEL2_VAL); @@ -1503,7 +1501,7 @@ sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, * and pci device BAR, check should be introduced manually * to avoid VFIO trying to map pci config space. */ - if (iova > SW64_BAR_ADDRESS) + if (iova >= SW64_BAR_ADDRESS) return 0; mutex_lock(&sdomain->api_lock); @@ -1521,7 +1519,7 @@ sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); size_t unmap_size; - if (iova > SW64_BAR_ADDRESS) + if (iova >= SW64_BAR_ADDRESS) return page_size; mutex_lock(&sdomain->api_lock); @@ -1678,6 +1676,9 @@ const struct iommu_ops sunway_iommu_ops = { /***************************************************************************** * * Boot param handle + * Each bit of iommu_enable bitmap represents an rc enable, and every 8 bits + * represents one cpu node. For example, iommu_enable=0x0100 means enabling + * rc0 for cpu node 1. * *****************************************************************************/ static int __init iommu_enable_setup(char *str) diff --git a/drivers/iommu/sw64/sunway_iommu.h b/drivers/iommu/sw64/sunway_iommu.h index 52d6452fa14c93b2cfb0fd9aee7f1cb5a88e6b96..d2a7e96657e9ae7f0a48cfefec776a599924c41e 100644 --- a/drivers/iommu/sw64/sunway_iommu.h +++ b/drivers/iommu/sw64/sunway_iommu.h @@ -21,7 +21,6 @@ struct sunway_iommu { int node; /* NUMA node */ struct pci_controller *hose_pt; - struct pci_dev *pdev; /* PCI device to this IOMMU */ struct iommu_device iommu; /* IOMMU core code handle */ }; @@ -60,11 +59,12 @@ struct sunway_iommu_group { }; #define SW64_IOMMU_ENTRY_VALID ((1UL) << 63) +#define SW64_PTE_LAST_MASK ((1UL) << 8) /*last stage valid*/ #define SW64_DMA_START 0x1000000 -#define SW64_IOMMU_GRN_8K ((0UL) << 4) /* page size as 8KB */ -#define SW64_IOMMU_GRN_8M ((0x2UL) << 4) /* page size as 8MB */ #define SW64_PTE_GRN_MASK ((0x3UL) << 4) #define PAGE_8M_SHIFT 23 +#define PAGE_512M_SHIFT 29 +#define PAGE_8G_SHIFT 33 #define SW64_IOMMU_ENABLE 3 #define SW64_IOMMU_DISABLE 0 #define SW64_IOMMU_LEVEL1_OFFSET 0x1ff @@ -75,3 +75,5 @@ struct sunway_iommu_group { #define PAGE_SHIFT_IOMMU 18 #define PAGE_SIZE_IOMMU (_AC(1, UL) << PAGE_SHIFT_IOMMU) + +#define PCACHE_FLUSHPADDR_MASK 0xffffffffff80UL diff --git a/drivers/iommu/sw64/sunway_iommu_v2.c b/drivers/iommu/sw64/sunway_iommu_v2.c new file mode 100644 index 0000000000000000000000000000000000000000..3bc87fbdaa4df2f4db2da34d0185ae63776529aa --- /dev/null +++ b/drivers/iommu/sw64/sunway_iommu_v2.c @@ -0,0 +1,1759 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * iommu.c: Generic sw64 IOMMU support + * + * This is designed and tested for 3231. If there are no changes in hardware + * in later chips, then it should work just as well. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sunway_iommu.h" + +#define MAX_DOMAIN_NUM 65536 +#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) +#define SW64_32BIT_DMA_LIMIT (0xe0000000 - 1) +#define SW64_64BIT_DMA_LIMIT ((1UL << 41) - 1) +#define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) + +#define SW64_IOMMU_PGSIZES (((1ULL) << PAGE_SHIFT) \ + | ((1ULL) << PAGE_8M_SHIFT) \ + | ((1ULL) << PAGE_512M_SHIFT) \ + | ((1ULL) << PAGE_8G_SHIFT)) + +#define IDENTMAP_ALL ((1U) << 0) +#define DMA_MASK64 ((1U) << 1) + +#define PTE_VALID 0x8000000000000000UL +#define LAST_STAGE 0x100UL +#define PTE_GRN_8M 0x10UL +#define PTE_GRN_512M 0x20UL +#define PTE_GRN_8G 0x30UL +#define PTE_WRITEE 0x2UL +#define PTE_READE 0x1UL +#define PTE_RWE 0x3UL +#define PTE_FLAGS_MASK 0x8000000000000133UL +#define PAGE_8G_OFFSET_MASK ((1UL << PAGE_8G_SHIFT) - 1) +#define PAGE_512M_OFFSET_MASK ((1UL << PAGE_512M_SHIFT) - 1) +#define PAGE_8M_OFFSET_MASK ((1UL << PAGE_8M_SHIFT) - 1) + +/* IOMMU Exceptional Status */ +enum exceptype { + DTE_LEVEL1 = 0x0, + DTE_LEVEL2, + PTE_LEVEL1, + PTE_LEVEL2, + PTE_LEVEL3, + UNAUTHORIZED_ACCESS, + ILLEGAL_RESPONSE, + DTE_LEVEL1_VAL, + DTE_LEVEL2_VAL, + PTE_LEVEL1_VAL, + PTE_LEVEL2_VAL, + PTE_LEVEL3_VAL, +}; + +u64 iommu_enable_cmd; /* default IOMMU boot param: 0 */ + +unsigned long *sunway_iommu_domain_bitmap; + +static DEFINE_SPINLOCK(domain_bitmap_lock); +static DEFINE_SPINLOCK(sunway_iommu_device_table_lock); +spinlock_t sunway_domain_lock; + +static LLIST_HEAD(dev_data_list); +LIST_HEAD(sunway_domain_list); + +struct dma_domain { + struct sunway_iommu_domain sdomain; + struct iova_domain iovad; +}; +const struct iommu_ops sunway_iommu_ops; +static const struct dma_map_ops sunway_dma_ops; + + +/* flush helpers */ +static void piu_flush_all(struct pci_controller *hose) +{ + write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); + write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); + write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); +} + +void flush_pcache_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) +{ + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + + list_for_each_entry(sdev, &sdomain->dev_list, list) { + hose = sdev->pdev->sysdata; + + flush_addr = __pa(flush_addr); + /* Set memory bar here */ + mb(); + write_piu_ior0(hose->node, hose->index, + PCACHE_FLUSHPADDR, flush_addr); + } +} + +void flush_ptlb_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) +{ + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + struct pci_dev *pdev; + + list_for_each_entry(sdev, &sdomain->dev_list, list) { + pdev = sdev->pdev; + hose = pdev->sysdata; + + flush_addr = (pdev->bus->number << 8) + | pdev->devfn | (flush_addr << 16); + write_piu_ior0(hose->node, hose->index, + PTLB_FLUSHVADDR, flush_addr); + } +} + +/* domain helpers */ +static struct sunway_iommu_domain *to_sunway_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct sunway_iommu_domain, domain); +} + +static struct dma_domain *to_dma_domain(struct sunway_iommu_domain *sdomain) +{ + return container_of(sdomain, struct dma_domain, sdomain); +} + +static void add_domain_to_list(struct sunway_iommu_domain *sdomain) +{ + unsigned long flags; + + spin_lock_irqsave(&sunway_domain_lock, flags); + list_add(&sdomain->list, &sunway_domain_list); + spin_unlock_irqrestore(&sunway_domain_lock, flags); +} + +static void del_domain_from_list(struct sunway_iommu_domain *sdomain) +{ + unsigned long flags; + + spin_lock_irqsave(&sunway_domain_lock, flags); + list_del(&sdomain->list); + spin_unlock_irqrestore(&sunway_domain_lock, flags); +} + +static void free_pagetable(struct sunway_iommu_domain *sdomain) +{ + unsigned long *l2_pte, *l3_pte; + unsigned long l2_pte_val, l3_pte_val; + int l2_index, l3_index, ptes_one_page; + + l2_pte = sdomain->pt_root; + if (!l2_pte) + return; + + ptes_one_page = PAGE_SIZE/sizeof(unsigned long); + for (l2_index = 0; l2_index < ptes_one_page; l2_index++, l2_pte++) { + l2_pte_val = *l2_pte; + if ((l2_pte_val & SW64_IOMMU_ENTRY_VALID) == 0) + continue; + + l2_pte_val &= ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK; + l2_pte_val |= PAGE_OFFSET; + l3_pte = (unsigned long *)l2_pte_val; + for (l3_index = 0; l3_index < ptes_one_page; l3_index++, l3_pte++) { + l3_pte_val = *l3_pte; + if ((l3_pte_val & SW64_IOMMU_ENTRY_VALID) == 0) + continue; + + l3_pte_val &= ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK; + l3_pte_val |= PAGE_OFFSET; + free_page(l3_pte_val); + } + free_page(l2_pte_val); + } + + free_page((unsigned long)sdomain->pt_root); +} + +static void domain_id_free(int id) +{ + spin_lock(&domain_bitmap_lock); + if (id > 0) + __clear_bit(id, sunway_iommu_domain_bitmap); + spin_unlock(&domain_bitmap_lock); +} + +static void dma_domain_free(struct dma_domain *dma_dom) +{ + if (!dma_dom) + return; + + del_domain_from_list(&dma_dom->sdomain); + put_iova_domain(&dma_dom->iovad); + free_pagetable(&dma_dom->sdomain); + if (dma_dom->sdomain.id) + domain_id_free(dma_dom->sdomain.id); + + kfree(dma_dom); +} + +static void sunway_domain_free(struct sunway_iommu_domain *sdomain) +{ + if (!sdomain) + return; + + del_domain_from_list(sdomain); + if (sdomain->id) + domain_id_free(sdomain->id); + + kfree(sdomain); +} + +static u16 sunway_domain_id_alloc(void) +{ + int id; + + spin_lock(&domain_bitmap_lock); + id = find_first_zero_bit(sunway_iommu_domain_bitmap, MAX_DOMAIN_NUM); + if (id > 0 && id < MAX_DOMAIN_NUM) + __set_bit(id, sunway_iommu_domain_bitmap); + else + id = 0; + spin_unlock(&domain_bitmap_lock); + + return id; +} + +static int sunway_domain_init(struct sunway_iommu_domain *sdomain) +{ + spin_lock_init(&sdomain->lock); + mutex_init(&sdomain->api_lock); + sdomain->id = sunway_domain_id_alloc(); + if (!sdomain->id) + return -ENOMEM; + INIT_LIST_HEAD(&sdomain->dev_list); + + return 1; +} + +static struct sunway_iommu_domain *sunway_domain_alloc(void) +{ + struct sunway_iommu_domain *sdomain; + + sdomain = kzalloc(sizeof(struct sunway_iommu_domain), GFP_KERNEL); + if (!sdomain) + return NULL; + + if (!sunway_domain_init(sdomain)) { + kfree(sdomain); + return NULL; + } + + add_domain_to_list(sdomain); + return sdomain; +} + +static struct dma_domain *dma_domain_alloc(void) +{ + struct dma_domain *dma_dom; + struct page; + + dma_dom = kzalloc(sizeof(struct dma_domain), GFP_KERNEL); + if (!dma_dom) + return NULL; + + sunway_domain_init(&dma_dom->sdomain); + dma_dom->sdomain.type = IOMMU_DOMAIN_DMA; + init_iova_domain(&dma_dom->iovad, PAGE_SIZE, IOVA_PFN(SW64_DMA_START)); + reserve_iova(&dma_dom->iovad, (0xe0000000UL >> PAGE_SHIFT), (0x100000000UL >> PAGE_SHIFT)); + + add_domain_to_list(&dma_dom->sdomain); + + return dma_dom; +} + +static void device_flush_all(struct sunway_iommu_dev *sdata) +{ + struct pci_controller *hose = sdata->pdev->sysdata; + + if (hose == NULL) + return; + + write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, sdata->devid); + write_piu_ior0(hose->node, hose->index, PTLB_FLUSHDEV, sdata->devid); + write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHDEV, sdata->devid); +} + +/* iommu_ops device attach/unattach helpers */ +static void +set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu *iommu; + struct pci_dev *pdev; + struct page *dt_page, *pt_page; + unsigned long *dte_l1, *dte_l2; + unsigned long dte_l1_val, dte_l2_base, dte_l2_val; + + pdev = sdev->pdev; + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + return; + + sdev->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); + iommu = sdev->iommu; + dte_l1 = iommu->iommu_dtbr + (pdev->bus->number); + dte_l1_val = *dte_l1; + + if (!dte_l1_val) { + /* Alloc a new level-2 device table page */ + dt_page = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, + get_order(PAGE_SIZE)); + + WARN_ON(!dt_page); + dte_l2_base = (unsigned long)page_address(dt_page); + dte_l1_val = (__pa(dte_l2_base) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; + *dte_l1 = dte_l1_val; + } + + if (!sdomain->pt_root) { + pt_page = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, 0); + WARN_ON(!pt_page); + sdomain->pt_root = page_address(pt_page); + } + + dte_l2 = __va(dte_l1_val & ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + (pdev->devfn << 3); + dte_l2_val = (__pa(sdomain->pt_root) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; + if (sdomain->type == IOMMU_DOMAIN_IDENTITY) { + dte_l2_val |= 0x1; + sdev->passthrough = IDENTMAP_ALL; + } + *dte_l2 = dte_l2_val; + device_flush_all(sdev); +} + +static void +do_attach(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomain) +{ + sdev_data->domain = sdomain; + list_add(&sdev_data->list, &sdomain->dev_list); + + sdomain->dev_cnt++; + set_dte_entry(sdev_data, sdomain); + + pr_debug("iommu: device %d add to domain: %d\n", + sdev_data->devid, sdomain->id); +} + +static void do_detach(struct sunway_iommu_dev *sdev_data) +{ + struct sunway_iommu_domain *sdomain = sdev_data->domain; + + sdev_data->domain = NULL; + list_del(&sdev_data->list); + device_flush_all(sdev_data); + + sdomain->dev_cnt--; + pr_debug("iommu: device %d detached from domain %d\n", + sdev_data->devid, sdomain->id); +} + +static int +__attach_device(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomain) +{ + int ret; + + spin_lock(&sdomain->lock); + ret = -EBUSY; + if (sdev_data->domain != NULL) + goto out_unlock; + + do_attach(sdev_data, sdomain); + ret = 0; + +out_unlock: + spin_unlock(&sdomain->lock); + return ret; +} + +static void __detach_device(struct sunway_iommu_dev *sunway_dev_data) +{ + struct sunway_iommu_domain *domain; + + domain = sunway_dev_data->domain; + + spin_lock(&domain->lock); + do_detach(sunway_dev_data); + spin_unlock(&domain->lock); +} + +static int attach_device(struct device *dev, struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu_dev *sdev; + unsigned long flags; + int ret; + + sdev = dev_iommu_priv_get(dev); + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + ret = __attach_device(sdev, sdomain); + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); + + return ret; +} + +static void detach_device(struct device *dev) +{ + struct sunway_iommu_domain *sunway_domain; + struct sunway_iommu_dev *sdev; + unsigned long flags; + + sdev = dev_iommu_priv_get(dev); + sunway_domain = sdev->domain; + + if (WARN_ON(!sdev->domain)) + return; + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + __detach_device(sdev); + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); + + if (!dev_is_pci(dev)) + return; +} + +static struct sunway_iommu_dev *search_dev_data(u16 devid) +{ + struct sunway_iommu_dev *sdev_data; + struct llist_node *node; + + if (llist_empty(&dev_data_list)) + return NULL; + + node = dev_data_list.first; + llist_for_each_entry(sdev_data, node, dev_data_list) { + if (sdev_data->devid == devid) + return sdev_data; + } + + return NULL; +} + +/* dma_ops helpers*/ +static struct sunway_iommu_domain *get_sunway_domain(struct device *dev) +{ + struct sunway_iommu_domain *sdomain; + struct iommu_domain *domain; + struct pci_dev *pdev; + struct sunway_iommu_dev *sdev; + + pdev = to_pci_dev(dev); + if (!pdev) + return ERR_PTR(-ENODEV); + + sdev = dev_iommu_priv_get(dev); + sdomain = sdev->domain; + if (sdomain == NULL) { + domain = iommu_get_domain_for_dev(dev); + sdomain = to_sunway_domain(domain); + attach_device(dev, sdomain); + } + + if (sdomain == NULL) + return ERR_PTR(-EBUSY); + + return sdomain; +} + +/********************************************************************** + * + * Following functions describe IOMMU init ops + * + **********************************************************************/ + +static struct sunway_iommu *sunway_iommu_early_init(struct pci_controller *hose) +{ + struct sunway_iommu *iommu; + struct page *page; + unsigned long base; + + hose->pci_iommu = kzalloc(sizeof(struct sunway_iommu), GFP_KERNEL); + if (!hose->pci_iommu) + return 0; + + iommu = hose->pci_iommu; + spin_lock_init(&iommu->dt_lock); + + iommu->node = hose->node; + if (!node_online(hose->node)) + iommu->node = -1; + + page = alloc_pages_node(iommu->node, __GFP_ZERO, get_order(PAGE_SIZE)); + iommu->iommu_dtbr = page_address(page); + + iommu->hose_pt = hose; + iommu->index = hose->index; + + iommu->enabled = true; + + base = __pa(iommu->iommu_dtbr) & PAGE_MASK; + write_piu_ior0(hose->node, hose->index, DTBASEADDR, base); + + return iommu; +} + +unsigned long fetch_dte(struct sunway_iommu *iommu, unsigned long devid, + enum exceptype type) +{ + unsigned long *dte_l1, *dte_l2; + unsigned long dte_l1_val, dte_l2_val; + + if (!iommu) + return 0; + dte_l1 = iommu->iommu_dtbr + (devid >> 8); + if (type == DTE_LEVEL1) + return (unsigned long)dte_l1; + + dte_l1_val = *dte_l1; + if (type == DTE_LEVEL1_VAL) + return dte_l1_val; + + dte_l1_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + dte_l1_val |= PAGE_OFFSET; + dte_l2 = (unsigned long *)(dte_l1_val + ((devid & 0xff) << 3)); + if (type == DTE_LEVEL2) + return (unsigned long)dte_l2; + + dte_l2_val = *dte_l2; + if (type == DTE_LEVEL2_VAL) + return dte_l2_val; + + return dte_l2_val; +} + +unsigned long fetch_pte(struct sunway_iommu_domain *sdomain, dma_addr_t iova, + enum exceptype type) +{ + unsigned long iova_pfn; + unsigned long pte_l1_val, pte_l2_val, pte_l3_val; + unsigned long *pte_l1, *pte_l2, *pte_l3; + unsigned long pte_root; + unsigned long offset; + + if (!sdomain) + return -EINVAL; + + pte_root = __pa(sdomain->pt_root) & PAGE_MASK; + iova_pfn = iova >> PAGE_SHIFT; + pte_root = ((pte_root) & (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK)); + pte_root |= PAGE_OFFSET; + offset = ((iova_pfn >> 20) & SW64_IOMMU_LEVEL1_OFFSET) << 3; + pte_l1 = (unsigned long *)(pte_root + offset); + if (type == PTE_LEVEL1) + return (unsigned long)pte_l1; + + pte_l1_val = *pte_l1; + if (type == PTE_LEVEL1_VAL) + return pte_l1_val; + + pte_l1_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + pte_l1_val |= PAGE_OFFSET; + offset = ((iova_pfn >> 10) & SW64_IOMMU_LEVEL2_OFFSET) << 3; + pte_l2 = (unsigned long *)(pte_l1_val + offset); + + if (type == PTE_LEVEL2) + return (unsigned long)pte_l2; + + pte_l2_val = *pte_l2; + if (type == PTE_LEVEL2_VAL) + return pte_l2_val; + + pte_l2_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + pte_l2_val |= PAGE_OFFSET; + offset = (iova_pfn & SW64_IOMMU_LEVEL3_OFFSET) << 3; + pte_l3 = (unsigned long *)(pte_l2_val + offset); + if (type == PTE_LEVEL3) + return (unsigned long)pte_l3; + + pte_l3_val = *pte_l3; + if (type == PTE_LEVEL3_VAL) + return pte_l3_val; + + return pte_l3_val; +} + +/* IOMMU Interrupt handle */ +irqreturn_t iommu_interrupt(int irq, void *dev) +{ + struct pci_controller *hose = (struct pci_controller *)dev; + struct sunway_iommu_domain *sdomain; + struct sunway_iommu_dev *sdev; + unsigned long iommu_status; + unsigned long type; + unsigned long devid, dva; + + iommu_status = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); + if (!(iommu_status >> 63)) + return IRQ_NONE; + + type = (iommu_status >> 58) & 0xf; + devid = (iommu_status >> 36) & 0xffff; + dva = ((iommu_status & 0xffffffff) >> 3) << 13; + pr_info("%s, iommu_status = %#lx, devid %#lx, dva %#lx, ", + __func__, iommu_status, devid, dva); + + sdev = search_dev_data(devid); + if (sdev == NULL) { + pr_info("no such dev!!!\n"); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + + return IRQ_HANDLED; + } + + sdomain = sdev->domain; + switch (type) { + case DTE_LEVEL1: + pr_info("invalid level1 dte, addr:%#lx, val:%#lx\n", + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1), + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1_VAL)); + break; + case DTE_LEVEL2: + pr_info("invalid level2 dte, addr:%#lx, val:%#lx\n", + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2), + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2_VAL)); + break; + case PTE_LEVEL1: + pr_info("invalid level1 pte, addr: %#lx, val:%#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL1), + fetch_pte(sdomain, dva, PTE_LEVEL1_VAL)); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + break; + case PTE_LEVEL2: + pr_info("invalid level2 pte, addr: %#lx, val: %#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL2), + fetch_pte(sdomain, dva, PTE_LEVEL2_VAL)); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + break; + + case PTE_LEVEL3: + pr_info("invalid level3 pte, addr: %#lx, val: %#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL3), + fetch_pte(sdomain, dva, PTE_LEVEL3_VAL)); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + break; + default: + pr_info("iommu exception type %ld\n", type); + break; + } + + return IRQ_HANDLED; +} + +struct irqaction iommu_irqaction = { + .handler = iommu_interrupt, + .flags = IRQF_SHARED | IRQF_NO_THREAD, + .name = "sunway_iommu", +}; + +void sunway_enable_iommu_func(struct pci_controller *hose) +{ + unsigned int iommu_irq, err; + unsigned long iommu_conf, iommu_ctrl; + + iommu_irq = hose->int_irq; + pr_debug("%s node %ld rc %ld iommu_irq %d\n", + __func__, hose->node, hose->index, iommu_irq); + err = request_irq(iommu_irq, iommu_interrupt, + IRQF_SHARED, "sunway_iommu", hose); + if (err < 0) + pr_info("sw iommu request irq failed!\n"); + + iommu_ctrl = (1UL << 63) | (0x100UL << 10); + write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_CTRL, iommu_ctrl); + iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + iommu_conf = iommu_conf | (0x3 << 7); + write_piu_ior0(hose->node, hose->index, PIUCONFIG0, iommu_conf); + write_piu_ior0(hose->node, hose->index, TIMEOUT_CONFIG, 0xf); + iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + pr_debug("SW arch configure node %ld hose-%ld iommu_conf = %#lx\n", + hose->node, hose->index, iommu_conf); +} + +static bool is_iommu_enable(struct pci_controller *hose) +{ + u64 rc_mask = 0x1; + + rc_mask <<= (8 * hose->node + hose->index); + if (iommu_enable_cmd & rc_mask) + return true; + + return false; +} + +static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type); + +int sunway_iommu_init(void) +{ + struct pci_controller *hose; + struct sunway_iommu *iommu; + int ret; + int iommu_index = 0; + + sunway_iommu_domain_bitmap = + (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(MAX_DOMAIN_NUM / 8)); + if (sunway_iommu_domain_bitmap == NULL) + return 0; + __set_bit(0, sunway_iommu_domain_bitmap); + + /* Do the loop */ + for (hose = hose_head; hose; hose = hose->next) { + if (!is_iommu_enable(hose)) { + hose->iommu_enable = false; + continue; + } + + iommu = sunway_iommu_early_init(hose); + iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", + iommu_index); + iommu_device_set_ops(&iommu->iommu, &sunway_iommu_ops); + iommu_device_register(&iommu->iommu); + iommu_index++; + sunway_enable_iommu_func(hose); + hose->iommu_enable = true; + } + + ret = iova_cache_get(); + if (ret) + return ret; + + ret = bus_set_iommu(&pci_bus_type, &sunway_iommu_ops); + if (ret) + return ret; + + for (hose = hose_head; hose; hose = hose->next) + if (hose->iommu_enable) + piu_flush_all(hose); + + return 1; +} +device_initcall(sunway_iommu_init); + +/* iommu cpu syscore ops */ +static int iommu_cpu_suspend(void) +{ + return 0; +} + +static void iommu_cpu_resume(void) +{ + +} + +struct syscore_ops iommu_cpu_syscore_ops = { + .suspend = iommu_cpu_suspend, + .resume = iommu_cpu_resume, +}; + +/******************************************************************************* + * + * DMA OPS Functions + * + ******************************************************************************/ + +struct sunway_iommu *get_first_iommu_from_domain(struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu *iommu; + struct sunway_iommu_dev *entry; + + entry = list_first_entry(&sdomain->dev_list, struct sunway_iommu_dev, list); + iommu = entry->iommu; + + return iommu; +} + +static unsigned long +sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, + unsigned long iova, unsigned long page_size) +{ + unsigned long offset, iova_pfn; + unsigned long *pte_base, *pte; + unsigned long grn; + int level, current_level; + int tmp = 1; + + pr_debug("%s iova %#lx, page_size %#lx\n", __func__, iova, page_size); + BUG_ON(!is_power_of_2(page_size)); + + switch (page_size) { + case (1UL << 33): + level = 1; + grn = PTE_GRN_8G; + break; + case (1UL << 29): + level = 2; + grn = PTE_GRN_512M; + break; + case (1UL << 23): + level = 2; + grn = PTE_GRN_8M; + break; + default: + level = 3; + break; + } + + pte_base = sunway_domain->pt_root; + iova_pfn = iova >> PAGE_SHIFT; + offset = (iova_pfn >> 20) & 0x1ff; + current_level = 1; + while (current_level <= level) { + pte = &pte_base[offset]; + if (current_level == level) { + if (grn == PTE_GRN_512M) { + int i; + + for (i = 0; i < 64; i++) { + *(pte + i) = 0; + flush_pcache_by_addr(sunway_domain, (unsigned long)pte); + } + + } else { + *pte = 0; + flush_pcache_by_addr(sunway_domain, (unsigned long)pte); + } + flush_ptlb_by_addr(sunway_domain, (iova >> PAGE_SHIFT)); + break; + } + + pte_base = (unsigned long *)((*pte & (~PTE_FLAGS_MASK)) | PAGE_OFFSET); + offset = (iova_pfn >> (tmp--) * 10) & 0x3ff; + current_level++; + } + + return page_size; +} + +int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, + unsigned long bus_addr, unsigned long paddr, + size_t page_size) +{ + struct page *page; + struct sunway_iommu *iommu; + unsigned long iova_pfn, pte_val; + unsigned long *pte_base, *pte; + unsigned long offset, grn = 0; + int level = 0, current_level; + int tmp = 1; + + iommu = get_first_iommu_from_domain(sunway_domain); + if (!iommu) + return -1; + iova_pfn = bus_addr >> PAGE_SHIFT; + pte_base = sunway_domain->pt_root; + + switch (page_size) { + case (1UL << 33): + level = 1; + grn = PTE_GRN_8G; + break; + case (1UL << 29): + level = 2; + grn = PTE_GRN_512M; + break; + case (1UL << 23): + grn = PTE_GRN_8M; + level = 2; + break; + default: + level = 3; + break; + } + + offset = (iova_pfn >> 20) & 0x1ff; + current_level = 1; + while (current_level <= level) { + pte = &pte_base[offset]; + + if (!(*pte) || (current_level == level)) { + pte_val = PTE_VALID | PTE_RWE | grn; + if (current_level == level) { + *(volatile u64 *)(pte) = 0; + pte_val |= ((paddr & PAGE_MASK) | LAST_STAGE); + } else { + page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0); + if (!page) { + pr_err("Allocating level%d page table pages failed.\n", (level + 1)); + return -ENOMEM; + } + + pte_val |= (page_to_phys(page) & PAGE_MASK); + } + + if ((grn == PTE_GRN_512M) && (current_level == 2)) { + int i; + + for (i = 0; i < 64; i++) { + cmpxchg64((volatile u64 *)(pte + i), 0UL, pte_val); + flush_pcache_by_addr(sunway_domain, (unsigned long)(pte + i)); + } + } else { + if (cmpxchg64((volatile u64 *)pte, 0UL, pte_val)) + free_page((unsigned long)page_address(page)); + else + flush_pcache_by_addr(sunway_domain, (unsigned long)pte); + } + } + + pte_base = (unsigned long *)__va((*pte) & (~PTE_FLAGS_MASK)); + offset = (iova_pfn >> (tmp--) * 10) & 0x3ff; + current_level++; + } + + return 0; +} + +static unsigned long +sunway_alloc_iova(struct dma_domain *dma_dom, unsigned long pages, struct pci_dev *pdev) +{ + struct device *dev; + unsigned long pfn = 0; + + pages = __roundup_pow_of_two(pages); + dev = &(pdev->dev); + if (min(dev->coherent_dma_mask, *dev->dma_mask) == DMA_BIT_MASK(32)) { + pfn = alloc_iova_fast(&dma_dom->iovad, pages, + IOVA_PFN(SW64_32BIT_DMA_LIMIT), true); + } else { + /* IOVA boundary should be 16M ~ 3.5G */ + pfn = alloc_iova_fast(&dma_dom->iovad, pages, + IOVA_PFN(SW64_64BIT_DMA_LIMIT), true); + } + + return (pfn << PAGE_SHIFT); +} + +static void sunway_free_iova(struct dma_domain *dma_dom, + unsigned long address, unsigned long pages) +{ + pages = __roundup_pow_of_two(pages); + address >>= PAGE_SHIFT; + + free_iova_fast(&dma_dom->iovad, address, pages); +} + +static dma_addr_t +__sunway_map_single(struct dma_domain *dma_dom, + struct pci_dev *pdev, phys_addr_t paddr, size_t size) +{ + dma_addr_t ret, address, start; + unsigned long npages, i; + + npages = iommu_num_pages(paddr, size, PAGE_SIZE); + + address = sunway_alloc_iova(dma_dom, npages, pdev); + if (!address) + return 0; + + start = address; + for (i = 0; i < npages; ++i) { + ret = sunway_iommu_map_page(&dma_dom->sdomain, start, + paddr, PAGE_SIZE); + if (ret) { + pr_info("error when map page.\n"); + goto out_unmap; + } + + start += PAGE_SIZE; + paddr += PAGE_SIZE; + } + + address += paddr & ~PAGE_MASK; + return address; + +out_unmap: + for (--i; i >= 0; --i) { + start -= PAGE_SIZE; + sunway_iommu_unmap_page(&dma_dom->sdomain, start, PAGE_SIZE); + } + + sunway_free_iova(dma_dom, address, npages); + return 0; +} + +static dma_addr_t +pci_iommu_map_single(struct pci_dev *pdev, + struct dma_domain *dma_dom, void *cpu_addr, size_t size) +{ + struct pci_controller *hose = pdev->sysdata; + unsigned long paddr; + + if (hose == NULL) { + pr_err("%s: hose does not exist!\n", __func__); + return 0; + } + + paddr = __sunway_map_single(dma_dom, pdev, __pa(cpu_addr), size); + + pr_debug("pci_alloc_consistent: %zx -> [%px,%lx] from %ps\n", + size, cpu_addr, paddr, __builtin_return_address(0)); + + return paddr; +} + +static void *sunway_alloc_coherent(struct device *dev, + size_t size, + dma_addr_t *dma_addr, gfp_t gfp, + unsigned long attrs) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_controller *hose; + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct sunway_iommu_dev *sdev; + struct page *page; + void *cpu_addr; + + if (!pdev) + return NULL; + + hose = pdev->sysdata; + if (!hose) + return NULL; + + gfp &= ~GFP_DMA; + +try_again: + page = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, get_order(size)); + cpu_addr = page_address(page); + if (!cpu_addr) { + pr_info + ("pci_alloc_consistent: get_free_pages failed from %ps\n", + __builtin_return_address(0)); + + return NULL; + } + + *dma_addr = __pa(cpu_addr); + if (!(hose->iommu_enable)) + return cpu_addr; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough & DMA_MASK64) + return cpu_addr; + else if (sdev->passthrough) { + if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { + sdev->passthrough |= DMA_MASK64; + return cpu_addr; + } + + __free_pages(page, get_order(size)); + set_dma_ops(dev, get_arch_dma_ops(dev->bus)); + return dev->dma_ops->alloc(dev, size, dma_addr, gfp, attrs); + } + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + *dma_addr = pci_iommu_map_single(pdev, dma_dom, cpu_addr, size); + if (*dma_addr == 0) { + free_pages((unsigned long)cpu_addr, get_order(size)); + if (gfp & GFP_DMA) + return NULL; + + gfp |= GFP_DMA; + goto try_again; + } + + return cpu_addr; +} + +static void +__sunway_unmap_single(struct dma_domain *dma_dom, dma_addr_t dma_addr, size_t size) +{ + dma_addr_t start; + unsigned long npages; + int i; + + npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); + dma_addr &= PAGE_MASK; + start = dma_addr; + + for (i = 0; i < npages; i++) { + sunway_iommu_unmap_page(&dma_dom->sdomain, start, PAGE_SIZE); + start += PAGE_SIZE; + } + + sunway_free_iova(dma_dom, dma_addr, npages); + pr_debug("pci_free_consistent: %zx -> [%llx] from %ps\n", + size, dma_addr, __builtin_return_address(0)); + +} + +static void +sunway_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_addr, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + + if (!pdev) + goto out_unmap; + + hose = pdev->sysdata; + if (!hose || !(hose->iommu_enable)) + goto out_unmap; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough) + goto out_unmap; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + __sunway_unmap_single(dma_dom, dma_addr, size); + goto out_free; + +out_unmap: + pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); + +out_free: + pr_debug("sunway_free_consistent: [%llx,%zx] from %ps\n", + dma_addr, size, __builtin_return_address(0)); + + free_pages((unsigned long)vaddr, get_order(size)); +} + +static dma_addr_t +sunway_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, unsigned long attrs) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + phys_addr_t paddr = page_to_phys(page) + offset; + + if (dir == PCI_DMA_NONE) + BUG(); + + if (!pdev) + return 0; + + hose = pdev->sysdata; + if (!hose || !(hose->iommu_enable)) + return paddr; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough & DMA_MASK64) + return paddr; + else if (sdev->passthrough) { + if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { + sdev->passthrough |= DMA_MASK64; + return paddr; + } + + set_dma_ops(dev, get_arch_dma_ops(dev->bus)); + return dev->dma_ops->map_page(dev, page, offset, size, dir, attrs); + } + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + return pci_iommu_map_single(pdev, dma_dom, + (char *)page_address(page) + offset, size); +} + +static void +sunway_unmap_page(struct device *dev, dma_addr_t dma_addr, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct pci_dev *pdev; + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + + pdev = to_pci_dev(dev); + if (!pdev) + return; + + hose = pdev->sysdata; + if (hose == NULL) + return; + + if (!hose->iommu_enable) + return; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough) + return; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + __sunway_unmap_single(dma_dom, dma_addr, size); +} + +#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) +static int +sunway_map_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom = NULL; + struct scatterlist *sg; + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + int i, out_nents = 0; + + if (dir == PCI_DMA_NONE) + BUG(); + + if (!pdev) + return 0; + + hose = pdev->sysdata; + if (!hose) + return 0; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + for_each_sg(sgl, sg, nents, i) { + BUG_ON(!sg_page(sg)); + + sg_dma_address(sg) = __pa(SG_ENT_VIRT_ADDRESS(sg)); + if (!(hose->iommu_enable)) + goto check; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough & DMA_MASK64) + goto check; + else if (sdev->passthrough) { + if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { + sdev->passthrough |= DMA_MASK64; + goto check; + } + + set_dma_ops(dev, get_arch_dma_ops(dev->bus)); + return dev->dma_ops->map_sg(dev, sgl, nents, dir, attrs); + } + + sg_dma_address(sg) = + pci_iommu_map_single(pdev, dma_dom, + SG_ENT_VIRT_ADDRESS(sg), sg->length); +check: + if (sg_dma_address(sg) == 0) + goto error; + + sg_dma_len(sg) = sg->length; + out_nents++; + } + + return nents; + +error: + pr_warn("pci_map_sg failed:"); + pr_warn("could not allocate dma page tables\n"); + + if (out_nents) + pci_unmap_sg(pdev, sgl, out_nents, dir); + return 0; +} + +static void +sunway_unmap_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct scatterlist *sg; + struct pci_dev *pdev; + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + dma_addr_t dma_addr; + long size; + int j; + + pdev = to_pci_dev(dev); + if (!pdev) + return; + + hose = pdev->sysdata; + if (!hose->iommu_enable) + return; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough) + return; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + for_each_sg(sgl, sg, nents, j) { + dma_addr = sg->dma_address; + size = sg->dma_length; + if (!size) + break; + + __sunway_unmap_single(dma_dom, dma_addr, size); + } +} + +static const struct dma_map_ops sunway_dma_ops = { + .alloc = sunway_alloc_coherent, + .free = sunway_free_coherent, + .map_sg = sunway_map_sg, + .unmap_sg = sunway_unmap_sg, + .map_page = sunway_map_page, + .unmap_page = sunway_unmap_page, + .dma_supported = dma_direct_supported, +}; + +/********************************************************************** + * + * IOMMU OPS Functions + * + **********************************************************************/ + +static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + sdomain = sunway_domain_alloc(); + if (!sdomain) { + pr_err("Allocating sunway_domain failed!\n"); + return NULL; + } + + sdomain->domain.geometry.aperture_start = 0UL; + sdomain->domain.geometry.aperture_end = ~0ULL; + sdomain->domain.geometry.force_aperture = true; + sdomain->type = IOMMU_DOMAIN_UNMANAGED; + break; + + case IOMMU_DOMAIN_DMA: + dma_dom = dma_domain_alloc(); + if (!dma_dom) { + pr_err("Failed to alloc dma domain!\n"); + return NULL; + } + + sdomain = &dma_dom->sdomain; + break; + + case IOMMU_DOMAIN_IDENTITY: + sdomain = sunway_domain_alloc(); + if (!sdomain) + return NULL; + + sdomain->type = IOMMU_DOMAIN_IDENTITY; + break; + + default: + return NULL; + } + + return &sdomain->domain; +} + +static void clean_domain(struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu_dev *entry; + unsigned long flags; + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + + while (!list_empty(&sdomain->dev_list)) { + entry = list_first_entry(&sdomain->dev_list, + struct sunway_iommu_dev, list); + + BUG_ON(!entry->domain); + __detach_device(entry); + } + + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); +} + +static void sunway_iommu_domain_free(struct iommu_domain *dom) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + + sdomain = to_sunway_domain(dom); + + if (sdomain->dev_cnt > 0) + clean_domain(sdomain); + + BUG_ON(sdomain->dev_cnt != 0); + + if (!dom) + return; + + switch (dom->type) { + case IOMMU_DOMAIN_DMA: + dma_dom = to_dma_domain(sdomain); + dma_domain_free(dma_dom); + break; + + default: + free_pagetable(sdomain); + sunway_domain_free(sdomain); + break; + } + +} + +static int sunway_iommu_attach_device(struct iommu_domain *dom, struct device *dev) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + struct sunway_iommu_dev *sdev; + struct pci_dev *pdev; + struct pci_controller *hose; + int ret; + + pdev = to_pci_dev(dev); + if (!pdev) + return -EINVAL; + + hose = pdev->sysdata; + if (!hose) + return -EINVAL; + + if (!hose->iommu_enable) + return -EINVAL; + + sdev = dev_iommu_priv_get(dev); + if (!sdev) + return -EINVAL; + + if (sdev->domain) + detach_device(dev); + + ret = attach_device(dev, sdomain); + + return ret; +} + +static void sunway_iommu_detach_device(struct iommu_domain *dom, struct device *dev) +{ + struct sunway_iommu_dev *sdev; + struct pci_dev *pdev = to_pci_dev(dev); + + if (!pdev) + return; + + sdev = dev_iommu_priv_get(dev); + if (sdev->domain != NULL) + detach_device(dev); +} + +static phys_addr_t +sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + unsigned long paddr, grn; + unsigned long is_last; + + if (iova > SW64_BAR_ADDRESS) + return iova; + + paddr = fetch_pte(sdomain, iova, PTE_LEVEL1_VAL); + if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) + return 0; + + is_last = paddr & SW64_PTE_LAST_MASK; + grn = paddr & SW64_PTE_GRN_MASK; + if (is_last) { + if (grn == PTE_GRN_8G) { + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_8G_OFFSET_MASK; + return paddr; + } + + return 0; + } + + paddr = fetch_pte(sdomain, iova, PTE_LEVEL2_VAL); + if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) + return 0; + + is_last = paddr & SW64_PTE_LAST_MASK; + grn = paddr & SW64_PTE_GRN_MASK; + if (is_last) { + if (grn == PTE_GRN_512M) { + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_512M_OFFSET_MASK; + return paddr; + } + + if (grn == PTE_GRN_8M) { + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_8M_OFFSET_MASK; + return paddr; + } + + return 0; + } + + paddr = fetch_pte(sdomain, iova, PTE_LEVEL3_VAL); + if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) + return 0; + + grn = paddr & SW64_PTE_GRN_MASK; + if (grn != 0) + return 0; + + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_MASK; + return paddr; +} + +static int +sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, + phys_addr_t paddr, size_t page_size, int iommu_prot, gfp_t gfp) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + int ret; + + /* + * As VFIO cannot distinguish between normal DMA request + * and pci device BAR, check should be introduced manually + * to avoid VFIO trying to map pci config space. + */ + if (iova > SW64_BAR_ADDRESS) + return 0; + + mutex_lock(&sdomain->api_lock); + ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size); + mutex_unlock(&sdomain->api_lock); + + return ret; +} + +static size_t +sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, + size_t page_size, + struct iommu_iotlb_gather *gather) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + size_t unmap_size; + + if (iova > SW64_BAR_ADDRESS) + return page_size; + + mutex_lock(&sdomain->api_lock); + unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); + mutex_unlock(&sdomain->api_lock); + + return unmap_size; +} + +static struct iommu_group *sunway_iommu_device_group(struct device *dev) +{ + return pci_device_group(dev); +} + +static void iommu_uninit_device(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + + sdev = dev_iommu_priv_get(dev); + if (!sdev) + return; + + if (sdev->domain) + detach_device(dev); + + dev_iommu_priv_set(dev, NULL); +} + +static void sunway_iommu_release_device(struct device *dev) +{ + struct pci_dev *pdev; + struct pci_controller *hose; + + pdev = to_pci_dev(dev); + if (!pdev) + return; + + hose = pdev->sysdata; + if (!hose->iommu_enable) + return; + + iommu_uninit_device(dev); +} + +static int iommu_init_device(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + struct sunway_iommu *iommu; + struct pci_dev *pdev; + struct pci_controller *hose; + + if (dev_iommu_priv_get(dev)) + return 0; + + sdev = kzalloc(sizeof(struct sunway_iommu_dev), GFP_KERNEL); + if (!sdev) + return -ENOMEM; + + pdev = to_pci_dev(dev); + hose = pdev->sysdata; + iommu = hose->pci_iommu; + llist_add(&sdev->dev_data_list, &dev_data_list); + sdev->pdev = pdev; + sdev->iommu = iommu; + + dev_iommu_priv_set(dev, sdev); + + return 0; +} + +static struct iommu_device *sunway_iommu_probe_device(struct device *dev) +{ + struct pci_dev *pdev; + struct pci_controller *hose; + struct sunway_iommu *iommu; + int ret; + + pdev = to_pci_dev(dev); + if (!pdev) + return ERR_PTR(-ENODEV); + + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + return ERR_PTR(-ENODEV); + + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) + return ERR_PTR(-ENODEV); + + hose = pdev->sysdata; + if (!hose) + return ERR_PTR(-ENODEV); + + if (!hose->iommu_enable) + return ERR_PTR(-ENODEV); + + if (dev_iommu_priv_get(dev)) { + iommu = hose->pci_iommu; + return &iommu->iommu; + } + + ret = iommu_init_device(dev); + if (ret) + return ERR_PTR(ret); + + iommu = hose->pci_iommu; + + return &iommu->iommu; +} + +static int sunway_iommu_def_domain_type(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + + sdev = dev_iommu_priv_get(dev); + if (sdev->domain) + return 0; + + return sdev->domain->type; +} + +static bool sunway_iommu_capable(enum iommu_cap cap) +{ + switch (cap) { + case IOMMU_CAP_INTR_REMAP: + return true; + default: + return false; + } +} + +static void sunway_iommu_probe_finalize(struct device *dev) +{ + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (domain) + set_dma_ops(dev, &sunway_dma_ops); +} + +const struct iommu_ops sunway_iommu_ops = { + .capable = sunway_iommu_capable, + .domain_alloc = sunway_iommu_domain_alloc, + .domain_free = sunway_iommu_domain_free, + .attach_dev = sunway_iommu_attach_device, + .detach_dev = sunway_iommu_detach_device, + .probe_device = sunway_iommu_probe_device, + .probe_finalize = sunway_iommu_probe_finalize, + .release_device = sunway_iommu_release_device, + .map = sunway_iommu_map, + .unmap = sunway_iommu_unmap, + .iova_to_phys = sunway_iommu_iova_to_phys, + .device_group = sunway_iommu_device_group, + .pgsize_bitmap = SW64_IOMMU_PGSIZES, + .def_domain_type = sunway_iommu_def_domain_type, +}; + +/***************************************************************************** + * + * Boot param handle + * Each bit of iommu_enable bitmap represents an rc enable, and every 8 bits + * represents one cpu node. For example, iommu_enable=0x0100 means enabling + * rc0 for cpu node 1. + * + *****************************************************************************/ +static int __init iommu_enable_setup(char *str) +{ + int ret; + unsigned long rc_bitmap = 0xffffffffUL; + + ret = kstrtoul(str, 16, &rc_bitmap); + iommu_enable_cmd = rc_bitmap; + + return ret; +} +__setup("iommu_enable=", iommu_enable_setup); diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 4f7d22b1ef4280900edff7bd40689197480702cc..8ddf6461b1751a9a1f5a3f068e9d0076c7e3eb48 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -13,7 +13,7 @@ config ARM_GIC config SW64_INTC_V2 bool "SW64 Interrupt Controller V2" - depends on SW64_CHIP3 + depends on UNCORE_XUELANG default y select GENERIC_IRQ_CHIP select IRQ_DOMAIN @@ -29,6 +29,21 @@ config SW64_LPC_INTC Say yes here to add support for the SW64 cpu builtin LPC IRQ controller. +config SW64_IRQ_CPU + bool + depends on SW64 + default y + +config SW64_IRQ_MSI + bool + depends on SW64 && PCI_MSI + default y + +config SW64_IRQ_MSI_VT + bool + depends on SW64_IRQ_MSI + default y + config ARM_GIC_PM bool depends on PM diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 0e69f31b450801ec15434afb3460271bd27c11ab..eaa41878ed29660991c50b95cd51ff48be22de3e 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -29,6 +29,15 @@ obj-$(CONFIG_ARCH_SPEAR3XX) += spear-shirq.o obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o obj-$(CONFIG_SW64_INTC_V2) += irq-sw64-intc-v2.o obj-$(CONFIG_SW64_LPC_INTC) += irq-sw64-lpc-intc.o +obj-$(CONFIG_SW64_IRQ_CPU) += irq-sunway-cpu.o + +ifeq ($(CONFIG_UNCORE_XUELANG),y) +obj-$(CONFIG_SW64_IRQ_MSI) += irq-sunway-msi.o +else +obj-$(CONFIG_SW64_IRQ_MSI) += irq-sunway-msi-v2.o +endif + +obj-$(CONFIG_SW64_IRQ_MSI_VT) += irq-sunway-msi-vt.o obj-$(CONFIG_ARM_GIC_PM) += irq-gic-pm.o obj-$(CONFIG_ARCH_REALVIEW) += irq-gic-realview.o obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c new file mode 100644 index 0000000000000000000000000000000000000000..ca1b62b2d64531279e5e8ab7e9bc1cddd88743a9 --- /dev/null +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include +#include +#include + +static void handle_intx(unsigned int offset) +{ + struct pci_controller *hose; + unsigned long value; + + hose = hose_head; + for (hose = hose_head; hose; hose = hose->next) { + value = read_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7)); + if (value >> 63) { + value = value & (~(1UL << 62)); + write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); + handle_irq(hose->int_irq); + value = value | (1UL << 62); + write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); + } + + if (IS_ENABLED(CONFIG_PCIE_PME)) { + value = read_piu_ior0(hose->node, hose->index, PMEINTCONFIG); + if (value >> 63) { + handle_irq(hose->service_irq); + write_piu_ior0(hose->node, hose->index, PMEINTCONFIG, value); + } + } + + if (IS_ENABLED(CONFIG_PCIEAER)) { + value = read_piu_ior0(hose->node, hose->index, AERERRINTCONFIG); + if (value >> 63) { + handle_irq(hose->service_irq); + write_piu_ior0(hose->node, hose->index, AERERRINTCONFIG, value); + } + } + + if (hose->iommu_enable) { + value = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); + if (value >> 63) + handle_irq(hose->int_irq); + } + } +} + +static void handle_device_interrupt(unsigned long irq_info) +{ + unsigned int i; + + if (is_guest_or_emul()) { + handle_irq(irq_info); + return; + } + + for (i = 0; i < 4; i++) { + if ((irq_info >> i) & 0x1) + handle_intx(i); + } +} + +/* Performance counter hook. A module can override this to do something useful. */ +static void dummy_perf(unsigned long vector, struct pt_regs *regs) +{ + irq_err_count++; + pr_crit("Performance counter interrupt!\n"); +} + +void (*perf_irq)(unsigned long, struct pt_regs*) = dummy_perf; +EXPORT_SYMBOL(perf_irq); + +static void handle_fault_int(void) +{ + int node; + unsigned long value; + + node = __this_cpu_read(hard_node_id); + pr_info("enter fault int, si_fault_stat = %#lx\n", + sw64_io_read(node, SI_FAULT_STAT)); + sw64_io_write(node, SI_FAULT_INT_EN, 0); + sw64_io_write(node, DLI_RLTD_FAULT_INTEN, 0); +#if defined(CONFIG_UNCORE_XUELANG) + value = 0; +#elif defined(CONFIG_UNCORE_JUNZHANG) + value = sw64_io_read(node, FAULT_INT_CONFIG); + value |= (1 << 8); +#endif + __io_write_fault_int_en(node, value); +} + +static void handle_mt_int(void) +{ + pr_info("enter mt int\n"); +} + +static void handle_nmi_int(void) +{ + pr_info("enter nmi int\n"); +} + +static void handle_dev_int(struct pt_regs *regs) +{ + unsigned long config_val, val, stat; + int node = 0; + unsigned int hwirq; + + config_val = sw64_io_read(node, DEV_INT_CONFIG); + val = config_val & (~(1UL << 8)); + sw64_io_write(node, DEV_INT_CONFIG, val); + stat = sw64_io_read(node, MCU_DVC_INT); + + while (stat) { + hwirq = ffs(stat) - 1; + handle_domain_irq(NULL, hwirq, regs); + stat &= ~(1UL << hwirq); + } + /*do handle irq */ + + sw64_io_write(node, DEV_INT_CONFIG, config_val); +} + +asmlinkage void do_entInt(unsigned long type, unsigned long vector, + unsigned long irq_arg, struct pt_regs *regs) +{ + struct pt_regs *old_regs; + extern char __idle_start[], __idle_end[]; + + if (is_guest_or_emul()) { + if ((type & 0xffff) > 15) { + vector = type; + if (vector == 16) + type = INT_INTx; + else + type = INT_MSI; + } + } + + /* restart idle routine if it is interrupted */ + if (regs->pc > (u64)__idle_start && regs->pc < (u64)__idle_end) + regs->pc = (u64)__idle_start; + + switch (type & 0xffff) { + case INT_MSI: + old_regs = set_irq_regs(regs); + handle_pci_msi_interrupt(type, vector, irq_arg); + set_irq_regs(old_regs); + return; + case INT_INTx: + old_regs = set_irq_regs(regs); + handle_device_interrupt(vector); + set_irq_regs(old_regs); + return; + + case INT_IPI: +#ifdef CONFIG_SMP + handle_ipi(regs); + return; +#else + irq_err_count++; + pr_crit("Interprocessor interrupt? You must be kidding!\n"); +#endif + break; + case INT_RTC: + old_regs = set_irq_regs(regs); + sw64_timer_interrupt(); + set_irq_regs(old_regs); + return; + case INT_VT_SERIAL: + old_regs = set_irq_regs(regs); + handle_irq(type); + set_irq_regs(old_regs); + return; + case INT_VT_HOTPLUG: + old_regs = set_irq_regs(regs); + handle_irq(type); + set_irq_regs(old_regs); + return; + case INT_PC0: + perf_irq(PMC_PC0, regs); + return; + case INT_PC1: + perf_irq(PMC_PC1, regs); + return; + case INT_DEV: + old_regs = set_irq_regs(regs); + handle_dev_int(regs); + set_irq_regs(old_regs); + return; + case INT_FAULT: + old_regs = set_irq_regs(regs); + handle_fault_int(); + set_irq_regs(old_regs); + return; + case INT_MT: + old_regs = set_irq_regs(regs); + handle_mt_int(); + set_irq_regs(old_regs); + return; + case INT_NMI: + old_regs = set_irq_regs(regs); + handle_nmi_int(); + set_irq_regs(old_regs); + return; + default: + pr_crit("Hardware intr %ld %lx? uh?\n", type, vector); + } + pr_crit("PC = %016lx PS = %04lx\n", regs->pc, regs->ps); +} +EXPORT_SYMBOL(do_entInt); diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c new file mode 100644 index 0000000000000000000000000000000000000000..213a22f2b54bb11ce25b8074e51e3bdc34c967c3 --- /dev/null +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include +#include + +static struct irq_domain *msi_default_domain; +static DEFINE_RAW_SPINLOCK(vector_lock); +DEFINE_PER_CPU(vector_irq_t, vector_irq) = { + [0 ... PERCPU_MSI_IRQS - 1] = 0, +}; + +static struct sw64_msi_chip_data *alloc_sw_msi_chip_data(struct irq_data *irq_data) +{ + struct sw64_msi_chip_data *data; + int node; + + node = irq_data_get_node(irq_data); + data = kzalloc_node(sizeof(*data), GFP_KERNEL, node); + if (!data) + return NULL; + spin_lock_init(&data->cdata_lock); + return data; +} + +static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct sw64_msi_chip_data *chip_data; + int rcid; + + chip_data = irq_data_get_irq_chip_data(data->parent_data); + rcid = cpu_to_rcid(chip_data->dst_cpu); + msg->address_hi = MSI_ADDR_BASE_HI; + msg->address_lo = + (unsigned int)chip_data->msiaddr | + (rcid_to_msicid(rcid) << MSI_ADDR_DEST_ID_SHIFT); + msg->data = chip_data->vector; +} + +bool find_free_cpu_vector(const struct cpumask *search_mask, + int *found_cpu, int *found_vector) +{ + int vector, max_vector, cpu; + bool find_once_global = false; + + cpu = cpumask_first(search_mask); +try_again: + if (is_guest_or_emul()) { + vector = IRQ_PENDING_MSI_VECTORS_SHIFT; + max_vector = SWVM_IRQS; + } else { + vector = 0; + max_vector = 256; + } + for (; vector < max_vector; vector++) { + while (per_cpu(vector_irq, cpu)[vector]) { + cpu = cpumask_next(cpu, search_mask); + if (cpu >= nr_cpu_ids) { + if (vector == 255) { + if (find_once_global) { + printk("No global free vector\n"); + return false; + } + printk("No local free vector\n"); + search_mask = cpu_online_mask; + cpu = cpumask_first(search_mask); + find_once_global = true; + goto try_again; + } + cpu = cpumask_first(search_mask); + break; + } + } + if (!per_cpu(vector_irq, cpu)[vector]) + break; + } + + *found_cpu = cpu; + *found_vector = vector; + return true; +} + +static bool find_free_cpu_vectors(const struct cpumask *search_mask, int *found_cpu, int *found_vector, unsigned int nr_irqs) +{ + int i, vector, cpu; + bool found = false, find_once_global = false; + + cpu = cpumask_first(search_mask); +try_again: + for (vector = 0; vector < 256; vector++) { + for (i = 0; i < nr_irqs; i++) + if (per_cpu(vector_irq, cpu)[vector + i]) + break; + + if (i == nr_irqs) { + found = true; + *found_cpu = cpu; + *found_vector = vector; + return found; + } + + vector += i; + } + + cpu = cpumask_next(cpu, search_mask); + if (cpu < nr_cpu_ids) + goto try_again; + else { + if (find_once_global) { + printk("No global free vectors\n"); + return found; + } + printk("No local free vectors\n"); + search_mask = cpu_online_mask; + cpu = cpumask_first(search_mask); + find_once_global = true; + goto try_again; + } +} + +static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, bool force) +{ + struct sw64_msi_chip_data *cdata; + struct irq_data *irqd; + struct msi_desc *entry; + struct cpumask searchmask; + unsigned long flags; + int vector, cpu; + int i; + struct msi_msg msg; + + /* Is this valid ? */ + if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) + return -EINVAL; + + irqd = irq_domain_get_irq_data(msi_default_domain->parent, d->irq); + /* Don't do anything if the interrupt isn't started */ + if (!irqd_is_started(irqd)) + return IRQ_SET_MASK_OK; + + cdata = irqd->chip_data; + if (!cdata) + return -ENOMEM; + + /* + * If existing target cpu is already in the new mask and is online + * then do nothing. + */ + if (cpu_online(cdata->dst_cpu) && cpumask_test_cpu(cdata->dst_cpu, cpumask)) + return IRQ_SET_MASK_OK; + + raw_spin_lock_irqsave(&vector_lock, flags); + + cpumask_and(&searchmask, cpumask, cpu_online_mask); + if (cdata->multi_msi > 1) { + if (!find_free_cpu_vectors(&searchmask, &cpu, + &vector, cdata->multi_msi)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); + return -ENOSPC; + } + } else { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); + return -ENOSPC; + } + } + + /* update new setting */ + entry = irq_get_msi_desc(irqd->irq); + spin_lock(&cdata->cdata_lock); + for (i = 0; i < cdata->multi_msi; i++) + per_cpu(vector_irq, cpu)[vector + i] = entry->irq + i; + BUG_ON(irq_chip_compose_msi_msg(irqd, &msg)); + __pci_write_msi_msg(entry, &msg); + cdata->prev_vector = cdata->vector; + cdata->prev_cpu = cdata->dst_cpu; + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->move_in_progress = true; + spin_unlock(&cdata->cdata_lock); + cpumask_copy(irq_data_get_affinity_mask(irqd), &searchmask); + + raw_spin_unlock_irqrestore(&vector_lock, flags); + + return 0; +} + +static void chip_irq_ack(struct irq_data *data) +{ +} + +static struct irq_chip pci_msi_controller = { + .name = "PCI-MSI", + .irq_unmask = pci_msi_unmask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_ack = chip_irq_ack, + .irq_compose_msi_msg = irq_msi_compose_msg, + .flags = IRQCHIP_SKIP_SET_WAKE, + .irq_set_affinity = sw64_set_affinity, +}; + +static int __assign_irq_vector(int virq, unsigned int nr_irqs, + struct irq_domain *domain, enum irq_alloc_type type) +{ + struct irq_data *irq_data; + const struct cpumask *mask; + struct cpumask searchmask; + struct sw64_msi_chip_data *cdata; + int node; + int i, vector, cpu; + unsigned long msiaddr; + + if (unlikely((nr_irqs > 1) && (!is_power_of_2(nr_irqs)))) + nr_irqs = __roundup_pow_of_two(nr_irqs); + + irq_data = irq_domain_get_irq_data(domain, virq); + BUG_ON(!irq_data); + irq_data->chip = &pci_msi_controller; + + if (irqd_affinity_is_managed(irq_data)) { + mask = irq_data_get_affinity_mask(irq_data); + cpumask_and(&searchmask, mask, cpu_online_mask); + } else { + node = irq_data_get_node(irq_data); + cpumask_copy(&searchmask, cpumask_of_node(node)); + } + + if (cpumask_first(&searchmask) >= nr_cpu_ids) + cpumask_copy(&searchmask, cpu_online_mask); + + if (type == IRQ_ALLOC_TYPE_MSI && nr_irqs > 1) { + if (!find_free_cpu_vectors(&searchmask, &cpu, + &vector, nr_irqs)) + return -ENOSPC; + + cdata = alloc_sw_msi_chip_data(irq_data); + if (!cdata) { + printk("error alloc irq chip data\n"); + return -ENOMEM; + } + + for (i = 0; i < nr_irqs; i++) { + per_cpu(vector_irq, cpu)[vector + i] = virq + i; + + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &pci_msi_controller; + } + + irq_data->chip_data = cdata; + } + + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->msiaddr = MSIX_MSG_ADDR; + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + cdata->multi_msi = nr_irqs; + cdata->move_in_progress = false; + } else { + for (i = 0; i < nr_irqs; i++) { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + return -ENOSPC; + + per_cpu(vector_irq, cpu)[vector] = virq + i; + + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &pci_msi_controller; + } + + cdata = alloc_sw_msi_chip_data(irq_data); + if (!cdata) { + printk("error alloc irq chip data\n"); + return -ENOMEM; + } + + irq_data->chip_data = cdata; + + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->msiaddr = MSIX_MSG_ADDR; + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + cdata->multi_msi = 1; + cdata->move_in_progress = false; + } + } + return 0; +} + +static int assign_irq_vector(int irq, unsigned int nr_irqs, + struct irq_domain *domain, enum irq_alloc_type type) +{ + int err; + unsigned long flags; + + raw_spin_lock_irqsave(&vector_lock, flags); + err = __assign_irq_vector(irq, nr_irqs, domain, type); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return err; +} + +static void sw64_vector_free_irqs(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + int i, j; + struct irq_data *irq_data; + unsigned long flags; + unsigned int multi_msi; + + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + if (irq_data && irq_data->chip_data) { + struct sw64_msi_chip_data *cdata; + + raw_spin_lock_irqsave(&vector_lock, flags); + cdata = irq_data->chip_data; + irq_domain_reset_irq_data(irq_data); + multi_msi = cdata->multi_msi; + for (j = 0; j < multi_msi; j++) + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector + j] = 0; + kfree(cdata); + raw_spin_unlock_irqrestore(&vector_lock, flags); + if (multi_msi > 1) + break; + } + } +} + +static void sw64_irq_free_descs(unsigned int virq, unsigned int nr_irqs) +{ + if (is_guest_or_emul()) { + vt_sw64_vector_free_irqs(virq, nr_irqs); + return irq_free_descs(virq, nr_irqs); + } + + return irq_domain_free_irqs(virq, nr_irqs); +} + +void arch_teardown_msi_irqs(struct pci_dev *dev) +{ + struct msi_desc *desc; + int i; + + for_each_pci_msi_entry(desc, dev) { + if (desc->irq) { + for (i = 0; i < desc->nvec_used; i++) + sw64_irq_free_descs(desc->irq + i, 1); + desc->irq = 0; + } + } +} + +static int sw64_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + int err; + struct irq_alloc_info *info = arg; + enum irq_alloc_type msi_type; + + if (arg == NULL) + return -ENODEV; + msi_type = info->type; + err = assign_irq_vector(virq, nr_irqs, domain, msi_type); + if (err) + goto error; + return 0; +error: + sw64_vector_free_irqs(domain, virq, nr_irqs); + return err; +} + +static int pci_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *arg) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct msi_desc *desc = first_pci_msi_entry(pdev); + + memset(arg, 0, sizeof(*arg)); + arg->msi_dev = pdev; + if (desc->msi_attrib.is_msix) + arg->type = IRQ_ALLOC_TYPE_MSIX; + else + arg->type = IRQ_ALLOC_TYPE_MSI; + return 0; +} + +static struct msi_domain_ops pci_msi_domain_ops = { + .msi_prepare = pci_msi_prepare, +}; + +static struct msi_domain_info pci_msi_domain_info = { + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, + .ops = &pci_msi_domain_ops, + .chip = &pci_msi_controller, + .handler = handle_edge_irq, + .handler_name = "edge", +}; + +static int sw64_irq_map(struct irq_domain *d, unsigned int virq, irq_hw_number_t hw) +{ + irq_set_chip_and_handler(virq, &sw64_irq_chip, handle_level_irq); + irq_set_status_flags(virq, IRQ_LEVEL); + return 0; +} + +const struct irq_domain_ops sw64_msi_domain_ops = { + .map = sw64_irq_map, + .alloc = sw64_vector_alloc_irqs, + .free = sw64_vector_free_irqs, +}; + +int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) +{ + struct irq_domain *domain; + int err; + + if (is_guest_or_emul()) + return sw64_setup_vt_msi_irqs(pdev, nvec, type); + + domain = msi_default_domain; + if (domain == NULL) + return -ENOSYS; + err = msi_domain_alloc_irqs(domain, &pdev->dev, nvec); + return err; +} + +void arch_init_msi_domain(struct irq_domain *parent) +{ + struct irq_domain *sw64_irq_domain; + + if (is_guest_or_emul()) + return; + + sw64_irq_domain = irq_domain_add_tree(NULL, &sw64_msi_domain_ops, NULL); + BUG_ON(sw64_irq_domain == NULL); + irq_set_default_host(sw64_irq_domain); + msi_default_domain = pci_msi_create_irq_domain(NULL, + &pci_msi_domain_info, sw64_irq_domain); + if (!msi_default_domain) + pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n"); +} + +static void irq_move_complete(struct sw64_msi_chip_data *cdata, int cpu, int vector) +{ + if (likely(!cdata->move_in_progress)) + return; + if (cdata->dst_cpu == cpu) { + if (vector >= cdata->vector && + vector < cdata->vector + cdata->multi_msi) { + int i; + + raw_spin_lock(&vector_lock); + cdata->move_in_progress = false; + for (i = 0; i < cdata->multi_msi; i++) + per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector + i] = 0; + raw_spin_unlock(&vector_lock); + } + } +} + +void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned long pci_msi1_addr) +{ + int i, irq, msi_index = 0; + int cpu, vector_index = 0; + unsigned long int_pci_msi[3]; + unsigned long *ptr; + struct irq_data *irq_data; + struct sw64_msi_chip_data *cdata; + + if (is_guest_or_emul()) { + cpu = smp_processor_id(); + irq = per_cpu(vector_irq, cpu)[vector]; + handle_irq(irq); + return; + } + + ptr = (unsigned long *)pci_msi1_addr; + int_pci_msi[0] = *ptr; + int_pci_msi[1] = *(ptr + 1); + int_pci_msi[2] = *(ptr + 2); + + cpu = smp_processor_id(); + + for (i = 0; i < 4; i++) { + vector_index = i * 64; + while (vector != 0) { + int irq = 0; + + msi_index = find_next_bit(&vector, 64, msi_index); + if (msi_index == 64) { + msi_index = 0; + continue; + } + + irq = per_cpu(vector_irq, cpu)[vector_index + msi_index]; + irq_data = irq_domain_get_irq_data(msi_default_domain->parent, irq); + cdata = irq_data_get_irq_chip_data(irq_data); + spin_lock(&cdata->cdata_lock); + irq_move_complete(cdata, cpu, vector_index + msi_index); + spin_unlock(&cdata->cdata_lock); + handle_irq(irq); + + vector = vector & (~(1UL << msi_index)); + } + + vector = int_pci_msi[i % 3]; + } +} + +MODULE_LICENSE("GPL v2"); diff --git a/arch/sw_64/chip/chip3/vt_msi.c b/drivers/irqchip/irq-sunway-msi-vt.c similarity index 84% rename from arch/sw_64/chip/chip3/vt_msi.c rename to drivers/irqchip/irq-sunway-msi-vt.c index 0cdf7f196e8a0f24ea1c4c23718f8cd9f866e086..3d7f56f3acdf29bf2d4d6b82ad4b540048f5f7c9 100644 --- a/arch/sw_64/chip/chip3/vt_msi.c +++ b/drivers/irqchip/irq-sunway-msi-vt.c @@ -32,12 +32,57 @@ static void vt_irq_msi_update_msg(struct irq_data *irqd, pci_write_msi_msg(irqd->irq, msg); } +static int +vt_set_affinity(struct irq_data *irqd, const struct cpumask *cpumask, + bool force) +{ + struct sw64_msi_chip_data *cdata; + struct cpumask searchmask; + int cpu, vector; + + /* Is this valid ? */ + if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) + return -EINVAL; + + if (!irqd_is_started(irqd)) + return IRQ_SET_MASK_OK; + + cdata = irqd->chip_data; + if (!cdata) + return -ENOMEM; + + /* + * If existing target coreid is already in the new mask, + * and is online then do nothing. + */ + if (cpu_online(cdata->dst_cpu) && cpumask_test_cpu(cdata->dst_cpu, cpumask)) + return IRQ_SET_MASK_OK; + + cpumask_and(&searchmask, cpumask, cpu_online_mask); + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + return -ENOSPC; + + per_cpu(vector_irq, cpu)[vector] = irqd->irq; + spin_lock(&cdata->cdata_lock); + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->prev_cpu = cdata->dst_cpu; + cdata->prev_vector = cdata->vector; + cdata->move_in_progress = true; + spin_unlock(&cdata->cdata_lock); + cpumask_copy(irq_data_get_affinity_mask(irqd), &searchmask); + vt_irq_msi_update_msg(irqd, irqd->chip_data); + + return 0; +} + static struct irq_chip vt_pci_msi_controller = { .name = "PCI-MSI", .irq_unmask = pci_msi_unmask_irq, .irq_mask = pci_msi_mask_irq, .irq_ack = sw64_irq_noop, .irq_compose_msi_msg = vt_irq_msi_compose_msg, + .irq_set_affinity = vt_set_affinity, }; int chip_setup_vt_msix_irq(struct pci_dev *dev, struct msi_desc *desc) diff --git a/arch/sw_64/chip/chip3/msi.c b/drivers/irqchip/irq-sunway-msi.c similarity index 99% rename from arch/sw_64/chip/chip3/msi.c rename to drivers/irqchip/irq-sunway-msi.c index a0ab4de8fa294e4f548f2feabafd3eca2a228bff..3f8fd9fc775b7a639e80f503ac970fcc424b59b7 100644 --- a/arch/sw_64/chip/chip3/msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -31,6 +31,7 @@ static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) struct sw64_msi_chip_data *chip_data; chip_data = irq_data_get_irq_chip_data(data->parent_data); + msg->address_hi = MSI_ADDR_BASE_HI; msg->address_lo = MSI_ADDR_BASE_LO; msg->data = chip_data->msi_config_index; diff --git a/drivers/irqchip/irq-sw64-intc-v2.c b/drivers/irqchip/irq-sw64-intc-v2.c index 070c12ede959baeb1e45feaa2c22676d30c2fbfc..bc2c8ef3ed2fca25a4742c8f4232f27854bea948 100644 --- a/drivers/irqchip/irq-sw64-intc-v2.c +++ b/drivers/irqchip/irq-sw64-intc-v2.c @@ -34,66 +34,6 @@ static const struct irq_domain_ops sw64_intc_domain_ops = { .map = sw64_intc_domain_map, }; -#ifdef CONFIG_ACPI -static int __init -intc_parse_madt(union acpi_subtable_headers *header, - const unsigned long end) -{ - struct acpi_madt_io_sapic *its_entry; - static struct irq_domain *root_domain; - int intc_irqs = 8, irq_base = NR_IRQS_LEGACY; - irq_hw_number_t hwirq_base = 0; - int irq_start = -1; - - its_entry = (struct acpi_madt_io_sapic *)header; - - intc_irqs -= hwirq_base; /* calculate # of irqs to allocate */ - - irq_base = irq_alloc_descs(irq_start, 16, intc_irqs, - numa_node_id()); - if (irq_base < 0) { - WARN(1, "Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n", - irq_start); - irq_base = irq_start; - } - - root_domain = irq_domain_add_legacy(NULL, intc_irqs, irq_base, - hwirq_base, &sw64_intc_domain_ops, NULL); - - if (!root_domain) - pr_err("Failed to create irqdomain"); - - irq_set_default_host(root_domain); - - sw64_io_write(0, MCU_DVC_INT_EN, 0xff); - - return 0; -} - -static int __init acpi_intc_init(void) -{ - int count = 0; - - count = acpi_table_parse_madt(ACPI_MADT_TYPE_IO_SAPIC, - intc_parse_madt, 0); - - if (count <= 0) { - pr_err("No valid intc entries exist\n"); - return -EINVAL; - } - return 0; -} - -static int __init intc_init(void) -{ - acpi_intc_init(); - - return 0; -} - -subsys_initcall(intc_init); -#endif - #ifdef CONFIG_OF static struct irq_domain *root_domain; diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index b4a59e131f08b8bcf8057b82c466a584aa31467b..30eca221a3a11e7228d78d99560f30551662fe5f 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -570,7 +570,7 @@ config LPC_SCH config LPC_CHIP3 tristate "CHIP3 LPC" - depends on SW64_CHIP3 + depends on UNCORE_XUELANG select MFD_CORE help LPC bridge function of the chip3 provides support for diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 5c4e184ac9dc88371338e965db13b017f7b80f42..215dcfbbbda54a9962b3fd4758de468fe564440b 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -305,6 +305,10 @@ config PCIE_HISI_ERR Say Y here if you want error handling support for the PCIe controller's errors on HiSilicon HIP SoCs +config PCI_SW64 + bool + depends on SW64 && PCI + source "drivers/pci/controller/dwc/Kconfig" source "drivers/pci/controller/mobiveil/Kconfig" source "drivers/pci/controller/cadence/Kconfig" diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 04c6edc285c511a8f470bdcf721ff4b5eeb65327..e633f0a38d9944f6b6c158cdf578b4d42320aff4 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_VMD) += vmd.o obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o obj-$(CONFIG_PCIE_HISI_ERR) += pcie-hisi-error.o +obj-$(CONFIG_PCI_SW64) += pci-sunway.o # pcie-hisi.o quirks are needed even without CONFIG_PCIE_DW obj-y += dwc/ obj-y += mobiveil/ diff --git a/arch/sw_64/chip/chip3/chip.c b/drivers/pci/controller/pci-sunway.c similarity index 33% rename from arch/sw_64/chip/chip3/chip.c rename to drivers/pci/controller/pci-sunway.c index 819b9c3728f643323f47ccad536e5768e2d3f2c5..c57c2350ffef09e5260c312134d25ccb13a3c647 100644 --- a/arch/sw_64/chip/chip3/chip.c +++ b/drivers/pci/controller/pci-sunway.c @@ -1,94 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -#include -#include #include -#include -#include -#include -#include -#include "../../../../drivers/pci/pci.h" - -static u64 read_longtime(struct clocksource *cs) -{ - u64 result; - unsigned long node; - - if (IS_ENABLED(CONFIG_SW64_FPGA) || IS_ENABLED(CONFIG_SW64_SIM)) - node = 0; - else - node = __this_cpu_read(hard_node_id); - result = sw64_io_read(node, LONG_TIME); - - return result; -} - -static int longtime_enable(struct clocksource *cs) -{ - switch (cpu_desc.model) { - case CPU_SW3231: - sw64_io_write(0, GPIO_SWPORTA_DR, 0); - sw64_io_write(0, GPIO_SWPORTA_DDR, 0xff); - break; - case CPU_SW831: - sw64_io_write(0, LONG_TIME_START_EN, 0x1); - break; - default: - break; - } - - return 0; -} - -static struct clocksource clocksource_longtime = { - .name = "longtime", - .rating = 100, - .enable = longtime_enable, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .mask = CLOCKSOURCE_MASK(64), - .shift = 0, - .mult = 0, - .read = read_longtime, -}; - -static u64 read_vtime(struct clocksource *cs) -{ - u64 result; - unsigned long vtime_addr = IO_BASE | LONG_TIME; - - result = rdio64(vtime_addr); - return result; -} - -static int vtime_enable(struct clocksource *cs) -{ - return 0; -} - -static struct clocksource clocksource_vtime = { - .name = "vtime", - .rating = 100, - .enable = vtime_enable, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .mask = CLOCKSOURCE_MASK(64), - .shift = 0, - .mult = 0, - .read = read_vtime, -}; - -void setup_chip_clocksource(void) -{ -#ifdef CONFIG_SW64_SIM - clocksource_register_khz(&clocksource_longtime, 400000); /* Hardware Simulator 400Mhz */ -#elif defined(CONFIG_SW64_FPGA) - clocksource_register_khz(&clocksource_longtime, 1000); /* FPGA 1Mhz */ -#else - if (is_in_host()) - clocksource_register_khz(&clocksource_longtime, 25000); - else - clocksource_register_khz(&clocksource_vtime, 25000); -#endif -} void set_devint_wken(int node) { @@ -100,6 +12,14 @@ void set_devint_wken(int node) sw64_io_write(node, DEVINTWK_INTEN, val); } +#ifdef CONFIG_UNCORE_JUNZHANG +void set_adr_int(int node) +{ + sw64_io_write(node, ADR_INT_CONFIG, (0x0 << 16 | 0x3f)); + sw64_io_write(node, ADR_CTL, 0xc); +} +#endif + void set_pcieport_service_irq(int node, int index) { if (IS_ENABLED(CONFIG_PCIE_PME)) @@ -109,81 +29,6 @@ void set_pcieport_service_irq(int node, int index) write_piu_ior0(node, index, AERERRINTCONFIG, AER_ENABLE_INTD_CORE0); } -static int chip3_get_cpu_nums(void) -{ - unsigned long trkmode; - int cpus; - - if (is_guest_or_emul()) - return 1; - - trkmode = sw64_io_read(0, TRKMODE); - trkmode = (trkmode >> 6) & 0x3; - cpus = 1 << trkmode; - - return cpus; -} - -static void chip3_get_vt_smp_info(void) -{ - unsigned long smp_info; - - smp_info = sw64_io_read(0, SMP_INFO); - if (smp_info == -1UL) - smp_info = 0; - topo_nr_threads = (smp_info >> VT_THREADS_SHIFT) & VT_THREADS_MASK; - topo_nr_cores = (smp_info >> VT_CORES_SHIFT) & VT_CORES_MASK; - topo_nr_maxcpus = (smp_info >> VT_MAX_CPUS_SHIFT) & VT_MAX_CPUS_MASK; -} - -static unsigned long chip3_get_vt_node_mem(int nodeid) -{ - return *(unsigned long *)MMSIZE & MMSIZE_MASK; -} - -static unsigned long chip3_get_node_mem(int nodeid) -{ - unsigned long mc_config, mc_online, mc_cap, mc_num; - unsigned long node_mem; - - mc_config = sw64_io_read(nodeid, MC_CAP_CFG) & 0xf; - mc_cap = (1UL << mc_config) << 28; - mc_online = sw64_io_read(nodeid, MC_ONLINE) & 0xff; - mc_num = __kernel_ctpop(mc_online); - node_mem = mc_cap * mc_num; - - return node_mem; -} - -static void chip3_setup_vt_core_start(struct cpumask *cpumask) -{ - int i; - unsigned long coreonline; - - coreonline = sw64_io_read(0, CORE_ONLINE); - - for (i = 0; i < 64 ; i++) { - if (coreonline & (1UL << i)) - cpumask_set_cpu(i, cpumask); - } -} - -static void chip3_setup_core_start(struct cpumask *cpumask) -{ - int i, j, cpus; - unsigned long coreonline; - - cpus = chip3_get_cpu_nums(); - for (i = 0; i < cpus; i++) { - coreonline = sw64_io_read(i, CORE_ONLINE); - for (j = 0; j < 32 ; j++) { - if (coreonline & (1UL << j)) - cpumask_set_cpu(i * 32 + j, cpumask); - } - } - -} - int chip_pcie_configure(struct pci_controller *hose) { struct pci_dev *dev; @@ -322,48 +167,26 @@ int chip_pcie_configure(struct pci_controller *hose) return bus_max_num; } -static int chip3_check_pci_vt_linkup(unsigned long node, unsigned long index) -{ - if (node == 0 && index == 0) - return 0; - else - return 1; -} - -static int chip3_check_pci_linkup(unsigned long node, unsigned long index) +static int check_pci_linkup(unsigned long node, unsigned long index) { unsigned long rc_debug; -#ifdef CONFIG_SW64_FPGA //for PCIE4.0 - printk("waiting for link up...\n"); - if (index == 0) - sw64_io_write(node, PIU_TOP0_CONFIG, 0x10011); - else - sw64_io_write(node, PIU_TOP1_CONFIG, 0x10011); - mdelay(10); - rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); - while (!(rc_debug & 0x1)) { - udelay(10); - rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); - } - mdelay(10); -#endif -#ifdef CONFIG_SW64_SIM - printk("waiting for link up...\n"); - rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); - while (!(rc_debug & 0x1)) { - udelay(10); + if (is_guest_or_emul()) { + if (node == 0 && index == 0) + return 0; + else + return 1; + } else { rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); } -#endif - rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); - return !(rc_debug & 0x1); + return !(rc_debug == 0x111); } -static void chip3_set_rc_piu(unsigned long node, unsigned long index) +static void set_rc_piu(unsigned long node, unsigned long index) { - unsigned int i, value; + unsigned int i __maybe_unused; + unsigned int value; u32 rc_misc_ctrl; if (is_guest_or_emul()) @@ -376,18 +199,12 @@ static void chip3_set_rc_piu(unsigned long node, unsigned long index) write_rc_conf(node, index, RC_EXP_DEVCTL2, 0x6); write_rc_conf(node, index, RC_ORDER_RULE_CTL, 0x0100); - if (IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_SW64_SIM)) { - value = read_rc_conf(node, index, RC_LINK_STAT); - value |= 0x3; - write_rc_conf(node, index, RC_LINK_STAT, value); - } - /* enable DBI_RO_WR_EN */ rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); /* fix up DEVICE_ID_VENDOR_ID register */ - value = (PCI_DEVICE_ID_CHIP3 << 16) | PCI_VENDOR_ID_JN; + value = (PCI_DEVICE_ID_SW64_ROOT_BRIDGE << 16) | PCI_VENDOR_ID_JN; write_rc_conf(node, index, RC_VENDOR_ID, value); /* set PCI-E root class code */ @@ -398,7 +215,8 @@ static void chip3_set_rc_piu(unsigned long node, unsigned long index) write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); write_rc_conf(node, index, RC_PRIMARY_BUS, 0xffffff); - write_piu_ior0(node, index, PIUCONFIG0, 0x38056); + write_piu_ior0(node, index, PIUCONFIG0, PIUCONFIG0_INIT_VAL); + write_piu_ior1(node, index, PIUCONFIG1, 0x2); write_piu_ior1(node, index, ERRENABLE, -1); @@ -406,40 +224,45 @@ static void chip3_set_rc_piu(unsigned long node, unsigned long index) write_piu_ior0(node, index, EPDMABAR, PCITODMA_OFFSET); if (IS_ENABLED(CONFIG_PCI_MSI)) { write_piu_ior0(node, index, MSIADDR, MSIX_MSG_ADDR); - for (i = 0; i < 256; i++) - write_piu_ior0(node, index, MSICONFIG0 + (i << 7), 0); +#ifdef CONFIG_UNCORE_XUELANG + for (i = 0; i < 256; i++) + write_piu_ior0(node, index, MSICONFIG0 + (i << 7), 0); +#endif } } -static void chip3_set_intx(unsigned long node, unsigned long index, +static void set_intx(unsigned long node, unsigned long index, unsigned long int_conf) { if (is_guest_or_emul()) return; +#if defined(CONFIG_UNCORE_XUELANG) write_piu_ior0(node, index, INTACONFIG, int_conf | (0x8UL << 10)); write_piu_ior0(node, index, INTBCONFIG, int_conf | (0x4UL << 10)); write_piu_ior0(node, index, INTCCONFIG, int_conf | (0x2UL << 10)); write_piu_ior0(node, index, INTDCONFIG, int_conf | (0x1UL << 10)); +#elif defined(CONFIG_UNCORE_JUNZHANG) + write_piu_ior0(node, index, INTACONFIG, int_conf | (0x1UL << 10)); + write_piu_ior0(node, index, INTBCONFIG, int_conf | (0x2UL << 10)); + write_piu_ior0(node, index, INTCCONFIG, int_conf | (0x4UL << 10)); + write_piu_ior0(node, index, INTDCONFIG, int_conf | (0x8UL << 10)); +#endif } -static unsigned long chip3_get_rc_enable(unsigned long node) +static unsigned long get_rc_enable(unsigned long node) { unsigned long rc_enable; if (is_guest_or_emul()) return 1; - if (!IS_ENABLED(CONFIG_SW64_ASIC)) { - rc_enable = 0x1; - sw64_io_write(node, IO_START, rc_enable); - } rc_enable = sw64_io_read(node, IO_START); return rc_enable; } -static int chip3_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +static int map_irq(const struct pci_dev *dev, u8 slot, u8 pin) { struct pci_controller *hose = dev->sysdata; @@ -449,216 +272,7 @@ static int chip3_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) return hose->int_irq; } -extern struct pci_controller *hose_head, **hose_tail; -static void sw6_handle_intx(unsigned int offset) -{ - struct pci_controller *hose; - unsigned long value; - - hose = hose_head; - for (hose = hose_head; hose; hose = hose->next) { - value = read_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7)); - if (value >> 63) { - value = value & (~(1UL << 62)); - write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); - handle_irq(hose->int_irq); - value = value | (1UL << 62); - write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); - } - - if (IS_ENABLED(CONFIG_PCIE_PME)) { - value = read_piu_ior0(hose->node, hose->index, PMEINTCONFIG); - if (value >> 63) { - handle_irq(hose->service_irq); - write_piu_ior0(hose->node, hose->index, PMEINTCONFIG, value); - } - } - - if (IS_ENABLED(CONFIG_PCIEAER)) { - value = read_piu_ior0(hose->node, hose->index, AERERRINTCONFIG); - if (value >> 63) { - handle_irq(hose->service_irq); - write_piu_ior0(hose->node, hose->index, AERERRINTCONFIG, value); - } - } - - if (hose->iommu_enable) { - value = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); - if (value >> 63) - handle_irq(hose->int_irq); - } - } -} - -static void chip3_device_interrupt(unsigned long irq_info) -{ - unsigned int i; - - if (is_guest_or_emul()) { - handle_irq(irq_info); - return; - } - - for (i = 0; i < 4; i++) { - if ((irq_info >> i) & 0x1) - sw6_handle_intx(i); - } -} - -static void chip3_i2c_srst(void) -{ - sw64_io_write(0, I2C0_SRST_L, 0x0); - sw64_io_write(0, I2C0_SRST_L, 0x1); - - sw64_io_write(0, I2C1_SRST_L, 0x0); - sw64_io_write(0, I2C1_SRST_L, 0x1); - - sw64_io_write(0, I2C2_SRST_L, 0x0); - sw64_io_write(0, I2C2_SRST_L, 0x1); -} - -static void chip3_pcie_save(void) -{ - struct pci_controller *hose; - struct piu_saved *piu_save; - unsigned long node, index; - unsigned long i; - - for (hose = hose_head; hose; hose = hose->next) { - piu_save = kzalloc(sizeof(*piu_save), GFP_KERNEL); - - node = hose->node; - index = hose->index; - hose->sysdata = piu_save; - - piu_save->piuconfig0 = read_piu_ior0(node, index, PIUCONFIG0); - piu_save->piuconfig1 = read_piu_ior1(node, index, PIUCONFIG1); - piu_save->epdmabar = read_piu_ior0(node, index, EPDMABAR); - piu_save->msiaddr = read_piu_ior0(node, index, MSIADDR); - - for (i = 0; i < 256; i++) { - piu_save->msiconfig[i] = read_piu_ior0(node, index, - MSICONFIG0 + (i << 7)); - } - - piu_save->iommuexcpt_ctrl = read_piu_ior0(node, index, IOMMUEXCPT_CTRL); - piu_save->dtbaseaddr = read_piu_ior0(node, index, DTBASEADDR); - - piu_save->intaconfig = read_piu_ior0(node, index, INTACONFIG); - piu_save->intbconfig = read_piu_ior0(node, index, INTBCONFIG); - piu_save->intcconfig = read_piu_ior0(node, index, INTCCONFIG); - piu_save->intdconfig = read_piu_ior0(node, index, INTDCONFIG); - piu_save->pmeintconfig = read_piu_ior0(node, index, PMEINTCONFIG); - piu_save->aererrintconfig = read_piu_ior0(node, index, AERERRINTCONFIG); - piu_save->hpintconfig = read_piu_ior0(node, index, HPINTCONFIG); - - } -} - -static void chip3_pcie_restore(void) -{ - struct pci_controller *hose; - struct piu_saved *piu_save; - unsigned long node, index; - u32 rc_misc_ctrl; - unsigned int value; - unsigned long i; - - for (hose = hose_head; hose; hose = hose->next) { - node = hose->node; - index = hose->index; - piu_save = hose->sysdata; - - write_piu_ior0(node, index, PIUCONFIG0, piu_save->piuconfig0); - write_piu_ior1(node, index, PIUCONFIG1, piu_save->piuconfig1); - write_piu_ior0(node, index, EPDMABAR, piu_save->epdmabar); - write_piu_ior0(node, index, MSIADDR, piu_save->msiaddr); - - for (i = 0; i < 256; i++) { - write_piu_ior0(node, index, MSICONFIG0 + (i << 7), - piu_save->msiconfig[i]); - } - - write_piu_ior0(node, index, IOMMUEXCPT_CTRL, piu_save->iommuexcpt_ctrl); - write_piu_ior0(node, index, DTBASEADDR, piu_save->dtbaseaddr); - - write_piu_ior0(node, index, INTACONFIG, piu_save->intaconfig); - write_piu_ior0(node, index, INTBCONFIG, piu_save->intbconfig); - write_piu_ior0(node, index, INTCCONFIG, piu_save->intcconfig); - write_piu_ior0(node, index, INTDCONFIG, piu_save->intdconfig); - write_piu_ior0(node, index, PMEINTCONFIG, piu_save->pmeintconfig); - write_piu_ior0(node, index, AERERRINTCONFIG, piu_save->aererrintconfig); - write_piu_ior0(node, index, HPINTCONFIG, piu_save->hpintconfig); - - /* Enable DBI_RO_WR_EN */ - rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); - write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); - - /* Fix up DEVICE_ID_VENDOR_ID register */ - value = (PCI_DEVICE_ID_CHIP3 << 16) | PCI_VENDOR_ID_JN; - write_rc_conf(node, index, RC_VENDOR_ID, value); - - /* Set PCI-E root class code */ - value = read_rc_conf(node, index, RC_REVISION_ID); - write_rc_conf(node, index, RC_REVISION_ID, (PCI_CLASS_BRIDGE_HOST << 16) | value); - - /* Disable DBI_RO_WR_EN */ - write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); - } - -} - -static unsigned long saved_dvc_int, saved_long_time; - -static inline void chip3_intpu_save(void) -{ - saved_long_time = sw64_io_read(0, LONG_TIME); -} - -static inline void chip3_intpu_restore(void) -{ - switch (cpu_desc.model) { - case CPU_SW831: - sw64_io_write(0, LONG_TIME, saved_long_time); - sw64_io_write(0, LONG_TIME_START_EN, 0x1); - break; - default: - pr_info("long time start is disable!"); - break; - } -} - -static inline void chip3_spbu_save(void) -{ - saved_dvc_int = sw64_io_read(0, MCU_DVC_INT_EN); -} - -static inline void chip3_spbu_restore(void) -{ - chip3_i2c_srst(); - sw64_io_write(0, MCU_DVC_INT_EN, saved_dvc_int); -} - -extern void cpld_write(uint8_t slave_addr, uint8_t reg, uint8_t data); - -static void chip3_suspend(bool wakeup) -{ - - if (wakeup) { - chip3_pcie_restore(); - chip3_intpu_restore(); - chip3_spbu_restore(); - } else { - /* Set S3 flag */ - cpld_write(0x64, 0x34, 0x33); - - chip3_spbu_save(); - chip3_intpu_save(); - chip3_pcie_save(); - } -} - -static void chip3_hose_init(struct pci_controller *hose) +static void hose_init(struct pci_controller *hose) { unsigned long pci_io_base; @@ -706,240 +320,16 @@ static void chip3_hose_init(struct pci_controller *hose) } }; -static void chip3_init_ops_fixup(void) -{ - if (is_guest_or_emul()) { - sw64_chip_init->early_init.setup_core_start = chip3_setup_vt_core_start; - sw64_chip_init->early_init.get_node_mem = chip3_get_vt_node_mem; - sw64_chip_init->early_init.get_smp_info = chip3_get_vt_smp_info; - sw64_chip_init->pci_init.check_pci_linkup = chip3_check_pci_vt_linkup; - } -}; - -static void chip3_ops_fixup(void) -{ - if (is_guest_or_emul()) - sw64_chip->suspend = NULL; -}; - -static struct sw64_chip_init_ops chip3_chip_init_ops = { - .early_init = { - .setup_core_start = chip3_setup_core_start, - .get_node_mem = chip3_get_node_mem, - }, - .pci_init = { - .map_irq = chip3_map_irq, - .get_rc_enable = chip3_get_rc_enable, - .hose_init = chip3_hose_init, - .set_rc_piu = chip3_set_rc_piu, - .check_pci_linkup = chip3_check_pci_linkup, - .set_intx = chip3_set_intx, - }, - .fixup = chip3_init_ops_fixup, +static struct sw64_pci_init_ops chip_pci_init_ops = { + .map_irq = map_irq, + .get_rc_enable = get_rc_enable, + .hose_init = hose_init, + .set_rc_piu = set_rc_piu, + .check_pci_linkup = check_pci_linkup, + .set_intx = set_intx, }; -static struct sw64_chip_ops chip3_chip_ops = { - .get_cpu_num = chip3_get_cpu_nums, - .suspend = chip3_suspend, - .fixup = chip3_ops_fixup, -}; - -void __init sw64_setup_chip_ops(void) +void __init setup_chip_pci_ops(void) { - sw64_chip_init = &chip3_chip_init_ops; - sw64_chip = &chip3_chip_ops; + sw64_chip_init->pci_init = chip_pci_init_ops; } - -/* Performance counter hook. A module can override this to do something useful. */ -static void dummy_perf(unsigned long vector, struct pt_regs *regs) -{ - irq_err_count++; - pr_crit("Performance counter interrupt!\n"); -} - -void (*perf_irq)(unsigned long, struct pt_regs*) = dummy_perf; -EXPORT_SYMBOL(perf_irq); - -#ifdef CONFIG_PCI_MSI -extern void handle_pci_msi_interrupt(unsigned long type, - unsigned long vector, - unsigned long pci_msi1_addr); -#else -void handle_pci_msi_interrupt(unsigned long type, - unsigned long vector, unsigned long pci_msi1_addr) -{ - pr_warn("SW arch disable CONFIG_PCI_MSI option.\n"); -} -#endif - -static void handle_fault_int(void) -{ - int node; - - node = __this_cpu_read(hard_node_id); - printk("enter fault int, si_fault_stat = %#lx\n", - sw64_io_read(node, SI_FAULT_STAT)); - sw64_io_write(node, SI_FAULT_INT_EN, 0); - sw64_io_write(node, DLI_RLTD_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG0_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG1_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG2_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG3_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG4_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG5_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG6_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG7_FAULT_INTEN, 0); -} - -static void handle_mt_int(void) -{ - printk("enter mt int\n"); -} - -static void handle_nmi_int(void) -{ - printk("enter nmi int\n"); -} - -static void handle_dev_int(struct pt_regs *regs) -{ - unsigned long config_val, val, stat; - int node = 0; - unsigned int hwirq; - - config_val = sw64_io_read(node, DEV_INT_CONFIG); - val = config_val & (~(1UL << 8)); - sw64_io_write(node, DEV_INT_CONFIG, val); - stat = sw64_io_read(node, MCU_DVC_INT); - - while (stat) { - hwirq = ffs(stat) - 1; - handle_domain_irq(NULL, hwirq, regs); - stat &= ~(1UL << hwirq); - } - /*do handle irq */ - - sw64_io_write(node, DEV_INT_CONFIG, config_val); -} - -asmlinkage void do_entInt(unsigned long type, unsigned long vector, - unsigned long irq_arg, struct pt_regs *regs) -{ - struct pt_regs *old_regs; - extern char __idle_start[], __idle_end[]; - - if (is_guest_or_emul()) { - if ((type & 0xffff) > 15) { - vector = type; - if (vector == 16) - type = INT_INTx; - else - type = INT_MSI; - } - } - - /* restart idle routine if it is interrupted */ - if (regs->pc > (u64)__idle_start && regs->pc < (u64)__idle_end) - regs->pc = (u64)__idle_start; - - switch (type & 0xffff) { - case INT_MSI: - old_regs = set_irq_regs(regs); - handle_pci_msi_interrupt(type, vector, irq_arg); - set_irq_regs(old_regs); - return; - case INT_INTx: - old_regs = set_irq_regs(regs); - chip3_device_interrupt(vector); - set_irq_regs(old_regs); - return; - - case INT_IPI: -#ifdef CONFIG_SMP - handle_ipi(regs); - return; -#else - irq_err_count++; - pr_crit("Interprocessor interrupt? You must be kidding!\n"); -#endif - break; - case INT_RTC: - old_regs = set_irq_regs(regs); - sw64_timer_interrupt(); - set_irq_regs(old_regs); - return; - case INT_VT_SERIAL: - old_regs = set_irq_regs(regs); - handle_irq(type); - set_irq_regs(old_regs); - return; - case INT_VT_HOTPLUG: - old_regs = set_irq_regs(regs); - handle_irq(type); - set_irq_regs(old_regs); - return; - case INT_PC0: - perf_irq(PERFMON_PC0, regs); - return; - case INT_PC1: - perf_irq(PERFMON_PC1, regs); - return; - case INT_DEV: - old_regs = set_irq_regs(regs); - handle_dev_int(regs); - set_irq_regs(old_regs); - return; - case INT_FAULT: - old_regs = set_irq_regs(regs); - handle_fault_int(); - set_irq_regs(old_regs); - return; - case INT_MT: - old_regs = set_irq_regs(regs); - handle_mt_int(); - set_irq_regs(old_regs); - return; - case INT_NMI: - old_regs = set_irq_regs(regs); - handle_nmi_int(); - set_irq_regs(old_regs); - return; - default: - pr_crit("Hardware intr %ld %lx? uh?\n", type, vector); - } - pr_crit("PC = %016lx PS = %04lx\n", regs->pc, regs->ps); -} -EXPORT_SYMBOL(do_entInt); - -/* - * Early fix up the chip3 Root Complex settings - */ -static void chip3_pci_fixup_root_complex(struct pci_dev *dev) -{ - int i; - struct pci_bus *bus = dev->bus; - struct pci_controller *hose = bus->sysdata; - - hose->self_busno = hose->busn_space->start; - - if (likely(bus->number == hose->self_busno)) { - if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) { - /* Check Root Complex port again */ - dev->is_hotplug_bridge = 0; - dev->current_state = PCI_D0; - } - - dev->class &= 0xff; - dev->class |= PCI_CLASS_BRIDGE_PCI << 8; - for (i = 0; i < PCI_NUM_RESOURCES; i++) { - dev->resource[i].start = 0; - dev->resource[i].end = 0; - dev->resource[i].flags = IORESOURCE_PCI_FIXED; - } - } - atomic_inc(&dev->enable_cnt); - - dev->no_msi = 1; -} - -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_CHIP3, chip3_pci_fixup_root_complex); diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 5a926c883a8918c4135cbb4cb1a7f9eadf72fea2..63c02b346e938227563d14011e24557adf3559e9 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -891,8 +891,6 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) bridge->bus = bus; - /* Temporarily move resources off the list */ - list_splice_init(&bridge->windows, &resources); bus->sysdata = bridge->sysdata; bus->msi = bridge->msi; bus->ops = bridge->ops; @@ -917,6 +915,8 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) if (err) goto free; + /* Temporarily move resources off the list */ + list_splice_init(&bridge->windows, &resources); err = device_add(&bridge->dev); if (err) { put_device(&bridge->dev); diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index ff3443e6d2e40137d35413db234e31e0683b1041..47c9f9aab43be9888b524dd8d925702ce5a88a80 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -983,7 +983,7 @@ config SPI_AMD # config SPI_CHIP3 tristate "Memory-mapped io interface driver for SUNWAY CHIP3 SPI core" - depends on SW64 + depends on UNCORE_XUELANG help general driver for SPI controller core from DesignWare diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c index ef08d68b9714924513a3e5b47d467f4793e18f73..8d341adf5353452d4ac72c78e06545b35805b8a3 100644 --- a/drivers/usb/host/pci-quirks.c +++ b/drivers/usb/host/pci-quirks.c @@ -1285,3 +1285,130 @@ static void quirk_usb_early_handoff(struct pci_dev *pdev) } DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_SERIAL_USB, 8, quirk_usb_early_handoff); + +#ifdef CONFIG_SW64 +#include +#define XHCI_STS_FATAL (1 << 2) +#define XHCI_STS_EINT (1 << 3) +#define XHCI_STS_PORT (1 << 4) +#define XHCI_STS_SRE (1 << 10) +#define STS_RW1C_BITS (XHCI_STS_FATAL | XHCI_STS_EINT | XHCI_STS_PORT | XHCI_STS_SRE) + +static void +fixup_usb_xhci_reset(struct pci_dev *dev) +{ + void __iomem *op_reg_base; + int timeout; + u32 xhci_command; + u32 tmp, val; + void __iomem *base; + struct pci_controller *hose = dev->sysdata; + unsigned long offset; + int ext_cap_offset; + int retries = 3; + + pci_read_config_dword(dev, PCI_COMMAND, &tmp); + tmp |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + pci_write_config_dword(dev, PCI_COMMAND, tmp); + + pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, &tmp); + if (tmp & PCI_BASE_ADDRESS_MEM_TYPE_MASK) { + pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, &val); + offset = (unsigned long)(val) << 32 | (tmp & (~0xf)); + } else + offset = (unsigned long)(tmp & (~0xf)); + + if (offset == 0) + return; + + base = (void *)__va(SW64_PCI_IO_BASE(hose->node, hose->index) | offset); + + ext_cap_offset = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_LEGACY); + if (!ext_cap_offset) + goto hc_init; + + val = readl(base + ext_cap_offset); + + if ((dev->vendor == PCI_VENDOR_ID_TI && dev->device == 0x8241) || + (dev->vendor == PCI_VENDOR_ID_RENESAS + && dev->device == 0x0014)) { + val = (val | XHCI_HC_OS_OWNED) & ~XHCI_HC_BIOS_OWNED; + writel(val, base + ext_cap_offset); + } + + if (val & XHCI_HC_BIOS_OWNED) { + writel(val | XHCI_HC_OS_OWNED, base + ext_cap_offset); + + timeout = handshake(base + ext_cap_offset, XHCI_HC_BIOS_OWNED, + 0, 1000000, 10); + if (timeout) { + pr_err("xHCI BIOS handoff failed (BIOS bug ?) %08x\n", val); + writel(val & ~XHCI_HC_BIOS_OWNED, base + ext_cap_offset); + } + } + + val = readl(base + ext_cap_offset + XHCI_LEGACY_CONTROL_OFFSET); + val &= XHCI_LEGACY_DISABLE_SMI; + val |= XHCI_LEGACY_SMI_EVENTS; + writel(val, base + ext_cap_offset + XHCI_LEGACY_CONTROL_OFFSET); + +hc_init: + if (dev->vendor == PCI_VENDOR_ID_INTEL) + usb_enable_intel_xhci_ports(dev); + + op_reg_base = base + XHCI_HC_LENGTH(readl(base)); + + timeout = handshake(op_reg_base + XHCI_STS_OFFSET, XHCI_STS_CNR, 0, + 5000000, 10); + if (timeout) { + val = readl(op_reg_base + XHCI_STS_OFFSET); + pr_err("xHCI HW not ready after 5 sec (HC bug?) status = 0x%x\n", val); + } + + xhci_command = readl(op_reg_base + XHCI_CMD_OFFSET); + xhci_command |= 0x2; + writel(xhci_command, op_reg_base + XHCI_CMD_OFFSET); + + timeout = handshake(op_reg_base + XHCI_CMD_OFFSET, + 0x2, 0, 10 * 1000 * 1000, 125); + if (timeout) + pr_err("xHCI BIOS handoff time out\n"); + +retry: + val = readl(op_reg_base + XHCI_STS_OFFSET); + val |= STS_RW1C_BITS; + writel(val, op_reg_base + XHCI_STS_OFFSET); + val = readl(op_reg_base + XHCI_STS_OFFSET); + + if ((val & STS_RW1C_BITS) && retries--) { + pr_err("clear USB Status Register (status = %#x) failed, retry\n", val); + goto retry; + } + + val = readl(op_reg_base + XHCI_CMD_OFFSET); + val &= ~(XHCI_CMD_RUN | XHCI_IRQS); + writel(val, op_reg_base + XHCI_CMD_OFFSET); + timeout = handshake(op_reg_base + XHCI_STS_OFFSET, XHCI_STS_HALT, 1, + XHCI_MAX_HALT_USEC, 125); + if (timeout) { + val = readl(op_reg_base + XHCI_STS_OFFSET); + pr_err("xHCI HW did not halt within %d usec status = 0x%x\n", + XHCI_MAX_HALT_USEC, val); + } + + xhci_command = readl(op_reg_base + XHCI_CMD_OFFSET); + xhci_command |= 0x2; + writel(xhci_command, op_reg_base + XHCI_CMD_OFFSET); + + timeout = handshake(op_reg_base + XHCI_CMD_OFFSET, + 0x2, 0, 10 * 1000 * 1000, 125); + if (timeout) + pr_err("xHCI BIOS handoff time out\n"); + + pci_read_config_dword(dev, PCI_COMMAND, &tmp); + tmp &= ~(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + pci_write_config_dword(dev, PCI_COMMAND, tmp); +} +DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_ANY_ID, PCI_ANY_ID, + PCI_CLASS_SERIAL_USB_XHCI, 0, fixup_usb_xhci_reset); +#endif diff --git a/include/linux/fb.h b/include/linux/fb.h index ecfbcc0553a5904d28cd6f3fb4a1952c43069439..916a4b4e5e8460ed4f45ee42f283f775c75c98da 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -545,7 +545,7 @@ static inline struct apertures_struct *alloc_apertures(unsigned int max_num) { #elif defined(__i386__) || defined(__alpha__) || defined(__x86_64__) || \ defined(__hppa__) || defined(__sh__) || defined(__powerpc__) || \ - defined(__arm__) || defined(__aarch64__) + defined(__arm__) || defined(__aarch64__) || defined(__sw_64__) #define fb_readb __raw_readb #define fb_readw __raw_readw diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 401f6cf22a6653505ee277642c1c181b7c6db3c7..6a8a35b8c7ea691cd000d53c86940616e89a0423 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -240,10 +240,18 @@ else endif $(OUTPUT)test-libbfd-buildid.bin: +ifeq ($(ARCH),sw_64) + $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl -liberty -lz +else $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl +endif $(OUTPUT)test-disassembler-four-args.bin: +ifeq ($(ARCH),sw_64) + $(BUILD) -DPACKAGE='"perf"' -lbfd -lopcodes -liberty -lz +else $(BUILD) -DPACKAGE='"perf"' -lbfd -lopcodes +endif $(OUTPUT)test-disassembler-init-styled.bin: $(BUILD) -DPACKAGE='"perf"' -lbfd -lopcodes diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc index 84285a6f60b079346a5c527ea99a5d400c678c0a..60506d6626a9dc209b11c2f2c16b654d6a0f472d 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc @@ -22,6 +22,9 @@ ppc64*) ppc*) ARG1=%r3 ;; +sw_64) + ARG1=%r16 +;; *) echo "Please implement other architecture here" exit_untested diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc index 474ca1a9a088528e2f15b5a642fcd775a518e5d2..8879f9517b428eeaf9d271fa988c61ff8038719b 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc @@ -32,6 +32,10 @@ ppc*) GOODREG=%r3 BADREG=%msr ;; +sw_64) + GOODREG=%r16 + BADREG=%ps +;; *) echo "Please implement other architecture here" exit_untested diff --git a/tools/testing/selftests/sigaltstack/current_stack_pointer.h b/tools/testing/selftests/sigaltstack/current_stack_pointer.h index ea9bdf3a90b16449a0ffcdfcdc54247125f1c806..47c0dc383430aee8d3ccb63419d15fbd233b3538 100644 --- a/tools/testing/selftests/sigaltstack/current_stack_pointer.h +++ b/tools/testing/selftests/sigaltstack/current_stack_pointer.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#if __alpha__ +#if __alpha__ || __sw_64__ register unsigned long sp asm("$30"); #elif __arm__ || __aarch64__ || __csky__ || __m68k__ || __mips__ || __riscv register unsigned long sp asm("sp"); diff --git a/tools/testing/selftests/vm/virtual_address_range.c b/tools/testing/selftests/vm/virtual_address_range.c index c0592646ed936ef094ab2758cb330864e701616c..d9eeaaee58ebd71cdbdb33773b124d8e77007687 100644 --- a/tools/testing/selftests/vm/virtual_address_range.c +++ b/tools/testing/selftests/vm/virtual_address_range.c @@ -48,6 +48,11 @@ #define HIGH_ADDR_SHIFT 49 #define NR_CHUNKS_LOW NR_CHUNKS_256TB #define NR_CHUNKS_HIGH 0 +#elif defined __sw_64__ +#define HIGH_ADDR_MARK ADDR_MARK_128TB * 32UL +#define HIGH_ADDR_SHIFT 53 +#define NR_CHUNKS_LOW NR_CHUNKS_128TB * 32UL +#define NR_CHUNKS_HIGH 0 #else #define HIGH_ADDR_MARK ADDR_MARK_128TB #define HIGH_ADDR_SHIFT 48