diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h
index 819044fefbe7641d3f1246503cff4cde5f61c1d9..9c0664fe1eb165e7f007e254d78af13571f98b72 100644
--- a/arch/arm64/include/asm/asm-extable.h
+++ b/arch/arm64/include/asm/asm-extable.h
@@ -5,12 +5,13 @@
 #include <linux/bits.h>
 #include <asm/gpr-num.h>
 
-#define EX_TYPE_NONE			0
-#define EX_TYPE_BPF			1
-#define EX_TYPE_UACCESS_ERR_ZERO	2
-#define EX_TYPE_KACCESS_ERR_ZERO	3
-#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD	4
-#define EX_TYPE_COPY_MC_PAGE_ERR_ZERO	5
+#define EX_TYPE_NONE				0
+#define EX_TYPE_BPF				1
+#define EX_TYPE_UACCESS_ERR_ZERO		2
+#define EX_TYPE_KACCESS_ERR_ZERO		3
+#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD		4
+/* kernel access memory error safe */
+#define EX_TYPE_KACCESS_ERR_ZERO_ME_SAFE	5
 
 /* Data fields for EX_TYPE_UACCESS_ERR_ZERO */
 #define EX_DATA_REG_ERR_SHIFT	0
@@ -52,16 +53,17 @@
 #define _ASM_EXTABLE_UACCESS(insn, fixup)				\
 	_ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, wzr, wzr)
 
-#define _ASM_EXTABLE_COPY_MC_PAGE_ERR_ZERO(insn, fixup, err, zero)	\
+#define _ASM_EXTABLE_KACCESS_ERR_ZERO_ME_SAFE(insn, fixup, err, zero)	\
 	__ASM_EXTABLE_RAW(insn, fixup, 					\
-			  EX_TYPE_COPY_MC_PAGE_ERR_ZERO,		\
+			  EX_TYPE_KACCESS_ERR_ZERO_ME_SAFE,		\
 			  (						\
 			    EX_DATA_REG(ERR, err) |			\
 			    EX_DATA_REG(ZERO, zero)			\
 			  ))
 
-#define _ASM_EXTABLE_COPY_MC_PAGE(insn, fixup)				\
-	_ASM_EXTABLE_COPY_MC_PAGE_ERR_ZERO(insn, fixup, wzr, wzr)
+#define _ASM_EXTABLE_KACCESS_ME_SAFE(insn, fixup)			\
+	_ASM_EXTABLE_KACCESS_ERR_ZERO_ME_SAFE(insn, fixup, wzr, wzr)
+
 /*
  * Create an exception table entry for uaccess `insn`, which will branch to `fixup`
  * when an unhandled fault is taken.
@@ -70,10 +72,6 @@
 	_ASM_EXTABLE_UACCESS(\insn, \fixup)
 	.endm
 
-	.macro          _asm_extable_copy_mc_page, insn, fixup
-	_ASM_EXTABLE_COPY_MC_PAGE(\insn, \fixup)
-	.endm
-
 /*
  * Create an exception table entry for `insn` if `fixup` is provided. Otherwise
  * do nothing.
@@ -84,6 +82,14 @@
 	.endif
 	.endm
 
+/*
+ * Create an exception table entry for kaccess me(memory error) safe `insn`, which
+ * will branch to `fixup` when an unhandled fault is taken.
+ */
+	.macro          _asm_extable_kaccess_me_safe, insn, fixup
+	_ASM_EXTABLE_KACCESS_ME_SAFE(\insn, \fixup)
+	.endm
+
 #else /* __ASSEMBLY__ */
 
 #include <linux/stringify.h>
diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h
index 5b6efe8abeeb4462ccfb033bdf71663f3aedfb5d..7bbebfa5b71039f4b6716656102418b80bc7cb33 100644
--- a/arch/arm64/include/asm/asm-uaccess.h
+++ b/arch/arm64/include/asm/asm-uaccess.h
@@ -57,6 +57,10 @@ alternative_else_nop_endif
 	.endm
 #endif
 
+#define KERNEL_ME_SAFE(l, x...)			\
+9999:	x;					\
+	_asm_extable_kaccess_me_safe	9999b, l
+
 #define USER(l, x...)				\
 9999:	x;					\
 	_asm_extable_uaccess	9999b, l
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 7712c532ce1e7aae56e163171f5848a908be4805..c738877b8c72af307fedea04a0b64f280bcb72ba 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -172,10 +172,6 @@ lr	.req	x30		// link register
 #define CPU_LE(code...) code
 #endif
 
-#define CPY_MC(l, x...)		\
-9999:   x;			\
-	_asm_extable_copy_mc_page    9999b, l
-
 /*
  * Define a macro that constructs a 64-bit value by concatenating two
  * 32-bit registers. Note that on big endian systems the order of the
diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h
index f80ebd0addfdea9af592e2c42712dd4c05dacd9a..bc49443bc50208f44b9ce00fb87178d09f0848b6 100644
--- a/arch/arm64/include/asm/extable.h
+++ b/arch/arm64/include/asm/extable.h
@@ -46,5 +46,5 @@ bool ex_handler_bpf(const struct exception_table_entry *ex,
 #endif /* !CONFIG_BPF_JIT */
 
 bool fixup_exception(struct pt_regs *regs);
-bool fixup_exception_mc(struct pt_regs *regs);
+bool fixup_exception_me(struct pt_regs *regs);
 #endif
diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h
index 9cdded082dd40a64bf124bedc40eb2a4697a3318..dc68337c26235ff3db003d46cf8d9073c1849ceb 100644
--- a/arch/arm64/include/asm/mte.h
+++ b/arch/arm64/include/asm/mte.h
@@ -92,7 +92,11 @@ static inline bool try_page_mte_tagging(struct page *page)
 void mte_zero_clear_page_tags(void *addr);
 void mte_sync_tags(pte_t pte, unsigned int nr_pages);
 void mte_copy_page_tags(void *kto, const void *kfrom);
+
+#ifdef CONFIG_ARCH_HAS_COPY_MC
 int mte_copy_mc_page_tags(void *kto, const void *kfrom);
+#endif
+
 void mte_thread_init_user(void);
 void mte_thread_switch(struct task_struct *next);
 void mte_cpu_setup(void);
diff --git a/arch/arm64/include/asm/string.h b/arch/arm64/include/asm/string.h
index 995b63c26e991ab29712b8bdbc0254ed7fed3e34..23eca4fb24fa6aa72c49b155f5d819fb58da085c 100644
--- a/arch/arm64/include/asm/string.h
+++ b/arch/arm64/include/asm/string.h
@@ -36,8 +36,8 @@ extern void *memcpy(void *, const void *, __kernel_size_t);
 extern void *__memcpy(void *, const void *, __kernel_size_t);
 
 #define __HAVE_ARCH_MEMCPY_MC
-extern int memcpy_mcs(void *, const void *, __kernel_size_t);
-extern int __memcpy_mcs(void *, const void *, __kernel_size_t);
+extern int memcpy_mc(void *, const void *, __kernel_size_t);
+extern int __memcpy_mc(void *, const void *, __kernel_size_t);
 
 #define __HAVE_ARCH_MEMMOVE
 extern void *memmove(void *, const void *, __kernel_size_t);
@@ -61,7 +61,7 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt);
  */
 
 #define memcpy(dst, src, len) __memcpy(dst, src, len)
-#define memcpy_mcs(dst, src, len) __memcpy_mcs(dst, src, len)
+#define memcpy_mc(dst, src, len) __memcpy_mc(dst, src, len)
 #define memmove(dst, src, len) __memmove(dst, src, len)
 #define memset(s, c, n) __memset(s, c, n)
 
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 61e28ef2112ad7b0220ee15788b9bab2243e9558..07c1aeaeb094fe0004cc4d25ce9978a609a5a38e 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -429,19 +429,16 @@ static inline size_t probe_subpage_writeable(const char __user *uaddr,
 /**
  * copy_mc_to_kernel - memory copy that handles source exceptions
  *
- * @dst:	destination address
- * @src:	source address
- * @len:	number of bytes to copy
+ * @to:		destination address
+ * @from:	source address
+ * @size:	number of bytes to copy
  *
- * Return 0 for success, or #size if there was an exception.
+ * Return 0 for success, or bytes not copied.
  */
 static inline unsigned long __must_check
 copy_mc_to_kernel(void *to, const void *from, unsigned long size)
 {
-	int ret;
-
-	ret = memcpy_mcs(to, from, size);
-	return (ret == -EFAULT) ? size : 0;
+	return memcpy_mc(to, from, size);
 }
 #define copy_mc_to_kernel copy_mc_to_kernel
 #endif
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 316c57b19c9344fa2eaa7878ca4d52adfb3b18f9..c0d5985012d75cdf07c7992c3d50d35a0806bb89 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1582,16 +1582,6 @@ static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry,
 	return has_sre;
 }
 
-static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused)
-{
-	u32 midr = read_cpuid_id();
-
-	/* Cavium ThunderX pass 1.x and 2.x */
-	return midr_is_cpu_model_range(midr, MIDR_THUNDERX,
-		MIDR_CPU_VAR_REV(0, 0),
-		MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK));
-}
-
 static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused)
 {
 	u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
@@ -2401,12 +2391,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, ATOMIC, IMP)
 	},
 #endif /* CONFIG_ARM64_LSE_ATOMICS */
-	{
-		.desc = "Software prefetching using PRFM",
-		.capability = ARM64_HAS_NO_HW_PREFETCH,
-		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
-		.matches = has_no_hw_prefetch,
-	},
 	{
 		.desc = "Virtualization Host Extensions",
 		.capability = ARM64_HAS_VIRT_HOST_EXTN,
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 899d6ae9698caac845c9657433f8025b0704c58a..3b6914bb8ec788f199cd2cd659591a34bf077efb 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -3,7 +3,7 @@ lib-y		:= clear_user.o delay.o copy_from_user.o		\
 		   copy_to_user.o copy_page.o				\
 		   clear_page.o csum.o insn.o memchr.o memcpy.o		\
 		   memset.o memcmp.o strcmp.o strncmp.o strlen.o	\
-		   strnlen.o strchr.o strrchr.o tishift.o memcpy_mc.o
+		   strnlen.o strchr.o strrchr.o tishift.o
 
 ifeq ($(CONFIG_KERNEL_MODE_NEON), y)
 obj-$(CONFIG_XOR_BLOCKS)	+= xor-neon.o
@@ -15,7 +15,7 @@ endif
 
 lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o
 
-lib-$(CONFIG_ARCH_HAS_COPY_MC) += copy_mc_page.o
+lib-$(CONFIG_ARCH_HAS_COPY_MC) += copy_mc_page.o memcpy_mc.o
 
 obj-$(CONFIG_CRC32) += crc32.o
 
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 1bf676e9201de6b5399f2e82e7515f65602c8696..34e3179075244932422dc708508cd2d097b498fe 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -25,7 +25,7 @@
 	.endm
 
 	.macro strb1 reg, ptr, val
-	USER(9998f, strb \reg, [\ptr], \val)
+	strb \reg, [\ptr], \val
 	.endm
 
 	.macro ldrh1 reg, ptr, val
@@ -33,7 +33,7 @@
 	.endm
 
 	.macro strh1 reg, ptr, val
-	USER(9998f, strh \reg, [\ptr], \val)
+	strh \reg, [\ptr], \val
 	.endm
 
 	.macro ldr1 reg, ptr, val
@@ -41,7 +41,7 @@
 	.endm
 
 	.macro str1 reg, ptr, val
-	USER(9998f, str \reg, [\ptr], \val)
+	str \reg, [\ptr], \val
 	.endm
 
 	.macro ldp1 reg1, reg2, ptr, val
@@ -49,7 +49,7 @@
 	.endm
 
 	.macro stp1 reg1, reg2, ptr, val
-	USER(9998f, stp \reg1, \reg2, [\ptr], \val)
+	stp \reg1, \reg2, [\ptr], \val
 	.endm
 
 end	.req	x5
@@ -66,7 +66,7 @@ SYM_FUNC_START(__arch_copy_from_user)
 	b.ne	9998f
 	// Before being absolutely sure we couldn't copy anything, try harder
 USER(9998f, ldtrb tmp1w, [srcin])
-USER(9998f, strb	tmp1w, [dst], #1)
+	strb	tmp1w, [dst], #1
 9998:	sub	x0, end, dst			// bytes not copied
 	ret
 SYM_FUNC_END(__arch_copy_from_user)
diff --git a/arch/arm64/lib/copy_mc_page.S b/arch/arm64/lib/copy_mc_page.S
index 656d831ef4b87c0813515a53680b119e3ea20603..a5b53a2bcac8f37e98c5ec9288eaa013a01b44cf 100644
--- a/arch/arm64/lib/copy_mc_page.S
+++ b/arch/arm64/lib/copy_mc_page.S
@@ -10,9 +10,10 @@
 #include <asm/cpufeature.h>
 #include <asm/alternative.h>
 #include <asm/asm-extable.h>
+#include <asm/asm-uaccess.h>
 
 /*
- * Copy a page from src to dest (both are page aligned) with machine check
+ * Copy a page from src to dest (both are page aligned) with memory error safe
  *
  * Parameters:
  *	x0 - dest
@@ -21,62 +22,12 @@
  * 	x0 - Return 0 if copy success, or -EFAULT if anything goes wrong
  *	     while copying.
  */
-SYM_FUNC_START(__pi_copy_mc_page)
-alternative_if ARM64_HAS_NO_HW_PREFETCH
-	// Prefetch three cache lines ahead.
-	prfm	pldl1strm, [x1, #128]
-	prfm	pldl1strm, [x1, #256]
-	prfm	pldl1strm, [x1, #384]
-alternative_else_nop_endif
-
-CPY_MC(9998f, ldp	x2, x3, [x1])
-CPY_MC(9998f, ldp	x4, x5, [x1, #16])
-CPY_MC(9998f, ldp	x6, x7, [x1, #32])
-CPY_MC(9998f, ldp	x8, x9, [x1, #48])
-CPY_MC(9998f, ldp	x10, x11, [x1, #64])
-CPY_MC(9998f, ldp	x12, x13, [x1, #80])
-CPY_MC(9998f, ldp	x14, x15, [x1, #96])
-CPY_MC(9998f, ldp	x16, x17, [x1, #112])
-
-	add	x0, x0, #256
-	add	x1, x1, #128
-1:
-	tst	x0, #(PAGE_SIZE - 1)
-
-alternative_if ARM64_HAS_NO_HW_PREFETCH
-	prfm	pldl1strm, [x1, #384]
-alternative_else_nop_endif
+	.macro ldp1 reg1, reg2, ptr, val
+	KERNEL_ME_SAFE(9998f, ldp \reg1, \reg2, [\ptr, \val])
+	.endm
 
-CPY_MC(9998f, stnp	x2, x3, [x0, #-256])
-CPY_MC(9998f, ldp	x2, x3, [x1])
-CPY_MC(9998f, stnp	x4, x5, [x0, #16 - 256])
-CPY_MC(9998f, ldp	x4, x5, [x1, #16])
-CPY_MC(9998f, stnp	x6, x7, [x0, #32 - 256])
-CPY_MC(9998f, ldp	x6, x7, [x1, #32])
-CPY_MC(9998f, stnp	x8, x9, [x0, #48 - 256])
-CPY_MC(9998f, ldp	x8, x9, [x1, #48])
-CPY_MC(9998f, stnp	x10, x11, [x0, #64 - 256])
-CPY_MC(9998f, ldp	x10, x11, [x1, #64])
-CPY_MC(9998f, stnp	x12, x13, [x0, #80 - 256])
-CPY_MC(9998f, ldp	x12, x13, [x1, #80])
-CPY_MC(9998f, stnp	x14, x15, [x0, #96 - 256])
-CPY_MC(9998f, ldp	x14, x15, [x1, #96])
-CPY_MC(9998f, stnp	x16, x17, [x0, #112 - 256])
-CPY_MC(9998f, ldp	x16, x17, [x1, #112])
-
-	add	x0, x0, #128
-	add	x1, x1, #128
-
-	b.ne	1b
-
-CPY_MC(9998f, stnp	x2, x3, [x0, #-256])
-CPY_MC(9998f, stnp	x4, x5, [x0, #16 - 256])
-CPY_MC(9998f, stnp	x6, x7, [x0, #32 - 256])
-CPY_MC(9998f, stnp	x8, x9, [x0, #48 - 256])
-CPY_MC(9998f, stnp	x10, x11, [x0, #64 - 256])
-CPY_MC(9998f, stnp	x12, x13, [x0, #80 - 256])
-CPY_MC(9998f, stnp	x14, x15, [x0, #96 - 256])
-CPY_MC(9998f, stnp	x16, x17, [x0, #112 - 256])
+SYM_FUNC_START(__pi_copy_mc_page)
+#include "copy_page_template.S"
 
 	mov x0, #0
 	ret
diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S
index c336d2ffdec55975d8beb31fe36c2b815cfa764e..5499f507bb75800a8c181b4c08d9e0324077b9cc 100644
--- a/arch/arm64/lib/copy_page.S
+++ b/arch/arm64/lib/copy_page.S
@@ -17,63 +17,12 @@
  *	x0 - dest
  *	x1 - src
  */
-SYM_FUNC_START(__pi_copy_page)
-alternative_if ARM64_HAS_NO_HW_PREFETCH
-	// Prefetch three cache lines ahead.
-	prfm	pldl1strm, [x1, #128]
-	prfm	pldl1strm, [x1, #256]
-	prfm	pldl1strm, [x1, #384]
-alternative_else_nop_endif
-
-	ldp	x2, x3, [x1]
-	ldp	x4, x5, [x1, #16]
-	ldp	x6, x7, [x1, #32]
-	ldp	x8, x9, [x1, #48]
-	ldp	x10, x11, [x1, #64]
-	ldp	x12, x13, [x1, #80]
-	ldp	x14, x15, [x1, #96]
-	ldp	x16, x17, [x1, #112]
-
-	add	x0, x0, #256
-	add	x1, x1, #128
-1:
-	tst	x0, #(PAGE_SIZE - 1)
-
-alternative_if ARM64_HAS_NO_HW_PREFETCH
-	prfm	pldl1strm, [x1, #384]
-alternative_else_nop_endif
-
-	stnp	x2, x3, [x0, #-256]
-	ldp	x2, x3, [x1]
-	stnp	x4, x5, [x0, #16 - 256]
-	ldp	x4, x5, [x1, #16]
-	stnp	x6, x7, [x0, #32 - 256]
-	ldp	x6, x7, [x1, #32]
-	stnp	x8, x9, [x0, #48 - 256]
-	ldp	x8, x9, [x1, #48]
-	stnp	x10, x11, [x0, #64 - 256]
-	ldp	x10, x11, [x1, #64]
-	stnp	x12, x13, [x0, #80 - 256]
-	ldp	x12, x13, [x1, #80]
-	stnp	x14, x15, [x0, #96 - 256]
-	ldp	x14, x15, [x1, #96]
-	stnp	x16, x17, [x0, #112 - 256]
-	ldp	x16, x17, [x1, #112]
-
-	add	x0, x0, #128
-	add	x1, x1, #128
-
-	b.ne	1b
-
-	stnp	x2, x3, [x0, #-256]
-	stnp	x4, x5, [x0, #16 - 256]
-	stnp	x6, x7, [x0, #32 - 256]
-	stnp	x8, x9, [x0, #48 - 256]
-	stnp	x10, x11, [x0, #64 - 256]
-	stnp	x12, x13, [x0, #80 - 256]
-	stnp	x14, x15, [x0, #96 - 256]
-	stnp	x16, x17, [x0, #112 - 256]
+	.macro ldp1 reg1, reg2, ptr, val
+	ldp \reg1, \reg2, [\ptr, \val]
+	.endm
 
+SYM_FUNC_START(__pi_copy_page)
+#include "copy_page_template.S"
 	ret
 SYM_FUNC_END(__pi_copy_page)
 SYM_FUNC_ALIAS(copy_page, __pi_copy_page)
diff --git a/arch/arm64/lib/copy_page_template.S b/arch/arm64/lib/copy_page_template.S
new file mode 100644
index 0000000000000000000000000000000000000000..b3ddec2c7a27a4563b58daba66d206d77b643d40
--- /dev/null
+++ b/arch/arm64/lib/copy_page_template.S
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2012 ARM Ltd.
+ */
+
+/*
+ * Copy a page from src to dest (both are page aligned)
+ *
+ * Parameters:
+ *	x0 - dest
+ *	x1 - src
+ */
+	ldp1	x2, x3, x1, #0
+	ldp1	x4, x5, x1, #16
+	ldp1	x6, x7, x1, #32
+	ldp1	x8, x9, x1, #48
+	ldp1	x10, x11, x1, #64
+	ldp1	x12, x13, x1, #80
+	ldp1	x14, x15, x1, #96
+	ldp1	x16, x17, x1, #112
+
+	add	x0, x0, #256
+	add	x1, x1, #128
+1:
+	tst	x0, #(PAGE_SIZE - 1)
+
+	stnp	x2, x3, [x0, #-256]
+	ldp1	x2, x3, x1, #0
+	stnp	x4, x5, [x0, #16 - 256]
+	ldp1	x4, x5, x1, #16
+	stnp	x6, x7, [x0, #32 - 256]
+	ldp1	x6, x7, x1, #32
+	stnp	x8, x9, [x0, #48 - 256]
+	ldp1	x8, x9, x1, #48
+	stnp	x10, x11, [x0, #64 - 256]
+	ldp1	x10, x11, x1, #64
+	stnp	x12, x13, [x0, #80 - 256]
+	ldp1	x12, x13, x1, #80
+	stnp	x14, x15, [x0, #96 - 256]
+	ldp1	x14, x15, x1, #96
+	stnp	x16, x17, [x0, #112 - 256]
+	ldp1	x16, x17, x1, #112
+
+	add	x0, x0, #128
+	add	x1, x1, #128
+
+	b.ne	1b
+
+	stnp	x2, x3, [x0, #-256]
+	stnp	x4, x5, [x0, #16 - 256]
+	stnp	x6, x7, [x0, #32 - 256]
+	stnp	x8, x9, [x0, #48 - 256]
+	stnp	x10, x11, [x0, #64 - 256]
+	stnp	x12, x13, [x0, #80 - 256]
+	stnp	x14, x15, [x0, #96 - 256]
+	stnp	x16, x17, [x0, #112 - 256]
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index cc031bd87455a08d8c5c0a30c095d22c46ea1513..2ac716c0d6d8cba5df698f7825dc19a75e53db78 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -20,7 +20,7 @@
  *	x0 - bytes not copied
  */
 	.macro ldrb1 reg, ptr, val
-	USER(9998f, ldrb  \reg, [\ptr], \val)
+	KERNEL_ME_SAFE(9998f, ldrb  \reg, [\ptr], \val)
 	.endm
 
 	.macro strb1 reg, ptr, val
@@ -28,7 +28,7 @@
 	.endm
 
 	.macro ldrh1 reg, ptr, val
-	USER(9998f, ldrh  \reg, [\ptr], \val)
+	KERNEL_ME_SAFE(9998f, ldrh  \reg, [\ptr], \val)
 	.endm
 
 	.macro strh1 reg, ptr, val
@@ -36,7 +36,7 @@
 	.endm
 
 	.macro ldr1 reg, ptr, val
-	USER(9998f, ldr \reg, [\ptr], \val)
+	KERNEL_ME_SAFE(9998f, ldr \reg, [\ptr], \val)
 	.endm
 
 	.macro str1 reg, ptr, val
@@ -44,7 +44,7 @@
 	.endm
 
 	.macro ldp1 reg1, reg2, ptr, val
-	USER(9998f, ldp \reg1, \reg2, [\ptr], \val)
+	KERNEL_ME_SAFE(9998f, ldp \reg1, \reg2, [\ptr], \val)
 	.endm
 
 	.macro stp1 reg1, reg2, ptr, val
@@ -64,7 +64,7 @@ SYM_FUNC_START(__arch_copy_to_user)
 9997:	cmp	dst, dstin
 	b.ne	9998f
 	// Before being absolutely sure we couldn't copy anything, try harder
-USER(9998f, ldrb	tmp1w, [srcin])
+KERNEL_ME_SAFE(9998f, ldrb	tmp1w, [srcin])
 USER(9998f, sttrb tmp1w, [dst])
 	add	dst, dst, #1
 9998:	sub	x0, end, dst			// bytes not copied
diff --git a/arch/arm64/lib/memcpy_mc.S b/arch/arm64/lib/memcpy_mc.S
index 7076b500d1545d9dd0a6eb52aae63d6a762eccb0..1798090eba060fb96444b9f43df20e639391b0c3 100644
--- a/arch/arm64/lib/memcpy_mc.S
+++ b/arch/arm64/lib/memcpy_mc.S
@@ -1,257 +1,73 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (c) 2012-2021, Arm Limited.
+ * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
  *
- * Adapted from the original at:
- * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/memcpy.S
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
  */
 
 #include <linux/linkage.h>
 #include <asm/assembler.h>
+#include <asm/cache.h>
+#include <asm/asm-uaccess.h>
 
-/* Assumptions:
- *
- * ARMv8-a, AArch64, unaligned accesses.
+/*
+ * Copy a buffer from src to dest (alignment handled by the hardware)
  *
+ * Parameters:
+ *	x0 - dest
+ *	x1 - src
+ *	x2 - n
+ * Returns:
+ *	x0 - dest
  */
-
-#define L(label) .L ## label
-
-#define dstin	x0
-#define src	x1
-#define count	x2
-#define dst	x3
-#define srcend	x4
-#define dstend	x5
-#define A_l	x6
-#define A_lw	w6
-#define A_h	x7
-#define B_l	x8
-#define B_lw	w8
-#define B_h	x9
-#define C_l	x10
-#define C_lw	w10
-#define C_h	x11
-#define D_l	x12
-#define D_h	x13
-#define E_l	x14
-#define E_h	x15
-#define F_l	x16
-#define F_h	x17
-#define G_l	count
-#define G_h	dst
-#define H_l	src
-#define H_h	srcend
-#define tmp1	x14
-
-/* This implementation handles overlaps and supports both memcpy and memmove
-   from a single entry point.  It uses unaligned accesses and branchless
-   sequences to keep the code small, simple and improve performance.
-
-   Copies are split into 3 main cases: small copies of up to 32 bytes, medium
-   copies of up to 128 bytes, and large copies.  The overhead of the overlap
-   check is negligible since it is only required for large copies.
-
-   Large copies use a software pipelined loop processing 64 bytes per iteration.
-   The destination pointer is 16-byte aligned to minimize unaligned accesses.
-   The loop tail is handled by always copying 64 bytes from the end.
-*/
-
-SYM_FUNC_START(__pi_memcpy_mcs)
-	add	srcend, src, count
-	add	dstend, dstin, count
-	cmp	count, 128
-	b.hi	L(copy_long)
-	cmp	count, 32
-	b.hi	L(copy32_128)
-
-	/* Small copies: 0..32 bytes.  */
-	cmp	count, 16
-	b.lo	L(copy16)
-	CPY_MC(9998f, ldp	A_l, A_h, [src])
-	CPY_MC(9998f, ldp	D_l, D_h, [srcend, -16])
-	CPY_MC(9998f, stp	A_l, A_h, [dstin])
-	CPY_MC(9998f, stp	D_l, D_h, [dstend, -16])
-	mov x0, #0
-	ret
-
-	/* Copy 8-15 bytes.  */
-L(copy16):
-	tbz	count, 3, L(copy8)
-	CPY_MC(9998f, ldr	A_l, [src])
-	CPY_MC(9998f, ldr	A_h, [srcend, -8])
-	CPY_MC(9998f, str	A_l, [dstin])
-	CPY_MC(9998f, str	A_h, [dstend, -8])
-	mov x0, #0
-	ret
-
-	.p2align 3
-	/* Copy 4-7 bytes.  */
-L(copy8):
-	tbz	count, 2, L(copy4)
-	CPY_MC(9998f, ldr	A_lw, [src])
-	CPY_MC(9998f, ldr	B_lw, [srcend, -4])
-	CPY_MC(9998f, str	A_lw, [dstin])
-	CPY_MC(9998f, str	B_lw, [dstend, -4])
-	mov x0, #0
-	ret
-
-	/* Copy 0..3 bytes using a branchless sequence.  */
-L(copy4):
-	cbz	count, L(copy0)
-	lsr	tmp1, count, 1
-	CPY_MC(9998f, ldrb	A_lw, [src])
-	CPY_MC(9998f, ldrb	C_lw, [srcend, -1])
-	CPY_MC(9998f, ldrb	B_lw, [src, tmp1])
-	CPY_MC(9998f, strb	A_lw, [dstin])
-	CPY_MC(9998f, strb	B_lw, [dstin, tmp1])
-	CPY_MC(9998f, strb	C_lw, [dstend, -1])
-L(copy0):
-	mov x0, #0
+	.macro ldrb1 reg, ptr, val
+	KERNEL_ME_SAFE(9997f, ldrb  \reg, [\ptr], \val)
+	.endm
+
+	.macro strb1 reg, ptr, val
+	strb \reg, [\ptr], \val
+	.endm
+
+	.macro ldrh1 reg, ptr, val
+	KERNEL_ME_SAFE(9997f, ldrh  \reg, [\ptr], \val)
+	.endm
+
+	.macro strh1 reg, ptr, val
+	strh \reg, [\ptr], \val
+	.endm
+
+	.macro ldr1 reg, ptr, val
+	KERNEL_ME_SAFE(9997f, ldr \reg, [\ptr], \val)
+	.endm
+
+	.macro str1 reg, ptr, val
+	str \reg, [\ptr], \val
+	.endm
+
+	.macro ldp1 reg1, reg2, ptr, val
+	KERNEL_ME_SAFE(9997f, ldp \reg1, \reg2, [\ptr], \val)
+	.endm
+
+	.macro stp1 reg1, reg2, ptr, val
+	stp \reg1, \reg2, [\ptr], \val
+	.endm
+
+end	.req	x5
+SYM_FUNC_START(__memcpy_mc)
+	add	end, x0, x2
+#include "copy_template.S"
+	mov	x0, #0				// Nothing to copy
 	ret
 
-	.p2align 4
-	/* Medium copies: 33..128 bytes.  */
-L(copy32_128):
-	CPY_MC(9998f, ldp	A_l, A_h, [src])
-	CPY_MC(9998f, ldp	B_l, B_h, [src, 16])
-	CPY_MC(9998f, ldp	C_l, C_h, [srcend, -32])
-	CPY_MC(9998f, ldp	D_l, D_h, [srcend, -16])
-	cmp	count, 64
-	b.hi	L(copy128)
-	CPY_MC(9998f, stp	A_l, A_h, [dstin])
-	CPY_MC(9998f, stp	B_l, B_h, [dstin, 16])
-	CPY_MC(9998f, stp	C_l, C_h, [dstend, -32])
-	CPY_MC(9998f, stp	D_l, D_h, [dstend, -16])
-	mov x0, #0
+	// Exception fixups
+9997:	sub	x0, end, dst			// bytes not copied
 	ret
-
-	.p2align 4
-	/* Copy 65..128 bytes.  */
-L(copy128):
-	CPY_MC(9998f, ldp	E_l, E_h, [src, 32])
-	CPY_MC(9998f, ldp	F_l, F_h, [src, 48])
-	cmp	count, 96
-	b.ls	L(copy96)
-	CPY_MC(9998f, ldp	G_l, G_h, [srcend, -64])
-	CPY_MC(9998f, ldp	H_l, H_h, [srcend, -48])
-	CPY_MC(9998f, stp	G_l, G_h, [dstend, -64])
-	CPY_MC(9998f, stp	H_l, H_h, [dstend, -48])
-L(copy96):
-	CPY_MC(9998f, stp	A_l, A_h, [dstin])
-	CPY_MC(9998f, stp	B_l, B_h, [dstin, 16])
-	CPY_MC(9998f, stp	E_l, E_h, [dstin, 32])
-	CPY_MC(9998f, stp	F_l, F_h, [dstin, 48])
-	CPY_MC(9998f, stp	C_l, C_h, [dstend, -32])
-	CPY_MC(9998f, stp	D_l, D_h, [dstend, -16])
-	mov x0, #0
-	ret
-
-	.p2align 4
-	/* Copy more than 128 bytes.  */
-L(copy_long):
-	/* Use backwards copy if there is an overlap.  */
-	sub	tmp1, dstin, src
-	cbz	tmp1, L(copy0)
-	cmp	tmp1, count
-	b.lo	L(copy_long_backwards)
-
-	/* Copy 16 bytes and then align dst to 16-byte alignment.  */
-
-	CPY_MC(9998f, ldp	D_l, D_h, [src])
-	and	tmp1, dstin, 15
-	bic	dst, dstin, 15
-	sub	src, src, tmp1
-	add	count, count, tmp1	/* Count is now 16 too large.  */
-	CPY_MC(9998f, ldp	A_l, A_h, [src, 16])
-	CPY_MC(9998f, stp	D_l, D_h, [dstin])
-	CPY_MC(9998f, ldp	B_l, B_h, [src, 32])
-	CPY_MC(9998f, ldp	C_l, C_h, [src, 48])
-	CPY_MC(9998f, ldp	D_l, D_h, [src, 64]!)
-	subs	count, count, 128 + 16	/* Test and readjust count.  */
-	b.ls	L(copy64_from_end)
-
-L(loop64):
-	CPY_MC(9998f, stp	A_l, A_h, [dst, 16])
-	CPY_MC(9998f, ldp	A_l, A_h, [src, 16])
-	CPY_MC(9998f, stp	B_l, B_h, [dst, 32])
-	CPY_MC(9998f, ldp	B_l, B_h, [src, 32])
-	CPY_MC(9998f, stp	C_l, C_h, [dst, 48])
-	CPY_MC(9998f, ldp	C_l, C_h, [src, 48])
-	CPY_MC(9998f, stp	D_l, D_h, [dst, 64]!)
-	CPY_MC(9998f, ldp	D_l, D_h, [src, 64]!)
-	subs	count, count, 64
-	b.hi	L(loop64)
-
-	/* Write the last iteration and copy 64 bytes from the end.  */
-L(copy64_from_end):
-	CPY_MC(9998f, ldp	E_l, E_h, [srcend, -64])
-	CPY_MC(9998f, stp	A_l, A_h, [dst, 16])
-	CPY_MC(9998f, ldp	A_l, A_h, [srcend, -48])
-	CPY_MC(9998f, stp	B_l, B_h, [dst, 32])
-	CPY_MC(9998f, ldp	B_l, B_h, [srcend, -32])
-	CPY_MC(9998f, stp	C_l, C_h, [dst, 48])
-	CPY_MC(9998f, ldp	C_l, C_h, [srcend, -16])
-	CPY_MC(9998f, stp	D_l, D_h, [dst, 64])
-	CPY_MC(9998f, stp	E_l, E_h, [dstend, -64])
-	CPY_MC(9998f, stp	A_l, A_h, [dstend, -48])
-	CPY_MC(9998f, stp	B_l, B_h, [dstend, -32])
-	CPY_MC(9998f, stp	C_l, C_h, [dstend, -16])
-	mov x0, #0
-	ret
-
-	.p2align 4
-
-	/* Large backwards copy for overlapping copies.
-	   Copy 16 bytes and then align dst to 16-byte alignment.  */
-L(copy_long_backwards):
-	CPY_MC(9998f, ldp	D_l, D_h, [srcend, -16])
-	and	tmp1, dstend, 15
-	sub	srcend, srcend, tmp1
-	sub	count, count, tmp1
-	CPY_MC(9998f, ldp	A_l, A_h, [srcend, -16])
-	CPY_MC(9998f, stp	D_l, D_h, [dstend, -16])
-	CPY_MC(9998f, ldp	B_l, B_h, [srcend, -32])
-	CPY_MC(9998f, ldp	C_l, C_h, [srcend, -48])
-	CPY_MC(9998f, ldp	D_l, D_h, [srcend, -64]!)
-	sub	dstend, dstend, tmp1
-	subs	count, count, 128
-	b.ls	L(copy64_from_start)
-
-L(loop64_backwards):
-	CPY_MC(9998f, stp	A_l, A_h, [dstend, -16])
-	CPY_MC(9998f, ldp	A_l, A_h, [srcend, -16])
-	CPY_MC(9998f, stp	B_l, B_h, [dstend, -32])
-	CPY_MC(9998f, ldp	B_l, B_h, [srcend, -32])
-	CPY_MC(9998f, stp	C_l, C_h, [dstend, -48])
-	CPY_MC(9998f, ldp	C_l, C_h, [srcend, -48])
-	CPY_MC(9998f, stp	D_l, D_h, [dstend, -64]!)
-	CPY_MC(9998f, ldp	D_l, D_h, [srcend, -64]!)
-	subs	count, count, 64
-	b.hi	L(loop64_backwards)
-
-	/* Write the last iteration and copy 64 bytes from the start.  */
-L(copy64_from_start):
-	CPY_MC(9998f, ldp	G_l, G_h, [src, 48])
-	CPY_MC(9998f, stp	A_l, A_h, [dstend, -16])
-	CPY_MC(9998f, ldp	A_l, A_h, [src, 32])
-	CPY_MC(9998f, stp	B_l, B_h, [dstend, -32])
-	CPY_MC(9998f, ldp	B_l, B_h, [src, 16])
-	CPY_MC(9998f, stp	C_l, C_h, [dstend, -48])
-	CPY_MC(9998f, ldp	C_l, C_h, [src])
-	CPY_MC(9998f, stp	D_l, D_h, [dstend, -64])
-	CPY_MC(9998f, stp	G_l, G_h, [dstin, 48])
-	CPY_MC(9998f, stp	A_l, A_h, [dstin, 32])
-	CPY_MC(9998f, stp	B_l, B_h, [dstin, 16])
-	CPY_MC(9998f, stp	C_l, C_h, [dstin])
-	mov x0, #0
-	ret
-
-9998:	mov x0, #-EFAULT
-	ret
-SYM_FUNC_END(__pi_memcpy_mcs)
-
-SYM_FUNC_ALIAS(__memcpy_mcs, __pi_memcpy_mcs)
-EXPORT_SYMBOL(__memcpy_mcs)
-SYM_FUNC_ALIAS_WEAK(memcpy_mcs, __memcpy_mcs)
-EXPORT_SYMBOL(memcpy_mcs)
+SYM_FUNC_END(__memcpy_mc)
+EXPORT_SYMBOL(__memcpy_mc)
+SYM_FUNC_ALIAS_WEAK(memcpy_mc, __memcpy_mc)
+EXPORT_SYMBOL(memcpy_mc)
diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S
index 2b748e83f6cf01359fd1b8dda0e7c1b481d4acf0..50ef243182814241a717ffab5e80430c02b6ed40 100644
--- a/arch/arm64/lib/mte.S
+++ b/arch/arm64/lib/mte.S
@@ -80,6 +80,7 @@ SYM_FUNC_START(mte_copy_page_tags)
 	ret
 SYM_FUNC_END(mte_copy_page_tags)
 
+#ifdef CONFIG_ARCH_HAS_COPY_MC
 /*
  * Copy the tags from the source page to the destination one wiht machine check safe
  *   x0 - address of the destination page
@@ -93,8 +94,8 @@ SYM_FUNC_START(mte_copy_mc_page_tags)
 	mov	x3, x1
 	multitag_transfer_size x5, x6
 1:
-CPY_MC(2f, ldgm	x4, [x3])
-CPY_MC(2f, stgm	x4, [x2])
+KERNEL_ME_SAFE(2f, ldgm	x4, [x3])
+	stgm	x4, [x2]
 	add	x2, x2, x5
 	add	x3, x3, x5
 	tst	x2, #(PAGE_SIZE - 1)
@@ -106,6 +107,7 @@ CPY_MC(2f, stgm	x4, [x2])
 2:	mov x0, #-EFAULT
 	ret
 SYM_FUNC_END(mte_copy_mc_page_tags)
+#endif
 
 /*
  * Read tags from a user buffer (one tag per byte) and set the corresponding
diff --git a/arch/arm64/mm/copypage.c b/arch/arm64/mm/copypage.c
index 9765e40cde6cf3766f2030ba264cc9c12bc71524..ff0d9ceea2a46b7d2e6c49e2ea33c2b839d8fc2a 100644
--- a/arch/arm64/mm/copypage.c
+++ b/arch/arm64/mm/copypage.c
@@ -14,25 +14,6 @@
 #include <asm/cpufeature.h>
 #include <asm/mte.h>
 
-static int do_mte(struct page *to, struct page *from, void *kto, void *kfrom, bool mc)
-{
-	int ret = 0;
-
-	if (system_supports_mte() && page_mte_tagged(from)) {
-		/* It's a new page, shouldn't have been tagged yet */
-		WARN_ON_ONCE(!try_page_mte_tagging(to));
-		if (mc)
-			ret = mte_copy_mc_page_tags(kto, kfrom);
-		else
-			mte_copy_page_tags(kto, kfrom);
-
-		if (!ret)
-			set_page_mte_tagged(to);
-	}
-
-	return ret;
-}
-
 void copy_highpage(struct page *to, struct page *from)
 {
 	void *kto = page_address(to);
@@ -43,7 +24,12 @@ void copy_highpage(struct page *to, struct page *from)
 	if (kasan_hw_tags_enabled())
 		page_kasan_tag_reset(to);
 
-	do_mte(to, from, kto, kfrom, false);
+	if (system_supports_mte() && page_mte_tagged(from)) {
+		/* It's a new page, shouldn't have been tagged yet */
+		WARN_ON_ONCE(!try_page_mte_tagging(to));
+		mte_copy_page_tags(kto, kfrom);
+		set_page_mte_tagged(to);
+	}
 }
 EXPORT_SYMBOL(copy_highpage);
 
@@ -72,9 +58,15 @@ int copy_mc_highpage(struct page *to, struct page *from)
 	if (kasan_hw_tags_enabled())
 		page_kasan_tag_reset(to);
 
-	ret = do_mte(to, from, kto, kfrom, true);
-	if (ret)
-		return -EFAULT;
+	if (system_supports_mte() && page_mte_tagged(from)) {
+		/* It's a new page, shouldn't have been tagged yet */
+		WARN_ON_ONCE(!try_page_mte_tagging(to));
+		ret = mte_copy_mc_page_tags(kto, kfrom);
+		if (ret)
+			return -EFAULT;
+
+		set_page_mte_tagged(to);
+	}
 
 	return 0;
 }
@@ -86,7 +78,6 @@ int copy_mc_user_highpage(struct page *to, struct page *from,
 	int ret;
 
 	ret = copy_mc_highpage(to, from);
-
 	if (!ret)
 		flush_dcache_page(to);
 
diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
index bdc81518d20788c1fc99aa5a49267ed14d957c9d..8c690ae619445021538dbf9cc86e5ea138f5f827 100644
--- a/arch/arm64/mm/extable.c
+++ b/arch/arm64/mm/extable.c
@@ -16,7 +16,7 @@ get_ex_fixup(const struct exception_table_entry *ex)
 	return ((unsigned long)&ex->fixup + ex->fixup);
 }
 
-static bool ex_handler_fixup_err_zero(const struct exception_table_entry *ex,
+static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
 					struct pt_regs *regs)
 {
 	int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data);
@@ -69,15 +69,17 @@ bool fixup_exception(struct pt_regs *regs)
 		return ex_handler_bpf(ex, regs);
 	case EX_TYPE_UACCESS_ERR_ZERO:
 	case EX_TYPE_KACCESS_ERR_ZERO:
-		return ex_handler_fixup_err_zero(ex, regs);
+		return ex_handler_uaccess_err_zero(ex, regs);
 	case EX_TYPE_LOAD_UNALIGNED_ZEROPAD:
 		return ex_handler_load_unaligned_zeropad(ex, regs);
+	case EX_TYPE_KACCESS_ERR_ZERO_ME_SAFE:
+		return false;
 	}
 
 	BUG();
 }
 
-bool fixup_exception_mc(struct pt_regs *regs)
+bool fixup_exception_me(struct pt_regs *regs)
 {
 	const struct exception_table_entry *ex;
 
@@ -87,8 +89,8 @@ bool fixup_exception_mc(struct pt_regs *regs)
 
 	switch (ex->type) {
 	case EX_TYPE_UACCESS_ERR_ZERO:
-	case EX_TYPE_COPY_MC_PAGE_ERR_ZERO:
-		return ex_handler_fixup_err_zero(ex, regs);
+	case EX_TYPE_KACCESS_ERR_ZERO_ME_SAFE:
+		return ex_handler_uaccess_err_zero(ex, regs);
 	}
 
 	return false;
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index e5b38a26bc73d4625aaf5572ab77d6c083bc1932..5d040bbfd247981a0d4c88f4076bb1381d4534d7 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -758,32 +758,23 @@ static int do_bad(unsigned long far, unsigned long esr, struct pt_regs *regs)
 	return 1; /* "fault" */
 }
 
-static bool arm64_do_kernel_sea(unsigned long addr, unsigned int esr,
-				     struct pt_regs *regs, int sig, int code)
+/*
+ * APEI claimed this as a firmware-first notification.
+ * Some processing deferred to task_work before ret_to_user().
+ */
+static bool do_apei_claim_sea(struct pt_regs *regs)
 {
-	if (!IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC))
-		return false;
-
-	if (!sysctl_machine_check_safe)
-		return false;
-
-	if (user_mode(regs))
-		return false;
-
-	if (apei_claim_sea(regs) < 0)
-		return false;
-
-	if (!fixup_exception_mc(regs))
-		return false;
-
-	if (current->flags & PF_KTHREAD)
-		return true;
-
-	set_thread_esr(0, esr);
-	arm64_force_sig_fault(sig, code, addr,
-		"Uncorrected memory error on access to user memory\n");
+	if (user_mode(regs)) {
+		if (!apei_claim_sea(regs))
+			return true;
+	} else if (IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC)) {
+		if (sysctl_machine_check_safe &&
+		    fixup_exception_me(regs) &&
+		    !apei_claim_sea(regs))
+			return true;
+	}
 
-	return true;
+	return false;
 }
 
 static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs)
@@ -791,16 +782,10 @@ static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs)
 	const struct fault_info *inf;
 	unsigned long siaddr;
 
-	inf = esr_to_fault_info(esr);
-
-	if (user_mode(regs) && apei_claim_sea(regs) == 0) {
-		/*
-		 * APEI claimed this as a firmware-first notification.
-		 * Some processing deferred to task_work before ret_to_user().
-		 */
+	if (do_apei_claim_sea(regs))
 		return 0;
-	}
 
+	inf = esr_to_fault_info(esr);
 	if (esr & ESR_ELx_FnV) {
 		siaddr = 0;
 	} else {
@@ -811,9 +796,7 @@ static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs)
 		 */
 		siaddr  = untagged_addr(far);
 	}
-
-	if (!arm64_do_kernel_sea(siaddr, esr, regs, inf->sig, inf->code))
-		arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
+	arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
 
 	return 0;
 }
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index f5e1e494ba847b76cb284560677ee6b48ad68752..2d3df8c7315852063dc0587d43bc3ffb92f7b387 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -41,7 +41,6 @@ HAS_MOPS
 HAS_NESTED_VIRT
 HAS_NMI
 HAS_NO_FPSIMD
-HAS_NO_HW_PREFETCH
 HAS_PAN
 HAS_S1PIE
 HAS_RAS_EXTN
diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index 211f83fbcb1bebb81914fcba0a143f6c1ef48379..ea56b6406e0bc5d163505ec5e0ae86e96e2f50e9 100644
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -1353,13 +1353,6 @@ static void kill_me_maybe(struct callback_head *cb)
 		return;
 	}
 
-	/*
-	 * -EHWPOISON from memory_failure() means that it already sent SIGBUS
-	 * to the current process with the proper error info,
-	 * -EOPNOTSUPP means hwpoison_filter() filtered the error event,
-	 *
-	 * In both cases, no further processing is required.
-	 */
 	if (ret == -EHWPOISON || ret == -EOPNOTSUPP)
 		return;
 
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index 7e0770722bc5a38d8c524dbbd874acc924e720ab..eab8cb40b6016053adad23df49228c5c0c492762 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -464,28 +464,41 @@ static void ghes_clear_estatus(struct ghes *ghes,
 }
 
 /*
- * Called as task_work before returning to user-space.
- * Ensure any queued work has been done before we return to the context that
- * triggered the notification.
+ * struct sync_task_work - for synchronous RAS event
+ *
+ * @twork:                callback_head for task work
+ * @pfn:                  page frame number of corrupted page
+ * @flags:                fine tune action taken
+ *
+ * Structure to pass task work to be handled before
+ * ret_to_user via task_work_add().
  */
-static void ghes_kick_task_work(struct callback_head *head)
+struct sync_task_work {
+	struct callback_head twork;
+	u64 pfn;
+	int flags;
+};
+
+static void memory_failure_cb(struct callback_head *twork)
 {
-	struct acpi_hest_generic_status *estatus;
-	struct ghes_estatus_node *estatus_node;
-	u32 node_len;
+	int ret;
+	struct sync_task_work *twcb =
+		container_of(twork, struct sync_task_work, twork);
 
-	estatus_node = container_of(head, struct ghes_estatus_node, task_work);
-	if (IS_ENABLED(CONFIG_ACPI_APEI_MEMORY_FAILURE))
-		memory_failure_queue_kick(estatus_node->task_work_cpu);
+	ret = memory_failure(twcb->pfn, twcb->flags);
+	gen_pool_free(ghes_estatus_pool, (unsigned long)twcb, sizeof(*twcb));
 
-	estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
-	node_len = GHES_ESTATUS_NODE_LEN(cper_estatus_len(estatus));
-	gen_pool_free(ghes_estatus_pool, (unsigned long)estatus_node, node_len);
+	if (!ret || ret == -EHWPOISON || ret == -EOPNOTSUPP)
+		return;
+
+	pr_err("Sending SIGBUS to current task due to memory error not recovered");
+	force_sig(SIGBUS);
 }
 
 static bool ghes_do_memory_failure(u64 physical_addr, int flags)
 {
 	unsigned long pfn;
+	struct sync_task_work *twcb;
 
 	if (!IS_ENABLED(CONFIG_ACPI_APEI_MEMORY_FAILURE))
 		return false;
@@ -498,6 +511,18 @@ static bool ghes_do_memory_failure(u64 physical_addr, int flags)
 		return false;
 	}
 
+	if (flags == MF_ACTION_REQUIRED && current->mm) {
+		twcb = (void *)gen_pool_alloc(ghes_estatus_pool, sizeof(*twcb));
+		if (!twcb)
+			return false;
+
+		twcb->pfn = pfn;
+		twcb->flags = flags;
+		init_task_work(&twcb->twork, memory_failure_cb);
+		task_work_add(current, &twcb->twork, TWA_RESUME);
+		return true;
+	}
+
 	memory_failure_queue(pfn, flags);
 	return true;
 }
@@ -676,7 +701,7 @@ static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata,
 	schedule_work(&entry->work);
 }
 
-static bool ghes_do_proc(struct ghes *ghes,
+static void ghes_do_proc(struct ghes *ghes,
 			 const struct acpi_hest_generic_status *estatus)
 {
 	int sev, sec_sev;
@@ -726,7 +751,14 @@ static bool ghes_do_proc(struct ghes *ghes,
 #endif
 	}
 
-	return queued;
+	/*
+	 * If no memory failure work is queued for abnormal synchronous
+	 * errors, do a force kill.
+	 */
+	if (sync && !queued) {
+		pr_err("Sending SIGBUS to current task due to memory error not recovered");
+		force_sig(SIGBUS);
+	}
 }
 
 static void __ghes_print_estatus(const char *pfx,
@@ -1028,9 +1060,7 @@ static void ghes_proc_in_irq(struct irq_work *irq_work)
 	struct ghes_estatus_node *estatus_node;
 	struct acpi_hest_generic *generic;
 	struct acpi_hest_generic_status *estatus;
-	bool task_work_pending;
 	u32 len, node_len;
-	int ret;
 
 	llnode = llist_del_all(&ghes_estatus_llist);
 	/*
@@ -1045,25 +1075,16 @@ static void ghes_proc_in_irq(struct irq_work *irq_work)
 		estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
 		len = cper_estatus_len(estatus);
 		node_len = GHES_ESTATUS_NODE_LEN(len);
-		task_work_pending = ghes_do_proc(estatus_node->ghes, estatus);
+
+		ghes_do_proc(estatus_node->ghes, estatus);
+
 		if (!ghes_estatus_cached(estatus)) {
 			generic = estatus_node->generic;
 			if (ghes_print_estatus(NULL, generic, estatus))
 				ghes_estatus_cache_add(generic, estatus);
 		}
-
-		if (task_work_pending && current->mm) {
-			estatus_node->task_work.func = ghes_kick_task_work;
-			estatus_node->task_work_cpu = smp_processor_id();
-			ret = task_work_add(current, &estatus_node->task_work,
-					    TWA_RESUME);
-			if (ret)
-				estatus_node->task_work.func = NULL;
-		}
-
-		if (!estatus_node->task_work.func)
-			gen_pool_free(ghes_estatus_pool,
-				      (unsigned long)estatus_node, node_len);
+		gen_pool_free(ghes_estatus_pool, (unsigned long)estatus_node,
+			      node_len);
 
 		llnode = next;
 	}
@@ -1124,7 +1145,6 @@ static int ghes_in_nmi_queue_one_entry(struct ghes *ghes,
 
 	estatus_node->ghes = ghes;
 	estatus_node->generic = ghes->generic;
-	estatus_node->task_work.func = NULL;
 	estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
 
 	if (__ghes_read_estatus(estatus, buf_paddr, fixmap_idx, len)) {
diff --git a/include/acpi/ghes.h b/include/acpi/ghes.h
index be1dd4c1a917447c1d20d59ef0de7df423758e1d..ebd21b05fe6ed1ffb64e64136b91cbfa62b8ee1e 100644
--- a/include/acpi/ghes.h
+++ b/include/acpi/ghes.h
@@ -35,9 +35,6 @@ struct ghes_estatus_node {
 	struct llist_node llnode;
 	struct acpi_hest_generic *generic;
 	struct ghes *ghes;
-
-	int task_work_cpu;
-	struct callback_head task_work;
 };
 
 struct ghes_estatus_cache {
diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c
index 9372f0c6f7ec78912b8061dc2466be0dee62a060..ac8d19bdcc957691ad060e571017dd5b422edd26 100644
--- a/mm/kasan/shadow.c
+++ b/mm/kasan/shadow.c
@@ -80,14 +80,14 @@ void *memcpy(void *dest, const void *src, size_t len)
 #endif
 
 #ifdef __HAVE_ARCH_MEMCPY_MC
-#undef memcpy_mcs
-int memcpy_mcs(void *dest, const void *src, size_t len)
+#undef memcpy_mc
+int memcpy_mc(void *dest, const void *src, size_t len)
 {
 	if (!kasan_check_range(src, len, false, _RET_IP_) ||
 	    !kasan_check_range(dest, len, true, _RET_IP_))
 		return (int)len;
 
-	return __memcpy_mcs(dest, src, len);
+	return __memcpy_mc(dest, src, len);
 }
 #endif
 
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index c9372bae352473b75e6f71de3819f507feb26af4..d710f6c550801c6bdb9ca0314ebd623e02725835 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -2171,9 +2171,12 @@ static int memory_failure_dev_pagemap(unsigned long pfn, int flags,
  * Must run in process context (e.g. a work queue) with interrupts
  * enabled and no spinlocks held.
  *
- * Return: 0 for successfully handled the memory error,
- *         -EOPNOTSUPP for hwpoison_filter() filtered the error event,
- *         < 0(except -EOPNOTSUPP) on failure.
+ * Return values:
+ *   0             - success
+ *   -EOPNOTSUPP   - hwpoison_filter() filtered the error event.
+ *   -EHWPOISON    - sent SIGBUS to the current process with the proper
+ *                   error info by kill_accessing_process().
+ *   other negative values - failure
  */
 int memory_failure(unsigned long pfn, int flags)
 {
@@ -2459,19 +2462,6 @@ static void memory_failure_work_func(struct work_struct *work)
 	}
 }
 
-/*
- * Process memory_failure work queued on the specified CPU.
- * Used to avoid return-to-userspace racing with the memory_failure workqueue.
- */
-void memory_failure_queue_kick(int cpu)
-{
-	struct memory_failure_cpu *mf_cpu;
-
-	mf_cpu = &per_cpu(memory_failure_cpu, cpu);
-	cancel_work_sync(&mf_cpu->work);
-	memory_failure_work_func(&mf_cpu->work);
-}
-
 static int __init memory_failure_init(void)
 {
 	struct memory_failure_cpu *mf_cpu;