diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index d8dd4163206e915d2e9ed05ff17152d4e451c07f..38b23786aeb4fc4b76120d998ae11df5497876ce 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -25,9 +25,6 @@ #include #include #include -#ifdef CONFIG_IEE -#include -#endif /* * Provide a wxN alias for each wN register so what we can paste a xN @@ -494,9 +491,6 @@ alternative_endif .macro load_ttbr1, pgtbl, tmp1, tmp2 phys_to_ttbr \tmp1, \pgtbl offset_ttbr1 \tmp1, \tmp2 - #ifdef CONFIG_IEE - orr \tmp1, \tmp1, #IEE_ASM_ASID - #endif msr ttbr1_el1, \tmp1 isb .endm diff --git a/arch/arm64/include/asm/haoc/iee.h b/arch/arm64/include/asm/haoc/iee.h index a509e6c2342df92e4f2aa8f01fa29d3cf82562b5..1b3b1cbe070ffa9d5acf7b1268d024bd926515b5 100644 --- a/arch/arm64/include/asm/haoc/iee.h +++ b/arch/arm64/include/asm/haoc/iee.h @@ -56,6 +56,7 @@ void iee_init_mappings(pgd_t *pgdp); void iee_init_post(void); void iee_stack_init(void); void iee_init_tcr(void); +void iee_setup_asid(void); #define IEE_STACK_ORDER 0x3 #define IEE_STACK_SIZE (PAGE_SIZE << IEE_STACK_ORDER) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 65f0d228e98fc7d7b03e520cd7898c612c0e9d98..a269aef75d00f4323d393b592ade43487404b306 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -28,6 +28,9 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#endif #ifdef CONFIG_IEE /* IEE code shall not be interrupted. */ diff --git a/arch/arm64/kernel/haoc/iee/iee-init.c b/arch/arm64/kernel/haoc/iee/iee-init.c index 8492a43a1f609684734dd17d2a2905662ea76f92..1f341424654efc7aada02bf87c1274ae4e640cd8 100644 --- a/arch/arm64/kernel/haoc/iee/iee-init.c +++ b/arch/arm64/kernel/haoc/iee/iee-init.c @@ -11,11 +11,12 @@ #include #include #include +#include __aligned(PAGE_SIZE) DEFINE_PER_CPU(u64*[(PAGE_SIZE/8)], iee_cpu_stack_ptr); -bool __initdata iee_init_done; +bool __ro_after_init iee_init_done; bool __ro_after_init haoc_enabled; /* Allocate pages from IEE data pool to use as per-cpu IEE stack. */ @@ -34,13 +35,30 @@ static void __init iee_stack_alloc(void) flush_tlb_all(); } +/* Setup TCR for this cpu and move ASID from ttbr1 to ttbr0 */ +void iee_setup_asid(void) +{ + unsigned long asid, ttbr0, ttbr1; + + ttbr1 = read_sysreg(ttbr1_el1); + asid = FIELD_GET(TTBR_ASID_MASK, ttbr1); + ttbr0 = read_sysreg(ttbr0_el1) | FIELD_PREP(TTBR_ASID_MASK, asid); + ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, IEE_ASID); + write_sysreg(ttbr1, ttbr1_el1); + write_sysreg(ttbr0, ttbr0_el1); + write_sysreg(read_sysreg(tcr_el1) & ~TCR_A1, tcr_el1); + isb(); + + /* Flush tlb to enable IEE. */ + local_flush_tlb_all(); +} + void __init iee_init_post(void) { if (!haoc_enabled) return; - /* Flush tlb to enable IEE. */ - flush_tlb_all(); + iee_setup_asid(); iee_init_done = true; } diff --git a/arch/arm64/kernel/haoc/iee/iee-mmu.c b/arch/arm64/kernel/haoc/iee/iee-mmu.c index 46f5245144ff7a91ab181f26f61a37abafc81ef4..db6b46984de21f8f17b23521b478cc194793556a 100644 --- a/arch/arm64/kernel/haoc/iee/iee-mmu.c +++ b/arch/arm64/kernel/haoc/iee/iee-mmu.c @@ -493,8 +493,11 @@ void __init iee_init_mappings(pgd_t *pgdp) /* Check if hardware supports IEE. */ if (!cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR1_EL1), - ID_AA64MMFR1_EL1_HPDS_SHIFT)) - panic("Architecture doesn't support HPDS, please disable CONFIG_IEE.\n"); + ID_AA64MMFR1_EL1_HPDS_SHIFT)) { + pr_err("Architecture doesn't support HPDS, please disable CONFIG_IEE.\n"); + haoc_enabled = false; + return; + } else pr_info("HAOC: ARM64 hardware support detected."); diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 34a65153c5b9aa3770a11b2117aaf8a89eae0743..49cf07d171d203d87348a76cd55d08c8bd40d6ed 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -53,6 +53,9 @@ #include #include #include +#ifdef CONFIG_IEE +#include +#endif #include @@ -212,6 +215,10 @@ asmlinkage notrace void secondary_start_kernel(void) #ifdef CONFIG_ARM64_TLBI_IPI cpumask_set_cpu(cpu, mm_cpumask(mm)); +#endif +#ifdef CONFIG_IEE + if (haoc_enabled) + iee_setup_asid(); #endif /* * Setup per-NUMA node page table if kernel diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index 00af510ef70dfbc4825e8dabb55fcf713c5d00ae..fe82d5d3005af7c55ed2e2a91ebb714abe6aa21a 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -372,16 +372,27 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid); #ifdef CONFIG_IEE - /* - * IEE requires the reserved ASID stored in TTBR1 and User ASID stored - * in TTBR0 to support ASID switch by changing TCR.A1. - */ - ttbr0 &= ~TTBR_ASID_MASK; - ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid); + if (iee_init_done) { + /* + * IEE requires the reserved ASID stored in TTBR1 and User ASID stored + * in TTBR0 to support ASID switch by changing TCR.A1. + */ + ttbr0 &= ~TTBR_ASID_MASK; + ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid); - cpu_set_reserved_ttbr0_nosync(); - write_sysreg(ttbr0, ttbr0_el1); - isb(); + cpu_set_reserved_ttbr0_nosync(); + write_sysreg(ttbr0, ttbr0_el1); + isb(); + } else { + /* Set ASID in TTBR1 since TCR.A1 is set */ + ttbr1 &= ~TTBR_ASID_MASK; + ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); + + cpu_set_reserved_ttbr0_nosync(); + write_sysreg(ttbr1, ttbr1_el1); + write_sysreg(ttbr0, ttbr0_el1); + isb(); + } #else /* Set ASID in TTBR1 since TCR.A1 is set */ ttbr1 &= ~TTBR_ASID_MASK; @@ -404,7 +415,7 @@ static int asids_update_limit(void) if (pinned_asid_map) set_kpti_asid_bits(pinned_asid_map); #ifdef CONFIG_IEE - if (pinned_asid_map) { + if (haoc_enabled && pinned_asid_map) { __set_bit(ctxid2asid(IEE_ASID), pinned_asid_map); __set_bit(ctxid2asid(IEE_ASID | ASID_BIT), pinned_asid_map); } @@ -448,12 +459,14 @@ static int asids_init(void) if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) set_kpti_asid_bits(asid_map); #ifdef CONFIG_IEE - #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - __set_bit(ctxid2asid(IEE_ASID | ASID_BIT), asid_map); - __set_bit(ctxid2asid(IEE_ASID | ASID_BIT), pinned_asid_map); - #endif - __set_bit(ctxid2asid(IEE_ASID), asid_map); - __set_bit(ctxid2asid(IEE_ASID), pinned_asid_map); + if (haoc_enabled) { + #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 + __set_bit(ctxid2asid(IEE_ASID | ASID_BIT), asid_map); + __set_bit(ctxid2asid(IEE_ASID | ASID_BIT), pinned_asid_map); + #endif + __set_bit(ctxid2asid(IEE_ASID), asid_map); + __set_bit(ctxid2asid(IEE_ASID), pinned_asid_map); + } #endif return 0; } diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index dafdc63c6aed1bdd5006cf0cc440c47235abb065..9ef7b07349d7f03d2853db6304b82cba9b715f1f 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -21,9 +21,6 @@ #include #include #include -#ifdef CONFIG_IEE -#include -#endif #ifdef CONFIG_ARM64_64K_PAGES #define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K @@ -176,9 +173,6 @@ SYM_FUNC_END(cpu_do_resume) adrp \tmp1, reserved_pg_dir phys_to_ttbr \tmp2, \tmp1 offset_ttbr1 \tmp2, \tmp1 - #ifdef CONFIG_IEE - orr \tmp2, \tmp2, #IEE_ASM_ASID - #endif msr ttbr1_el1, \tmp2 isb tlbi vmalle1 @@ -196,9 +190,6 @@ SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1) __idmap_cpu_set_reserved_ttbr1 x1, x3 offset_ttbr1 x0, x3 - #ifdef CONFIG_IEE - orr x0, x0, #IEE_ASM_ASID - #endif msr ttbr1_el1, x0 isb @@ -431,15 +422,9 @@ SYM_FUNC_START(__cpu_setup) tcr .req x16 tcr2 .req x15 mov_q mair, MAIR_EL1_SET -#ifdef CONFIG_IEE - mov_q tcr, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ - TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ - TCR_TBI0 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS -#else mov_q tcr, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS -#endif mov tcr2, xzr tcr_clear_errata_bits tcr, x9, x5