From 1e4c834dbd35c145aa39c5174c7b3c5a46cc5b93 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Tue, 30 Apr 2019 10:57:37 +0200 Subject: [PATCH 01/10] arm.h: add CTR_WORD_SIZE Adds a common define for the word size used by the CTR (cache type) register. Reviewed-by: Jerome Forissier Signed-off-by: Jens Wiklander --- core/arch/arm/include/arm.h | 1 + core/arch/arm/kernel/cache_helpers_a32.S | 6 ++---- core/arch/arm/kernel/cache_helpers_a64.S | 7 ++----- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/core/arch/arm/include/arm.h b/core/arch/arm/include/arm.h index f825e3b9a2b..6fe50812d52 100644 --- a/core/arch/arm/include/arm.h +++ b/core/arch/arm/include/arm.h @@ -52,6 +52,7 @@ #define CTR_L1IP_MASK 0x3 #define CTR_IMINLINE_SHIFT 0 #define CTR_IMINLINE_MASK 0xf +#define CTR_WORD_SIZE 4 #define ARM32_CPSR_MODE_MASK 0x1f #define ARM32_CPSR_MODE_USR 0x10 diff --git a/core/arch/arm/kernel/cache_helpers_a32.S b/core/arch/arm/kernel/cache_helpers_a32.S index 8c80a939417..501433dafab 100644 --- a/core/arch/arm/kernel/cache_helpers_a32.S +++ b/core/arch/arm/kernel/cache_helpers_a32.S @@ -11,22 +11,20 @@ #include #include -#define WORD_SIZE 4 - /* * Cache line size helpers */ .macro dcache_line_size reg, tmp read_ctr \tmp ubfx \tmp, \tmp, #CTR_DMINLINE_SHIFT, #CTR_DMINLINE_WIDTH - mov \reg, #WORD_SIZE + mov \reg, #CTR_WORD_SIZE lsl \reg, \reg, \tmp .endm .macro icache_line_size reg, tmp read_ctr \tmp and \tmp, \tmp, #CTR_IMINLINE_MASK - mov \reg, #WORD_SIZE + mov \reg, #CTR_WORD_SIZE lsl \reg, \reg, \tmp .endm diff --git a/core/arch/arm/kernel/cache_helpers_a64.S b/core/arch/arm/kernel/cache_helpers_a64.S index 20e3ac5a9ce..3d1c92e4647 100644 --- a/core/arch/arm/kernel/cache_helpers_a64.S +++ b/core/arch/arm/kernel/cache_helpers_a64.S @@ -8,20 +8,17 @@ #include #include -#define WORD_SIZE 4 - - .macro dcache_line_size reg, tmp mrs \tmp, ctr_el0 ubfx \tmp, \tmp, #CTR_DMINLINE_SHIFT, #CTR_DMINLINE_WIDTH - mov \reg, #WORD_SIZE + mov \reg, #CTR_WORD_SIZE lsl \reg, \reg, \tmp .endm .macro icache_line_size reg, tmp mrs \tmp, ctr_el0 and \tmp, \tmp, #CTR_IMINLINE_MASK - mov \reg, #WORD_SIZE + mov \reg, #CTR_WORD_SIZE lsl \reg, \reg, \tmp .endm From d61bcfe498eb8f8c80a586620b0545b26479d9a5 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Mon, 29 Apr 2019 16:41:37 +0200 Subject: [PATCH 02/10] core: add icache_inv_user_range() Adds icache_inv_user_range() which is used when invalidating currently mapped user space memory. This is needed since a different ASID is usually in use while in kernel mode. So using icache_inv_range() would normally not have any effect on user mode mappings. Reviewed-by: Etienne Carriere Signed-off-by: Jens Wiklander --- core/arch/arm/include/kernel/cache_helpers.h | 1 + core/arch/arm/kernel/thread_a32.S | 107 +++++++++++++++++++ core/arch/arm/kernel/thread_a64.S | 86 +++++++++++++++ 3 files changed, 194 insertions(+) diff --git a/core/arch/arm/include/kernel/cache_helpers.h b/core/arch/arm/include/kernel/cache_helpers.h index d6f7fb7244d..38a11716ba5 100644 --- a/core/arch/arm/include/kernel/cache_helpers.h +++ b/core/arch/arm/include/kernel/cache_helpers.h @@ -22,6 +22,7 @@ void dcache_inv_range(void *addr, size_t size); void icache_inv_all(void); void icache_inv_range(void *addr, size_t size); +void icache_inv_user_range(void *addr, size_t size); void dcache_op_louis(unsigned long op_type); void dcache_op_all(unsigned long op_type); diff --git a/core/arch/arm/kernel/thread_a32.S b/core/arch/arm/kernel/thread_a32.S index a0a9f743bb1..27c5f690c8b 100644 --- a/core/arch/arm/kernel/thread_a32.S +++ b/core/arch/arm/kernel/thread_a32.S @@ -1048,6 +1048,113 @@ eret_to_user_mode: #endif movs pc, lr + + /* + * void icache_inv_user_range(void *addr, size_t size); + * + * This function has to execute with the user space ASID active, + * this means executing with reduced mapping and the code needs + * to be located here together with the vector. + */ + .global icache_inv_user_range + .type icache_inv_user_range , %function +icache_inv_user_range: + push {r4-r7} + + /* Mask all exceptions */ + mrs r4, cpsr /* This register must be preserved */ + cpsid aif + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + ldr r2, =thread_user_kcode_offset + ldr r2, [r2] + read_vbar r5 /* This register must be preserved */ + sub r3, r5, r2 + write_vbar r3 + isb + + /* Jump into the reduced mapping before the full mapping is removed */ + ldr r3, =1f + sub r3, r3, r2 + bx r3 +1: +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + +#ifdef CFG_WITH_LPAE + read_ttbr0_64bit r6, r7 /* These registers must be preseved */ +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + add r2, r6, #CORE_MMU_L1_TBL_OFFSET +#endif + /* switch to user ASID */ + orr r3, r7, #BIT(TTBR_ASID_SHIFT - 32) + write_ttbr0_64bit r2, r3 + isb +#else /*!CFG_WITH_LPAE*/ +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + read_ttbr1 r6 /* This register must be preserved */ + add r2, r6, #CORE_MMU_L1_TBL_OFFSET + write_ttbr1 r2 + isb +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + read_contextidr r7 /* This register must be preserved */ + orr r2, r7, #BIT(0) + write_contextidr r2 + isb +#endif /*!CFG_WITH_LPAE*/ + + /* + * Do the actual icache invalidation + */ + + /* Calculate minimum icache line size, result in r2 */ + read_ctr r3 + and r3, r3, #CTR_IMINLINE_MASK + mov r2, #CTR_WORD_SIZE + lsl r2, r2, r3 + + add r1, r0, r1 + sub r3, r2, #1 + bic r0, r0, r3 +1: + write_icimvau r0 + add r0, r0, r2 + cmp r0, r1 + blo 1b + + /* Invalidate entire branch predictor array inner shareable */ + write_bpiallis + + dsb ishst + isb + +#ifdef CFG_WITH_LPAE + write_ttbr0_64bit r6, r7 + isb +#else /*!CFG_WITH_LPAE*/ + write_contextidr r7 + isb +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + write_ttbr1 r6 + isb +#endif +#endif /*!CFG_WITH_LPAE*/ + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + write_vbar r5 + isb + /* + * The PC is adjusted unconditionally to guard against the + * case there was an FIQ just before we did the "cpsid aif". + */ + ldr r0, =1f + bx r0 +1: +#endif + + msr cpsr_fsxc, r4 /* Restore exceptions */ + pop {r4-r7} + bx lr /* End of icache_inv_user_range() */ + /* * Make sure that literals are placed before the * thread_excp_vect_end label. diff --git a/core/arch/arm/kernel/thread_a64.S b/core/arch/arm/kernel/thread_a64.S index 1a03524483a..5325e4a450f 100644 --- a/core/arch/arm/kernel/thread_a64.S +++ b/core/arch/arm/kernel/thread_a64.S @@ -703,6 +703,92 @@ eret_to_el0: #endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ eret + + /* + * void icache_inv_user_range(void *addr, size_t size); + * + * This function has to execute with the user space ASID active, + * this means executing with reduced mapping and the code needs + * to be located here together with the vector. + */ + .global icache_inv_user_range + .type icache_inv_user_range , %function +icache_inv_user_range: + /* Mask all exceptions */ + mrs x6, daif /* this register must be preserved */ + msr daifset, #DAIFBIT_ALL + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + /* Point to the vector into the reduced mapping */ + adr x2, thread_user_kcode_offset + ldr x2, [x2] + mrs x4, vbar_el1 /* this register must be preserved */ + sub x3, x4, x2 + msr vbar_el1, x3 + isb + + /* Jump into the reduced mapping and continue execution */ + ldr x3, =1f + sub x3, x3, x2 + br x3 +1: + + /* Update the mapping to exclude the full kernel mapping */ + mrs x5, ttbr0_el1 /* this register must be preserved */ + add x2, x5, #CORE_MMU_L1_TBL_OFFSET + orr x2, x2, #BIT(TTBR_ASID_SHIFT) /* switch to user mode ASID */ + msr ttbr0_el1, x2 + isb + +#else + mrs x5, ttbr0_el1 /* this register must be preserved */ + orr x2, x2, #BIT(TTBR_ASID_SHIFT) /* switch to user mode ASID */ + msr ttbr0_el1, x2 + isb +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + + /* + * Do the actual icache invalidation + */ + + /* Calculate minimum icache line size, result in x2 */ + mrs x3, ctr_el0 + and x3, x3, #CTR_IMINLINE_MASK + mov x2, #CTR_WORD_SIZE + lsl x2, x2, x3 + + add x1, x0, x1 + sub x3, x2, #1 + bic x0, x0, x3 +1: + ic ivau, x0 + add x0, x0, x2 + cmp x0, x1 + b.lo 1b + dsb ish + +#ifdef CFG_CORE_UNMAP_CORE_AT_EL0 + /* Update the mapping to use the full kernel mapping and ASID */ + msr ttbr0_el1, x5 + isb + + /* Jump into the full mapping and continue execution */ + ldr x0, =1f + br x0 +1: + + /* Point to the vector into the full mapping */ + msr vbar_el1, x4 + isb +#else + /* switch to kernel mode ASID */ + msr ttbr0_el1, x5 + isb +#endif /*CFG_CORE_UNMAP_CORE_AT_EL0*/ + + msr daif, x6 /* restore exceptions */ + ret /* End of icache_inv_user_range() */ + /* * Make sure that literals are placed before the * thread_excp_vect_end label. From 2b405e8037462c5000bb8fb02cf0d939c62dfeca Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Tue, 30 Apr 2019 13:11:13 +0200 Subject: [PATCH 03/10] core: cache_helpers_a{32,64}.S: remove section assignments Since the FUNC and LOCAL_FUNC assembly macros now assign a section to each assembly function the explicitly assigned sections in cache_helpers_a{32,64}.S are ignored. So remove the ignored section assignments. Reviewed-by: Jerome Forissier Signed-off-by: Jens Wiklander --- core/arch/arm/kernel/cache_helpers_a32.S | 11 ----------- core/arch/arm/kernel/cache_helpers_a64.S | 11 ----------- 2 files changed, 22 deletions(-) diff --git a/core/arch/arm/kernel/cache_helpers_a32.S b/core/arch/arm/kernel/cache_helpers_a32.S index 501433dafab..7f908d04ed6 100644 --- a/core/arch/arm/kernel/cache_helpers_a32.S +++ b/core/arch/arm/kernel/cache_helpers_a32.S @@ -50,7 +50,6 @@ loop_\reg: * size. 'r0' = addr, 'r1' = size * ------------------------------------------ */ -.section .text.dcache_cleaninv_range FUNC dcache_cleaninv_range , : UNWIND( .fnstart) do_dcache_maintenance_by_mva dccimvac @@ -62,7 +61,6 @@ END_FUNC dcache_cleaninv_range * 'r0' = addr, 'r1' = size * ------------------------------------------ */ -.section .text.dcache_clean_range FUNC dcache_clean_range , : UNWIND( .fnstart) do_dcache_maintenance_by_mva dccmvac @@ -74,7 +72,6 @@ END_FUNC dcache_clean_range * size. 'r0' = addr, 'r1' = size * ------------------------------------------ */ -.section .text.dcache_inv_range FUNC dcache_inv_range , : UNWIND( .fnstart) do_dcache_maintenance_by_mva dcimvac @@ -106,7 +103,6 @@ END_FUNC dcache_inv_range b do_dcsw_op .endm -.section .text.do_dcsw_op LOCAL_FUNC do_dcsw_op , : UNWIND( .fnstart) push {r4-r12,lr} @@ -170,7 +166,6 @@ END_FUNC do_dcsw_op * DCACHE_OP_CLEAN), as defined in cache_helpers.h * --------------------------------------------------------------- */ -.section .text.dcache_op_louis FUNC dcache_op_louis , : UNWIND( .fnstart) dcsw_op #CLIDR_LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT @@ -185,7 +180,6 @@ END_FUNC dcache_op_louis * DCACHE_OP_CLEAN), as defined in cache_helpers.h * --------------------------------------------------------------- */ -.section .text.dcache_op_all FUNC dcache_op_all , : UNWIND( .fnstart) dcsw_op #CLIDR_LOC_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT @@ -213,7 +207,6 @@ END_FUNC dcache_op_all * DCACHE_OP_CLEAN), as defined in cache_helpers.h * --------------------------------------------------------------- */ -.section .text.dcache_op_level1 FUNC dcache_op_level1 , : UNWIND( .fnstart) dcsw_op_level #(1 << CSSELR_LEVEL_SHIFT) @@ -228,7 +221,6 @@ END_FUNC dcache_op_level1 * DCACHE_OP_CLEAN), as defined in cache_helpers.h * --------------------------------------------------------------- */ -.section .text.dcache_op_level2 FUNC dcache_op_level2 , : UNWIND( .fnstart) dcsw_op_level #(2 << CSSELR_LEVEL_SHIFT) @@ -243,14 +235,12 @@ END_FUNC dcache_op_level2 * DCACHE_OP_CLEAN), as defined in cache_helpers.h * --------------------------------------------------------------- */ -.section .text.dcache_op_level3 FUNC dcache_op_level3 , : UNWIND( .fnstart) dcsw_op_level #(3 << CSSELR_LEVEL_SHIFT) UNWIND( .fnend) END_FUNC dcache_op_level3 -.section .text.icache_inv_all FUNC icache_inv_all , : UNWIND( .fnstart) /* Invalidate Entire Instruction Cache (and branch predictors) */ @@ -268,7 +258,6 @@ END_FUNC icache_inv_all * size. 'r0' = addr, 'r1' = size * ------------------------------------------ */ -.section .text.icache_inv_range FUNC icache_inv_range , : UNWIND( .fnstart) icache_line_size r2, r3 diff --git a/core/arch/arm/kernel/cache_helpers_a64.S b/core/arch/arm/kernel/cache_helpers_a64.S index 3d1c92e4647..1c68d30ea7f 100644 --- a/core/arch/arm/kernel/cache_helpers_a64.S +++ b/core/arch/arm/kernel/cache_helpers_a64.S @@ -43,7 +43,6 @@ loop_\op: * size. 'x0' = addr, 'x1' = size * ------------------------------------------ */ -.section .text.dcache_cleaninv_range FUNC dcache_cleaninv_range , : do_dcache_maintenance_by_mva civac END_FUNC dcache_cleaninv_range @@ -53,7 +52,6 @@ END_FUNC dcache_cleaninv_range * 'x0' = addr, 'x1' = size * ------------------------------------------ */ -.section .text.dcache_clean_range FUNC dcache_clean_range , : do_dcache_maintenance_by_mva cvac END_FUNC dcache_clean_range @@ -63,7 +61,6 @@ END_FUNC dcache_clean_range * size. 'x0' = addr, 'x1' = size * ------------------------------------------ */ -.section .text.dcache_inv_range FUNC dcache_inv_range , : do_dcache_maintenance_by_mva ivac END_FUNC dcache_inv_range @@ -93,7 +90,6 @@ END_FUNC dcache_inv_range b do_dcsw_op .endm -.section .text.do_dcsw_op LOCAL_FUNC do_dcsw_op , : cbz x3, exit adr x14, dcsw_loop_table // compute inner loop address @@ -154,12 +150,10 @@ dcsw_loop_table: dcsw_loop csw END_FUNC do_dcsw_op -.section .text.dcache_op_louis FUNC dcache_op_louis , : dcsw_op #CLIDR_LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT END_FUNC dcache_op_louis -.section .text.dcache_op_all FUNC dcache_op_all , : dcsw_op #CLIDR_LOC_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT END_FUNC dcache_op_all @@ -183,7 +177,6 @@ END_FUNC dcache_op_all * x0: The operation type (0-2), as defined in cache_helpers.h * --------------------------------------------------------------- */ -.section .text.dcache_op_level1 FUNC dcache_op_level1 , : dcsw_op_level #(1 << CSSELR_LEVEL_SHIFT) END_FUNC dcache_op_level1 @@ -195,7 +188,6 @@ END_FUNC dcache_op_level1 * x0: The operation type (0-2), as defined in cache_helpers.h * --------------------------------------------------------------- */ -.section .text.dcache_op_level2 FUNC dcache_op_level2 , : dcsw_op_level #(2 << CSSELR_LEVEL_SHIFT) END_FUNC dcache_op_level2 @@ -207,12 +199,10 @@ END_FUNC dcache_op_level2 * x0: The operation type (0-2), as defined in cache_helpers.h * --------------------------------------------------------------- */ -.section .text.dcache_op_level3 FUNC dcache_op_level3 , : dcsw_op_level #(3 << CSSELR_LEVEL_SHIFT) END_FUNC dcache_op_level3 -.section .text.icache_inv_all FUNC icache_inv_all , : /* Invalidate Entire Instruction Cache */ ic ialluis @@ -223,7 +213,6 @@ FUNC icache_inv_all , : ret END_FUNC icache_inv_all -.section .text.icache_inv_range FUNC icache_inv_range , : icache_line_size x2, x3 add x1, x0, x1 From 51ffb7113cc9043fe725fc1bc7a3b5fa480a2eee Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Tue, 30 Apr 2019 13:16:32 +0200 Subject: [PATCH 04/10] core: add dcache_clean_range_pou() Adds dcache_clean_range_pou() which cleans the data cache to the point of unification. This is exactly what's needed when later invalidating the icache due to updates in a page. Reviewed-by: Etienne Carriere Signed-off-by: Jens Wiklander --- core/arch/arm/include/kernel/cache_helpers.h | 1 + core/arch/arm/kernel/cache_helpers_a32.S | 12 ++++++++++++ core/arch/arm/kernel/cache_helpers_a64.S | 8 ++++++++ 3 files changed, 21 insertions(+) diff --git a/core/arch/arm/include/kernel/cache_helpers.h b/core/arch/arm/include/kernel/cache_helpers.h index 38a11716ba5..39a9d32389f 100644 --- a/core/arch/arm/include/kernel/cache_helpers.h +++ b/core/arch/arm/include/kernel/cache_helpers.h @@ -19,6 +19,7 @@ void dcache_cleaninv_range(void *addr, size_t size); void dcache_clean_range(void *addr, size_t size); void dcache_inv_range(void *addr, size_t size); +void dcache_clean_range_pou(void *addr, size_t size); void icache_inv_all(void); void icache_inv_range(void *addr, size_t size); diff --git a/core/arch/arm/kernel/cache_helpers_a32.S b/core/arch/arm/kernel/cache_helpers_a32.S index 7f908d04ed6..de8f0e4c726 100644 --- a/core/arch/arm/kernel/cache_helpers_a32.S +++ b/core/arch/arm/kernel/cache_helpers_a32.S @@ -78,6 +78,18 @@ UNWIND( .fnstart) UNWIND( .fnend) END_FUNC dcache_inv_range + + /* ------------------------------------------ + * Clean from base address till size to point of unification + * 'r0' = addr, 'r1' = size + * ------------------------------------------ + */ +FUNC dcache_clean_range_pou , : +UNWIND( .fnstart) + do_dcache_maintenance_by_mva dccmvau +UNWIND( .fnend) +END_FUNC dcache_clean_range_pou + /* ---------------------------------------------------------------- * Data cache operations by set/way to the level specified * diff --git a/core/arch/arm/kernel/cache_helpers_a64.S b/core/arch/arm/kernel/cache_helpers_a64.S index 1c68d30ea7f..cf427e3dc3c 100644 --- a/core/arch/arm/kernel/cache_helpers_a64.S +++ b/core/arch/arm/kernel/cache_helpers_a64.S @@ -65,6 +65,14 @@ FUNC dcache_inv_range , : do_dcache_maintenance_by_mva ivac END_FUNC dcache_inv_range + /* ------------------------------------------ + * Clean from base address till size to point of unification + * 'x0' = addr, 'x1' = size + * ------------------------------------------ + */ +FUNC dcache_clean_range_pou , : + do_dcache_maintenance_by_mva cvau +END_FUNC dcache_clean_range_pou /* --------------------------------------------------------------- * Data cache operations by set/way to the level specified From 0105e1b878c1a0f73a76dd8408449c9d2227ba39 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Tue, 30 Apr 2019 13:21:38 +0200 Subject: [PATCH 05/10] core: pager: use dcache_clean_range_pou() Pager uses dcache_clean_range_pou() when cleaning pages before invalidating icache for that page. Prior to this patch dcache_clean_range() was used indirectly which cleans the range to point of coherency instead of point of unification. With this patch we're likely to save one data cache level by only cleaning level 1 instead of level 1 and 2. This assumes separate data and instructions caches level 1 and a unified data cache at level 2 Acked-by: Etienne Carriere Signed-off-by: Jens Wiklander --- core/arch/arm/mm/tee_pager.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/core/arch/arm/mm/tee_pager.c b/core/arch/arm/mm/tee_pager.c index 6033b7e9e28..5b02d3eb03c 100644 --- a/core/arch/arm/mm/tee_pager.c +++ b/core/arch/arm/mm/tee_pager.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -948,8 +949,7 @@ bool tee_pager_set_uta_area_attr(struct user_ta_ctx *utc, vaddr_t base, /* Assert that the pmem isn't shared. */ assert(same_context(pmem)); - cache_op_inner(DCACHE_AREA_CLEAN, va, - SMALL_PAGE_SIZE); + dcache_clean_range_pou(va, SMALL_PAGE_SIZE); cache_op_inner(ICACHE_INVALIDATE, NULL, 0); } } @@ -1355,17 +1355,13 @@ bool tee_pager_handle_fault(struct abort_info *ai) if (area->flags & (TEE_MATTR_PX | TEE_MATTR_UX)) { uint32_t mask = TEE_MATTR_PX | TEE_MATTR_UX | TEE_MATTR_PW | TEE_MATTR_UW; + void *va = (void *)page_va; /* Set a temporary read-only mapping */ area_set_entry(area, tblidx, pa, attr & ~mask); tlbi_mva_allasid(page_va); - /* - * Doing these operations to LoUIS (Level of - * unification, Inner Shareable) would be enough - */ - cache_op_inner(DCACHE_AREA_CLEAN, (void *)page_va, - SMALL_PAGE_SIZE); + dcache_clean_range_pou(va, SMALL_PAGE_SIZE); cache_op_inner(ICACHE_INVALIDATE, NULL, 0); /* Set the final mapping */ From c2c2439cb2cc00806cffa75b69056cd569048525 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Tue, 30 Apr 2019 15:10:20 +0200 Subject: [PATCH 06/10] core: arm32.h: add TLBI_{MVA_SHIFT,ASID_MASK} Adds TLBI macros to help formatting source register for TLB invalidations. Reviewed-by: Jerome Forissier Signed-off-by: Jens Wiklander --- core/arch/arm/include/arm32.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/arch/arm/include/arm32.h b/core/arch/arm/include/arm32.h index e7c3b2d23d0..6b5bf7bbb14 100644 --- a/core/arch/arm/include/arm32.h +++ b/core/arch/arm/include/arm32.h @@ -143,6 +143,8 @@ #define TTBR_ASID_MASK 0xff #define TTBR_ASID_SHIFT 48 +#define TLBI_MVA_SHIFT 12 +#define TLBI_ASID_MASK 0xff #define FSR_LPAE BIT32(9) #define FSR_WNR BIT32(11) From 70e5314f755513726fd8f79e516619de3519781f Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Mon, 13 May 2019 12:09:29 +0200 Subject: [PATCH 07/10] core: arm64.h: add tlbi_vale1is() Adds tlbi_vale1is() which is a wrapper around inline assembly code to execute "tlbi vale1is". The operation is described as "TLB Invalidate by VA, Last level, EL1, Inner Shareable" in the ARM ARM. Reviewed-by: Etienne Carriere Signed-off-by: Jens Wiklander --- core/arch/arm/include/arm64.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/arch/arm/include/arm64.h b/core/arch/arm/include/arm64.h index 8fc30214123..e36f43518a2 100644 --- a/core/arch/arm/include/arm64.h +++ b/core/arch/arm/include/arm64.h @@ -249,6 +249,11 @@ static inline void tlbi_vaae1is(uint64_t mva) asm volatile ("tlbi vaae1is, %0" : : "r" (mva)); } +static inline void tlbi_vale1is(uint64_t mva) +{ + asm volatile ("tlbi vale1is, %0" : : "r" (mva)); +} + /* * Templates for register read/write functions based on mrs/msr */ From 11c0157e37f791d16ada86345be09a7a162b2a76 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Tue, 30 Apr 2019 15:11:36 +0200 Subject: [PATCH 08/10] core: add tlbi_mva_asid() Adds tlbi_mva_asid() to invalidate one TLB entry, typically page sized, selected by virtual address and address identifier. The function targets both the kernel mode and user mode address identifiers at the same time. Reviewed-by: Etienne Carriere Signed-off-by: Jens Wiklander --- core/arch/arm/include/kernel/tlb_helpers.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/arch/arm/include/kernel/tlb_helpers.h b/core/arch/arm/include/kernel/tlb_helpers.h index 5d4d35adc12..19994c68866 100644 --- a/core/arch/arm/include/kernel/tlb_helpers.h +++ b/core/arch/arm/include/kernel/tlb_helpers.h @@ -15,6 +15,7 @@ void tlbi_all(void); void tlbi_asid(unsigned long asid); void tlbi_mva_allasid(unsigned long addr); + static inline void tlbi_mva_allasid_nosync(vaddr_t va) { #ifdef ARM64 @@ -23,6 +24,23 @@ static inline void tlbi_mva_allasid_nosync(vaddr_t va) write_tlbimvaais(va); #endif } + +static inline void tlbi_mva_asid(vaddr_t va, uint32_t asid) +{ + uint32_t a = asid & TLBI_ASID_MASK; + + dsb_ishst(); +#ifdef ARM64 + tlbi_vale1is((va >> TLBI_MVA_SHIFT) | SHIFT_U64(a, TLBI_ASID_SHIFT)); + tlbi_vale1is((va >> TLBI_MVA_SHIFT) | + SHIFT_U64(a | 1, TLBI_ASID_SHIFT)); +#else + write_tlbimvais((va & ~(BIT32(TLBI_MVA_SHIFT) - 1)) | a); + write_tlbimvais((va & ~(BIT32(TLBI_MVA_SHIFT) - 1)) | a | 1); +#endif + dsb_ish(); + isb(); +} #endif /*!ASM*/ #endif /* TLB_HELPERS_H */ From 11d6ccc8178709101ffd9d09ea699215cd1a3fef Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Tue, 30 Apr 2019 16:21:08 +0200 Subject: [PATCH 09/10] core: pager: use tlbi_mva_asid() where applicable Instead of invalidating a virtual address for all ASIDs only target the relevant ones. For kernel mode mappings all ASIDs still needs to be targeted though. Reviewed-by: Etienne Carriere Signed-off-by: Jens Wiklander --- core/arch/arm/mm/tee_pager.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/core/arch/arm/mm/tee_pager.c b/core/arch/arm/mm/tee_pager.c index 5b02d3eb03c..821758f4969 100644 --- a/core/arch/arm/mm/tee_pager.c +++ b/core/arch/arm/mm/tee_pager.c @@ -411,6 +411,22 @@ static vaddr_t area_idx2va(struct tee_pager_area *area, size_t idx) return (idx << SMALL_PAGE_SHIFT) + (area->base & ~CORE_MMU_PGDIR_MASK); } +static void area_tlbi_entry(struct tee_pager_area *area, size_t idx) +{ + vaddr_t va = area_idx2va(area, idx); + +#if defined(CFG_PAGED_USER_TA) + assert(area->pgt); + if (area->pgt->ctx) { + uint32_t asid = to_user_ta_ctx(area->pgt->ctx)->vm_info->asid; + + tlbi_mva_asid(va, asid); + return; + } +#endif + tlbi_mva_allasid(va); +} + static void pmem_unmap(struct tee_pager_pmem *pmem, struct pgt *only_this_pgt) { struct tee_pager_area *area = NULL; @@ -431,7 +447,7 @@ static void pmem_unmap(struct tee_pager_pmem *pmem, struct pgt *only_this_pgt) if (a & TEE_MATTR_VALID_BLOCK) { area_set_entry(area, tblidx, 0, 0); pgt_dec_used_entries(area->pgt); - tlbi_mva_allasid(area_idx2va(area, tblidx)); + area_tlbi_entry(area, tblidx); } } } @@ -821,7 +837,7 @@ static void rem_area(struct tee_pager_area_head *area_head, continue; area_set_entry(area, idx, 0, 0); - tlbi_mva_allasid(area_idx2va(area, idx)); + area_tlbi_entry(area, idx); pgt_dec_used_entries(area->pgt); } @@ -921,7 +937,7 @@ bool tee_pager_set_uta_area_attr(struct user_ta_ctx *utc, vaddr_t base, if (a == f) continue; area_set_entry(area, tblidx, 0, 0); - tlbi_mva_allasid(area_idx2va(area, tblidx)); + area_tlbi_entry(area, tblidx); pmem->flags &= ~PMEM_FLAG_HIDDEN; if (pmem_is_dirty(pmem)) @@ -1190,7 +1206,7 @@ static bool pager_update_permissions(struct tee_pager_area *area, pmem->flags |= PMEM_FLAG_DIRTY; area_set_entry(area, pgidx, pa, get_area_mattr(area->flags)); - tlbi_mva_allasid(ai->va & ~SMALL_PAGE_MASK); + area_tlbi_entry(area, pgidx); } } else { @@ -1359,14 +1375,14 @@ bool tee_pager_handle_fault(struct abort_info *ai) /* Set a temporary read-only mapping */ area_set_entry(area, tblidx, pa, attr & ~mask); - tlbi_mva_allasid(page_va); + area_tlbi_entry(area, tblidx); dcache_clean_range_pou(va, SMALL_PAGE_SIZE); cache_op_inner(ICACHE_INVALIDATE, NULL, 0); /* Set the final mapping */ area_set_entry(area, tblidx, pa, attr); - tlbi_mva_allasid(page_va); + area_tlbi_entry(area, tblidx); } else { area_set_entry(area, tblidx, pa, attr); /* From 2b0ebf9533053a4174b7fd419c5e2e989e6f06e7 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Mon, 29 Apr 2019 16:47:07 +0200 Subject: [PATCH 10/10] core: pager: use icache_inv_user_range() Prior to this patch the entire icache was invalidated when icache invalidations was needed, even if it only was for a single page. This was needed to reach a stable state with regards to paging user TAs. With this patch a new function, icache_inv_user_range(), is used to invalidate pages used by user TAs and icache_inv_range() is used instead to invalidate kernel mode pages. Reviewed-by: Etienne Carriere Signed-off-by: Jens Wiklander --- core/arch/arm/mm/tee_pager.c | 66 +++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/core/arch/arm/mm/tee_pager.c b/core/arch/arm/mm/tee_pager.c index 821758f4969..fc7380bde82 100644 --- a/core/arch/arm/mm/tee_pager.c +++ b/core/arch/arm/mm/tee_pager.c @@ -966,7 +966,7 @@ bool tee_pager_set_uta_area_attr(struct user_ta_ctx *utc, vaddr_t base, assert(same_context(pmem)); dcache_clean_range_pou(va, SMALL_PAGE_SIZE); - cache_op_inner(ICACHE_INVALIDATE, NULL, 0); + icache_inv_user_range(va, SMALL_PAGE_SIZE); } } @@ -1020,6 +1020,7 @@ static bool tee_pager_unhide_page(struct tee_pager_area *area, struct tee_pager_pmem *pmem = pmem_find(area, tblidx); uint32_t a = get_area_mattr(area->flags); uint32_t attr = 0; + paddr_t pa = 0; if (!pmem) return false; @@ -1028,22 +1029,55 @@ static bool tee_pager_unhide_page(struct tee_pager_area *area, if (attr & TEE_MATTR_VALID_BLOCK) return false; - /* page is hidden, show and move to back */ + /* + * The page is hidden, or not not mapped yet. Unhide the page and + * move it to the tail. + * + * Since the page isn't mapped there doesn't exist a valid TLB entry + * for this address, so no TLB invalidation is required after setting + * the new entry. A DSB is needed though, to make the write visible. + * + * For user executable pages it's more complicated. Those pages can + * be shared between multiple TA mappings and thus populated by + * another TA. The reference manual states that: + * + * "instruction cache maintenance is required only after writing + * new data to a physical address that holds an instruction." + * + * So for hidden pages we would not need to invalidate i-cache, but + * for newly populated pages we do. Since we don't know which we + * have to assume the worst and always invalidate the i-cache. We + * don't need to clean the d-cache though, since that has already + * been done earlier. + * + * Additional bookkeeping to tell if the i-cache invalidation is + * needed or not is left as a future optimization. + */ /* If it's not a dirty block, then it should be read only. */ if (!pmem_is_dirty(pmem)) a &= ~(TEE_MATTR_PW | TEE_MATTR_UW); + pa = get_pmem_pa(pmem); pmem->flags &= ~PMEM_FLAG_HIDDEN; - area_set_entry(area, tblidx, get_pmem_pa(pmem), a); + if (area->flags & TEE_MATTR_UX) { + void *va = (void *)area_idx2va(area, tblidx); + + /* Set a temporary read-only mapping */ + assert(!(a & (TEE_MATTR_UW | TEE_MATTR_PW))); + area_set_entry(area, tblidx, pa, a & ~TEE_MATTR_UX); + dsb_ishst(); + + icache_inv_user_range(va, SMALL_PAGE_SIZE); + + /* Set the final mapping */ + area_set_entry(area, tblidx, pa, a); + area_tlbi_entry(area, tblidx); + } else { + area_set_entry(area, tblidx, pa, a); + dsb_ishst(); + } pgt_inc_used_entries(area->pgt); - /* - * Note that TLB invalidation isn't needed since - * there wasn't a valid mapping before. We should - * use a barrier though, to make sure that the - * change is visible. - */ - dsb_ishst(); TAILQ_REMOVE(&tee_pager_pmem_head, pmem, link); TAILQ_INSERT_TAIL(&tee_pager_pmem_head, pmem, link); @@ -1266,6 +1300,7 @@ bool tee_pager_handle_fault(struct abort_info *ai) vaddr_t page_va = ai->va & ~SMALL_PAGE_MASK; uint32_t exceptions; bool ret; + bool clean_user_cache = false; #ifdef TEE_PAGER_DEBUG_PRINT if (!abort_is_user_exception(ai)) @@ -1290,11 +1325,13 @@ bool tee_pager_handle_fault(struct abort_info *ai) /* check if the access is valid */ if (abort_is_user_exception(ai)) { area = find_uta_area(ai->va); - + clean_user_cache = true; } else { area = find_area(&tee_pager_area_head, ai->va); - if (!area) + if (!area) { area = find_uta_area(ai->va); + clean_user_cache = true; + } } if (!area || !area->pgt) { ret = false; @@ -1378,7 +1415,10 @@ bool tee_pager_handle_fault(struct abort_info *ai) area_tlbi_entry(area, tblidx); dcache_clean_range_pou(va, SMALL_PAGE_SIZE); - cache_op_inner(ICACHE_INVALIDATE, NULL, 0); + if (clean_user_cache) + icache_inv_user_range(va, SMALL_PAGE_SIZE); + else + icache_inv_range(va, SMALL_PAGE_SIZE); /* Set the final mapping */ area_set_entry(area, tblidx, pa, attr);