Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cache optimizations #3005

Merged
merged 10 commits into from
Jul 1, 2019
Merged

Cache optimizations #3005

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
1e4c834
arm.h: add CTR_WORD_SIZE
jenswi-linaro Apr 30, 2019
d61bcfe
core: add icache_inv_user_range()
jenswi-linaro Apr 29, 2019
2b405e8
core: cache_helpers_a{32,64}.S: remove section assignments
jenswi-linaro Apr 30, 2019
51ffb71
core: add dcache_clean_range_pou()
jenswi-linaro Apr 30, 2019
0105e1b
core: pager: use dcache_clean_range_pou()
jenswi-linaro Apr 30, 2019
c2c2439
core: arm32.h: add TLBI_{MVA_SHIFT,ASID_MASK}
jenswi-linaro Apr 30, 2019
70e5314
core: arm64.h: add tlbi_vale1is()
jenswi-linaro May 13, 2019
11c0157
core: add tlbi_mva_asid()
jenswi-linaro Apr 30, 2019
11d6ccc
core: pager: use tlbi_mva_asid() where applicable
jenswi-linaro Apr 30, 2019
2b0ebf9
core: pager: use icache_inv_user_range()
jenswi-linaro Apr 29, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/arch/arm/include/arm.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
#define CTR_L1IP_MASK 0x3
#define CTR_IMINLINE_SHIFT 0
#define CTR_IMINLINE_MASK 0xf
#define CTR_WORD_SIZE 4

#define ARM32_CPSR_MODE_MASK 0x1f
#define ARM32_CPSR_MODE_USR 0x10
Expand Down
2 changes: 2 additions & 0 deletions core/arch/arm/include/arm32.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@
#define TTBR_ASID_MASK 0xff
#define TTBR_ASID_SHIFT 48

#define TLBI_MVA_SHIFT 12
#define TLBI_ASID_MASK 0xff

#define FSR_LPAE BIT32(9)
#define FSR_WNR BIT32(11)
Expand Down
5 changes: 5 additions & 0 deletions core/arch/arm/include/arm64.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,11 @@ static inline void tlbi_vaae1is(uint64_t mva)
asm volatile ("tlbi vaae1is, %0" : : "r" (mva));
}

static inline void tlbi_vale1is(uint64_t mva)
{
asm volatile ("tlbi vale1is, %0" : : "r" (mva));
}

/*
* Templates for register read/write functions based on mrs/msr
*/
Expand Down
2 changes: 2 additions & 0 deletions core/arch/arm/include/kernel/cache_helpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@
void dcache_cleaninv_range(void *addr, size_t size);
void dcache_clean_range(void *addr, size_t size);
void dcache_inv_range(void *addr, size_t size);
void dcache_clean_range_pou(void *addr, size_t size);

void icache_inv_all(void);
void icache_inv_range(void *addr, size_t size);
void icache_inv_user_range(void *addr, size_t size);

void dcache_op_louis(unsigned long op_type);
void dcache_op_all(unsigned long op_type);
Expand Down
18 changes: 18 additions & 0 deletions core/arch/arm/include/kernel/tlb_helpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
void tlbi_all(void);
void tlbi_asid(unsigned long asid);
void tlbi_mva_allasid(unsigned long addr);

static inline void tlbi_mva_allasid_nosync(vaddr_t va)
{
#ifdef ARM64
Expand All @@ -23,6 +24,23 @@ static inline void tlbi_mva_allasid_nosync(vaddr_t va)
write_tlbimvaais(va);
#endif
}

static inline void tlbi_mva_asid(vaddr_t va, uint32_t asid)
{
uint32_t a = asid & TLBI_ASID_MASK;

dsb_ishst();
#ifdef ARM64
tlbi_vale1is((va >> TLBI_MVA_SHIFT) | SHIFT_U64(a, TLBI_ASID_SHIFT));
tlbi_vale1is((va >> TLBI_MVA_SHIFT) |
SHIFT_U64(a | 1, TLBI_ASID_SHIFT));
#else
write_tlbimvais((va & ~(BIT32(TLBI_MVA_SHIFT) - 1)) | a);
write_tlbimvais((va & ~(BIT32(TLBI_MVA_SHIFT) - 1)) | a | 1);
#endif
dsb_ish();
isb();
}
#endif /*!ASM*/

#endif /* TLB_HELPERS_H */
29 changes: 14 additions & 15 deletions core/arch/arm/kernel/cache_helpers_a32.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,20 @@
#include <kernel/cache_helpers.h>
#include <kernel/unwind.h>

#define WORD_SIZE 4

/*
* Cache line size helpers
*/
.macro dcache_line_size reg, tmp
read_ctr \tmp
ubfx \tmp, \tmp, #CTR_DMINLINE_SHIFT, #CTR_DMINLINE_WIDTH
mov \reg, #WORD_SIZE
mov \reg, #CTR_WORD_SIZE
lsl \reg, \reg, \tmp
.endm

.macro icache_line_size reg, tmp
read_ctr \tmp
and \tmp, \tmp, #CTR_IMINLINE_MASK
mov \reg, #WORD_SIZE
mov \reg, #CTR_WORD_SIZE
lsl \reg, \reg, \tmp
.endm

Expand All @@ -52,7 +50,6 @@ loop_\reg:
* size. 'r0' = addr, 'r1' = size
* ------------------------------------------
*/
.section .text.dcache_cleaninv_range
FUNC dcache_cleaninv_range , :
UNWIND( .fnstart)
do_dcache_maintenance_by_mva dccimvac
Expand All @@ -64,7 +61,6 @@ END_FUNC dcache_cleaninv_range
* 'r0' = addr, 'r1' = size
* ------------------------------------------
*/
.section .text.dcache_clean_range
FUNC dcache_clean_range , :
UNWIND( .fnstart)
do_dcache_maintenance_by_mva dccmvac
Expand All @@ -76,13 +72,24 @@ END_FUNC dcache_clean_range
* size. 'r0' = addr, 'r1' = size
* ------------------------------------------
*/
.section .text.dcache_inv_range
FUNC dcache_inv_range , :
UNWIND( .fnstart)
do_dcache_maintenance_by_mva dcimvac
UNWIND( .fnend)
END_FUNC dcache_inv_range


/* ------------------------------------------
* Clean from base address till size to point of unification
* 'r0' = addr, 'r1' = size
* ------------------------------------------
*/
FUNC dcache_clean_range_pou , :
UNWIND( .fnstart)
do_dcache_maintenance_by_mva dccmvau
UNWIND( .fnend)
END_FUNC dcache_clean_range_pou

/* ----------------------------------------------------------------
* Data cache operations by set/way to the level specified
*
Expand All @@ -108,7 +115,6 @@ END_FUNC dcache_inv_range
b do_dcsw_op
.endm

.section .text.do_dcsw_op
LOCAL_FUNC do_dcsw_op , :
UNWIND( .fnstart)
push {r4-r12,lr}
Expand Down Expand Up @@ -172,7 +178,6 @@ END_FUNC do_dcsw_op
* DCACHE_OP_CLEAN), as defined in cache_helpers.h
* ---------------------------------------------------------------
*/
.section .text.dcache_op_louis
FUNC dcache_op_louis , :
UNWIND( .fnstart)
dcsw_op #CLIDR_LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT
Expand All @@ -187,7 +192,6 @@ END_FUNC dcache_op_louis
* DCACHE_OP_CLEAN), as defined in cache_helpers.h
* ---------------------------------------------------------------
*/
.section .text.dcache_op_all
FUNC dcache_op_all , :
UNWIND( .fnstart)
dcsw_op #CLIDR_LOC_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT
Expand Down Expand Up @@ -215,7 +219,6 @@ END_FUNC dcache_op_all
* DCACHE_OP_CLEAN), as defined in cache_helpers.h
* ---------------------------------------------------------------
*/
.section .text.dcache_op_level1
FUNC dcache_op_level1 , :
UNWIND( .fnstart)
dcsw_op_level #(1 << CSSELR_LEVEL_SHIFT)
Expand All @@ -230,7 +233,6 @@ END_FUNC dcache_op_level1
* DCACHE_OP_CLEAN), as defined in cache_helpers.h
* ---------------------------------------------------------------
*/
.section .text.dcache_op_level2
FUNC dcache_op_level2 , :
UNWIND( .fnstart)
dcsw_op_level #(2 << CSSELR_LEVEL_SHIFT)
Expand All @@ -245,14 +247,12 @@ END_FUNC dcache_op_level2
* DCACHE_OP_CLEAN), as defined in cache_helpers.h
* ---------------------------------------------------------------
*/
.section .text.dcache_op_level3
FUNC dcache_op_level3 , :
UNWIND( .fnstart)
dcsw_op_level #(3 << CSSELR_LEVEL_SHIFT)
UNWIND( .fnend)
END_FUNC dcache_op_level3

.section .text.icache_inv_all
FUNC icache_inv_all , :
UNWIND( .fnstart)
/* Invalidate Entire Instruction Cache (and branch predictors) */
Expand All @@ -270,7 +270,6 @@ END_FUNC icache_inv_all
* size. 'r0' = addr, 'r1' = size
* ------------------------------------------
*/
.section .text.icache_inv_range
FUNC icache_inv_range , :
UNWIND( .fnstart)
icache_line_size r2, r3
Expand Down
26 changes: 10 additions & 16 deletions core/arch/arm/kernel/cache_helpers_a64.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,17 @@
#include <arm.h>
#include <asm.S>

#define WORD_SIZE 4


.macro dcache_line_size reg, tmp
mrs \tmp, ctr_el0
ubfx \tmp, \tmp, #CTR_DMINLINE_SHIFT, #CTR_DMINLINE_WIDTH
mov \reg, #WORD_SIZE
mov \reg, #CTR_WORD_SIZE
lsl \reg, \reg, \tmp
.endm

.macro icache_line_size reg, tmp
mrs \tmp, ctr_el0
and \tmp, \tmp, #CTR_IMINLINE_MASK
mov \reg, #WORD_SIZE
mov \reg, #CTR_WORD_SIZE
lsl \reg, \reg, \tmp
.endm

Expand All @@ -46,7 +43,6 @@ loop_\op:
* size. 'x0' = addr, 'x1' = size
* ------------------------------------------
*/
.section .text.dcache_cleaninv_range
FUNC dcache_cleaninv_range , :
do_dcache_maintenance_by_mva civac
END_FUNC dcache_cleaninv_range
Expand All @@ -56,7 +52,6 @@ END_FUNC dcache_cleaninv_range
* 'x0' = addr, 'x1' = size
* ------------------------------------------
*/
.section .text.dcache_clean_range
FUNC dcache_clean_range , :
do_dcache_maintenance_by_mva cvac
END_FUNC dcache_clean_range
Expand All @@ -66,11 +61,18 @@ END_FUNC dcache_clean_range
* size. 'x0' = addr, 'x1' = size
* ------------------------------------------
*/
.section .text.dcache_inv_range
FUNC dcache_inv_range , :
do_dcache_maintenance_by_mva ivac
END_FUNC dcache_inv_range

/* ------------------------------------------
* Clean from base address till size to point of unification
* 'x0' = addr, 'x1' = size
* ------------------------------------------
*/
FUNC dcache_clean_range_pou , :
do_dcache_maintenance_by_mva cvau
END_FUNC dcache_clean_range_pou

/* ---------------------------------------------------------------
* Data cache operations by set/way to the level specified
Expand All @@ -96,7 +98,6 @@ END_FUNC dcache_inv_range
b do_dcsw_op
.endm

.section .text.do_dcsw_op
LOCAL_FUNC do_dcsw_op , :
cbz x3, exit
adr x14, dcsw_loop_table // compute inner loop address
Expand Down Expand Up @@ -157,12 +158,10 @@ dcsw_loop_table:
dcsw_loop csw
END_FUNC do_dcsw_op

.section .text.dcache_op_louis
FUNC dcache_op_louis , :
dcsw_op #CLIDR_LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT
END_FUNC dcache_op_louis

.section .text.dcache_op_all
FUNC dcache_op_all , :
dcsw_op #CLIDR_LOC_SHIFT, #CLIDR_FIELD_WIDTH, #CSSELR_LEVEL_SHIFT
END_FUNC dcache_op_all
Expand All @@ -186,7 +185,6 @@ END_FUNC dcache_op_all
* x0: The operation type (0-2), as defined in cache_helpers.h
* ---------------------------------------------------------------
*/
.section .text.dcache_op_level1
FUNC dcache_op_level1 , :
dcsw_op_level #(1 << CSSELR_LEVEL_SHIFT)
END_FUNC dcache_op_level1
Expand All @@ -198,7 +196,6 @@ END_FUNC dcache_op_level1
* x0: The operation type (0-2), as defined in cache_helpers.h
* ---------------------------------------------------------------
*/
.section .text.dcache_op_level2
FUNC dcache_op_level2 , :
dcsw_op_level #(2 << CSSELR_LEVEL_SHIFT)
END_FUNC dcache_op_level2
Expand All @@ -210,12 +207,10 @@ END_FUNC dcache_op_level2
* x0: The operation type (0-2), as defined in cache_helpers.h
* ---------------------------------------------------------------
*/
.section .text.dcache_op_level3
FUNC dcache_op_level3 , :
dcsw_op_level #(3 << CSSELR_LEVEL_SHIFT)
END_FUNC dcache_op_level3

.section .text.icache_inv_all
FUNC icache_inv_all , :
/* Invalidate Entire Instruction Cache */
ic ialluis
Expand All @@ -226,7 +221,6 @@ FUNC icache_inv_all , :
ret
END_FUNC icache_inv_all

.section .text.icache_inv_range
FUNC icache_inv_range , :
icache_line_size x2, x3
add x1, x0, x1
Expand Down
Loading