Implement workaround for AT speculative behaviour

During context switching from higher EL (EL2 or higher)
to lower EL can cause incorrect translation in TLB due to
speculative execution of AT instruction using out-of-context
translation regime.

Workaround is implemented as below during EL's (EL1 or EL2)
"context_restore" operation:
1. Disable page table walk using SCTLR.M and TCR.EPD0 & EPD1
   bits for EL1 or EL2 (stage1 and stage2 disabled)
2. Save all system registers except TCR and SCTLR (for EL1 and EL2)
3. Do memory barrier operation (isb) to ensure all
   system register writes are done.
4. Restore TCR and SCTLR registers (for EL1 and EL2)

Errata details are available for various CPUs as below:
Cortex-A76: 1165522
Cortex-A72: 1319367
Cortex-A57: 1319537
Cortex-A55: 1530923
Cortex-A53: 1530924

More details can be found in mail-chain:
https://lists.trustedfirmware.org/pipermail/tf-a/2020-April/000445.html

Currently, Workaround is implemented as build option which is default
disabled.

Signed-off-by: Manish V Badarkhe <Manish.Badarkhe@arm.com>
Change-Id: If8545e61f782cb0c2dda7ffbaf50681c825bd2f0

Makefile

@@ -891,6 +891,7 @@ $(eval $(call assert_boolean,BL2_INV_DCACHE))
 $(eval $(call assert_boolean,USE_SPINLOCK_CAS))
 $(eval $(call assert_boolean,ENCRYPT_BL31))
 $(eval $(call assert_boolean,ENCRYPT_BL32))
+$(eval $(call assert_boolean,ERRATA_SPECULATIVE_AT))
 
 $(eval $(call assert_numeric,ARM_ARCH_MAJOR))
 $(eval $(call assert_numeric,ARM_ARCH_MINOR))
@@ -967,6 +968,7 @@ $(eval $(call add_define,BL2_AT_EL3))
 $(eval $(call add_define,BL2_IN_XIP_MEM))
 $(eval $(call add_define,BL2_INV_DCACHE))
 $(eval $(call add_define,USE_SPINLOCK_CAS))
+$(eval $(call add_define,ERRATA_SPECULATIVE_AT))
 
 ifeq (${SANITIZE_UB},trap)
         $(eval $(call add_define,MONITOR_TRAPS))

docs/getting_started/build-options.rst

@@ -673,6 +673,29 @@ Common build options
    default value of this flag is ``no``. Note this option must be enabled only
    for ARM architecture greater than Armv8.5-A.
 
+-  ``ERRATA_SPECULATIVE_AT``: This flag enables/disables page table walk during
+   context restore as speculative AT instructions using an out-of-context
+   translation regime could cause subsequent requests to generate an incorrect
+   translation.
+   System registers are not updated during context save, hence this workaround
+   need not be applied in the context save path.
+
+   This boolean option enables errata for all below CPUs.
+
+   +---------+--------------+
+   | Errata  |      CPU     |
+   +=========+==============+
+   | 1165522 |  Cortex-A76  |
+   +---------+--------------+
+   | 1319367 |  Cortex-A72  |
+   +---------+--------------+
+   | 1319537 |  Cortex-A57  |
+   +---------+--------------+
+   | 1530923 |  Cortex-A55  |
+   +---------+--------------+
+   | 1530924 |  Cortex-A53  |
+   +---------+--------------+
+
 GICv3 driver options
 --------------------
 

include/arch/aarch64/arch.h

@@ -381,6 +381,7 @@
 /* HCR definitions */
 #define HCR_API_BIT		(ULL(1) << 41)
 #define HCR_APK_BIT		(ULL(1) << 40)
+#define HCR_E2H_BIT		(ULL(1) << 34)
 #define HCR_TGE_BIT		(ULL(1) << 27)
 #define HCR_RW_SHIFT		U(31)
 #define HCR_RW_BIT		(ULL(1) << HCR_RW_SHIFT)

lib/el3_runtime/aarch64/context.S

@@ -234,6 +234,21 @@ endfunc el2_sysregs_context_save
  */
 func el2_sysregs_context_restore
 
+#if ERRATA_SPECULATIVE_AT
+/* Clear EPD0 and EPD1 bit and M bit to disable PTW */
+	mrs	x9, hcr_el2
+	tst	x9, #HCR_E2H_BIT
+	bne	1f
+	mrs	x9, tcr_el2
+	orr	x9, x9, #TCR_EPD0_BIT
+	orr	x9, x9, #TCR_EPD1_BIT
+	msr	tcr_el2, x9
+1:	mrs	x9, sctlr_el2
+	bic	x9, x9, #SCTLR_M_BIT
+	msr	sctlr_el2, x9
+	isb
+#endif
+
 	ldp	x9, x10, [x0, #CTX_ACTLR_EL2]
 	msr	actlr_el2, x9
 	msr	afsr0_el2, x10
@@ -282,17 +297,15 @@ func el2_sysregs_context_restore
 	msr	mair_el2, x15
 	msr	mdcr_el2, x16
 
-	ldp	x17, x9, [x0, #CTX_PMSCR_EL2]
+	ldr	x17, [x0, #CTX_PMSCR_EL2]
 	msr	PMSCR_EL2, x17
-	msr	sctlr_el2, x9
 
 	ldp	x10, x11, [x0, #CTX_SPSR_EL2]
 	msr	spsr_el2, x10
 	msr	sp_el2, x11
 
-	ldp	x12, x13, [x0, #CTX_TCR_EL2]
-	msr	tcr_el2, x12
-	msr	tpidr_el2, x13
+	ldr	x12, [x0, #CTX_TPIDR_EL2]
+	msr	tpidr_el2, x12
 
 	ldp	x14, x15, [x0, #CTX_TTBR0_EL2]
 	msr	ttbr0_el2, x14
@@ -404,6 +417,19 @@ func el2_sysregs_context_restore
 	msr	scxtnum_el2, x9
 #endif
 
+#if ERRATA_SPECULATIVE_AT
+/*
+ * Make sure all registers are stored successfully except
+ * SCTLR_EL2 and TCR_EL2
+ */
+	isb
+#endif
+
+	ldr	x9, [x0, #CTX_SCTLR_EL2]
+	msr	sctlr_el2, x9
+	ldr	x9, [x0, #CTX_TCR_EL2]
+	msr	tcr_el2, x9
+
 	ret
 endfunc el2_sysregs_context_restore
 
@@ -515,12 +541,22 @@ endfunc el1_sysregs_context_save
  */
 func el1_sysregs_context_restore
 
+#if ERRATA_SPECULATIVE_AT
+	mrs	x9, tcr_el1
+	orr	x9, x9, #TCR_EPD0_BIT
+	orr	x9, x9, #TCR_EPD1_BIT
+	msr	tcr_el1, x9
+	mrs	x9, sctlr_el1
+	bic	x9, x9, #SCTLR_M_BIT
+	msr	sctlr_el1, x9
+	isb
+#endif
+
 	ldp	x9, x10, [x0, #CTX_SPSR_EL1]
 	msr	spsr_el1, x9
 	msr	elr_el1, x10
 
-	ldp	x15, x16, [x0, #CTX_SCTLR_EL1]
-	msr	sctlr_el1, x15
+	ldr	x16, [x0, #CTX_ACTLR_EL1]
 	msr	actlr_el1, x16
 
 	ldp	x17, x9, [x0, #CTX_CPACR_EL1]
@@ -539,9 +575,8 @@ func el1_sysregs_context_restore
 	msr	mair_el1, x14
 	msr	amair_el1, x15
 
-	ldp	x16, x17, [x0, #CTX_TCR_EL1]
-	msr	tcr_el1, x16
-	msr	tpidr_el1, x17
+	ldr	x16,[x0, #CTX_TPIDR_EL1]
+	msr	tpidr_el1, x16
 
 	ldp	x9, x10, [x0, #CTX_TPIDR_EL0]
 	msr	tpidr_el0, x9
@@ -597,6 +632,19 @@ func el1_sysregs_context_restore
 	msr	GCR_EL1, x14
 #endif
 
+#if ERRATA_SPECULATIVE_AT
+/*
+ * Make sure all registers are stored successfully except
+ * SCTLR_EL1 and TCR_EL1
+ */
+	isb
+#endif
+
+	ldr	x9, [x0, #CTX_SCTLR_EL1]
+	msr	sctlr_el1, x9
+	ldr	x9, [x0, #CTX_TCR_EL1]
+	msr	tcr_el1, x9
+
 	/* No explict ISB required here as ERET covers it */
 	ret
 endfunc el1_sysregs_context_restore

make_helpers/defaults.mk

@@ -293,3 +293,6 @@ CTX_INCLUDE_EL2_REGS		:= 0
 # than Armv8.5-A
 # By default it is set to "no"
 SUPPORT_STACK_MEMTAG		:= no
+
+# Select workaround for AT speculative behaviour.
+ERRATA_SPECULATIVE_AT           := 0