ENGR00309881 ARM:imx6: Disable L1 and L2 when DDR is in self-refresh.

When switching to page tables in IRAM, we are switching from a cacheable
to non-cacheable page table. Based on recommendation from ARM, we need to
ensure that branch prediction, L1 data nd L2 are disabled when DDR is put
into self-refresh.

Signed-off-by: Ranjani Vaidyanathan <Ranjani.Vaidyanathan@freescale.com>

5 files changed, 201 insertions, 38 deletions

diff --git a/arch/arm/mach-imx/ddr3_freq_imx6.S b/arch/arm/mach-imx/ddr3_freq_imx6.S
index cbba4c2a9322..26a601805b8a 100644
--- a/arch/arm/mach-imx/ddr3_freq_imx6.S
+++ b/arch/arm/mach-imx/ddr3_freq_imx6.S
@@ -358,15 +358,23 @@ ddr_freq_change:
          *     and 2-4G is translated by TTBR1.
          */
 
+        ldr     r6, =iram_tlb_phys_addr
+        ldr     r7, [r6]
+
         /* Flush the Branch Target Address Cache (BTAC) */
         ldr     r6, =0x0
         mcr     p15, 0, r6, c7, c1, 6
-        ldr     r6, =iram_tlb_phys_addr
-        ldr     r6, [r6]
+
+        /* Disable Branch Prediction, Z bit in SCTLR. */
+        mrc     p15, 0, r6, c1, c0, 0
+        bic     r6, r6, #0x800
+        mcr     p15, 0, r6, c1, c0, 0
+
         dsb
         isb
+
         /* Store the IRAM table in TTBR1 */
-        mcr     p15, 0, r6, c2, c0, 1
+        mcr     p15, 0, r7, c2, c0, 1
 
         /* Read TTBCR and set PD0=1, N = 1 */
         mrc     p15, 0, r6, c2, c0, 2
@@ -379,22 +387,37 @@ ddr_freq_change:
         /* flush the TLB */
         ldr     r6, =0x0
         mcr     p15, 0, r6, c8, c3, 0
+
         dsb
         isb
 
-        ldr     r5, =IMX_IO_P2V(MX6Q_MMDC_P0_BASE_ADDR)
-        ldr     r6, =IMX_IO_P2V(MX6Q_CCM_BASE_ADDR)
-        ldr     r7, =IMX_IO_P2V(MX6Q_IOMUXC_BASE_ADDR)
-        ldr     r12, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
+        /* Disable L1 data cache. */
+        mrc     p15, 0, r6, c1, c0, 0
+        bic     r6, r6, #0x4
+        mcr     p15, 0, r6, c1, c0, 0
 
 #ifdef CONFIG_CACHE_L2X0
         /*
-          * Make sure the L2 buffers are drained.
-          * Sync operation on L2 drains the buffers.
-          */
-        mov    r1, #0x0
-        str      r1, [r12, #L2_CACHE_SYNC]
+         * Make sure the L2 buffers are drained.
+         * Sync operation on L2 drains the buffers.
+         */
+        ldr     r12, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
+        mov     r1, #0x0
+        str     r1, [r12, #L2_CACHE_SYNC]
+
+        /* Disable L2. */
+        str     r1, [r12, #0x100]
+
+        /*
+         * The second dsb might be needed to keep cache sync (device write)
+         * ordering with the memory accesses before it.
+         */
+        dsb
+        isb
 #endif
+        ldr     r5, =IMX_IO_P2V(MX6Q_MMDC_P0_BASE_ADDR)
+        ldr     r6, =IMX_IO_P2V(MX6Q_CCM_BASE_ADDR)
+        ldr     r7, =IMX_IO_P2V(MX6Q_IOMUXC_BASE_ADDR)
 
         /* disable automatic power saving. */
         ldr     r0, [r5, #MMDC0_MAPSR]
@@ -887,9 +910,23 @@ poll_conreq_clear_2:
         beq     poll_conreq_clear_2
 
 done:
+
+#ifdef CONFIG_CACHE_L2X0
+        /* Enable L2. */
+        ldr     r7, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
+        ldr     r6, =0x1
+        str     r6, [r7, #0x100]
+#endif
+
+        /* Enable L1 data cache. */
+        mrc     p15, 0, r6, c1, c0, 0
+        orr     r6, r6, #0x4
+        mcr     p15, 0, r6, c1, c0, 0
+
         /* Restore the TTBCR */
         dsb
         isb
+
         /* Read TTBCR and set PD0=0, N = 0 */
         mrc     p15, 0, r6, c2, c0, 2
         bic     r6, r6, #0x11
@@ -900,9 +937,19 @@ done:
         /* flush the TLB */
         ldr     r6, =0x0
         mcr     p15, 0, r6, c8, c3, 0
+
         dsb
         isb
 
+        /* Enable Branch Prediction, Z bit in SCTLR. */
+        mrc     p15, 0, r6, c1, c0, 0
+        orr     r6, r6, #0x800
+        mcr     p15, 0, r6, c1, c0, 0
+
+        /* Flush the Branch Target Address Cache (BTAC) */
+        ldr     r6, =0x0
+        mcr     p15, 0, r6, c7, c1, 6
+
         /* restore registers */
 
         ldmfd   sp!, {r4-r12}
diff --git a/arch/arm/mach-imx/imx6sl_wfi.S b/arch/arm/mach-imx/imx6sl_wfi.S
index f8556e67631e..e4ce82faaa8e 100644
--- a/arch/arm/mach-imx/imx6sl_wfi.S
+++ b/arch/arm/mach-imx/imx6sl_wfi.S
@@ -186,17 +186,24 @@ mx6sl_lpm_wfi:
          * 4. Set TTBR0.N=1, implying 0-2G is translated by TTBR0
          *     and 2-4G is translated by TTBR1.
          */
+
+        ldr     r6, =iram_tlb_phys_addr
+        ldr     r7, [r6]
+
         /* Flush the BTAC. */
         ldr     r6, =0x0
         mcr     p15, 0, r6, c7, c1, 6
 
-        ldr     r6, =iram_tlb_phys_addr
-        ldr     r6, [r6]
+        /* Disable Branch Prediction, Z bit in SCTLR. */
+        mrc     p15, 0, r6, c1, c0, 0
+        bic     r6, r6, #0x800
+        mcr     p15, 0, r6, c1, c0, 0
+
         dsb
         isb
 
         /* Store the IRAM table in TTBR1 */
-        mcr     p15, 0, r6, c2, c0, 1
+        mcr     p15, 0, r7, c2, c0, 1
 
         /* Read TTBCR and set PD0=1, N = 1 */
         mrc     p15, 0, r6, c2, c0, 2
@@ -210,29 +217,25 @@ mx6sl_lpm_wfi:
         ldr     r6, =0x0
         mcr     p15, 0, r6, c8, c3, 0
 
-        dsb
-        isb
-
-        ldr     r1, =IMX_IO_P2V(MX6Q_IOMUXC_BASE_ADDR)
-        ldr     r3, =IMX_IO_P2V(MX6Q_ANATOP_BASE_ADDR)
-        ldr     r2, =IMX_IO_P2V(MX6Q_CCM_BASE_ADDR)
-        ldr     r8, =IMX_IO_P2V(MX6Q_MMDC_P0_BASE_ADDR)
-        ldr     r7, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
-
-        /* Store the original ARM PODF. */
-        ldr     r0, [r2, #0x10]
+        /* Disable L1 data cache. */
+        mrc     p15, 0, r6, c1, c0, 0
+        bic     r6, r6, #0x4
+        mcr     p15, 0, r6, c1, c0, 0
 
-        /* Drain all the L1 buffers. */
         dsb
+        isb
 
 #ifdef CONFIG_CACHE_L2X0
         /*
          * Need to make sure the buffers in L2 are drained.
          * Performing a sync operation does this.
          */
+        ldr     r7, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
         mov     r6, #0x0
         str     r6, [r7, #0x730]
-#endif
+
+        /* Disable L2. */
+        str     r6, [r7, #0x100]
 
         /*
          * The second dsb might be needed to keep cache sync (device write)
@@ -240,6 +243,16 @@ mx6sl_lpm_wfi:
          */
         dsb
         isb
+#endif
+
+
+        ldr     r1, =IMX_IO_P2V(MX6Q_IOMUXC_BASE_ADDR)
+        ldr     r3, =IMX_IO_P2V(MX6Q_ANATOP_BASE_ADDR)
+        ldr     r2, =IMX_IO_P2V(MX6Q_CCM_BASE_ADDR)
+        ldr     r8, =IMX_IO_P2V(MX6Q_MMDC_P0_BASE_ADDR)
+
+        /* Store the original ARM PODF. */
+        ldr     r0, [r2, #0x10]
 
         /* Save the DDR IO state. */
         sl_ddr_io_save
@@ -595,7 +608,7 @@ audio_arm_clk_restore:
         str     r6, [r2, #0xC]
 
 wfi_restore:
-        /* get suspend_iram_base */
+        /* get wfi_iram_base */
         mov     r9, r10
         add     r9, r9, #MX6SL_WFI_IRAM_CODE_SIZE
 
@@ -637,10 +650,23 @@ poll_dvfs_clear_1:
         bic     r6, r6, #0x100
         str     r6, [r8, #0x410]
 
+#ifdef CONFIG_CACHE_L2X0
+        /* Enable L2. */
+        ldr     r7, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
+        ldr     r6, =0x1
+        str     r6, [r7, #0x100]
+#endif
+
+        /* Enable L1 data cache. */
+        mrc     p15, 0, r6, c1, c0, 0
+        orr     r6, r6, #0x4
+        mcr     p15, 0, r6, c1, c0, 0
+
         /* Restore the TTBCR */
 
         dsb
         isb
+
         /* Read TTBCR and set PD0=0, N = 0 */
         mrc     p15, 0, r6, c2, c0, 2
         bic     r6, r6, #0x11
@@ -656,6 +682,15 @@ poll_dvfs_clear_1:
         dsb
         isb
 
+        /* Enable Branch Prediction, Z bit in SCTLR. */
+        mrc     p15, 0, r6, c1, c0, 0
+        orr     r6, r6, #0x800
+        mcr     p15, 0, r6, c1, c0, 0
+
+        /* Flush the Branch Target Address Cache (BTAC) */
+        ldr     r6, =0x0
+        mcr     p15, 0, r6, c7, c1, 6
+
         /*
          * Add these nops so that the
          * prefetcher will not try to get
@@ -693,7 +728,6 @@ poll_dvfs_clear_1:
         nop
         nop
 
-
         pop {r4-r12}
 
         /* Restore registers */
diff --git a/arch/arm/mach-imx/lpddr2_freq_imx6.S b/arch/arm/mach-imx/lpddr2_freq_imx6.S
index 3c1c076112ef..121a88bb62e8 100644
--- a/arch/arm/mach-imx/lpddr2_freq_imx6.S
+++ b/arch/arm/mach-imx/lpddr2_freq_imx6.S
@@ -359,16 +359,23 @@ ENTRY(mx6_lpddr2_freq_change)
          *     and 2-4G is translated by TTBR1.
          */
 
+        ldr     r6, =iram_tlb_phys_addr
+        ldr     r7, [r6]
+
+
         /* Flush the Branch Target Address Cache (BTAC) */
         ldr     r6, =0x0
         mcr     p15, 0, r6, c7, c1, 6
 
-        ldr     r6, =iram_tlb_phys_addr
-        ldr     r6, [r6]
+        /* Disable Branch Prediction, Z bit in SCTLR. */
+        mrc     p15, 0, r6, c1, c0, 0
+        bic     r6, r6, #0x800
+        mcr     p15, 0, r6, c1, c0, 0
+
         dsb
         isb
         /* Store the IRAM table in TTBR1 */
-        mcr     p15, 0, r6, c2, c0, 1
+        mcr     p15, 0, r7, c2, c0, 1
 
         /* Read TTBCR and set PD0=1, N = 1 */
         mrc     p15, 0, r6, c2, c0, 2
@@ -382,29 +389,39 @@ ENTRY(mx6_lpddr2_freq_change)
         ldr     r6, =0x0
         mcr     p15, 0, r6, c8, c3, 0
 
+        /* Disable L1 data cache. */
+        mrc     p15, 0, r6, c1, c0, 0
+        bic     r6, r6, #0x4
+        mcr     p15, 0, r6, c1, c0, 0
+
         dsb
         isb
 
-        ldr     r3, =IMX_IO_P2V(MX6Q_ANATOP_BASE_ADDR)
-        ldr     r2, =IMX_IO_P2V(MX6Q_CCM_BASE_ADDR)
-        ldr     r8, =IMX_IO_P2V(MX6Q_MMDC_P0_BASE_ADDR)
-        ldr     r7, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
 
 #ifdef CONFIG_CACHE_L2X0
         /*
          * Need to make sure the buffers in L2 are drained.
          * Performing a sync operation does this.
          */
+        ldr     r7, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
         mov     r6, #0x0
         str     r6, [r7, #0x730]
+
         /*
          * The second dsb might be needed to keep cache sync (device write)
          * ordering with the memory accesses before it.
          */
         dsb
         isb
+
+        /* Disable L2. */
+        str     r6, [r7, #0x100]
 #endif
 
+        ldr     r3, =IMX_IO_P2V(MX6Q_ANATOP_BASE_ADDR)
+        ldr     r2, =IMX_IO_P2V(MX6Q_CCM_BASE_ADDR)
+        ldr     r8, =IMX_IO_P2V(MX6Q_MMDC_P0_BASE_ADDR)
+
         /* Disable Automatic power savings. */
         ldr     r6, [r8, #0x404]
         orr     r6, r6, #0x01
@@ -498,9 +515,22 @@ skip_power_down:
         bic     r6, r6, #0x100
         str     r6, [r8, #0x410]
 
+#ifdef CONFIG_CACHE_L2X0
+        /* Enable L2. */
+        ldr     r7, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
+        ldr     r6, =0x1
+        str     r6, [r7, #0x100]
+#endif
+
+        /* Enable L1 data cache. */
+        mrc     p15, 0, r6, c1, c0, 0
+        orr     r6, r6, #0x4
+        mcr     p15, 0, r6, c1, c0, 0
+
         /* Restore the TTBCR */
         dsb
         isb
+
         /* Read TTBCR and set PD0=0, N = 0 */
         mrc     p15, 0, r6, c2, c0, 2
         bic     r6, r6, #0x11
@@ -515,6 +545,15 @@ skip_power_down:
         dsb
         isb
 
+        /* Enable Branch Prediction, Z bit in SCTLR. */
+        mrc     p15, 0, r6, c1, c0, 0
+        orr     r6, r6, #0x800
+        mcr     p15, 0, r6, c1, c0, 0
+
+        /* Flush the Branch Target Address Cache (BTAC) */
+        ldr     r6, =0x0
+        mcr     p15, 0, r6, c7, c1, 6
+
         nop
         nop
         nop
diff --git a/arch/arm/mach-imx/mx6.h b/arch/arm/mach-imx/mx6.h
index cac661ac7533..477c523745a2 100644
--- a/arch/arm/mach-imx/mx6.h
+++ b/arch/arm/mach-imx/mx6.h
@@ -44,7 +44,7 @@
 #define LPDDR2_FREQ_CODE_SIZE           0x600
 #define DDR3_FREQ_CODE_SIZE             0xC00
 #define DDR3_IOMUX_SETTINGS_SIZE        0x400
-#define MX6SL_WFI_IRAM_CODE_SIZE        0x600
+#define MX6SL_WFI_IRAM_CODE_SIZE        0x700
 
 #define MX6_SUSPEND_IRAM_ADDR_OFFSET            0
 #define DDR3_FREQ_CODE_ADDR_OFFSET              (MX6_SUSPEND_IRAM_ADDR_OFFSET + MX6_SUSPEND_IRAM_SIZE)
diff --git a/arch/arm/mach-imx/suspend-imx6.S b/arch/arm/mach-imx/suspend-imx6.S
index f21e1359bc5c..620181a4886b 100644
--- a/arch/arm/mach-imx/suspend-imx6.S
+++ b/arch/arm/mach-imx/suspend-imx6.S
@@ -640,6 +640,11 @@ ddr_io_save_dsm_done:
         ldr r6, =0x0
         mcr p15, 0, r6, c7, c1, 6
 
+        /* Disable Branch Prediction, Z bit in SCTLR. */
+        mrc     p15, 0, r6, c1, c0, 0
+        bic     r6, r6, #0x800
+        mcr     p15, 0, r6, c1, c0, 0
+
         ldr r6, =iram_tlb_phys_addr
         ldr r6, [r6]
         dsb
@@ -659,6 +664,23 @@ ddr_io_save_dsm_done:
         ldr r6, =0x0
         mcr p15, 0, r6, c8, c3, 0
 
+        /* Disable L1 data cache. */
+        mrc     p15, 0, r6, c1, c0, 0
+        bic     r6, r6, #0x4
+        mcr     p15, 0, r6, c1, c0, 0
+
+        dsb
+        isb
+
+#ifdef CONFIG_CACHE_L2X0
+        ldr     r8, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
+        mov     r6, #0x0
+        str     r6, [r8, #0x100]
+
+        dsb
+        isb
+#endif
+
         ldr     r8, =IMX_IO_P2V(MX6Q_MMDC_P0_BASE_ADDR)
         /*
          * put DDR explicitly into self-refresh and
@@ -861,6 +883,18 @@ poll_dvfs_clear_2:
         bic     r7, r7, #0x1
         str     r7, [r8, #MX6Q_MMDC_MAPSR]
 
+#ifdef CONFIG_CACHE_L2X0
+        /* Enable L2. */
+        ldr     r8, =IMX_IO_P2V(MX6Q_L2_BASE_ADDR)
+        ldr     r7, =0x1
+        str     r7, [r8, #0x100]
+#endif
+
+        /* Enable L1 data cache. */
+        mrc     p15, 0, r6, c1, c0, 0
+        orr     r6, r6, #0x4
+        mcr     p15, 0, r6, c1, c0, 0
+
         /* Restore TTBCR */
         dsb
         isb
@@ -875,6 +909,15 @@ poll_dvfs_clear_2:
         ldr     r6, =0x0
         mcr     p15, 0, r6, c8, c3, 0
 
+        /* Enable Branch Prediction, Z bit in SCTLR. */
+        mrc     p15, 0, r6, c1, c0, 0
+        orr     r6, r6, #0x800
+        mcr     p15, 0, r6, c1, c0, 0
+
+        /* Flush the Branch Target Address Cache (BTAC) */
+        ldr     r6, =0x0
+        mcr     p15, 0, r6, c7, c1, 6
+
         pop {r4-r12}
 
         /* return to suspend finish */