arch/tile: use interrupt critical sections less

In general we want to avoid ever touching memory while within an
interrupt critical section, since the page fault path goes through
a different path from the hypervisor when in an interrupt critical
section, and we carefully decided with tilegx that we didn't need
to support this path in the kernel.  (On tilepro we did implement
that path as part of supporting atomic instructions in software.)

In practice we always need to touch the kernel stack, since that's
where we store the interrupt state before releasing the critical
section, but this change cleans up a few things.  The IRQ_ENABLE
macro is split up so that when we want to enable interrupts in a
deferred way (e.g. for cpu_idle or for interrupt return) we can
read the per-cpu enable mask before entering the critical section.
The cache-migration code is changed to use interrupt masking instead
of interrupt critical sections.  And, the interrupt-entry code is
changed so that we defer loading "tp" from per-cpu data until after
we have released the interrupt critical section.

Signed-off-by: Chris Metcalf <cmetcalf@tilera.com>

4 files changed, 25 insertions, 55 deletions

diff --git a/arch/tile/mm/init.c b/arch/tile/mm/init.c
index 6a9d20ddc34f..1e4633520b35 100644
--- a/arch/tile/mm/init.c
+++ b/arch/tile/mm/init.c
@@ -444,6 +444,7 @@ static pgd_t pgtables[PTRS_PER_PGD]
  */
 static void __init kernel_physical_mapping_init(pgd_t *pgd_base)
 {
+        unsigned long long irqmask;
         unsigned long address, pfn;
         pmd_t *pmd;
         pte_t *pte;
@@ -633,10 +634,13 @@ static void __init kernel_physical_mapping_init(pgd_t *pgd_base)
          *  - install pgtables[] as the real page table
          *  - flush the TLB so the new page table takes effect
          */
+        irqmask = interrupt_mask_save_mask();
+        interrupt_mask_set_mask(-1ULL);
         rc = flush_and_install_context(__pa(pgtables),
                                        init_pgprot((unsigned long)pgtables),
                                        __get_cpu_var(current_asid),
                                        cpumask_bits(my_cpu_mask));
+        interrupt_mask_restore_mask(irqmask);
         BUG_ON(rc != 0);
 
         /* Copy the page table back to the normal swapper_pg_dir. */
diff --git a/arch/tile/mm/migrate.h b/arch/tile/mm/migrate.h
index cd45a0837fa6..91683d97917e 100644
--- a/arch/tile/mm/migrate.h
+++ b/arch/tile/mm/migrate.h
@@ -24,6 +24,9 @@
 /*
  * This function is used as a helper when setting up the initial
  * page table (swapper_pg_dir).
+ *
+ * You must mask ALL interrupts prior to invoking this code, since
+ * you can't legally touch the stack during the cache flush.
  */
 extern int flush_and_install_context(HV_PhysAddr page_table, HV_PTE access,
                                      HV_ASID asid,
@@ -39,6 +42,9 @@ extern int flush_and_install_context(HV_PhysAddr page_table, HV_PTE access,
  *
  * Note that any non-NULL pointers must not point to the page that
  * is handled by the stack_pte itself.
+ *
+ * You must mask ALL interrupts prior to invoking this code, since
+ * you can't legally touch the stack during the cache flush.
  */
 extern int homecache_migrate_stack_and_flush(pte_t stack_pte, unsigned long va,
                                      size_t length, pte_t *stack_ptep,
diff --git a/arch/tile/mm/migrate_32.S b/arch/tile/mm/migrate_32.S
index ac01a7cdf77f..5305814bf187 100644
--- a/arch/tile/mm/migrate_32.S
+++ b/arch/tile/mm/migrate_32.S
@@ -40,8 +40,7 @@
 #define FRAME_R32       16
 #define FRAME_R33       20
 #define FRAME_R34       24
-#define FRAME_R35       28
-#define FRAME_SIZE      32
+#define FRAME_SIZE      28
 
 
 
@@ -66,12 +65,11 @@
 #define r_my_cpumask    r5
 
 /* Locals (callee-save); must not be more than FRAME_xxx above. */
-#define r_save_ics      r30
-#define r_context_lo    r31
-#define r_context_hi    r32
-#define r_access_lo     r33
-#define r_access_hi     r34
-#define r_asid          r35
+#define r_context_lo    r30
+#define r_context_hi    r31
+#define r_access_lo     r32
+#define r_access_hi     r33
+#define r_asid          r34
 
 STD_ENTRY(flush_and_install_context)
         /*
@@ -104,11 +102,7 @@ STD_ENTRY(flush_and_install_context)
          sw r_tmp, r33
          addi r_tmp, sp, FRAME_R34
         }
-        {
-         sw r_tmp, r34
-         addi r_tmp, sp, FRAME_R35
-        }
-        sw r_tmp, r35
+        sw r_tmp, r34
 
         /* Move some arguments to callee-save registers. */
         {
@@ -121,13 +115,6 @@ STD_ENTRY(flush_and_install_context)
         }
         move r_asid, r_asid_in
 
-        /* Disable interrupts, since we can't use our stack. */
-        {
-         mfspr r_save_ics, INTERRUPT_CRITICAL_SECTION
-         movei r_tmp, 1
-        }
-        mtspr INTERRUPT_CRITICAL_SECTION, r_tmp
-
         /* First, flush our L2 cache. */
         {
          move r0, zero  /* cache_pa */
@@ -163,7 +150,7 @@ STD_ENTRY(flush_and_install_context)
         }
         {
          move r4, r_asid
-         movei r5, HV_CTX_DIRECTIO
+         moveli r5, HV_CTX_DIRECTIO | CTX_PAGE_FLAG
         }
         jal hv_install_context
         bnz r0, .Ldone
@@ -175,9 +162,6 @@ STD_ENTRY(flush_and_install_context)
         }
 
 .Ldone:
-        /* Reset interrupts back how they were before. */
-        mtspr INTERRUPT_CRITICAL_SECTION, r_save_ics
-
         /* Restore the callee-saved registers and return. */
         addli lr, sp, FRAME_SIZE
         {
@@ -202,10 +186,6 @@ STD_ENTRY(flush_and_install_context)
         }
         {
          lw r34, r_tmp
-         addli r_tmp, sp, FRAME_R35
-        }
-        {
-         lw r35, r_tmp
          addi sp, sp, FRAME_SIZE
         }
         jrp lr
diff --git a/arch/tile/mm/migrate_64.S b/arch/tile/mm/migrate_64.S
index e76fea688beb..1d15b10833d1 100644
--- a/arch/tile/mm/migrate_64.S
+++ b/arch/tile/mm/migrate_64.S
@@ -38,8 +38,7 @@
 #define FRAME_R30       16
 #define FRAME_R31       24
 #define FRAME_R32       32
-#define FRAME_R33       40
-#define FRAME_SIZE      48
+#define FRAME_SIZE      40
 
 
 
@@ -60,10 +59,9 @@
 #define r_my_cpumask    r3
 
 /* Locals (callee-save); must not be more than FRAME_xxx above. */
-#define r_save_ics      r30
-#define r_context       r31
-#define r_access        r32
-#define r_asid          r33
+#define r_context       r30
+#define r_access        r31
+#define r_asid          r32
 
 /*
  * Caller-save locals and frame constants are the same as
@@ -93,11 +91,7 @@ STD_ENTRY(flush_and_install_context)
          st r_tmp, r31
          addi r_tmp, sp, FRAME_R32
         }
-        {
-         st r_tmp, r32
-         addi r_tmp, sp, FRAME_R33
-        }
-        st r_tmp, r33
+        st r_tmp, r32
 
         /* Move some arguments to callee-save registers. */
         {
@@ -106,13 +100,6 @@ STD_ENTRY(flush_and_install_context)
         }
         move r_asid, r_asid_in
 
-        /* Disable interrupts, since we can't use our stack. */
-        {
-         mfspr r_save_ics, INTERRUPT_CRITICAL_SECTION
-         movei r_tmp, 1
-        }
-        mtspr INTERRUPT_CRITICAL_SECTION, r_tmp
-
         /* First, flush our L2 cache. */
         {
          move r0, zero  /* cache_pa */
@@ -147,7 +134,7 @@ STD_ENTRY(flush_and_install_context)
         }
         {
          move r2, r_asid
-         movei r3, HV_CTX_DIRECTIO
+         moveli r3, HV_CTX_DIRECTIO | CTX_PAGE_FLAG
         }
         jal hv_install_context
         bnez r0, 1f
@@ -158,10 +145,7 @@ STD_ENTRY(flush_and_install_context)
          jal hv_flush_all
         }
 
-1:      /* Reset interrupts back how they were before. */
-        mtspr INTERRUPT_CRITICAL_SECTION, r_save_ics
-
-        /* Restore the callee-saved registers and return. */
+1:      /* Restore the callee-saved registers and return. */
         addli lr, sp, FRAME_SIZE
         {
          ld lr, lr
@@ -177,10 +161,6 @@ STD_ENTRY(flush_and_install_context)
         }
         {
          ld r32, r_tmp
-         addli r_tmp, sp, FRAME_R33
-        }
-        {
-         ld r33, r_tmp
          addi sp, sp, FRAME_SIZE
         }
         jrp lr