13 files changed, 616 insertions, 260 deletions

diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c
index 72dfd9e7de0f..cda06f88c66e 100644
--- a/arch/ia64/kernel/acpi.c
+++ b/arch/ia64/kernel/acpi.c
@@ -236,9 +236,7 @@ acpi_parse_iosapic (acpi_table_entry_header *header, const unsigned long end)
         if (BAD_MADT_ENTRY(iosapic, end))
                 return -EINVAL;
 
-        iosapic_init(iosapic->address, iosapic->global_irq_base);
-
-        return 0;
+        return iosapic_init(iosapic->address, iosapic->global_irq_base);
 }
 
 
@@ -772,7 +770,7 @@ EXPORT_SYMBOL(acpi_unmap_lsapic);
  
 
 #ifdef CONFIG_ACPI_NUMA
-acpi_status __init
+acpi_status __devinit
 acpi_map_iosapic (acpi_handle handle, u32 depth, void *context, void **ret)
 {
         struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER, NULL};
@@ -825,4 +823,28 @@ acpi_map_iosapic (acpi_handle handle, u32 depth, void *context, void **ret)
         return AE_OK;
 }
 #endif /* CONFIG_NUMA */
+
+int
+acpi_register_ioapic (acpi_handle handle, u64 phys_addr, u32 gsi_base)
+{
+        int err;
+
+        if ((err = iosapic_init(phys_addr, gsi_base)))
+                return err;
+
+#if CONFIG_ACPI_NUMA
+        acpi_map_iosapic(handle, 0, NULL, NULL);
+#endif /* CONFIG_ACPI_NUMA */
+
+        return 0;
+}
+EXPORT_SYMBOL(acpi_register_ioapic);
+
+int
+acpi_unregister_ioapic (acpi_handle handle, u32 gsi_base)
+{
+        return iosapic_remove(gsi_base);
+}
+EXPORT_SYMBOL(acpi_unregister_ioapic);
+
 #endif /* CONFIG_ACPI_BOOT */
diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S
index b1d5d3d5276c..69f88d561d62 100644
--- a/arch/ia64/kernel/entry.S
+++ b/arch/ia64/kernel/entry.S
@@ -470,18 +470,6 @@ ENTRY(load_switch_stack)
         br.cond.sptk.many b7
 END(load_switch_stack)
 
-GLOBAL_ENTRY(__ia64_syscall)
-        .regstk 6,0,0,0
-        mov r15=in5                             // put syscall number in place
-        break __BREAK_SYSCALL
-        movl r2=errno
-        cmp.eq p6,p7=-1,r10
-        ;;
-(p6)    st4 [r2]=r8
-(p6)    mov r8=-1
-        br.ret.sptk.many rp
-END(__ia64_syscall)
-
 GLOBAL_ENTRY(execve)
         mov r15=__NR_execve                     // put syscall number in place
         break __BREAK_SYSCALL
@@ -637,7 +625,7 @@ END(ia64_ret_from_syscall)
  *            r8-r11: restored (syscall return value(s))
  *               r12: restored (user-level stack pointer)
  *               r13: restored (user-level thread pointer)
- *               r14: cleared
+ *               r14: set to __kernel_syscall_via_epc
  *               r15: restored (syscall #)
  *           r16-r17: cleared
  *               r18: user-level b6
@@ -658,7 +646,7 @@ END(ia64_ret_from_syscall)
  *                pr: restored (user-level pr)
  *                b0: restored (user-level rp)
  *                b6: restored
- *                b7: cleared
+ *                b7: set to __kernel_syscall_via_epc
  *           ar.unat: restored (user-level ar.unat)
  *            ar.pfs: restored (user-level ar.pfs)
  *            ar.rsc: restored (user-level ar.rsc)
@@ -704,72 +692,79 @@ ENTRY(ia64_leave_syscall)
         ;;
 (p6)    ld4 r31=[r18]                           // load current_thread_info()->flags
         ld8 r19=[r2],PT(B6)-PT(LOADRS)          // load ar.rsc value for "loadrs"
-        mov b7=r0               // clear b7
+        nop.i 0
         ;;
-        ld8 r23=[r3],PT(R11)-PT(AR_BSPSTORE)    // load ar.bspstore (may be garbage)
+        mov r16=ar.bsp                          // M2  get existing backing store pointer
         ld8 r18=[r2],PT(R9)-PT(B6)              // load b6
 (p6)    and r15=TIF_WORK_MASK,r31               // any work other than TIF_SYSCALL_TRACE?
         ;;
-        mov r16=ar.bsp                          // M2  get existing backing store pointer
+        ld8 r23=[r3],PT(R11)-PT(AR_BSPSTORE)    // load ar.bspstore (may be garbage)
 (p6)    cmp4.ne.unc p6,p0=r15, r0               // any special work pending?
 (p6)    br.cond.spnt .work_pending_syscall
         ;;
         // start restoring the state saved on the kernel stack (struct pt_regs):
         ld8 r9=[r2],PT(CR_IPSR)-PT(R9)
         ld8 r11=[r3],PT(CR_IIP)-PT(R11)
-        mov f6=f0               // clear f6
+(pNonSys) break 0               //      bug check: we shouldn't be here if pNonSys is TRUE!
         ;;
         invala                  // M0|1 invalidate ALAT
-        rsm psr.i | psr.ic      // M2 initiate turning off of interrupt and interruption collection
-        mov f9=f0               // clear f9
+        rsm psr.i | psr.ic      // M2   turn off interrupts and interruption collection
+        cmp.eq p9,p0=r0,r0      // A    set p9 to indicate that we should restore cr.ifs
 
-        ld8 r29=[r2],16         // load cr.ipsr
-        ld8 r28=[r3],16                 // load cr.iip
-        mov f8=f0               // clear f8
+        ld8 r29=[r2],16         // M0|1 load cr.ipsr
+        ld8 r28=[r3],16         // M0|1 load cr.iip
+        mov r22=r0              // A    clear r22
         ;;
         ld8 r30=[r2],16         // M0|1 load cr.ifs
         ld8 r25=[r3],16         // M0|1 load ar.unat
-        cmp.eq p9,p0=r0,r0      // set p9 to indicate that we should restore cr.ifs
+(pUStk) add r14=IA64_TASK_THREAD_ON_USTACK_OFFSET,r13
         ;;
         ld8 r26=[r2],PT(B0)-PT(AR_PFS)  // M0|1 load ar.pfs
-(pKStk) mov r22=psr             // M2 read PSR now that interrupts are disabled
-        mov f10=f0              // clear f10
+(pKStk) mov r22=psr                     // M2   read PSR now that interrupts are disabled
+        nop 0
         ;;
-        ld8 r21=[r2],PT(AR_RNAT)-PT(B0) // load b0
-        ld8 r27=[r3],PT(PR)-PT(AR_RSC)  // load ar.rsc
-        mov f11=f0              // clear f11
+        ld8 r21=[r2],PT(AR_RNAT)-PT(B0) // M0|1 load b0
+        ld8 r27=[r3],PT(PR)-PT(AR_RSC)  // M0|1 load ar.rsc
+        mov f6=f0                       // F    clear f6
         ;;
-        ld8 r24=[r2],PT(AR_FPSR)-PT(AR_RNAT)    // load ar.rnat (may be garbage)
-        ld8 r31=[r3],PT(R1)-PT(PR)              // load predicates
-(pUStk) add r14=IA64_TASK_THREAD_ON_USTACK_OFFSET,r13
+        ld8 r24=[r2],PT(AR_FPSR)-PT(AR_RNAT)    // M0|1 load ar.rnat (may be garbage)
+        ld8 r31=[r3],PT(R1)-PT(PR)              // M0|1 load predicates
+        mov f7=f0                               // F    clear f7
         ;;
-        ld8 r20=[r2],PT(R12)-PT(AR_FPSR)        // load ar.fpsr
-        ld8.fill r1=[r3],16     // load r1
-(pUStk) mov r17=1
+        ld8 r20=[r2],PT(R12)-PT(AR_FPSR)        // M0|1 load ar.fpsr
+        ld8.fill r1=[r3],16                     // M0|1 load r1
+(pUStk) mov r17=1                               // A
         ;;
-        srlz.d                  // M0  ensure interruption collection is off
-        ld8.fill r13=[r3],16
-        mov f7=f0               // clear f7
+(pUStk) st1 [r14]=r17                           // M2|3
+        ld8.fill r13=[r3],16                    // M0|1
+        mov f8=f0                               // F    clear f8
         ;;
-        ld8.fill r12=[r2]       // restore r12 (sp)
-        mov.m ar.ssd=r0         // M2 clear ar.ssd
-        mov r22=r0              // clear r22
+        ld8.fill r12=[r2]                       // M0|1 restore r12 (sp)
+        ld8.fill r15=[r3]                       // M0|1 restore r15
+        mov b6=r18                              // I0   restore b6
 
-        ld8.fill r15=[r3]       // restore r15
-(pUStk) st1 [r14]=r17
-        addl r3=THIS_CPU(ia64_phys_stacked_size_p8),r0
+        addl r17=THIS_CPU(ia64_phys_stacked_size_p8),r0 // A
+        mov f9=f0                                       // F    clear f9
+(pKStk) br.cond.dpnt.many skip_rbs_switch               // B
+
+        srlz.d                          // M0   ensure interruption collection is off (for cover)
+        shr.u r18=r19,16                // I0|1 get byte size of existing "dirty" partition
+        cover                           // B    add current frame into dirty partition & set cr.ifs
         ;;
-(pUStk) ld4 r17=[r3]            // r17 = cpu_data->phys_stacked_size_p8
-        mov.m ar.csd=r0         // M2 clear ar.csd
-        mov b6=r18              // I0  restore b6
+(pUStk) ld4 r17=[r17]                   // M0|1 r17 = cpu_data->phys_stacked_size_p8
+        mov r19=ar.bsp                  // M2   get new backing store pointer
+        mov f10=f0                      // F    clear f10
+
+        nop.m 0
+        movl r14=__kernel_syscall_via_epc // X
         ;;
-        mov r14=r0              // clear r14
-        shr.u r18=r19,16        // I0|1 get byte size of existing "dirty" partition
-(pKStk) br.cond.dpnt.many skip_rbs_switch
+        mov.m ar.csd=r0                 // M2   clear ar.csd
+        mov.m ar.ccv=r0                 // M2   clear ar.ccv
+        mov b7=r14                      // I0   clear b7 (hint with __kernel_syscall_via_epc)
 
-        mov.m ar.ccv=r0         // clear ar.ccv
-(pNonSys) br.cond.dpnt.many dont_preserve_current_frame
-        br.cond.sptk.many rbs_switch
+        mov.m ar.ssd=r0                 // M2   clear ar.ssd
+        mov f11=f0                      // F    clear f11
+        br.cond.sptk.many rbs_switch    // B
 END(ia64_leave_syscall)
 
 #ifdef CONFIG_IA32_SUPPORT
@@ -885,7 +880,7 @@ GLOBAL_ENTRY(ia64_leave_kernel)
         ldf.fill f7=[r2],PT(F11)-PT(F7)
         ldf.fill f8=[r3],32
         ;;
-        srlz.i                  // ensure interruption collection is off
+        srlz.d  // ensure that inter. collection is off (VHPT is don't care, since text is pinned)
         mov ar.ccv=r15
         ;;
         ldf.fill f11=[r2]
@@ -945,11 +940,10 @@ GLOBAL_ENTRY(ia64_leave_kernel)
          * NOTE: alloc, loadrs, and cover can't be predicated.
          */
 (pNonSys) br.cond.dpnt dont_preserve_current_frame
-
-rbs_switch:
         cover                           // add current frame into dirty partition and set cr.ifs
         ;;
         mov r19=ar.bsp                  // get new backing store pointer
+rbs_switch:
         sub r16=r16,r18                 // krbs = old bsp - size of dirty partition
         cmp.ne p9,p0=r0,r0              // clear p9 to skip restore of cr.ifs
         ;;
@@ -1024,14 +1018,14 @@ rse_clear_invalid:
         mov loc5=0
         mov loc6=0
         mov loc7=0
-(pRecurse) br.call.sptk.few b0=rse_clear_invalid
+(pRecurse) br.call.dptk.few b0=rse_clear_invalid
         ;;
         mov loc8=0
         mov loc9=0
         cmp.ne pReturn,p0=r0,in1        // if recursion count != 0, we need to do a br.ret
         mov loc10=0
         mov loc11=0
-(pReturn) br.ret.sptk.many b0
+(pReturn) br.ret.dptk.many b0
 #endif /* !CONFIG_ITANIUM */
 #       undef pRecurse
 #       undef pReturn
@@ -1577,8 +1571,8 @@ sys_call_table:
         data8 sys_add_key
         data8 sys_request_key
         data8 sys_keyctl
-        data8 sys_ni_syscall
-        data8 sys_ni_syscall                    // 1275
+        data8 sys_ioprio_set
+        data8 sys_ioprio_get                    // 1275
         data8 sys_set_zone_reclaim
         data8 sys_ni_syscall
         data8 sys_ni_syscall
diff --git a/arch/ia64/kernel/fsys.S b/arch/ia64/kernel/fsys.S
index 962b6c4e32b5..7d7684a369d3 100644
--- a/arch/ia64/kernel/fsys.S
+++ b/arch/ia64/kernel/fsys.S
@@ -531,93 +531,114 @@ GLOBAL_ENTRY(fsys_bubble_down)
         .altrp b6
         .body
         /*
-         * We get here for syscalls that don't have a lightweight handler.  For those, we
-         * need to bubble down into the kernel and that requires setting up a minimal
-         * pt_regs structure, and initializing the CPU state more or less as if an
-         * interruption had occurred.  To make syscall-restarts work, we setup pt_regs
-         * such that cr_iip points to the second instruction in syscall_via_break.
-         * Decrementing the IP hence will restart the syscall via break and not
-         * decrementing IP will return us to the caller, as usual.  Note that we preserve
-         * the value of psr.pp rather than initializing it from dcr.pp.  This makes it
-         * possible to distinguish fsyscall execution from other privileged execution.
+         * We get here for syscalls that don't have a lightweight
+         * handler.  For those, we need to bubble down into the kernel
+         * and that requires setting up a minimal pt_regs structure,
+         * and initializing the CPU state more or less as if an
+         * interruption had occurred.  To make syscall-restarts work,
+         * we setup pt_regs such that cr_iip points to the second
+         * instruction in syscall_via_break.  Decrementing the IP
+         * hence will restart the syscall via break and not
+         * decrementing IP will return us to the caller, as usual.
+         * Note that we preserve the value of psr.pp rather than
+         * initializing it from dcr.pp.  This makes it possible to
+         * distinguish fsyscall execution from other privileged
+         * execution.
          *
          * On entry:
-         *      - normal fsyscall handler register usage, except that we also have:
+         *      - normal fsyscall handler register usage, except
+         *        that we also have:
          *      - r18: address of syscall entry point
          *      - r21: ar.fpsr
          *      - r26: ar.pfs
          *      - r27: ar.rsc
          *      - r29: psr
+         *
+         * We used to clear some PSR bits here but that requires slow
+         * serialization.  Fortuntely, that isn't really necessary.
+         * The rationale is as follows: we used to clear bits
+         * ~PSR_PRESERVED_BITS in PSR.L.  Since
+         * PSR_PRESERVED_BITS==PSR.{UP,MFL,MFH,PK,DT,PP,SP,RT,IC}, we
+         * ended up clearing PSR.{BE,AC,I,DFL,DFH,DI,DB,SI,TB}.
+         * However,
+         *
+         * PSR.BE : already is turned off in __kernel_syscall_via_epc()
+         * PSR.AC : don't care (kernel normally turns PSR.AC on)
+         * PSR.I  : already turned off by the time fsys_bubble_down gets
+         *          invoked
+         * PSR.DFL: always 0 (kernel never turns it on)
+         * PSR.DFH: don't care --- kernel never touches f32-f127 on its own
+         *          initiative
+         * PSR.DI : always 0 (kernel never turns it on)
+         * PSR.SI : always 0 (kernel never turns it on)
+         * PSR.DB : don't care --- kernel never enables kernel-level
+         *          breakpoints
+         * PSR.TB : must be 0 already; if it wasn't zero on entry to
+         *          __kernel_syscall_via_epc, the branch to fsys_bubble_down
+         *          will trigger a taken branch; the taken-trap-handler then
+         *          converts the syscall into a break-based system-call.
          */
-#       define PSR_PRESERVED_BITS       (IA64_PSR_UP | IA64_PSR_MFL | IA64_PSR_MFH | IA64_PSR_PK \
-                                         | IA64_PSR_DT | IA64_PSR_PP | IA64_PSR_SP | IA64_PSR_RT \
-                                         | IA64_PSR_IC)
         /*
-         * Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.  The rest we have
-         * to synthesize.
+         * Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.
+         * The rest we have to synthesize.
          */
-#       define PSR_ONE_BITS             ((3 << IA64_PSR_CPL0_BIT) | (0x1 << IA64_PSR_RI_BIT) \
+#       define PSR_ONE_BITS             ((3 << IA64_PSR_CPL0_BIT)       \
+                                         | (0x1 << IA64_PSR_RI_BIT)     \
                                          | IA64_PSR_BN | IA64_PSR_I)
 
-        invala
-        movl r8=PSR_ONE_BITS
+        invala                                  // M0|1
+        movl r14=ia64_ret_from_syscall          // X
 
-        mov r25=ar.unat                 // save ar.unat (5 cyc)
-        movl r9=PSR_PRESERVED_BITS
+        nop.m 0
+        movl r28=__kernel_syscall_via_break     // X    create cr.iip
+        ;;
 
-        mov ar.rsc=0                    // set enforced lazy mode, pl 0, little-endian, loadrs=0
-        movl r28=__kernel_syscall_via_break
+        mov r2=r16                              // A    get task addr to addl-addressable register
+        adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // A
+        mov r31=pr                              // I0   save pr (2 cyc)
         ;;
-        mov r23=ar.bspstore             // save ar.bspstore (12 cyc)
-        mov r31=pr                      // save pr (2 cyc)
-        mov r20=r1                      // save caller's gp in r20
+        st1 [r16]=r0                            // M2|3 clear current->thread.on_ustack flag
+        addl r22=IA64_RBS_OFFSET,r2             // A    compute base of RBS
+        add r3=TI_FLAGS+IA64_TASK_SIZE,r2       // A
         ;;
-        mov r2=r16                      // copy current task addr to addl-addressable register
-        and r9=r9,r29
-        mov r19=b6                      // save b6 (2 cyc)
+        ld4 r3=[r3]                             // M0|1 r3 = current_thread_info()->flags
+        lfetch.fault.excl.nt1 [r22]             // M0|1 prefetch register backing-store
+        nop.i 0
         ;;
-        mov psr.l=r9                    // slam the door (17 cyc to srlz.i)
-        or r29=r8,r29                   // construct cr.ipsr value to save
-        addl r22=IA64_RBS_OFFSET,r2     // compute base of RBS
+        mov ar.rsc=0                            // M2   set enforced lazy mode, pl 0, LE, loadrs=0
+        nop.m 0
+        nop.i 0
         ;;
-        // GAS reports a spurious RAW hazard on the read of ar.rnat because it thinks
-        // we may be reading ar.itc after writing to psr.l.  Avoid that message with
-        // this directive:
-        dv_serialize_data
-        mov.m r24=ar.rnat               // read ar.rnat (5 cyc lat)
-        lfetch.fault.excl.nt1 [r22]
-        adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r2
-
-        // ensure previous insn group is issued before we stall for srlz.i:
+        mov r23=ar.bspstore                     // M2 (12 cyc) save ar.bspstore
+        mov.m r24=ar.rnat                       // M2 (5 cyc) read ar.rnat (dual-issues!)
+        nop.i 0
         ;;
-        srlz.i                          // ensure new psr.l has been established
-        /////////////////////////////////////////////////////////////////////////////
-        ////////// from this point on, execution is not interruptible anymore
-        /////////////////////////////////////////////////////////////////////////////
-        addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2    // compute base of memory stack
-        cmp.ne pKStk,pUStk=r0,r0        // set pKStk <- 0, pUStk <- 1
+        mov ar.bspstore=r22                     // M2 (6 cyc) switch to kernel RBS
+        movl r8=PSR_ONE_BITS                    // X
         ;;
-        st1 [r16]=r0                    // clear current->thread.on_ustack flag
-        mov ar.bspstore=r22             // switch to kernel RBS
-        mov b6=r18                      // copy syscall entry-point to b6 (7 cyc)
-        add r3=TI_FLAGS+IA64_TASK_SIZE,r2
+        mov r25=ar.unat                         // M2 (5 cyc) save ar.unat
+        mov r19=b6                              // I0   save b6 (2 cyc)
+        mov r20=r1                              // A    save caller's gp in r20
         ;;
-        ld4 r3=[r3]                             // r2 = current_thread_info()->flags
-        mov r18=ar.bsp                  // save (kernel) ar.bsp (12 cyc)
-        mov ar.rsc=0x3                  // set eager mode, pl 0, little-endian, loadrs=0
-        br.call.sptk.many b7=ia64_syscall_setup
-        ;;
-        ssm psr.i
-        movl r2=ia64_ret_from_syscall
+        or r29=r8,r29                           // A    construct cr.ipsr value to save
+        mov b6=r18                              // I0   copy syscall entry-point to b6 (7 cyc)
+        addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2 // A compute base of memory stack
+
+        mov r18=ar.bsp                          // M2   save (kernel) ar.bsp (12 cyc)
+        cmp.ne pKStk,pUStk=r0,r0                // A    set pKStk <- 0, pUStk <- 1
+        br.call.sptk.many b7=ia64_syscall_setup // B
         ;;
-        mov rp=r2                               // set the real return addr
-        and r3=_TIF_SYSCALL_TRACEAUDIT,r3
+        mov ar.rsc=0x3                          // M2   set eager mode, pl 0, LE, loadrs=0
+        mov rp=r14                              // I0   set the real return addr
+        and r3=_TIF_SYSCALL_TRACEAUDIT,r3       // A
         ;;
-        cmp.eq p8,p0=r3,r0
+        ssm psr.i                               // M2   we're on kernel stacks now, reenable irqs
+        cmp.eq p8,p0=r3,r0                      // A
+(p10)   br.cond.spnt.many ia64_ret_from_syscall // B    return if bad call-frame or r15 is a NaT
 
-(p10)   br.cond.spnt.many ia64_ret_from_syscall // p10==true means out registers are more than 8
-(p8)    br.call.sptk.many b6=b6         // ignore this return addr
-        br.cond.sptk ia64_trace_syscall
+        nop.m 0
+(p8)    br.call.sptk.many b6=b6                 // B    (ignore return address)
+        br.cond.spnt ia64_trace_syscall         // B
 END(fsys_bubble_down)
 
         .rodata
diff --git a/arch/ia64/kernel/gate.S b/arch/ia64/kernel/gate.S
index facf75acdc85..86948ce63e43 100644
--- a/arch/ia64/kernel/gate.S
+++ b/arch/ia64/kernel/gate.S
@@ -72,38 +72,40 @@ GLOBAL_ENTRY(__kernel_syscall_via_epc)
          * bundle get executed.  The remaining code must be safe even if
          * they do not get executed.
          */
-        adds r17=-1024,r15
-        mov r10=0                               // default to successful syscall execution
-        epc
+        adds r17=-1024,r15                      // A
+        mov r10=0                               // A    default to successful syscall execution
+        epc                                     // B    causes split-issue
 }
         ;;
-        rsm psr.be // note: on McKinley "rsm psr.be/srlz.d" is slightly faster than "rum psr.be"
-        LOAD_FSYSCALL_TABLE(r14)
-
-        mov r16=IA64_KR(CURRENT)                // 12 cycle read latency
-        tnat.nz p10,p9=r15
-        mov r19=NR_syscalls-1
+        rsm psr.be | psr.i                      // M2 (5 cyc to srlz.d)
+        LOAD_FSYSCALL_TABLE(r14)                // X
         ;;
-        shladd r18=r17,3,r14
-
-        srlz.d
-        cmp.ne p8,p0=r0,r0                      // p8 <- FALSE
-        /* Note: if r17 is a NaT, p6 will be set to zero.  */
-        cmp.geu p6,p7=r19,r17                   // (syscall > 0 && syscall < 1024+NR_syscalls)?
-        ;;
-(p6)    ld8 r18=[r18]
-        mov r21=ar.fpsr
-        add r14=-8,r14                          // r14 <- addr of fsys_bubble_down entry
-        ;;
-(p6)    mov b7=r18
-(p6)    tbit.z p8,p0=r18,0
-(p8)    br.dptk.many b7
-
-(p6)    rsm psr.i
-        mov r27=ar.rsc
-        mov r26=ar.pfs
+        mov r16=IA64_KR(CURRENT)                // M2 (12 cyc)
+        shladd r18=r17,3,r14                    // A
+        mov r19=NR_syscalls-1                   // A
+        ;;
+        lfetch [r18]                            // M0|1
+        mov r29=psr                             // M2 (12 cyc)
+        // If r17 is a NaT, p6 will be zero
+        cmp.geu p6,p7=r19,r17                   // A    (sysnr > 0 && sysnr < 1024+NR_syscalls)?
+        ;;
+        mov r21=ar.fpsr                         // M2 (12 cyc)
+        tnat.nz p10,p9=r15                      // I0
+        mov.i r26=ar.pfs                        // I0 (would stall anyhow due to srlz.d...)
+        ;;
+        srlz.d                                  // M0 (forces split-issue) ensure PSR.BE==0
+(p6)    ld8 r18=[r18]                           // M0|1
+        nop.i 0
+        ;;
+        nop.m 0
+(p6)    tbit.z.unc p8,p0=r18,0                  // I0 (dual-issues with "mov b7=r18"!)
+        nop.i 0
         ;;
-        mov r29=psr                             // read psr (12 cyc load latency)
+(p8)    ssm psr.i
+(p6)    mov b7=r18                              // I0
+(p8)    br.dptk.many b7                         // B
+
+        mov r27=ar.rsc                          // M2 (12 cyc)
 /*
  * brl.cond doesn't work as intended because the linker would convert this branch
  * into a branch to a PLT.  Perhaps there will be a way to avoid this with some
@@ -111,6 +113,8 @@ GLOBAL_ENTRY(__kernel_syscall_via_epc)
  * instead.
  */
 #ifdef CONFIG_ITANIUM
+(p6)    add r14=-8,r14                          // r14 <- addr of fsys_bubble_down entry
+        ;;
 (p6)    ld8 r14=[r14]                           // r14 <- fsys_bubble_down
         ;;
 (p6)    mov b7=r14
@@ -118,7 +122,7 @@ GLOBAL_ENTRY(__kernel_syscall_via_epc)
 #else
         BRL_COND_FSYS_BUBBLE_DOWN(p6)
 #endif
-
+        ssm psr.i
         mov r10=-1
 (p10)   mov r8=EINVAL
 (p9)    mov r8=ENOSYS
diff --git a/arch/ia64/kernel/ia64_ksyms.c b/arch/ia64/kernel/ia64_ksyms.c
index 7bbf019c9867..01572814abe4 100644
--- a/arch/ia64/kernel/ia64_ksyms.c
+++ b/arch/ia64/kernel/ia64_ksyms.c
@@ -58,9 +58,6 @@ EXPORT_SYMBOL(__strlen_user);
 EXPORT_SYMBOL(__strncpy_from_user);
 EXPORT_SYMBOL(__strnlen_user);
 
-#include <asm/unistd.h>
-EXPORT_SYMBOL(__ia64_syscall);
-
 /* from arch/ia64/lib */
 extern void __divsi3(void);
 extern void __udivsi3(void);
diff --git a/arch/ia64/kernel/iosapic.c b/arch/ia64/kernel/iosapic.c
index 88b014381df5..c170be095ccd 100644
--- a/arch/ia64/kernel/iosapic.c
+++ b/arch/ia64/kernel/iosapic.c
@@ -129,14 +129,13 @@ static struct iosapic {
         char __iomem    *addr;          /* base address of IOSAPIC */
         unsigned int    gsi_base;       /* first GSI assigned to this IOSAPIC */
         unsigned short  num_rte;        /* number of RTE in this IOSAPIC */
+        int             rtes_inuse;     /* # of RTEs in use on this IOSAPIC */
 #ifdef CONFIG_NUMA
         unsigned short  node;           /* numa node association via pxm */
 #endif
 } iosapic_lists[NR_IOSAPICS];
 
-static int num_iosapic;
-
-static unsigned char pcat_compat __initdata;    /* 8259 compatibility flag */
+static unsigned char pcat_compat __devinitdata; /* 8259 compatibility flag */
 
 static int iosapic_kmalloc_ok;
 static LIST_HEAD(free_rte_list);
@@ -149,7 +148,7 @@ find_iosapic (unsigned int gsi)
 {
         int i;
 
-        for (i = 0; i < num_iosapic; i++) {
+        for (i = 0; i < NR_IOSAPICS; i++) {
                 if ((unsigned) (gsi - iosapic_lists[i].gsi_base) < iosapic_lists[i].num_rte)
                         return i;
         }
@@ -598,6 +597,7 @@ register_intr (unsigned int gsi, int vector, unsigned char delivery,
                 rte->refcnt++;
                 list_add_tail(&rte->rte_list, &iosapic_intr_info[vector].rtes);
                 iosapic_intr_info[vector].count++;
+                iosapic_lists[index].rtes_inuse++;
         }
         else if (vector_is_shared(vector)) {
                 struct iosapic_intr_info *info = &iosapic_intr_info[vector];
@@ -778,7 +778,7 @@ void
 iosapic_unregister_intr (unsigned int gsi)
 {
         unsigned long flags;
-        int irq, vector;
+        int irq, vector, index;
         irq_desc_t *idesc;
         u32 low32;
         unsigned long trigger, polarity;
@@ -819,6 +819,9 @@ iosapic_unregister_intr (unsigned int gsi)
                 list_del(&rte->rte_list);
                 iosapic_intr_info[vector].count--;
                 iosapic_free_rte(rte);
+                index = find_iosapic(gsi);
+                iosapic_lists[index].rtes_inuse--;
+                WARN_ON(iosapic_lists[index].rtes_inuse < 0);
 
                 trigger  = iosapic_intr_info[vector].trigger;
                 polarity = iosapic_intr_info[vector].polarity;
@@ -952,30 +955,86 @@ iosapic_system_init (int system_pcat_compat)
         }
 }
 
-void __init
+static inline int
+iosapic_alloc (void)
+{
+        int index;
+
+        for (index = 0; index < NR_IOSAPICS; index++)
+                if (!iosapic_lists[index].addr)
+                        return index;
+
+        printk(KERN_WARNING "%s: failed to allocate iosapic\n", __FUNCTION__);
+        return -1;
+}
+
+static inline void
+iosapic_free (int index)
+{
+        memset(&iosapic_lists[index], 0, sizeof(iosapic_lists[0]));
+}
+
+static inline int
+iosapic_check_gsi_range (unsigned int gsi_base, unsigned int ver)
+{
+        int index;
+        unsigned int gsi_end, base, end;
+
+        /* check gsi range */
+        gsi_end = gsi_base + ((ver >> 16) & 0xff);
+        for (index = 0; index < NR_IOSAPICS; index++) {
+                if (!iosapic_lists[index].addr)
+                        continue;
+
+                base = iosapic_lists[index].gsi_base;
+                end  = base + iosapic_lists[index].num_rte - 1;
+
+                if (gsi_base < base && gsi_end < base)
+                        continue;/* OK */
+
+                if (gsi_base > end && gsi_end > end)
+                        continue; /* OK */
+
+                return -EBUSY;
+        }
+        return 0;
+}
+
+int __devinit
 iosapic_init (unsigned long phys_addr, unsigned int gsi_base)
 {
-        int num_rte;
+        int num_rte, err, index;
         unsigned int isa_irq, ver;
         char __iomem *addr;
+        unsigned long flags;
+
+        spin_lock_irqsave(&iosapic_lock, flags);
+        {
+                addr = ioremap(phys_addr, 0);
+                ver = iosapic_version(addr);
 
-        addr = ioremap(phys_addr, 0);
-        ver = iosapic_version(addr);
+                if ((err = iosapic_check_gsi_range(gsi_base, ver))) {
+                        iounmap(addr);
+                        spin_unlock_irqrestore(&iosapic_lock, flags);
+                        return err;
+                }
 
-        /*
-         * The MAX_REDIR register holds the highest input pin
-         * number (starting from 0).
-         * We add 1 so that we can use it for number of pins (= RTEs)
-         */
-        num_rte = ((ver >> 16) & 0xff) + 1;
+                /*
+                 * The MAX_REDIR register holds the highest input pin
+                 * number (starting from 0).
+                 * We add 1 so that we can use it for number of pins (= RTEs)
+                 */
+                num_rte = ((ver >> 16) & 0xff) + 1;
 
-        iosapic_lists[num_iosapic].addr = addr;
-        iosapic_lists[num_iosapic].gsi_base = gsi_base;
-        iosapic_lists[num_iosapic].num_rte = num_rte;
+                index = iosapic_alloc();
+                iosapic_lists[index].addr = addr;
+                iosapic_lists[index].gsi_base = gsi_base;
+                iosapic_lists[index].num_rte = num_rte;
 #ifdef CONFIG_NUMA
-        iosapic_lists[num_iosapic].node = MAX_NUMNODES;
+                iosapic_lists[index].node = MAX_NUMNODES;
 #endif
-        num_iosapic++;
+        }
+        spin_unlock_irqrestore(&iosapic_lock, flags);
 
         if ((gsi_base == 0) && pcat_compat) {
                 /*
@@ -986,10 +1045,43 @@ iosapic_init (unsigned long phys_addr, unsigned int gsi_base)
                 for (isa_irq = 0; isa_irq < 16; ++isa_irq)
                         iosapic_override_isa_irq(isa_irq, isa_irq, IOSAPIC_POL_HIGH, IOSAPIC_EDGE);
         }
+        return 0;
+}
+
+#ifdef CONFIG_HOTPLUG
+int
+iosapic_remove (unsigned int gsi_base)
+{
+        int index, err = 0;
+        unsigned long flags;
+
+        spin_lock_irqsave(&iosapic_lock, flags);
+        {
+                index = find_iosapic(gsi_base);
+                if (index < 0) {
+                        printk(KERN_WARNING "%s: No IOSAPIC for GSI base %u\n",
+                               __FUNCTION__, gsi_base);
+                        goto out;
+                }
+
+                if (iosapic_lists[index].rtes_inuse) {
+                        err = -EBUSY;
+                        printk(KERN_WARNING "%s: IOSAPIC for GSI base %u is busy\n",
+                               __FUNCTION__, gsi_base);
+                        goto out;
+                }
+
+                iounmap(iosapic_lists[index].addr);
+                iosapic_free(index);
+        }
+ out:
+        spin_unlock_irqrestore(&iosapic_lock, flags);
+        return err;
 }
+#endif /* CONFIG_HOTPLUG */
 
 #ifdef CONFIG_NUMA
-void __init
+void __devinit
 map_iosapic_to_node(unsigned int gsi_base, int node)
 {
         int index;
diff --git a/arch/ia64/kernel/ivt.S b/arch/ia64/kernel/ivt.S
index 2bc085a73e30..3bb3a13c4047 100644
--- a/arch/ia64/kernel/ivt.S
+++ b/arch/ia64/kernel/ivt.S
@@ -1,7 +1,7 @@
 /*
  * arch/ia64/kernel/ivt.S
  *
- * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co
+ * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co
  *      Stephane Eranian <eranian@hpl.hp.com>
  *      David Mosberger <davidm@hpl.hp.com>
  * Copyright (C) 2000, 2002-2003 Intel Co
@@ -692,82 +692,118 @@ ENTRY(break_fault)
          * to prevent leaking bits from kernel to user level.
          */
         DBG_FAULT(11)
-        mov r16=IA64_KR(CURRENT)                // r16 = current task; 12 cycle read lat.
-        mov r17=cr.iim
-        mov r18=__IA64_BREAK_SYSCALL
-        mov r21=ar.fpsr
-        mov r29=cr.ipsr
-        mov r19=b6
-        mov r25=ar.unat
-        mov r27=ar.rsc
-        mov r26=ar.pfs
-        mov r28=cr.iip
-        mov r31=pr                              // prepare to save predicates
-        mov r20=r1
-        ;;
+        mov.m r16=IA64_KR(CURRENT)              // M2 r16 <- current task (12 cyc)
+        mov r29=cr.ipsr                         // M2 (12 cyc)
+        mov r31=pr                              // I0 (2 cyc)
+
+        mov r17=cr.iim                          // M2 (2 cyc)
+        mov.m r27=ar.rsc                        // M2 (12 cyc)
+        mov r18=__IA64_BREAK_SYSCALL            // A
+
+        mov.m ar.rsc=0                          // M2
+        mov.m r21=ar.fpsr                       // M2 (12 cyc)
+        mov r19=b6                              // I0 (2 cyc)
+        ;;
+        mov.m r23=ar.bspstore                   // M2 (12 cyc)
+        mov.m r24=ar.rnat                       // M2 (5 cyc)
+        mov.i r26=ar.pfs                        // I0 (2 cyc)
+
+        invala                                  // M0|1
+        nop.m 0                                 // M
+        mov r20=r1                              // A                    save r1
+
+        nop.m 0
+        movl r30=sys_call_table                 // X
+
+        mov r28=cr.iip                          // M2 (2 cyc)
+        cmp.eq p0,p7=r18,r17                    // I0 is this a system call?
+(p7)    br.cond.spnt non_syscall                // B  no ->
+        //
+        // From this point on, we are definitely on the syscall-path
+        // and we can use (non-banked) scratch registers.
+        //
+///////////////////////////////////////////////////////////////////////
+        mov r1=r16                              // A    move task-pointer to "addl"-addressable reg
+        mov r2=r16                              // A    setup r2 for ia64_syscall_setup
+        add r9=TI_FLAGS+IA64_TASK_SIZE,r16      // A    r9 = &current_thread_info()->flags
+
         adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16
-        cmp.eq p0,p7=r18,r17                    // is this a system call? (p7 <- false, if so)
-(p7)    br.cond.spnt non_syscall
+        adds r15=-1024,r15                      // A    subtract 1024 from syscall number
+        mov r3=NR_syscalls - 1
         ;;
-        ld1 r17=[r16]                           // load current->thread.on_ustack flag
-        st1 [r16]=r0                            // clear current->thread.on_ustack flag
-        add r1=-IA64_TASK_THREAD_ON_USTACK_OFFSET,r16   // set r1 for MINSTATE_START_SAVE_MIN_VIRT
+        ld1.bias r17=[r16]                      // M0|1 r17 = current->thread.on_ustack flag
+        ld4 r9=[r9]                             // M0|1 r9 = current_thread_info()->flags
+        extr.u r8=r29,41,2                      // I0   extract ei field from cr.ipsr
+
+        shladd r30=r15,3,r30                    // A    r30 = sys_call_table + 8*(syscall-1024)
+        addl r22=IA64_RBS_OFFSET,r1             // A    compute base of RBS
+        cmp.leu p6,p7=r15,r3                    // A    syscall number in range?
         ;;
-        invala
 
-        /* adjust return address so we skip over the break instruction: */
+        lfetch.fault.excl.nt1 [r22]             // M0|1 prefetch RBS
+(p6)    ld8 r30=[r30]                           // M0|1 load address of syscall entry point
+        tnat.nz.or p7,p0=r15                    // I0   is syscall nr a NaT?
 
-        extr.u r8=r29,41,2                      // extract ei field from cr.ipsr
-        ;;
-        cmp.eq p6,p7=2,r8                       // isr.ei==2?
-        mov r2=r1                               // setup r2 for ia64_syscall_setup
-        ;;
-(p6)    mov r8=0                                // clear ei to 0
-(p6)    adds r28=16,r28                         // switch cr.iip to next bundle cr.ipsr.ei wrapped
-(p7)    adds r8=1,r8                            // increment ei to next slot
-        ;;
-        cmp.eq pKStk,pUStk=r0,r17               // are we in kernel mode already?
-        dep r29=r8,r29,41,2                     // insert new ei into cr.ipsr
+        mov.m ar.bspstore=r22                   // M2   switch to kernel RBS
+        cmp.eq p8,p9=2,r8                       // A    isr.ei==2?
         ;;
 
-        // switch from user to kernel RBS:
-        MINSTATE_START_SAVE_MIN_VIRT
-        br.call.sptk.many b7=ia64_syscall_setup
-        ;;
-        MINSTATE_END_SAVE_MIN_VIRT              // switch to bank 1
-        ssm psr.ic | PSR_DEFAULT_BITS
-        ;;
-        srlz.i                                  // guarantee that interruption collection is on
-        mov r3=NR_syscalls - 1
-        ;;
-(p15)   ssm psr.i                               // restore psr.i
-        // p10==true means out registers are more than 8 or r15's Nat is true
-(p10)   br.cond.spnt.many ia64_ret_from_syscall
-        ;;
-        movl r16=sys_call_table
+(p8)    mov r8=0                                // A    clear ei to 0
+(p7)    movl r30=sys_ni_syscall                 // X
 
-        adds r15=-1024,r15                      // r15 contains the syscall number---subtract 1024
-        movl r2=ia64_ret_from_syscall
-        ;;
-        shladd r20=r15,3,r16                    // r20 = sys_call_table + 8*(syscall-1024)
-        cmp.leu p6,p7=r15,r3                    // (syscall > 0 && syscall < 1024 + NR_syscalls) ?
-        mov rp=r2                               // set the real return addr
+(p8)    adds r28=16,r28                         // A    switch cr.iip to next bundle
+(p9)    adds r8=1,r8                            // A    increment ei to next slot
+        nop.i 0
         ;;
-(p6)    ld8 r20=[r20]                           // load address of syscall entry point
-(p7)    movl r20=sys_ni_syscall
 
-        add r2=TI_FLAGS+IA64_TASK_SIZE,r13
-        ;;
-        ld4 r2=[r2]                             // r2 = current_thread_info()->flags
-        ;;
-        and r2=_TIF_SYSCALL_TRACEAUDIT,r2       // mask trace or audit
+        mov.m r25=ar.unat                       // M2 (5 cyc)
+        dep r29=r8,r29,41,2                     // I0   insert new ei into cr.ipsr
+        adds r15=1024,r15                       // A    restore original syscall number
+        //
+        // If any of the above loads miss in L1D, we'll stall here until
+        // the data arrives.
+        //
+///////////////////////////////////////////////////////////////////////
+        st1 [r16]=r0                            // M2|3 clear current->thread.on_ustack flag
+        mov b6=r30                              // I0   setup syscall handler branch reg early
+        cmp.eq pKStk,pUStk=r0,r17               // A    were we on kernel stacks already?
+
+        and r9=_TIF_SYSCALL_TRACEAUDIT,r9       // A    mask trace or audit
+        mov r18=ar.bsp                          // M2 (12 cyc)
+(pKStk) br.cond.spnt .break_fixup               // B    we're already in kernel-mode -- fix up RBS
+        ;;
+.back_from_break_fixup:
+(pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A    compute base of memory stack
+        cmp.eq p14,p0=r9,r0                     // A    are syscalls being traced/audited?
+        br.call.sptk.many b7=ia64_syscall_setup // B
+1:
+        mov ar.rsc=0x3                          // M2   set eager mode, pl 0, LE, loadrs=0
+        nop 0
+        bsw.1                                   // B (6 cyc) regs are saved, switch to bank 1
         ;;
-        cmp.eq p8,p0=r2,r0
-        mov b6=r20
+
+        ssm psr.ic | PSR_DEFAULT_BITS           // M2   now it's safe to re-enable intr.-collection
+        movl r3=ia64_ret_from_syscall           // X
         ;;
-(p8)    br.call.sptk.many b6=b6                 // ignore this return addr
-        br.cond.sptk ia64_trace_syscall
+
+        srlz.i                                  // M0   ensure interruption collection is on
+        mov rp=r3                               // I0   set the real return addr
+(p10)   br.cond.spnt.many ia64_ret_from_syscall // B    return if bad call-frame or r15 is a NaT
+
+(p15)   ssm psr.i                               // M2   restore psr.i
+(p14)   br.call.sptk.many b6=b6                 // B    invoke syscall-handker (ignore return addr)
+        br.cond.spnt.many ia64_trace_syscall    // B    do syscall-tracing thingamagic
         // NOT REACHED
+///////////////////////////////////////////////////////////////////////
+        // On entry, we optimistically assumed that we're coming from user-space.
+        // For the rare cases where a system-call is done from within the kernel,
+        // we fix things up at this point:
+.break_fixup:
+        add r1=-IA64_PT_REGS_SIZE,sp            // A    allocate space for pt_regs structure
+        mov ar.rnat=r24                         // M2   restore kernel's AR.RNAT
+        ;;
+        mov ar.bspstore=r23                     // M2   restore kernel's AR.BSPSTORE
+        br.cond.sptk .back_from_break_fixup
 END(break_fault)
 
         .org ia64_ivt+0x3000
@@ -842,8 +878,6 @@ END(interrupt)
          *      - r31: saved pr
          *      -  b0: original contents (to be saved)
          * On exit:
-         *      - executing on bank 1 registers
-         *      - psr.ic enabled, interrupts restored
          *      -  p10: TRUE if syscall is invoked with more than 8 out
          *              registers or r15's Nat is true
          *      -  r1: kernel's gp
@@ -851,8 +885,11 @@ END(interrupt)
          *      -  r8: -EINVAL if p10 is true
          *      - r12: points to kernel stack
          *      - r13: points to current task
+         *      - r14: preserved (same as on entry)
+         *      - p13: preserved
          *      - p15: TRUE if interrupts need to be re-enabled
          *      - ar.fpsr: set to kernel settings
+         *      -  b6: preserved (same as on entry)
          */
 GLOBAL_ENTRY(ia64_syscall_setup)
 #if PT(B6) != 0
@@ -920,10 +957,10 @@ GLOBAL_ENTRY(ia64_syscall_setup)
 (p13)   mov in5=-1
         ;;
         st8 [r16]=r21,PT(R8)-PT(AR_FPSR)        // save ar.fpsr
-        tnat.nz p14,p0=in6
+        tnat.nz p13,p0=in6
         cmp.lt p10,p9=r11,r8    // frame size can't be more than local+8
         ;;
-        stf8 [r16]=f1           // ensure pt_regs.r8 != 0 (see handle_syscall_error)
+        mov r8=1
 (p9)    tnat.nz p10,p0=r15
         adds r12=-16,r1         // switch to kernel memory stack (with 16 bytes of scratch)
 
@@ -934,9 +971,9 @@ GLOBAL_ENTRY(ia64_syscall_setup)
         mov r13=r2                              // establish `current'
         movl r1=__gp                            // establish kernel global pointer
         ;;
-(p14)   mov in6=-1
+        st8 [r16]=r8            // ensure pt_regs.r8 != 0 (see handle_syscall_error)
+(p13)   mov in6=-1
 (p8)    mov in7=-1
-        nop.i 0
 
         cmp.eq pSys,pNonSys=r0,r0               // set pSys=1, pNonSys=0
         movl r17=FPSR_DEFAULT
@@ -1007,6 +1044,8 @@ END(dispatch_illegal_op_fault)
         FAULT(17)
 
 ENTRY(non_syscall)
+        mov ar.rsc=r27                  // restore ar.rsc before SAVE_MIN_WITH_COVER
+        ;;
         SAVE_MIN_WITH_COVER
 
         // There is no particular reason for this code to be here, other than that
@@ -1204,6 +1243,25 @@ END(disabled_fp_reg)
 // 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50)
 ENTRY(nat_consumption)
         DBG_FAULT(26)
+
+        mov r16=cr.ipsr
+        mov r17=cr.isr
+        mov r31=pr                              // save PR
+        ;;
+        and r18=0xf,r17                         // r18 = cr.ipsr.code{3:0}
+        tbit.z p6,p0=r17,IA64_ISR_NA_BIT
+        ;;
+        cmp.ne.or p6,p0=IA64_ISR_CODE_LFETCH,r18
+        dep r16=-1,r16,IA64_PSR_ED_BIT,1
+(p6)    br.cond.spnt 1f         // branch if (cr.ispr.na == 0 || cr.ipsr.code{3:0} != LFETCH)
+        ;;
+        mov cr.ipsr=r16         // set cr.ipsr.na
+        mov pr=r31,-1
+        ;;
+        rfi
+
+1:      mov pr=r31,-1
+        ;;
         FAULT(26)
 END(nat_consumption)
 
diff --git a/arch/ia64/kernel/kprobes.c b/arch/ia64/kernel/kprobes.c
index 5978823d5c63..3aa3167edbec 100644
--- a/arch/ia64/kernel/kprobes.c
+++ b/arch/ia64/kernel/kprobes.c
@@ -34,6 +34,7 @@
 
 #include <asm/pgtable.h>
 #include <asm/kdebug.h>
+#include <asm/sections.h>
 
 extern void jprobe_inst_return(void);
 
@@ -263,13 +264,33 @@ static inline void get_kprobe_inst(bundle_t *bundle, uint slot,
         }
 }
 
+/* Returns non-zero if the addr is in the Interrupt Vector Table */
+static inline int in_ivt_functions(unsigned long addr)
+{
+        return (addr >= (unsigned long)__start_ivt_text
+                && addr < (unsigned long)__end_ivt_text);
+}
+
 static int valid_kprobe_addr(int template, int slot, unsigned long addr)
 {
         if ((slot > 2) || ((bundle_encoding[template][1] == L) && slot > 1)) {
-                printk(KERN_WARNING "Attempting to insert unaligned kprobe at 0x%lx\n",
-                                addr);
+                printk(KERN_WARNING "Attempting to insert unaligned kprobe "
+                                "at 0x%lx\n", addr);
                 return -EINVAL;
         }
+
+        if (in_ivt_functions(addr)) {
+                printk(KERN_WARNING "Kprobes can't be inserted inside "
+                                "IVT functions at 0x%lx\n", addr);
+                return -EINVAL;
+        }
+
+        if (slot == 1 && bundle_encoding[template][1] != L) {
+                printk(KERN_WARNING "Inserting kprobes on slot #1 "
+                       "is not supported\n");
+                return -EINVAL;
+        }
+
         return 0;
 }
 
@@ -290,6 +311,94 @@ static inline void set_current_kprobe(struct kprobe *p)
         current_kprobe = p;
 }
 
+static void kretprobe_trampoline(void)
+{
+}
+
+/*
+ * At this point the target function has been tricked into
+ * returning into our trampoline.  Lookup the associated instance
+ * and then:
+ *    - call the handler function
+ *    - cleanup by marking the instance as unused
+ *    - long jump back to the original return address
+ */
+int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
+{
+        struct kretprobe_instance *ri = NULL;
+        struct hlist_head *head;
+        struct hlist_node *node, *tmp;
+        unsigned long orig_ret_address = 0;
+        unsigned long trampoline_address =
+                ((struct fnptr *)kretprobe_trampoline)->ip;
+
+        head = kretprobe_inst_table_head(current);
+
+        /*
+         * It is possible to have multiple instances associated with a given
+         * task either because an multiple functions in the call path
+         * have a return probe installed on them, and/or more then one return
+         * return probe was registered for a target function.
+         *
+         * We can handle this because:
+         *     - instances are always inserted at the head of the list
+         *     - when multiple return probes are registered for the same
+         *       function, the first instance's ret_addr will point to the
+         *       real return address, and all the rest will point to
+         *       kretprobe_trampoline
+         */
+        hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
+                if (ri->task != current)
+                        /* another task is sharing our hash bucket */
+                        continue;
+
+                if (ri->rp && ri->rp->handler)
+                        ri->rp->handler(ri, regs);
+
+                orig_ret_address = (unsigned long)ri->ret_addr;
+                recycle_rp_inst(ri);
+
+                if (orig_ret_address != trampoline_address)
+                        /*
+                         * This is the real return address. Any other
+                         * instances associated with this task are for
+                         * other calls deeper on the call stack
+                         */
+                        break;
+        }
+
+        BUG_ON(!orig_ret_address || (orig_ret_address == trampoline_address));
+        regs->cr_iip = orig_ret_address;
+
+        unlock_kprobes();
+        preempt_enable_no_resched();
+
+        /*
+         * By returning a non-zero value, we are telling
+         * kprobe_handler() that we have handled unlocking
+         * and re-enabling preemption.
+         */
+        return 1;
+}
+
+void arch_prepare_kretprobe(struct kretprobe *rp, struct pt_regs *regs)
+{
+        struct kretprobe_instance *ri;
+
+        if ((ri = get_free_rp_inst(rp)) != NULL) {
+                ri->rp = rp;
+                ri->task = current;
+                ri->ret_addr = (kprobe_opcode_t *)regs->b0;
+
+                /* Replace the return addr with trampoline addr */
+                regs->b0 = ((struct fnptr *)kretprobe_trampoline)->ip;
+
+                add_rp_inst(ri);
+        } else {
+                rp->nmissed++;
+        }
+}
+
 int arch_prepare_kprobe(struct kprobe *p)
 {
         unsigned long addr = (unsigned long) p->addr;
@@ -492,8 +601,8 @@ static int pre_kprobes_handler(struct die_args *args)
         if (p->pre_handler && p->pre_handler(p, regs))
                 /*
                  * Our pre-handler is specifically requesting that we just
-                 * do a return.  This is handling the case where the
-                 * pre-handler is really our special jprobe pre-handler.
+                 * do a return.  This is used for both the jprobe pre-handler
+                 * and the kretprobe trampoline
                  */
                 return 1;
 
@@ -599,3 +708,14 @@ int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
         *regs = jprobe_saved_regs;
         return 1;
 }
+
+static struct kprobe trampoline_p = {
+        .pre_handler = trampoline_probe_handler
+};
+
+int __init arch_init(void)
+{
+        trampoline_p.addr =
+                (kprobe_opcode_t *)((struct fnptr *)kretprobe_trampoline)->ip;
+        return register_kprobe(&trampoline_p);
+}
diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c
index ebb71f3d6d19..6e35bff05d59 100644
--- a/arch/ia64/kernel/process.c
+++ b/arch/ia64/kernel/process.c
@@ -27,6 +27,7 @@
 #include <linux/efi.h>
 #include <linux/interrupt.h>
 #include <linux/delay.h>
+#include <linux/kprobes.h>
 
 #include <asm/cpu.h>
 #include <asm/delay.h>
@@ -707,6 +708,13 @@ kernel_thread_helper (int (*fn)(void *), void *arg)
 void
 flush_thread (void)
 {
+        /*
+         * Remove function-return probe instances associated with this task
+         * and put them back on the free list. Do not insert an exit probe for
+         * this function, it will be disabled by kprobe_flush_task if you do.
+         */
+        kprobe_flush_task(current);
+
         /* drop floating-point and debug-register state if it exists: */
         current->thread.flags &= ~(IA64_THREAD_FPH_VALID | IA64_THREAD_DBG_VALID);
         ia64_drop_fpu(current);
@@ -721,6 +729,14 @@ flush_thread (void)
 void
 exit_thread (void)
 {
+
+        /*
+         * Remove function-return probe instances associated with this task
+         * and put them back on the free list. Do not insert an exit probe for
+         * this function, it will be disabled by kprobe_flush_task if you do.
+         */
+        kprobe_flush_task(current);
+
         ia64_drop_fpu(current);
 #ifdef CONFIG_PERFMON
        /* if needed, stop monitoring and flush state to perfmon context */
diff --git a/arch/ia64/kernel/ptrace.c b/arch/ia64/kernel/ptrace.c
index 6d57aebad485..bbb8bc7c0552 100644
--- a/arch/ia64/kernel/ptrace.c
+++ b/arch/ia64/kernel/ptrace.c
@@ -725,12 +725,32 @@ convert_to_non_syscall (struct task_struct *child, struct pt_regs  *pt,
                         break;
         }
 
+        /*
+         * Note: at the time of this call, the target task is blocked
+         * in notify_resume_user() and by clearling PRED_LEAVE_SYSCALL
+         * (aka, "pLvSys") we redirect execution from
+         * .work_pending_syscall_end to .work_processed_kernel.
+         */
         unw_get_pr(&prev_info, &pr);
-        pr &= ~(1UL << PRED_SYSCALL);
+        pr &= ~((1UL << PRED_SYSCALL) | (1UL << PRED_LEAVE_SYSCALL));
         pr |=  (1UL << PRED_NON_SYSCALL);
         unw_set_pr(&prev_info, pr);
 
         pt->cr_ifs = (1UL << 63) | cfm;
+        /*
+         * Clear the memory that is NOT written on syscall-entry to
+         * ensure we do not leak kernel-state to user when execution
+         * resumes.
+         */
+        pt->r2 = 0;
+        pt->r3 = 0;
+        pt->r14 = 0;
+        memset(&pt->r16, 0, 16*8);      /* clear r16-r31 */
+        memset(&pt->f6, 0, 6*16);       /* clear f6-f11 */
+        pt->b7 = 0;
+        pt->ar_ccv = 0;
+        pt->ar_csd = 0;
+        pt->ar_ssd = 0;
 }
 
 static int
diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c
index d14692e0920a..2693e1522d7c 100644
--- a/arch/ia64/kernel/setup.c
+++ b/arch/ia64/kernel/setup.c
@@ -72,6 +72,8 @@ DEFINE_PER_CPU(unsigned long, ia64_phys_stacked_size_p8);
 unsigned long ia64_cycles_per_usec;
 struct ia64_boot_param *ia64_boot_param;
 struct screen_info screen_info;
+unsigned long vga_console_iobase;
+unsigned long vga_console_membase;
 
 unsigned long ia64_max_cacheline_size;
 unsigned long ia64_iobase;      /* virtual address for I/O accesses */
@@ -273,23 +275,25 @@ io_port_init (void)
 static inline int __init
 early_console_setup (char *cmdline)
 {
+        int earlycons = 0;
+
 #ifdef CONFIG_SERIAL_SGI_L1_CONSOLE
         {
                 extern int sn_serial_console_early_setup(void);
                 if (!sn_serial_console_early_setup())
-                        return 0;
+                        earlycons++;
         }
 #endif
 #ifdef CONFIG_EFI_PCDP
         if (!efi_setup_pcdp_console(cmdline))
-                return 0;
+                earlycons++;
 #endif
 #ifdef CONFIG_SERIAL_8250_CONSOLE
         if (!early_serial_console_init(cmdline))
-                return 0;
+                earlycons++;
 #endif
 
-        return -1;
+        return (earlycons) ? 0 : -1;
 }
 
 static inline void
diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c
index b49d4ddaab93..0166a9847095 100644
--- a/arch/ia64/kernel/smp.c
+++ b/arch/ia64/kernel/smp.c
@@ -231,13 +231,16 @@ smp_flush_tlb_all (void)
 void
 smp_flush_tlb_mm (struct mm_struct *mm)
 {
+        preempt_disable();
         /* this happens for the common case of a single-threaded fork():  */
         if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
         {
                 local_finish_flush_tlb_mm(mm);
+                preempt_enable();
                 return;
         }
 
+        preempt_enable();
         /*
          * We could optimize this further by using mm->cpu_vm_mask to track which CPUs
          * have been running in the address space.  It's not clear that this is worth the
diff --git a/arch/ia64/kernel/vmlinux.lds.S b/arch/ia64/kernel/vmlinux.lds.S
index b9f0db4c1b04..a676e79e0681 100644
--- a/arch/ia64/kernel/vmlinux.lds.S
+++ b/arch/ia64/kernel/vmlinux.lds.S
@@ -8,6 +8,11 @@
 #define LOAD_OFFSET     (KERNEL_START - KERNEL_TR_PAGE_SIZE)
 #include <asm-generic/vmlinux.lds.h>
 
+#define IVT_TEXT                                                        \
+                VMLINUX_SYMBOL(__start_ivt_text) = .;                   \
+                *(.text.ivt)                                            \
+                VMLINUX_SYMBOL(__end_ivt_text) = .;
+
 OUTPUT_FORMAT("elf64-ia64-little")
 OUTPUT_ARCH(ia64)
 ENTRY(phys_start)
@@ -39,7 +44,7 @@ SECTIONS
 
   .text : AT(ADDR(.text) - LOAD_OFFSET)
     {
-        *(.text.ivt)
+        IVT_TEXT
         *(.text)
         SCHED_TEXT
         LOCK_TEXT