x86_64 ia32 syscall audit fast-path

This adds fast paths for 32-bit syscall entry and exit when
TIF_SYSCALL_AUDIT is set, but no other kind of syscall tracing.
These paths does not need to save and restore all registers as
the general case of tracing does.  Avoiding the iret return path
when syscall audit is enabled helps performance a lot.

Signed-off-by: Roland McGrath <roland@redhat.com>

2 files changed, 88 insertions, 4 deletions

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 23d146ce676..021d71bc69b 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -15,6 +15,16 @@
 #include <asm/irqflags.h>
 #include <linux/linkage.h>
 
+/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
+#include <linux/elf-em.h>
+#define AUDIT_ARCH_I386         (EM_386|__AUDIT_ARCH_LE)
+#define __AUDIT_ARCH_LE    0x40000000
+
+#ifndef CONFIG_AUDITSYSCALL
+#define sysexit_audit int_ret_from_sys_call
+#define sysretl_audit int_ret_from_sys_call
+#endif
+
 #define IA32_NR_syscalls ((ia32_syscall_end - ia32_sys_call_table)/8)
 
         .macro IA32_ARG_FIXUP noebp=0
@@ -148,13 +158,15 @@ ENTRY(ia32_sysenter_target)
         ja      ia32_badsys
 sysenter_do_call:
         IA32_ARG_FIXUP 1
+sysenter_dispatch:
         call    *ia32_sys_call_table(,%rax,8)
         movq    %rax,RAX-ARGOFFSET(%rsp)
         GET_THREAD_INFO(%r10)
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         testl   $_TIF_ALLWORK_MASK,TI_flags(%r10)
-        jnz     int_ret_from_sys_call
+        jnz     sysexit_audit
+sysexit_from_sys_call:
         andl    $~TS_COMPAT,TI_status(%r10)
         /* clear IF, that popfq doesn't enable interrupts early */
         andl  $~0x200,EFLAGS-R11(%rsp) 
@@ -170,9 +182,63 @@ sysenter_do_call:
         TRACE_IRQS_ON
         ENABLE_INTERRUPTS_SYSEXIT32
 
-sysenter_tracesys:
+#ifdef CONFIG_AUDITSYSCALL
+        .macro auditsys_entry_common
+        movl %esi,%r9d                  /* 6th arg: 4th syscall arg */
+        movl %edx,%r8d                  /* 5th arg: 3rd syscall arg */
+        /* (already in %ecx)               4th arg: 2nd syscall arg */
+        movl %ebx,%edx                  /* 3rd arg: 1st syscall arg */
+        movl %eax,%esi                  /* 2nd arg: syscall number */
+        movl $AUDIT_ARCH_I386,%edi      /* 1st arg: audit arch */
+        call audit_syscall_entry
+        movl RAX-ARGOFFSET(%rsp),%eax   /* reload syscall number */
+        cmpl $(IA32_NR_syscalls-1),%eax
+        ja ia32_badsys
+        movl %ebx,%edi                  /* reload 1st syscall arg */
+        movl RCX-ARGOFFSET(%rsp),%esi   /* reload 2nd syscall arg */
+        movl RDX-ARGOFFSET(%rsp),%edx   /* reload 3rd syscall arg */
+        movl RSI-ARGOFFSET(%rsp),%ecx   /* reload 4th syscall arg */
+        movl RDI-ARGOFFSET(%rsp),%r8d   /* reload 5th syscall arg */
+        .endm
+
+        .macro auditsys_exit exit
+        testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)
+        jnz int_ret_from_sys_call
+        TRACE_IRQS_ON
+        sti
+        movl %eax,%esi          /* second arg, syscall return value */
+        cmpl $0,%eax            /* is it < 0? */
+        setl %al                /* 1 if so, 0 if not */
+        movzbl %al,%edi         /* zero-extend that into %edi */
+        inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
+        call audit_syscall_exit
+        GET_THREAD_INFO(%r10)
+        movl RAX-ARGOFFSET(%rsp),%eax   /* reload syscall return value */
+        movl RBP-ARGOFFSET(%rsp),%ebp   /* reload user register value */
+        movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
+        cli
+        TRACE_IRQS_OFF
+        testl %edi,TI_flags(%r10)
+        jnz int_with_check
+        jmp \exit
+        .endm
+
+sysenter_auditsys:
         CFI_RESTORE_STATE
+        auditsys_entry_common
+        movl %ebp,%r9d                  /* reload 6th syscall arg */
+        jmp sysenter_dispatch
+
+sysexit_audit:
+        auditsys_exit sysexit_from_sys_call
+#endif
+
+sysenter_tracesys:
         xchgl   %r9d,%ebp
+#ifdef CONFIG_AUDITSYSCALL
+        testl   $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)
+        jz      sysenter_auditsys
+#endif
         SAVE_REST
         CLEAR_RREGS
         movq    %r9,R9(%rsp)
@@ -252,13 +318,15 @@ cstar_do_call:
         cmpl $IA32_NR_syscalls-1,%eax
         ja  ia32_badsys
         IA32_ARG_FIXUP 1
+cstar_dispatch:
         call *ia32_sys_call_table(,%rax,8)
         movq %rax,RAX-ARGOFFSET(%rsp)
         GET_THREAD_INFO(%r10)
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         testl $_TIF_ALLWORK_MASK,TI_flags(%r10)
-        jnz  int_ret_from_sys_call
+        jnz sysretl_audit
+sysretl_from_sys_call:
         andl $~TS_COMPAT,TI_status(%r10)
         RESTORE_ARGS 1,-ARG_SKIP,1,1,1
         movl RIP-ARGOFFSET(%rsp),%ecx
@@ -270,8 +338,23 @@ cstar_do_call:
         CFI_RESTORE rsp
         USERGS_SYSRET32
         
-cstar_tracesys: 
+#ifdef CONFIG_AUDITSYSCALL
+cstar_auditsys:
         CFI_RESTORE_STATE
+        movl %r9d,R9-ARGOFFSET(%rsp)    /* register to be clobbered by call */
+        auditsys_entry_common
+        movl R9-ARGOFFSET(%rsp),%r9d    /* reload 6th syscall arg */
+        jmp cstar_dispatch
+
+sysretl_audit:
+        auditsys_exit sysretl_from_sys_call
+#endif
+
+cstar_tracesys:
+#ifdef CONFIG_AUDITSYSCALL
+        testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)
+        jz cstar_auditsys
+#endif
         xchgl %r9d,%ebp
         SAVE_REST
         CLEAR_RREGS
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index db7d34a89d2..89434d43960 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -492,6 +492,7 @@ tracesys:
  * Has correct top of stack, but partial stack frame.
  */
         .globl int_ret_from_sys_call
+        .globl int_with_check
 int_ret_from_sys_call:
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF