Added KUTrace Support

Moved sched_litmus back to the top scheduling class

10 files changed, 117 insertions, 0 deletions

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 3765164809c5..aa63a2cafc1f 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -32,6 +32,13 @@ config X86_64
         select SWIOTLB
         select ARCH_HAS_SYSCALL_WRAPPER
 
+config KUTRACE
+        bool "kernel/user tracing, dsites"
+        depends on 64BIT
+        def_bool y
+        ---help---
+         Enables kernel/user tracing patches
+
 config FORCE_DYNAMIC_FTRACE
         def_bool y
         depends on X86_32
diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index 3f8e22615812..24f30bb65ac4 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -37,6 +37,8 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/syscalls.h>
 
+#include <linux/kutrace.h>
+
 #ifdef CONFIG_CONTEXT_TRACKING
 /* Called on entry from user mode with IRQs off. */
 __visible inline void enter_from_user_mode(void)
@@ -287,7 +289,29 @@ __visible void do_syscall_64(unsigned long nr, struct pt_regs *regs)
 
         if (likely(nr < NR_syscalls)) {
                 nr = array_index_nospec(nr, NR_syscalls);
+                /* dsites 2019.03.05 track all syscalls and normal returns */
+                /* Pass in low 16 bits of call arg0 and return value */
+                kutrace1(KUTRACE_SYSCALL64 | kutrace_map_nr(nr), regs->di & 0xFFFFul);
+
                 regs->ax = sys_call_table[nr](regs);
+
+                /* dsites 2019.03.05 track all syscalls and normal returns */
+                /* Pass in low 16 bits of return value */
+                kutrace1(KUTRACE_SYSRET64 | kutrace_map_nr(nr), regs->ax & 0xFFFFul);
+#ifdef CONFIG_KUTRACE
+        } else if ((nr == __NR_kutrace_control) &&
+                           (kutrace_global_ops.kutrace_trace_control != NULL)) {
+                BUILD_BUG_ON_MSG(NR_syscalls > __NR_kutrace_control,
+                                "__NR_kutrace_control is too small");
+                BUILD_BUG_ON_MSG(16 > TASK_COMM_LEN,
+                                "TASK_COMM_LEN is less than 16");
+
+                /* Calling kutrace_control(u64 command, u64 arg) */
+                /* see arch/x86/calling.h: */
+                /* syscall arg0 in rdi (command), arg1 in rsi (arg) */
+                regs->ax = (*kutrace_global_ops.kutrace_trace_control)(
+                        regs->di, regs->si);
+#endif
 #ifdef CONFIG_X86_X32_ABI
         } else if (likely((nr & __X32_SYSCALL_BIT) &&
                           (nr & ~__X32_SYSCALL_BIT) < X32_NR_syscalls)) {
diff --git a/arch/x86/kernel/acpi/cstate.c b/arch/x86/kernel/acpi/cstate.c
index caf2edccbad2..503712c45066 100644
--- a/arch/x86/kernel/acpi/cstate.c
+++ b/arch/x86/kernel/acpi/cstate.c
@@ -16,6 +16,8 @@
 #include <asm/mwait.h>
 #include <asm/special_insns.h>
 
+#include <linux/kutrace.h>
+
 /*
  * Initialize bm_flags based on the CPU cache properties
  * On SMP it depends on cache configuration
@@ -186,6 +188,9 @@ void __cpuidle acpi_processor_ffh_cstate_enter(struct acpi_processor_cx *cx)
         struct cstate_entry *percpu_entry;
 
         percpu_entry = per_cpu_ptr(cpu_cstate_entry, cpu);
+
+        kutrace1(KUTRACE_MWAIT, percpu_entry->states[cx->index].eax);
+
         mwait_idle_with_hints(percpu_entry->states[cx->index].eax,
                               percpu_entry->states[cx->index].ecx);
 }
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 2b0faf86da1b..4d6a2a7569db 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -60,6 +60,9 @@
 #include <asm/intel-family.h>
 #include <asm/irq_regs.h>
 
+#include <linux/kutrace.h>
+#include <linux/cpufreq.h>
+
 unsigned int num_processors;
 
 unsigned disabled_cpus;
@@ -1131,9 +1134,28 @@ __visible void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
          * interrupt lock, which is the WrongThing (tm) to do.
          */
         entering_ack_irq();
+
+        kutrace1(KUTRACE_IRQ + LOCAL_TIMER_VECTOR, 0);
+
         trace_local_timer_entry(LOCAL_TIMER_VECTOR);
         local_apic_timer_interrupt();
         trace_local_timer_exit(LOCAL_TIMER_VECTOR);
+
+        kutrace1(KUTRACE_IRQRET + LOCAL_TIMER_VECTOR, 0);
+
+        /* dsites 2020.05.02 Trace current CPU frequency in MHz */
+        /* NOT FINISHED. Recent Intel only. Not AMD. Not ARM */
+        //{
+        //      u64 mhz;
+        //      kutrace_freq(mhz);
+        //      kutrace1(KUTRACE_PSTATE, mhz);
+        //}
+
+        /* dsites 2020.01.29 Trace return address -- we are also a profiler now */
+        if (kutrace_tracing) {
+                (*kutrace_global_ops.kutrace_trace_2)(KUTRACE_PC, 0, regs->ip);
+        }
+
         exiting_irq();
 
         set_irq_regs(old_regs);
diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c
index 6ca0f91372fd..b8fac56af4d8 100644
--- a/arch/x86/kernel/apic/ipi.c
+++ b/arch/x86/kernel/apic/ipi.c
@@ -5,6 +5,8 @@
 
 #include "local.h"
 
+#include <linux/kutrace.h>
+
 DEFINE_STATIC_KEY_FALSE(apic_use_ipi_shorthand);
 
 #ifdef CONFIG_SMP
@@ -67,16 +69,25 @@ void native_smp_send_reschedule(int cpu)
                 WARN(1, "sched: Unexpected reschedule of offline CPU#%d!\n", cpu);
                 return;
         }
+
+        kutrace1(KUTRACE_IPI, cpu);
+
         apic->send_IPI(cpu, RESCHEDULE_VECTOR);
 }
 
 void native_send_call_func_single_ipi(int cpu)
 {
+        kutrace1(KUTRACE_IPI, cpu);
+
         apic->send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR);
 }
 
 void native_send_call_func_ipi(const struct cpumask *mask)
 {
+        /* dsites 2019.03.06 */
+        /* Use CPU 0 as a placeholder to indicate when mask was sent */
+        kutrace1(KUTRACE_IPI, 0);
+
         if (static_branch_likely(&apic_use_ipi_shorthand)) {
                 unsigned int cpu = smp_processor_id();
 
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 21efee32e2b1..e643db0fb67e 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -23,6 +23,8 @@
 #define CREATE_TRACE_POINTS
 #include <asm/trace/irq_vectors.h>
 
+#include <linux/kutrace.h>
+
 DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 EXPORT_PER_CPU_SYMBOL(irq_stat);
 
@@ -239,6 +241,8 @@ __visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
 
         entering_irq();
 
+        kutrace1(KUTRACE_IRQ + (vector & 0xFF), 0);
+
         /* entering_irq() tells RCU that we're not quiescent.  Check it. */
         RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");
 
@@ -260,6 +264,8 @@ __visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
                 }
         }
 
+        kutrace1(KUTRACE_IRQRET + (vector & 0xFF), 0);
+
         exiting_irq();
 
         set_irq_regs(old_regs);
@@ -277,11 +283,17 @@ __visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs)
         struct pt_regs *old_regs = set_irq_regs(regs);
 
         entering_ack_irq();
+
+        kutrace1(KUTRACE_IRQ + X86_PLATFORM_IPI_VECTOR, 0);
+
         trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
         inc_irq_stat(x86_platform_ipis);
         if (x86_platform_ipi_callback)
                 x86_platform_ipi_callback();
         trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
+
+        kutrace1(KUTRACE_IRQRET + X86_PLATFORM_IPI_VECTOR, 0);
+
         exiting_irq();
         set_irq_regs(old_regs);
 }
diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c
index 80bee7695a20..964610b2d6dc 100644
--- a/arch/x86/kernel/irq_work.c
+++ b/arch/x86/kernel/irq_work.c
@@ -12,14 +12,22 @@
 #include <asm/trace/irq_vectors.h>
 #include <linux/interrupt.h>
 
+#include <linux/kutrace.h>
+
 #ifdef CONFIG_X86_LOCAL_APIC
 __visible void __irq_entry smp_irq_work_interrupt(struct pt_regs *regs)
 {
         ipi_entering_ack_irq();
+
+        kutrace1(KUTRACE_IRQ + IRQ_WORK_VECTOR, 0);
+
         trace_irq_work_entry(IRQ_WORK_VECTOR);
         inc_irq_stat(apic_irq_work_irqs);
         irq_work_run();
         trace_irq_work_exit(IRQ_WORK_VECTOR);
+
+        kutrace1(KUTRACE_IRQRET + IRQ_WORK_VECTOR, 0);
+
         exiting_irq();
 }
 
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index b8d4e9c3c070..d9d8fed828e4 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -33,6 +33,8 @@
 #include <asm/kexec.h>
 #include <asm/virtext.h>
 
+#include <linux/kutrace.h>
+
 /*
  *      Some notes on x86 processor bugs affecting SMP operation:
  *
@@ -226,6 +228,9 @@ static void native_stop_other_cpus(int wait)
 __visible void __irq_entry smp_reschedule_interrupt(struct pt_regs *regs)
 {
         ack_APIC_irq();
+
+        kutrace1(KUTRACE_IRQ + RESCHEDULE_VECTOR, 0);
+
         inc_irq_stat(irq_resched_count);
         kvm_set_cpu_l1tf_flush_l1d();
 
@@ -242,25 +247,38 @@ __visible void __irq_entry smp_reschedule_interrupt(struct pt_regs *regs)
                 return;
         }
         scheduler_ipi();
+        /* dsites 2019.03.06 continued in sched/core.c */
 }
 
 __visible void __irq_entry smp_call_function_interrupt(struct pt_regs *regs)
 {
         ipi_entering_ack_irq();
+
+        kutrace1(KUTRACE_IRQ + CALL_FUNCTION_VECTOR, 0);
+
         trace_call_function_entry(CALL_FUNCTION_VECTOR);
         inc_irq_stat(irq_call_count);
         generic_smp_call_function_interrupt();
         trace_call_function_exit(CALL_FUNCTION_VECTOR);
+
+        kutrace1(KUTRACE_IRQRET + CALL_FUNCTION_VECTOR, 0);
+
         exiting_irq();
 }
 
 __visible void __irq_entry smp_call_function_single_interrupt(struct pt_regs *r)
 {
         ipi_entering_ack_irq();
+
+        kutrace1(KUTRACE_IRQ + CALL_FUNCTION_SINGLE_VECTOR, 0);
+
         trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR);
         inc_irq_stat(irq_call_count);
         generic_smp_call_function_single_interrupt();
         trace_call_function_single_exit(CALL_FUNCTION_SINGLE_VECTOR);
+
+        kutrace1(KUTRACE_IRQRET + CALL_FUNCTION_SINGLE_VECTOR, 0);
+
         exiting_irq();
 }
 
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 9ceacd1156db..6bc4011901a0 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -33,6 +33,8 @@
 #define CREATE_TRACE_POINTS
 #include <asm/trace/exceptions.h>
 
+#include <linux/kutrace.h>
+
 /*
  * Returns 0 if mmiotrace is disabled, or if the fault is not
  * handled by mmiotrace:
@@ -1527,7 +1529,13 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long addr
 
         prev_state = exception_enter();
         trace_page_fault_entries(regs, error_code, address);
+
+        kutrace1(KUTRACE_TRAP + KUTRACE_PAGEFAULT, 0);
+
         __do_page_fault(regs, error_code, address);
+
+        kutrace1(KUTRACE_TRAPRET + KUTRACE_PAGEFAULT, 0);
+
         exception_exit(prev_state);
 }
 NOKPROBE_SYMBOL(do_page_fault);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index e6a9edc5baaf..36fac9e7b37f 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -18,6 +18,8 @@
 
 #include "mm_internal.h"
 
+#include <linux/kutrace.h>
+
 /*
  *      TLB flushing, formerly SMP-only
  *              c/o Linus Torvalds.