performance counters: core code

Implement the core kernel bits of Performance Counters subsystem.

The Linux Performance Counter subsystem provides an abstraction of
performance counter hardware capabilities. It provides per task and per
CPU counters, and it provides event capabilities on top of those.

Performance counters are accessed via special file descriptors.
There's one file descriptor per virtual counter used.

The special file descriptor is opened via the perf_counter_open()
system call:

 int
 perf_counter_open(u32 hw_event_type,
                   u32 hw_event_period,
                   u32 record_type,
                   pid_t pid,
                   int cpu);

The syscall returns the new fd. The fd can be used via the normal
VFS system calls: read() can be used to read the counter, fcntl()
can be used to set the blocking mode, etc.

Multiple counters can be kept open at a time, and the counters
can be poll()ed.

See more details in Documentation/perf-counters.txt.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

5 files changed, 972 insertions, 0 deletions

diff --git a/kernel/Makefile b/kernel/Makefile
index 19fad003b19d..1f184a1dc406 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -89,6 +89,7 @@ obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += dma-coherent.o
 obj-$(CONFIG_FUNCTION_TRACER) += trace/
 obj-$(CONFIG_TRACING) += trace/
 obj-$(CONFIG_SMP) += sched_cpupri.o
+obj-$(CONFIG_PERF_COUNTERS) += perf_counter.o
 
 ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
 # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
diff --git a/kernel/fork.c b/kernel/fork.c
index 2a372a0e206f..441fadff1fa4 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -975,6 +975,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
                 goto fork_out;
 
         rt_mutex_init_task(p);
+        perf_counter_init_task(p);
 
 #ifdef CONFIG_PROVE_LOCKING
         DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
new file mode 100644
index 000000000000..20508f053658
--- /dev/null
+++ b/kernel/perf_counter.c
@@ -0,0 +1,943 @@
+/*
+ * Performance counter core code
+ *
+ *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
+ *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
+ *
+ *  For licencing details see kernel-base/COPYING
+ */
+
+#include <linux/fs.h>
+#include <linux/cpu.h>
+#include <linux/smp.h>
+#include <linux/poll.h>
+#include <linux/sysfs.h>
+#include <linux/ptrace.h>
+#include <linux/percpu.h>
+#include <linux/uaccess.h>
+#include <linux/syscalls.h>
+#include <linux/anon_inodes.h>
+#include <linux/perf_counter.h>
+
+/*
+ * Each CPU has a list of per CPU counters:
+ */
+DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
+
+int perf_max_counters __read_mostly;
+static int perf_reserved_percpu __read_mostly;
+static int perf_overcommit __read_mostly = 1;
+
+/*
+ * Mutex for (sysadmin-configurable) counter reservations:
+ */
+static DEFINE_MUTEX(perf_resource_mutex);
+
+/*
+ * Architecture provided APIs - weak aliases:
+ */
+
+int __weak hw_perf_counter_init(struct perf_counter *counter, u32 hw_event_type)
+{
+        return -EINVAL;
+}
+
+void __weak hw_perf_counter_enable(struct perf_counter *counter)         { }
+void __weak hw_perf_counter_disable(struct perf_counter *counter)        { }
+void __weak hw_perf_counter_read(struct perf_counter *counter)           { }
+void __weak hw_perf_disable_all(void) { }
+void __weak hw_perf_enable_all(void) { }
+void __weak hw_perf_counter_setup(void) { }
+
+#if BITS_PER_LONG == 64
+
+/*
+ * Read the cached counter in counter safe against cross CPU / NMI
+ * modifications. 64 bit version - no complications.
+ */
+static inline u64 perf_read_counter_safe(struct perf_counter *counter)
+{
+        return (u64) atomic64_read(&counter->count);
+}
+
+#else
+
+/*
+ * Read the cached counter in counter safe against cross CPU / NMI
+ * modifications. 32 bit version.
+ */
+static u64 perf_read_counter_safe(struct perf_counter *counter)
+{
+        u32 cntl, cnth;
+
+        local_irq_disable();
+        do {
+                cnth = atomic_read(&counter->count32[1]);
+                cntl = atomic_read(&counter->count32[0]);
+        } while (cnth != atomic_read(&counter->count32[1]));
+
+        local_irq_enable();
+
+        return cntl | ((u64) cnth) << 32;
+}
+
+#endif
+
+/*
+ * Cross CPU call to remove a performance counter
+ *
+ * We disable the counter on the hardware level first. After that we
+ * remove it from the context list.
+ */
+static void __perf_remove_from_context(void *info)
+{
+        struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+        struct perf_counter *counter = info;
+        struct perf_counter_context *ctx = counter->ctx;
+
+        /*
+         * If this is a task context, we need to check whether it is
+         * the current task context of this cpu. If not it has been
+         * scheduled out before the smp call arrived.
+         */
+        if (ctx->task && cpuctx->task_ctx != ctx)
+                return;
+
+        spin_lock(&ctx->lock);
+
+        if (counter->active) {
+                hw_perf_counter_disable(counter);
+                counter->active = 0;
+                ctx->nr_active--;
+                cpuctx->active_oncpu--;
+                counter->task = NULL;
+        }
+        ctx->nr_counters--;
+
+        /*
+         * Protect the list operation against NMI by disabling the
+         * counters on a global level. NOP for non NMI based counters.
+         */
+        hw_perf_disable_all();
+        list_del_init(&counter->list);
+        hw_perf_enable_all();
+
+        if (!ctx->task) {
+                /*
+                 * Allow more per task counters with respect to the
+                 * reservation:
+                 */
+                cpuctx->max_pertask =
+                        min(perf_max_counters - ctx->nr_counters,
+                            perf_max_counters - perf_reserved_percpu);
+        }
+
+        spin_unlock(&ctx->lock);
+}
+
+
+/*
+ * Remove the counter from a task's (or a CPU's) list of counters.
+ *
+ * Must be called with counter->mutex held.
+ *
+ * CPU counters are removed with a smp call. For task counters we only
+ * call when the task is on a CPU.
+ */
+static void perf_remove_from_context(struct perf_counter *counter)
+{
+        struct perf_counter_context *ctx = counter->ctx;
+        struct task_struct *task = ctx->task;
+
+        if (!task) {
+                /*
+                 * Per cpu counters are removed via an smp call and
+                 * the removal is always sucessful.
+                 */
+                smp_call_function_single(counter->cpu,
+                                         __perf_remove_from_context,
+                                         counter, 1);
+                return;
+        }
+
+retry:
+        task_oncpu_function_call(task, __perf_remove_from_context,
+                                 counter);
+
+        spin_lock_irq(&ctx->lock);