Merge branch 'wip-mc' of ssh://rtsrv.cs.unc.edu/home/litmus/litmus-rt into wip-mc

13 files changed, 134 insertions, 42 deletions

diff --git a/arch/arm/include/asm/glue-cache.h b/arch/arm/include/asm/glue-cache.h
index 7e30874377e..4f8d2c0dc44 100644
--- a/arch/arm/include/asm/glue-cache.h
+++ b/arch/arm/include/asm/glue-cache.h
@@ -110,19 +110,19 @@
 #endif
 
 #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K)
-//# ifdef _CACHE
+# ifdef _CACHE
 #  define MULTI_CACHE 1
-//# else
-//#  define _CACHE v6
-//# endif
+# else
+#  define _CACHE v6
+# endif
 #endif
 
 #if defined(CONFIG_CPU_V7)
-//# ifdef _CACHE
+# ifdef _CACHE
 #  define MULTI_CACHE 1
-//# else
-//#  define _CACHE v7
-//# endif
+# else
+#  define _CACHE v7
+# endif
 #endif
 
 #if !defined(_CACHE) && !defined(MULTI_CACHE)
diff --git a/arch/arm/include/asm/hardware/cache-l2x0.h b/arch/arm/include/asm/hardware/cache-l2x0.h
index 16bd4803158..9e2fd1889b6 100644
--- a/arch/arm/include/asm/hardware/cache-l2x0.h
+++ b/arch/arm/include/asm/hardware/cache-l2x0.h
@@ -45,8 +45,13 @@
 #define L2X0_CLEAN_INV_LINE_PA          0x7F0
 #define L2X0_CLEAN_INV_LINE_IDX         0x7F8
 #define L2X0_CLEAN_INV_WAY              0x7FC
-#define L2X0_LOCKDOWN_WAY_D             0x900
-#define L2X0_LOCKDOWN_WAY_I             0x904
+/*
+ * The lockdown registers repeat 8 times for L310, the L210 has only one
+ * D and one I lockdown register at 0x0900 and 0x0904.
+ */
+#define L2X0_LOCKDOWN_WAY_D_BASE        0x900
+#define L2X0_LOCKDOWN_WAY_I_BASE        0x904
+#define L2X0_LOCKDOWN_STRIDE            0x08
 #define L2X0_TEST_OPERATION             0xF00
 #define L2X0_LINE_DATA                  0xF10
 #define L2X0_LINE_TAG                   0xF30
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index 83a9c076931..4f98e8e0783 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -279,6 +279,25 @@ static void l2x0_disable(void)
         spin_unlock_irqrestore(&l2x0_lock, flags);
 }
 
+static void __init l2x0_unlock(__u32 cache_id)
+{
+        int lockregs;
+        int i;
+
+        if (cache_id == L2X0_CACHE_ID_PART_L310)
+                lockregs = 8;
+        else
+                /* L210 and unknown types */
+                lockregs = 1;
+
+        for (i = 0; i < lockregs; i++) {
+                writel_relaxed(0x0, l2x0_base + L2X0_LOCKDOWN_WAY_D_BASE +
+                               i * L2X0_LOCKDOWN_STRIDE);
+                writel_relaxed(0x0, l2x0_base + L2X0_LOCKDOWN_WAY_I_BASE +
+                               i * L2X0_LOCKDOWN_STRIDE);
+        }
+}
+
 void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
 {
         __u32 aux;
@@ -330,6 +349,8 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
          * accessing the below registers will fault.
          */
         if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) {
+                /* Make sure that I&D is not locked down when starting */
+                l2x0_unlock(l2x0_cache_id);
 
                 /* l2x0 controller is disabled */
                 writel_relaxed(aux, l2x0_base + L2X0_AUX_CTRL);
diff --git a/include/litmus/color.h b/include/litmus/color.h
index eefb6c6dddf..615ebd80703 100644
--- a/include/litmus/color.h
+++ b/include/litmus/color.h
@@ -6,6 +6,12 @@
 #define ONE_COLOR_LEN 11
 #define ONE_COLOR_FMT "%4lu: %4d\n"
 
+#ifdef CONFIG_PHYS_ADDR_T_64BIT
+#define FMT_PA "llx"
+#else
+#define FMT_PA "lx"
+#endif
+
 struct color_cache_info {
         unsigned long size;
         unsigned long line_size;
@@ -18,6 +24,9 @@ struct color_cache_info {
 extern struct color_cache_info color_cache_info;
 extern unsigned long color_chunk;
 
+struct vm_area_struct;
+struct ctl_table;
+
 struct page* get_colored_page(unsigned long);
 void add_page_to_color_list(struct page*);
 void add_page_to_alloced_list(struct page*, struct vm_area_struct*);
diff --git a/include/litmus/fpmath.h b/include/litmus/fpmath.h
index 04d4bcaeae9..642de98542c 100644
--- a/include/litmus/fpmath.h
+++ b/include/litmus/fpmath.h
@@ -1,6 +1,8 @@
 #ifndef __FP_MATH_H__
 #define __FP_MATH_H__
 
+#include <linux/math64.h>
+
 #ifndef __KERNEL__
 #include <stdint.h>
 #define abs(x) (((x) < 0) ? -(x) : x)
@@ -32,7 +34,7 @@ static inline fp_t FP(fpbuf_t x)
 /* divide two integers to obtain a fixed point value  */
 static inline fp_t _frac(fpbuf_t a, fpbuf_t b)
 {
-        return _fp(FP(a).val / (b));
+        return _fp(div64_s64(FP(a).val, (b)));
 }
 
 static inline fpbuf_t _point(fp_t x)
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index a78f7f94344..b273618fb83 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -14,6 +14,10 @@ extern atomic_t release_master_cpu;
 
 void litmus_schedule_deallocation(struct task_struct *t);
 
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+void * remap_noncached(void *addr, size_t len);
+#endif
+
 /* in_list - is a given list_head queued on some list?
  */
 static inline int in_list(struct list_head* list)
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 1ee65e19d7b..e9e26faddce 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -277,6 +277,9 @@ struct rt_param {
         lt_t last_exec_time;
         lt_t orig_cost;
         struct color_ctrl_page *color_ctrl_page;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        void *color_ctrl_page_orig;
+#endif
         struct dgl_group_req *req;
 };
 
diff --git a/include/litmus/trace_irq.h b/include/litmus/trace_irq.h
index b717b1d5539..01e4b98fc3e 100644
--- a/include/litmus/trace_irq.h
+++ b/include/litmus/trace_irq.h
@@ -3,6 +3,7 @@
 
 #ifdef CONFIG_SCHED_OVERHEAD_TRACE
 
+#include <asm/atomic.h>
 #include <linux/percpu.h>
 
 extern DEFINE_PER_CPU(atomic_t, irq_fired_count);
diff --git a/litmus/color.c b/litmus/color.c
index b2834b36939..976366e7e39 100644
--- a/litmus/color.c
+++ b/litmus/color.c
@@ -198,7 +198,7 @@ void reclaim_pages(struct vm_area_struct *vma)
                         put_page(ap->page);
                         add_page_to_color_list(ap->page);
                         nr_reclaimed++;
-                        TRACE_CUR("reclaiming page (pa:0x%10llx, pfn:%8lu, "
+                        TRACE_CUR("reclaiming page (pa:0x%10"FMT_PA", pfn:%8lu, "
                                         "color:%3lu)\n", page_to_phys(ap->page),
                                         page_to_pfn(ap->page), page_color(ap->page));
                         kfree(ap);
diff --git a/litmus/color_dev.c b/litmus/color_dev.c
index 3ed584ad7af..6287d14405e 100644
--- a/litmus/color_dev.c
+++ b/litmus/color_dev.c
@@ -7,6 +7,8 @@
 #include <linux/highmem.h>
 #include <asm/io.h> /* page_to_phys on SPARC */
 
+#include <asm/glue-cache.h>
+
 #include <litmus/litmus.h>
 #include <litmus/color.h>
 
@@ -82,17 +84,35 @@ static struct vm_operations_struct litmus_color_ctrl_vm_ops = {
 
 static int alloc_color_ctrl_page(void)
 {
+        void *mem;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        void *mem_remap;
+#endif
         struct task_struct *t = current;
         int err = 0;
 
         /* only allocate if the task doesn't have one yet */
         if (!tsk_rt(t)->color_ctrl_page) {
-                tsk_rt(t)->color_ctrl_page = (void*) get_zeroed_page(GFP_KERNEL);
-                if (!tsk_rt(t)->color_ctrl_page) {
-                        TRACE_TASK(t, "could not allocate control page\n");
+                mem = (void*) get_zeroed_page(GFP_KERNEL);
+                if (!mem) {
+                        TRACE_TASK(t, "cannot allocate color control page\n");
                         err = -ENOMEM;
                         goto out;
                 }
+
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+                mem_remap = remap_noncached(mem, PAGE_SIZE);
+                if (IS_ERR(mem_remap)) {
+                        err = PTR_ERR(mem_remap);
+                        free_page((unsigned long) mem);
+                        goto out;
+                }
+                tsk_rt(t)->color_ctrl_page_orig = mem;
+                tsk_rt(t)->color_ctrl_page = mem_remap;
+#else
+                tsk_rt(t)->color_ctrl_page = mem;
+#endif
+
                 /* will get de-allocated in task teardown */
                 TRACE_TASK(t, "color_ctrl_page = %p\n",
                            tsk_rt(t)->color_ctrl_page);
@@ -105,7 +125,14 @@ static int map_color_ctrl_page(struct vm_area_struct *vma)
 {
         int err = 0;
         struct task_struct *t = current;
-        struct page *color_ctrl = virt_to_page(tsk_rt(t)->color_ctrl_page);
+        struct page *color_ctrl;
+
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        /* vm_insert_page() using the "real" vaddr, not the shadow mapping. */
+        color_ctrl = virt_to_page(tsk_rt(t)->color_ctrl_page_orig);
+#else
+        color_ctrl = virt_to_page(tsk_rt(t)->color_ctrl_page);
+#endif
 
         TRACE_CUR("mapping %p (pfn:%lx) to 0x%lx (flags:%lx prot:%lx)\n",
                   tsk_rt(t)->color_ctrl_page, page_to_pfn(color_ctrl),
@@ -139,6 +166,10 @@ static int litmus_color_ctrl_mmap(struct file *filp, struct vm_area_struct *vma)
         vma->vm_ops = &litmus_color_ctrl_vm_ops;
         mmap_common_vma_flags(vma);
 
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+#endif
+
         err = alloc_color_ctrl_page();
         if (!err)
                 err = map_color_ctrl_page(vma);
@@ -164,6 +195,9 @@ static int do_map_colored_page_set(struct vm_area_struct *vma,
 {
         int i, err = 0;
 
+        TRACE_CUR("vma: %p  starting_at: 0x%10lx  color_no: %lu  nr_pages: %lu\n",
+                        vma, starting_at, color_no, nr_pages);
+
         for (i = 0; i < nr_pages; ++i) {
                 const unsigned long addr = starting_at + (i << PAGE_SHIFT);
                 struct page *page = get_colored_page(color_no);
@@ -176,16 +210,13 @@ static int do_map_colored_page_set(struct vm_area_struct *vma,
                         goto out;
                 }
 
-                clear_user_highpage(page, addr);
+                /* TODO For some reason this causes problems on ARM. */
+                /* clear_user_highpage(page, addr); */
 
-                TRACE_CUR("insert page (pa:0x%10llx, pfn:%8lu, color:%3lu) "
-                                "at 0x%lx vma:(flags:0x%16lx prot:0x%16lx)\n",
-                                page_to_phys(page),
-                                page_to_pfn(page),
-                                color_no,
-                                addr,
-                                vma->vm_flags,
-                                pgprot_val(vma->vm_page_prot));
+                TRACE_CUR("inserting page (pa: 0x%"FMT_PA") at vaddr: 0x%10lx  "
+                                "flags: 0x%10lx  prot: 0x%10lx\n",
+                                page_to_phys(page), addr,
+                                vma->vm_flags, pgprot_val(vma->vm_page_prot));
 
                 err = vm_insert_page(vma, addr, page);
                 if (err) {
@@ -207,14 +238,14 @@ static int do_map_colored_pages(struct vm_area_struct *vma)
         unsigned long nr_mapped;
         int i, err = 0;
 
-        TRACE_CUR("allocating %lu pages (flags:%lx prot:%lx)\n",
-                        nr_pages, vma->vm_flags, pgprot_val(vma->vm_page_prot));
-
 #ifdef CONFIG_PLUGIN_COLOR_UNCACHABLE
         /* Adds non-cached on x86 */
         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 #endif
 
+        TRACE_CUR("allocating %lu pages (flags:%lx prot:%lx)\n",
+                        nr_pages, vma->vm_flags, pgprot_val(vma->vm_page_prot));
+
         for (i = 0, nr_mapped = 0; nr_mapped < nr_pages; ++i) {
                 const unsigned long color_no = color_ctrl->colors[i];
                 const unsigned long color_quantity = color_ctrl->pages[i];
@@ -231,7 +262,7 @@ static int do_map_colored_pages(struct vm_area_struct *vma)
                         goto out;
                 }
 
-                TRACE_CUR("addr: %lu  nr_mapped: %lu color number: %lu  "
+                TRACE_CUR("addr: 0x%10lx  nr_mapped: %lu color number: %lu  "
                                 "color quantity: %lu\n",
                                 addr, nr_mapped, color_no,
                                 color_quantity);
diff --git a/litmus/color_proc.c b/litmus/color_proc.c
index 78657bd4c7a..8ee624860d7 100644
--- a/litmus/color_proc.c
+++ b/litmus/color_proc.c
@@ -50,6 +50,13 @@ static struct ctl_table cache_table[] =
                 .data           = &color_cache_info.line_size,
                 .maxlen         = sizeof(color_cache_info.line_size),
         },
+        {
+                .procname       = "nr_colors",
+                .mode           = 0444,
+                .proc_handler   = proc_doulongvec_minmax,
+                .data           = &color_cache_info.nr_colors,
+                .maxlen         = sizeof(color_cache_info.nr_colors),
+        },
         { }
 };
 
diff --git a/litmus/ctrldev.c b/litmus/ctrldev.c
index 07c3a680ca6..87c21609964 100644
--- a/litmus/ctrldev.c
+++ b/litmus/ctrldev.c
@@ -19,8 +19,10 @@ static struct workqueue_struct *wq_litmus_dealloc;
 struct litmus_dealloc_work {
         struct work_struct work_struct;
         void *ctrl_page_mem;
+        void *color_ctrl_page_mem;
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
         void *ctrl_page_unmap;
+        void *color_ctrl_page_unmap;
 #endif
 };
 
@@ -31,10 +33,16 @@ static void litmus_dealloc(struct work_struct *work_in)
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
         TRACE("vunmap() control page %p.\n", work->ctrl_page_unmap);
         vunmap(work->ctrl_page_unmap);
+
+        TRACE("vunmap() color control page %p.\n", work->color_ctrl_page_unmap);
+        vunmap(work->color_ctrl_page_unmap);
 #endif
         TRACE("freeing ctrl_page %p\n", work->ctrl_page_mem);
         free_page((unsigned long) work->ctrl_page_mem);
 
+        TRACE("freeing color_ctrl_page %p\n", work->color_ctrl_page_mem);
+        free_page((unsigned long) work->color_ctrl_page_mem);
+
         kfree((void*) work);
 }
 
@@ -42,7 +50,7 @@ void litmus_schedule_deallocation(struct task_struct *t)
 {
         struct litmus_dealloc_work *work;
 
-        if (NULL == tsk_rt(t)->ctrl_page)
+        if (NULL == tsk_rt(t)->ctrl_page && NULL == tsk_rt(t)->color_ctrl_page)
                 return;
 
         work = kmalloc(sizeof(*work), GFP_ATOMIC);
@@ -56,8 +64,11 @@ void litmus_schedule_deallocation(struct task_struct *t)
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
         work->ctrl_page_mem = tsk_rt(t)->ctrl_page_orig;
         work->ctrl_page_unmap = tsk_rt(t)->ctrl_page;
+        work->color_ctrl_page_mem = tsk_rt(t)->color_ctrl_page_orig;
+        work->color_ctrl_page_unmap = tsk_rt(t)->color_ctrl_page;
 #else
         work->ctrl_page_mem = tsk_rt(t)->ctrl_page;
+        work->color_ctrl_page_mem = tsk_rt(t)->color_ctrl_page;
 #endif
         queue_work(wq_litmus_dealloc, &work->work_struct);
 }
@@ -71,7 +82,7 @@ void litmus_schedule_deallocation(struct task_struct *t)
  * The caller should vunmap(addr) when the mapping is no longer needed.
  * The caller should also save the original @addr to free it later.
  */
-static void * remap_noncached(void *addr, size_t len)
+void * remap_noncached(void *addr, size_t len)
 {
         void *vaddr;
         int nr_pages = DIV_ROUND_UP(offset_in_page(addr) + len, PAGE_SIZE);
diff --git a/litmus/litmus.c b/litmus/litmus.c
index f8397898ce0..1bbad8db1a5 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -393,7 +393,10 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
         void * ctrl_page_orig = NULL;
 #endif
 
-        void*  color_ctrl_page = NULL;
+        void *color_ctrl_page = NULL;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        void *color_ctrl_page_orig = NULL;
+#endif
 
         if (restore) {
                 /* Safe user-space provided configuration data.
@@ -403,6 +406,7 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
                 color_ctrl_page = p->rt_param.color_ctrl_page;
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
                 ctrl_page_orig = p->rt_param.ctrl_page_orig;
+                color_ctrl_page_orig = p->rt_param.color_ctrl_page_orig;
 #endif
         }
 
@@ -421,6 +425,7 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
                 p->rt_param.color_ctrl_page     = color_ctrl_page;
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
                 p->rt_param.ctrl_page_orig = ctrl_page_orig;
+                p->rt_param.color_ctrl_page_orig = color_ctrl_page_orig;
 #endif
         }
 }
@@ -573,6 +578,7 @@ void litmus_fork(struct task_struct* p)
                 tsk_rt(p)->color_ctrl_page = NULL;
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
                 tsk_rt(p)->ctrl_page_orig = NULL;
+                tsk_rt(p)->color_ctrl_page_orig = NULL;
 #endif
         }
 
@@ -590,14 +596,12 @@ void litmus_exec(void)
 
         if (is_realtime(p)) {
                 WARN_ON(p->rt_param.inh_task);
-                if (tsk_rt(p)->color_ctrl_page) {
-                        free_page((unsigned long) tsk_rt(p)->color_ctrl_page);
-                        tsk_rt(p)->color_ctrl_page = NULL;
-                }
                 litmus_schedule_deallocation(p);
                 tsk_rt(p)->ctrl_page = NULL;
+                tsk_rt(p)->color_ctrl_page = NULL;
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
                 tsk_rt(p)->ctrl_page_orig = NULL;
+                tsk_rt(p)->color_ctrl_page_orig = NULL;
 #endif
         }
 }
@@ -611,12 +615,6 @@ void exit_litmus(struct task_struct *dead_tsk)
          * So check if we need to free the page
          * in any case.
          */
-        if (tsk_rt(dead_tsk)->color_ctrl_page) {
-                TRACE_TASK(dead_tsk,
-                                "freeing color_ctrl_page %p\n",
-                                tsk_rt(dead_tsk)->color_ctrl_page);
-                free_page((unsigned long) tsk_rt(dead_tsk)->color_ctrl_page);
-        }
 
 #ifdef CONFIG_PLUGIN_MC
         /* The MC-setup syscall might succeed and allocate mc_data, but the