uncachable color control page

Signed-off-by: Christopher Kenna <cjk@cs.unc.edu>

9 files changed, 93 insertions, 31 deletions

diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index 83a9c076931b..2cbd6ca0ce23 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -354,6 +354,8 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask)
         color_cache_info.ways = l2x0_ways;
         color_cache_info.sets = l2x0_sets;
 
+        litmus_setup_lockdown(l2x0_base);
+
         printk(KERN_INFO "%s cache controller enabled\n", type);
         printk(KERN_INFO "l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x, Cache size: %d B\n",
                         ways, cache_id, aux, l2x0_size);
diff --git a/include/litmus/color.h b/include/litmus/color.h
index eefb6c6dddf5..0e67028f8ea3 100644
--- a/include/litmus/color.h
+++ b/include/litmus/color.h
@@ -6,6 +6,12 @@
 #define ONE_COLOR_LEN 11
 #define ONE_COLOR_FMT "%4lu: %4d\n"
 
+#ifdef CONFIG_PHYS_ADDR_T_64BIT
+#define FMT_PA "llx"
+#else
+#define FMT_PA "lx"
+#endif
+
 struct color_cache_info {
         unsigned long size;
         unsigned long line_size;
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 93961d08efd5..91623528ad9e 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -14,6 +14,10 @@ extern atomic_t release_master_cpu;
 
 void litmus_schedule_deallocation(struct task_struct *t);
 
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+void * remap_noncached(void *addr, size_t len);
+#endif
+
 /* in_list - is a given list_head queued on some list?
  */
 static inline int in_list(struct list_head* list)
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 1ee65e19d7b7..e9e26faddcea 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -277,6 +277,9 @@ struct rt_param {
         lt_t last_exec_time;
         lt_t orig_cost;
         struct color_ctrl_page *color_ctrl_page;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        void *color_ctrl_page_orig;
+#endif
         struct dgl_group_req *req;
 };
 
diff --git a/litmus/color.c b/litmus/color.c
index b2834b369396..976366e7e39c 100644
--- a/litmus/color.c
+++ b/litmus/color.c
@@ -198,7 +198,7 @@ void reclaim_pages(struct vm_area_struct *vma)
                         put_page(ap->page);
                         add_page_to_color_list(ap->page);
                         nr_reclaimed++;
-                        TRACE_CUR("reclaiming page (pa:0x%10llx, pfn:%8lu, "
+                        TRACE_CUR("reclaiming page (pa:0x%10"FMT_PA", pfn:%8lu, "
                                         "color:%3lu)\n", page_to_phys(ap->page),
                                         page_to_pfn(ap->page), page_color(ap->page));
                         kfree(ap);
diff --git a/litmus/color_dev.c b/litmus/color_dev.c
index 3ed584ad7aff..6287d14405e8 100644
--- a/litmus/color_dev.c
+++ b/litmus/color_dev.c
@@ -7,6 +7,8 @@
 #include <linux/highmem.h>
 #include <asm/io.h> /* page_to_phys on SPARC */
 
+#include <asm/glue-cache.h>
+
 #include <litmus/litmus.h>
 #include <litmus/color.h>
 
@@ -82,17 +84,35 @@ static struct vm_operations_struct litmus_color_ctrl_vm_ops = {
 
 static int alloc_color_ctrl_page(void)
 {
+        void *mem;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        void *mem_remap;
+#endif
         struct task_struct *t = current;
         int err = 0;
 
         /* only allocate if the task doesn't have one yet */
         if (!tsk_rt(t)->color_ctrl_page) {
-                tsk_rt(t)->color_ctrl_page = (void*) get_zeroed_page(GFP_KERNEL);
-                if (!tsk_rt(t)->color_ctrl_page) {
-                        TRACE_TASK(t, "could not allocate control page\n");
+                mem = (void*) get_zeroed_page(GFP_KERNEL);
+                if (!mem) {
+                        TRACE_TASK(t, "cannot allocate color control page\n");
                         err = -ENOMEM;
                         goto out;
                 }
+
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+                mem_remap = remap_noncached(mem, PAGE_SIZE);
+                if (IS_ERR(mem_remap)) {
+                        err = PTR_ERR(mem_remap);
+                        free_page((unsigned long) mem);
+                        goto out;
+                }
+                tsk_rt(t)->color_ctrl_page_orig = mem;
+                tsk_rt(t)->color_ctrl_page = mem_remap;
+#else
+                tsk_rt(t)->color_ctrl_page = mem;
+#endif
+
                 /* will get de-allocated in task teardown */
                 TRACE_TASK(t, "color_ctrl_page = %p\n",
                            tsk_rt(t)->color_ctrl_page);
@@ -105,7 +125,14 @@ static int map_color_ctrl_page(struct vm_area_struct *vma)
 {
         int err = 0;
         struct task_struct *t = current;
-        struct page *color_ctrl = virt_to_page(tsk_rt(t)->color_ctrl_page);
+        struct page *color_ctrl;
+
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        /* vm_insert_page() using the "real" vaddr, not the shadow mapping. */
+        color_ctrl = virt_to_page(tsk_rt(t)->color_ctrl_page_orig);
+#else
+        color_ctrl = virt_to_page(tsk_rt(t)->color_ctrl_page);
+#endif
 
         TRACE_CUR("mapping %p (pfn:%lx) to 0x%lx (flags:%lx prot:%lx)\n",
                   tsk_rt(t)->color_ctrl_page, page_to_pfn(color_ctrl),
@@ -139,6 +166,10 @@ static int litmus_color_ctrl_mmap(struct file *filp, struct vm_area_struct *vma)
         vma->vm_ops = &litmus_color_ctrl_vm_ops;
         mmap_common_vma_flags(vma);
 
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+#endif
+
         err = alloc_color_ctrl_page();
         if (!err)
                 err = map_color_ctrl_page(vma);
@@ -164,6 +195,9 @@ static int do_map_colored_page_set(struct vm_area_struct *vma,
 {
         int i, err = 0;
 
+        TRACE_CUR("vma: %p  starting_at: 0x%10lx  color_no: %lu  nr_pages: %lu\n",
+                        vma, starting_at, color_no, nr_pages);
+
         for (i = 0; i < nr_pages; ++i) {
                 const unsigned long addr = starting_at + (i << PAGE_SHIFT);
                 struct page *page = get_colored_page(color_no);
@@ -176,16 +210,13 @@ static int do_map_colored_page_set(struct vm_area_struct *vma,
                         goto out;
                 }
 
-                clear_user_highpage(page, addr);
+                /* TODO For some reason this causes problems on ARM. */
+                /* clear_user_highpage(page, addr); */
 
-                TRACE_CUR("insert page (pa:0x%10llx, pfn:%8lu, color:%3lu) "
-                                "at 0x%lx vma:(flags:0x%16lx prot:0x%16lx)\n",
-                                page_to_phys(page),
-                                page_to_pfn(page),
-                                color_no,
-                                addr,
-                                vma->vm_flags,
-                                pgprot_val(vma->vm_page_prot));
+                TRACE_CUR("inserting page (pa: 0x%"FMT_PA") at vaddr: 0x%10lx  "
+                                "flags: 0x%10lx  prot: 0x%10lx\n",
+                                page_to_phys(page), addr,
+                                vma->vm_flags, pgprot_val(vma->vm_page_prot));
 
                 err = vm_insert_page(vma, addr, page);
                 if (err) {
@@ -207,14 +238,14 @@ static int do_map_colored_pages(struct vm_area_struct *vma)
         unsigned long nr_mapped;
         int i, err = 0;
 
-        TRACE_CUR("allocating %lu pages (flags:%lx prot:%lx)\n",
-                        nr_pages, vma->vm_flags, pgprot_val(vma->vm_page_prot));
-
 #ifdef CONFIG_PLUGIN_COLOR_UNCACHABLE
         /* Adds non-cached on x86 */
         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 #endif
 
+        TRACE_CUR("allocating %lu pages (flags:%lx prot:%lx)\n",
+                        nr_pages, vma->vm_flags, pgprot_val(vma->vm_page_prot));
+
         for (i = 0, nr_mapped = 0; nr_mapped < nr_pages; ++i) {
                 const unsigned long color_no = color_ctrl->colors[i];
                 const unsigned long color_quantity = color_ctrl->pages[i];
@@ -231,7 +262,7 @@ static int do_map_colored_pages(struct vm_area_struct *vma)
                         goto out;
                 }
 
-                TRACE_CUR("addr: %lu  nr_mapped: %lu color number: %lu  "
+                TRACE_CUR("addr: 0x%10lx  nr_mapped: %lu color number: %lu  "
                                 "color quantity: %lu\n",
                                 addr, nr_mapped, color_no,
                                 color_quantity);
diff --git a/litmus/color_proc.c b/litmus/color_proc.c
index 78657bd4c7ac..8ee624860d71 100644
--- a/litmus/color_proc.c
+++ b/litmus/color_proc.c
@@ -50,6 +50,13 @@ static struct ctl_table cache_table[] =
                 .data           = &color_cache_info.line_size,
                 .maxlen         = sizeof(color_cache_info.line_size),
         },
+        {
+                .procname       = "nr_colors",
+                .mode           = 0444,
+                .proc_handler   = proc_doulongvec_minmax,
+                .data           = &color_cache_info.nr_colors,
+                .maxlen         = sizeof(color_cache_info.nr_colors),
+        },
         { }
 };
 
diff --git a/litmus/ctrldev.c b/litmus/ctrldev.c
index 07c3a680ca6b..87c216099646 100644
--- a/litmus/ctrldev.c
+++ b/litmus/ctrldev.c
@@ -19,8 +19,10 @@ static struct workqueue_struct *wq_litmus_dealloc;
 struct litmus_dealloc_work {
         struct work_struct work_struct;
         void *ctrl_page_mem;
+        void *color_ctrl_page_mem;
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
         void *ctrl_page_unmap;
+        void *color_ctrl_page_unmap;
 #endif
 };
 
@@ -31,10 +33,16 @@ static void litmus_dealloc(struct work_struct *work_in)
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
         TRACE("vunmap() control page %p.\n", work->ctrl_page_unmap);
         vunmap(work->ctrl_page_unmap);
+
+        TRACE("vunmap() color control page %p.\n", work->color_ctrl_page_unmap);
+        vunmap(work->color_ctrl_page_unmap);
 #endif
         TRACE("freeing ctrl_page %p\n", work->ctrl_page_mem);
         free_page((unsigned long) work->ctrl_page_mem);
 
+        TRACE("freeing color_ctrl_page %p\n", work->color_ctrl_page_mem);
+        free_page((unsigned long) work->color_ctrl_page_mem);
+
         kfree((void*) work);
 }
 
@@ -42,7 +50,7 @@ void litmus_schedule_deallocation(struct task_struct *t)
 {
         struct litmus_dealloc_work *work;
 
-        if (NULL == tsk_rt(t)->ctrl_page)
+        if (NULL == tsk_rt(t)->ctrl_page && NULL == tsk_rt(t)->color_ctrl_page)
                 return;
 
         work = kmalloc(sizeof(*work), GFP_ATOMIC);
@@ -56,8 +64,11 @@ void litmus_schedule_deallocation(struct task_struct *t)
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
         work->ctrl_page_mem = tsk_rt(t)->ctrl_page_orig;
         work->ctrl_page_unmap = tsk_rt(t)->ctrl_page;
+        work->color_ctrl_page_mem = tsk_rt(t)->color_ctrl_page_orig;
+        work->color_ctrl_page_unmap = tsk_rt(t)->color_ctrl_page;
 #else
         work->ctrl_page_mem = tsk_rt(t)->ctrl_page;
+        work->color_ctrl_page_mem = tsk_rt(t)->color_ctrl_page;
 #endif
         queue_work(wq_litmus_dealloc, &work->work_struct);
 }
@@ -71,7 +82,7 @@ void litmus_schedule_deallocation(struct task_struct *t)
  * The caller should vunmap(addr) when the mapping is no longer needed.
  * The caller should also save the original @addr to free it later.
  */
-static void * remap_noncached(void *addr, size_t len)
+void * remap_noncached(void *addr, size_t len)
 {
         void *vaddr;
         int nr_pages = DIV_ROUND_UP(offset_in_page(addr) + len, PAGE_SIZE);
diff --git a/litmus/litmus.c b/litmus/litmus.c
index f8397898ce0a..1bbad8db1a55 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -393,7 +393,10 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
         void * ctrl_page_orig = NULL;
 #endif
 
-        void*  color_ctrl_page = NULL;
+        void *color_ctrl_page = NULL;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        void *color_ctrl_page_orig = NULL;
+#endif
 
         if (restore) {
                 /* Safe user-space provided configuration data.
@@ -403,6 +406,7 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
                 color_ctrl_page = p->rt_param.color_ctrl_page;
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
                 ctrl_page_orig = p->rt_param.ctrl_page_orig;
+                color_ctrl_page_orig = p->rt_param.color_ctrl_page_orig;
 #endif
         }
 
@@ -421,6 +425,7 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
                 p->rt_param.color_ctrl_page     = color_ctrl_page;
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
                 p->rt_param.ctrl_page_orig = ctrl_page_orig;
+                p->rt_param.color_ctrl_page_orig = color_ctrl_page_orig;
 #endif
         }
 }
@@ -573,6 +578,7 @@ void litmus_fork(struct task_struct* p)
                 tsk_rt(p)->color_ctrl_page = NULL;
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
                 tsk_rt(p)->ctrl_page_orig = NULL;
+                tsk_rt(p)->color_ctrl_page_orig = NULL;
 #endif
         }
 
@@ -590,14 +596,12 @@ void litmus_exec(void)
 
         if (is_realtime(p)) {
                 WARN_ON(p->rt_param.inh_task);
-                if (tsk_rt(p)->color_ctrl_page) {
-                        free_page((unsigned long) tsk_rt(p)->color_ctrl_page);
-                        tsk_rt(p)->color_ctrl_page = NULL;
-                }
                 litmus_schedule_deallocation(p);
                 tsk_rt(p)->ctrl_page = NULL;
+                tsk_rt(p)->color_ctrl_page = NULL;
 #ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
                 tsk_rt(p)->ctrl_page_orig = NULL;
+                tsk_rt(p)->color_ctrl_page_orig = NULL;
 #endif
         }
 }
@@ -611,12 +615,6 @@ void exit_litmus(struct task_struct *dead_tsk)
          * So check if we need to free the page
          * in any case.
          */
-        if (tsk_rt(dead_tsk)->color_ctrl_page) {
-                TRACE_TASK(dead_tsk,
-                                "freeing color_ctrl_page %p\n",
-                                tsk_rt(dead_tsk)->color_ctrl_page);
-                free_page((unsigned long) tsk_rt(dead_tsk)->color_ctrl_page);
-        }
 
 #ifdef CONFIG_PLUGIN_MC
         /* The MC-setup syscall might succeed and allocate mc_data, but the