Architecture dependent uncachable control page.

While working on an ODROID-X (Samsung Exynos4412 ARM Cortex-A9), I
experienced non-determinism when reading and writing values to the
LITMUS^RT control page. Writes to the control page from user- (kernel-)
space where not always reflected in kernel- (user-) space.

Neither restricting the task to run on a single CPU nor inserting
general memory barriers (with mb()) fixed the problem. Mapping the
control page as uncachable in both kernel and user space did fix the
problem, which is what this patch does.

Also, since vunmap() cannot be called from an interrupt context, I had
to add a workqueue that unmaps and frees the control page when it is no
longer needed. (On my system, exit_litmus() was called from interrupt
context while the kernel reaped the task_struct.)

Conflicts:

	include/litmus/rt_param.h
	litmus/litmus.c

Does not make the color control page uncachable yet!

Signed-off-by: Christopher Kenna <cjk@cs.unc.edu>

5 files changed, 178 insertions, 17 deletions

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index fb228eaaab2b..662cb7d9e093 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -2047,4 +2047,7 @@ config ARCH_HAS_SEND_PULL_TIMERS
 config ARCH_HAS_FEATHER_TRACE
         def_bool n
 
+config ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        def_bool y
+
 source "litmus/Kconfig"
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 2cd68ba3b752..93961d08efd5 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -12,6 +12,8 @@
 extern atomic_t release_master_cpu;
 #endif
 
+void litmus_schedule_deallocation(struct task_struct *t);
+
 /* in_list - is a given list_head queued on some list?
  */
 static inline int in_list(struct list_head* list)
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 2eaa0c532414..28a4d538a2ac 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -264,6 +264,9 @@ struct rt_param {
 
         /* Pointer to the page shared between userspace and kernel. */
         struct control_page * ctrl_page;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        void *ctrl_page_orig;
+#endif
 
         lt_t            total_tardy;
         lt_t            max_tardy;
diff --git a/litmus/ctrldev.c b/litmus/ctrldev.c
index 9969ab17c190..07c3a680ca6b 100644
--- a/litmus/ctrldev.c
+++ b/litmus/ctrldev.c
@@ -3,6 +3,10 @@
 #include <linux/fs.h>
 #include <linux/miscdevice.h>
 #include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/kernel.h>
+#include <linux/workqueue.h>
 
 #include <litmus/litmus.h>
 
@@ -10,32 +14,157 @@
 
 #define CTRL_NAME        "litmus/ctrl"
 
+static struct workqueue_struct *wq_litmus_dealloc;
+
+struct litmus_dealloc_work {
+        struct work_struct work_struct;
+        void *ctrl_page_mem;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        void *ctrl_page_unmap;
+#endif
+};
+
+static void litmus_dealloc(struct work_struct *work_in)
+{
+        struct litmus_dealloc_work *work = container_of(work_in,
+                        struct litmus_dealloc_work, work_struct);
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        TRACE("vunmap() control page %p.\n", work->ctrl_page_unmap);
+        vunmap(work->ctrl_page_unmap);
+#endif
+        TRACE("freeing ctrl_page %p\n", work->ctrl_page_mem);
+        free_page((unsigned long) work->ctrl_page_mem);
+
+        kfree((void*) work);
+}
+
+void litmus_schedule_deallocation(struct task_struct *t)
+{
+        struct litmus_dealloc_work *work;
+
+        if (NULL == tsk_rt(t)->ctrl_page)
+                return;
+
+        work = kmalloc(sizeof(*work), GFP_ATOMIC);
+        if (!work) {
+                WARN(1, "Could not allocate LITMUS deallocation work.\n");
+                return;
+        }
+
+        INIT_WORK(&work->work_struct, litmus_dealloc);
+
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        work->ctrl_page_mem = tsk_rt(t)->ctrl_page_orig;
+        work->ctrl_page_unmap = tsk_rt(t)->ctrl_page;
+#else
+        work->ctrl_page_mem = tsk_rt(t)->ctrl_page;
+#endif
+        queue_work(wq_litmus_dealloc, &work->work_struct);
+}
+
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+/*
+ * remap_noncached - creates a non-cached memory "shadow mapping"
+ * @addr:       memory base virtual address
+ * @len:        length to remap
+ *
+ * The caller should vunmap(addr) when the mapping is no longer needed.
+ * The caller should also save the original @addr to free it later.
+ */
+static void * remap_noncached(void *addr, size_t len)
+{
+        void *vaddr;
+        int nr_pages = DIV_ROUND_UP(offset_in_page(addr) + len, PAGE_SIZE);
+        struct page **pages = kmalloc(nr_pages * sizeof(*pages), GFP_KERNEL);
+        void *page_addr = (void *)((unsigned long)addr & PAGE_MASK);
+        int i;
+
+        if (NULL == pages) {
+                TRACE_CUR("No memory!\n");
+                return ERR_PTR(-ENOMEM);
+        }
+
+        for (i = 0; i < nr_pages; i++) {
+                if (is_vmalloc_or_module_addr(page_addr)) {
+                        kfree(pages);
+                        TRACE_CUR("Remapping vmalloc or module memory?\n");
+                        return ERR_PTR(-EINVAL);
+                }
+
+                pages[i] = virt_to_page(page_addr);
+                if (NULL == pages[i]) {
+                        kfree(pages);
+                        TRACE_CUR("Bad virtual address.\n");
+                        return ERR_PTR(-EINVAL);
+                }
+                page_addr += PAGE_SIZE;
+        }
+
+        vaddr = vmap(pages, nr_pages, VM_MAP, pgprot_noncached(PAGE_KERNEL));
+        kfree(pages);
+        if (NULL == vaddr) {
+                TRACE_CUR("vmap() failed.\n");
+                return ERR_PTR(-ENOMEM);
+        }
+        return vaddr + offset_in_page(addr);
+}
+#endif
+
 /* allocate t->rt_param.ctrl_page*/
 static int alloc_ctrl_page(struct task_struct *t)
 {
+        void *mem;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        void *mem_remap;
+#endif
         int err = 0;
 
         /* only allocate if the task doesn't have one yet */
         if (!tsk_rt(t)->ctrl_page) {
-                tsk_rt(t)->ctrl_page = (void*) get_zeroed_page(GFP_KERNEL);
-                if (!tsk_rt(t)->ctrl_page)
+                mem = (void*) get_zeroed_page(GFP_KERNEL);
+                if (!mem) {
                         err = -ENOMEM;
+                        goto out;
+                }
+
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+                mem_remap = remap_noncached(mem, PAGE_SIZE);
+                if (IS_ERR(mem_remap)) {
+                        err = PTR_ERR(mem_remap);
+                        free_page((unsigned long) mem);
+                        goto out;
+                }
+                tsk_rt(t)->ctrl_page_orig = mem;
+                tsk_rt(t)->ctrl_page = mem_remap;
+                TRACE_TASK(t, "ctrl_page_orig = %p\n",
+                                tsk_rt(t)->ctrl_page_orig);
+#else
+                tsk_rt(t)->ctrl_page = mem;
+#endif
+
                 /* will get de-allocated in task teardown */
                 TRACE_TASK(t, "%s ctrl_page = %p\n", __FUNCTION__,
                            tsk_rt(t)->ctrl_page);
         }
+out:
         return err;
 }
 
 static int map_ctrl_page(struct task_struct *t, struct vm_area_struct* vma)
 {
+        struct page *ctrl;
         int err;
 
-        struct page* ctrl = virt_to_page(tsk_rt(t)->ctrl_page);
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        /* vm_insert_page() using the "real" vaddr, not the shadow mapping. */
+        ctrl = virt_to_page(tsk_rt(t)->ctrl_page_orig);
+#else
+        ctrl = virt_to_page(tsk_rt(t)->ctrl_page);
+#endif
 
         TRACE_CUR(CTRL_NAME
                   ": mapping %p (pfn:%lx) to 0x%lx (prot:%lx)\n",
-                  tsk_rt(t)->ctrl_page,page_to_pfn(ctrl), vma->vm_start,
+                  tsk_rt(t)->ctrl_page, page_to_pfn(ctrl), vma->vm_start,
                   vma->vm_page_prot);
 
         /* Map it into the vma. */
@@ -104,7 +233,11 @@ static int litmus_ctrl_mmap(struct file* filp, struct vm_area_struct* vma)
          * don't care if it was touched or not. __S011 means RW access, but not
          * execute, and avoids copy-on-write behavior.
          * See protection_map in mmap.c.  */
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        vma->vm_page_prot = pgprot_noncached(__S011);
+#else
         vma->vm_page_prot = __S011;
+#endif
 
         err = alloc_ctrl_page(current);
         if (!err)
@@ -137,11 +270,20 @@ static int __init init_litmus_ctrl_dev(void)
         err = misc_register(&litmus_ctrl_dev);
         if (err)
                 printk("Could not allocate %s device (%d).\n", CTRL_NAME, err);
+
+        wq_litmus_dealloc = alloc_workqueue("litmus_dealloc",
+                        WQ_NON_REENTRANT|WQ_MEM_RECLAIM, 0);
+        if (NULL == wq_litmus_dealloc) {
+                printk("Could not allocate vunmap workqueue.\n");
+                misc_deregister(&litmus_ctrl_dev);
+        }
         return err;
 }
 
 static void __exit exit_litmus_ctrl_dev(void)
 {
+        flush_workqueue(wq_litmus_dealloc);
+        destroy_workqueue(wq_litmus_dealloc);
         misc_deregister(&litmus_ctrl_dev);
 }
 
diff --git a/litmus/litmus.c b/litmus/litmus.c
index 07d9d70346a0..f8397898ce0a 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -9,6 +9,7 @@
 #include <linux/sched.h>
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <linux/vmalloc.h>
 
 #include <litmus/litmus.h>
 #include <litmus/bheap.h>
@@ -388,14 +389,21 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
 {
         struct rt_task  user_config = {};
         void*  ctrl_page     = NULL;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+        void * ctrl_page_orig = NULL;
+#endif
+
         void*  color_ctrl_page = NULL;
 
         if (restore) {
                 /* Safe user-space provided configuration data.
                  * and allocated page. */
-                user_config     = p->rt_param.task_params;
-                ctrl_page       = p->rt_param.ctrl_page;
+                user_config     = p->rt_param.task_params;
+                ctrl_page       = p->rt_param.ctrl_page;
                 color_ctrl_page = p->rt_param.color_ctrl_page;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+                ctrl_page_orig = p->rt_param.ctrl_page_orig;
+#endif
         }
 
         /* We probably should not be inheriting any task's priority
@@ -409,8 +417,11 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
         /* Restore preserved fields. */
         if (restore) {
                 p->rt_param.task_params         = user_config;
-                p->rt_param.ctrl_page           = ctrl_page;
+                p->rt_param.ctrl_page           = ctrl_page;
                 p->rt_param.color_ctrl_page     = color_ctrl_page;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+                p->rt_param.ctrl_page_orig = ctrl_page_orig;
+#endif
         }
 }
 
@@ -560,6 +571,9 @@ void litmus_fork(struct task_struct* p)
                 /* non-rt tasks might have ctrl_page set */
                 tsk_rt(p)->ctrl_page = NULL;
                 tsk_rt(p)->color_ctrl_page = NULL;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+                tsk_rt(p)->ctrl_page_orig = NULL;
+#endif
         }
 
         /* od tables are never inherited across a fork */
@@ -576,31 +590,27 @@ void litmus_exec(void)
 
         if (is_realtime(p)) {
                 WARN_ON(p->rt_param.inh_task);
-                if (tsk_rt(p)->ctrl_page) {
-                        free_page((unsigned long) tsk_rt(p)->ctrl_page);
-                        tsk_rt(p)->ctrl_page = NULL;
-                }
                 if (tsk_rt(p)->color_ctrl_page) {
                         free_page((unsigned long) tsk_rt(p)->color_ctrl_page);
                         tsk_rt(p)->color_ctrl_page = NULL;
                 }
+                litmus_schedule_deallocation(p);
+                tsk_rt(p)->ctrl_page = NULL;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+                tsk_rt(p)->ctrl_page_orig = NULL;
+#endif
         }
 }
 
 void exit_litmus(struct task_struct *dead_tsk)
 {
+
         /* We also allow non-RT tasks to
          * allocate control pages to allow
          * measurements with non-RT tasks.
          * So check if we need to free the page
          * in any case.
          */
-        if (tsk_rt(dead_tsk)->ctrl_page) {
-                TRACE_TASK(dead_tsk,
-                           "freeing ctrl_page %p\n",
-                           tsk_rt(dead_tsk)->ctrl_page);
-                free_page((unsigned long) tsk_rt(dead_tsk)->ctrl_page);
-        }
         if (tsk_rt(dead_tsk)->color_ctrl_page) {
                 TRACE_TASK(dead_tsk,
                                 "freeing color_ctrl_page %p\n",
@@ -616,6 +626,7 @@ void exit_litmus(struct task_struct *dead_tsk)
         if (tsk_rt(dead_tsk)->mc_data)
                 kmem_cache_free(mc_data_cache, tsk_rt(dead_tsk)->mc_data);
 #endif
+        litmus_schedule_deallocation(dead_tsk);
 
         /* main cleanup only for RT tasks */
         if (is_realtime(dead_tsk))