Page reclaiming, control devices, cleanup.

Track allocated pages and add a proc handler to reclaim free pages and
add control device for allocating colored memory with mmap.

5 files changed, 421 insertions, 25 deletions

diff --git a/litmus/Makefile b/litmus/Makefile
index e4c937bc2850..2d77d11e905e 100644
--- a/litmus/Makefile
+++ b/litmus/Makefile
@@ -19,7 +19,8 @@ obj-y     = sched_plugin.o litmus.o \
             sched_gsn_edf.o \
             sched_psn_edf.o \
             color.o \
-            color_proc.o
+            color_proc.o \
+            color_dev.o
 
 obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o
 obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o
diff --git a/litmus/color.c b/litmus/color.c
index aefb76e36626..a3cc193418c0 100644
--- a/litmus/color.c
+++ b/litmus/color.c
@@ -1,15 +1,14 @@
 #include <linux/spinlock.h>
 
-#include <linux/mmzone.h>
 #include <linux/module.h>
-#include <linux/cpu.h>
-#include <linux/mm_types.h>
+#include <linux/mm.h>
 #include <linux/slab.h>
-#include <linux/page-flags.h>
 #include <linux/sysctl.h>
 #include <linux/lockdep.h>
+#include <linux/sched.h> /* required by litmus.h */
 
 #include <litmus/color.h>
+#include <litmus/litmus.h> /* for in_list(...) */
 
 #define MPRINT(fmt, args...) \
         printk(KERN_INFO "[%s@%s:%d]: " fmt, \
@@ -31,13 +30,20 @@ struct color_group {
         atomic_t nr_pages;
 };
 
+static struct alloced_pages {
+        spinlock_t lock;
+        struct list_head list;
+} alloced_pages;
+
 static unsigned long color_mask;
 static struct color_group *color_groups;
 
+
 /* non-static: extern'ed in various files */
 unsigned long nr_colors;
 struct color_cache_info color_cache_info;
 int color_sysctl_add_pages_data;
+int color_sysctl_reclaim_pages_data;
 
 static inline unsigned long page_color(struct page *page)
 {
@@ -48,12 +54,25 @@ void add_page_to_color_list(struct page *page)
 {
         const unsigned long color = page_color(page);
         struct color_group *cgroup = &color_groups[color];
+        BUG_ON(in_list(&page->lru) || PageLRU(page));
+        BUG_ON(page_mapped(page) || page_count(page) > 1);
         spin_lock(&cgroup->lock);
         list_add_tail(&page->lru, &cgroup->list);
         atomic_inc(&cgroup->nr_pages);
+        SetPageLRU(page);
         spin_unlock(&cgroup->lock);
 }
 
+void add_page_to_alloced_list(struct page *page)
+{
+        BUG_ON(in_list(&page->lru) || PageLRU(page));
+        BUG_ON(!page_mapped(page) || page_count(page) < 2);
+        spin_lock(&alloced_pages.lock);
+        list_add_tail(&page->lru, &alloced_pages.list);
+        SetPageLRU(page);
+        spin_unlock(&alloced_pages.lock);
+}
+
 struct page* get_colored_page(unsigned long color)
 {
         struct color_group *cgroup;
@@ -66,8 +85,10 @@ struct page* get_colored_page(unsigned long color)
                 goto out_unlock;
         }
         page = list_first_entry(&cgroup->list, struct page, lru);
+        BUG_ON(page_mapped(page) || page_count(page) > 1);
         list_del(&page->lru);
         atomic_dec(&cgroup->nr_pages);
+        ClearPageLRU(page);
 out_unlock:
         spin_unlock(&cgroup->lock);
         return page;
@@ -88,12 +109,10 @@ static unsigned long smallest_nr_pages(void)
 static int do_add_pages(void)
 {
         struct page *page, *page_tmp;
-        struct list_head free_later;
+        LIST_HEAD(free_later);
         unsigned long color;
         int ret = 0;
 
-        INIT_LIST_HEAD(&free_later);
-
         while (smallest_nr_pages() < PAGES_PER_COLOR) {
                 page = alloc_page(GFP_HIGHUSER | __GFP_ZERO |
                                 __GFP_MOVABLE);
@@ -116,6 +135,25 @@ out:
         return ret;
 }
 
+static int do_reclaim_pages(void)
+{
+        struct page *page, *page_tmp;
+        unsigned long nr_reclaimed = 0;
+        spin_lock(&alloced_pages.lock);
+        list_for_each_entry_safe(page, page_tmp, &alloced_pages.list, lru) {
+                if (1 == page_count(page) && !page_mapped(page)) {
+                        list_del(&page->lru);
+                        ClearPageLRU(page);
+                        add_page_to_color_list(page);
+                        nr_reclaimed++;
+                        TRACE_CUR("Reclaimed page (pfn:%lu phys:0x%lx).\n",
+                                        page_to_pfn(page), page_to_phys(page));
+                }
+        }
+        spin_unlock(&alloced_pages.lock);
+        TRACE_CUR("Reclaimed %lu pages.\n", nr_reclaimed);
+        return 0;
+}
 
 /***********************************************************
  * Proc
@@ -158,6 +196,19 @@ out:
         return ret;
 }
 
+int color_reclaim_pages_handler(struct ctl_table *table, int write,
+                void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+        int ret = 0;
+        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+        if (ret)
+                goto out;
+        if (write && color_sysctl_reclaim_pages_data)
+                ret = do_reclaim_pages();
+out:
+        return ret;
+}
+
 /***********************************************************
  * Initialization
 ***********************************************************/
@@ -181,7 +232,7 @@ static void __init init_mask(void)
         color_mask = ((color_cache_info.sets << line_size_log) - 1) ^
                 (PAGE_SIZE - 1);
         nr_colors = (color_mask >> PAGE_SHIFT) + 1;
-        MPRINT("color mask: 0x%lx  total colors: %lu\n", color_mask,
+        printk("Color mask: 0x%lx  Total colors: %lu\n", color_mask,
                         nr_colors);
         BUG_ON(LOCKDEP_MAX_NR_COLORS < nr_colors);
 }
@@ -214,18 +265,23 @@ out:
 static int __init init_color(void)
 {
         int ret = 0;
+        printk("Initializing LITMUS^RT cache coloring.\n");
 
-        MPRINT("Cache size: %lu  line-size: %lu  ways: %lu  sets: %lu\n",
+        BUG_ON(color_cache_info.size <= 1048576 ||
+                        color_cache_info.ways < 15 ||
+                        color_cache_info.line_size != 64);
+        INIT_LIST_HEAD(&alloced_pages.list);
+        spin_lock_init(&alloced_pages.lock);
+
+        printk("Cache size: %lu  line-size: %lu  ways: %lu  sets: %lu\n",
                         color_cache_info.size, color_cache_info.line_size,
                         color_cache_info.ways, color_cache_info.sets);
-        if (!color_cache_info.size){
+        if (!color_cache_info.size) {
                 printk(KERN_WARNING "No cache information found.\n");
                 ret = -EINVAL;
                 goto out;
         }
-        BUG_ON(color_cache_info.size <= 1048576 ||
-                        color_cache_info.ways < 15 ||
-                        color_cache_info.line_size != 64);
+
         init_mask();
         ret = init_color_groups();
 out:
diff --git a/litmus/color_dev.c b/litmus/color_dev.c
new file mode 100644
index 000000000000..b8218b6d1d9c
--- /dev/null
+++ b/litmus/color_dev.c
@@ -0,0 +1,312 @@
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/fs.h>
+#include <linux/miscdevice.h>
+#include <linux/spinlock.h>
+#include <linux/module.h>
+
+#include <litmus/litmus.h>
+#include <litmus/color.h>
+
+#define ALLOC_NAME      "litmus/color_alloc"
+#define CTRL_NAME       "litmus/color_ctrl"
+
+static struct non_rt_colors {
+        spinlock_t lock;
+        unsigned long color;
+} non_rt_colors;
+
+extern unsigned long nr_colors;
+
+/***********************************************************
+ * Control device
+***********************************************************/
+
+static void litmus_color_ctrl_vm_close(struct vm_area_struct *vma)
+{
+        TRACE_CUR("%s flags=0x%lx prot=0x%lx\n", __FUNCTION__,
+                        vma->vm_flags, pgprot_val(vma->vm_page_prot));
+
+        TRACE_CUR(CTRL_NAME ": %p:%p vma:%p vma->vm_private_data:%p closed.\n",
+                        (void*) vma->vm_start, (void*) vma->vm_end, vma,
+                        vma->vm_private_data);
+}
+
+static int litmus_color_ctrl_vm_fault(struct vm_area_struct *vma,
+                struct vm_fault *vmf)
+{
+        /* This function should never be called, since
+         * all pages should have been mapped by mmap()
+         * already. */
+        TRACE_CUR("%s flags=0x%lx\n", __FUNCTION__, vma->vm_flags);
+        printk(KERN_WARNING "fault: %s flags=0x%lx\n", __FUNCTION__,
+                        vma->vm_flags);
+
+        /* nope, you only get one page */
+        return VM_FAULT_SIGBUS;
+}
+
+static struct vm_operations_struct litmus_color_ctrl_vm_ops = {
+        .close  = litmus_color_ctrl_vm_close,
+        .fault  = litmus_color_ctrl_vm_fault,
+};
+
+static int mmap_common_checks(struct vm_area_struct *vma)
+{
+        /* you can only map the "first" page */
+        if (vma->vm_pgoff != 0)
+                return -EINVAL;
+
+        /* you can't share it with anyone */
+        if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED))
+                return -EINVAL;
+
+        return 0;
+}
+
+static int alloc_color_ctrl_page(void)
+{
+        struct task_struct *t;
+        int err = 0;
+
+        t = current;
+        /* only allocate if the task doesn't have one yet */
+        if (!tsk_rt(t)->color_ctrl_page) {
+                tsk_rt(t)->color_ctrl_page = (void*) get_zeroed_page(GFP_KERNEL);
+                if (!tsk_rt(t)->color_ctrl_page)
+                        err = -ENOMEM;
+                /* will get de-allocated in task teardown */
+                TRACE_TASK(t, "%s color_ctrl_page = %p\n", __FUNCTION__,
+                           tsk_rt(t)->color_ctrl_page);
+        }
+        return err;
+}
+
+static int map_color_ctrl_page(struct vm_area_struct *vma)
+{
+        int err;
+        unsigned long pfn;
+        struct task_struct *t = current;
+        struct page *color_ctrl = virt_to_page(tsk_rt(t)->color_ctrl_page);
+
+        t = current;
+        /* Increase ref count. Is decreased when vma is destroyed. */
+        get_page(color_ctrl);
+        pfn = page_to_pfn(color_ctrl);
+
+        TRACE_CUR(CTRL_NAME
+                  ": mapping %p (pfn:%lx, %lx) to 0x%lx (prot:%lx)\n",
+                  tsk_rt(t)->color_ctrl_page, pfn, page_to_pfn(color_ctrl),
+                  vma->vm_start, vma->vm_page_prot);
+
+        /* Map it into the vma. Make sure to use PAGE_SHARED, otherwise
+         * userspace actually gets a copy-on-write page. */
+        err = remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, PAGE_SHARED);
+        TRACE_CUR("page shared: guess:0x1(63)...1??111  actual:0x%lx\n", PAGE_SHARED);
+        /* present, RW, user, accessed, NX=63 */
+
+        if (err)
+                TRACE_CUR(CTRL_NAME ": remap_pfn_range() failed (%d)\n", err);
+
+        return err;
+}
+
+static int litmus_color_ctrl_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+        int err = 0;
+
+        /* you can only get one page */
+        if (vma->vm_end - vma->vm_start != PAGE_SIZE) {
+                err = -EINVAL;
+                goto out;
+        }
+
+        err = mmap_common_checks(vma);
+        if (err)
+                goto out;
+
+        vma->vm_ops = &litmus_color_ctrl_vm_ops;
+        /* this mapping should not be kept across forks,
+         * and cannot be expanded */
+        vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
+
+        err = alloc_color_ctrl_page();
+        if (!err)
+                err = map_color_ctrl_page(vma);
+
+        TRACE_CUR("%s flags=0x%lx prot=0x%lx\n", __FUNCTION__, vma->vm_flags,
+                        pgprot_val(vma->vm_page_prot));
+out:
+        return err;
+}
+
+
+/***********************************************************
+ * Allocation device
+***********************************************************/
+
+static int map_colored_pages_non_rt(struct vm_area_struct *vma)
+{
+        unsigned long color, mapped;
+        int err;
+        const unsigned long nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+
+        spin_lock(&non_rt_colors.lock);
+        color = non_rt_colors.color;
+        non_rt_colors.color = (non_rt_colors.color + nr_pages) % nr_colors;
+        spin_unlock(&non_rt_colors.lock);
+
+        TRACE_CUR(ALLOC_NAME ": allocating %lu pages from color %lu.\n",
+                        nr_pages, color);
+
+        for (mapped = 0; mapped < nr_pages;
+                        mapped++, color = (color + 1) % nr_colors)
+        {
+                struct page *page = get_colored_page(color);
+                const unsigned long addr = vma->vm_start + PAGE_SIZE * mapped;
+
+                if (!page) {
+                        TRACE_CUR(ALLOC_NAME ": Could not get page with "
+                                        " color %lu.\n", color);
+                        /* TODO unmap mapped pages */
+                        break;
+                }
+                TRACE_CUR(ALLOC_NAME ": insert page (pa:0x%llx, pfn:%lu, "
+                                "color:%lu count:%d LRU:%d) at 0x%lx "
+                                "(prot: 0x%lx)\n",
+                                page_to_phys(page), page_to_pfn(page), color,
+                                page_count(page), PageLRU(page), addr,
+                                pgprot_val(vma->vm_page_prot));
+                err = vm_insert_page(vma, addr, page);
+                if (err) {
+                        TRACE_CUR(ALLOC_NAME ": vm_insert_page() failed "
+                                        "(%d)\n", err);
+                        /* TODO unmap mapped pages */
+                        break;
+                }
+                add_page_to_alloced_list(page);
+        }
+        return err;
+}
+
+static int map_colored_pages_rt(struct vm_area_struct *vma)
+{
+        /* TODO */
+        return -EINVAL;
+}
+
+static int map_colored_pages(struct vm_area_struct *vma)
+{
+        if (likely(is_realtime(current)))
+                return map_colored_pages_rt(vma);
+        return map_colored_pages_non_rt(vma);
+}
+
+static void litmus_color_alloc_vm_close(struct vm_area_struct *vma)
+{
+        TRACE_CUR("%s flags=0x%lx prot=0x%lx\n", __FUNCTION__,
+                        vma->vm_flags, pgprot_val(vma->vm_page_prot));
+
+        TRACE_CUR(ALLOC_NAME ": %p:%p vma:%p vma->vm_private_data:%p closed.\n",
+                        (void*) vma->vm_start, (void*) vma->vm_end, vma,
+                        vma->vm_private_data);
+}
+
+static int litmus_color_alloc_vm_fault(struct vm_area_struct *vma,
+                struct vm_fault *vmf)
+{
+        /* This function should never be called, since
+         * all pages should have been mapped by mmap()
+         * already. */
+        TRACE_CUR("%s flags=0x%lx\n", __FUNCTION__, vma->vm_flags);
+        printk(KERN_WARNING "fault: %s flags=0x%lx\n", __FUNCTION__,
+                        vma->vm_flags);
+
+        /* nope, you only get one page */
+        return VM_FAULT_SIGBUS;
+}
+
+static struct vm_operations_struct litmus_color_alloc_vm_ops = {
+        .close  = litmus_color_alloc_vm_close,
+        .fault  = litmus_color_alloc_vm_fault,
+};
+
+static int litmus_color_alloc_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+        int err = 0;
+
+        /* you may only request integer multiple of PAGE_SIZE */
+        if (offset_in_page(vma->vm_end - vma->vm_start)) {
+                err = -EINVAL;
+                goto out;
+        }
+
+        err = mmap_common_checks(vma);
+        if (err)
+                goto out;
+
+        vma->vm_ops = &litmus_color_alloc_vm_ops;
+        /* this mapping should not be kept across forks,
+         * and cannot be expanded */
+        vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
+
+        err = map_colored_pages(vma);
+
+        TRACE_CUR("%s flags=0x%lx prot=0x%lx\n", __FUNCTION__, vma->vm_flags,
+                        pgprot_val(vma->vm_page_prot));
+out:
+        return err;
+}
+
+/***********************************************************
+ * Initilization
+***********************************************************/
+
+static struct file_operations litmus_color_ctrl_fops = {
+        .owner  = THIS_MODULE,
+        .mmap   = litmus_color_ctrl_mmap,
+};
+
+static struct miscdevice litmus_color_ctrl_dev = {
+        .name   = CTRL_NAME,
+        .minor  = MISC_DYNAMIC_MINOR,
+        .fops   = &litmus_color_ctrl_fops,
+};
+
+static struct file_operations litmus_color_alloc_fops = {
+        .owner  = THIS_MODULE,
+        .mmap   = litmus_color_alloc_mmap,
+};
+
+static struct miscdevice litmus_color_alloc_dev = {
+        .name   = ALLOC_NAME,
+        .minor  = MISC_DYNAMIC_MINOR,
+        .fops   = &litmus_color_alloc_fops,
+};
+
+static int __init init_dev(const char* name, struct miscdevice *dev)
+{
+        int err;
+        err = misc_register(dev);
+        if (err)
+                printk(KERN_WARNING "Could not allocate %s device (%d).\n",
+                                name, err);
+        return err;
+}
+
+static int __init init_color_devices(void)
+{
+        int err;
+        spin_lock_init(&non_rt_colors.lock);
+        non_rt_colors.color = 0;
+
+        printk("Allocating LITMUS^RT color devices.\n");
+        err = init_dev(ALLOC_NAME, &litmus_color_alloc_dev);
+        if (err)
+                goto out;
+        err = init_dev(CTRL_NAME, &litmus_color_ctrl_dev);
+out:
+        return err;
+}
+
+module_init(init_color_devices);
diff --git a/litmus/color_proc.c b/litmus/color_proc.c
index 31eec0d728a5..cac336ac1731 100644
--- a/litmus/color_proc.c
+++ b/litmus/color_proc.c
@@ -5,14 +5,23 @@
 #include <litmus/color.h>
 
 extern int color_sysctl_add_pages_data; /* litmus/color.c */
+extern int color_sysctl_reclaim_pages_data; /* litmus/color.c */
 
 static int zero = 0;
 static int one = 1;
 
-#define NR_PAGES_INDEX 1 /* location of nr_pages in the table below */
+#define NR_PAGES_INDEX 0 /* location of nr_pages in the table below */
 static struct ctl_table color_table[] =
 {
         {
+                /* you MUST update NR_PAGES_INDEX if you move this entry */
+                .procname       = "nr_pages",
+                .mode           = 0444,
+                .proc_handler   = color_nr_pages_handler,
+                .data           = NULL,         /* dynamically set later */
+                .maxlen         = 0,            /* also set later */
+        },
+        {
                 .procname       = "add_pages",
                 .data           = &color_sysctl_add_pages_data,
                 .maxlen         = sizeof(int),
@@ -22,11 +31,13 @@ static struct ctl_table color_table[] =
                 .extra2         = &one,
         },
         {
-                .procname       = "nr_pages",
-                .mode           = 0444,
-                .proc_handler   = color_nr_pages_handler,
-                .data           = NULL, /* dynamically later */
-                .maxlen         = 0,    /* also set later */
+                .procname       = "reclaim_pages",
+                .data           = &color_sysctl_reclaim_pages_data,
+                .maxlen         = sizeof(int),
+                .mode           = 0644,
+                .proc_handler   = color_reclaim_pages_handler,
+                .extra1         = &zero,
+                .extra2         = &one,
         },
         { }
 };
@@ -71,6 +82,7 @@ static struct ctl_table_header *litmus_sysctls;
 static int __init litmus_sysctl_init(void)
 {
         int ret = 0;
+        printk(KERN_INFO "Registering LITMUS^RT proc sysctl.\n");
         litmus_sysctls = register_sysctl_table(litmus_dir_table);
         if (!litmus_sysctls) {
                 printk(KERN_WARNING "Could not register LITMUS^RT sysctl.\n");
diff --git a/litmus/litmus.c b/litmus/litmus.c
index 301390148d02..eaa2070d28ce 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -291,12 +291,14 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
 {
         struct rt_task  user_config = {};
         void*  ctrl_page     = NULL;
+        void*  color_ctrl_page = NULL;
 
         if (restore) {
                 /* Safe user-space provided configuration data.
                  * and allocated page. */
-                user_config = p->rt_param.task_params;
-                ctrl_page   = p->rt_param.ctrl_page;
+                user_config     = p->rt_param.task_params;
+                ctrl_page       = p->rt_param.ctrl_page;
+                color_ctrl_page = p->rt_param.color_ctrl_page;
         }
 
         /* We probably should not be inheriting any task's priority
@@ -309,8 +311,9 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
 
         /* Restore preserved fields. */
         if (restore) {
-                p->rt_param.task_params = user_config;
-                p->rt_param.ctrl_page   = ctrl_page;
+                p->rt_param.task_params         = user_config;
+                p->rt_param.ctrl_page           = ctrl_page;
+                p->rt_param.color_ctrl_page     = color_ctrl_page;
         }
 }
 
@@ -451,9 +454,11 @@ void litmus_fork(struct task_struct* p)
                 reinit_litmus_state(p, 0);
                 /* Don't let the child be a real-time task.  */
                 p->sched_reset_on_fork = 1;
-        } else
+        } else {
                 /* non-rt tasks might have ctrl_page set */
                 tsk_rt(p)->ctrl_page = NULL;
+                tsk_rt(p)->color_ctrl_page = NULL;
+        }
 
         /* od tables are never inherited across a fork */
         p->od_table = NULL;
@@ -473,6 +478,10 @@ void litmus_exec(void)
                         free_page((unsigned long) tsk_rt(p)->ctrl_page);
                         tsk_rt(p)->ctrl_page = NULL;
                 }
+                if (tsk_rt(p)->color_ctrl_page) {
+                        free_page((unsigned long) tsk_rt(p)->color_ctrl_page);
+                        tsk_rt(p)->color_ctrl_page = NULL;
+                }
         }
 }
 
@@ -490,6 +499,12 @@ void exit_litmus(struct task_struct *dead_tsk)
                            tsk_rt(dead_tsk)->ctrl_page);
                 free_page((unsigned long) tsk_rt(dead_tsk)->ctrl_page);
         }
+        if (tsk_rt(dead_tsk)->color_ctrl_page) {
+                TRACE_TASK(dead_tsk,
+                                "freeing color_ctrl_page %p\n",
+                                tsk_rt(dead_tsk)->color_ctrl_page);
+                free_page((unsigned long) tsk_rt(dead_tsk)->color_ctrl_page);
+        }
 
         /* main cleanup only for RT tasks */
         if (is_realtime(dead_tsk))