From ae95a4582d707de8a57a8159ea81b16ba7bddd54 Mon Sep 17 00:00:00 2001 From: Christopher Kenna Date: Sun, 1 Apr 2012 19:24:46 -0400 Subject: Page reclaiming, control devices, cleanup. Track allocated pages and add a proc handler to reclaim free pages and add control device for allocating colored memory with mmap. --- litmus/Makefile | 3 +- litmus/color.c | 82 +++++++++++--- litmus/color_dev.c | 312 ++++++++++++++++++++++++++++++++++++++++++++++++++++ litmus/color_proc.c | 24 +++- litmus/litmus.c | 25 ++++- 5 files changed, 421 insertions(+), 25 deletions(-) create mode 100644 litmus/color_dev.c (limited to 'litmus') diff --git a/litmus/Makefile b/litmus/Makefile index e4c937bc2850..2d77d11e905e 100644 --- a/litmus/Makefile +++ b/litmus/Makefile @@ -19,7 +19,8 @@ obj-y = sched_plugin.o litmus.o \ sched_gsn_edf.o \ sched_psn_edf.o \ color.o \ - color_proc.o + color_proc.o \ + color_dev.o obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o diff --git a/litmus/color.c b/litmus/color.c index aefb76e36626..a3cc193418c0 100644 --- a/litmus/color.c +++ b/litmus/color.c @@ -1,15 +1,14 @@ #include -#include #include -#include -#include +#include #include -#include #include #include +#include /* required by litmus.h */ #include +#include /* for in_list(...) */ #define MPRINT(fmt, args...) \ printk(KERN_INFO "[%s@%s:%d]: " fmt, \ @@ -31,13 +30,20 @@ struct color_group { atomic_t nr_pages; }; +static struct alloced_pages { + spinlock_t lock; + struct list_head list; +} alloced_pages; + static unsigned long color_mask; static struct color_group *color_groups; + /* non-static: extern'ed in various files */ unsigned long nr_colors; struct color_cache_info color_cache_info; int color_sysctl_add_pages_data; +int color_sysctl_reclaim_pages_data; static inline unsigned long page_color(struct page *page) { @@ -48,12 +54,25 @@ void add_page_to_color_list(struct page *page) { const unsigned long color = page_color(page); struct color_group *cgroup = &color_groups[color]; + BUG_ON(in_list(&page->lru) || PageLRU(page)); + BUG_ON(page_mapped(page) || page_count(page) > 1); spin_lock(&cgroup->lock); list_add_tail(&page->lru, &cgroup->list); atomic_inc(&cgroup->nr_pages); + SetPageLRU(page); spin_unlock(&cgroup->lock); } +void add_page_to_alloced_list(struct page *page) +{ + BUG_ON(in_list(&page->lru) || PageLRU(page)); + BUG_ON(!page_mapped(page) || page_count(page) < 2); + spin_lock(&alloced_pages.lock); + list_add_tail(&page->lru, &alloced_pages.list); + SetPageLRU(page); + spin_unlock(&alloced_pages.lock); +} + struct page* get_colored_page(unsigned long color) { struct color_group *cgroup; @@ -66,8 +85,10 @@ struct page* get_colored_page(unsigned long color) goto out_unlock; } page = list_first_entry(&cgroup->list, struct page, lru); + BUG_ON(page_mapped(page) || page_count(page) > 1); list_del(&page->lru); atomic_dec(&cgroup->nr_pages); + ClearPageLRU(page); out_unlock: spin_unlock(&cgroup->lock); return page; @@ -88,12 +109,10 @@ static unsigned long smallest_nr_pages(void) static int do_add_pages(void) { struct page *page, *page_tmp; - struct list_head free_later; + LIST_HEAD(free_later); unsigned long color; int ret = 0; - INIT_LIST_HEAD(&free_later); - while (smallest_nr_pages() < PAGES_PER_COLOR) { page = alloc_page(GFP_HIGHUSER | __GFP_ZERO | __GFP_MOVABLE); @@ -116,6 +135,25 @@ out: return ret; } +static int do_reclaim_pages(void) +{ + struct page *page, *page_tmp; + unsigned long nr_reclaimed = 0; + spin_lock(&alloced_pages.lock); + list_for_each_entry_safe(page, page_tmp, &alloced_pages.list, lru) { + if (1 == page_count(page) && !page_mapped(page)) { + list_del(&page->lru); + ClearPageLRU(page); + add_page_to_color_list(page); + nr_reclaimed++; + TRACE_CUR("Reclaimed page (pfn:%lu phys:0x%lx).\n", + page_to_pfn(page), page_to_phys(page)); + } + } + spin_unlock(&alloced_pages.lock); + TRACE_CUR("Reclaimed %lu pages.\n", nr_reclaimed); + return 0; +} /*********************************************************** * Proc @@ -158,6 +196,19 @@ out: return ret; } +int color_reclaim_pages_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret = 0; + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + if (write && color_sysctl_reclaim_pages_data) + ret = do_reclaim_pages(); +out: + return ret; +} + /*********************************************************** * Initialization ***********************************************************/ @@ -181,7 +232,7 @@ static void __init init_mask(void) color_mask = ((color_cache_info.sets << line_size_log) - 1) ^ (PAGE_SIZE - 1); nr_colors = (color_mask >> PAGE_SHIFT) + 1; - MPRINT("color mask: 0x%lx total colors: %lu\n", color_mask, + printk("Color mask: 0x%lx Total colors: %lu\n", color_mask, nr_colors); BUG_ON(LOCKDEP_MAX_NR_COLORS < nr_colors); } @@ -214,18 +265,23 @@ out: static int __init init_color(void) { int ret = 0; + printk("Initializing LITMUS^RT cache coloring.\n"); - MPRINT("Cache size: %lu line-size: %lu ways: %lu sets: %lu\n", + BUG_ON(color_cache_info.size <= 1048576 || + color_cache_info.ways < 15 || + color_cache_info.line_size != 64); + INIT_LIST_HEAD(&alloced_pages.list); + spin_lock_init(&alloced_pages.lock); + + printk("Cache size: %lu line-size: %lu ways: %lu sets: %lu\n", color_cache_info.size, color_cache_info.line_size, color_cache_info.ways, color_cache_info.sets); - if (!color_cache_info.size){ + if (!color_cache_info.size) { printk(KERN_WARNING "No cache information found.\n"); ret = -EINVAL; goto out; } - BUG_ON(color_cache_info.size <= 1048576 || - color_cache_info.ways < 15 || - color_cache_info.line_size != 64); + init_mask(); ret = init_color_groups(); out: diff --git a/litmus/color_dev.c b/litmus/color_dev.c new file mode 100644 index 000000000000..b8218b6d1d9c --- /dev/null +++ b/litmus/color_dev.c @@ -0,0 +1,312 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +#define ALLOC_NAME "litmus/color_alloc" +#define CTRL_NAME "litmus/color_ctrl" + +static struct non_rt_colors { + spinlock_t lock; + unsigned long color; +} non_rt_colors; + +extern unsigned long nr_colors; + +/*********************************************************** + * Control device +***********************************************************/ + +static void litmus_color_ctrl_vm_close(struct vm_area_struct *vma) +{ + TRACE_CUR("%s flags=0x%lx prot=0x%lx\n", __FUNCTION__, + vma->vm_flags, pgprot_val(vma->vm_page_prot)); + + TRACE_CUR(CTRL_NAME ": %p:%p vma:%p vma->vm_private_data:%p closed.\n", + (void*) vma->vm_start, (void*) vma->vm_end, vma, + vma->vm_private_data); +} + +static int litmus_color_ctrl_vm_fault(struct vm_area_struct *vma, + struct vm_fault *vmf) +{ + /* This function should never be called, since + * all pages should have been mapped by mmap() + * already. */ + TRACE_CUR("%s flags=0x%lx\n", __FUNCTION__, vma->vm_flags); + printk(KERN_WARNING "fault: %s flags=0x%lx\n", __FUNCTION__, + vma->vm_flags); + + /* nope, you only get one page */ + return VM_FAULT_SIGBUS; +} + +static struct vm_operations_struct litmus_color_ctrl_vm_ops = { + .close = litmus_color_ctrl_vm_close, + .fault = litmus_color_ctrl_vm_fault, +}; + +static int mmap_common_checks(struct vm_area_struct *vma) +{ + /* you can only map the "first" page */ + if (vma->vm_pgoff != 0) + return -EINVAL; + + /* you can't share it with anyone */ + if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED)) + return -EINVAL; + + return 0; +} + +static int alloc_color_ctrl_page(void) +{ + struct task_struct *t; + int err = 0; + + t = current; + /* only allocate if the task doesn't have one yet */ + if (!tsk_rt(t)->color_ctrl_page) { + tsk_rt(t)->color_ctrl_page = (void*) get_zeroed_page(GFP_KERNEL); + if (!tsk_rt(t)->color_ctrl_page) + err = -ENOMEM; + /* will get de-allocated in task teardown */ + TRACE_TASK(t, "%s color_ctrl_page = %p\n", __FUNCTION__, + tsk_rt(t)->color_ctrl_page); + } + return err; +} + +static int map_color_ctrl_page(struct vm_area_struct *vma) +{ + int err; + unsigned long pfn; + struct task_struct *t = current; + struct page *color_ctrl = virt_to_page(tsk_rt(t)->color_ctrl_page); + + t = current; + /* Increase ref count. Is decreased when vma is destroyed. */ + get_page(color_ctrl); + pfn = page_to_pfn(color_ctrl); + + TRACE_CUR(CTRL_NAME + ": mapping %p (pfn:%lx, %lx) to 0x%lx (prot:%lx)\n", + tsk_rt(t)->color_ctrl_page, pfn, page_to_pfn(color_ctrl), + vma->vm_start, vma->vm_page_prot); + + /* Map it into the vma. Make sure to use PAGE_SHARED, otherwise + * userspace actually gets a copy-on-write page. */ + err = remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, PAGE_SHARED); + TRACE_CUR("page shared: guess:0x1(63)...1??111 actual:0x%lx\n", PAGE_SHARED); + /* present, RW, user, accessed, NX=63 */ + + if (err) + TRACE_CUR(CTRL_NAME ": remap_pfn_range() failed (%d)\n", err); + + return err; +} + +static int litmus_color_ctrl_mmap(struct file *filp, struct vm_area_struct *vma) +{ + int err = 0; + + /* you can only get one page */ + if (vma->vm_end - vma->vm_start != PAGE_SIZE) { + err = -EINVAL; + goto out; + } + + err = mmap_common_checks(vma); + if (err) + goto out; + + vma->vm_ops = &litmus_color_ctrl_vm_ops; + /* this mapping should not be kept across forks, + * and cannot be expanded */ + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND; + + err = alloc_color_ctrl_page(); + if (!err) + err = map_color_ctrl_page(vma); + + TRACE_CUR("%s flags=0x%lx prot=0x%lx\n", __FUNCTION__, vma->vm_flags, + pgprot_val(vma->vm_page_prot)); +out: + return err; +} + + +/*********************************************************** + * Allocation device +***********************************************************/ + +static int map_colored_pages_non_rt(struct vm_area_struct *vma) +{ + unsigned long color, mapped; + int err; + const unsigned long nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + + spin_lock(&non_rt_colors.lock); + color = non_rt_colors.color; + non_rt_colors.color = (non_rt_colors.color + nr_pages) % nr_colors; + spin_unlock(&non_rt_colors.lock); + + TRACE_CUR(ALLOC_NAME ": allocating %lu pages from color %lu.\n", + nr_pages, color); + + for (mapped = 0; mapped < nr_pages; + mapped++, color = (color + 1) % nr_colors) + { + struct page *page = get_colored_page(color); + const unsigned long addr = vma->vm_start + PAGE_SIZE * mapped; + + if (!page) { + TRACE_CUR(ALLOC_NAME ": Could not get page with " + " color %lu.\n", color); + /* TODO unmap mapped pages */ + break; + } + TRACE_CUR(ALLOC_NAME ": insert page (pa:0x%llx, pfn:%lu, " + "color:%lu count:%d LRU:%d) at 0x%lx " + "(prot: 0x%lx)\n", + page_to_phys(page), page_to_pfn(page), color, + page_count(page), PageLRU(page), addr, + pgprot_val(vma->vm_page_prot)); + err = vm_insert_page(vma, addr, page); + if (err) { + TRACE_CUR(ALLOC_NAME ": vm_insert_page() failed " + "(%d)\n", err); + /* TODO unmap mapped pages */ + break; + } + add_page_to_alloced_list(page); + } + return err; +} + +static int map_colored_pages_rt(struct vm_area_struct *vma) +{ + /* TODO */ + return -EINVAL; +} + +static int map_colored_pages(struct vm_area_struct *vma) +{ + if (likely(is_realtime(current))) + return map_colored_pages_rt(vma); + return map_colored_pages_non_rt(vma); +} + +static void litmus_color_alloc_vm_close(struct vm_area_struct *vma) +{ + TRACE_CUR("%s flags=0x%lx prot=0x%lx\n", __FUNCTION__, + vma->vm_flags, pgprot_val(vma->vm_page_prot)); + + TRACE_CUR(ALLOC_NAME ": %p:%p vma:%p vma->vm_private_data:%p closed.\n", + (void*) vma->vm_start, (void*) vma->vm_end, vma, + vma->vm_private_data); +} + +static int litmus_color_alloc_vm_fault(struct vm_area_struct *vma, + struct vm_fault *vmf) +{ + /* This function should never be called, since + * all pages should have been mapped by mmap() + * already. */ + TRACE_CUR("%s flags=0x%lx\n", __FUNCTION__, vma->vm_flags); + printk(KERN_WARNING "fault: %s flags=0x%lx\n", __FUNCTION__, + vma->vm_flags); + + /* nope, you only get one page */ + return VM_FAULT_SIGBUS; +} + +static struct vm_operations_struct litmus_color_alloc_vm_ops = { + .close = litmus_color_alloc_vm_close, + .fault = litmus_color_alloc_vm_fault, +}; + +static int litmus_color_alloc_mmap(struct file *filp, struct vm_area_struct *vma) +{ + int err = 0; + + /* you may only request integer multiple of PAGE_SIZE */ + if (offset_in_page(vma->vm_end - vma->vm_start)) { + err = -EINVAL; + goto out; + } + + err = mmap_common_checks(vma); + if (err) + goto out; + + vma->vm_ops = &litmus_color_alloc_vm_ops; + /* this mapping should not be kept across forks, + * and cannot be expanded */ + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND; + + err = map_colored_pages(vma); + + TRACE_CUR("%s flags=0x%lx prot=0x%lx\n", __FUNCTION__, vma->vm_flags, + pgprot_val(vma->vm_page_prot)); +out: + return err; +} + +/*********************************************************** + * Initilization +***********************************************************/ + +static struct file_operations litmus_color_ctrl_fops = { + .owner = THIS_MODULE, + .mmap = litmus_color_ctrl_mmap, +}; + +static struct miscdevice litmus_color_ctrl_dev = { + .name = CTRL_NAME, + .minor = MISC_DYNAMIC_MINOR, + .fops = &litmus_color_ctrl_fops, +}; + +static struct file_operations litmus_color_alloc_fops = { + .owner = THIS_MODULE, + .mmap = litmus_color_alloc_mmap, +}; + +static struct miscdevice litmus_color_alloc_dev = { + .name = ALLOC_NAME, + .minor = MISC_DYNAMIC_MINOR, + .fops = &litmus_color_alloc_fops, +}; + +static int __init init_dev(const char* name, struct miscdevice *dev) +{ + int err; + err = misc_register(dev); + if (err) + printk(KERN_WARNING "Could not allocate %s device (%d).\n", + name, err); + return err; +} + +static int __init init_color_devices(void) +{ + int err; + spin_lock_init(&non_rt_colors.lock); + non_rt_colors.color = 0; + + printk("Allocating LITMUS^RT color devices.\n"); + err = init_dev(ALLOC_NAME, &litmus_color_alloc_dev); + if (err) + goto out; + err = init_dev(CTRL_NAME, &litmus_color_ctrl_dev); +out: + return err; +} + +module_init(init_color_devices); diff --git a/litmus/color_proc.c b/litmus/color_proc.c index 31eec0d728a5..cac336ac1731 100644 --- a/litmus/color_proc.c +++ b/litmus/color_proc.c @@ -5,13 +5,22 @@ #include extern int color_sysctl_add_pages_data; /* litmus/color.c */ +extern int color_sysctl_reclaim_pages_data; /* litmus/color.c */ static int zero = 0; static int one = 1; -#define NR_PAGES_INDEX 1 /* location of nr_pages in the table below */ +#define NR_PAGES_INDEX 0 /* location of nr_pages in the table below */ static struct ctl_table color_table[] = { + { + /* you MUST update NR_PAGES_INDEX if you move this entry */ + .procname = "nr_pages", + .mode = 0444, + .proc_handler = color_nr_pages_handler, + .data = NULL, /* dynamically set later */ + .maxlen = 0, /* also set later */ + }, { .procname = "add_pages", .data = &color_sysctl_add_pages_data, @@ -22,11 +31,13 @@ static struct ctl_table color_table[] = .extra2 = &one, }, { - .procname = "nr_pages", - .mode = 0444, - .proc_handler = color_nr_pages_handler, - .data = NULL, /* dynamically later */ - .maxlen = 0, /* also set later */ + .procname = "reclaim_pages", + .data = &color_sysctl_reclaim_pages_data, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = color_reclaim_pages_handler, + .extra1 = &zero, + .extra2 = &one, }, { } }; @@ -71,6 +82,7 @@ static struct ctl_table_header *litmus_sysctls; static int __init litmus_sysctl_init(void) { int ret = 0; + printk(KERN_INFO "Registering LITMUS^RT proc sysctl.\n"); litmus_sysctls = register_sysctl_table(litmus_dir_table); if (!litmus_sysctls) { printk(KERN_WARNING "Could not register LITMUS^RT sysctl.\n"); diff --git a/litmus/litmus.c b/litmus/litmus.c index 301390148d02..eaa2070d28ce 100644 --- a/litmus/litmus.c +++ b/litmus/litmus.c @@ -291,12 +291,14 @@ static void reinit_litmus_state(struct task_struct* p, int restore) { struct rt_task user_config = {}; void* ctrl_page = NULL; + void* color_ctrl_page = NULL; if (restore) { /* Safe user-space provided configuration data. * and allocated page. */ - user_config = p->rt_param.task_params; - ctrl_page = p->rt_param.ctrl_page; + user_config = p->rt_param.task_params; + ctrl_page = p->rt_param.ctrl_page; + color_ctrl_page = p->rt_param.color_ctrl_page; } /* We probably should not be inheriting any task's priority @@ -309,8 +311,9 @@ static void reinit_litmus_state(struct task_struct* p, int restore) /* Restore preserved fields. */ if (restore) { - p->rt_param.task_params = user_config; - p->rt_param.ctrl_page = ctrl_page; + p->rt_param.task_params = user_config; + p->rt_param.ctrl_page = ctrl_page; + p->rt_param.color_ctrl_page = color_ctrl_page; } } @@ -451,9 +454,11 @@ void litmus_fork(struct task_struct* p) reinit_litmus_state(p, 0); /* Don't let the child be a real-time task. */ p->sched_reset_on_fork = 1; - } else + } else { /* non-rt tasks might have ctrl_page set */ tsk_rt(p)->ctrl_page = NULL; + tsk_rt(p)->color_ctrl_page = NULL; + } /* od tables are never inherited across a fork */ p->od_table = NULL; @@ -473,6 +478,10 @@ void litmus_exec(void) free_page((unsigned long) tsk_rt(p)->ctrl_page); tsk_rt(p)->ctrl_page = NULL; } + if (tsk_rt(p)->color_ctrl_page) { + free_page((unsigned long) tsk_rt(p)->color_ctrl_page); + tsk_rt(p)->color_ctrl_page = NULL; + } } } @@ -490,6 +499,12 @@ void exit_litmus(struct task_struct *dead_tsk) tsk_rt(dead_tsk)->ctrl_page); free_page((unsigned long) tsk_rt(dead_tsk)->ctrl_page); } + if (tsk_rt(dead_tsk)->color_ctrl_page) { + TRACE_TASK(dead_tsk, + "freeing color_ctrl_page %p\n", + tsk_rt(dead_tsk)->color_ctrl_page); + free_page((unsigned long) tsk_rt(dead_tsk)->color_ctrl_page); + } /* main cleanup only for RT tasks */ if (is_realtime(dead_tsk)) -- cgit v1.2.2