From d7352bf3c9392104c34b56e2c0756a14db81b68a Mon Sep 17 00:00:00 2001 From: Namhoon Kim Date: Thu, 22 Sep 2016 14:49:49 -0400 Subject: 9/22/2016 --- include/linux/fs.h | 1 + include/linux/mm.h | 2 + include/linux/mm_types.h | 7 +++ include/linux/mmzone.h | 1 + init/main.c | 1 + litmus/litmus.c | 5 +- litmus/sched_mc2.c | 15 ++++-- mm/Makefile | 2 +- mm/filemap.c | 129 ++++++++++++++++++++++++++++++++++++++++++----- mm/internal.h | 28 ++++++++++ mm/memory.c | 2 + mm/page-writeback.c | 3 +- 12 files changed, 175 insertions(+), 21 deletions(-) diff --git a/include/linux/fs.h b/include/linux/fs.h index 1ea979d984c2..9bddbf91dcbf 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -489,6 +489,7 @@ struct block_device { #define PAGECACHE_TAG_DIRTY 0 #define PAGECACHE_TAG_WRITEBACK 1 #define PAGECACHE_TAG_TOWRITE 2 +#define PAGECACHE_TAG_REPLICATED 3 int mapping_tagged(struct address_space *mapping, int tag); diff --git a/include/linux/mm.h b/include/linux/mm.h index 0755b9fd03a7..55df1f8bf4cb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2200,5 +2200,7 @@ void __init setup_nr_node_ids(void); static inline void setup_nr_node_ids(void) {} #endif +extern void replication_init(void); + #endif /* __KERNEL__ */ #endif /* _LINUX_MM_H */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 8d37e26a1007..76603d077455 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -14,6 +14,7 @@ #include #include #include +#include #ifndef AT_VECTOR_SIZE_ARCH #define AT_VECTOR_SIZE_ARCH 0 @@ -545,4 +546,10 @@ typedef struct { unsigned long val; } swp_entry_t; +struct pcache_desc { + struct page *master; + cpumask_t cpus_present; + struct radix_tree_root page_tree; +}; + #endif /* _LINUX_MM_TYPES_H */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 54d74f6eb233..d3762dec9dea 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -128,6 +128,7 @@ enum zone_stat_item { NR_FILE_PAGES, NR_FILE_DIRTY, NR_WRITEBACK, + NR_REPL_PAGES, NR_SLAB_RECLAIMABLE, NR_SLAB_UNRECLAIMABLE, NR_PAGETABLE, /* used for pagetables */ diff --git a/init/main.c b/init/main.c index 2a89545e0a5d..88917d93fbe4 100644 --- a/init/main.c +++ b/init/main.c @@ -628,6 +628,7 @@ asmlinkage __visible void __init start_kernel(void) kmemleak_init(); setup_per_cpu_pageset(); numa_policy_init(); + replication_init(); if (late_time_init) late_time_init(); sched_clock_init(); diff --git a/litmus/litmus.c b/litmus/litmus.c index ad3d50c78bb1..cad0e5adb5a3 100644 --- a/litmus/litmus.c +++ b/litmus/litmus.c @@ -444,6 +444,7 @@ asmlinkage long sys_set_page_color(int cpu) nr_shared_pages++; TRACE_TASK(current, "SHARED\n"); + put_page(old_page); } else { ret = isolate_lru_page(old_page); @@ -504,7 +505,7 @@ asmlinkage long sys_set_page_color(int cpu) } } } - +/* if (!list_empty(&task_shared_pagelist)) { ret = replicate_pages(&task_shared_pagelist, new_alloc_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL); TRACE_TASK(current, "%ld shared pages not migrated.\n", ret); @@ -513,7 +514,7 @@ asmlinkage long sys_set_page_color(int cpu) putback_movable_pages(&task_shared_pagelist); } } - +*/ /* handle sigpage and litmus ctrl_page */ /* vma_itr = current->mm->mmap; while (vma_itr != NULL) { diff --git a/litmus/sched_mc2.c b/litmus/sched_mc2.c index 5c88a36aacec..d0a39ff96b13 100644 --- a/litmus/sched_mc2.c +++ b/litmus/sched_mc2.c @@ -341,7 +341,7 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state) local = local_cpu_state() == state; raw_spin_lock(&_global_env.lock); - +// TRACE("P%d acquired GE lock L344\n"); list_for_each_entry_safe(event, next, &_global_env.next_events, list) { /* If the event time is already passed, we call schedule() on the lowest priority cpu */ @@ -376,6 +376,7 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state) /* Must drop state lock before calling into hrtimer_start(), which * may raise a softirq, which in turn may wake ksoftirqd. */ raw_spin_unlock(&_global_env.lock); +// TRACE("P%d releases GE lock L379\n"); raw_spin_unlock(&state->lock); if (update <= now || reschedule[state->cpu]) { @@ -600,6 +601,7 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer) } raw_spin_lock(&_global_env.lock); +// TRACE("P%d acquired GE lock L604\n"); global_schedule_now = gmp_update_time(&_global_env, now); BUG_ON(global_schedule_now < 0 || global_schedule_now > 4); @@ -619,7 +621,7 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer) } } raw_spin_unlock(&_global_env.lock); - +// TRACE("P%d releases GE lock L624\n"); raw_spin_unlock_irqrestore(&state->lock, flags); //raw_spin_unlock_irqrestore(&_global_env.lock, flags); @@ -679,6 +681,7 @@ static long mc2_complete_job(void) state = local_cpu_state(); raw_spin_lock(&state->lock); raw_spin_lock(&_global_env.lock); +// TRACE("P%d acquired GE lock L684\n"); res = gmp_find_by_id(&_global_env, tinfo->mc2_param.res_id); _global_env.env.time_zero = tsk_rt(current)->sporadic_release_time; } @@ -705,8 +708,10 @@ static long mc2_complete_job(void) //if (lv < CRIT_LEVEL_C) // raw_spin_unlock(&state->lock); //else - if (lv == CRIT_LEVEL_C) + if (lv == CRIT_LEVEL_C) { raw_spin_unlock(&_global_env.lock); +// TRACE("P%d releases GE lock L713\n"); + } raw_spin_unlock(&state->lock); local_irq_restore(flags); @@ -991,7 +996,7 @@ static struct task_struct* mc2_schedule(struct task_struct * prev) } } - if (to_schedule != 0) { + if (to_schedule > 0) { raw_spin_lock(&_global_env.lock); while (to_schedule--) { int cpu = get_lowest_prio_cpu(0); @@ -1078,9 +1083,11 @@ static void mc2_task_resume(struct task_struct *tsk) task_arrives(state, tsk); } else { raw_spin_lock(&_global_env.lock); +// TRACE("P%d acquired GE lock L1086\n"); gmp_update_time(&_global_env, litmus_clock()); task_arrives(state, tsk); raw_spin_unlock(&_global_env.lock); +// TRACE("P%d releases GE lock L1090\n"); } /* 9/20/2015 fix diff --git a/mm/Makefile b/mm/Makefile index 98c4eaeabdcb..98d28edd36a5 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -56,7 +56,7 @@ obj-$(CONFIG_KASAN) += kasan/ obj-$(CONFIG_FAILSLAB) += failslab.o obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o obj-$(CONFIG_MEMTEST) += memtest.o -obj-$(CONFIG_MIGRATION) += migrate.o +obj-$(CONFIG_MIGRATION) += migrate.o replication.o obj-$(CONFIG_QUICKLIST) += quicklist.o obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o obj-$(CONFIG_PAGE_COUNTER) += page_counter.o diff --git a/mm/filemap.c b/mm/filemap.c index 8f378ac675d7..60d7e38cf73d 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -36,6 +36,9 @@ #include #include "internal.h" +#include +#include + #define CREATE_TRACE_POINTS #include @@ -970,13 +973,23 @@ repeat: page = NULL; pagep = radix_tree_lookup_slot(&mapping->page_tree, offset); if (pagep) { - void *pdesc; - pdesc = radix_tree_deref_slot(pagep); - if (pdesc) - page = (struct page*)pdesc; - //page = radix_tree_deref_slot(pagep); + page = radix_tree_deref_slot(pagep); if (unlikely(!page)) goto out; + if (is_pcache_desc(page)) { + struct pcache_desc *pcd; + +printk(KERN_INFO "PCACHE_DESC\n"); + + pcd = ptr_to_pcache_desc(page); + page = pcd->master; + page_cache_get_speculative(page); + + unreplicate_pcache(mapping, page->index); + + goto out; + } + if (radix_tree_exception(page)) { if (radix_tree_deref_retry(page)) goto repeat; @@ -1175,6 +1188,21 @@ repeat: page = radix_tree_deref_slot(slot); if (unlikely(!page)) continue; + + if (is_pcache_desc(page)) { + struct pcache_desc *pcd; + +printk(KERN_INFO "PCACHE_DESC\n"); + + pcd = ptr_to_pcache_desc(page); + page = pcd->master; + page_cache_get_speculative(page); + + unreplicate_pcache(mapping, page->index); + + goto export; + } + if (radix_tree_exception(page)) { if (radix_tree_deref_retry(page)) goto restart; @@ -1238,6 +1266,20 @@ repeat: if (unlikely(!page)) continue; + if (is_pcache_desc(page)) { + struct pcache_desc *pcd; + +printk(KERN_INFO "PCACHE_DESC\n"); + + pcd = ptr_to_pcache_desc(page); + page = pcd->master; + page_cache_get_speculative(page); + + unreplicate_pcache(mapping, page->index); + + goto export; + } + if (radix_tree_exception(page)) { if (radix_tree_deref_retry(page)) { /* @@ -1264,7 +1306,7 @@ repeat: page_cache_release(page); goto repeat; } - +export: pages[ret] = page; if (++ret == nr_pages) break; @@ -1306,6 +1348,21 @@ repeat: if (unlikely(!page)) break; + if (is_pcache_desc(page)) { + struct pcache_desc *pcd; + +printk(KERN_INFO "PCACHE_DESC\n"); + + pcd = ptr_to_pcache_desc(page); + page = pcd->master; + if (!page_cache_get_speculative(page)) + goto repeat; + + unreplicate_pcache(mapping, page->index); + + goto export; + } + if (radix_tree_exception(page)) { if (radix_tree_deref_retry(page)) { /* @@ -1331,7 +1388,7 @@ repeat: page_cache_release(page); goto repeat; } - +export: /* * must check mapping and index after taking the ref. * otherwise we can get both false positives and false @@ -1382,6 +1439,10 @@ repeat: if (unlikely(!page)) continue; + if (is_pcache_desc(page)) { + BUG(); + } + if (radix_tree_exception(page)) { if (radix_tree_deref_retry(page)) { /* @@ -1489,7 +1550,13 @@ static ssize_t do_generic_file_read(struct file *filp, loff_t *ppos, cond_resched(); find_page: - page = find_get_page(mapping, index); + if (tsk_mc2_data(current)) { + printk(KERN_INFO "RT Task file_read1\n"); + page = find_get_page_readonly(mapping, index); + } + else + page = find_get_page(mapping, index); + if (!page) { page_cache_sync_readahead(mapping, ra, filp, @@ -1526,7 +1593,6 @@ page_ok: * part of the page is not copied back to userspace (unless * another truncate extends the file - this is desired though). */ - isize = i_size_read(inode); end_index = (isize - 1) >> PAGE_CACHE_SHIFT; if (unlikely(!isize || index > end_index)) { @@ -1544,7 +1610,16 @@ page_ok: } } nr = nr - offset; - + + if (tsk_mc2_data(current)) { + printk(KERN_INFO "RT Task file_read2\n"); + page_cache_release(page); + page = find_get_page_readonly(mapping, index); + if (!page) { + BUG(); + } + } + /* If users can be writing to this page using arbitrary * virtual addresses, take care about potential aliasing * before reading the page on the kernel side. @@ -1885,7 +1960,13 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) /* * Do we have something in the page cache already? */ - page = find_get_page(mapping, offset); + if (tsk_mc2_data(current)) { + printk(KERN_INFO "RT Task filemap_fault1\n"); + page = find_get_page_readonly(mapping, offset); + } + else + page = find_get_page(mapping, offset); + if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) { /* * We found the page, so try async readahead before @@ -1899,7 +1980,13 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); ret = VM_FAULT_MAJOR; retry_find: - page = find_get_page(mapping, offset); + if (tsk_mc2_data(current)) { + printk(KERN_INFO "RT Task filemap_fault2\n"); + page = find_get_page_readonly(mapping, offset); + } + else + page = find_get_page(mapping, offset); + if (!page) goto no_cached_page; } @@ -2007,6 +2094,22 @@ repeat: page = radix_tree_deref_slot(slot); if (unlikely(!page)) goto next; + + if (is_pcache_desc(page)) { + struct pcache_desc *pcd; + +printk(KERN_INFO "PCACHE_DESC\n"); + + pcd = ptr_to_pcache_desc(page); + page = pcd->master; + if (!page_cache_get_speculative(page)) + goto repeat; + + unreplicate_pcache(mapping, page->index); + + goto export; + } + if (radix_tree_exception(page)) { if (radix_tree_deref_retry(page)) break; @@ -2022,7 +2125,7 @@ repeat: page_cache_release(page); goto repeat; } - +export: if (!PageUptodate(page) || PageReadahead(page) || PageHWPoison(page)) diff --git a/mm/internal.h b/mm/internal.h index a25e359a4039..8bdf34edb5c2 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -13,6 +13,34 @@ #include #include +#include + +/* + * Pagecache replication + */ +static inline int pcache_replicated(struct address_space *mapping, unsigned long offset) +{ + return radix_tree_tag_get(&mapping->page_tree, offset, + PAGECACHE_TAG_REPLICATED); +} + +extern int reclaim_replicated_page(struct address_space *mapping, + struct page *page); +extern struct page *get_unreplicated_page(struct address_space *mapping, + unsigned long offset, struct page *page); +extern void get_unreplicated_pages(struct address_space *mapping, + struct page **pages, int nr); +extern struct page *find_get_page_readonly(struct address_space *mapping, + unsigned long offset); +extern int is_pcache_desc(void *ptr); +extern struct pcache_desc *ptr_to_pcache_desc(void *ptr); +extern void *pcache_desc_to_ptr(struct pcache_desc *pcd); +extern void unreplicate_pcache(struct address_space *mapping, unsigned long offset); + +struct page *get_unreplicated_page_fault(struct page *page); + +/* End Pagecache replication */ + void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, unsigned long floor, unsigned long ceiling); diff --git a/mm/memory.c b/mm/memory.c index 22e037e3364e..c9fa78dfccb9 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2235,6 +2235,8 @@ static int wp_page_shared(struct mm_struct *mm, struct vm_area_struct *vma, int tmp; pte_unmap_unlock(page_table, ptl); + old_page = get_unreplicated_page_fault(old_page); + tmp = do_page_mkwrite(vma, old_page, address); if (unlikely(!tmp || (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 7e39ffceb566..161af608b7e2 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2287,7 +2287,8 @@ int clear_page_dirty_for_io(struct page *page) BUG_ON(!PageLocked(page)); - if (mapping && mapping_cap_account_dirty(mapping)) { + //if (mapping && mapping_cap_account_dirty(mapping)) { + if (mapping) { /* * Yes, Virginia, this is indeed insane. * -- cgit v1.2.2