From 2dd395b092b13e26aaa7a1ce8795be7b6d4a24c9 Mon Sep 17 00:00:00 2001 From: Namhoon Kim Date: Wed, 7 Feb 2018 22:13:48 -0500 Subject: IPC supports --- arch/arm/mm/dma-mapping.c | 2 +- drivers/media/usb/uvc/uvc_video.c | 7 ++- drivers/media/v4l2-core/videobuf2-core.c | 9 ++-- drivers/media/v4l2-core/videobuf2-vmalloc.c | 5 +- drivers/net/ethernet/freescale/fec_main.c | 2 +- drivers/usb/core/urb.c | 10 +++- include/linux/msg.h | 2 + include/litmus/cache_proc.h | 4 ++ ipc/mqueue.c | 2 +- ipc/msg.c | 14 +++-- ipc/msgutil.c | 68 ++++++++++++++++++++---- ipc/util.h | 2 +- kernel/softirq.c | 11 ++-- litmus/bank_proc.c | 12 +++-- litmus/cache_proc.c | 82 ++++++++++++++++++++++++++--- litmus/litmus.c | 38 ++++++++++++- mm/page_alloc.c | 4 +- mm/slab_common.c | 9 +++- mm/vmalloc.c | 2 +- net/core/dev.c | 29 ++++++---- net/core/skbuff.c | 42 +++++++-------- 21 files changed, 273 insertions(+), 83 deletions(-) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index bd7dfb1b7ebd..b27199c9ae15 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -260,7 +260,7 @@ static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gf if (!page) return NULL; if (gfp&GFP_COLOR) - printk(KERN_INFO "__dma_alloc_buffer(): size %d, order %ld requested\n", size, order); + //printk(KERN_INFO "__dma_alloc_buffer(): size %d, order %ld requested\n", size, order); /* * Now split the huge page and free the excess pages */ diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index 9daef917557b..3dd00a6b51fe 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -26,7 +26,7 @@ #include "uvcvideo.h" -#define ENABLE_WORST_CASE 1 +//#define ENABLE_WORST_CASE 1 #ifdef ENABLE_WORST_CASE #define UVC_FLAG (GFP_COLOR|GFP_CPU1) #else @@ -343,6 +343,7 @@ int uvc_probe_video(struct uvc_streaming *stream, break; if (stream->dev->quirks & UVC_QUIRK_PROBE_MINMAX) { + printk(KERN_INFO "uvc_probe_video(): no space error\n"); ret = -ENOSPC; goto done; } @@ -1619,6 +1620,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags) /* Isochronous endpoint, select the alternate setting. */ bandwidth = stream->ctrl.dwMaxPayloadTransferSize; + //bandwidth = 1; if (bandwidth == 0) { uvc_trace(UVC_TRACE_VIDEO, "Device requested null " @@ -1656,7 +1658,8 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags) uvc_trace(UVC_TRACE_VIDEO, "Selecting alternate setting %u " "(%u B/frame bandwidth).\n", altsetting, best_psize); - + printk(KERN_INFO "Selecting alternate setting %u " + "(%u B/frame bandwidth).\n", altsetting, best_psize); ret = usb_set_interface(stream->dev->udev, intfnum, altsetting); if (ret < 0) return ret; diff --git a/drivers/media/v4l2-core/videobuf2-core.c b/drivers/media/v4l2-core/videobuf2-core.c index 54058877f467..0447135e89f1 100644 --- a/drivers/media/v4l2-core/videobuf2-core.c +++ b/drivers/media/v4l2-core/videobuf2-core.c @@ -30,7 +30,7 @@ #include #include -#define ENABLE_WORST_CASE 1 +//#define ENABLE_WORST_CASE 1 #ifdef ENABLE_WORST_CASE #define VB2_CORE_FLAG (GFP_COLOR|GFP_CPU1) #else @@ -207,9 +207,9 @@ static int __vb2_buf_mem_alloc(struct vb2_buffer *vb) */ for (plane = 0; plane < vb->num_planes; ++plane) { unsigned long size = PAGE_ALIGN(q->plane_sizes[plane]); - printk(KERN_INFO "__vb2_buf_mem_alloc(): size %ld, func %pF GFP_COLOR? %d\n", size, vb->vb2_queue->mem_ops->alloc, q->gfp_flags&GFP_COLOR); + printk(KERN_INFO "__vb2_buf_mem_alloc(): size %ld, func %pF GFP_COLOR? %d\n", size, vb->vb2_queue->mem_ops->alloc, q->gfp_flags|VB2_CORE_FLAG); mem_priv = call_ptr_memop(vb, alloc, q->alloc_ctx[plane], - size, dma_dir, q->gfp_flags); + size, dma_dir, q->gfp_flags|VB2_CORE_FLAG); if (IS_ERR_OR_NULL(mem_priv)) goto free; @@ -1783,8 +1783,10 @@ static int vb2_start_streaming(struct vb2_queue *q) /* Tell the driver to start streaming */ q->start_streaming_called = 1; + printk(KERN_INFO "vb2_start_streaming(): %pF\n", q->ops->start_streaming); ret = call_qop(q, start_streaming, q, atomic_read(&q->owned_by_drv_count)); + printk(KERN_INFO "vb2_start_streaming() : ret1 = %d\n", ret); if (!ret) return 0; @@ -1818,6 +1820,7 @@ static int vb2_start_streaming(struct vb2_queue *q) * STATE_DONE. */ WARN_ON(!list_empty(&q->done_list)); + printk(KERN_INFO "vb2_start_streaming() : ret2 = %d\n", ret); return ret; } diff --git a/drivers/media/v4l2-core/videobuf2-vmalloc.c b/drivers/media/v4l2-core/videobuf2-vmalloc.c index 813cc718c116..97ff2c45b8b2 100644 --- a/drivers/media/v4l2-core/videobuf2-vmalloc.c +++ b/drivers/media/v4l2-core/videobuf2-vmalloc.c @@ -21,14 +21,13 @@ #include #include - -#define ENABLE_WORST_CASE 1 +//#define ENABLE_WORST_CASE 1 #ifdef ENABLE_WORST_CASE #define VB2_FLAG (GFP_COLOR|GFP_CPU1) #else #define VB2_FLAG (GFP_COLOR) #endif - + struct vb2_vmalloc_buf { void *vaddr; struct page **pages; diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 2f7aab0b60cf..faaea8abddd0 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -65,7 +65,7 @@ #include "fec.h" -#define ENABLE_WORST_CASE 1 +//#define ENABLE_WORST_CASE 1 #ifdef ENABLE_WORST_CASE #define FEC_FLAG (GFP_COLOR|GFP_CPU1) #else diff --git a/drivers/usb/core/urb.c b/drivers/usb/core/urb.c index c9e8ee81b6b7..468334a94b41 100644 --- a/drivers/usb/core/urb.c +++ b/drivers/usb/core/urb.c @@ -10,6 +10,12 @@ #define to_urb(d) container_of(d, struct urb, kref) +//#define ENABLE_WORST_CASE 1 +#ifdef ENABLE_WORST_CASE +#define URB_FLAG (GFP_COLOR|GFP_CPU1) +#else +#define URB_FLAG (GFP_COLOR) +#endif static void urb_destroy(struct kref *kref) { @@ -67,7 +73,7 @@ struct urb *usb_alloc_urb(int iso_packets, gfp_t mem_flags) urb = kmalloc(sizeof(struct urb) + iso_packets * sizeof(struct usb_iso_packet_descriptor), - mem_flags); + mem_flags|URB_FLAG); if (!urb) { printk(KERN_ERR "alloc_urb: kmalloc failed\n"); return NULL; @@ -539,7 +545,7 @@ int usb_submit_urb(struct urb *urb, gfp_t mem_flags) } } - return usb_hcd_submit_urb(urb, mem_flags); + return usb_hcd_submit_urb(urb, mem_flags|URB_FLAG); } EXPORT_SYMBOL_GPL(usb_submit_urb); diff --git a/include/linux/msg.h b/include/linux/msg.h index f3f302f9c197..9bf01c93f56e 100644 --- a/include/linux/msg.h +++ b/include/linux/msg.h @@ -12,6 +12,8 @@ struct msg_msg { struct msg_msgseg *next; void *security; /* the actual message follows immediately */ + dma_addr_t handle; + size_t alloc_len; }; /* one msq_queue structure for each present queue on the system */ diff --git a/include/litmus/cache_proc.h b/include/litmus/cache_proc.h index e9440de504fc..1d9d443da3db 100644 --- a/include/litmus/cache_proc.h +++ b/include/litmus/cache_proc.h @@ -8,9 +8,13 @@ void enter_irq_mode(void); void exit_irq_mode(void); void flush_cache(int all); void lock_cache(int cpu, u32 val); +void cache_lockdown(u32 lock_val, int cpu); extern struct page *new_alloc_page_color(unsigned long color); +u32 color_read_in_mem_lock(u32 lock_val, u32 unlock_val, void *start, void *end); +u32 color_read_in_mem(u32 lock_val, u32 unlock_val, void *start, void *end); + #endif #endif diff --git a/ipc/mqueue.c b/ipc/mqueue.c index 3aaea7ffd077..4cb1b7f0dcbf 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -1004,7 +1004,7 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr, /* First try to allocate memory, before doing anything with * existing queues. */ - msg_ptr = load_msg(u_msg_ptr, msg_len); + msg_ptr = load_msg(u_msg_ptr, msg_len, 0); if (IS_ERR(msg_ptr)) { ret = PTR_ERR(msg_ptr); goto out_fput; diff --git a/ipc/msg.c b/ipc/msg.c index 2b6fdbb9e0e9..b874d700023f 100644 --- a/ipc/msg.c +++ b/ipc/msg.c @@ -38,6 +38,10 @@ #include #include +#include +#include +#include + #include #include #include "util.h" @@ -619,7 +623,7 @@ long do_msgsnd(int msqid, long mtype, void __user *mtext, if (mtype < 1) return -EINVAL; - msg = load_msg(mtext, msgsz); + msg = load_msg(mtext, msgsz, mtype); if (IS_ERR(msg)) return PTR_ERR(msg); @@ -752,13 +756,13 @@ static long do_msg_fill(void __user *dest, struct msg_msg *msg, size_t bufsz) { struct msgbuf __user *msgp = dest; size_t msgsz; - if (put_user(msg->m_type, &msgp->mtype)) return -EFAULT; - msgsz = (bufsz > msg->m_ts) ? msg->m_ts : bufsz; + if (store_msg(msgp->mtext, msg, msgsz)) return -EFAULT; + return msgsz; } @@ -976,10 +980,10 @@ out_unlock1: free_copy(copy); return PTR_ERR(msg); } - +TS_NET_RX_HARDIRQ_START; bufsz = msg_handler(buf, msg, bufsz); free_msg(msg); - +TS_NET_RX_HARDIRQ_END; return bufsz; } diff --git a/ipc/msgutil.c b/ipc/msgutil.c index 2b491590ebab..7570da15a053 100644 --- a/ipc/msgutil.c +++ b/ipc/msgutil.c @@ -18,6 +18,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include "util.h" @@ -42,20 +48,37 @@ atomic_t nr_ipc_ns = ATOMIC_INIT(1); struct msg_msgseg { struct msg_msgseg *next; /* the next part of the message follows immediately */ + dma_addr_t seg_handle; + size_t seg_len; }; #define DATALEN_MSG ((size_t)PAGE_SIZE-sizeof(struct msg_msg)) #define DATALEN_SEG ((size_t)PAGE_SIZE-sizeof(struct msg_msgseg)) +static dma_addr_t handle; -static struct msg_msg *alloc_msg(size_t len) +mempool_t *msgpool; +extern void *msgvaddr; + +static struct msg_msg *alloc_msg(size_t len, long mtype) { struct msg_msg *msg; struct msg_msgseg **pseg; size_t alen; + int n_seg = 0; alen = min(len, DATALEN_MSG); - msg = kmalloc(sizeof(*msg) + alen, GFP_KERNEL); + if (mtype == 1) { + //printk(KERN_INFO "SBP message\n"); + msg = dma_alloc_coherent(NULL, sizeof(*msg) + alen, &handle, GFP_KERNEL|GFP_COLOR|GFP_CPU1); + msg->handle = handle; + msg->alloc_len = sizeof(*msg) + alen; + } else if (mtype == 2) { + msg = kmalloc(sizeof(*msg) + alen, GFP_KERNEL|GFP_COLOR); + } else { + //msg = kmalloc(sizeof(*msg) + alen, GFP_KERNEL); + msg = msgvaddr; + } if (msg == NULL) return NULL; @@ -67,7 +90,18 @@ static struct msg_msg *alloc_msg(size_t len) while (len > 0) { struct msg_msgseg *seg; alen = min(len, DATALEN_SEG); - seg = kmalloc(sizeof(*seg) + alen, GFP_KERNEL); + if (mtype == 1) { + seg = dma_alloc_coherent(NULL, sizeof(*seg) + alen, &handle, GFP_KERNEL|GFP_COLOR|GFP_CPU1); + seg->seg_handle = handle; + seg->seg_len = alen; + //printk(KERN_INFO "SBP message seg %d\n", seg->seg_len); + } else if (mtype == 2) { + seg = kmalloc(sizeof(*seg) + alen, GFP_KERNEL|GFP_COLOR); + } else { + //seg = kmalloc(sizeof(*seg) + alen, GFP_KERNEL); + n_seg++; + seg = msgvaddr + PAGE_SIZE*n_seg; + } if (seg == NULL) goto out_err; *pseg = seg; @@ -83,18 +117,20 @@ out_err: return NULL; } -struct msg_msg *load_msg(const void __user *src, size_t len) +struct msg_msg *load_msg(const void __user *src, size_t len, long mtype) { struct msg_msg *msg; struct msg_msgseg *seg; int err = -EFAULT; size_t alen; - msg = alloc_msg(len); +TS_NET_RX_SOFTIRQ_START; + msg = alloc_msg(len, mtype); if (msg == NULL) return ERR_PTR(-ENOMEM); alen = min(len, DATALEN_MSG); + if (copy_from_user(msg + 1, src, alen)) goto out_err; @@ -105,7 +141,11 @@ struct msg_msg *load_msg(const void __user *src, size_t len) if (copy_from_user(seg + 1, src, alen)) goto out_err; } - +TS_NET_RX_SOFTIRQ_END; +/* if (mtype == 3) { + cache_lockdown(0xFFFF8000, smp_processor_id()); + } +*/ err = security_msg_msg_alloc(msg); if (err) goto out_err; @@ -172,14 +212,24 @@ int store_msg(void __user *dest, struct msg_msg *msg, size_t len) void free_msg(struct msg_msg *msg) { struct msg_msgseg *seg; - + long mtype = msg->m_type; + security_msg_msg_free(msg); seg = msg->next; - kfree(msg); + if (mtype == 1) { + //printk(KERN_INFO "free_msg(): SBP message\n"); + dma_free_coherent(NULL, msg->alloc_len, msg, msg->handle); + } else if (mtype != 3) { + kfree(msg); + } while (seg != NULL) { struct msg_msgseg *tmp = seg->next; - kfree(seg); + if (mtype == 1) { + //printk(KERN_INFO "free_msg(): SBP message seg %d\n", seg->seg_len); + dma_free_coherent(NULL, sizeof(*seg)+(seg->seg_len), seg, seg->seg_handle); + } else if (mtype != 3) + kfree(seg); seg = tmp; } } diff --git a/ipc/util.h b/ipc/util.h index 1a5a0fcd099c..9a2998308088 100644 --- a/ipc/util.h +++ b/ipc/util.h @@ -149,7 +149,7 @@ int ipc_parse_version(int *cmd); #endif extern void free_msg(struct msg_msg *msg); -extern struct msg_msg *load_msg(const void __user *src, size_t len); +extern struct msg_msg *load_msg(const void __user *src, size_t len, long mtype); extern struct msg_msg *copy_msg(struct msg_msg *src, struct msg_msg *dst); extern int store_msg(void __user *dest, struct msg_msg *msg, size_t len); diff --git a/kernel/softirq.c b/kernel/softirq.c index 5c694353763b..a546e77b0c50 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -276,12 +276,11 @@ restart: kstat_incr_softirqs_this_cpu(vec_nr); trace_softirq_entry(vec_nr); -// if (vec_nr == 3) -// TS_NET_RX_SOFTIRQ_START; -// net_rx_action() is called here - h->action(h); -// if (vec_nr == 3) -// TS_NET_RX_SOFTIRQ_END; + if (vec_nr == 3) + TS_NET_RX_SOFTIRQ_START; + h->action(h); //net_rx_action() + if (vec_nr == 3) + TS_NET_RX_SOFTIRQ_END; trace_softirq_exit(vec_nr); if (unlikely(prev_count != preempt_count())) { pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", diff --git a/litmus/bank_proc.c b/litmus/bank_proc.c index 353d38dbe9d6..3c4f70349133 100644 --- a/litmus/bank_proc.c +++ b/litmus/bank_proc.c @@ -548,6 +548,7 @@ out: return ret; } +/* static struct ctl_table cache_table[] = { @@ -728,7 +729,8 @@ static struct ctl_table cache_table[] = }, { } }; - +*/ +/* static struct ctl_table litmus_dir_table[] = { { .procname = "litmus", @@ -737,9 +739,9 @@ static struct ctl_table litmus_dir_table[] = { }, { } }; +*/ - -static struct ctl_table_header *litmus_sysctls; +//static struct ctl_table_header *litmus_sysctls; /* @@ -749,7 +751,7 @@ static int __init litmus_color_init(void) { int err=0; printk("Init bankproc.c\n"); - +/* init_variables(); printk(KERN_INFO "Registering LITMUS^RT proc color sysctl.\n"); @@ -763,7 +765,7 @@ static int __init litmus_color_init(void) init_color_groups(); do_add_pages(); - +*/ printk(KERN_INFO "Registering LITMUS^RT color and bank proc.\n"); out: return err; diff --git a/litmus/cache_proc.c b/litmus/cache_proc.c index 102feaf5c9e6..85ba5485acf9 100644 --- a/litmus/cache_proc.c +++ b/litmus/cache_proc.c @@ -161,6 +161,9 @@ static int l2_data_prefetch_proc; static int os_isolation; static int use_part; +static u32 debug_test_val; +struct mutex debug_mutex; + u32 lockdown_reg[9] = { 0x00000000, 0x00000000, @@ -283,6 +286,7 @@ void litmus_setup_lockdown(void __iomem *base, u32 id) mutex_init(&actlr_mutex); mutex_init(&l2x0_prefetch_mutex); mutex_init(&lockdown_proc); + mutex_init(&debug_mutex); raw_spin_lock_init(&cache_lock); raw_spin_lock_init(&prefetch_lock); @@ -676,6 +680,41 @@ int litmus_l2_data_prefetch_proc_handler(struct ctl_table *table, int write, return ret; } +extern void *msgvaddr; + +int do_measure(void) { + lt_t t1, t2; + int i; + + barrier(); + t1 = litmus_clock(); + color_read_in_mem_lock(0xFFFF7FFF, 0xFFFF8000, msgvaddr, msgvaddr + 65536); + t2 = litmus_clock() - t1; + barrier(); + + for (i = 0; i < 8; i++) { + cache_lockdown(0xFFFF8000, i); + } + printk("mem read time %lld\n", t2); + + return 0; +} + +int debug_test_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret; + + //mutex_lock(&debug_mutex); + //ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (write) { + ret = do_measure(); + } + + return ret; +} + + int do_perf_test_proc_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); @@ -871,6 +910,11 @@ static struct ctl_table cache_table[] = .extra1 = &way_partition_min, .extra2 = &way_partition_max, }, + { + .procname = "debug_test", + .mode = 0644, + .proc_handler = debug_test_handler, + }, { } }; @@ -1201,12 +1245,13 @@ asmlinkage long sys_lock_buffer(void* vaddr, size_t size, u32 lock_way, u32 unlo static int perf_test(void) { struct timespec before, after; - struct page *page; - void *vaddr; - u32 *data; + struct page *page, *page2; + void *vaddr, *vaddr2; + u32 *data, *data2; long time, flush_time; - int i, num_pages = 1; - unsigned int order = 4; + int i, n, num_pages = 1; + unsigned int order = 2; + lt_t t1 = 0, t2 = 0; for (i = 0; i < order; i++) { num_pages = num_pages*2; @@ -1220,11 +1265,35 @@ static int perf_test(void) { return -ENOMEM; } + page2 = alloc_pages(__GFP_MOVABLE, order); + if (!page2) { + printk(KERN_WARNING "No memory\n"); + return -ENOMEM; + } + vaddr = page_address(page); if (!vaddr) printk(KERN_WARNING "%s: vaddr is null\n", __FUNCTION__); data = (u32*) vaddr; + vaddr2 = page_address(page2); + if (!vaddr2) + printk(KERN_WARNING "%s: vaddr2 is null\n", __FUNCTION__); + data2 = (u32*) vaddr2; + + for (i = 32; i < 4096; i *= 2) { + for (n = 0; n < TRIALS; n++) { + invalidate_kernel_vmap_range(vaddr, 8192); + invalidate_kernel_vmap_range(vaddr2, 8192); + barrier(); + t1 = litmus_clock(); + memcpy(vaddr2, vaddr, i); + barrier(); + t2 += litmus_clock() - t1; + } + printk("Size %d, average for memcpy %lld\n", i, t2>>9); + } +/* getnstimeofday(&before); barrier(); for (i = 0; i < TRIALS; i++) { @@ -1277,9 +1346,10 @@ static int perf_test(void) { time = update_timeval(before, after); printk("Average for read in after write: %ld\n", time / TRIALS); - +*/ //free_page((unsigned long)vaddr); free_pages((unsigned long)vaddr, order); + free_pages((unsigned long)vaddr2, order); return 0; } diff --git a/litmus/litmus.c b/litmus/litmus.c index ec9379979e1a..144e3bfdcfc1 100644 --- a/litmus/litmus.c +++ b/litmus/litmus.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -604,7 +605,7 @@ asmlinkage long sys_test_call(unsigned int param) } TRACE_TASK(current, "addr: %08x, phy: %08x, color: %d, bank: %d, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s mapping: %p\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_phys(old_page), page_color(old_page), page_bank(old_page), page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-", &(old_page->mapping)); - printk(KERN_INFO "addr: %08x, phy: %08x, color: %d, bank: %d, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s mapping: %p\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_phys(old_page), page_color(old_page), page_bank(old_page), page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-", &(old_page->mapping)); + //printk(KERN_INFO "addr: %08x, phy: %08x, color: %d, bank: %d, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s mapping: %p\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_phys(old_page), page_color(old_page), page_bank(old_page), page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-", &(old_page->mapping)); put_page(old_page); } vma_itr = vma_itr->vm_next; @@ -623,6 +624,8 @@ asmlinkage long sys_test_call(unsigned int param) } rcu_read_unlock(); } + } else if (param == 2) { + flush_cache_all(); } return ret; @@ -959,6 +962,35 @@ static struct notifier_block shutdown_notifier = { .notifier_call = litmus_shutdown_nb, }; +extern mempool_t *msgpool; +struct page *msgpages; +void *msgvaddr; + +static int litmus_msgpool_init(void) +{ + int i; + lt_t t1, t2; + + msgpages = alloc_pages(GFP_KERNEL, 4); + if (!msgpages) { + printk(KERN_WARNING "No memory\n"); + return -ENOMEM; + } + msgvaddr = page_address(msgpages); + + printk(KERN_INFO "pfn %05lx addr %p\n", page_to_pfn(msgpages), msgvaddr); + + for (i = 0; i < 8; i++) { + cache_lockdown(0xFFFF8000, i); + } + t1 = litmus_clock(); + color_read_in_mem_lock(0xFFFF7FFF, 0xFFFF8000, msgvaddr, msgvaddr + 65536); + t2 = litmus_clock() - t1; + printk(KERN_INFO "mem read time %lld\n", t2); + + return 0; +} + static int __init _init_litmus(void) { /* Common initializers, @@ -993,7 +1025,9 @@ static int __init _init_litmus(void) color_mask = ((cache_info_sets << line_size_log) - 1) ^ (PAGE_SIZE - 1); printk("Page color mask %lx\n", color_mask); #endif - + + litmus_msgpool_init(); + return 0; } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 8c22d10b0c23..3fc137b3ac36 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1299,7 +1299,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, } set_freepage_migratetype(page, migratetype); - //printk(KERN_INFO "__rmqueue_smallest(): CPU%d COLOR %d BANK %d page return %p pfn:%05lx\n", cpu, page_color(page), page_bank(page), page, page_to_pfn(page)); + //printk(KERN_INFO "__rmqueue_smallest(): CPU%d COLOR %d BANK %d page order %d return %p pfn:%05lx\n", cpu, page_color(page), page_bank(page), order, page, page_to_pfn(page)); return page; } } else { @@ -1630,7 +1630,7 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, trace_mm_page_alloc_extfrag(page, order, current_order, start_migratetype, fallback_mt); - //printk(KERN_INFO "__rmqueue_fallback(): CPU%d COLOR %d BANK %d page return %p pfn:%05lx\n", cpu, page_color(page), page_bank(page), page, page_to_pfn(page)); + //printk(KERN_INFO "P%d __rmqueue_fallback(): cpu%d's bank COLOR %d BANK %d page order %d return %p pfn:%05lx\n", cpu, area_index, page_color(page), page_bank(page), order, page, page_to_pfn(page)); return page; } } else { diff --git a/mm/slab_common.c b/mm/slab_common.c index bbd0ddc0b029..dec9075e486b 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -30,6 +30,13 @@ LIST_HEAD(slab_caches); DEFINE_MUTEX(slab_mutex); struct kmem_cache *kmem_cache; +//#define ENABLE_WORST_CASE 1 +#ifdef ENABLE_WORST_CASE +#define KMEM_FLAG (GFP_COLOR|GFP_CPU1) +#else +#define KMEM_FLAG (0) +#endif + /* * Set of flags that will prevent slab merging */ @@ -303,7 +310,7 @@ do_kmem_cache_create(const char *name, size_t object_size, size_t size, int err; err = -ENOMEM; - s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); + s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL|KMEM_FLAG); if (!s) goto out; diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 7a6d7de8fff8..c7ac2adfd667 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -1757,7 +1757,7 @@ EXPORT_SYMBOL(vmalloc); void *vmalloc_color(unsigned long size) { return __vmalloc_node_flags(size, NUMA_NO_NODE, - GFP_KERNEL | __GFP_HIGHMEM | GFP_COLOR); + GFP_KERNEL | __GFP_HIGHMEM | GFP_COLOR | GFP_CPU1); } EXPORT_SYMBOL(vmalloc_color); diff --git a/net/core/dev.c b/net/core/dev.c index f02d0c582e84..7291e7ec8696 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -138,6 +138,13 @@ #include "net-sysfs.h" +//#define ENABLE_WORST_CASE 1 +#ifdef ENABLE_WORST_CASE +#define DEV_FLAG (GFP_COLOR|GFP_CPU1) +#else +#define DEV_FLAG (0) +#endif + /* Instead of increasing this, you should create a hash table. */ #define MAX_GRO_SKBS 8 @@ -1718,7 +1725,7 @@ EXPORT_SYMBOL_GPL(is_skb_forwardable); int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) { - if (skb_orphan_frags(skb, GFP_ATOMIC) || + if (skb_orphan_frags(skb, GFP_ATOMIC|DEV_FLAG) || unlikely(!is_skb_forwardable(dev, skb))) { atomic_long_inc(&dev->rx_dropped); kfree_skb(skb); @@ -1762,7 +1769,7 @@ static inline int deliver_skb(struct sk_buff *skb, struct packet_type *pt_prev, struct net_device *orig_dev) { - if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) + if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC|DEV_FLAG))) return -ENOMEM; atomic_inc(&skb->users); return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); @@ -1827,7 +1834,7 @@ again: } /* need to clone skb, done only once */ - skb2 = skb_clone(skb, GFP_ATOMIC); + skb2 = skb_clone(skb, GFP_ATOMIC|DEV_FLAG); if (!skb2) goto out_unlock; @@ -1986,7 +1993,7 @@ static struct xps_map *expand_xps_map(struct xps_map *map, } /* Need to allocate new map to store queue on this CPU's map */ - new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, + new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL|DEV_FLAG, cpu_to_node(cpu)); if (!new_map) return NULL; @@ -2018,7 +2025,7 @@ int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, continue; if (!new_dev_maps) - new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); + new_dev_maps = kzalloc(maps_sz, GFP_KERNEL|DEV_FLAG); if (!new_dev_maps) { mutex_unlock(&xps_map_mutex); return -ENOMEM; @@ -2398,7 +2405,7 @@ int skb_checksum_help(struct sk_buff *skb) if (skb_cloned(skb) && !skb_clone_writable(skb, offset + sizeof(__sum16))) { - ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); + ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC|DEV_FLAG); if (ret) goto out; } @@ -3777,7 +3784,7 @@ ncls: } if (pt_prev) { - if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) + if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC|DEV_FLAG))) goto drop; else ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); @@ -6259,7 +6266,7 @@ static int netif_alloc_rx_queues(struct net_device *dev) BUG_ON(count < 1); - rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); + rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT | DEV_FLAG); if (!rx) { rx = vzalloc(sz); if (!rx) @@ -6300,7 +6307,7 @@ static int netif_alloc_netdev_queues(struct net_device *dev) BUG_ON(count < 1 || count > 0xffff); - tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); + tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT | DEV_FLAG); if (!tx) { tx = vzalloc(sz); if (!tx) @@ -6735,7 +6742,7 @@ struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) #ifdef CONFIG_NET_CLS_ACT if (queue) return queue; - queue = kzalloc(sizeof(*queue), GFP_KERNEL); + queue = kzalloc(sizeof(*queue), GFP_KERNEL|DEV_FLAG); if (!queue) return NULL; netdev_init_one_queue(dev, queue, NULL); @@ -6808,7 +6815,7 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, /* ensure 32-byte alignment of whole construct */ alloc_size += NETDEV_ALIGN - 1; - p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); + p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT | DEV_FLAG); if (!p) p = vzalloc(alloc_size); if (!p) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 92f091ce1d47..0407fe78a4f1 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -77,7 +77,7 @@ #include #include -#define ENABLE_WORST_CASE 1 +//#define ENABLE_WORST_CASE 1 #ifdef ENABLE_WORST_CASE #define SKB_FLAG (GFP_COLOR|GFP_CPU1) #else @@ -1255,11 +1255,11 @@ struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) int delta = headroom - skb_headroom(skb); if (delta <= 0) - skb2 = pskb_copy(skb, GFP_ATOMIC); + skb2 = pskb_copy(skb, GFP_ATOMIC | SKB_FLAG); else { - skb2 = skb_clone(skb, GFP_ATOMIC); + skb2 = skb_clone(skb, GFP_ATOMIC | SKB_FLAG); if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, - GFP_ATOMIC)) { + GFP_ATOMIC | SKB_FLAG)) { kfree_skb(skb2); skb2 = NULL; } @@ -1481,7 +1481,7 @@ int ___pskb_trim(struct sk_buff *skb, unsigned int len) int err; if (skb_cloned(skb) && - unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) + unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC|SKB_FLAG)))) return err; i = 0; @@ -1516,7 +1516,7 @@ drop_pages: if (skb_shared(frag)) { struct sk_buff *nfrag; - nfrag = skb_clone(frag, GFP_ATOMIC); + nfrag = skb_clone(frag, GFP_ATOMIC|SKB_FLAG); if (unlikely(!nfrag)) return -ENOMEM; @@ -1589,7 +1589,7 @@ unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta) if (eat > 0 || skb_cloned(skb)) { if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, - GFP_ATOMIC)) + GFP_ATOMIC|SKB_FLAG)) return NULL; } @@ -1637,7 +1637,7 @@ unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta) if (skb_shared(list)) { /* Sucks! We need to fork list. :-( */ - clone = skb_clone(list, GFP_ATOMIC); + clone = skb_clone(list, GFP_ATOMIC|SKB_FLAG); if (!clone) return NULL; insp = list->next; @@ -2313,7 +2313,7 @@ skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) to->len += len + plen; to->data_len += len + plen; - if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { + if (unlikely(skb_orphan_frags(from, GFP_ATOMIC|SKB_FLAG))) { skb_tx_error(from); return -ENOMEM; } @@ -2607,7 +2607,7 @@ EXPORT_SYMBOL(skb_split); */ static int skb_prepare_for_shift(struct sk_buff *skb) { - return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC); + return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC|SKB_FLAG); } /** @@ -3073,7 +3073,7 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, frag++; } - nskb = skb_clone(list_skb, GFP_ATOMIC); + nskb = skb_clone(list_skb, GFP_ATOMIC|SKB_FLAG); list_skb = list_skb->next; if (unlikely(!nskb)) @@ -3095,7 +3095,7 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, __skb_push(nskb, doffset); } else { nskb = __alloc_skb(hsize + doffset + headroom, - GFP_ATOMIC, skb_alloc_rx_flag(head_skb), + GFP_ATOMIC|SKB_FLAG, skb_alloc_rx_flag(head_skb), NUMA_NO_NODE); if (unlikely(!nskb)) @@ -3163,7 +3163,7 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, goto err; } - if (unlikely(skb_orphan_frags(frag_skb, GFP_ATOMIC))) + if (unlikely(skb_orphan_frags(frag_skb, GFP_ATOMIC|SKB_FLAG))) goto err; *nskb_frag = *frag; @@ -3497,7 +3497,7 @@ int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) * space, 128 bytes is fair. */ if (skb_tailroom(skb) < tailbits && - pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) + pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC|SKB_FLAG)) return -ENOMEM; /* Voila! */ @@ -3539,12 +3539,12 @@ int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) /* Fuck, we are miserable poor guys... */ if (ntail == 0) - skb2 = skb_copy(skb1, GFP_ATOMIC); + skb2 = skb_copy(skb1, GFP_ATOMIC|SKB_FLAG); else skb2 = skb_copy_expand(skb1, skb_headroom(skb1), ntail, - GFP_ATOMIC); + GFP_ATOMIC|SKB_FLAG); if (unlikely(skb2 == NULL)) return -ENOMEM; @@ -3641,7 +3641,7 @@ struct sk_buff *skb_clone_sk(struct sk_buff *skb) if (!sk || !atomic_inc_not_zero(&sk->sk_refcnt)) return NULL; - clone = skb_clone(skb, GFP_ATOMIC); + clone = skb_clone(skb, GFP_ATOMIC|SKB_FLAG); if (!clone) { sock_put(sk); return NULL; @@ -3725,9 +3725,9 @@ void __skb_tstamp_tx(struct sk_buff *orig_skb, return; if (tsonly) - skb = alloc_skb(0, GFP_ATOMIC); + skb = alloc_skb(0, GFP_ATOMIC|SKB_FLAG); else - skb = skb_clone(orig_skb, GFP_ATOMIC); + skb = skb_clone(orig_skb, GFP_ATOMIC|SKB_FLAG); if (!skb) return; @@ -4222,7 +4222,7 @@ struct sk_buff *skb_vlan_untag(struct sk_buff *skb) return skb; } - skb = skb_share_check(skb, GFP_ATOMIC); + skb = skb_share_check(skb, GFP_ATOMIC|SKB_FLAG); if (unlikely(!skb)) goto err_free; @@ -4260,7 +4260,7 @@ int skb_ensure_writable(struct sk_buff *skb, int write_len) if (!skb_cloned(skb) || skb_clone_writable(skb, write_len)) return 0; - return pskb_expand_head(skb, 0, 0, GFP_ATOMIC); + return pskb_expand_head(skb, 0, 0, GFP_ATOMIC|SKB_FLAG); } EXPORT_SYMBOL(skb_ensure_writable); -- cgit v1.2.2