From bda9461f8f90ed9ae69a57a797daed44340c8484 Mon Sep 17 00:00:00 2001 From: Namhoon Kim Date: Mon, 9 Oct 2017 11:51:16 -0400 Subject: rtas18 done --- arch/arm/kernel/irq.c | 1 + arch/arm/mm/dma-mapping.c | 20 ++- drivers/media/usb/uvc/uvc_v4l2.c | 4 +- drivers/media/usb/uvc/uvc_video.c | 37 +++--- drivers/media/v4l2-core/videobuf2-core.c | 21 ++- drivers/media/v4l2-core/videobuf2-vmalloc.c | 29 ++-- drivers/net/ethernet/freescale/fec_main.c | 17 ++- include/linux/gfp.h | 8 +- include/linux/mm.h | 6 + include/linux/vmalloc.h | 2 + include/litmus/page_dev.h | 8 +- include/litmus/trace.h | 10 ++ kernel/irq/handle.c | 19 ++- kernel/softirq.c | 8 ++ litmus/Makefile | 3 +- litmus/cache_proc.c | 4 +- litmus/fakedev0.c | 123 +++++++++++++++++ litmus/litmus.c | 44 ++++++- litmus/page_dev.c | 49 +++++-- litmus/sched_mc2.c | 198 +++++++++++++++------------- litmus/uncachedev.c | 4 +- mm/dmapool.c | 2 +- mm/migrate.c | 13 +- mm/page_alloc.c | 182 +++++++++++++++++-------- mm/slub.c | 56 +++++--- mm/vmalloc.c | 41 ++++++ net/core/dev.c | 1 + net/core/skbuff.c | 68 +++++----- 28 files changed, 706 insertions(+), 272 deletions(-) create mode 100644 litmus/fakedev0.c diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index 720b45e232f2..a9ba6e5ce317 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -45,6 +45,7 @@ #include #include +#include unsigned long irq_err_count; diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index e272fdcccc48..bd7dfb1b7ebd 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -259,7 +259,8 @@ static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gf page = alloc_pages(gfp, order); if (!page) return NULL; - + if (gfp&GFP_COLOR) + printk(KERN_INFO "__dma_alloc_buffer(): size %d, order %ld requested\n", size, order); /* * Now split the huge page and free the excess pages */ @@ -665,18 +666,24 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs); #ifdef CONFIG_SCHED_DEBUG_TRACE - if (gfp&GFP_COLOR) + if (gfp&GFP_COLOR) { printk(KERN_INFO "__dma_alloc() for usb buffer\n"); + if (gfp&GFP_CPU1) { + printk(KERN_INFO "__dma_alloc() GFP_CPU1 is set\n"); + } + } #endif if (is_coherent || nommu()) addr = __alloc_simple_buffer(dev, size, gfp, &page); else if (!(gfp & __GFP_WAIT)) addr = __alloc_from_pool(size, &page); - else if (!dev_get_cma_area(dev)) + else if (!dev_get_cma_area(dev)) { addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller, want_vaddr); - else + //printk(KERN_INFO "__alloc_remap_buffer returned %p page, size %d, color %d, bank %d, pfn %05lx\n", page, size, page_color(page), page_bank(page), page_to_pfn(page)); + } else { addr = __alloc_from_contiguous(dev, size, prot, &page, caller, want_vaddr); + } if (page) *handle = pfn_to_dma(dev, page_to_pfn(page)); @@ -694,16 +701,17 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); void *memory; + /* if ((gfp&GFP_COLOR) && (size > PAGE_SIZE*4)) { #ifdef CONFIG_SCHED_DEBUG_TRACE printk(KERN_INFO "arm_dma_alloc(): original prot %08x\n", prot); #endif - prot = pgprot_noncached(prot); + //prot = pgprot_noncached(prot); #ifdef CONFIG_SCHED_DEBUG_TRACE printk(KERN_INFO "arm_dma_alloc(): set as uncacheable prot %08x\n", prot); #endif } - + */ if (dma_alloc_from_coherent(dev, size, handle, &memory)) return memory; diff --git a/drivers/media/usb/uvc/uvc_v4l2.c b/drivers/media/usb/uvc/uvc_v4l2.c index c4b1ac6750d8..e40daf90d388 100644 --- a/drivers/media/usb/uvc/uvc_v4l2.c +++ b/drivers/media/usb/uvc/uvc_v4l2.c @@ -1437,7 +1437,9 @@ static int uvc_v4l2_mmap(struct file *file, struct vm_area_struct *vma) struct uvc_streaming *stream = handle->stream; uvc_trace(UVC_TRACE_CALLS, "uvc_v4l2_mmap\n"); - +#if 0 + printk(KERN_INFO "uvc_mmap entry point\n"); +#endif return uvc_queue_mmap(&stream->queue, vma); } diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index 454e6e83aa56..9daef917557b 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -26,6 +26,13 @@ #include "uvcvideo.h" +#define ENABLE_WORST_CASE 1 +#ifdef ENABLE_WORST_CASE +#define UVC_FLAG (GFP_COLOR|GFP_CPU1) +#else +#define UVC_FLAG (GFP_COLOR) +#endif + /* ------------------------------------------------------------------------ * UVC Controls */ @@ -167,7 +174,7 @@ static int uvc_get_video_ctrl(struct uvc_streaming *stream, query == UVC_GET_DEF) return -EIO; - data = kmalloc(size, GFP_KERNEL); + data = kmalloc(size, GFP_KERNEL|UVC_FLAG); if (data == NULL) return -ENOMEM; @@ -251,7 +258,7 @@ static int uvc_set_video_ctrl(struct uvc_streaming *stream, int ret; size = stream->dev->uvc_version >= 0x0110 ? 34 : 26; - data = kzalloc(size, GFP_KERNEL); + data = kzalloc(size, GFP_KERNEL|UVC_FLAG); if (data == NULL) return -ENOMEM; @@ -494,7 +501,7 @@ static int uvc_video_clock_init(struct uvc_streaming *stream) clock->size = 32; clock->samples = kmalloc(clock->size * sizeof(*clock->samples), - GFP_KERNEL); + GFP_KERNEL|UVC_FLAG); if (clock->samples == NULL) return -ENOMEM; @@ -1343,7 +1350,7 @@ static void uvc_video_complete(struct urb *urb) stream->decode(urb, stream, buf); - if ((ret = usb_submit_urb(urb, GFP_ATOMIC)) < 0) { + if ((ret = usb_submit_urb(urb, GFP_ATOMIC|UVC_FLAG)) < 0) { uvc_printk(KERN_ERR, "Failed to resubmit video URB (%d).\n", ret); } @@ -1406,10 +1413,10 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, #ifndef CONFIG_DMA_NONCOHERENT stream->urb_buffer[i] = usb_alloc_coherent( stream->dev->udev, stream->urb_size, - gfp_flags | __GFP_NOWARN | GFP_COLOR, &stream->urb_dma[i]); + gfp_flags | __GFP_NOWARN | UVC_FLAG, &stream->urb_dma[i]); #else stream->urb_buffer[i] = - kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN | GFP_COLOR); + kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN | UVC_FLAG; #endif if (!stream->urb_buffer[i]) { uvc_free_urb_buffers(stream); @@ -1492,14 +1499,14 @@ static int uvc_init_video_isoc(struct uvc_streaming *stream, psize = uvc_endpoint_max_bpi(stream->dev->udev, ep); size = stream->ctrl.dwMaxVideoFrameSize; - npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags); + npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags|UVC_FLAG); if (npackets == 0) return -ENOMEM; size = npackets * psize; for (i = 0; i < UVC_URBS; ++i) { - urb = usb_alloc_urb(npackets, gfp_flags); + urb = usb_alloc_urb(npackets, gfp_flags|UVC_FLAG); if (urb == NULL) { uvc_uninit_video(stream, 1); return -ENOMEM; @@ -1548,7 +1555,7 @@ static int uvc_init_video_bulk(struct uvc_streaming *stream, size = stream->ctrl.dwMaxPayloadTransferSize; stream->bulk.max_payload_size = size; - npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags); + npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags|UVC_FLAG); if (npackets == 0) return -ENOMEM; @@ -1565,7 +1572,7 @@ static int uvc_init_video_bulk(struct uvc_streaming *stream, size = 0; for (i = 0; i < UVC_URBS; ++i) { - urb = usb_alloc_urb(0, gfp_flags); + urb = usb_alloc_urb(0, gfp_flags|UVC_FLAG); if (urb == NULL) { uvc_uninit_video(stream, 1); return -ENOMEM; @@ -1654,7 +1661,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags) if (ret < 0) return ret; - ret = uvc_init_video_isoc(stream, best_ep, gfp_flags); + ret = uvc_init_video_isoc(stream, best_ep, gfp_flags|UVC_FLAG); } else { /* Bulk endpoint, proceed to URB initialization. */ ep = uvc_find_endpoint(&intf->altsetting[0], @@ -1662,7 +1669,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags) if (ep == NULL) return -EIO; - ret = uvc_init_video_bulk(stream, ep, gfp_flags); + ret = uvc_init_video_bulk(stream, ep, gfp_flags|UVC_FLAG); } if (ret < 0) @@ -1670,7 +1677,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags) /* Submit the URBs. */ for (i = 0; i < UVC_URBS; ++i) { - ret = usb_submit_urb(stream->urb[i], gfp_flags); + ret = usb_submit_urb(stream->urb[i], gfp_flags|UVC_FLAG); if (ret < 0) { uvc_printk(KERN_ERR, "Failed to submit URB %u " "(%d).\n", i, ret); @@ -1741,7 +1748,7 @@ int uvc_video_resume(struct uvc_streaming *stream, int reset) if (ret < 0) return ret; - return uvc_init_video(stream, GFP_NOIO); + return uvc_init_video(stream, GFP_NOIO|UVC_FLAG); } /* ------------------------------------------------------------------------ @@ -1892,7 +1899,7 @@ int uvc_video_enable(struct uvc_streaming *stream, int enable) if (ret < 0) goto error_commit; - ret = uvc_init_video(stream, GFP_KERNEL); + ret = uvc_init_video(stream, GFP_KERNEL|UVC_FLAG); if (ret < 0) goto error_video; diff --git a/drivers/media/v4l2-core/videobuf2-core.c b/drivers/media/v4l2-core/videobuf2-core.c index 66ada01c796c..54058877f467 100644 --- a/drivers/media/v4l2-core/videobuf2-core.c +++ b/drivers/media/v4l2-core/videobuf2-core.c @@ -30,6 +30,13 @@ #include #include +#define ENABLE_WORST_CASE 1 +#ifdef ENABLE_WORST_CASE +#define VB2_CORE_FLAG (GFP_COLOR|GFP_CPU1) +#else +#define VB2_CORE_FLAG (GFP_COLOR) +#endif + static int debug; module_param(debug, int, 0644); @@ -200,7 +207,7 @@ static int __vb2_buf_mem_alloc(struct vb2_buffer *vb) */ for (plane = 0; plane < vb->num_planes; ++plane) { unsigned long size = PAGE_ALIGN(q->plane_sizes[plane]); - + printk(KERN_INFO "__vb2_buf_mem_alloc(): size %ld, func %pF GFP_COLOR? %d\n", size, vb->vb2_queue->mem_ops->alloc, q->gfp_flags&GFP_COLOR); mem_priv = call_ptr_memop(vb, alloc, q->alloc_ctx[plane], size, dma_dir, q->gfp_flags); if (IS_ERR_OR_NULL(mem_priv)) @@ -352,7 +359,7 @@ static int __vb2_queue_alloc(struct vb2_queue *q, enum v4l2_memory memory, for (buffer = 0; buffer < num_buffers; ++buffer) { /* Allocate videobuf buffer structures */ - vb = kzalloc(q->buf_struct_size, GFP_KERNEL); + vb = kzalloc(q->buf_struct_size, GFP_KERNEL|VB2_CORE_FLAG); if (!vb) { dprintk(1, "memory alloc for buffer struct failed\n"); break; @@ -402,7 +409,8 @@ static int __vb2_queue_alloc(struct vb2_queue *q, enum v4l2_memory memory, dprintk(1, "allocated %d buffers, %d plane(s) each\n", buffer, num_planes); - + printk(KERN_INFO "allocated %d buffers, %d plane(s) each\n", + buffer, num_planes); return buffer; } @@ -2237,6 +2245,7 @@ static int vb2_internal_streamon(struct vb2_queue *q, enum v4l2_buf_type type) * Tell driver to start streaming provided sufficient buffers * are available. */ +printk(KERN_INFO "vb2_internal_streamon()\n"); if (q->queued_count >= q->min_buffers_needed) { ret = vb2_start_streaming(q); if (ret) { @@ -2525,7 +2534,7 @@ int vb2_mmap(struct vb2_queue *q, struct vm_area_struct *vma) "MMAP invalid, as it would overflow buffer length\n"); return -EINVAL; } - +printk(KERN_INFO "memop mmap %pF\n", vb->vb2_queue->mem_ops->mmap); mutex_lock(&q->mmap_lock); ret = call_memop(vb, mmap, vb->planes[plane].mem_priv, vma); mutex_unlock(&q->mmap_lock); @@ -2830,7 +2839,7 @@ static int __vb2_init_fileio(struct vb2_queue *q, int read) (read) ? "read" : "write", count, q->fileio_read_once, q->fileio_write_immediately); - fileio = kzalloc(sizeof(struct vb2_fileio_data), GFP_KERNEL); + fileio = kzalloc(sizeof(struct vb2_fileio_data), GFP_KERNEL|VB2_CORE_FLAG); if (fileio == NULL) return -ENOMEM; @@ -3223,7 +3232,7 @@ int vb2_thread_start(struct vb2_queue *q, vb2_thread_fnc fnc, void *priv, if (WARN_ON(q->fileio)) return -EBUSY; - threadio = kzalloc(sizeof(*threadio), GFP_KERNEL); + threadio = kzalloc(sizeof(*threadio), GFP_KERNEL|VB2_CORE_FLAG); if (threadio == NULL) return -ENOMEM; threadio->fnc = fnc; diff --git a/drivers/media/v4l2-core/videobuf2-vmalloc.c b/drivers/media/v4l2-core/videobuf2-vmalloc.c index 657ab302a5cf..813cc718c116 100644 --- a/drivers/media/v4l2-core/videobuf2-vmalloc.c +++ b/drivers/media/v4l2-core/videobuf2-vmalloc.c @@ -21,6 +21,14 @@ #include #include + +#define ENABLE_WORST_CASE 1 +#ifdef ENABLE_WORST_CASE +#define VB2_FLAG (GFP_COLOR|GFP_CPU1) +#else +#define VB2_FLAG (GFP_COLOR) +#endif + struct vb2_vmalloc_buf { void *vaddr; struct page **pages; @@ -39,13 +47,18 @@ static void *vb2_vmalloc_alloc(void *alloc_ctx, unsigned long size, enum dma_data_direction dma_dir, gfp_t gfp_flags) { struct vb2_vmalloc_buf *buf; - - buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags); +/* video buffer allocation */ +printk(KERN_INFO "vb2_vmalloc_alloc(): size %ld requested\n", size); + buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags | VB2_FLAG); if (!buf) return NULL; buf->size = size; - buf->vaddr = vmalloc_user(buf->size); +#ifdef ENABLE_WORST_CASE + buf->vaddr = vmalloc_color_user_cpu1(buf->size); +#else + buf->vaddr = vmalloc_color_user(buf->size); +#endif buf->dma_dir = dma_dir; buf->handler.refcount = &buf->refcount; buf->handler.put = vb2_vmalloc_put; @@ -81,7 +94,7 @@ static void *vb2_vmalloc_get_userptr(void *alloc_ctx, unsigned long vaddr, struct vm_area_struct *vma; dma_addr_t physp; - buf = kzalloc(sizeof(*buf), GFP_KERNEL); + buf = kzalloc(sizeof(*buf), GFP_KERNEL | VB2_FLAG); if (!buf) return NULL; @@ -103,7 +116,7 @@ static void *vb2_vmalloc_get_userptr(void *alloc_ctx, unsigned long vaddr, last = (vaddr + size - 1) >> PAGE_SHIFT; buf->n_pages = last - first + 1; buf->pages = kzalloc(buf->n_pages * sizeof(struct page *), - GFP_KERNEL); + GFP_KERNEL | VB2_FLAG); if (!buf->pages) goto fail_pages_array_alloc; @@ -233,12 +246,12 @@ static int vb2_vmalloc_dmabuf_ops_attach(struct dma_buf *dbuf, struct device *de int ret; int i; - attach = kzalloc(sizeof(*attach), GFP_KERNEL); + attach = kzalloc(sizeof(*attach), GFP_KERNEL | VB2_FLAG); if (!attach) return -ENOMEM; sgt = &attach->sgt; - ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL); + ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL | VB2_FLAG); if (ret) { kfree(attach); return ret; @@ -429,7 +442,7 @@ static void *vb2_vmalloc_attach_dmabuf(void *alloc_ctx, struct dma_buf *dbuf, if (dbuf->size < size) return ERR_PTR(-EFAULT); - buf = kzalloc(sizeof(*buf), GFP_KERNEL); + buf = kzalloc(sizeof(*buf), GFP_KERNEL | VB2_FLAG); if (!buf) return ERR_PTR(-ENOMEM); diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index ad3759e8a749..2f7aab0b60cf 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -65,6 +65,13 @@ #include "fec.h" +#define ENABLE_WORST_CASE 1 +#ifdef ENABLE_WORST_CASE +#define FEC_FLAG (GFP_COLOR|GFP_CPU1) +#else +#define FEC_FLAG (0) +#endif + static void set_multicast_list(struct net_device *ndev); static void fec_enet_itr_coal_init(struct net_device *ndev); @@ -2628,7 +2635,7 @@ static int fec_enet_alloc_queue(struct net_device *ndev) struct fec_enet_priv_tx_q *txq; for (i = 0; i < fep->num_tx_queues; i++) { - txq = kzalloc(sizeof(*txq), GFP_KERNEL); + txq = kzalloc(sizeof(*txq), GFP_KERNEL|FEC_FLAG); if (!txq) { ret = -ENOMEM; goto alloc_failed; @@ -2645,7 +2652,7 @@ static int fec_enet_alloc_queue(struct net_device *ndev) txq->tso_hdrs = dma_alloc_coherent(NULL, txq->tx_ring_size * TSO_HEADER_SIZE, &txq->tso_hdrs_dma, - GFP_KERNEL); + GFP_KERNEL|FEC_FLAG); if (!txq->tso_hdrs) { ret = -ENOMEM; goto alloc_failed; @@ -2654,7 +2661,7 @@ static int fec_enet_alloc_queue(struct net_device *ndev) for (i = 0; i < fep->num_rx_queues; i++) { fep->rx_queue[i] = kzalloc(sizeof(*fep->rx_queue[i]), - GFP_KERNEL); + GFP_KERNEL|FEC_FLAG); if (!fep->rx_queue[i]) { ret = -ENOMEM; goto alloc_failed; @@ -2723,7 +2730,7 @@ fec_enet_alloc_txq_buffers(struct net_device *ndev, unsigned int queue) txq = fep->tx_queue[queue]; bdp = txq->tx_bd_base; for (i = 0; i < txq->tx_ring_size; i++) { - txq->tx_bounce[i] = kmalloc(FEC_ENET_TX_FRSIZE, GFP_KERNEL); + txq->tx_bounce[i] = kmalloc(FEC_ENET_TX_FRSIZE, GFP_KERNEL|FEC_FLAG); if (!txq->tx_bounce[i]) goto err_alloc; @@ -3037,7 +3044,7 @@ static int fec_enet_init(struct net_device *ndev) /* Allocate memory for buffer descriptors. */ cbd_base = dma_alloc_coherent(NULL, bd_size, &bd_dma, - GFP_KERNEL); + GFP_KERNEL|FEC_FLAG); if (!cbd_base) { return -ENOMEM; } diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 10a4601c558b..69bdfb464e5c 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -36,6 +36,7 @@ struct vm_area_struct; #define ___GFP_OTHER_NODE 0x800000u #define ___GFP_WRITE 0x1000000u #define ___GFP_COLOR 0x2000000u +#define ___GFP_CPU1 0x4000000u /* If the above are modified, __GFP_BITS_SHIFT may need updating */ /* @@ -96,6 +97,7 @@ struct vm_area_struct; #define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) /* On behalf of other node */ #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) /* Allocator intends to dirty page */ #define __GFP_COLOR ((__force gfp_t)___GFP_COLOR) /* Colored page request */ +#define __GFP_CPU1 ((__force gfp_t)___GFP_CPU1) /* use cpu1 bank */ /* * This may seem redundant, but it's a way of annotating false positives vs. @@ -103,7 +105,7 @@ struct vm_area_struct; */ #define __GFP_NOTRACK_FALSE_POSITIVE (__GFP_NOTRACK) -#define __GFP_BITS_SHIFT 26 /* Room for N __GFP_FOO bits */ +#define __GFP_BITS_SHIFT 27 /* Room for N __GFP_FOO bits */ #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /* This equals 0, but use constants in case they ever change */ @@ -129,7 +131,7 @@ struct vm_area_struct; /* Control page allocator reclaim behavior */ #define GFP_RECLAIM_MASK (__GFP_WAIT|__GFP_HIGH|__GFP_IO|__GFP_FS|\ __GFP_NOWARN|__GFP_REPEAT|__GFP_NOFAIL|\ - __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC) + __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|__GFP_COLOR|__GFP_CPU1) /* Control slab gfp mask during early boot */ #define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_WAIT|__GFP_IO|__GFP_FS)) @@ -151,6 +153,8 @@ struct vm_area_struct; /* Colored page requests */ #define GFP_COLOR __GFP_COLOR +#define GFP_CPU1 __GFP_CPU1 + /* Convert GFP flags to their corresponding migrate type */ static inline int gfpflags_to_migratetype(const gfp_t gfp_flags) { diff --git a/include/linux/mm.h b/include/linux/mm.h index 0755b9fd03a7..6008a33bfeac 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -172,6 +172,12 @@ extern unsigned int kobjsize(const void *objp); # define VM_MPX VM_ARCH_2 #endif +#if defined(CONFIG_ARM) +/* MPX specific bounds table or bounds directory */ +# define VM_DONOTMOVE VM_ARCH_2 +#endif + + #ifndef VM_GROWSUP # define VM_GROWSUP VM_NONE #endif diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index c82054890d77..2323fbe02aa9 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -70,6 +70,8 @@ extern void *vmalloc(unsigned long size); extern void *vmalloc_color(unsigned long size); extern void *vzalloc(unsigned long size); extern void *vmalloc_user(unsigned long size); +extern void *vmalloc_color_user(unsigned long size); +extern void *vmalloc_color_user_cpu1(unsigned long size); extern void *vmalloc_node(unsigned long size, int node); extern void *vzalloc_node(unsigned long size, int node); extern void *vmalloc_exec(unsigned long size); diff --git a/include/litmus/page_dev.h b/include/litmus/page_dev.h index f1791469cba1..37953928e6f3 100644 --- a/include/litmus/page_dev.h +++ b/include/litmus/page_dev.h @@ -1,7 +1,5 @@ /* * page_dev.h - Implementation of the page coloring for cache and bank partition. - * The file will keep a pool of colored pages. MMU can allocate pages with - * specific color or bank number. * Author: Namhoon Kim (namhoonk@cs.unc.edu) */ @@ -16,17 +14,15 @@ #include #include #include -#include -#include -#include -#include #include #include int llc_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); int dram_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); int bank_to_partition(unsigned int bank); +int get_area_index(int cpu); +int is_in_correct_bank(struct page* page, int cpu); int is_in_llc_partition(struct page* page, int cpu); #endif /* _LITMUS_PAGE_DEV_H */ \ No newline at end of file diff --git a/include/litmus/trace.h b/include/litmus/trace.h index 24ca412e1184..7d36a119d045 100644 --- a/include/litmus/trace.h +++ b/include/litmus/trace.h @@ -143,6 +143,16 @@ feather_callback void save_cpu_task_latency(unsigned long event, unsigned long w #define TS_ISR_START CPU_TIMESTAMP_CUR(192) #define TS_ISR_END CPU_TIMESTAMP_CUR(193) +/* For RTAS2018 */ +#define TS_NET_RX_HARDIRQ_START CPU_TIMESTAMP_CUR(194) +#define TS_NET_RX_HARDIRQ_END CPU_TIMESTAMP_CUR(195) + +#define TS_NET_RX_SOFTIRQ_START CPU_TIMESTAMP_CUR(196) +#define TS_NET_RX_SOFTIRQ_END CPU_TIMESTAMP_CUR(197) + +#define TS_UVC_IRQ_START CPU_TIMESTAMP_CUR(198) +#define TS_UVC_IRQ_END CPU_TIMESTAMP_CUR(199) + #define TS_RELEASE_LATENCY(when) CPU_LTIMESTAMP(208, &(when)) #define TS_RELEASE_LATENCY_A(when) CPU_LTIMESTAMP(209, &(when)) #define TS_RELEASE_LATENCY_B(when) CPU_LTIMESTAMP(210, &(when)) diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index 635480270858..8da6ba6a3ae6 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c @@ -18,6 +18,9 @@ #include +#include +#include + #include "internals.h" /** @@ -138,11 +141,23 @@ handle_irq_event_percpu(struct irq_desc *desc, struct irqaction *action) do { irqreturn_t res; - +/* + if (irq == 282) + TS_UVC_IRQ_START; + if (irq == 284) { + TS_NET_RX_HARDIRQ_START; + } +*/ trace_irq_handler_entry(irq, action); res = action->handler(irq, action->dev_id); trace_irq_handler_exit(irq, action, res); - +/* + if (irq == 282) + TS_UVC_IRQ_END; + if (irq == 284) { + TS_NET_RX_HARDIRQ_END; + } +*/ if (WARN_ONCE(!irqs_disabled(),"irq %u handler %pF enabled interrupts\n", irq, action->handler)) local_irq_disable(); diff --git a/kernel/softirq.c b/kernel/softirq.c index 99fe8b877e53..5c694353763b 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -27,6 +27,9 @@ #include #include +/* for measuring NET_RX bottom half */ +#include + #define CREATE_TRACE_POINTS #include @@ -273,7 +276,12 @@ restart: kstat_incr_softirqs_this_cpu(vec_nr); trace_softirq_entry(vec_nr); +// if (vec_nr == 3) +// TS_NET_RX_SOFTIRQ_START; +// net_rx_action() is called here h->action(h); +// if (vec_nr == 3) +// TS_NET_RX_SOFTIRQ_END; trace_softirq_exit(vec_nr); if (unlikely(prev_count != preempt_count())) { pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", diff --git a/litmus/Makefile b/litmus/Makefile index 29ae4b04f046..ccd532d81b78 100644 --- a/litmus/Makefile +++ b/litmus/Makefile @@ -30,7 +30,8 @@ obj-y = sched_plugin.o litmus.o \ color_shm.o \ replicate_lib.o \ cache_proc.o \ - page_dev.o + page_dev.o \ + fakedev0.o obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o diff --git a/litmus/cache_proc.c b/litmus/cache_proc.c index 2d90454ad5cc..102feaf5c9e6 100644 --- a/litmus/cache_proc.c +++ b/litmus/cache_proc.c @@ -10,7 +10,10 @@ #include #include #include +#include +#include +#include #include #include #include @@ -19,7 +22,6 @@ #include #include - #define UNLOCK_ALL 0x00000000 /* allocation in any way */ #define LOCK_ALL (~UNLOCK_ALL) #define MAX_NR_WAYS 16 diff --git a/litmus/fakedev0.c b/litmus/fakedev0.c new file mode 100644 index 000000000000..0b8909e77777 --- /dev/null +++ b/litmus/fakedev0.c @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* device for allocating pages not cached by the CPU */ + +#define FAKEDEV0_NAME "litmus/fakedev0" + +#define NUM_BANKS 8 +#define BANK_MASK 0x38000000 +#define BANK_SHIFT 27 + +#define NUM_COLORS 16 +#define CACHE_MASK 0x0000f000 +#define CACHE_SHIFT 12 + +/* Decoding page color, 0~15 */ +static inline unsigned int page_color(struct page *page) +{ + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT); +} + +/* Decoding page bank number, 0~7 */ +static inline unsigned int page_bank(struct page *page) +{ + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT); +} + +void litmus_fakedev0_vm_open(struct vm_area_struct *vma) +{ +} + +void litmus_fakedev0_vm_close(struct vm_area_struct *vma) +{ +} + +int litmus_fakedev0_vm_fault(struct vm_area_struct* vma, + struct vm_fault* vmf) +{ + /* modeled after SG DMA video4linux, but without DMA. */ + /* (see drivers/media/video/videobuf-dma-sg.c) */ + struct page *page; + + page = alloc_page(GFP_USER|GFP_COLOR|GFP_CPU1); + if (!page) + return VM_FAULT_OOM; + + clear_user_highpage(page, (unsigned long)vmf->virtual_address); + vmf->page = page; + + return 0; +} + +static struct vm_operations_struct litmus_fakedev0_vm_ops = { + .open = litmus_fakedev0_vm_open, + .close = litmus_fakedev0_vm_close, + .fault = litmus_fakedev0_vm_fault, +}; + +static int litmus_fakedev0_mmap(struct file* filp, struct vm_area_struct* vma) +{ + /* first make sure mapper knows what he's doing */ + + /* you can only map the "first" page */ + if (vma->vm_pgoff != 0) + return -EINVAL; + + /* you can't share it with anyone */ + if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED)) + return -EINVAL; + + /* cannot be expanded, and is not a "normal" page. */ + vma->vm_flags |= (VM_DONTEXPAND|VM_DONOTMOVE); + + /* noncached pages are not explicitly locked in memory (for now). */ + //vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + vma->vm_ops = &litmus_fakedev0_vm_ops; + + return 0; +} + +static struct file_operations litmus_fakedev0_fops = { + .owner = THIS_MODULE, + .mmap = litmus_fakedev0_mmap, +}; + +static struct miscdevice litmus_fakedev0_dev = { + .name = FAKEDEV0_NAME, + .minor = MISC_DYNAMIC_MINOR, + .fops = &litmus_fakedev0_fops, + /* pages are not locked, so there is no reason why + anyone cannot allocate an fakedev0 pages */ + .mode = (S_IRUGO | S_IWUGO), +}; + +static int __init init_litmus_fakedev0_dev(void) +{ + int err; + + printk("Initializing LITMUS^RT fakedev0 device.\n"); + err = misc_register(&litmus_fakedev0_dev); + if (err) + printk("Could not allocate %s device (%d).\n", FAKEDEV0_NAME, err); + return err; +} + +static void __exit exit_litmus_fakedev0_dev(void) +{ + misc_deregister(&litmus_fakedev0_dev); +} + +module_init(init_litmus_fakedev0_dev); +module_exit(exit_litmus_fakedev0_dev); + diff --git a/litmus/litmus.c b/litmus/litmus.c index 1105408e405a..ec9379979e1a 100644 --- a/litmus/litmus.c +++ b/litmus/litmus.c @@ -28,6 +28,7 @@ #include #include #include +#include #ifdef CONFIG_SCHED_CPU_AFFINITY #include @@ -350,8 +351,15 @@ extern struct page *new_alloc_page(struct page *page, unsigned long node, int ** static struct page *alloc_colored_page(struct page *page, unsigned long node, int **result) { struct page *newpage; + gfp_t gfp_mask; - newpage = alloc_pages(GFP_HIGHUSER_MOVABLE|GFP_COLOR, 0); + gfp_mask = GFP_HIGHUSER_MOVABLE; + if (node != 8) + gfp_mask |= GFP_COLOR; + if (node == 9) + gfp_mask |= GFP_CPU1; + + newpage = alloc_pages(gfp_mask, 0); return newpage; } @@ -378,6 +386,7 @@ asmlinkage long sys_set_page_color(int cpu) LIST_HEAD(pagelist); LIST_HEAD(task_shared_pagelist); + LIST_HEAD(fakedev_pagelist); migrate_prep(); @@ -396,7 +405,11 @@ asmlinkage long sys_set_page_color(int cpu) unsigned int num_pages = 0, i; struct page *old_page = NULL; int pages_in_vma = 0; + int fakedev_pages = 0; + if (vma_itr->vm_flags & VM_DONOTMOVE) { + fakedev_pages = 1; + } num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE; /* Traverse all pages in vm_area_struct */ for (i = 0; i < num_pages; i++) { @@ -412,7 +425,13 @@ asmlinkage long sys_set_page_color(int cpu) put_page(old_page); continue; } - + /* + if (PageDirty(old_page)) { + TRACE("Dirty Page!\n"); + put_page(old_page); + continue; + } + */ TRACE_TASK(current, "addr: %08x, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-"); pages_in_vma++; @@ -460,12 +479,18 @@ asmlinkage long sys_set_page_color(int cpu) else { ret = isolate_lru_page(old_page); if (!ret) { - list_add_tail(&old_page->lru, &pagelist); + if (fakedev_pages == 0) + list_add_tail(&old_page->lru, &pagelist); + else + list_add_tail(&old_page->lru, &fakedev_pagelist); + inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page)); nr_pages++; - } else { + } else if (!is_in_correct_bank(old_page, cpu)) { TRACE_TASK(current, "isolate_lru_page for a private page failed\n"); nr_failed++; + } else { + TRACE_TASK(current, "page is already in the correct bank\n"); } put_page(old_page); } @@ -491,6 +516,16 @@ asmlinkage long sys_set_page_color(int cpu) } } + /* Migrate fakedev pages */ + if (!list_empty(&fakedev_pagelist)) { + ret = migrate_pages(&fakedev_pagelist, alloc_colored_page, NULL, 9, MIGRATE_SYNC, MR_SYSCALL); + TRACE_TASK(current, "%ld pages not migrated.\n", ret); + nr_not_migrated = ret; + if (ret) { + putback_movable_pages(&fakedev_pagelist); + } + } + /* Replicate shared pages */ if (!list_empty(&task_shared_pagelist)) { ret = replicate_pages(&task_shared_pagelist, alloc_colored_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL); @@ -569,6 +604,7 @@ asmlinkage long sys_test_call(unsigned int param) } TRACE_TASK(current, "addr: %08x, phy: %08x, color: %d, bank: %d, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s mapping: %p\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_phys(old_page), page_color(old_page), page_bank(old_page), page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-", &(old_page->mapping)); + printk(KERN_INFO "addr: %08x, phy: %08x, color: %d, bank: %d, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s mapping: %p\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_phys(old_page), page_color(old_page), page_bank(old_page), page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-", &(old_page->mapping)); put_page(old_page); } vma_itr = vma_itr->vm_next; diff --git a/litmus/page_dev.c b/litmus/page_dev.c index ea5d5f5cb36d..cde8e52d9111 100644 --- a/litmus/page_dev.c +++ b/litmus/page_dev.c @@ -44,15 +44,6 @@ unsigned int dram_partition[NR_DRAM_PARTITIONS] = { 0x00000080, 0x0000000f, }; -/* -unsigned int dram_partition[NR_DRAM_PARTITIONS] = { - 0x00000001, - 0x00000002, - 0x00000004, - 0x00000008, - 0x000000f0, -}; -*/ /* Decoding page color, 0~15 */ static inline unsigned int page_color(struct page *page) @@ -79,6 +70,35 @@ int bank_to_partition(unsigned int bank) return -EINVAL; } +int get_area_index(int cpu) +{ + int index = 0x10, area_index = 0; + + while (index < 0x100) { + if (dram_partition[cpu]&index) + break; + index = index << 1; + area_index++; + } + + return area_index; +} + +/* use this function ONLY for Lv.A/B pages */ +int is_in_correct_bank(struct page* page, int cpu) +{ + int bank; + unsigned int page_bank_bit; + + bank = page_bank(page); + page_bank_bit = 1 << bank; + + if (cpu == -1 || cpu == NR_CPUS) + return (page_bank_bit & dram_partition[NR_CPUS]); + else + return (page_bank_bit & dram_partition[cpu]); +} + int is_in_llc_partition(struct page* page, int cpu) { int color; @@ -87,8 +107,8 @@ int is_in_llc_partition(struct page* page, int cpu) color = page_color(page); page_color_bit = 1 << color; - if (cpu == NR_CPUS) - return (page_color_bit&llc_partition[cpu*2]); + if (cpu == -1 || cpu == NR_CPUS) + return (page_color_bit & llc_partition[8]); else return (page_color_bit & (llc_partition[cpu*2] | llc_partition[cpu*2+1])); } @@ -117,12 +137,14 @@ int slabtest_handler(struct ctl_table *table, int write, void __user *buffer, si int idx; int n_data = buf_size/sizeof(int); - testbuffer = kmalloc(sizeof(int*)*buf_num, GFP_KERNEL|GFP_COLOR); + printk(KERN_INFO "-------SLABTEST on CPU%d with %d buffer size\n", raw_smp_processor_id(), buf_size); + + testbuffer = kmalloc(sizeof(int*)*buf_num, GFP_KERNEL|GFP_COLOR|GFP_CPU1); for (idx=0; idxres_info.client.reservation; client = &tinfo->res_info.client; @@ -169,14 +169,15 @@ static void task_arrives(struct mc2_cpu_state *state, struct task_struct *tsk) res->ops->client_arrives(res, client); TRACE_TASK(tsk, "Client arrives at %llu\n", litmus_clock()); - +/* if (lv != NUM_CRIT_LEVELS) { struct crit_entry *ce; ce = &state->crit_entries[lv]; - /* if the currrent task is a ghost job, remove it */ + // if the currrent task is a ghost job, remove it if (ce->running == tsk) ce->running = NULL; } +*/ } /* get_lowest_prio_cpu - return the lowest priority cpu @@ -190,6 +191,9 @@ static int get_lowest_prio_cpu(lt_t priority) int cpu, ret = NO_CPU; lt_t latest_deadline = 0; + if (priority == LITMUS_NO_PRIORITY) + return ret; + ce = &_lowest_prio_cpu.cpu_entries[local_cpu_state()->cpu]; if (!ce->will_schedule && !ce->scheduled) { TRACE("CPU %d (local) is the lowest!\n", ce->cpu); @@ -202,10 +206,12 @@ static int get_lowest_prio_cpu(lt_t priority) ce = &_lowest_prio_cpu.cpu_entries[cpu]; /* If a CPU will call schedule() in the near future, we don't return that CPU. */ + /* TRACE("CPU %d will_schedule=%d, scheduled=(%s/%d:%d)\n", cpu, ce->will_schedule, ce->scheduled ? (ce->scheduled)->comm : "null", ce->scheduled ? (ce->scheduled)->pid : 0, ce->scheduled ? (ce->scheduled)->rt_param.job_params.job_no : 0); + */ if (!ce->will_schedule) { if (!ce->scheduled) { /* Idle cpu, return this. */ @@ -242,6 +248,9 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state) lt_t update, now; struct next_timer_event *event, *next; int reschedule[NR_CPUS]; + unsigned long flags; + + local_irq_save(flags); for (cpus = 0; cpustimer_armed_on == NO_CPU) { struct reservation *res = gmp_find_by_id(&_global_env, event->id); int cpu = get_lowest_prio_cpu(res?res->priority:0); - TRACE("GLOBAL EVENT PASSED!! poking CPU %d to reschedule\n", cpu); + //TRACE("GLOBAL EVENT PASSED!! poking CPU %d to reschedule\n", cpu); list_del(&event->list); kfree(event); if (cpu != NO_CPU) { _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; - if (cpu == local_cpu_state()->cpu) - litmus_reschedule_local(); - else - reschedule[cpu] = 1; + reschedule[cpu] = 1; } } } else if (event->next_update < update && (event->timer_armed_on == NO_CPU || event->timer_armed_on == state->cpu)) { @@ -289,6 +295,7 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state) /* Must drop state lock before calling into hrtimer_start(), which * may raise a softirq, which in turn may wake ksoftirqd. */ raw_spin_unlock(&_global_env.lock); + local_irq_restore(flags); raw_spin_unlock(&state->lock); if (update <= now || reschedule[state->cpu]) { @@ -325,9 +332,7 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state) */ TRACE("mc2_update_timer for remote CPU %d (update=%llu, " "active:%d, set:%llu)\n", - state->cpu, - update, - hrtimer_active(&state->timer), + state->cpu, update, hrtimer_active(&state->timer), ktime_to_ns(hrtimer_get_expires(&state->timer))); if (!hrtimer_active(&state->timer) || ktime_to_ns(hrtimer_get_expires(&state->timer)) > update) { @@ -336,17 +341,19 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state) state->cpu, hrtimer_active(&state->timer), ktime_to_ns(hrtimer_get_expires(&state->timer))); - raw_spin_lock(&state->lock); - preempt_if_preemptable(state->scheduled, state->cpu); - raw_spin_unlock(&state->lock); - reschedule[state->cpu] = 0; + //raw_spin_lock(&state->lock); + //preempt_if_preemptable(state->scheduled, state->cpu); + //raw_spin_unlock(&state->lock); + //reschedule[state->cpu] = 0; } } + for (cpus = 0; cpuscpu && update > now) - litmus_reschedule_local(); + ; //litmus_reschedule_local(); else reschedule[cpu] = 1; } } raw_spin_unlock(&_global_env.lock); - raw_spin_unlock_irqrestore(&state->lock, flags); TS_ISR_END; @@ -447,7 +452,6 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer) } } - return restart; } @@ -470,7 +474,7 @@ static long mc2_complete_job(void) unsigned long flags; enum crit_level lv; - preempt_disable(); + //preempt_disable(); local_irq_save(flags); tinfo = get_mc2_state(current); @@ -504,7 +508,7 @@ static long mc2_complete_job(void) raw_spin_unlock(&state->lock); local_irq_restore(flags); - preempt_enable(); + //preempt_enable(); } sched_trace_task_completion(current, 0); @@ -568,7 +572,6 @@ struct task_struct* mc2_global_dispatch(struct mc2_cpu_state* state) { struct reservation *res, *next; struct task_struct *tsk = NULL; - enum crit_level lv; lt_t time_slice; @@ -578,31 +581,20 @@ struct task_struct* mc2_global_dispatch(struct mc2_cpu_state* state) tsk = res->ops->dispatch_client(res, &time_slice); if (likely(tsk)) { lv = get_task_crit_level(tsk); - if (lv == NUM_CRIT_LEVELS) { + if (lv != CRIT_LEVEL_C) + BUG(); #if BUDGET_ENFORCEMENT_AT_C - gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN); -#endif - res->event_added = 1; - res->blocked_by_ghost = 0; - res->is_ghost = NO_CPU; - res->scheduled_on = state->cpu; - return tsk; - } else if (lv == CRIT_LEVEL_C) { -#if BUDGET_ENFORCEMENT_AT_C - gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN); + gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN); #endif - res->event_added = 1; - res->blocked_by_ghost = 0; - res->is_ghost = NO_CPU; - res->scheduled_on = state->cpu; - return tsk; - } else { - BUG(); - } + res->event_added = 1; + res->blocked_by_ghost = 0; + res->is_ghost = NO_CPU; + res->scheduled_on = state->cpu; + return tsk; } } } - + return NULL; } @@ -621,7 +613,7 @@ static inline void post_schedule(struct task_struct *next, int cpu) { enum crit_level lev; if ((!next) || !is_realtime(next)) { - do_partition(NUM_CRIT_LEVELS, -1); + //do_partition(NUM_CRIT_LEVELS, -1); return; } @@ -646,15 +638,15 @@ static inline void post_schedule(struct task_struct *next, int cpu) */ static struct task_struct* mc2_schedule(struct task_struct * prev) { - int np, blocks, exists, preempt, to_schedule; + int np, blocks, exists, cpu; //preempt, to_schedule; /* next == NULL means "schedule background work". */ - lt_t now; + lt_t now = litmus_clock(); struct mc2_cpu_state *state = local_cpu_state(); - pre_schedule(prev, state->cpu); - raw_spin_lock(&state->lock); + pre_schedule(prev, state->cpu); + if (state->scheduled && state->scheduled != prev) printk(KERN_ALERT "BUG1!!!!!!!! %s %s\n", state->scheduled ? (state->scheduled)->comm : "null", prev ? (prev)->comm : "null"); if (state->scheduled && !is_realtime(prev)) @@ -664,16 +656,9 @@ static struct task_struct* mc2_schedule(struct task_struct * prev) exists = state->scheduled != NULL; blocks = exists && !is_current_running(); np = exists && is_np(state->scheduled); - - raw_spin_lock(&_global_env.lock); - preempt = resched_cpu[state->cpu]; - resched_cpu[state->cpu] = 0; - raw_spin_unlock(&_global_env.lock); /* update time */ state->sup_env.will_schedule = true; - - now = litmus_clock(); sup_update_time(&state->sup_env, now); if (is_realtime(current) && blocks) { @@ -690,7 +675,8 @@ static struct task_struct* mc2_schedule(struct task_struct * prev) if (!state->scheduled) { raw_spin_lock(&_global_env.lock); - to_schedule = gmp_update_time(&_global_env, now); + if (is_realtime(prev)) + gmp_update_time(&_global_env, now); state->scheduled = mc2_global_dispatch(state); _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false; update_cpu_prio(state); @@ -711,18 +697,18 @@ static struct task_struct* mc2_schedule(struct task_struct * prev) /* NOTE: drops state->lock */ mc2_update_timer_and_unlock(state); + raw_spin_lock(&state->lock); if (prev != state->scheduled && is_realtime(prev)) { struct mc2_task_state* tinfo = get_mc2_state(prev); struct reservation* res = tinfo->res_info.client.reservation; - TRACE_TASK(prev, "PREV JOB scheduled_on = P%d\n", res->scheduled_on); res->scheduled_on = NO_CPU; - TRACE_TASK(prev, "descheduled.\n"); + TRACE_TASK(prev, "descheduled at %llu.\n", litmus_clock()); /* if prev is preempted and a global task, find the lowest cpu and reschedule */ if (tinfo->has_departed == false && get_task_crit_level(prev) == CRIT_LEVEL_C) { int cpu; raw_spin_lock(&_global_env.lock); - cpu = get_lowest_prio_cpu(res?res->priority:0); - TRACE("LEVEL-C TASK PREEMPTED!! poking CPU %d to reschedule\n", cpu); + cpu = get_lowest_prio_cpu(res?res->priority:LITMUS_NO_PRIORITY); + //TRACE("LEVEL-C TASK PREEMPTED!! poking CPU %d to reschedule\n", cpu); if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) { _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; resched_cpu[cpu] = 1; @@ -730,7 +716,8 @@ static struct task_struct* mc2_schedule(struct task_struct * prev) raw_spin_unlock(&_global_env.lock); } } - + +/* if (to_schedule != 0) { raw_spin_lock(&_global_env.lock); while (to_schedule--) { @@ -742,13 +729,15 @@ static struct task_struct* mc2_schedule(struct task_struct * prev) } raw_spin_unlock(&_global_env.lock); } +*/ + post_schedule(state->scheduled, state->cpu); + + raw_spin_unlock(&state->lock); if (state->scheduled) { TRACE_TASK(state->scheduled, "scheduled.\n"); } - post_schedule(state->scheduled, state->cpu); - return state->scheduled; } @@ -758,38 +747,40 @@ static struct task_struct* mc2_schedule(struct task_struct * prev) static void mc2_task_resume(struct task_struct *tsk) { unsigned long flags; - struct mc2_task_state* tinfo = get_mc2_state(tsk); + struct mc2_task_state* tinfo; struct mc2_cpu_state *state; TRACE_TASK(tsk, "thread wakes up at %llu\n", litmus_clock()); - local_irq_save(flags); + preempt_disable(); + tinfo = get_mc2_state(tsk); if (tinfo->cpu != -1) state = cpu_state_for(tinfo->cpu); else state = local_cpu_state(); + preempt_enable(); /* Requeue only if self-suspension was already processed. */ if (tinfo->has_departed) { - /* We don't want to consider jobs before synchronous releases */ - if (tsk_rt(tsk)->job_params.job_no > 5) { - switch(get_task_crit_level(tsk)) { - case CRIT_LEVEL_A: - TS_RELEASE_LATENCY_A(get_release(tsk)); - break; - case CRIT_LEVEL_B: - TS_RELEASE_LATENCY_B(get_release(tsk)); - break; - case CRIT_LEVEL_C: - TS_RELEASE_LATENCY_C(get_release(tsk)); - break; - default: - break; - } +#ifdef CONFIG_SCHED_OVERHEAD_TRACE + switch(get_task_crit_level(tsk)) { + case CRIT_LEVEL_A: + TS_RELEASE_LATENCY_A(get_release(tsk)); + break; + case CRIT_LEVEL_B: + TS_RELEASE_LATENCY_B(get_release(tsk)); + break; + case CRIT_LEVEL_C: + TS_RELEASE_LATENCY_C(get_release(tsk)); + break; + default: + break; } +#endif + + raw_spin_lock_irqsave(&state->lock, flags); - raw_spin_lock(&state->lock); /* Assumption: litmus_clock() is synchronized across cores, * since we might not actually be executing on tinfo->cpu * at the moment. */ @@ -805,12 +796,14 @@ static void mc2_task_resume(struct task_struct *tsk) /* NOTE: drops state->lock */ TRACE_TASK(tsk, "mc2_resume()\n"); + raw_spin_unlock_irqrestore(&state->lock, flags); + + raw_spin_lock(&state->lock); mc2_update_timer_and_unlock(state); } else { TRACE_TASK(tsk, "resume event ignored, still scheduled\n"); } - local_irq_restore(flags); } @@ -818,7 +811,7 @@ static void mc2_task_resume(struct task_struct *tsk) */ static long mc2_admit_task(struct task_struct *tsk) { - long err = -ESRCH; + long err = 0; unsigned long flags; struct reservation *res; struct mc2_cpu_state *state; @@ -831,11 +824,10 @@ static long mc2_admit_task(struct task_struct *tsk) if (!mp) { printk(KERN_ERR "mc2_admit_task: criticality level has not been set\n"); - return err; + return -ESRCH; } lv = mp->crit; - preempt_disable(); if (lv < CRIT_LEVEL_C) { state = cpu_state_for(task_cpu(tsk)); @@ -858,6 +850,9 @@ static long mc2_admit_task(struct task_struct *tsk) /* disable LITMUS^RT's per-thread budget enforcement */ tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT; } + else { + err = -ESRCH; + } raw_spin_unlock_irqrestore(&state->lock, flags); } else if (lv == CRIT_LEVEL_C) { @@ -882,12 +877,13 @@ static long mc2_admit_task(struct task_struct *tsk) /* disable LITMUS^RT's per-thread budget enforcement */ tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT; } + else { + err = -ESRCH; + } raw_spin_unlock(&_global_env.lock); raw_spin_unlock_irqrestore(&state->lock, flags); } - - preempt_enable(); if (err) kfree(tinfo); @@ -908,6 +904,8 @@ static void mc2_task_new(struct task_struct *tsk, int on_runqueue, enum crit_level lv = get_task_crit_level(tsk); lt_t release = 0; + BUG_ON(lv < CRIT_LEVEL_A || lv > CRIT_LEVEL_C); + TRACE_TASK(tsk, "new RT task %llu (on_rq:%d, running:%d)\n", litmus_clock(), on_runqueue, is_running); @@ -934,8 +932,7 @@ static void mc2_task_new(struct task_struct *tsk, int on_runqueue, else { res = sup_find_by_id(&state->sup_env, tinfo->mc2_param.res_id); } - release = res->next_replenishment; - + if (on_runqueue || is_running) { /* Assumption: litmus_clock() is synchronized across cores * [see comment in pres_task_resume()] */ @@ -944,22 +941,29 @@ static void mc2_task_new(struct task_struct *tsk, int on_runqueue, } else sup_update_time(&state->sup_env, litmus_clock()); + task_arrives(state, tsk); if (lv == CRIT_LEVEL_C) raw_spin_unlock(&_global_env.lock); /* NOTE: drops state->lock */ + raw_spin_unlock(&state->lock); + local_irq_restore(flags); + TRACE("mc2_new()\n"); + raw_spin_lock(&state->lock); mc2_update_timer_and_unlock(state); } else { if (lv == CRIT_LEVEL_C) raw_spin_unlock(&_global_env.lock); raw_spin_unlock(&state->lock); + local_irq_restore(flags); } - local_irq_restore(flags); + release = res->next_replenishment; if (!release) { TRACE_TASK(tsk, "mc2_task_new() next_release = %llu\n", release); + BUG(); } else TRACE_TASK(tsk, "mc2_task_new() next_release = NULL\n"); @@ -977,7 +981,10 @@ static long mc2_reservation_destroy(unsigned int reservation_id, int cpu) unsigned long flags; if (cpu == -1) { + struct next_timer_event *event, *e_next; + /* if the reservation is global reservation */ + local_irq_save(flags); raw_spin_lock(&_global_env.lock); @@ -1009,6 +1016,13 @@ static long mc2_reservation_destroy(unsigned int reservation_id, int cpu) } } } + /* delete corresponding events */ + list_for_each_entry_safe(event, e_next, &_global_env.next_events, list) { + if (event->id == reservation_id) { + list_del(&event->list); + kfree(event); + } + } raw_spin_unlock(&_global_env.lock); local_irq_restore(flags); @@ -1105,7 +1119,6 @@ static void mc2_task_exit(struct task_struct *tsk) mc2_update_timer_and_unlock(state); } else { raw_spin_unlock(&state->lock); - } if (lv == CRIT_LEVEL_C) { @@ -1412,7 +1425,7 @@ static long mc2_activate_plugin(void) struct cpu_entry *ce; gmp_init(&_global_env); - raw_spin_lock_init(&_lowest_prio_cpu.lock); + //raw_spin_lock_init(&_lowest_prio_cpu.lock); for_each_online_cpu(cpu) { TRACE("Initializing CPU%d...\n", cpu); @@ -1456,7 +1469,8 @@ static void mc2_finish_switch(struct task_struct *prev) state->scheduled = is_realtime(current) ? current : NULL; if (lv == CRIT_LEVEL_C) { for (cpus = 0; cpuscpu != cpus) { + resched_cpu[cpus] = 0; litmus_reschedule(cpus); } } diff --git a/litmus/uncachedev.c b/litmus/uncachedev.c index 06a6a7c17983..86875816c6ef 100644 --- a/litmus/uncachedev.c +++ b/litmus/uncachedev.c @@ -28,8 +28,8 @@ int litmus_uncache_vm_fault(struct vm_area_struct* vma, /* modeled after SG DMA video4linux, but without DMA. */ /* (see drivers/media/video/videobuf-dma-sg.c) */ struct page *page; - - page = alloc_page(GFP_USER); + + page = alloc_page(GFP_USER|GFP_COLOR); if (!page) return VM_FAULT_OOM; diff --git a/mm/dmapool.c b/mm/dmapool.c index fd5fe4342e93..b69dc139bcc4 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -333,7 +333,7 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, /* pool_alloc_page() might sleep, so temporarily drop &pool->lock */ spin_unlock_irqrestore(&pool->lock, flags); - + printk(KERN_INFO "dma_pool_alloc(): called with %x flags\n", mem_flags); page = pool_alloc_page(pool, mem_flags); if (!page) return NULL; diff --git a/mm/migrate.c b/mm/migrate.c index 8dd685be20d8..29b69cd39d13 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -638,7 +638,18 @@ void replicate_page_copy(struct page *newpage, struct page *page) SetPageMappedToDisk(newpage); if (PageDirty(page)) { - BUG(); + clear_page_dirty_for_io(page); + /* + * Want to mark the page and the radix tree as dirty, and + * redo the accounting that clear_page_dirty_for_io undid, + * but we can't use set_page_dirty because that function + * is actually a signal that all of the page has become dirty. + * Whereas only part of our page may be dirty. + */ + if (PageSwapBacked(page)) + SetPageDirty(newpage); + else + __set_page_dirty_nobuffers(newpage); } /* diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 90cf3ea441e0..8c22d10b0c23 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1032,7 +1032,7 @@ static inline void expand(struct zone *zone, struct page *page, } } -static inline void expand_middle(struct zone *zone, struct page *page, +static inline int expand_middle(struct zone *zone, struct page *page, int offset, int low, int high, struct free_area *area, int migratetype) { @@ -1056,8 +1056,10 @@ static inline void expand_middle(struct zone *zone, struct page *page, area->nr_free++; set_page_order(&page[0], high); - if (offset == size) - return; + if (offset == size) { + //printk(KERN_INFO "offset == size %d high = %d\n", offset, high); + return high; + } area--; high--; @@ -1065,6 +1067,8 @@ static inline void expand_middle(struct zone *zone, struct page *page, list_add(&page[size].lru, &area->free_list[migratetype]); area->nr_free++; set_page_order(&page[size], high); + + return high; } /* @@ -1224,52 +1228,78 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, // if (order <= 2 && color_req == 1) { /* The max. order of color_req is <= 2 */ - if (color_req == 1) { + if (color_req != 0) { int found = 0; + int area_index; unsigned long s_pfn = zone->zone_start_pfn; unsigned long e_pfn = zone_end_pfn(zone); - TRACE("COLOR PAGE requested on CPU%d with order = %d migratetype = %d\n", cpu, order, migratetype); + + if (color_req == 2) + area_index = get_area_index(1); + else + area_index = get_area_index(cpu); + + //printk(KERN_INFO "CPU%d color_request %d, area_index %d\n", cpu, color_req, area_index); + //printk(KERN_INFO "COLOR PAGE requested on CPU%d with order = %d migratetype = %d\n", cpu, order, migratetype); /* Find a page of the appropriate size in the preferred list */ for (current_order = order; current_order < MAX_PARTITIONED_ORDER; ++current_order) { int offset = 0; - area = &(zone->free_area_d[cpu][current_order]); + area = &(zone->free_area_d[area_index][current_order]); if (list_empty(&area->free_list[migratetype])) { - TRACE("order %d list empty\n", current_order); + //printk(KERN_INFO "P%d order %d list empty\n", cpu, current_order); continue; } - list_for_each_entry(page, &area->free_list[migratetype], lru) { - TRACE("__rmqueue_smallest list entry %p color %d pfn:%05lx\n", page, page_color(page), page_to_pfn(page)); - if (current_order < MAX_CONTIG_ORDER) { - if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) { - found = 1; - offset = 0; - break; - } - } else { // order >= 3 , must be uncacheable. - int size = 1 << current_order; - for (offset = 0; offset < size; offset += 4) { - if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) { + if (order >= MAX_CONTIG_ORDER) { // requested order >= 3 , must be uncacheable. + page = list_entry(area->free_list[migratetype].next, struct page, lru); + found = 1; + } else { + list_for_each_entry(page, &area->free_list[migratetype], lru) { + //printk(KERN_INFO "P%d __rmqueue_smallest order [%d] list entry %p color %d pfn:%05lx\n", cpu, current_order, page, page_color(page), page_to_pfn(page)); + if (current_order < MAX_CONTIG_ORDER) { + if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) { + offset = 0; found = 1; break; } + } else { + int size = 1 << current_order; + for (offset = 0; offset < size; offset++) { + if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) { + found = 1; + break; + } + } + if (found) + break; } - if (found) - break; } } - TRACE("__rmqueue_smallest LAST list entry %p\n", page); - + //printk(KERN_INFO "P%d __rmqueue_smallest LAST list entry %p\n", cpu, page); + if (!found) - return NULL; + continue; + //printk(KERN_INFO "P%d __rmqueue_smallest LAST list entry %p, order %d current_order %d offset %d\n", cpu, page, order, current_order, offset); list_del(&page->lru); rmv_page_order(page); area->nr_free--; - expand(zone, page, order, current_order, area, migratetype); + + if (offset == 0) { + expand(zone, page, order, current_order, area, migratetype); + } else { + int frac = expand_middle(zone, page, offset, order, current_order, area, migratetype); + page = &page[offset]; + //list_del(&page->lru); + //rmv_page_order(page); + area = &(zone->free_area_d[area_index][frac]); + //area->nr_free--; + expand(zone, page, order, frac, area, migratetype); + } + set_freepage_migratetype(page, migratetype); - TRACE("COLOR %d page return %p\n", page_color(page), page); + //printk(KERN_INFO "__rmqueue_smallest(): CPU%d COLOR %d BANK %d page return %p pfn:%05lx\n", cpu, page_color(page), page_bank(page), page, page_to_pfn(page)); return page; } } else { @@ -1511,47 +1541,59 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, int fallback_mt; bool can_steal; - if (color_req == 1) { + if (color_req != 0) { int cpu = raw_smp_processor_id(); int found = 0; + int area_index; unsigned long s_pfn = zone->zone_start_pfn; unsigned long e_pfn = zone_end_pfn(zone); + if (color_req == 2) + area_index = get_area_index(1); + else + area_index = get_area_index(cpu); + /* Find the largest possible block of pages in the other list */ for (current_order = MAX_PARTITIONED_ORDER-1; current_order >= order && current_order <= MAX_PARTITIONED_ORDER-1; --current_order) { int offset = 0; - area = &(zone->free_area_d[cpu][current_order]); + area = &(zone->free_area_d[area_index][current_order]); fallback_mt = find_suitable_fallback(area, current_order, start_migratetype, false, &can_steal); if (fallback_mt == -1) continue; - list_for_each_entry(page, &area->free_list[fallback_mt], lru) { - TRACE("__rmqueue_fallback list entry %p color %d pfn:%05lx\n", page, page_color(page), page_to_pfn(page)); - if (current_order < MAX_CONTIG_ORDER) { - if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) { - found = 1; - offset = 0; - break; - } - } else { // order >= 3 , must be uncacheable. - int size = 1 << current_order; - for (offset = 0; offset < size; offset += 4) { - if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) { + if (order >= MAX_CONTIG_ORDER) { + page = list_entry(area->free_list[fallback_mt].next, struct page, lru); + found = 1; + //printk(KERN_INFO "__rmqueue_fallback order >= MAX_CONTIG_ORDER page = %p color %d pfn:%05lx\n", page, page_color(page), page_to_pfn(page)); + } else { + list_for_each_entry(page, &area->free_list[fallback_mt], lru) { + //printk(KERN_INFO "__rmqueue_fallback list entry %p color %d pfn:%05lx\n", page, page_color(page), page_to_pfn(page)); + if (current_order < MAX_CONTIG_ORDER) { + if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) { found = 1; + offset = 0; break; } + } else { + int size = 1 << current_order; + for (offset = 0; offset < size; offset++) { + if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) { + found = 1; + break; + } + } + if (found) + break; } - if (found) - break; } } - TRACE("__rmqueue_fallback LAST list entry %p\n", page); + //printk(KERN_INFO "__rmqueue_fallback LAST list entry %p\n", page); if (!found) - return NULL; + continue; if (can_steal) steal_suitable_fallback(zone, page, start_migratetype); @@ -1561,7 +1603,19 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, list_del(&page->lru); rmv_page_order(page); - expand(zone, page, order, current_order, area, start_migratetype); + if (offset == 0) + expand(zone, page, order, current_order, area, start_migratetype); + else { + int frac = expand_middle(zone, page, offset, order, current_order, area, start_migratetype); + page = &page[offset]; + //list_del(&page->lru); + //rmv_page_order(page); + area = &(zone->free_area_d[area_index][frac]); + //area->nr_free--; + expand(zone, page, order, frac, area, start_migratetype); + + } + /* * The freepage_migratetype may differ from pageblock's @@ -1576,7 +1630,7 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, trace_mm_page_alloc_extfrag(page, order, current_order, start_migratetype, fallback_mt); - TRACE("__rmqueue_fallback(): CPU%d COLOR %d page return %p pfn:%05lx\n", cpu, page_color(page), page, page_to_pfn(page)); + //printk(KERN_INFO "__rmqueue_fallback(): CPU%d COLOR %d BANK %d page return %p pfn:%05lx\n", cpu, page_color(page), page_bank(page), page, page_to_pfn(page)); return page; } } else { @@ -1640,9 +1694,10 @@ retry_reserve: if (!page) { page = __rmqueue_fallback(zone, order, migratetype, color_req); -#ifdef CONFIG_SCHED_DEBUG_TRACE +#if 0 +//#ifdef CONFIG_SCHED_DEBUG_TRACE if (color_req) - TRACE("page received from __rmqueue_fallback()"); + printk(KERN_INFO "page received from __rmqueue_fallback()"); #endif } @@ -1651,7 +1706,7 @@ retry_reserve: * is used because __rmqueue_smallest is an inline function * and we want just one call site */ - if (!page) { + if (!page && !color_req) { migratetype = MIGRATE_RESERVE; goto retry_reserve; } @@ -1897,9 +1952,11 @@ void free_hot_cold_page(struct page *page, bool cold) if (bank_to_partition(page_bank(page)) == NR_CPUS) __count_vm_event(PGFREE); - else if (bank_to_partition(page_bank(page)) < NR_CPUS) - //__count_vm_event(PGFREE_HC); - BUG(); + else if (bank_to_partition(page_bank(page)) < NR_CPUS) { + __count_vm_event(PGFREE_HC); + free_one_page(zone, page, pfn, 0, migratetype); + goto out; + } /* * We only track unmovable, reclaimable and movable on pcp lists. @@ -1954,8 +2011,12 @@ void free_hot_cold_page_list(struct list_head *list, bool cold) struct page *page, *next; list_for_each_entry_safe(page, next, list, lru) { + int parti_no = bank_to_partition(page_bank(page)); trace_mm_page_free_batched(page, cold); - free_hot_cold_page(page, cold); + if (parti_no == NR_CPUS) + free_hot_cold_page(page, cold); + else + __free_pages_ok(page, 0); } } @@ -2069,11 +2130,15 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, unsigned long flags; struct page *page; bool cold = ((gfp_flags & __GFP_COLD) != 0); - bool colored_req = ((gfp_flags & __GFP_COLOR) != 0); + int colored_req = ((gfp_flags & __GFP_COLOR) != 0); -#ifdef CONFIG_SCHED_DEBUG_TRACE + if ((gfp_flags & __GFP_CPU1) != 0) + colored_req = 2; + +//#ifdef CONFIG_SCHED_DEBUG_TRACE +#if 0 if (colored_req) - TRACE("buffered_rmqueue(): colored_req received\n"); + printk(KERN_INFO "buffered_rmqueue(): colored_req %d received\n", colored_req); #endif if (likely(order == 0) && !colored_req) { @@ -3226,9 +3291,10 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, .migratetype = gfpflags_to_migratetype(gfp_mask), }; -#ifdef CONFIG_SCHED_DEBUG_TRACE +//#ifdef CONFIG_SCHED_DEBUG_TRACE +#if 0 if (gfp_mask&GFP_COLOR) - TRACE("__alloc_pages_nodemask(): called gfp %08x gfp_allowed_mask %08x mt = %d\n", gfp_mask, gfp_allowed_mask, ac.migratetype); + printk(KERN_INFO "__alloc_pages_nodemask(): called gfp %08x gfp_allowed_mask %08x mt = %d\n", gfp_mask, gfp_allowed_mask, ac.migratetype); #endif gfp_mask &= gfp_allowed_mask; diff --git a/mm/slub.c b/mm/slub.c index 0e8ce1f912fb..1fc997111aac 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -115,6 +115,26 @@ * the fast path and disables lockless freelists. */ +// This Address Decoding is used in imx6-sabredsd platform +#define BANK_MASK 0x38000000 +#define BANK_SHIFT 27 + +#define CACHE_MASK 0x0000f000 +#define CACHE_SHIFT 12 +#define MAX_COLOR_NODE 128 + +/* Decoding page color, 0~15 */ +static inline unsigned int page_color(struct page *page) +{ + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT); +} + +/* Decoding page bank number, 0~7 */ +static inline unsigned int page_bank(struct page *page) +{ + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT); +} + static inline int kmem_cache_debug(struct kmem_cache *s) { #ifdef CONFIG_SLUB_DEBUG @@ -1322,8 +1342,8 @@ static inline struct page *alloc_slab_page(struct kmem_cache *s, if (node == NUMA_NO_NODE) { #ifdef CONFIG_SCHED_DEBUG_TRACE - if (flags&GFP_COLOR) - printk(KERN_INFO "alloc_pages calls with GFP_COLOR order = %d\n", order); +// if (flags&GFP_COLOR) +// printk(KERN_INFO "alloc_pages calls with GFP_COLOR order = %d\n", order); #endif page = alloc_pages(flags, order); } @@ -1343,8 +1363,8 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) gfp_t alloc_gfp; #ifdef CONFIG_SCHED_DEBUG_TRACE - if (flags&GFP_COLOR) - printk(KERN_INFO "gfp_allowed_mask = %08x\n", gfp_allowed_mask); +// if (flags&GFP_COLOR) +// printk(KERN_INFO "gfp_allowed_mask = %08x\n", gfp_allowed_mask); #endif flags &= gfp_allowed_mask; @@ -1361,8 +1381,8 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; #ifdef CONFIG_SCHED_DEBUG_TRACE - if (flags&__GFP_COLOR) - printk(KERN_INFO "allocate_slab with GFP_COLOR alloc_gfp = %08x\n", alloc_gfp); +// if (flags&__GFP_COLOR) +// printk(KERN_INFO "allocate_slab with GFP_COLOR alloc_gfp = %08x\n", alloc_gfp); #endif page = alloc_slab_page(s, alloc_gfp, node, oo); @@ -2240,8 +2260,8 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, page = new_slab(s, flags, node); #ifdef CONFIG_SCHED_DEBUG_TRACE - if (flags&GFP_COLOR) - printk(KERN_INFO "new_slab_objects(): gets page %p\n", page); +// if (flags&GFP_COLOR) +// printk(KERN_INFO "new_slab_objects(): gets page %p color %d, bank %d\n", page, page_color(page), page_bank(page)); #endif if (page) { @@ -2331,8 +2351,8 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long flags; #ifdef CONFIG_SCHED_DEBUG_TRACE - if (gfpflags&GFP_COLOR) - printk(KERN_INFO "__slab_alloc slow_path\n"); +// if (gfpflags&GFP_COLOR) +// printk(KERN_INFO "__slab_alloc slow_path\n"); #endif local_irq_save(flags); @@ -2346,8 +2366,8 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, #endif #ifdef CONFIG_SCHED_DEBUG_TRACE - if (gfpflags&GFP_COLOR) - printk(KERN_INFO "__slab_alloc : page %p, partial %p\n", c->page, c->partial); +// if (gfpflags&GFP_COLOR) +// printk(KERN_INFO "__slab_alloc : page %p, partial %p\n", c->page, c->partial); #endif page = c->page; @@ -3340,8 +3360,8 @@ void *__kmalloc(size_t size, gfp_t flags) void *ret; #ifdef CONFIG_SCHED_DEBUG_TRACE - if (flags & GFP_COLOR) - printk(KERN_INFO "kmalloc size %d\n", size); +// if (flags & GFP_COLOR) +// printk(KERN_INFO "kmalloc size %d\n", size); #endif if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) @@ -3350,16 +3370,16 @@ void *__kmalloc(size_t size, gfp_t flags) s = kmalloc_slab(size, flags); #ifdef CONFIG_SCHED_DEBUG_TRACE - if (flags & GFP_COLOR) - printk(KERN_INFO "kmalloc_slab %p\n", s); +// if (flags & GFP_COLOR) +// printk(KERN_INFO "kmalloc_slab %p\n", s); #endif if (unlikely(ZERO_OR_NULL_PTR(s))) return s; #ifdef CONFIG_SCHED_DEBUG_TRACE - if (flags & GFP_COLOR) - printk(KERN_INFO "slab_alloc calls!!\n"); +// if (flags & GFP_COLOR) +// printk(KERN_INFO "slab_alloc calls!!\n"); #endif ret = slab_alloc(s, flags, _RET_IP_); diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 27600f419126..7a6d7de8fff8 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -1802,6 +1802,47 @@ void *vmalloc_user(unsigned long size) } EXPORT_SYMBOL(vmalloc_user); +/** + * vmalloc_color_user - allocate zeroed virtually contiguous memory for userspace + * @size: allocation size + * + * The resulting memory area is zeroed so it can be mapped to userspace + * without leaking data. + */ +void *vmalloc_color_user(unsigned long size) +{ + struct vm_struct *area; + void *ret; + + ret = __vmalloc_node(size, SHMLBA, + GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | GFP_COLOR, + PAGE_KERNEL, NUMA_NO_NODE, + __builtin_return_address(0)); + if (ret) { + area = find_vm_area(ret); + area->flags |= VM_USERMAP; + } + return ret; +} +EXPORT_SYMBOL(vmalloc_color_user); + +void *vmalloc_color_user_cpu1(unsigned long size) +{ + struct vm_struct *area; + void *ret; + + ret = __vmalloc_node(size, SHMLBA, + GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | GFP_COLOR | GFP_CPU1, + PAGE_KERNEL, NUMA_NO_NODE, + __builtin_return_address(0)); + if (ret) { + area = find_vm_area(ret); + area->flags |= VM_USERMAP; + } + return ret; +} +EXPORT_SYMBOL(vmalloc_color_user_cpu1); + /** * vmalloc_node - allocate memory on a specific node * @size: allocation size diff --git a/net/core/dev.c b/net/core/dev.c index aa82f9ab6a36..f02d0c582e84 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4629,6 +4629,7 @@ static int napi_poll(struct napi_struct *n, struct list_head *repoll) */ work = 0; if (test_bit(NAPI_STATE_SCHED, &n->state)) { + // fec_enet_rx_napi() is called work = n->poll(n, weight); trace_napi_poll(n); } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 41ec02242ea7..92f091ce1d47 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -77,6 +77,13 @@ #include #include +#define ENABLE_WORST_CASE 1 +#ifdef ENABLE_WORST_CASE +#define SKB_FLAG (GFP_COLOR|GFP_CPU1) +#else +#define SKB_FLAG (0) +#endif + struct kmem_cache *skbuff_head_cache __read_mostly; static struct kmem_cache *skbuff_fclone_cache __read_mostly; @@ -133,14 +140,14 @@ static void *__kmalloc_reserve(size_t size, gfp_t flags, int node, * to the reserves, fail. */ obj = kmalloc_node_track_caller(size, - flags | __GFP_NOMEMALLOC | __GFP_NOWARN, + flags | __GFP_NOMEMALLOC | __GFP_NOWARN | SKB_FLAG, node); if (obj || !(gfp_pfmemalloc_allowed(flags))) goto out; /* Try again but now we are using pfmemalloc reserves */ ret_pfmemalloc = true; - obj = kmalloc_node_track_caller(size, flags, node); + obj = kmalloc_node_track_caller(size, flags | SKB_FLAG, node); out: if (pfmemalloc) @@ -161,7 +168,7 @@ struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node) /* Get the HEAD */ skb = kmem_cache_alloc_node(skbuff_head_cache, - gfp_mask & ~__GFP_DMA, node); + (gfp_mask & ~__GFP_DMA) | SKB_FLAG, node); if (!skb) goto out; @@ -213,7 +220,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, gfp_mask |= __GFP_MEMALLOC; /* Get the HEAD */ - skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); + skb = kmem_cache_alloc_node(cache, (gfp_mask & ~__GFP_DMA) | SKB_FLAG, node); if (!skb) goto out; prefetchw(skb); @@ -225,7 +232,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, */ size = SKB_DATA_ALIGN(size); size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); - data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc); + data = kmalloc_reserve(size, gfp_mask | SKB_FLAG, node, &pfmemalloc); if (!data) goto nodata; /* kmalloc(size) might give us more room than requested. @@ -304,7 +311,7 @@ struct sk_buff *__build_skb(void *data, unsigned int frag_size) struct sk_buff *skb; unsigned int size = frag_size ? : ksize(data); - skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); + skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC | SKB_FLAG); if (!skb) return NULL; @@ -367,12 +374,12 @@ static struct page *__page_frag_refill(struct netdev_alloc_cache *nc, if (order) { gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC; - page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order); + page = alloc_pages_node(NUMA_NO_NODE, gfp_mask | SKB_FLAG, order); nc->frag.size = PAGE_SIZE << (page ? order : 0); } if (unlikely(!page)) - page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); + page = alloc_pages_node(NUMA_NO_NODE, gfp | SKB_FLAG, 0); nc->frag.page = page; @@ -389,7 +396,7 @@ static void *__alloc_page_frag(struct netdev_alloc_cache __percpu *cache, if (unlikely(!page)) { refill: - page = __page_frag_refill(nc, gfp_mask); + page = __page_frag_refill(nc, gfp_mask | SKB_FLAG); if (!page) return NULL; @@ -434,7 +441,7 @@ static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) void *data; local_irq_save(flags); - data = __alloc_page_frag(&netdev_alloc_cache, fragsz, gfp_mask); + data = __alloc_page_frag(&netdev_alloc_cache, fragsz, gfp_mask | SKB_FLAG); local_irq_restore(flags); return data; } @@ -448,18 +455,18 @@ static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) */ void *netdev_alloc_frag(unsigned int fragsz) { - return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD); + return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD | SKB_FLAG); } EXPORT_SYMBOL(netdev_alloc_frag); static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) { - return __alloc_page_frag(&napi_alloc_cache, fragsz, gfp_mask); + return __alloc_page_frag(&napi_alloc_cache, fragsz, gfp_mask | SKB_FLAG); } void *napi_alloc_frag(unsigned int fragsz) { - return __napi_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD); + return __napi_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD | SKB_FLAG); } EXPORT_SYMBOL(napi_alloc_frag); @@ -493,8 +500,8 @@ static struct sk_buff *__alloc_rx_skb(unsigned int length, gfp_t gfp_mask, gfp_mask |= __GFP_MEMALLOC; data = (flags & SKB_ALLOC_NAPI) ? - __napi_alloc_frag(fragsz, gfp_mask) : - __netdev_alloc_frag(fragsz, gfp_mask); + __napi_alloc_frag(fragsz, gfp_mask | SKB_FLAG) : + __netdev_alloc_frag(fragsz, gfp_mask | SKB_FLAG); if (likely(data)) { skb = build_skb(data, fragsz); @@ -502,7 +509,7 @@ static struct sk_buff *__alloc_rx_skb(unsigned int length, gfp_t gfp_mask, put_page(virt_to_head_page(data)); } } else { - skb = __alloc_skb(length, gfp_mask, + skb = __alloc_skb(length, gfp_mask | SKB_FLAG, SKB_ALLOC_RX, NUMA_NO_NODE); } return skb; @@ -527,7 +534,8 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev, struct sk_buff *skb; length += NET_SKB_PAD; - skb = __alloc_rx_skb(length, gfp_mask, 0); + + skb = __alloc_rx_skb(length, gfp_mask | SKB_FLAG, 0); if (likely(skb)) { skb_reserve(skb, NET_SKB_PAD); @@ -557,7 +565,7 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, struct sk_buff *skb; length += NET_SKB_PAD + NET_IP_ALIGN; - skb = __alloc_rx_skb(length, gfp_mask, SKB_ALLOC_NAPI); + skb = __alloc_rx_skb(length, gfp_mask | SKB_FLAG, SKB_ALLOC_NAPI); if (likely(skb)) { skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); @@ -932,7 +940,7 @@ int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) u8 *vaddr; skb_frag_t *f = &skb_shinfo(skb)->frags[i]; - page = alloc_page(gfp_mask); + page = alloc_page(gfp_mask | SKB_FLAG); if (!page) { while (head) { struct page *next = (struct page *)page_private(head); @@ -988,7 +996,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) skb1); struct sk_buff *n; - if (skb_orphan_frags(skb, gfp_mask)) + if (skb_orphan_frags(skb, gfp_mask | SKB_FLAG)) return NULL; if (skb->fclone == SKB_FCLONE_ORIG && @@ -999,7 +1007,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) if (skb_pfmemalloc(skb)) gfp_mask |= __GFP_MEMALLOC; - n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); + n = kmem_cache_alloc(skbuff_head_cache, gfp_mask | SKB_FLAG); if (!n) return NULL; @@ -1063,7 +1071,7 @@ struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) { int headerlen = skb_headroom(skb); unsigned int size = skb_end_offset(skb) + skb->data_len; - struct sk_buff *n = __alloc_skb(size, gfp_mask, + struct sk_buff *n = __alloc_skb(size, gfp_mask | SKB_FLAG, skb_alloc_rx_flag(skb), NUMA_NO_NODE); if (!n) @@ -1104,7 +1112,7 @@ struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, { unsigned int size = skb_headlen(skb) + headroom; int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); - struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); + struct sk_buff *n = __alloc_skb(size, gfp_mask | SKB_FLAG, flags, NUMA_NO_NODE); if (!n) goto out; @@ -1123,7 +1131,7 @@ struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, if (skb_shinfo(skb)->nr_frags) { int i; - if (skb_orphan_frags(skb, gfp_mask)) { + if (skb_orphan_frags(skb, gfp_mask | SKB_FLAG)) { kfree_skb(n); n = NULL; goto out; @@ -1180,7 +1188,7 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, if (skb_pfmemalloc(skb)) gfp_mask |= __GFP_MEMALLOC; data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), - gfp_mask, NUMA_NO_NODE, NULL); + gfp_mask | SKB_FLAG, NUMA_NO_NODE, NULL); if (!data) goto nodata; size = SKB_WITH_OVERHEAD(ksize(data)); @@ -1201,7 +1209,7 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, */ if (skb_cloned(skb)) { /* copy this zero copy skb frags */ - if (skb_orphan_frags(skb, gfp_mask)) + if (skb_orphan_frags(skb, gfp_mask | SKB_FLAG)) goto nofrags; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) skb_frag_ref(skb, i); @@ -1286,7 +1294,7 @@ struct sk_buff *skb_copy_expand(const struct sk_buff *skb, * Allocate the copy buffer */ struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, - gfp_mask, skb_alloc_rx_flag(skb), + gfp_mask | SKB_FLAG, skb_alloc_rx_flag(skb), NUMA_NO_NODE); int oldheadroom = skb_headroom(skb); int head_copy_len, head_copy_off; @@ -4387,7 +4395,7 @@ struct sk_buff *alloc_skb_with_frags(unsigned long header_len, gfp_head |= __GFP_REPEAT; *errcode = -ENOBUFS; - skb = alloc_skb(header_len, gfp_head); + skb = alloc_skb(header_len, gfp_head | SKB_FLAG); if (!skb) return NULL; @@ -4401,7 +4409,7 @@ struct sk_buff *alloc_skb_with_frags(unsigned long header_len, page = alloc_pages((gfp_mask & ~__GFP_WAIT) | __GFP_COMP | __GFP_NOWARN | - __GFP_NORETRY, + __GFP_NORETRY | SKB_FLAG, order); if (page) goto fill_page; @@ -4411,7 +4419,7 @@ struct sk_buff *alloc_skb_with_frags(unsigned long header_len, } order--; } - page = alloc_page(gfp_mask); + page = alloc_page(gfp_mask | SKB_FLAG); if (!page) goto failure; fill_page: -- cgit v1.2.2