litmus-rt.git - The LITMUS^RT kernel.

diff options

author	Fengguang Wu <wfg@mail.ustc.edu.cn>	2007-07-19 04:48:02 -0400
committer	Linus Torvalds <torvalds@woody.linux-foundation.org>	2007-07-19 13:04:44 -0400
commit	3ea89ee86a82e9fbde37018d9b9e92a552e5fd13 (patch)
tree	5fef7216ef39d66684754e4aadf368ce9adfd72e /mm
parent	122a21d11cbfda6d1e33cbc8ae9e4c4ee2f1886e (diff)

readahead: convert filemap invocations

Convert filemap reads to use on-demand readahead. The new call scheme is to - call readahead on non-cached page - call readahead on look-ahead page - update prev_index when finished with the read request Signed-off-by: Fengguang Wu <wfg@mail.ustc.edu.cn> Cc: Steven Pratt <slpratt@austin.ibm.com> Cc: Ram Pai <linuxram@us.ibm.com> Cc: Rusty Russell <rusty@rustcorp.com.au> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Diffstat (limited to 'mm')

-rw-r--r--

mm/filemap.c

1 files changed, 31 insertions, 20 deletions

                 unsigned long nr, ret;
                 cond_resched();
-                if (index == next_index)
-                        next_index = page_cache_readahead(mapping, &ra, filp,
-                                        index, last_index - index);
 find_page:
                 page = find_get_page(mapping, index);
-                if (unlikely(page == NULL)) {
+                if (!page) {
-                        handle_ra_miss(mapping, &ra, index);
+                        page_cache_readahead_ondemand(mapping,
-                        goto no_cached_page;
+                                        &ra, filp, page,
+                                        index, last_index - index);
+                        page = find_get_page(mapping, index);
+                        if (unlikely(page == NULL))
+                                goto no_cached_page;
+                }
+                if (PageReadahead(page)) {
+                        page_cache_readahead_ondemand(mapping,
+                                        &ra, filp, page,
+                                        index, last_index - index);
                 }
                 if (!PageUptodate(page))
                         goto page_not_up_to_date;
 out:
         *_ra = ra;
+        _ra->prev_index = prev_index;
         *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
         if (cached_page)
                 goto no_cached_page;
         /*
-         * The readahead code wants to be told about each and every page
-         * so it can build and shrink its windows appropriately
-         *
-         * For sequential accesses, we use the generic readahead logic.
-         */
-        if (VM_SequentialReadHint(vma))
-                page_cache_readahead(mapping, ra, file, vmf->pgoff, 1);
-        /*
          * Do we have something in the page cache already?
          */
 retry_find:
         page = find_lock_page(mapping, vmf->pgoff);
+        /*
+         * For sequential accesses, we use the generic readahead logic.
+         */
+        if (VM_SequentialReadHint(vma)) {
+                if (!page) {
+                        page_cache_readahead_ondemand(mapping, ra, file, page,
+                                                           vmf->pgoff, 1);
+                        page = find_lock_page(mapping, vmf->pgoff);
+                        if (!page)
+                                goto no_cached_page;
+                }
+                if (PageReadahead(page)) {
+                        page_cache_readahead_ondemand(mapping, ra, file, page,
+                                                           vmf->pgoff, 1);
+                }
+        }
         if (!page) {
                 unsigned long ra_pages;
-                if (VM_SequentialReadHint(vma)) {
-                        handle_ra_miss(mapping, ra, vmf->pgoff);
-                        goto no_cached_page;
-                }
                 ra->mmap_miss++;
                 /*
          * Found the page and have a reference on it.
          */
         mark_page_accessed(page);
+        ra->prev_index = page->index;
         vmf->page = page;
         return ret | VM_FAULT_LOCKED;

768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820

/* pci_sun4v.c: SUN4V specific PCI controller support. * * Copyright (C) 2006, 2007, 2008 David S. Miller (davem@davemloft.net) */ #include <linux/kernel.h> #include <linux/types.h> #include <linux/pci.h> #include <linux/init.h> #include <linux/slab.h> #include <linux/interrupt.h> #include <linux/percpu.h> #include <linux/irq.h> #include <linux/msi.h> #include <linux/log2.h> #include <asm/iommu.h> #include <asm/irq.h> #include <asm/upa.h> #include <asm/pstate.h> #include <asm/oplib.h> #include <asm/hypervisor.h> #include <asm/prom.h> #include "pci_impl.h" #include "iommu_common.h" #include "pci_sun4v.h" static unsigned long vpci_major = 1; static unsigned long vpci_minor = 1; #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) struct iommu_batch { struct device *dev; /* Device mapping is for. */ unsigned long prot; /* IOMMU page protections */ unsigned long entry; /* Index into IOTSB. */ u64 *pglist; /* List of physical pages */ unsigned long npages; /* Number of pages in list. */ }; static DEFINE_PER_CPU(struct iommu_batch, iommu_batch); /* Interrupts must be disabled. */ static inline void iommu_batch_start(struct device *dev, unsigned long prot, unsigned long entry) { struct iommu_batch *p = &__get_cpu_var(iommu_batch); p->dev = dev; p->prot = prot; p->entry = entry; p->npages = 0; } /* Interrupts must be disabled. */ static long iommu_batch_flush(struct iommu_batch *p) { struct pci_pbm_info *pbm = p->dev->archdata.host_controller; unsigned long devhandle = pbm->devhandle; unsigned long prot = p->prot; unsigned long entry = p->entry; u64 *pglist = p->pglist; unsigned long npages = p->npages; while (npages != 0) { long num; num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry), npages, prot, __pa(pglist)); if (unlikely(num < 0)) { if (printk_ratelimit()) printk("iommu_batch_flush: IOMMU map of " "[%08lx:%08lx:%lx:%lx:%lx] failed with " "status %ld\n", devhandle, HV_PCI_TSBID(0, entry), npages, prot, __pa(pglist), num); return -1; } entry += num; npages -= num; pglist += num; } p->entry = entry; p->npages = 0; return 0; } static inline void iommu_batch_new_entry(unsigned long entry) { struct iommu_batch *p = &__get_cpu_var(iommu_batch); if (p->entry + p->npages == entry) return; if (p->entry != ~0UL) iommu_batch_flush(p); p->entry = entry; } /* Interrupts must be disabled. */ static inline long iommu_batch_add(u64 phys_page) { struct iommu_batch *p = &__get_cpu_var(iommu_batch); BUG_ON(p->npages >= PGLIST_NENTS); p->pglist[p->npages++] = phys_page; if (p->npages == PGLIST_NENTS) return iommu_batch_flush(p); return 0; } /* Interrupts must be disabled. */ static inline long iommu_batch_end(void) { struct iommu_batch *p = &__get_cpu_var(iommu_batch); BUG_ON(p->npages >= PGLIST_NENTS); return iommu_batch_flush(p); } static void *dma_4v_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp) { struct iommu *iommu; unsigned long flags, order, first_page, npages, n; void *ret; long entry; size = IO_PAGE_ALIGN(size); order = get_order(size); if (unlikely(order >= MAX_ORDER)) return NULL; npages = size >> IO_PAGE_SHIFT; first_page = __get_free_pages(gfp, order); if (unlikely(first_page == 0UL)) return NULL; memset((char *)first_page, 0, PAGE_SIZE << order); iommu = dev->archdata.iommu; spin_lock_irqsave(&iommu->lock, flags); entry = iommu_range_alloc(dev, iommu, npages, NULL); spin_unlock_irqrestore(&iommu->lock, flags); if (unlikely(entry == DMA_ERROR_CODE)) goto range_alloc_fail; *dma_addrp = (iommu->page_table_map_base + (entry << IO_PAGE_SHIFT)); ret = (void *) first_page; first_page = __pa(first_page); local_irq_save(flags); iommu_batch_start(dev, (HV_PCI_MAP_ATTR_READ | HV_PCI_MAP_ATTR_WRITE), entry); for (n = 0; n < npages; n++) { long err = iommu_batch_add(first_page + (n * PAGE_SIZE)); if (unlikely(err < 0L)) goto iommu_map_fail; } if (unlikely(iommu_batch_end() < 0L)) goto iommu_map_fail; local_irq_restore(flags); return ret; iommu_map_fail: /* Interrupts are disabled. */ spin_lock(&iommu->lock); iommu_range_free(iommu, *dma_addrp, npages); spin_unlock_irqrestore(&iommu->lock, flags); range_alloc_fail: free_pages(first_page, order); return NULL; } static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, dma_addr_t dvma) { struct pci_pbm_info *pbm; struct iommu *iommu; unsigned long flags, order, npages, entry; u32 devhandle; npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; iommu = dev->archdata.iommu; pbm = dev->archdata.host_controller; devhandle = pbm->devhandle; entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); spin_lock_irqsave(&iommu->lock, flags); iommu_range_free(iommu, dvma, npages); do { unsigned long num; num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), npages); entry += num; npages -= num; } while (npages != 0); spin_unlock_irqrestore(&iommu->lock, flags); order = get_order(size); if (order < 10) free_pages((unsigned long)cpu, order); } static dma_addr_t dma_4v_map_single(struct device *dev, void *ptr, size_t sz, enum dma_data_direction direction) { struct iommu *iommu; unsigned long flags, npages, oaddr; unsigned long i, base_paddr; u32 bus_addr, ret; unsigned long prot; long entry; iommu = dev->archdata.iommu; if (unlikely(direction == DMA_NONE)) goto bad; oaddr = (unsigned long)ptr; npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); npages >>= IO_PAGE_SHIFT; spin_lock_irqsave(&iommu->lock, flags); entry = iommu_range_alloc(dev, iommu, npages, NULL); spin_unlock_irqrestore(&iommu->lock, flags); if (unlikely(entry == DMA_ERROR_CODE)) goto bad; bus_addr = (iommu->page_table_map_base + (entry << IO_PAGE_SHIFT)); ret = bus_addr | (oaddr & ~IO_PAGE_MASK); base_paddr = __pa(oaddr & IO_PAGE_MASK); prot = HV_PCI_MAP_ATTR_READ; if (direction != DMA_TO_DEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; local_irq_save(flags); iommu_batch_start(dev, prot, entry); for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) { long err = iommu_batch_add(base_paddr); if (unlikely(err < 0L)) goto iommu_map_fail; } if (unlikely(iommu_batch_end() < 0L)) goto iommu_map_fail; local_irq_restore(flags); return ret; bad: if (printk_ratelimit()) WARN_ON(1); return DMA_ERROR_CODE; iommu_map_fail: /* Interrupts are disabled. */ spin_lock(&iommu->lock); iommu_range_free(iommu, bus_addr, npages); spin_unlock_irqrestore(&iommu->lock, flags); return DMA_ERROR_CODE; } static void dma_4v_unmap_single(struct device *dev, dma_addr_t bus_addr, size_t sz, enum dma_data_direction direction) { struct pci_pbm_info *pbm; struct iommu *iommu; unsigned long flags, npages; long entry; u32 devhandle; if (unlikely(direction == DMA_NONE)) { if (printk_ratelimit()) WARN_ON(1); return; } iommu = dev->archdata.iommu; pbm = dev->archdata.host_controller; devhandle = pbm->devhandle; npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); npages >>= IO_PAGE_SHIFT; bus_addr &= IO_PAGE_MASK; spin_lock_irqsave(&iommu->lock, flags); iommu_range_free(iommu, bus_addr, npages); entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; do { unsigned long num; num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), npages); entry += num; npages -= num; } while (npages != 0); spin_unlock_irqrestore(&iommu->lock, flags); } static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction) { struct scatterlist *s, *outs, *segstart; unsigned long flags, handle, prot; dma_addr_t dma_next = 0, dma_addr; unsigned int max_seg_size; int outcount, incount, i; struct iommu *iommu; long err; BUG_ON(direction == DMA_NONE); iommu = dev->archdata.iommu; if (nelems == 0 || !iommu) return 0; prot = HV_PCI_MAP_ATTR_READ; if (direction != DMA_TO_DEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; outs = s = segstart = &sglist[0]; outcount = 1; incount = nelems; handle = 0; /* Init first segment length for backout at failure */ outs->dma_length = 0; spin_lock_irqsave(&iommu->lock, flags); iommu_batch_start(dev, prot, ~0UL); max_seg_size = dma_get_max_seg_size(dev); for_each_sg(sglist, s, nelems, i) { unsigned long paddr, npages, entry, slen; slen = s->length; /* Sanity check */ if (slen == 0) { dma_next = 0; continue; } /* Allocate iommu entries for that segment */ paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); npages = iommu_num_pages(paddr, slen); entry = iommu_range_alloc(dev, iommu, npages, &handle); /* Handle failure */ if (unlikely(entry == DMA_ERROR_CODE)) { if (printk_ratelimit()) printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx" " npages %lx\n", iommu, paddr, npages); goto iommu_map_failed; } iommu_batch_new_entry(entry); /* Convert entry to a dma_addr_t */ dma_addr = iommu->page_table_map_base + (entry << IO_PAGE_SHIFT); dma_addr |= (s->offset & ~IO_PAGE_MASK); /* Insert into HW table */ paddr &= IO_PAGE_MASK; while (npages--) { err = iommu_batch_add(paddr); if (unlikely(err < 0L)) goto iommu_map_failed; paddr += IO_PAGE_SIZE; } /* If we are in an open segment, try merging */ if (segstart != s) { /* We cannot merge if: * - allocated dma_addr isn't contiguous to previous allocation */ if ((dma_addr != dma_next) || (outs->dma_length + s->length > max_seg_size)) { /* Can't merge: create a new segment */ segstart = s; outcount++; outs = sg_next(outs); } else { outs->dma_length += s->length; } } if (segstart == s) { /* This is a new segment, fill entries */ outs->dma_address = dma_addr; outs->dma_length = slen; } /* Calculate next page pointer for contiguous check */ dma_next = dma_addr + slen; } err = iommu_batch_end(); if (unlikely(err < 0L)) goto iommu_map_failed; spin_unlock_irqrestore(&iommu->lock, flags); if (outcount < incount) { outs = sg_next(outs); outs->dma_address = DMA_ERROR_CODE; outs->dma_length = 0; } return outcount; iommu_map_failed: for_each_sg(sglist, s, nelems, i) { if (s->dma_length != 0) { unsigned long vaddr, npages; vaddr = s->dma_address & IO_PAGE_MASK; npages = iommu_num_pages(s->dma_address, s->dma_length); iommu_range_free(iommu, vaddr, npages); /* XXX demap? XXX */ s->dma_address = DMA_ERROR_CODE; s->dma_length = 0; } if (s == outs) break; } spin_unlock_irqrestore(&iommu->lock, flags); return 0; } static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction) { struct pci_pbm_info *pbm; struct scatterlist *sg; struct iommu *iommu; unsigned long flags; u32 devhandle; BUG_ON(direction == DMA_NONE); iommu = dev->archdata.iommu; pbm = dev->archdata.host_controller; devhandle = pbm->devhandle; spin_lock_irqsave(&iommu->lock, flags); sg = sglist; while (nelems--) { dma_addr_t dma_handle = sg->dma_address; unsigned int len = sg->dma_length; unsigned long npages, entry; if (!len) break; npages = iommu_num_pages(dma_handle, len); iommu_range_free(iommu, dma_handle, npages); entry = ((dma_handle - iommu->page_table_map_base) >> IO_PAGE_SHIFT); while (npages) { unsigned long num; num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), npages); entry += num; npages -= num; } sg = sg_next(sg); } spin_unlock_irqrestore(&iommu->lock, flags); } static void dma_4v_sync_single_for_cpu(struct device *dev, dma_addr_t bus_addr, size_t sz, enum dma_data_direction direction) { /* Nothing to do... */ } static void dma_4v_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction) { /* Nothing to do... */ } const struct dma_ops sun4v_dma_ops = { .alloc_coherent = dma_4v_alloc_coherent, .free_coherent = dma_4v_free_coherent, .map_single = dma_4v_map_single, .unmap_single = dma_4v_unmap_single, .map_sg = dma_4v_map_sg, .unmap_sg = dma_4v_unmap_sg, .sync_single_for_cpu = dma_4v_sync_single_for_cpu, .sync_sg_for_cpu = dma_4v_sync_sg_for_cpu, }; static void __init pci_sun4v_scan_bus(struct pci_pbm_info *pbm) { struct property *prop; struct device_node *dp; dp = pbm->prom_node; prop = of_find_property(dp, "66mhz-capable", NULL); pbm->is_66mhz_capable = (prop != NULL); pbm->pci_bus = pci_scan_one_pbm(pbm); /* XXX register error interrupt handlers XXX */ } static unsigned long __init probe_existing_entries(struct pci_pbm_info *pbm,


context:
space:
mode: