diff options
-rw-r--r-- | mm/slub.c | 41 |
1 files changed, 22 insertions, 19 deletions
@@ -1790,7 +1790,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, | |||
1790 | unsigned long addr, struct kmem_cache_cpu *c) | 1790 | unsigned long addr, struct kmem_cache_cpu *c) |
1791 | { | 1791 | { |
1792 | void **object; | 1792 | void **object; |
1793 | struct page *new; | 1793 | struct page *page; |
1794 | #ifdef CONFIG_CMPXCHG_LOCAL | 1794 | #ifdef CONFIG_CMPXCHG_LOCAL |
1795 | unsigned long flags; | 1795 | unsigned long flags; |
1796 | 1796 | ||
@@ -1808,28 +1808,30 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, | |||
1808 | /* We handle __GFP_ZERO in the caller */ | 1808 | /* We handle __GFP_ZERO in the caller */ |
1809 | gfpflags &= ~__GFP_ZERO; | 1809 | gfpflags &= ~__GFP_ZERO; |
1810 | 1810 | ||
1811 | if (!c->page) | 1811 | page = c->page; |
1812 | if (!page) | ||
1812 | goto new_slab; | 1813 | goto new_slab; |
1813 | 1814 | ||
1814 | slab_lock(c->page); | 1815 | slab_lock(page); |
1815 | if (unlikely(!node_match(c, node))) | 1816 | if (unlikely(!node_match(c, node))) |
1816 | goto another_slab; | 1817 | goto another_slab; |
1817 | 1818 | ||
1818 | stat(s, ALLOC_REFILL); | 1819 | stat(s, ALLOC_REFILL); |
1819 | 1820 | ||
1820 | load_freelist: | 1821 | load_freelist: |
1821 | object = c->page->freelist; | 1822 | object = page->freelist; |
1822 | if (unlikely(!object)) | 1823 | if (unlikely(!object)) |
1823 | goto another_slab; | 1824 | goto another_slab; |
1824 | if (kmem_cache_debug(s)) | 1825 | if (kmem_cache_debug(s)) |
1825 | goto debug; | 1826 | goto debug; |
1826 | 1827 | ||
1827 | c->freelist = get_freepointer(s, object); | 1828 | c->freelist = get_freepointer(s, object); |
1828 | c->page->inuse = c->page->objects; | 1829 | page->inuse = page->objects; |
1829 | c->page->freelist = NULL; | 1830 | page->freelist = NULL; |
1830 | c->node = page_to_nid(c->page); | 1831 | c->node = page_to_nid(page); |
1832 | |||
1831 | unlock_out: | 1833 | unlock_out: |
1832 | slab_unlock(c->page); | 1834 | slab_unlock(page); |
1833 | #ifdef CONFIG_CMPXCHG_LOCAL | 1835 | #ifdef CONFIG_CMPXCHG_LOCAL |
1834 | c->tid = next_tid(c->tid); | 1836 | c->tid = next_tid(c->tid); |
1835 | local_irq_restore(flags); | 1837 | local_irq_restore(flags); |
@@ -1841,9 +1843,9 @@ another_slab: | |||
1841 | deactivate_slab(s, c); | 1843 | deactivate_slab(s, c); |
1842 | 1844 | ||
1843 | new_slab: | 1845 | new_slab: |
1844 | new = get_partial(s, gfpflags, node); | 1846 | page = get_partial(s, gfpflags, node); |
1845 | if (new) { | 1847 | if (page) { |
1846 | c->page = new; | 1848 | c->page = page; |
1847 | stat(s, ALLOC_FROM_PARTIAL); | 1849 | stat(s, ALLOC_FROM_PARTIAL); |
1848 | goto load_freelist; | 1850 | goto load_freelist; |
1849 | } | 1851 | } |
@@ -1852,19 +1854,20 @@ new_slab: | |||
1852 | if (gfpflags & __GFP_WAIT) | 1854 | if (gfpflags & __GFP_WAIT) |
1853 | local_irq_enable(); | 1855 | local_irq_enable(); |
1854 | 1856 | ||
1855 | new = new_slab(s, gfpflags, node); | 1857 | page = new_slab(s, gfpflags, node); |
1856 | 1858 | ||
1857 | if (gfpflags & __GFP_WAIT) | 1859 | if (gfpflags & __GFP_WAIT) |
1858 | local_irq_disable(); | 1860 | local_irq_disable(); |
1859 | 1861 | ||
1860 | if (new) { | 1862 | if (page) { |
1861 | c = __this_cpu_ptr(s->cpu_slab); | 1863 | c = __this_cpu_ptr(s->cpu_slab); |
1862 | stat(s, ALLOC_SLAB); | 1864 | stat(s, ALLOC_SLAB); |
1863 | if (c->page) | 1865 | if (c->page) |
1864 | flush_slab(s, c); | 1866 | flush_slab(s, c); |
1865 | slab_lock(new); | 1867 | |
1866 | __SetPageSlubFrozen(new); | 1868 | slab_lock(page); |
1867 | c->page = new; | 1869 | __SetPageSlubFrozen(page); |
1870 | c->page = page; | ||
1868 | goto load_freelist; | 1871 | goto load_freelist; |
1869 | } | 1872 | } |
1870 | if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) | 1873 | if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) |
@@ -1874,11 +1877,11 @@ new_slab: | |||
1874 | #endif | 1877 | #endif |
1875 | return NULL; | 1878 | return NULL; |
1876 | debug: | 1879 | debug: |
1877 | if (!alloc_debug_processing(s, c->page, object, addr)) | 1880 | if (!alloc_debug_processing(s, page, object, addr)) |
1878 | goto another_slab; | 1881 | goto another_slab; |
1879 | 1882 | ||
1880 | c->page->inuse++; | 1883 | page->inuse++; |
1881 | c->page->freelist = get_freepointer(s, object); | 1884 | page->freelist = get_freepointer(s, object); |
1882 | c->node = NUMA_NO_NODE; | 1885 | c->node = NUMA_NO_NODE; |
1883 | goto unlock_out; | 1886 | goto unlock_out; |
1884 | } | 1887 | } |