aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--mm/slub.c41
1 files changed, 22 insertions, 19 deletions
diff --git a/mm/slub.c b/mm/slub.c
index 837f932671a1..ab44368ed692 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1790,7 +1790,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
1790 unsigned long addr, struct kmem_cache_cpu *c) 1790 unsigned long addr, struct kmem_cache_cpu *c)
1791{ 1791{
1792 void **object; 1792 void **object;
1793 struct page *new; 1793 struct page *page;
1794#ifdef CONFIG_CMPXCHG_LOCAL 1794#ifdef CONFIG_CMPXCHG_LOCAL
1795 unsigned long flags; 1795 unsigned long flags;
1796 1796
@@ -1808,28 +1808,30 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
1808 /* We handle __GFP_ZERO in the caller */ 1808 /* We handle __GFP_ZERO in the caller */
1809 gfpflags &= ~__GFP_ZERO; 1809 gfpflags &= ~__GFP_ZERO;
1810 1810
1811 if (!c->page) 1811 page = c->page;
1812 if (!page)
1812 goto new_slab; 1813 goto new_slab;
1813 1814
1814 slab_lock(c->page); 1815 slab_lock(page);
1815 if (unlikely(!node_match(c, node))) 1816 if (unlikely(!node_match(c, node)))
1816 goto another_slab; 1817 goto another_slab;
1817 1818
1818 stat(s, ALLOC_REFILL); 1819 stat(s, ALLOC_REFILL);
1819 1820
1820load_freelist: 1821load_freelist:
1821 object = c->page->freelist; 1822 object = page->freelist;
1822 if (unlikely(!object)) 1823 if (unlikely(!object))
1823 goto another_slab; 1824 goto another_slab;
1824 if (kmem_cache_debug(s)) 1825 if (kmem_cache_debug(s))
1825 goto debug; 1826 goto debug;
1826 1827
1827 c->freelist = get_freepointer(s, object); 1828 c->freelist = get_freepointer(s, object);
1828 c->page->inuse = c->page->objects; 1829 page->inuse = page->objects;
1829 c->page->freelist = NULL; 1830 page->freelist = NULL;
1830 c->node = page_to_nid(c->page); 1831 c->node = page_to_nid(page);
1832
1831unlock_out: 1833unlock_out:
1832 slab_unlock(c->page); 1834 slab_unlock(page);
1833#ifdef CONFIG_CMPXCHG_LOCAL 1835#ifdef CONFIG_CMPXCHG_LOCAL
1834 c->tid = next_tid(c->tid); 1836 c->tid = next_tid(c->tid);
1835 local_irq_restore(flags); 1837 local_irq_restore(flags);
@@ -1841,9 +1843,9 @@ another_slab:
1841 deactivate_slab(s, c); 1843 deactivate_slab(s, c);
1842 1844
1843new_slab: 1845new_slab:
1844 new = get_partial(s, gfpflags, node); 1846 page = get_partial(s, gfpflags, node);
1845 if (new) { 1847 if (page) {
1846 c->page = new; 1848 c->page = page;
1847 stat(s, ALLOC_FROM_PARTIAL); 1849 stat(s, ALLOC_FROM_PARTIAL);
1848 goto load_freelist; 1850 goto load_freelist;
1849 } 1851 }
@@ -1852,19 +1854,20 @@ new_slab:
1852 if (gfpflags & __GFP_WAIT) 1854 if (gfpflags & __GFP_WAIT)
1853 local_irq_enable(); 1855 local_irq_enable();
1854 1856
1855 new = new_slab(s, gfpflags, node); 1857 page = new_slab(s, gfpflags, node);
1856 1858
1857 if (gfpflags & __GFP_WAIT) 1859 if (gfpflags & __GFP_WAIT)
1858 local_irq_disable(); 1860 local_irq_disable();
1859 1861
1860 if (new) { 1862 if (page) {
1861 c = __this_cpu_ptr(s->cpu_slab); 1863 c = __this_cpu_ptr(s->cpu_slab);
1862 stat(s, ALLOC_SLAB); 1864 stat(s, ALLOC_SLAB);
1863 if (c->page) 1865 if (c->page)
1864 flush_slab(s, c); 1866 flush_slab(s, c);
1865 slab_lock(new); 1867
1866 __SetPageSlubFrozen(new); 1868 slab_lock(page);
1867 c->page = new; 1869 __SetPageSlubFrozen(page);
1870 c->page = page;
1868 goto load_freelist; 1871 goto load_freelist;
1869 } 1872 }
1870 if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) 1873 if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
@@ -1874,11 +1877,11 @@ new_slab:
1874#endif 1877#endif
1875 return NULL; 1878 return NULL;
1876debug: 1879debug:
1877 if (!alloc_debug_processing(s, c->page, object, addr)) 1880 if (!alloc_debug_processing(s, page, object, addr))
1878 goto another_slab; 1881 goto another_slab;
1879 1882
1880 c->page->inuse++; 1883 page->inuse++;
1881 c->page->freelist = get_freepointer(s, object); 1884 page->freelist = get_freepointer(s, object);
1882 c->node = NUMA_NO_NODE; 1885 c->node = NUMA_NO_NODE;
1883 goto unlock_out; 1886 goto unlock_out;
1884} 1887}