diff options
| -rw-r--r-- | include/linux/slub_def.h | 11 | ||||
| -rw-r--r-- | mm/slub.c | 33 |
2 files changed, 15 insertions, 29 deletions
diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 55695c8d2f8a..4ba59cfc1f75 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h | |||
| @@ -75,12 +75,6 @@ struct kmem_cache { | |||
| 75 | int offset; /* Free pointer offset. */ | 75 | int offset; /* Free pointer offset. */ |
| 76 | struct kmem_cache_order_objects oo; | 76 | struct kmem_cache_order_objects oo; |
| 77 | 77 | ||
| 78 | /* | ||
| 79 | * Avoid an extra cache line for UP, SMP and for the node local to | ||
| 80 | * struct kmem_cache. | ||
| 81 | */ | ||
| 82 | struct kmem_cache_node local_node; | ||
| 83 | |||
| 84 | /* Allocation and freeing of slabs */ | 78 | /* Allocation and freeing of slabs */ |
| 85 | struct kmem_cache_order_objects max; | 79 | struct kmem_cache_order_objects max; |
| 86 | struct kmem_cache_order_objects min; | 80 | struct kmem_cache_order_objects min; |
| @@ -102,6 +96,9 @@ struct kmem_cache { | |||
| 102 | */ | 96 | */ |
| 103 | int remote_node_defrag_ratio; | 97 | int remote_node_defrag_ratio; |
| 104 | struct kmem_cache_node *node[MAX_NUMNODES]; | 98 | struct kmem_cache_node *node[MAX_NUMNODES]; |
| 99 | #else | ||
| 100 | /* Avoid an extra cache line for UP */ | ||
| 101 | struct kmem_cache_node local_node; | ||
| 105 | #endif | 102 | #endif |
| 106 | }; | 103 | }; |
| 107 | 104 | ||
| @@ -140,7 +137,7 @@ struct kmem_cache { | |||
| 140 | #ifdef CONFIG_ZONE_DMA | 137 | #ifdef CONFIG_ZONE_DMA |
| 141 | #define SLUB_DMA __GFP_DMA | 138 | #define SLUB_DMA __GFP_DMA |
| 142 | /* Reserve extra caches for potential DMA use */ | 139 | /* Reserve extra caches for potential DMA use */ |
| 143 | #define KMALLOC_CACHES (2 * SLUB_PAGE_SHIFT - 6) | 140 | #define KMALLOC_CACHES (2 * SLUB_PAGE_SHIFT) |
| 144 | #else | 141 | #else |
| 145 | /* Disable DMA functionality */ | 142 | /* Disable DMA functionality */ |
| 146 | #define SLUB_DMA (__force gfp_t)0 | 143 | #define SLUB_DMA (__force gfp_t)0 |
| @@ -2137,7 +2137,7 @@ static void free_kmem_cache_nodes(struct kmem_cache *s) | |||
| 2137 | 2137 | ||
| 2138 | for_each_node_state(node, N_NORMAL_MEMORY) { | 2138 | for_each_node_state(node, N_NORMAL_MEMORY) { |
| 2139 | struct kmem_cache_node *n = s->node[node]; | 2139 | struct kmem_cache_node *n = s->node[node]; |
| 2140 | if (n && n != &s->local_node) | 2140 | if (n) |
| 2141 | kmem_cache_free(kmalloc_caches, n); | 2141 | kmem_cache_free(kmalloc_caches, n); |
| 2142 | s->node[node] = NULL; | 2142 | s->node[node] = NULL; |
| 2143 | } | 2143 | } |
| @@ -2146,33 +2146,22 @@ static void free_kmem_cache_nodes(struct kmem_cache *s) | |||
| 2146 | static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags) | 2146 | static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags) |
| 2147 | { | 2147 | { |
| 2148 | int node; | 2148 | int node; |
| 2149 | int local_node; | ||
| 2150 | |||
| 2151 | if (slab_state >= UP && (s < kmalloc_caches || | ||
| 2152 | s >= kmalloc_caches + KMALLOC_CACHES)) | ||
| 2153 | local_node = page_to_nid(virt_to_page(s)); | ||
| 2154 | else | ||
| 2155 | local_node = 0; | ||
| 2156 | 2149 | ||
| 2157 | for_each_node_state(node, N_NORMAL_MEMORY) { | 2150 | for_each_node_state(node, N_NORMAL_MEMORY) { |
| 2158 | struct kmem_cache_node *n; | 2151 | struct kmem_cache_node *n; |
| 2159 | 2152 | ||
| 2160 | if (local_node == node) | 2153 | if (slab_state == DOWN) { |
| 2161 | n = &s->local_node; | 2154 | early_kmem_cache_node_alloc(gfpflags, node); |
| 2162 | else { | 2155 | continue; |
| 2163 | if (slab_state == DOWN) { | 2156 | } |
| 2164 | early_kmem_cache_node_alloc(gfpflags, node); | 2157 | n = kmem_cache_alloc_node(kmalloc_caches, |
| 2165 | continue; | 2158 | gfpflags, node); |
| 2166 | } | ||
| 2167 | n = kmem_cache_alloc_node(kmalloc_caches, | ||
| 2168 | gfpflags, node); | ||
| 2169 | |||
| 2170 | if (!n) { | ||
| 2171 | free_kmem_cache_nodes(s); | ||
| 2172 | return 0; | ||
| 2173 | } | ||
| 2174 | 2159 | ||
| 2160 | if (!n) { | ||
| 2161 | free_kmem_cache_nodes(s); | ||
| 2162 | return 0; | ||
| 2175 | } | 2163 | } |
| 2164 | |||
| 2176 | s->node[node] = n; | 2165 | s->node[node] = n; |
| 2177 | init_kmem_cache_node(n, s); | 2166 | init_kmem_cache_node(n, s); |
| 2178 | } | 2167 | } |
