aboutsummaryrefslogtreecommitdiffstats
path: root/fs/btrfs/extent_map.c
diff options
context:
space:
mode:
authorChris Mason <chris.mason@oracle.com>2007-10-15 16:14:37 -0400
committerChris Mason <chris.mason@oracle.com>2008-09-25 11:03:56 -0400
commit6d36dcd48f1e4e7446d603a3df9638bd314a182d (patch)
tree321f6f36de71adaab1754b113401e05d50cd5ebd /fs/btrfs/extent_map.c
parent479965d66e320f1a095bb76027171daa675a9c72 (diff)
Btrfs: Avoid memcpy where possible in extent_buffers
Signed-off-by: Chris Mason <chris.mason@oracle.com>
Diffstat (limited to 'fs/btrfs/extent_map.c')
-rw-r--r--fs/btrfs/extent_map.c132
1 files changed, 102 insertions, 30 deletions
diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c
index 8bef309e1b37..d2c733c68b4c 100644
--- a/fs/btrfs/extent_map.c
+++ b/fs/btrfs/extent_map.c
@@ -18,6 +18,11 @@ struct kmem_cache *btrfs_cache_create(const char *name, size_t size,
18 18
19static struct kmem_cache *extent_map_cache; 19static struct kmem_cache *extent_map_cache;
20static struct kmem_cache *extent_state_cache; 20static struct kmem_cache *extent_state_cache;
21static struct kmem_cache *extent_buffer_cache;
22static LIST_HEAD(extent_buffers);
23static spinlock_t extent_buffers_lock;
24static int nr_extent_buffers;
25#define MAX_EXTENT_BUFFER_CACHE 128
21 26
22struct tree_entry { 27struct tree_entry {
23 u64 start; 28 u64 start;
@@ -29,21 +34,33 @@ struct tree_entry {
29void __init extent_map_init(void) 34void __init extent_map_init(void)
30{ 35{
31 extent_map_cache = btrfs_cache_create("extent_map", 36 extent_map_cache = btrfs_cache_create("extent_map",
32 sizeof(struct extent_map), 37 sizeof(struct extent_map), 0,
33 SLAB_DESTROY_BY_RCU,
34 NULL); 38 NULL);
35 extent_state_cache = btrfs_cache_create("extent_state", 39 extent_state_cache = btrfs_cache_create("extent_state",
36 sizeof(struct extent_state), 40 sizeof(struct extent_state), 0,
37 SLAB_DESTROY_BY_RCU,
38 NULL); 41 NULL);
42 extent_buffer_cache = btrfs_cache_create("extent_buffers",
43 sizeof(struct extent_buffer), 0,
44 NULL);
45 spin_lock_init(&extent_buffers_lock);
39} 46}
40 47
41void __exit extent_map_exit(void) 48void __exit extent_map_exit(void)
42{ 49{
50 struct extent_buffer *eb;
51
52 while (!list_empty(&extent_buffers)) {
53 eb = list_entry(extent_buffers.next,
54 struct extent_buffer, list);
55 list_del(&eb->list);
56 kmem_cache_free(extent_buffer_cache, eb);
57 }
43 if (extent_map_cache) 58 if (extent_map_cache)
44 kmem_cache_destroy(extent_map_cache); 59 kmem_cache_destroy(extent_map_cache);
45 if (extent_state_cache) 60 if (extent_state_cache)
46 kmem_cache_destroy(extent_state_cache); 61 kmem_cache_destroy(extent_state_cache);
62 if (extent_buffer_cache)
63 kmem_cache_destroy(extent_buffer_cache);
47} 64}
48 65
49void extent_map_tree_init(struct extent_map_tree *tree, 66void extent_map_tree_init(struct extent_map_tree *tree,
@@ -1858,6 +1875,48 @@ sector_t extent_bmap(struct address_space *mapping, sector_t iblock,
1858 return (em->block_start + start - em->start) >> inode->i_blkbits; 1875 return (em->block_start + start - em->start) >> inode->i_blkbits;
1859} 1876}
1860 1877
1878static struct extent_buffer *__alloc_extent_buffer(gfp_t mask)
1879{
1880 struct extent_buffer *eb = NULL;
1881 spin_lock(&extent_buffers_lock);
1882 if (!list_empty(&extent_buffers)) {
1883 eb = list_entry(extent_buffers.next, struct extent_buffer,
1884 list);
1885 list_del(&eb->list);
1886 WARN_ON(nr_extent_buffers == 0);
1887 nr_extent_buffers--;
1888 }
1889 spin_unlock(&extent_buffers_lock);
1890 if (eb) {
1891 memset(eb, 0, sizeof(*eb));
1892 return eb;
1893 }
1894 return kmem_cache_zalloc(extent_buffer_cache, mask);
1895}
1896
1897static void __free_extent_buffer(struct extent_buffer *eb)
1898{
1899 if (nr_extent_buffers >= MAX_EXTENT_BUFFER_CACHE) {
1900 kmem_cache_free(extent_buffer_cache, eb);
1901 } else {
1902 spin_lock(&extent_buffers_lock);
1903 list_add(&eb->list, &extent_buffers);
1904 nr_extent_buffers++;
1905 spin_unlock(&extent_buffers_lock);
1906 }
1907}
1908
1909static inline struct page *extent_buffer_page(struct extent_buffer *eb, int i)
1910{
1911 struct page *p;
1912 if (i == 0)
1913 return eb->first_page;
1914 i += eb->start >> PAGE_CACHE_SHIFT;
1915 p = find_get_page(eb->first_page->mapping, i);
1916 page_cache_release(p);
1917 return p;
1918}
1919
1861struct extent_buffer *alloc_extent_buffer(struct extent_map_tree *tree, 1920struct extent_buffer *alloc_extent_buffer(struct extent_map_tree *tree,
1862 u64 start, unsigned long len, 1921 u64 start, unsigned long len,
1863 gfp_t mask) 1922 gfp_t mask)
@@ -1871,7 +1930,7 @@ struct extent_buffer *alloc_extent_buffer(struct extent_map_tree *tree,
1871 struct address_space *mapping = tree->mapping; 1930 struct address_space *mapping = tree->mapping;
1872 int uptodate = 0; 1931 int uptodate = 0;
1873 1932
1874 eb = kzalloc(EXTENT_BUFFER_SIZE(num_pages), mask); 1933 eb = __alloc_extent_buffer(mask);
1875 if (!eb || IS_ERR(eb)) 1934 if (!eb || IS_ERR(eb))
1876 return NULL; 1935 return NULL;
1877 1936
@@ -1881,9 +1940,16 @@ struct extent_buffer *alloc_extent_buffer(struct extent_map_tree *tree,
1881 1940
1882 for (i = 0; i < num_pages; i++, index++) { 1941 for (i = 0; i < num_pages; i++, index++) {
1883 p = find_or_create_page(mapping, index, mask | __GFP_HIGHMEM); 1942 p = find_or_create_page(mapping, index, mask | __GFP_HIGHMEM);
1884 if (!p) 1943 if (!p) {
1944 /* make sure the free only frees the pages we've
1945 * grabbed a reference on
1946 */
1947 eb->len = i << PAGE_CACHE_SHIFT;
1948 eb->start &= ~((u64)PAGE_CACHE_SIZE - 1);
1885 goto fail; 1949 goto fail;
1886 eb->pages[i] = p; 1950 }
1951 if (i == 0)
1952 eb->first_page = p;
1887 if (!PageUptodate(p)) 1953 if (!PageUptodate(p))
1888 uptodate = 0; 1954 uptodate = 0;
1889 unlock_page(p); 1955 unlock_page(p);
@@ -1909,7 +1975,7 @@ struct extent_buffer *find_extent_buffer(struct extent_map_tree *tree,
1909 struct page *p; 1975 struct page *p;
1910 struct address_space *mapping = tree->mapping; 1976 struct address_space *mapping = tree->mapping;
1911 1977
1912 eb = kzalloc(EXTENT_BUFFER_SIZE(num_pages), mask); 1978 eb = __alloc_extent_buffer(mask);
1913 if (!eb || IS_ERR(eb)) 1979 if (!eb || IS_ERR(eb))
1914 return NULL; 1980 return NULL;
1915 1981
@@ -1919,9 +1985,16 @@ struct extent_buffer *find_extent_buffer(struct extent_map_tree *tree,
1919 1985
1920 for (i = 0; i < num_pages; i++, index++) { 1986 for (i = 0; i < num_pages; i++, index++) {
1921 p = find_get_page(mapping, index); 1987 p = find_get_page(mapping, index);
1922 if (!p) 1988 if (!p) {
1989 /* make sure the free only frees the pages we've
1990 * grabbed a reference on
1991 */
1992 eb->len = i << PAGE_CACHE_SHIFT;
1993 eb->start &= ~((u64)PAGE_CACHE_SIZE - 1);
1923 goto fail; 1994 goto fail;
1924 eb->pages[i] = p; 1995 }
1996 if (i == 0)
1997 eb->first_page = p;
1925 } 1998 }
1926 return eb; 1999 return eb;
1927fail: 2000fail:
@@ -1944,11 +2017,12 @@ void free_extent_buffer(struct extent_buffer *eb)
1944 num_pages = ((eb->start + eb->len - 1) >> PAGE_CACHE_SHIFT) - 2017 num_pages = ((eb->start + eb->len - 1) >> PAGE_CACHE_SHIFT) -
1945 (eb->start >> PAGE_CACHE_SHIFT) + 1; 2018 (eb->start >> PAGE_CACHE_SHIFT) + 1;
1946 2019
1947 for (i = 0; i < num_pages; i++) { 2020 if (eb->first_page)
1948 if (eb->pages[i]) 2021 page_cache_release(eb->first_page);
1949 page_cache_release(eb->pages[i]); 2022 for (i = 1; i < num_pages; i++) {
2023 page_cache_release(extent_buffer_page(eb, i));
1950 } 2024 }
1951 kfree(eb); 2025 __free_extent_buffer(eb);
1952} 2026}
1953EXPORT_SYMBOL(free_extent_buffer); 2027EXPORT_SYMBOL(free_extent_buffer);
1954 2028
@@ -1968,7 +2042,7 @@ int clear_extent_buffer_dirty(struct extent_map_tree *tree,
1968 (eb->start >> PAGE_CACHE_SHIFT) + 1; 2042 (eb->start >> PAGE_CACHE_SHIFT) + 1;
1969 2043
1970 for (i = 0; i < num_pages; i++) { 2044 for (i = 0; i < num_pages; i++) {
1971 page = eb->pages[i]; 2045 page = extent_buffer_page(eb, i);
1972 lock_page(page); 2046 lock_page(page);
1973 /* 2047 /*
1974 * if we're on the last page or the first page and the 2048 * if we're on the last page or the first page and the
@@ -2021,7 +2095,7 @@ int set_extent_buffer_uptodate(struct extent_map_tree *tree,
2021 set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1, 2095 set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
2022 GFP_NOFS); 2096 GFP_NOFS);
2023 for (i = 0; i < num_pages; i++) { 2097 for (i = 0; i < num_pages; i++) {
2024 page = eb->pages[i]; 2098 page = extent_buffer_page(eb, i);
2025 if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) || 2099 if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
2026 ((i == num_pages - 1) && 2100 ((i == num_pages - 1) &&
2027 ((eb->start + eb->len - 1) & (PAGE_CACHE_SIZE - 1)))) { 2101 ((eb->start + eb->len - 1) & (PAGE_CACHE_SIZE - 1)))) {
@@ -2064,7 +2138,7 @@ int read_extent_buffer_pages(struct extent_map_tree *tree,
2064 num_pages = ((eb->start + eb->len - 1) >> PAGE_CACHE_SHIFT) - 2138 num_pages = ((eb->start + eb->len - 1) >> PAGE_CACHE_SHIFT) -
2065 (eb->start >> PAGE_CACHE_SHIFT) + 1; 2139 (eb->start >> PAGE_CACHE_SHIFT) + 1;
2066 for (i = 0; i < num_pages; i++) { 2140 for (i = 0; i < num_pages; i++) {
2067 page = eb->pages[i]; 2141 page = extent_buffer_page(eb, i);
2068 if (PageUptodate(page)) { 2142 if (PageUptodate(page)) {
2069 continue; 2143 continue;
2070 } 2144 }
@@ -2090,7 +2164,7 @@ int read_extent_buffer_pages(struct extent_map_tree *tree,
2090 } 2164 }
2091 2165
2092 for (i = 0; i < num_pages; i++) { 2166 for (i = 0; i < num_pages; i++) {
2093 page = eb->pages[i]; 2167 page = extent_buffer_page(eb, i);
2094 wait_on_page_locked(page); 2168 wait_on_page_locked(page);
2095 if (!PageUptodate(page)) { 2169 if (!PageUptodate(page)) {
2096 ret = -EIO; 2170 ret = -EIO;
@@ -2116,12 +2190,12 @@ void read_extent_buffer(struct extent_buffer *eb, void *dstv,
2116 WARN_ON(start > eb->len); 2190 WARN_ON(start > eb->len);
2117 WARN_ON(start + len > eb->start + eb->len); 2191 WARN_ON(start + len > eb->start + eb->len);
2118 2192
2119 page = eb->pages[i];
2120 offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1); 2193 offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1);
2121 if (i == 0) 2194 if (i == 0)
2122 offset += start_offset; 2195 offset += start_offset;
2123 2196
2124 while(len > 0) { 2197 while(len > 0) {
2198 page = extent_buffer_page(eb, i);
2125 WARN_ON(!PageUptodate(page)); 2199 WARN_ON(!PageUptodate(page));
2126 2200
2127 cur = min(len, (PAGE_CACHE_SIZE - offset)); 2201 cur = min(len, (PAGE_CACHE_SIZE - offset));
@@ -2134,7 +2208,6 @@ void read_extent_buffer(struct extent_buffer *eb, void *dstv,
2134 len -= cur; 2208 len -= cur;
2135 offset = 0; 2209 offset = 0;
2136 i++; 2210 i++;
2137 page = eb->pages[i];
2138 } 2211 }
2139} 2212}
2140EXPORT_SYMBOL(read_extent_buffer); 2213EXPORT_SYMBOL(read_extent_buffer);
@@ -2165,7 +2238,7 @@ int map_extent_buffer(struct extent_buffer *eb, unsigned long start,
2165 } 2238 }
2166 2239
2167 // kaddr = kmap_atomic(eb->pages[i], km); 2240 // kaddr = kmap_atomic(eb->pages[i], km);
2168 kaddr = page_address(eb->pages[i]); 2241 kaddr = page_address(extent_buffer_page(eb, i));
2169 *token = kaddr; 2242 *token = kaddr;
2170 *map = kaddr + offset; 2243 *map = kaddr + offset;
2171 *map_len = PAGE_CACHE_SIZE - offset; 2244 *map_len = PAGE_CACHE_SIZE - offset;
@@ -2195,12 +2268,12 @@ int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
2195 WARN_ON(start > eb->len); 2268 WARN_ON(start > eb->len);
2196 WARN_ON(start + len > eb->start + eb->len); 2269 WARN_ON(start + len > eb->start + eb->len);
2197 2270
2198 page = eb->pages[i];
2199 offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1); 2271 offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1);
2200 if (i == 0) 2272 if (i == 0)
2201 offset += start_offset; 2273 offset += start_offset;
2202 2274
2203 while(len > 0) { 2275 while(len > 0) {
2276 page = extent_buffer_page(eb, i);
2204 WARN_ON(!PageUptodate(page)); 2277 WARN_ON(!PageUptodate(page));
2205 2278
2206 cur = min(len, (PAGE_CACHE_SIZE - offset)); 2279 cur = min(len, (PAGE_CACHE_SIZE - offset));
@@ -2216,7 +2289,6 @@ int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
2216 len -= cur; 2289 len -= cur;
2217 offset = 0; 2290 offset = 0;
2218 i++; 2291 i++;
2219 page = eb->pages[i];
2220 } 2292 }
2221 return ret; 2293 return ret;
2222} 2294}
@@ -2236,12 +2308,12 @@ void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
2236 WARN_ON(start > eb->len); 2308 WARN_ON(start > eb->len);
2237 WARN_ON(start + len > eb->start + eb->len); 2309 WARN_ON(start + len > eb->start + eb->len);
2238 2310
2239 page = eb->pages[i];
2240 offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1); 2311 offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1);
2241 if (i == 0) 2312 if (i == 0)
2242 offset += start_offset; 2313 offset += start_offset;
2243 2314
2244 while(len > 0) { 2315 while(len > 0) {
2316 page = extent_buffer_page(eb, i);
2245 WARN_ON(!PageUptodate(page)); 2317 WARN_ON(!PageUptodate(page));
2246 2318
2247 cur = min(len, PAGE_CACHE_SIZE - offset); 2319 cur = min(len, PAGE_CACHE_SIZE - offset);
@@ -2254,7 +2326,6 @@ void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
2254 len -= cur; 2326 len -= cur;
2255 offset = 0; 2327 offset = 0;
2256 i++; 2328 i++;
2257 page = eb->pages[i];
2258 } 2329 }
2259} 2330}
2260EXPORT_SYMBOL(write_extent_buffer); 2331EXPORT_SYMBOL(write_extent_buffer);
@@ -2272,12 +2343,12 @@ void memset_extent_buffer(struct extent_buffer *eb, char c,
2272 WARN_ON(start > eb->len); 2343 WARN_ON(start > eb->len);
2273 WARN_ON(start + len > eb->start + eb->len); 2344 WARN_ON(start + len > eb->start + eb->len);
2274 2345
2275 page = eb->pages[i];
2276 offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1); 2346 offset = start & ((unsigned long)PAGE_CACHE_SIZE - 1);
2277 if (i == 0) 2347 if (i == 0)
2278 offset += start_offset; 2348 offset += start_offset;
2279 2349
2280 while(len > 0) { 2350 while(len > 0) {
2351 page = extent_buffer_page(eb, i);
2281 WARN_ON(!PageUptodate(page)); 2352 WARN_ON(!PageUptodate(page));
2282 2353
2283 cur = min(len, PAGE_CACHE_SIZE - offset); 2354 cur = min(len, PAGE_CACHE_SIZE - offset);
@@ -2289,7 +2360,6 @@ void memset_extent_buffer(struct extent_buffer *eb, char c,
2289 len -= cur; 2360 len -= cur;
2290 offset = 0; 2361 offset = 0;
2291 i++; 2362 i++;
2292 page = eb->pages[i];
2293 } 2363 }
2294} 2364}
2295EXPORT_SYMBOL(memset_extent_buffer); 2365EXPORT_SYMBOL(memset_extent_buffer);
@@ -2313,7 +2383,7 @@ void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
2313 offset += start_offset; 2383 offset += start_offset;
2314 2384
2315 while(len > 0) { 2385 while(len > 0) {
2316 page = dst->pages[i]; 2386 page = extent_buffer_page(dst, i);
2317 WARN_ON(!PageUptodate(page)); 2387 WARN_ON(!PageUptodate(page));
2318 2388
2319 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset)); 2389 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
@@ -2414,7 +2484,8 @@ void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
2414 cur = min(cur, (unsigned long)(PAGE_CACHE_SIZE - 2484 cur = min(cur, (unsigned long)(PAGE_CACHE_SIZE -
2415 dst_off_in_page)); 2485 dst_off_in_page));
2416 2486
2417 copy_pages(dst->pages[dst_i], dst->pages[src_i], 2487 copy_pages(extent_buffer_page(dst, dst_i),
2488 extent_buffer_page(dst, src_i),
2418 dst_off_in_page, src_off_in_page, cur); 2489 dst_off_in_page, src_off_in_page, cur);
2419 2490
2420 src_offset += cur; 2491 src_offset += cur;
@@ -2467,7 +2538,8 @@ void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
2467 cur = min(len, src_off_in_page + 1); 2538 cur = min(len, src_off_in_page + 1);
2468 cur = min(cur, dst_off_in_page + 1); 2539 cur = min(cur, dst_off_in_page + 1);
2469// printk("move pages orig dst %lu src %lu len %lu, this %lu %lu %lu\n", dst_offset, src_offset, len, dst_off_in_page - cur + 1, src_off_in_page - cur + 1, cur); 2540// printk("move pages orig dst %lu src %lu len %lu, this %lu %lu %lu\n", dst_offset, src_offset, len, dst_off_in_page - cur + 1, src_off_in_page - cur + 1, cur);
2470 move_pages(dst->pages[dst_i], dst->pages[src_i], 2541 move_pages(extent_buffer_page(dst, dst_i),
2542 extent_buffer_page(dst, src_i),
2471 dst_off_in_page - cur + 1, 2543 dst_off_in_page - cur + 1,
2472 src_off_in_page - cur + 1, cur); 2544 src_off_in_page - cur + 1, cur);
2473 2545