From 534acc057b5a08ec33fa57cdd2f5a09ef124e7f2 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Wed, 29 Jul 2009 15:04:18 -0700 Subject: lib: flexible array implementation Once a structure goes over PAGE_SIZE*2, we see occasional allocation failures. Some people have chosen to switch over to things like vmalloc() that will let them keep array-like access to such a large structures. But, vmalloc() has plenty of downsides. Here's an alternative. I think it's what Andrew was suggesting here: http://lkml.org/lkml/2009/7/2/518 I call it a flexible array. It does all of its work in PAGE_SIZE bits, so never does an order>0 allocation. The base level has PAGE_SIZE-2*sizeof(int) bytes of storage for pointers to the second level. So, with a 32-bit arch, you get about 4MB (4183112 bytes) of total storage when the objects pack nicely into a page. It is half that on 64-bit because the pointers are twice the size. There's a table detailing this in the code. There are kerneldocs for the functions, but here's an overview: flex_array_alloc() - dynamically allocate a base structure flex_array_free() - free the array and all of the second-level pages flex_array_free_parts() - free the second-level pages, but not the base (for static bases) flex_array_put() - copy into the array at the given index flex_array_get() - copy out of the array at the given index flex_array_prealloc() - preallocate the second-level pages between the given indexes to guarantee no allocs will occur at put() time. We could also potentially just pass the "element_size" into each of the API functions instead of storing it internally. That would get us one more base pointer on 32-bit. I've been testing this by running it in userspace. The header and patch that I've been using are here, as well as the little script I'm using to generate the size table which goes in the kerneldocs. http://sr71.net/~dave/linux/flexarray/ [akpm@linux-foundation.org: coding-style fixes] Signed-off-by: Dave Hansen Reviewed-by: KAMEZAWA Hiroyuki Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 269 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 lib/flex_array.c (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c new file mode 100644 index 000000000000..0e7894ce8882 --- /dev/null +++ b/lib/flex_array.c @@ -0,0 +1,269 @@ +/* + * Flexible array managed in PAGE_SIZE parts + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright IBM Corporation, 2009 + * + * Author: Dave Hansen + */ + +#include +#include +#include + +struct flex_array_part { + char elements[FLEX_ARRAY_PART_SIZE]; +}; + +static inline int __elements_per_part(int element_size) +{ + return FLEX_ARRAY_PART_SIZE / element_size; +} + +static inline int bytes_left_in_base(void) +{ + int element_offset = offsetof(struct flex_array, parts); + int bytes_left = FLEX_ARRAY_BASE_SIZE - element_offset; + return bytes_left; +} + +static inline int nr_base_part_ptrs(void) +{ + return bytes_left_in_base() / sizeof(struct flex_array_part *); +} + +/* + * If a user requests an allocation which is small + * enough, we may simply use the space in the + * flex_array->parts[] array to store the user + * data. + */ +static inline int elements_fit_in_base(struct flex_array *fa) +{ + int data_size = fa->element_size * fa->total_nr_elements; + if (data_size <= bytes_left_in_base()) + return 1; + return 0; +} + +/** + * flex_array_alloc - allocate a new flexible array + * @element_size: the size of individual elements in the array + * @total: total number of elements that this should hold + * + * Note: all locking must be provided by the caller. + * + * @total is used to size internal structures. If the user ever + * accesses any array indexes >=@total, it will produce errors. + * + * The maximum number of elements is defined as: the number of + * elements that can be stored in a page times the number of + * page pointers that we can fit in the base structure or (using + * integer math): + * + * (PAGE_SIZE/element_size) * (PAGE_SIZE-8)/sizeof(void *) + * + * Here's a table showing example capacities. Note that the maximum + * index that the get/put() functions is just nr_objects-1. This + * basically means that you get 4MB of storage on 32-bit and 2MB on + * 64-bit. + * + * + * Element size | Objects | Objects | + * PAGE_SIZE=4k | 32-bit | 64-bit | + * ---------------------------------| + * 1 bytes | 4186112 | 2093056 | + * 2 bytes | 2093056 | 1046528 | + * 3 bytes | 1395030 | 697515 | + * 4 bytes | 1046528 | 523264 | + * 32 bytes | 130816 | 65408 | + * 33 bytes | 126728 | 63364 | + * 2048 bytes | 2044 | 1022 | + * 2049 bytes | 1022 | 511 | + * void * | 1046528 | 261632 | + * + * Since 64-bit pointers are twice the size, we lose half the + * capacity in the base structure. Also note that no effort is made + * to efficiently pack objects across page boundaries. + */ +struct flex_array *flex_array_alloc(int element_size, int total, gfp_t flags) +{ + struct flex_array *ret; + int max_size = nr_base_part_ptrs() * __elements_per_part(element_size); + + /* max_size will end up 0 if element_size > PAGE_SIZE */ + if (total > max_size) + return NULL; + ret = kzalloc(sizeof(struct flex_array), flags); + if (!ret) + return NULL; + ret->element_size = element_size; + ret->total_nr_elements = total; + return ret; +} + +static int fa_element_to_part_nr(struct flex_array *fa, int element_nr) +{ + return element_nr / __elements_per_part(fa->element_size); +} + +/** + * flex_array_free_parts - just free the second-level pages + * @src: address of data to copy into the array + * @element_nr: index of the position in which to insert + * the new element. + * + * This is to be used in cases where the base 'struct flex_array' + * has been statically allocated and should not be free. + */ +void flex_array_free_parts(struct flex_array *fa) +{ + int part_nr; + int max_part = nr_base_part_ptrs(); + + if (elements_fit_in_base(fa)) + return; + for (part_nr = 0; part_nr < max_part; part_nr++) + kfree(fa->parts[part_nr]); +} + +void flex_array_free(struct flex_array *fa) +{ + flex_array_free_parts(fa); + kfree(fa); +} + +static int fa_index_inside_part(struct flex_array *fa, int element_nr) +{ + return element_nr % __elements_per_part(fa->element_size); +} + +static int index_inside_part(struct flex_array *fa, int element_nr) +{ + int part_offset = fa_index_inside_part(fa, element_nr); + return part_offset * fa->element_size; +} + +static struct flex_array_part * +__fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) +{ + struct flex_array_part *part = fa->parts[part_nr]; + if (!part) { + /* + * This leaves the part pages uninitialized + * and with potentially random data, just + * as if the user had kmalloc()'d the whole. + * __GFP_ZERO can be used to zero it. + */ + part = kmalloc(FLEX_ARRAY_PART_SIZE, flags); + if (!part) + return NULL; + fa->parts[part_nr] = part; + } + return part; +} + +/** + * flex_array_put - copy data into the array at @element_nr + * @src: address of data to copy into the array + * @element_nr: index of the position in which to insert + * the new element. + * + * Note that this *copies* the contents of @src into + * the array. If you are trying to store an array of + * pointers, make sure to pass in &ptr instead of ptr. + * + * Locking must be provided by the caller. + */ +int flex_array_put(struct flex_array *fa, int element_nr, void *src, gfp_t flags) +{ + int part_nr = fa_element_to_part_nr(fa, element_nr); + struct flex_array_part *part; + void *dst; + + if (element_nr >= fa->total_nr_elements) + return -ENOSPC; + if (elements_fit_in_base(fa)) + part = (struct flex_array_part *)&fa->parts[0]; + else + part = __fa_get_part(fa, part_nr, flags); + if (!part) + return -ENOMEM; + dst = &part->elements[index_inside_part(fa, element_nr)]; + memcpy(dst, src, fa->element_size); + return 0; +} + +/** + * flex_array_prealloc - guarantee that array space exists + * @start: index of first array element for which space is allocated + * @end: index of last (inclusive) element for which space is allocated + * + * This will guarantee that no future calls to flex_array_put() + * will allocate memory. It can be used if you are expecting to + * be holding a lock or in some atomic context while writing + * data into the array. + * + * Locking must be provided by the caller. + */ +int flex_array_prealloc(struct flex_array *fa, int start, int end, gfp_t flags) +{ + int start_part; + int end_part; + int part_nr; + struct flex_array_part *part; + + if (start >= fa->total_nr_elements || end >= fa->total_nr_elements) + return -ENOSPC; + if (elements_fit_in_base(fa)) + return 0; + start_part = fa_element_to_part_nr(fa, start); + end_part = fa_element_to_part_nr(fa, end); + for (part_nr = start_part; part_nr <= end_part; part_nr++) { + part = __fa_get_part(fa, part_nr, flags); + if (!part) + return -ENOMEM; + } + return 0; +} + +/** + * flex_array_get - pull data back out of the array + * @element_nr: index of the element to fetch from the array + * + * Returns a pointer to the data at index @element_nr. Note + * that this is a copy of the data that was passed in. If you + * are using this to store pointers, you'll get back &ptr. + * + * Locking must be provided by the caller. + */ +void *flex_array_get(struct flex_array *fa, int element_nr) +{ + int part_nr = fa_element_to_part_nr(fa, element_nr); + struct flex_array_part *part; + int index; + + if (element_nr >= fa->total_nr_elements) + return NULL; + if (!fa->parts[part_nr]) + return NULL; + if (elements_fit_in_base(fa)) + part = (struct flex_array_part *)&fa->parts[0]; + else + part = fa->parts[part_nr]; + index = index_inside_part(fa, element_nr); + return &part->elements[index_inside_part(fa, element_nr)]; +} -- cgit v1.2.2 From 07868201070d87484bd00610a4921e879be78746 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Tue, 4 Aug 2009 13:35:17 -0600 Subject: flex_array: remove unneeded index calculation flex_array_get() calculates an index value, then drops it on the floor; simply remove it. Signed-off-by: Jonathan Corbet Acked-by: Dave Hansen Signed-off-by: Linus Torvalds --- lib/flex_array.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index 0e7894ce8882..08f1636d296a 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -254,7 +254,6 @@ void *flex_array_get(struct flex_array *fa, int element_nr) { int part_nr = fa_element_to_part_nr(fa, element_nr); struct flex_array_part *part; - int index; if (element_nr >= fa->total_nr_elements) return NULL; @@ -264,6 +263,5 @@ void *flex_array_get(struct flex_array *fa, int element_nr) part = (struct flex_array_part *)&fa->parts[0]; else part = fa->parts[part_nr]; - index = index_inside_part(fa, element_nr); return &part->elements[index_inside_part(fa, element_nr)]; } -- cgit v1.2.2 From a30b595d2ca6d39e784a1bed5f2b35f3d7a03af7 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Wed, 26 Aug 2009 14:29:20 -0700 Subject: flex_array: fix get function for elements in base starting at non-zero If all array elements fit into the base structure and data is copied using flex_array_put() starting at a non-zero index, flex_array_get() will fail to return the data. This fixes the bug by only checking for NULL parts when all elements do not fit in the base structure when flex_array_get() is used. Otherwise, fa_element_to_part_nr() will always be 0 since there are no parts structures needed and such element may never have been put. Thus, it will remain NULL due to the kzalloc() of the base. Additionally, flex_array_put() now only checks for a NULL part when all elements do not fit in the base structure. This is otherwise unnecessary since the base structure is guaranteed to exist (or we would have already hit a NULL pointer). Signed-off-by: David Rientjes Acked-by: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index 08f1636d296a..e73c691aec36 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -198,10 +198,11 @@ int flex_array_put(struct flex_array *fa, int element_nr, void *src, gfp_t flags return -ENOSPC; if (elements_fit_in_base(fa)) part = (struct flex_array_part *)&fa->parts[0]; - else + else { part = __fa_get_part(fa, part_nr, flags); - if (!part) - return -ENOMEM; + if (!part) + return -ENOMEM; + } dst = &part->elements[index_inside_part(fa, element_nr)]; memcpy(dst, src, fa->element_size); return 0; @@ -257,11 +258,12 @@ void *flex_array_get(struct flex_array *fa, int element_nr) if (element_nr >= fa->total_nr_elements) return NULL; - if (!fa->parts[part_nr]) - return NULL; if (elements_fit_in_base(fa)) part = (struct flex_array_part *)&fa->parts[0]; - else + else { part = fa->parts[part_nr]; + if (!part) + return NULL; + } return &part->elements[index_inside_part(fa, element_nr)]; } -- cgit v1.2.2 From 105b6e8a74cac11cdf70903877593c7f202075cc Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Wed, 26 Aug 2009 14:29:20 -0700 Subject: flex_array: fix flex_array_free_parts comment flex_array_free_parts() does not take `src' or `element_nr' formals, so remove their respective comments. Signed-off-by: David Rientjes Acked-by: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index e73c691aec36..cf4e372cae90 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -122,9 +122,6 @@ static int fa_element_to_part_nr(struct flex_array *fa, int element_nr) /** * flex_array_free_parts - just free the second-level pages - * @src: address of data to copy into the array - * @element_nr: index of the position in which to insert - * the new element. * * This is to be used in cases where the base 'struct flex_array' * has been statically allocated and should not be free. -- cgit v1.2.2 From b62e408c05228f40e69bb38a48db8961cac6cd23 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Wed, 26 Aug 2009 14:29:22 -0700 Subject: flex_array: convert element_nr formals to unsigned It's problematic to allow signed element_nr's or total's to be passed as part of the flex array API. flex_array_alloc() allows total_nr_elements to be set to a negative quantity, which is obviously erroneous. flex_array_get() and flex_array_put() allows negative array indices in dereferencing an array part, which could address memory mapped before struct flex_array. The fix is to convert all existing element_nr formals to be qualified as unsigned. Existing checks to compare it to total_nr_elements or the max array size based on element_size need not be changed. Signed-off-by: David Rientjes Cc: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index cf4e372cae90..7baed2fc3bc8 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -99,7 +99,8 @@ static inline int elements_fit_in_base(struct flex_array *fa) * capacity in the base structure. Also note that no effort is made * to efficiently pack objects across page boundaries. */ -struct flex_array *flex_array_alloc(int element_size, int total, gfp_t flags) +struct flex_array *flex_array_alloc(int element_size, unsigned int total, + gfp_t flags) { struct flex_array *ret; int max_size = nr_base_part_ptrs() * __elements_per_part(element_size); @@ -115,7 +116,8 @@ struct flex_array *flex_array_alloc(int element_size, int total, gfp_t flags) return ret; } -static int fa_element_to_part_nr(struct flex_array *fa, int element_nr) +static int fa_element_to_part_nr(struct flex_array *fa, + unsigned int element_nr) { return element_nr / __elements_per_part(fa->element_size); } @@ -143,14 +145,12 @@ void flex_array_free(struct flex_array *fa) kfree(fa); } -static int fa_index_inside_part(struct flex_array *fa, int element_nr) +static unsigned int index_inside_part(struct flex_array *fa, + unsigned int element_nr) { - return element_nr % __elements_per_part(fa->element_size); -} + unsigned int part_offset; -static int index_inside_part(struct flex_array *fa, int element_nr) -{ - int part_offset = fa_index_inside_part(fa, element_nr); + part_offset = element_nr % __elements_per_part(fa->element_size); return part_offset * fa->element_size; } @@ -185,7 +185,8 @@ __fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) * * Locking must be provided by the caller. */ -int flex_array_put(struct flex_array *fa, int element_nr, void *src, gfp_t flags) +int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, + gfp_t flags) { int part_nr = fa_element_to_part_nr(fa, element_nr); struct flex_array_part *part; @@ -217,7 +218,8 @@ int flex_array_put(struct flex_array *fa, int element_nr, void *src, gfp_t flags * * Locking must be provided by the caller. */ -int flex_array_prealloc(struct flex_array *fa, int start, int end, gfp_t flags) +int flex_array_prealloc(struct flex_array *fa, unsigned int start, + unsigned int end, gfp_t flags) { int start_part; int end_part; @@ -248,7 +250,7 @@ int flex_array_prealloc(struct flex_array *fa, int start, int end, gfp_t flags) * * Locking must be provided by the caller. */ -void *flex_array_get(struct flex_array *fa, int element_nr) +void *flex_array_get(struct flex_array *fa, unsigned int element_nr) { int part_nr = fa_element_to_part_nr(fa, element_nr); struct flex_array_part *part; -- cgit v1.2.2 From e6de3988aa52debb25a427d085061f3bf1181d54 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 21 Sep 2009 17:04:30 -0700 Subject: flex_array: add flex_array_clear function Add a new function to the flex_array API: int flex_array_clear(struct flex_array *fa, unsigned int element_nr) This function will zero the element at element_nr in the flex_array. Although this is equivalent to using flex_array_put() and passing a pointer to zero'd memory, flex_array_clear() does not require such a pointer to memory that would most likely need to be allocated on the caller's stack which could be significantly large depending on element_size. Signed-off-by: David Rientjes Cc: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index 7baed2fc3bc8..b68f99be4080 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -206,6 +206,32 @@ int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, return 0; } +/** + * flex_array_clear - clear element in array at @element_nr + * @element_nr: index of the position to clear. + * + * Locking must be provided by the caller. + */ +int flex_array_clear(struct flex_array *fa, unsigned int element_nr) +{ + int part_nr = fa_element_to_part_nr(fa, element_nr); + struct flex_array_part *part; + void *dst; + + if (element_nr >= fa->total_nr_elements) + return -ENOSPC; + if (elements_fit_in_base(fa)) + part = (struct flex_array_part *)&fa->parts[0]; + else { + part = fa->parts[part_nr]; + if (!part) + return -EINVAL; + } + dst = &part->elements[index_inside_part(fa, element_nr)]; + memset(dst, 0, fa->element_size); + return 0; +} + /** * flex_array_prealloc - guarantee that array space exists * @start: index of first array element for which space is allocated -- cgit v1.2.2 From 19da3dd157f8db6fe727ff268dab4791d55a6371 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 21 Sep 2009 17:04:31 -0700 Subject: flex_array: poison free elements Newly initialized flex_array's and/or flex_array_part's are now poisoned with a new poison value, FLEX_ARRAY_FREE. It's value is similar to POISON_FREE used in the various slab allocators, but is different to distinguish between flex array's poisoned kmem and slab allocator poisoned kmem. This will allow us to identify flex_array_part's that only contain free elements (and free them with an addition to the flex_array API). This could also be extended in the future to identify `get' uses on elements that have not been `put'. If __GFP_ZERO is passed for a part's gfp mask, the poisoning is avoided. These elements are considered to be in-use since they have been initialized. Signed-off-by: David Rientjes Cc: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index b68f99be4080..e22d0e9776aa 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -113,6 +113,8 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, return NULL; ret->element_size = element_size; ret->total_nr_elements = total; + if (elements_fit_in_base(ret) && !(flags & __GFP_ZERO)) + memset(ret->parts[0], FLEX_ARRAY_FREE, bytes_left_in_base()); return ret; } @@ -159,15 +161,12 @@ __fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) { struct flex_array_part *part = fa->parts[part_nr]; if (!part) { - /* - * This leaves the part pages uninitialized - * and with potentially random data, just - * as if the user had kmalloc()'d the whole. - * __GFP_ZERO can be used to zero it. - */ - part = kmalloc(FLEX_ARRAY_PART_SIZE, flags); + part = kmalloc(sizeof(struct flex_array_part), flags); if (!part) return NULL; + if (!(flags & __GFP_ZERO)) + memset(part, FLEX_ARRAY_FREE, + sizeof(struct flex_array_part)); fa->parts[part_nr] = part; } return part; @@ -228,7 +227,7 @@ int flex_array_clear(struct flex_array *fa, unsigned int element_nr) return -EINVAL; } dst = &part->elements[index_inside_part(fa, element_nr)]; - memset(dst, 0, fa->element_size); + memset(dst, FLEX_ARRAY_FREE, fa->element_size); return 0; } -- cgit v1.2.2 From 4af5a2f770cc8575840ccb1514ec76ecb592985c Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 21 Sep 2009 17:04:31 -0700 Subject: flex_array: add flex_array_shrink function Add a new function to the flex_array API: int flex_array_shrink(struct flex_array *fa) This function will free all unused second-level pages. Since elements are now poisoned if they are not allocated with __GFP_ZERO, it's possible to identify parts that consist solely of unused elements. flex_array_shrink() returns the number of pages freed. Signed-off-by: David Rientjes Cc: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index e22d0e9776aa..1b03bb553410 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -291,3 +291,43 @@ void *flex_array_get(struct flex_array *fa, unsigned int element_nr) } return &part->elements[index_inside_part(fa, element_nr)]; } + +static int part_is_free(struct flex_array_part *part) +{ + int i; + + for (i = 0; i < sizeof(struct flex_array_part); i++) + if (part->elements[i] != FLEX_ARRAY_FREE) + return 0; + return 1; +} + +/** + * flex_array_shrink - free unused second-level pages + * + * Frees all second-level pages that consist solely of unused + * elements. Returns the number of pages freed. + * + * Locking must be provided by the caller. + */ +int flex_array_shrink(struct flex_array *fa) +{ + struct flex_array_part *part; + int max_part = nr_base_part_ptrs(); + int part_nr; + int ret = 0; + + if (elements_fit_in_base(fa)) + return ret; + for (part_nr = 0; part_nr < max_part; part_nr++) { + part = fa->parts[part_nr]; + if (!part) + continue; + if (part_is_free(part)) { + fa->parts[part_nr] = NULL; + kfree(part); + ret++; + } + } + return ret; +} -- cgit v1.2.2 From 45b588d6e5cc172704bac0c998ce54873b149b22 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 21 Sep 2009 17:04:33 -0700 Subject: flex_array: introduce DEFINE_FLEX_ARRAY FLEX_ARRAY_INIT(element_size, total_nr_elements) cannot determine if either parameter is valid, so flex arrays which are statically allocated with this interface can easily become corrupted or reference beyond its allocated memory. This removes FLEX_ARRAY_INIT() as a struct flex_array initializer since no initializer may perform the required checking. Instead, the array is now defined with a new interface: DEFINE_FLEX_ARRAY(name, element_size, total_nr_elements) This may be prefixed with `static' for file scope. This interface includes compile-time checking of the parameters to ensure they are valid. Since the validity of both element_size and total_nr_elements depend on FLEX_ARRAY_BASE_SIZE and FLEX_ARRAY_PART_SIZE, the kernel build will fail if either of these predefined values changes such that the array parameters are no longer valid. Since BUILD_BUG_ON() requires compile time constants, several of the static inline functions that were once local to lib/flex_array.c had to be moved to include/linux/flex_array.h. Signed-off-by: David Rientjes Acked-by: Dave Hansen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index 1b03bb553410..b62ce6cffd0a 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -28,23 +28,6 @@ struct flex_array_part { char elements[FLEX_ARRAY_PART_SIZE]; }; -static inline int __elements_per_part(int element_size) -{ - return FLEX_ARRAY_PART_SIZE / element_size; -} - -static inline int bytes_left_in_base(void) -{ - int element_offset = offsetof(struct flex_array, parts); - int bytes_left = FLEX_ARRAY_BASE_SIZE - element_offset; - return bytes_left; -} - -static inline int nr_base_part_ptrs(void) -{ - return bytes_left_in_base() / sizeof(struct flex_array_part *); -} - /* * If a user requests an allocation which is small * enough, we may simply use the space in the @@ -54,7 +37,7 @@ static inline int nr_base_part_ptrs(void) static inline int elements_fit_in_base(struct flex_array *fa) { int data_size = fa->element_size * fa->total_nr_elements; - if (data_size <= bytes_left_in_base()) + if (data_size <= FLEX_ARRAY_BASE_BYTES_LEFT) return 1; return 0; } @@ -103,7 +86,8 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, gfp_t flags) { struct flex_array *ret; - int max_size = nr_base_part_ptrs() * __elements_per_part(element_size); + int max_size = FLEX_ARRAY_NR_BASE_PTRS * + FLEX_ARRAY_ELEMENTS_PER_PART(element_size); /* max_size will end up 0 if element_size > PAGE_SIZE */ if (total > max_size) @@ -114,14 +98,15 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, ret->element_size = element_size; ret->total_nr_elements = total; if (elements_fit_in_base(ret) && !(flags & __GFP_ZERO)) - memset(ret->parts[0], FLEX_ARRAY_FREE, bytes_left_in_base()); + memset(ret->parts[0], FLEX_ARRAY_FREE, + FLEX_ARRAY_BASE_BYTES_LEFT); return ret; } static int fa_element_to_part_nr(struct flex_array *fa, unsigned int element_nr) { - return element_nr / __elements_per_part(fa->element_size); + return element_nr / FLEX_ARRAY_ELEMENTS_PER_PART(fa->element_size); } /** @@ -133,11 +118,10 @@ static int fa_element_to_part_nr(struct flex_array *fa, void flex_array_free_parts(struct flex_array *fa) { int part_nr; - int max_part = nr_base_part_ptrs(); if (elements_fit_in_base(fa)) return; - for (part_nr = 0; part_nr < max_part; part_nr++) + for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) kfree(fa->parts[part_nr]); } @@ -152,7 +136,8 @@ static unsigned int index_inside_part(struct flex_array *fa, { unsigned int part_offset; - part_offset = element_nr % __elements_per_part(fa->element_size); + part_offset = element_nr % + FLEX_ARRAY_ELEMENTS_PER_PART(fa->element_size); return part_offset * fa->element_size; } @@ -313,13 +298,12 @@ static int part_is_free(struct flex_array_part *part) int flex_array_shrink(struct flex_array *fa) { struct flex_array_part *part; - int max_part = nr_base_part_ptrs(); int part_nr; int ret = 0; if (elements_fit_in_base(fa)) return ret; - for (part_nr = 0; part_nr < max_part; part_nr++) { + for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) { part = fa->parts[part_nr]; if (!part) continue; -- cgit v1.2.2 From fc0d8d944df0c58cd810f33db82f87dcf5dcc190 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Mon, 21 Sep 2009 17:04:33 -0700 Subject: flex_array: add missing kerneldoc annotations Add kerneldoc annotations for function formals of type struct flex_array and gfp_t which are currently lacking. Signed-off-by: David Rientjes Cc: Dave Hansen Cc: Randy Dunlap Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index b62ce6cffd0a..66eef2e4483e 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -46,6 +46,7 @@ static inline int elements_fit_in_base(struct flex_array *fa) * flex_array_alloc - allocate a new flexible array * @element_size: the size of individual elements in the array * @total: total number of elements that this should hold + * @flags: page allocation flags to use for base array * * Note: all locking must be provided by the caller. * @@ -111,6 +112,7 @@ static int fa_element_to_part_nr(struct flex_array *fa, /** * flex_array_free_parts - just free the second-level pages + * @fa: the flex array from which to free parts * * This is to be used in cases where the base 'struct flex_array' * has been statically allocated and should not be free. @@ -159,9 +161,12 @@ __fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) /** * flex_array_put - copy data into the array at @element_nr - * @src: address of data to copy into the array + * @fa: the flex array to copy data into * @element_nr: index of the position in which to insert * the new element. + * @src: address of data to copy into the array + * @flags: page allocation flags to use for array expansion + * * * Note that this *copies* the contents of @src into * the array. If you are trying to store an array of @@ -192,6 +197,7 @@ int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, /** * flex_array_clear - clear element in array at @element_nr + * @fa: the flex array of the element. * @element_nr: index of the position to clear. * * Locking must be provided by the caller. @@ -218,8 +224,10 @@ int flex_array_clear(struct flex_array *fa, unsigned int element_nr) /** * flex_array_prealloc - guarantee that array space exists + * @fa: the flex array for which to preallocate parts * @start: index of first array element for which space is allocated * @end: index of last (inclusive) element for which space is allocated + * @flags: page allocation flags * * This will guarantee that no future calls to flex_array_put() * will allocate memory. It can be used if you are expecting to @@ -252,6 +260,7 @@ int flex_array_prealloc(struct flex_array *fa, unsigned int start, /** * flex_array_get - pull data back out of the array + * @fa: the flex array from which to extract data * @element_nr: index of the element to fetch from the array * * Returns a pointer to the data at index @element_nr. Note @@ -289,6 +298,7 @@ static int part_is_free(struct flex_array_part *part) /** * flex_array_shrink - free unused second-level pages + * @fa: the flex array to shrink * * Frees all second-level pages that consist solely of unused * elements. Returns the number of pages freed. -- cgit v1.2.2 From e59464c735db19619cde2aa331609adb02005f5b Mon Sep 17 00:00:00 2001 From: Changli Gao Date: Fri, 23 Apr 2010 13:17:45 -0400 Subject: flex_array: fix the panic when calling flex_array_alloc() without __GFP_ZERO memset() is called with the wrong address and the kernel panics. Signed-off-by: Changli Gao Cc: Patrick McHardy Acked-by: David Rientjes Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index 66eef2e4483e..41b1804fa728 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -99,7 +99,7 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, ret->element_size = element_size; ret->total_nr_elements = total; if (elements_fit_in_base(ret) && !(flags & __GFP_ZERO)) - memset(ret->parts[0], FLEX_ARRAY_FREE, + memset(&ret->parts[0], FLEX_ARRAY_FREE, FLEX_ARRAY_BASE_BYTES_LEFT); return ret; } -- cgit v1.2.2 From ea98eed9bcb62d1319db8b1210712c6a110a886c Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Mon, 9 Aug 2010 17:20:56 -0700 Subject: flex_array: add helpers to get and put to make pointers easy to use Getting and putting arrays of pointers with flex arrays is a PITA. You have to remember to pass &ptr to the _put and you have to do weird and wacky casting to get the ptr back from the _get. Add two functions flex_array_get_ptr() and flex_array_put_ptr() to handle all of the magic. [akpm@linux-foundation.org: simplification suggested by Joe] Signed-off-by: Eric Paris Cc: David Rientjes Cc: Dave Hansen Cc: Joe Perches Cc: James Morris Cc: Joe Perches Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/flex_array.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index 41b1804fa728..77a6fea7481e 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -171,6 +171,8 @@ __fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) * Note that this *copies* the contents of @src into * the array. If you are trying to store an array of * pointers, make sure to pass in &ptr instead of ptr. + * You may instead wish to use the flex_array_put_ptr() + * helper function. * * Locking must be provided by the caller. */ @@ -265,7 +267,8 @@ int flex_array_prealloc(struct flex_array *fa, unsigned int start, * * Returns a pointer to the data at index @element_nr. Note * that this is a copy of the data that was passed in. If you - * are using this to store pointers, you'll get back &ptr. + * are using this to store pointers, you'll get back &ptr. You + * may instead wish to use the flex_array_get_ptr helper. * * Locking must be provided by the caller. */ @@ -286,6 +289,26 @@ void *flex_array_get(struct flex_array *fa, unsigned int element_nr) return &part->elements[index_inside_part(fa, element_nr)]; } +/** + * flex_array_get_ptr - pull a ptr back out of the array + * @fa: the flex array from which to extract data + * @element_nr: index of the element to fetch from the array + * + * Returns the pointer placed in the flex array at element_nr using + * flex_array_put_ptr(). This function should not be called if the + * element in question was not set using the _put_ptr() helper. + */ +void *flex_array_get_ptr(struct flex_array *fa, unsigned int element_nr) +{ + void **tmp; + + tmp = flex_array_get(fa, element_nr); + if (!tmp) + return NULL; + + return *tmp; +} + static int part_is_free(struct flex_array_part *part) { int i; -- cgit v1.2.2