diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/Kconfig.debug | 19 | ||||
| -rw-r--r-- | lib/Makefile | 1 | ||||
| -rw-r--r-- | lib/klist.c | 26 | ||||
| -rw-r--r-- | lib/radix-tree.c | 176 | ||||
| -rw-r--r-- | lib/semaphore-sleepers.c | 177 |
5 files changed, 309 insertions, 90 deletions
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 299f7f3b5b08..3754c9a8f5c8 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug | |||
| @@ -46,6 +46,25 @@ config LOG_BUF_SHIFT | |||
| 46 | 13 => 8 KB | 46 | 13 => 8 KB |
| 47 | 12 => 4 KB | 47 | 12 => 4 KB |
| 48 | 48 | ||
| 49 | config DETECT_SOFTLOCKUP | ||
| 50 | bool "Detect Soft Lockups" | ||
| 51 | depends on DEBUG_KERNEL | ||
| 52 | default y | ||
| 53 | help | ||
| 54 | Say Y here to enable the kernel to detect "soft lockups", | ||
| 55 | which are bugs that cause the kernel to loop in kernel | ||
| 56 | mode for more than 10 seconds, without giving other tasks a | ||
| 57 | chance to run. | ||
| 58 | |||
| 59 | When a soft-lockup is detected, the kernel will print the | ||
| 60 | current stack trace (which you should report), but the | ||
| 61 | system will stay locked up. This feature has negligible | ||
| 62 | overhead. | ||
| 63 | |||
| 64 | (Note that "hard lockups" are separate type of bugs that | ||
| 65 | can be detected via the NMI-watchdog, on platforms that | ||
| 66 | support it.) | ||
| 67 | |||
| 49 | config SCHEDSTATS | 68 | config SCHEDSTATS |
| 50 | bool "Collect scheduler statistics" | 69 | bool "Collect scheduler statistics" |
| 51 | depends on DEBUG_KERNEL && PROC_FS | 70 | depends on DEBUG_KERNEL && PROC_FS |
diff --git a/lib/Makefile b/lib/Makefile index 52f83380f704..3e2bd0df23bb 100644 --- a/lib/Makefile +++ b/lib/Makefile | |||
| @@ -18,6 +18,7 @@ endif | |||
| 18 | 18 | ||
| 19 | lib-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o | 19 | lib-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o |
| 20 | lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o | 20 | lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o |
| 21 | lib-$(CONFIG_SEMAPHORE_SLEEPERS) += semaphore-sleepers.o | ||
| 21 | lib-$(CONFIG_GENERIC_FIND_NEXT_BIT) += find_next_bit.o | 22 | lib-$(CONFIG_GENERIC_FIND_NEXT_BIT) += find_next_bit.o |
| 22 | obj-$(CONFIG_LOCK_KERNEL) += kernel_lock.o | 23 | obj-$(CONFIG_LOCK_KERNEL) += kernel_lock.o |
| 23 | obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o | 24 | obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o |
diff --git a/lib/klist.c b/lib/klist.c index 738ab810160a..bb2f3551d50a 100644 --- a/lib/klist.c +++ b/lib/klist.c | |||
| @@ -42,12 +42,23 @@ | |||
| 42 | /** | 42 | /** |
| 43 | * klist_init - Initialize a klist structure. | 43 | * klist_init - Initialize a klist structure. |
| 44 | * @k: The klist we're initializing. | 44 | * @k: The klist we're initializing. |
| 45 | * @get: The get function for the embedding object (NULL if none) | ||
| 46 | * @put: The put function for the embedding object (NULL if none) | ||
| 47 | * | ||
| 48 | * Initialises the klist structure. If the klist_node structures are | ||
| 49 | * going to be embedded in refcounted objects (necessary for safe | ||
| 50 | * deletion) then the get/put arguments are used to initialise | ||
| 51 | * functions that take and release references on the embedding | ||
| 52 | * objects. | ||
| 45 | */ | 53 | */ |
| 46 | 54 | ||
| 47 | void klist_init(struct klist * k) | 55 | void klist_init(struct klist * k, void (*get)(struct klist_node *), |
| 56 | void (*put)(struct klist_node *)) | ||
| 48 | { | 57 | { |
| 49 | INIT_LIST_HEAD(&k->k_list); | 58 | INIT_LIST_HEAD(&k->k_list); |
| 50 | spin_lock_init(&k->k_lock); | 59 | spin_lock_init(&k->k_lock); |
| 60 | k->get = get; | ||
| 61 | k->put = put; | ||
| 51 | } | 62 | } |
| 52 | 63 | ||
| 53 | EXPORT_SYMBOL_GPL(klist_init); | 64 | EXPORT_SYMBOL_GPL(klist_init); |
| @@ -74,16 +85,18 @@ static void klist_node_init(struct klist * k, struct klist_node * n) | |||
| 74 | init_completion(&n->n_removed); | 85 | init_completion(&n->n_removed); |
| 75 | kref_init(&n->n_ref); | 86 | kref_init(&n->n_ref); |
| 76 | n->n_klist = k; | 87 | n->n_klist = k; |
| 88 | if (k->get) | ||
| 89 | k->get(n); | ||
| 77 | } | 90 | } |
| 78 | 91 | ||
| 79 | 92 | ||
| 80 | /** | 93 | /** |
| 81 | * klist_add_head - Initialize a klist_node and add it to front. | 94 | * klist_add_head - Initialize a klist_node and add it to front. |
| 82 | * @k: klist it's going on. | ||
| 83 | * @n: node we're adding. | 95 | * @n: node we're adding. |
| 96 | * @k: klist it's going on. | ||
| 84 | */ | 97 | */ |
| 85 | 98 | ||
| 86 | void klist_add_head(struct klist * k, struct klist_node * n) | 99 | void klist_add_head(struct klist_node * n, struct klist * k) |
| 87 | { | 100 | { |
| 88 | klist_node_init(k, n); | 101 | klist_node_init(k, n); |
| 89 | add_head(k, n); | 102 | add_head(k, n); |
| @@ -94,11 +107,11 @@ EXPORT_SYMBOL_GPL(klist_add_head); | |||
| 94 | 107 | ||
| 95 | /** | 108 | /** |
| 96 | * klist_add_tail - Initialize a klist_node and add it to back. | 109 | * klist_add_tail - Initialize a klist_node and add it to back. |
| 97 | * @k: klist it's going on. | ||
| 98 | * @n: node we're adding. | 110 | * @n: node we're adding. |
| 111 | * @k: klist it's going on. | ||
| 99 | */ | 112 | */ |
| 100 | 113 | ||
| 101 | void klist_add_tail(struct klist * k, struct klist_node * n) | 114 | void klist_add_tail(struct klist_node * n, struct klist * k) |
| 102 | { | 115 | { |
| 103 | klist_node_init(k, n); | 116 | klist_node_init(k, n); |
| 104 | add_tail(k, n); | 117 | add_tail(k, n); |
| @@ -110,9 +123,12 @@ EXPORT_SYMBOL_GPL(klist_add_tail); | |||
| 110 | static void klist_release(struct kref * kref) | 123 | static void klist_release(struct kref * kref) |
| 111 | { | 124 | { |
| 112 | struct klist_node * n = container_of(kref, struct klist_node, n_ref); | 125 | struct klist_node * n = container_of(kref, struct klist_node, n_ref); |
| 126 | void (*put)(struct klist_node *) = n->n_klist->put; | ||
| 113 | list_del(&n->n_node); | 127 | list_del(&n->n_node); |
| 114 | complete(&n->n_removed); | 128 | complete(&n->n_removed); |
| 115 | n->n_klist = NULL; | 129 | n->n_klist = NULL; |
| 130 | if (put) | ||
| 131 | put(n); | ||
| 116 | } | 132 | } |
| 117 | 133 | ||
| 118 | static int klist_dec_and_del(struct klist_node * n) | 134 | static int klist_dec_and_del(struct klist_node * n) |
diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 10bed1c8c3c3..b972dd29289d 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c | |||
| @@ -1,6 +1,7 @@ | |||
| 1 | /* | 1 | /* |
| 2 | * Copyright (C) 2001 Momchil Velikov | 2 | * Copyright (C) 2001 Momchil Velikov |
| 3 | * Portions Copyright (C) 2001 Christoph Hellwig | 3 | * Portions Copyright (C) 2001 Christoph Hellwig |
| 4 | * Copyright (C) 2005 SGI, Christoph Lameter <clameter@sgi.com> | ||
| 4 | * | 5 | * |
| 5 | * This program is free software; you can redistribute it and/or | 6 | * This program is free software; you can redistribute it and/or |
| 6 | * modify it under the terms of the GNU General Public License as | 7 | * modify it under the terms of the GNU General Public License as |
| @@ -51,7 +52,7 @@ struct radix_tree_node { | |||
| 51 | }; | 52 | }; |
| 52 | 53 | ||
| 53 | struct radix_tree_path { | 54 | struct radix_tree_path { |
| 54 | struct radix_tree_node *node, **slot; | 55 | struct radix_tree_node *node; |
| 55 | int offset; | 56 | int offset; |
| 56 | }; | 57 | }; |
| 57 | 58 | ||
| @@ -227,7 +228,7 @@ out: | |||
| 227 | int radix_tree_insert(struct radix_tree_root *root, | 228 | int radix_tree_insert(struct radix_tree_root *root, |
| 228 | unsigned long index, void *item) | 229 | unsigned long index, void *item) |
| 229 | { | 230 | { |
| 230 | struct radix_tree_node *node = NULL, *tmp, **slot; | 231 | struct radix_tree_node *node = NULL, *slot; |
| 231 | unsigned int height, shift; | 232 | unsigned int height, shift; |
| 232 | int offset; | 233 | int offset; |
| 233 | int error; | 234 | int error; |
| @@ -240,38 +241,42 @@ int radix_tree_insert(struct radix_tree_root *root, | |||
| 240 | return error; | 241 | return error; |
| 241 | } | 242 | } |
| 242 | 243 | ||
| 243 | slot = &root->rnode; | 244 | slot = root->rnode; |
| 244 | height = root->height; | 245 | height = root->height; |
| 245 | shift = (height-1) * RADIX_TREE_MAP_SHIFT; | 246 | shift = (height-1) * RADIX_TREE_MAP_SHIFT; |
| 246 | 247 | ||
| 247 | offset = 0; /* uninitialised var warning */ | 248 | offset = 0; /* uninitialised var warning */ |
| 248 | while (height > 0) { | 249 | while (height > 0) { |
| 249 | if (*slot == NULL) { | 250 | if (slot == NULL) { |
| 250 | /* Have to add a child node. */ | 251 | /* Have to add a child node. */ |
| 251 | if (!(tmp = radix_tree_node_alloc(root))) | 252 | if (!(slot = radix_tree_node_alloc(root))) |
| 252 | return -ENOMEM; | 253 | return -ENOMEM; |
| 253 | *slot = tmp; | 254 | if (node) { |
| 254 | if (node) | 255 | node->slots[offset] = slot; |
| 255 | node->count++; | 256 | node->count++; |
| 257 | } else | ||
| 258 | root->rnode = slot; | ||
| 256 | } | 259 | } |
| 257 | 260 | ||
| 258 | /* Go a level down */ | 261 | /* Go a level down */ |
| 259 | offset = (index >> shift) & RADIX_TREE_MAP_MASK; | 262 | offset = (index >> shift) & RADIX_TREE_MAP_MASK; |
| 260 | node = *slot; | 263 | node = slot; |
| 261 | slot = (struct radix_tree_node **)(node->slots + offset); | 264 | slot = node->slots[offset]; |
| 262 | shift -= RADIX_TREE_MAP_SHIFT; | 265 | shift -= RADIX_TREE_MAP_SHIFT; |
| 263 | height--; | 266 | height--; |
| 264 | } | 267 | } |
| 265 | 268 | ||
| 266 | if (*slot != NULL) | 269 | if (slot != NULL) |
| 267 | return -EEXIST; | 270 | return -EEXIST; |
| 271 | |||
| 268 | if (node) { | 272 | if (node) { |
| 269 | node->count++; | 273 | node->count++; |
| 274 | node->slots[offset] = item; | ||
| 270 | BUG_ON(tag_get(node, 0, offset)); | 275 | BUG_ON(tag_get(node, 0, offset)); |
| 271 | BUG_ON(tag_get(node, 1, offset)); | 276 | BUG_ON(tag_get(node, 1, offset)); |
| 272 | } | 277 | } else |
| 278 | root->rnode = item; | ||
| 273 | 279 | ||
| 274 | *slot = item; | ||
| 275 | return 0; | 280 | return 0; |
| 276 | } | 281 | } |
| 277 | EXPORT_SYMBOL(radix_tree_insert); | 282 | EXPORT_SYMBOL(radix_tree_insert); |
| @@ -286,27 +291,25 @@ EXPORT_SYMBOL(radix_tree_insert); | |||
| 286 | void *radix_tree_lookup(struct radix_tree_root *root, unsigned long index) | 291 | void *radix_tree_lookup(struct radix_tree_root *root, unsigned long index) |
| 287 | { | 292 | { |
| 288 | unsigned int height, shift; | 293 | unsigned int height, shift; |
| 289 | struct radix_tree_node **slot; | 294 | struct radix_tree_node *slot; |
| 290 | 295 | ||
| 291 | height = root->height; | 296 | height = root->height; |
| 292 | if (index > radix_tree_maxindex(height)) | 297 | if (index > radix_tree_maxindex(height)) |
| 293 | return NULL; | 298 | return NULL; |
| 294 | 299 | ||
| 295 | shift = (height-1) * RADIX_TREE_MAP_SHIFT; | 300 | shift = (height-1) * RADIX_TREE_MAP_SHIFT; |
| 296 | slot = &root->rnode; | 301 | slot = root->rnode; |
| 297 | 302 | ||
| 298 | while (height > 0) { | 303 | while (height > 0) { |
| 299 | if (*slot == NULL) | 304 | if (slot == NULL) |
| 300 | return NULL; | 305 | return NULL; |
| 301 | 306 | ||
| 302 | slot = (struct radix_tree_node **) | 307 | slot = slot->slots[(index >> shift) & RADIX_TREE_MAP_MASK]; |
| 303 | ((*slot)->slots + | ||
| 304 | ((index >> shift) & RADIX_TREE_MAP_MASK)); | ||
| 305 | shift -= RADIX_TREE_MAP_SHIFT; | 308 | shift -= RADIX_TREE_MAP_SHIFT; |
| 306 | height--; | 309 | height--; |
| 307 | } | 310 | } |
| 308 | 311 | ||
| 309 | return *slot; | 312 | return slot; |
| 310 | } | 313 | } |
| 311 | EXPORT_SYMBOL(radix_tree_lookup); | 314 | EXPORT_SYMBOL(radix_tree_lookup); |
| 312 | 315 | ||
| @@ -326,27 +329,27 @@ void *radix_tree_tag_set(struct radix_tree_root *root, | |||
| 326 | unsigned long index, int tag) | 329 | unsigned long index, int tag) |
| 327 | { | 330 | { |
| 328 | unsigned int height, shift; | 331 | unsigned int height, shift; |
| 329 | struct radix_tree_node **slot; | 332 | struct radix_tree_node *slot; |
| 330 | 333 | ||
| 331 | height = root->height; | 334 | height = root->height; |
| 332 | if (index > radix_tree_maxindex(height)) | 335 | if (index > radix_tree_maxindex(height)) |
| 333 | return NULL; | 336 | return NULL; |
| 334 | 337 | ||
| 335 | shift = (height - 1) * RADIX_TREE_MAP_SHIFT; | 338 | shift = (height - 1) * RADIX_TREE_MAP_SHIFT; |
| 336 | slot = &root->rnode; | 339 | slot = root->rnode; |
| 337 | 340 | ||
| 338 | while (height > 0) { | 341 | while (height > 0) { |
| 339 | int offset; | 342 | int offset; |
| 340 | 343 | ||
| 341 | offset = (index >> shift) & RADIX_TREE_MAP_MASK; | 344 | offset = (index >> shift) & RADIX_TREE_MAP_MASK; |
| 342 | tag_set(*slot, tag, offset); | 345 | tag_set(slot, tag, offset); |
| 343 | slot = (struct radix_tree_node **)((*slot)->slots + offset); | 346 | slot = slot->slots[offset]; |
| 344 | BUG_ON(*slot == NULL); | 347 | BUG_ON(slot == NULL); |
| 345 | shift -= RADIX_TREE_MAP_SHIFT; | 348 | shift -= RADIX_TREE_MAP_SHIFT; |
| 346 | height--; | 349 | height--; |
| 347 | } | 350 | } |
| 348 | 351 | ||
| 349 | return *slot; | 352 | return slot; |
| 350 | } | 353 | } |
| 351 | EXPORT_SYMBOL(radix_tree_tag_set); | 354 | EXPORT_SYMBOL(radix_tree_tag_set); |
| 352 | 355 | ||
| @@ -367,6 +370,7 @@ void *radix_tree_tag_clear(struct radix_tree_root *root, | |||
| 367 | unsigned long index, int tag) | 370 | unsigned long index, int tag) |
| 368 | { | 371 | { |
| 369 | struct radix_tree_path path[RADIX_TREE_MAX_PATH], *pathp = path; | 372 | struct radix_tree_path path[RADIX_TREE_MAX_PATH], *pathp = path; |
| 373 | struct radix_tree_node *slot; | ||
| 370 | unsigned int height, shift; | 374 | unsigned int height, shift; |
| 371 | void *ret = NULL; | 375 | void *ret = NULL; |
| 372 | 376 | ||
| @@ -376,38 +380,37 @@ void *radix_tree_tag_clear(struct radix_tree_root *root, | |||
| 376 | 380 | ||
| 377 | shift = (height - 1) * RADIX_TREE_MAP_SHIFT; | 381 | shift = (height - 1) * RADIX_TREE_MAP_SHIFT; |
| 378 | pathp->node = NULL; | 382 | pathp->node = NULL; |
| 379 | pathp->slot = &root->rnode; | 383 | slot = root->rnode; |
| 380 | 384 | ||
| 381 | while (height > 0) { | 385 | while (height > 0) { |
| 382 | int offset; | 386 | int offset; |
| 383 | 387 | ||
| 384 | if (*pathp->slot == NULL) | 388 | if (slot == NULL) |
| 385 | goto out; | 389 | goto out; |
| 386 | 390 | ||
| 387 | offset = (index >> shift) & RADIX_TREE_MAP_MASK; | 391 | offset = (index >> shift) & RADIX_TREE_MAP_MASK; |
| 388 | pathp[1].offset = offset; | 392 | pathp[1].offset = offset; |
| 389 | pathp[1].node = *pathp[0].slot; | 393 | pathp[1].node = slot; |
| 390 | pathp[1].slot = (struct radix_tree_node **) | 394 | slot = slot->slots[offset]; |
| 391 | (pathp[1].node->slots + offset); | ||
| 392 | pathp++; | 395 | pathp++; |
| 393 | shift -= RADIX_TREE_MAP_SHIFT; | 396 | shift -= RADIX_TREE_MAP_SHIFT; |
| 394 | height--; | 397 | height--; |
| 395 | } | 398 | } |
| 396 | 399 | ||
| 397 | ret = *pathp[0].slot; | 400 | ret = slot; |
| 398 | if (ret == NULL) | 401 | if (ret == NULL) |
| 399 | goto out; | 402 | goto out; |
| 400 | 403 | ||
| 401 | do { | 404 | do { |
| 402 | int idx; | 405 | int idx; |
| 403 | 406 | ||
| 404 | tag_clear(pathp[0].node, tag, pathp[0].offset); | 407 | tag_clear(pathp->node, tag, pathp->offset); |
| 405 | for (idx = 0; idx < RADIX_TREE_TAG_LONGS; idx++) { | 408 | for (idx = 0; idx < RADIX_TREE_TAG_LONGS; idx++) { |
| 406 | if (pathp[0].node->tags[tag][idx]) | 409 | if (pathp->node->tags[tag][idx]) |
| 407 | goto out; | 410 | goto out; |
| 408 | } | 411 | } |
| 409 | pathp--; | 412 | pathp--; |
| 410 | } while (pathp[0].node); | 413 | } while (pathp->node); |
| 411 | out: | 414 | out: |
| 412 | return ret; | 415 | return ret; |
| 413 | } | 416 | } |
| @@ -415,21 +418,22 @@ EXPORT_SYMBOL(radix_tree_tag_clear); | |||
| 415 | 418 | ||
| 416 | #ifndef __KERNEL__ /* Only the test harness uses this at present */ | 419 | #ifndef __KERNEL__ /* Only the test harness uses this at present */ |
| 417 | /** | 420 | /** |
| 418 | * radix_tree_tag_get - get a tag on a radix tree node | 421 | * radix_tree_tag_get - get a tag on a radix tree node |
| 419 | * @root: radix tree root | 422 | * @root: radix tree root |
| 420 | * @index: index key | 423 | * @index: index key |
| 421 | * @tag: tag index | 424 | * @tag: tag index |
| 422 | * | 425 | * |
| 423 | * Return the search tag corresponging to @index in the radix tree. | 426 | * Return values: |
| 424 | * | 427 | * |
| 425 | * Returns zero if the tag is unset, or if there is no corresponding item | 428 | * 0: tag not present |
| 426 | * in the tree. | 429 | * 1: tag present, set |
| 430 | * -1: tag present, unset | ||
| 427 | */ | 431 | */ |
| 428 | int radix_tree_tag_get(struct radix_tree_root *root, | 432 | int radix_tree_tag_get(struct radix_tree_root *root, |
| 429 | unsigned long index, int tag) | 433 | unsigned long index, int tag) |
| 430 | { | 434 | { |
| 431 | unsigned int height, shift; | 435 | unsigned int height, shift; |
| 432 | struct radix_tree_node **slot; | 436 | struct radix_tree_node *slot; |
| 433 | int saw_unset_tag = 0; | 437 | int saw_unset_tag = 0; |
| 434 | 438 | ||
| 435 | height = root->height; | 439 | height = root->height; |
| @@ -437,12 +441,12 @@ int radix_tree_tag_get(struct radix_tree_root *root, | |||
| 437 | return 0; | 441 | return 0; |
| 438 | 442 | ||
| 439 | shift = (height - 1) * RADIX_TREE_MAP_SHIFT; | 443 | shift = (height - 1) * RADIX_TREE_MAP_SHIFT; |
| 440 | slot = &root->rnode; | 444 | slot = root->rnode; |
| 441 | 445 | ||
| 442 | for ( ; ; ) { | 446 | for ( ; ; ) { |
| 443 | int offset; | 447 | int offset; |
| 444 | 448 | ||
| 445 | if (*slot == NULL) | 449 | if (slot == NULL) |
| 446 | return 0; | 450 | return 0; |
| 447 | 451 | ||
| 448 | offset = (index >> shift) & RADIX_TREE_MAP_MASK; | 452 | offset = (index >> shift) & RADIX_TREE_MAP_MASK; |
| @@ -451,15 +455,15 @@ int radix_tree_tag_get(struct radix_tree_root *root, | |||
| 451 | * This is just a debug check. Later, we can bale as soon as | 455 | * This is just a debug check. Later, we can bale as soon as |
| 452 | * we see an unset tag. | 456 | * we see an unset tag. |
| 453 | */ | 457 | */ |
| 454 | if (!tag_get(*slot, tag, offset)) | 458 | if (!tag_get(slot, tag, offset)) |
| 455 | saw_unset_tag = 1; | 459 | saw_unset_tag = 1; |
| 456 | if (height == 1) { | 460 | if (height == 1) { |
| 457 | int ret = tag_get(*slot, tag, offset); | 461 | int ret = tag_get(slot, tag, offset); |
| 458 | 462 | ||
| 459 | BUG_ON(ret && saw_unset_tag); | 463 | BUG_ON(ret && saw_unset_tag); |
| 460 | return ret; | 464 | return ret ? 1 : -1; |
| 461 | } | 465 | } |
| 462 | slot = (struct radix_tree_node **)((*slot)->slots + offset); | 466 | slot = slot->slots[offset]; |
| 463 | shift -= RADIX_TREE_MAP_SHIFT; | 467 | shift -= RADIX_TREE_MAP_SHIFT; |
| 464 | height--; | 468 | height--; |
| 465 | } | 469 | } |
| @@ -472,17 +476,21 @@ __lookup(struct radix_tree_root *root, void **results, unsigned long index, | |||
| 472 | unsigned int max_items, unsigned long *next_index) | 476 | unsigned int max_items, unsigned long *next_index) |
| 473 | { | 477 | { |
| 474 | unsigned int nr_found = 0; | 478 | unsigned int nr_found = 0; |
| 475 | unsigned int shift; | 479 | unsigned int shift, height; |
| 476 | unsigned int height = root->height; | ||
| 477 | struct radix_tree_node *slot; | 480 | struct radix_tree_node *slot; |
| 481 | unsigned long i; | ||
| 482 | |||
| 483 | height = root->height; | ||
| 484 | if (height == 0) | ||
| 485 | goto out; | ||
| 478 | 486 | ||
| 479 | shift = (height-1) * RADIX_TREE_MAP_SHIFT; | 487 | shift = (height-1) * RADIX_TREE_MAP_SHIFT; |
| 480 | slot = root->rnode; | 488 | slot = root->rnode; |
| 481 | 489 | ||
| 482 | while (height > 0) { | 490 | for ( ; height > 1; height--) { |
| 483 | unsigned long i = (index >> shift) & RADIX_TREE_MAP_MASK; | ||
| 484 | 491 | ||
| 485 | for ( ; i < RADIX_TREE_MAP_SIZE; i++) { | 492 | for (i = (index >> shift) & RADIX_TREE_MAP_MASK ; |
| 493 | i < RADIX_TREE_MAP_SIZE; i++) { | ||
| 486 | if (slot->slots[i] != NULL) | 494 | if (slot->slots[i] != NULL) |
| 487 | break; | 495 | break; |
| 488 | index &= ~((1UL << shift) - 1); | 496 | index &= ~((1UL << shift) - 1); |
| @@ -492,22 +500,20 @@ __lookup(struct radix_tree_root *root, void **results, unsigned long index, | |||
| 492 | } | 500 | } |
| 493 | if (i == RADIX_TREE_MAP_SIZE) | 501 | if (i == RADIX_TREE_MAP_SIZE) |
| 494 | goto out; | 502 | goto out; |
| 495 | height--; | ||
| 496 | if (height == 0) { /* Bottom level: grab some items */ | ||
| 497 | unsigned long j = index & RADIX_TREE_MAP_MASK; | ||
| 498 | 503 | ||
| 499 | for ( ; j < RADIX_TREE_MAP_SIZE; j++) { | ||
| 500 | index++; | ||
| 501 | if (slot->slots[j]) { | ||
| 502 | results[nr_found++] = slot->slots[j]; | ||
| 503 | if (nr_found == max_items) | ||
| 504 | goto out; | ||
| 505 | } | ||
| 506 | } | ||
| 507 | } | ||
| 508 | shift -= RADIX_TREE_MAP_SHIFT; | 504 | shift -= RADIX_TREE_MAP_SHIFT; |
| 509 | slot = slot->slots[i]; | 505 | slot = slot->slots[i]; |
| 510 | } | 506 | } |
| 507 | |||
| 508 | /* Bottom level: grab some items */ | ||
| 509 | for (i = index & RADIX_TREE_MAP_MASK; i < RADIX_TREE_MAP_SIZE; i++) { | ||
| 510 | index++; | ||
| 511 | if (slot->slots[i]) { | ||
| 512 | results[nr_found++] = slot->slots[i]; | ||
| 513 | if (nr_found == max_items) | ||
| 514 | goto out; | ||
| 515 | } | ||
| 516 | } | ||
| 511 | out: | 517 | out: |
| 512 | *next_index = index; | 518 | *next_index = index; |
| 513 | return nr_found; | 519 | return nr_found; |
| @@ -655,6 +661,7 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index) | |||
| 655 | { | 661 | { |
| 656 | struct radix_tree_path path[RADIX_TREE_MAX_PATH], *pathp = path; | 662 | struct radix_tree_path path[RADIX_TREE_MAX_PATH], *pathp = path; |
| 657 | struct radix_tree_path *orig_pathp; | 663 | struct radix_tree_path *orig_pathp; |
| 664 | struct radix_tree_node *slot; | ||
| 658 | unsigned int height, shift; | 665 | unsigned int height, shift; |
| 659 | void *ret = NULL; | 666 | void *ret = NULL; |
| 660 | char tags[RADIX_TREE_TAGS]; | 667 | char tags[RADIX_TREE_TAGS]; |
| @@ -666,25 +673,23 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index) | |||
| 666 | 673 | ||
| 667 | shift = (height - 1) * RADIX_TREE_MAP_SHIFT; | 674 | shift = (height - 1) * RADIX_TREE_MAP_SHIFT; |
| 668 | pathp->node = NULL; | 675 | pathp->node = NULL; |
| 669 | pathp->slot = &root->rnode; | 676 | slot = root->rnode; |
| 670 | 677 | ||
| 671 | while (height > 0) { | 678 | for ( ; height > 0; height--) { |
| 672 | int offset; | 679 | int offset; |
| 673 | 680 | ||
| 674 | if (*pathp->slot == NULL) | 681 | if (slot == NULL) |
| 675 | goto out; | 682 | goto out; |
| 676 | 683 | ||
| 677 | offset = (index >> shift) & RADIX_TREE_MAP_MASK; | 684 | offset = (index >> shift) & RADIX_TREE_MAP_MASK; |
| 678 | pathp[1].offset = offset; | 685 | pathp[1].offset = offset; |
| 679 | pathp[1].node = *pathp[0].slot; | 686 | pathp[1].node = slot; |
| 680 | pathp[1].slot = (struct radix_tree_node **) | 687 | slot = slot->slots[offset]; |
| 681 | (pathp[1].node->slots + offset); | ||
| 682 | pathp++; | 688 | pathp++; |
| 683 | shift -= RADIX_TREE_MAP_SHIFT; | 689 | shift -= RADIX_TREE_MAP_SHIFT; |
| 684 | height--; | ||
| 685 | } | 690 | } |
| 686 | 691 | ||
| 687 | ret = *pathp[0].slot; | 692 | ret = slot; |
| 688 | if (ret == NULL) | 693 | if (ret == NULL) |
| 689 | goto out; | 694 | goto out; |
| 690 | 695 | ||
| @@ -704,10 +709,10 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index) | |||
| 704 | if (tags[tag]) | 709 | if (tags[tag]) |
| 705 | continue; | 710 | continue; |
| 706 | 711 | ||
| 707 | tag_clear(pathp[0].node, tag, pathp[0].offset); | 712 | tag_clear(pathp->node, tag, pathp->offset); |
| 708 | 713 | ||
| 709 | for (idx = 0; idx < RADIX_TREE_TAG_LONGS; idx++) { | 714 | for (idx = 0; idx < RADIX_TREE_TAG_LONGS; idx++) { |
| 710 | if (pathp[0].node->tags[tag][idx]) { | 715 | if (pathp->node->tags[tag][idx]) { |
| 711 | tags[tag] = 1; | 716 | tags[tag] = 1; |
| 712 | nr_cleared_tags--; | 717 | nr_cleared_tags--; |
| 713 | break; | 718 | break; |
| @@ -715,18 +720,19 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index) | |||
| 715 | } | 720 | } |
| 716 | } | 721 | } |
| 717 | pathp--; | 722 | pathp--; |
| 718 | } while (pathp[0].node && nr_cleared_tags); | 723 | } while (pathp->node && nr_cleared_tags); |
| 719 | 724 | ||
| 720 | pathp = orig_pathp; | 725 | /* Now free the nodes we do not need anymore */ |
| 721 | *pathp[0].slot = NULL; | 726 | for (pathp = orig_pathp; pathp->node; pathp--) { |
| 722 | while (pathp[0].node && --pathp[0].node->count == 0) { | 727 | pathp->node->slots[pathp->offset] = NULL; |
| 723 | pathp--; | 728 | if (--pathp->node->count) |
| 724 | BUG_ON(*pathp[0].slot == NULL); | 729 | goto out; |
| 725 | *pathp[0].slot = NULL; | 730 | |
| 726 | radix_tree_node_free(pathp[1].node); | 731 | /* Node with zero slots in use so free it */ |
| 732 | radix_tree_node_free(pathp->node); | ||
| 727 | } | 733 | } |
| 728 | if (root->rnode == NULL) | 734 | root->rnode = NULL; |
| 729 | root->height = 0; | 735 | root->height = 0; |
| 730 | out: | 736 | out: |
| 731 | return ret; | 737 | return ret; |
| 732 | } | 738 | } |
diff --git a/lib/semaphore-sleepers.c b/lib/semaphore-sleepers.c new file mode 100644 index 000000000000..4d5f18889fa5 --- /dev/null +++ b/lib/semaphore-sleepers.c | |||
| @@ -0,0 +1,177 @@ | |||
| 1 | /* | ||
| 2 | * i386 and x86-64 semaphore implementation. | ||
| 3 | * | ||
| 4 | * (C) Copyright 1999 Linus Torvalds | ||
| 5 | * | ||
| 6 | * Portions Copyright 1999 Red Hat, Inc. | ||
| 7 | * | ||
| 8 | * This program is free software; you can redistribute it and/or | ||
| 9 | * modify it under the terms of the GNU General Public License | ||
| 10 | * as published by the Free Software Foundation; either version | ||
| 11 | * 2 of the License, or (at your option) any later version. | ||
| 12 | * | ||
| 13 | * rw semaphores implemented November 1999 by Benjamin LaHaise <bcrl@kvack.org> | ||
| 14 | */ | ||
| 15 | #include <linux/config.h> | ||
| 16 | #include <linux/sched.h> | ||
| 17 | #include <linux/err.h> | ||
| 18 | #include <linux/init.h> | ||
| 19 | #include <asm/semaphore.h> | ||
| 20 | |||
| 21 | /* | ||
| 22 | * Semaphores are implemented using a two-way counter: | ||
| 23 | * The "count" variable is decremented for each process | ||
| 24 | * that tries to acquire the semaphore, while the "sleeping" | ||
| 25 | * variable is a count of such acquires. | ||
| 26 | * | ||
| 27 | * Notably, the inline "up()" and "down()" functions can | ||
| 28 | * efficiently test if they need to do any extra work (up | ||
| 29 | * needs to do something only if count was negative before | ||
| 30 | * the increment operation. | ||
| 31 | * | ||
| 32 | * "sleeping" and the contention routine ordering is protected | ||
| 33 | * by the spinlock in the semaphore's waitqueue head. | ||
| 34 | * | ||
| 35 | * Note that these functions are only called when there is | ||
| 36 | * contention on the lock, and as such all this is the | ||
| 37 | * "non-critical" part of the whole semaphore business. The | ||
| 38 | * critical part is the inline stuff in <asm/semaphore.h> | ||
| 39 | * where we want to avoid any extra jumps and calls. | ||
| 40 | */ | ||
| 41 | |||
| 42 | /* | ||
| 43 | * Logic: | ||
| 44 | * - only on a boundary condition do we need to care. When we go | ||
| 45 | * from a negative count to a non-negative, we wake people up. | ||
| 46 | * - when we go from a non-negative count to a negative do we | ||
| 47 | * (a) synchronize with the "sleeper" count and (b) make sure | ||
| 48 | * that we're on the wakeup list before we synchronize so that | ||
| 49 | * we cannot lose wakeup events. | ||
| 50 | */ | ||
| 51 | |||
| 52 | fastcall void __up(struct semaphore *sem) | ||
| 53 | { | ||
| 54 | wake_up(&sem->wait); | ||
| 55 | } | ||
| 56 | |||
| 57 | fastcall void __sched __down(struct semaphore * sem) | ||
| 58 | { | ||
| 59 | struct task_struct *tsk = current; | ||
| 60 | DECLARE_WAITQUEUE(wait, tsk); | ||
| 61 | unsigned long flags; | ||
| 62 | |||
| 63 | tsk->state = TASK_UNINTERRUPTIBLE; | ||
| 64 | spin_lock_irqsave(&sem->wait.lock, flags); | ||
| 65 | add_wait_queue_exclusive_locked(&sem->wait, &wait); | ||
| 66 | |||
| 67 | sem->sleepers++; | ||
| 68 | for (;;) { | ||
| 69 | int sleepers = sem->sleepers; | ||
| 70 | |||
| 71 | /* | ||
| 72 | * Add "everybody else" into it. They aren't | ||
| 73 | * playing, because we own the spinlock in | ||
| 74 | * the wait_queue_head. | ||
| 75 | */ | ||
| 76 | if (!atomic_add_negative(sleepers - 1, &sem->count)) { | ||
| 77 | sem->sleepers = 0; | ||
| 78 | break; | ||
| 79 | } | ||
| 80 | sem->sleepers = 1; /* us - see -1 above */ | ||
| 81 | spin_unlock_irqrestore(&sem->wait.lock, flags); | ||
| 82 | |||
| 83 | schedule(); | ||
| 84 | |||
| 85 | spin_lock_irqsave(&sem->wait.lock, flags); | ||
| 86 | tsk->state = TASK_UNINTERRUPTIBLE; | ||
| 87 | } | ||
| 88 | remove_wait_queue_locked(&sem->wait, &wait); | ||
| 89 | wake_up_locked(&sem->wait); | ||
| 90 | spin_unlock_irqrestore(&sem->wait.lock, flags); | ||
| 91 | tsk->state = TASK_RUNNING; | ||
| 92 | } | ||
| 93 | |||
| 94 | fastcall int __sched __down_interruptible(struct semaphore * sem) | ||
| 95 | { | ||
| 96 | int retval = 0; | ||
| 97 | struct task_struct *tsk = current; | ||
| 98 | DECLARE_WAITQUEUE(wait, tsk); | ||
| 99 | unsigned long flags; | ||
| 100 | |||
| 101 | tsk->state = TASK_INTERRUPTIBLE; | ||
| 102 | spin_lock_irqsave(&sem->wait.lock, flags); | ||
| 103 | add_wait_queue_exclusive_locked(&sem->wait, &wait); | ||
| 104 | |||
| 105 | sem->sleepers++; | ||
| 106 | for (;;) { | ||
| 107 | int sleepers = sem->sleepers; | ||
| 108 | |||
| 109 | /* | ||
| 110 | * With signals pending, this turns into | ||
| 111 | * the trylock failure case - we won't be | ||
| 112 | * sleeping, and we* can't get the lock as | ||
| 113 | * it has contention. Just correct the count | ||
| 114 | * and exit. | ||
| 115 | */ | ||
| 116 | if (signal_pending(current)) { | ||
| 117 | retval = -EINTR; | ||
| 118 | sem->sleepers = 0; | ||
| 119 | atomic_add(sleepers, &sem->count); | ||
| 120 | break; | ||
| 121 | } | ||
| 122 | |||
| 123 | /* | ||
| 124 | * Add "everybody else" into it. They aren't | ||
| 125 | * playing, because we own the spinlock in | ||
| 126 | * wait_queue_head. The "-1" is because we're | ||
| 127 | * still hoping to get the semaphore. | ||
| 128 | */ | ||
| 129 | if (!atomic_add_negative(sleepers - 1, &sem->count)) { | ||
| 130 | sem->sleepers = 0; | ||
| 131 | break; | ||
| 132 | } | ||
| 133 | sem->sleepers = 1; /* us - see -1 above */ | ||
| 134 | spin_unlock_irqrestore(&sem->wait.lock, flags); | ||
| 135 | |||
| 136 | schedule(); | ||
| 137 | |||
| 138 | spin_lock_irqsave(&sem->wait.lock, flags); | ||
| 139 | tsk->state = TASK_INTERRUPTIBLE; | ||
| 140 | } | ||
| 141 | remove_wait_queue_locked(&sem->wait, &wait); | ||
| 142 | wake_up_locked(&sem->wait); | ||
| 143 | spin_unlock_irqrestore(&sem->wait.lock, flags); | ||
| 144 | |||
| 145 | tsk->state = TASK_RUNNING; | ||
| 146 | return retval; | ||
| 147 | } | ||
| 148 | |||
| 149 | /* | ||
| 150 | * Trylock failed - make sure we correct for | ||
| 151 | * having decremented the count. | ||
| 152 | * | ||
| 153 | * We could have done the trylock with a | ||
| 154 | * single "cmpxchg" without failure cases, | ||
| 155 | * but then it wouldn't work on a 386. | ||
| 156 | */ | ||
| 157 | fastcall int __down_trylock(struct semaphore * sem) | ||
| 158 | { | ||
| 159 | int sleepers; | ||
| 160 | unsigned long flags; | ||
| 161 | |||
| 162 | spin_lock_irqsave(&sem->wait.lock, flags); | ||
| 163 | sleepers = sem->sleepers + 1; | ||
| 164 | sem->sleepers = 0; | ||
| 165 | |||
| 166 | /* | ||
| 167 | * Add "everybody else" and us into it. They aren't | ||
| 168 | * playing, because we own the spinlock in the | ||
| 169 | * wait_queue_head. | ||
| 170 | */ | ||
| 171 | if (!atomic_add_negative(sleepers, &sem->count)) { | ||
| 172 | wake_up_locked(&sem->wait); | ||
| 173 | } | ||
| 174 | |||
| 175 | spin_unlock_irqrestore(&sem->wait.lock, flags); | ||
| 176 | return 1; | ||
| 177 | } | ||
