diff options
| author | Qu Wenruo <quwenruo@cn.fujitsu.com> | 2014-02-27 21:46:18 -0500 |
|---|---|---|
| committer | Josef Bacik <jbacik@fb.com> | 2014-03-10 15:17:15 -0400 |
| commit | a046e9c88b0f46677923864295eac7c92cd962cb (patch) | |
| tree | 72cff91c3bb1426473d5d70a2b99684c86102594 /fs/btrfs | |
| parent | 0339ef2f42bcfbb2d4021ad6f38fe20580082c85 (diff) | |
btrfs: Cleanup the old btrfs_worker.
Since all the btrfs_worker is replaced with the newly created
btrfs_workqueue, the old codes can be easily remove.
Signed-off-by: Quwenruo <quwenruo@cn.fujitsu.com>
Tested-by: David Sterba <dsterba@suse.cz>
Signed-off-by: Josef Bacik <jbacik@fb.com>
Diffstat (limited to 'fs/btrfs')
| -rw-r--r-- | fs/btrfs/async-thread.c | 707 | ||||
| -rw-r--r-- | fs/btrfs/async-thread.h | 100 | ||||
| -rw-r--r-- | fs/btrfs/ctree.h | 1 | ||||
| -rw-r--r-- | fs/btrfs/disk-io.c | 12 | ||||
| -rw-r--r-- | fs/btrfs/super.c | 8 |
5 files changed, 3 insertions, 825 deletions
diff --git a/fs/btrfs/async-thread.c b/fs/btrfs/async-thread.c index 977bce2ec887..2a5f383c3636 100644 --- a/fs/btrfs/async-thread.c +++ b/fs/btrfs/async-thread.c | |||
| @@ -25,714 +25,13 @@ | |||
| 25 | #include <linux/workqueue.h> | 25 | #include <linux/workqueue.h> |
| 26 | #include "async-thread.h" | 26 | #include "async-thread.h" |
| 27 | 27 | ||
| 28 | #define WORK_QUEUED_BIT 0 | 28 | #define WORK_DONE_BIT 0 |
| 29 | #define WORK_DONE_BIT 1 | 29 | #define WORK_ORDER_DONE_BIT 1 |
| 30 | #define WORK_ORDER_DONE_BIT 2 | 30 | #define WORK_HIGH_PRIO_BIT 2 |
| 31 | #define WORK_HIGH_PRIO_BIT 3 | ||
| 32 | 31 | ||
| 33 | #define NO_THRESHOLD (-1) | 32 | #define NO_THRESHOLD (-1) |
| 34 | #define DFT_THRESHOLD (32) | 33 | #define DFT_THRESHOLD (32) |
| 35 | 34 | ||
| 36 | /* | ||
| 37 | * container for the kthread task pointer and the list of pending work | ||
| 38 | * One of these is allocated per thread. | ||
| 39 | */ | ||
| 40 | struct btrfs_worker_thread { | ||
| 41 | /* pool we belong to */ | ||
| 42 | struct btrfs_workers *workers; | ||
| 43 | |||
| 44 | /* list of struct btrfs_work that are waiting for service */ | ||
| 45 | struct list_head pending; | ||
| 46 | struct list_head prio_pending; | ||
| 47 | |||
| 48 | /* list of worker threads from struct btrfs_workers */ | ||
| 49 | struct list_head worker_list; | ||
| 50 | |||
| 51 | /* kthread */ | ||
| 52 | struct task_struct *task; | ||
| 53 | |||
| 54 | /* number of things on the pending list */ | ||
| 55 | atomic_t num_pending; | ||
| 56 | |||
| 57 | /* reference counter for this struct */ | ||
| 58 | atomic_t refs; | ||
| 59 | |||
| 60 | unsigned long sequence; | ||
| 61 | |||
| 62 | /* protects the pending list. */ | ||
| 63 | spinlock_t lock; | ||
| 64 | |||
| 65 | /* set to non-zero when this thread is already awake and kicking */ | ||
| 66 | int working; | ||
| 67 | |||
| 68 | /* are we currently idle */ | ||
| 69 | int idle; | ||
| 70 | }; | ||
| 71 | |||
| 72 | static int __btrfs_start_workers(struct btrfs_workers *workers); | ||
| 73 | |||
| 74 | /* | ||
| 75 | * btrfs_start_workers uses kthread_run, which can block waiting for memory | ||
| 76 | * for a very long time. It will actually throttle on page writeback, | ||
| 77 | * and so it may not make progress until after our btrfs worker threads | ||
| 78 | * process all of the pending work structs in their queue | ||
| 79 | * | ||
| 80 | * This means we can't use btrfs_start_workers from inside a btrfs worker | ||
| 81 | * thread that is used as part of cleaning dirty memory, which pretty much | ||
| 82 | * involves all of the worker threads. | ||
| 83 | * | ||
| 84 | * Instead we have a helper queue who never has more than one thread | ||
| 85 | * where we scheduler thread start operations. This worker_start struct | ||
| 86 | * is used to contain the work and hold a pointer to the queue that needs | ||
| 87 | * another worker. | ||
| 88 | */ | ||
| 89 | struct worker_start { | ||
| 90 | struct btrfs_work work; | ||
| 91 | struct btrfs_workers *queue; | ||
| 92 | }; | ||
| 93 | |||
| 94 | static void start_new_worker_func(struct btrfs_work *work) | ||
| 95 | { | ||
| 96 | struct worker_start *start; | ||
| 97 | start = container_of(work, struct worker_start, work); | ||
| 98 | __btrfs_start_workers(start->queue); | ||
| 99 | kfree(start); | ||
| 100 | } | ||
| 101 | |||
| 102 | /* | ||
| 103 | * helper function to move a thread onto the idle list after it | ||
| 104 | * has finished some requests. | ||
| 105 | */ | ||
| 106 | static void check_idle_worker(struct btrfs_worker_thread *worker) | ||
| 107 | { | ||
| 108 | if (!worker->idle && atomic_read(&worker->num_pending) < | ||
| 109 | worker->workers->idle_thresh / 2) { | ||
| 110 | unsigned long flags; | ||
| 111 | spin_lock_irqsave(&worker->workers->lock, flags); | ||
| 112 | worker->idle = 1; | ||
| 113 | |||
| 114 | /* the list may be empty if the worker is just starting */ | ||
| 115 | if (!list_empty(&worker->worker_list) && | ||
| 116 | !worker->workers->stopping) { | ||
| 117 | list_move(&worker->worker_list, | ||
| 118 | &worker->workers->idle_list); | ||
| 119 | } | ||
| 120 | spin_unlock_irqrestore(&worker->workers->lock, flags); | ||
| 121 | } | ||
| 122 | } | ||
| 123 | |||
| 124 | /* | ||
| 125 | * helper function to move a thread off the idle list after new | ||
| 126 | * pending work is added. | ||
| 127 | */ | ||
| 128 | static void check_busy_worker(struct btrfs_worker_thread *worker) | ||
| 129 | { | ||
| 130 | if (worker->idle && atomic_read(&worker->num_pending) >= | ||
| 131 | worker->workers->idle_thresh) { | ||
| 132 | unsigned long flags; | ||
| 133 | spin_lock_irqsave(&worker->workers->lock, flags); | ||
| 134 | worker->idle = 0; | ||
| 135 | |||
| 136 | if (!list_empty(&worker->worker_list) && | ||
| 137 | !worker->workers->stopping) { | ||
| 138 | list_move_tail(&worker->worker_list, | ||
| 139 | &worker->workers->worker_list); | ||
| 140 | } | ||
| 141 | spin_unlock_irqrestore(&worker->workers->lock, flags); | ||
| 142 | } | ||
| 143 | } | ||
| 144 | |||
| 145 | static void check_pending_worker_creates(struct btrfs_worker_thread *worker) | ||
| 146 | { | ||
| 147 | struct btrfs_workers *workers = worker->workers; | ||
| 148 | struct worker_start *start; | ||
| 149 | unsigned long flags; | ||
| 150 | |||
| 151 | rmb(); | ||
| 152 | if (!workers->atomic_start_pending) | ||
| 153 | return; | ||
| 154 | |||
| 155 | start = kzalloc(sizeof(*start), GFP_NOFS); | ||
| 156 | if (!start) | ||
| 157 | return; | ||
| 158 | |||
| 159 | start->work.func = start_new_worker_func; | ||
| 160 | start->queue = workers; | ||
| 161 | |||
| 162 | spin_lock_irqsave(&workers->lock, flags); | ||
| 163 | if (!workers->atomic_start_pending) | ||
| 164 | goto out; | ||
| 165 | |||
| 166 | workers->atomic_start_pending = 0; | ||
| 167 | if (workers->num_workers + workers->num_workers_starting >= | ||
| 168 | workers->max_workers) | ||
| 169 | goto out; | ||
| 170 | |||
| 171 | workers->num_workers_starting += 1; | ||
| 172 | spin_unlock_irqrestore(&workers->lock, flags); | ||
| 173 | btrfs_queue_worker(workers->atomic_worker_start, &start->work); | ||
| 174 | return; | ||
| 175 | |||
| 176 | out: | ||
| 177 | kfree(start); | ||
| 178 | spin_unlock_irqrestore(&workers->lock, flags); | ||
| 179 | } | ||
| 180 | |||
| 181 | static noinline void run_ordered_completions(struct btrfs_workers *workers, | ||
| 182 | struct btrfs_work *work) | ||
| 183 | { | ||
| 184 | if (!workers->ordered) | ||
| 185 | return; | ||
| 186 | |||
| 187 | set_bit(WORK_DONE_BIT, &work->flags); | ||
| 188 | |||
| 189 | spin_lock(&workers->order_lock); | ||
| 190 | |||
| 191 | while (1) { | ||
| 192 | if (!list_empty(&workers->prio_order_list)) { | ||
| 193 | work = list_entry(workers->prio_order_list.next, | ||
| 194 | struct btrfs_work, order_list); | ||
| 195 | } else if (!list_empty(&workers->order_list)) { | ||
| 196 | work = list_entry(workers->order_list.next, | ||
| 197 | struct btrfs_work, order_list); | ||
| 198 | } else { | ||
| 199 | break; | ||
| 200 | } | ||
| 201 | if (!test_bit(WORK_DONE_BIT, &work->flags)) | ||
| 202 | break; | ||
| 203 | |||
| 204 | /* we are going to call the ordered done function, but | ||
| 205 | * we leave the work item on the list as a barrier so | ||
| 206 | * that later work items that are done don't have their | ||
| 207 | * functions called before this one returns | ||
| 208 | */ | ||
| 209 | if (test_and_set_bit(WORK_ORDER_DONE_BIT, &work->flags)) | ||
| 210 | break; | ||
| 211 | |||
| 212 | spin_unlock(&workers->order_lock); | ||
| 213 | |||
| 214 | work->ordered_func(work); | ||
| 215 | |||
| 216 | /* now take the lock again and drop our item from the list */ | ||
| 217 | spin_lock(&workers->order_lock); | ||
| 218 | list_del(&work->order_list); | ||
| 219 | spin_unlock(&workers->order_lock); | ||
| 220 | |||
| 221 | /* | ||
| 222 | * we don't want to call the ordered free functions | ||
| 223 | * with the lock held though | ||
| 224 | */ | ||
| 225 | work->ordered_free(work); | ||
| 226 | spin_lock(&workers->order_lock); | ||
| 227 | } | ||
| 228 | |||
| 229 | spin_unlock(&workers->order_lock); | ||
| 230 | } | ||
| 231 | |||
| 232 | static void put_worker(struct btrfs_worker_thread *worker) | ||
| 233 | { | ||
| 234 | if (atomic_dec_and_test(&worker->refs)) | ||
| 235 | kfree(worker); | ||
| 236 | } | ||
| 237 | |||
| 238 | static int try_worker_shutdown(struct btrfs_worker_thread *worker) | ||
| 239 | { | ||
| 240 | int freeit = 0; | ||
| 241 | |||
| 242 | spin_lock_irq(&worker->lock); | ||
| 243 | spin_lock(&worker->workers->lock); | ||
| 244 | if (worker->workers->num_workers > 1 && | ||
| 245 | worker->idle && | ||
| 246 | !worker->working && | ||
| 247 | !list_empty(&worker->worker_list) && | ||
| 248 | list_empty(&worker->prio_pending) && | ||
| 249 | list_empty(&worker->pending) && | ||
| 250 | atomic_read(&worker->num_pending) == 0) { | ||
| 251 | freeit = 1; | ||
| 252 | list_del_init(&worker->worker_list); | ||
| 253 | worker->workers->num_workers--; | ||
| 254 | } | ||
| 255 | spin_unlock(&worker->workers->lock); | ||
| 256 | spin_unlock_irq(&worker->lock); | ||
| 257 | |||
| 258 | if (freeit) | ||
| 259 | put_worker(worker); | ||
| 260 | return freeit; | ||
| 261 | } | ||
| 262 | |||
| 263 | static struct btrfs_work *get_next_work(struct btrfs_worker_thread *worker, | ||
| 264 | struct list_head *prio_head, | ||
| 265 | struct list_head *head) | ||
| 266 | { | ||
| 267 | struct btrfs_work *work = NULL; | ||
| 268 | struct list_head *cur = NULL; | ||
| 269 | |||
| 270 | if (!list_empty(prio_head)) { | ||
| 271 | cur = prio_head->next; | ||
| 272 | goto out; | ||
| 273 | } | ||
| 274 | |||
| 275 | smp_mb(); | ||
| 276 | if (!list_empty(&worker->prio_pending)) | ||
| 277 | goto refill; | ||
| 278 | |||
| 279 | if (!list_empty(head)) { | ||
| 280 | cur = head->next; | ||
| 281 | goto out; | ||
| 282 | } | ||
| 283 | |||
| 284 | refill: | ||
| 285 | spin_lock_irq(&worker->lock); | ||
| 286 | list_splice_tail_init(&worker->prio_pending, prio_head); | ||
| 287 | list_splice_tail_init(&worker->pending, head); | ||
| 288 | |||
| 289 | if (!list_empty(prio_head)) | ||
| 290 | cur = prio_head->next; | ||
| 291 | else if (!list_empty(head)) | ||
| 292 | cur = head->next; | ||
| 293 | spin_unlock_irq(&worker->lock); | ||
| 294 | |||
| 295 | if (!cur) | ||
| 296 | goto out_fail; | ||
| 297 | |||
| 298 | out: | ||
| 299 | work = list_entry(cur, struct btrfs_work, list); | ||
| 300 | |||
| 301 | out_fail: | ||
| 302 | return work; | ||
| 303 | } | ||
| 304 | |||
| 305 | /* | ||
| 306 | * main loop for servicing work items | ||
| 307 | */ | ||
| 308 | static int worker_loop(void *arg) | ||
| 309 | { | ||
| 310 | struct btrfs_worker_thread *worker = arg; | ||
| 311 | struct list_head head; | ||
| 312 | struct list_head prio_head; | ||
| 313 | struct btrfs_work *work; | ||
| 314 | |||
| 315 | INIT_LIST_HEAD(&head); | ||
| 316 | INIT_LIST_HEAD(&prio_head); | ||
| 317 | |||
| 318 | do { | ||
| 319 | again: | ||
| 320 | while (1) { | ||
| 321 | |||
| 322 | |||
| 323 | work = get_next_work(worker, &prio_head, &head); | ||
| 324 | if (!work) | ||
| 325 | break; | ||
| 326 | |||
| 327 | list_del(&work->list); | ||
| 328 | clear_bit(WORK_QUEUED_BIT, &work->flags); | ||
| 329 | |||
| 330 | work->worker = worker; | ||
| 331 | |||
| 332 | work->func(work); | ||
| 333 | |||
| 334 | atomic_dec(&worker->num_pending); | ||
| 335 | /* | ||
| 336 | * unless this is an ordered work queue, | ||
| 337 | * 'work' was probably freed by func above. | ||
| 338 | */ | ||
| 339 | run_ordered_completions(worker->workers, work); | ||
| 340 | |||
| 341 | check_pending_worker_creates(worker); | ||
| 342 | cond_resched(); | ||
| 343 | } | ||
| 344 | |||
| 345 | spin_lock_irq(&worker->lock); | ||
| 346 | check_idle_worker(worker); | ||
| 347 | |||
| 348 | if (freezing(current)) { | ||
| 349 | worker->working = 0; | ||
| 350 | spin_unlock_irq(&worker->lock); | ||
| 351 | try_to_freeze(); | ||
| 352 | } else { | ||
| 353 | spin_unlock_irq(&worker->lock); | ||
| 354 | if (!kthread_should_stop()) { | ||
| 355 | cpu_relax(); | ||
| 356 | /* | ||
| 357 | * we've dropped the lock, did someone else | ||
| 358 | * jump_in? | ||
| 359 | */ | ||
| 360 | smp_mb(); | ||
| 361 | if (!list_empty(&worker->pending) || | ||
| 362 | !list_empty(&worker->prio_pending)) | ||
| 363 | continue; | ||
| 364 | |||
| 365 | /* | ||
| 366 | * this short schedule allows more work to | ||
| 367 | * come in without the queue functions | ||
| 368 | * needing to go through wake_up_process() | ||
| 369 | * | ||
| 370 | * worker->working is still 1, so nobody | ||
| 371 | * is going to try and wake us up | ||
| 372 | */ | ||
| 373 | schedule_timeout(1); | ||
| 374 | smp_mb(); | ||
| 375 | if (!list_empty(&worker->pending) || | ||
| 376 | !list_empty(&worker->prio_pending)) | ||
| 377 | continue; | ||
| 378 | |||
| 379 | if (kthread_should_stop()) | ||
| 380 | break; | ||
| 381 | |||
| 382 | /* still no more work?, sleep for real */ | ||
| 383 | spin_lock_irq(&worker->lock); | ||
| 384 | set_current_state(TASK_INTERRUPTIBLE); | ||
| 385 | if (!list_empty(&worker->pending) || | ||
| 386 | !list_empty(&worker->prio_pending)) { | ||
| 387 | spin_unlock_irq(&worker->lock); | ||
| 388 | set_current_state(TASK_RUNNING); | ||
| 389 | goto again; | ||
| 390 | } | ||
| 391 | |||
| 392 | /* | ||
| 393 | * this makes sure we get a wakeup when someone | ||
| 394 | * adds something new to the queue | ||
| 395 | */ | ||
| 396 | worker->working = 0; | ||
| 397 | spin_unlock_irq(&worker->lock); | ||
| 398 | |||
| 399 | if (!kthread_should_stop()) { | ||
| 400 | schedule_timeout(HZ * 120); | ||
| 401 | if (!worker->working && | ||
| 402 | try_worker_shutdown(worker)) { | ||
| 403 | return 0; | ||
| 404 | } | ||
| 405 | } | ||
| 406 | } | ||
| 407 | __set_current_state(TASK_RUNNING); | ||
| 408 | } | ||
| 409 | } while (!kthread_should_stop()); | ||
| 410 | return 0; | ||
| 411 | } | ||
| 412 | |||
| 413 | /* | ||
| 414 | * this will wait for all the worker threads to shutdown | ||
| 415 | */ | ||
| 416 | void btrfs_stop_workers(struct btrfs_workers *workers) | ||
| 417 | { | ||
| 418 | struct list_head *cur; | ||
| 419 | struct btrfs_worker_thread *worker; | ||
| 420 | int can_stop; | ||
| 421 | |||
| 422 | spin_lock_irq(&workers->lock); | ||
| 423 | workers->stopping = 1; | ||
| 424 | list_splice_init(&workers->idle_list, &workers->worker_list); | ||
| 425 | while (!list_empty(&workers->worker_list)) { | ||
| 426 | cur = workers->worker_list.next; | ||
| 427 | worker = list_entry(cur, struct btrfs_worker_thread, | ||
| 428 | worker_list); | ||
| 429 | |||
| 430 | atomic_inc(&worker->refs); | ||
| 431 | workers->num_workers -= 1; | ||
| 432 | if (!list_empty(&worker->worker_list)) { | ||
| 433 | list_del_init(&worker->worker_list); | ||
| 434 | put_worker(worker); | ||
| 435 | can_stop = 1; | ||
| 436 | } else | ||
| 437 | can_stop = 0; | ||
| 438 | spin_unlock_irq(&workers->lock); | ||
| 439 | if (can_stop) | ||
| 440 | kthread_stop(worker->task); | ||
| 441 | spin_lock_irq(&workers->lock); | ||
| 442 | put_worker(worker); | ||
| 443 | } | ||
| 444 | spin_unlock_irq(&workers->lock); | ||
| 445 | } | ||
| 446 | |||
| 447 | /* | ||
| 448 | * simple init on struct btrfs_workers | ||
| 449 | */ | ||
| 450 | void btrfs_init_workers(struct btrfs_workers *workers, char *name, int max, | ||
| 451 | struct btrfs_workers *async_helper) | ||
| 452 | { | ||
| 453 | workers->num_workers = 0; | ||
| 454 | workers->num_workers_starting = 0; | ||
| 455 | INIT_LIST_HEAD(&workers->worker_list); | ||
| 456 | INIT_LIST_HEAD(&workers->idle_list); | ||
| 457 | INIT_LIST_HEAD(&workers->order_list); | ||
| 458 | INIT_LIST_HEAD(&workers->prio_order_list); | ||
| 459 | spin_lock_init(&workers->lock); | ||
| 460 | spin_lock_init(&workers->order_lock); | ||
| 461 | workers->max_workers = max; | ||
| 462 | workers->idle_thresh = 32; | ||
| 463 | workers->name = name; | ||
| 464 | workers->ordered = 0; | ||
| 465 | workers->atomic_start_pending = 0; | ||
| 466 | workers->atomic_worker_start = async_helper; | ||
| 467 | workers->stopping = 0; | ||
| 468 | } | ||
| 469 | |||
| 470 | /* | ||
| 471 | * starts new worker threads. This does not enforce the max worker | ||
| 472 | * count in case you need to temporarily go past it. | ||
| 473 | */ | ||
| 474 | static int __btrfs_start_workers(struct btrfs_workers *workers) | ||
| 475 | { | ||
| 476 | struct btrfs_worker_thread *worker; | ||
| 477 | int ret = 0; | ||
| 478 | |||
| 479 | worker = kzalloc(sizeof(*worker), GFP_NOFS); | ||
| 480 | if (!worker) { | ||
| 481 | ret = -ENOMEM; | ||
| 482 | goto fail; | ||
| 483 | } | ||
| 484 | |||
| 485 | INIT_LIST_HEAD(&worker->pending); | ||
| 486 | INIT_LIST_HEAD(&worker->prio_pending); | ||
| 487 | INIT_LIST_HEAD(&worker->worker_list); | ||
| 488 | spin_lock_init(&worker->lock); | ||
| 489 | |||
| 490 | atomic_set(&worker->num_pending, 0); | ||
| 491 | atomic_set(&worker->refs, 1); | ||
| 492 | worker->workers = workers; | ||
| 493 | worker->task = kthread_create(worker_loop, worker, | ||
| 494 | "btrfs-%s-%d", workers->name, | ||
| 495 | workers->num_workers + 1); | ||
| 496 | if (IS_ERR(worker->task)) { | ||
| 497 | ret = PTR_ERR(worker->task); | ||
| 498 | goto fail; | ||
| 499 | } | ||
| 500 | |||
| 501 | spin_lock_irq(&workers->lock); | ||
| 502 | if (workers->stopping) { | ||
| 503 | spin_unlock_irq(&workers->lock); | ||
| 504 | ret = -EINVAL; | ||
| 505 | goto fail_kthread; | ||
| 506 | } | ||
| 507 | list_add_tail(&worker->worker_list, &workers->idle_list); | ||
| 508 | worker->idle = 1; | ||
| 509 | workers->num_workers++; | ||
| 510 | workers->num_workers_starting--; | ||
| 511 | WARN_ON(workers->num_workers_starting < 0); | ||
| 512 | spin_unlock_irq(&workers->lock); | ||
| 513 | |||
| 514 | wake_up_process(worker->task); | ||
| 515 | return 0; | ||
| 516 | |||
| 517 | fail_kthread: | ||
| 518 | kthread_stop(worker->task); | ||
| 519 | fail: | ||
| 520 | kfree(worker); | ||
| 521 | spin_lock_irq(&workers->lock); | ||
| 522 | workers->num_workers_starting--; | ||
| 523 | spin_unlock_irq(&workers->lock); | ||
| 524 | return ret; | ||
| 525 | } | ||
| 526 | |||
| 527 | int btrfs_start_workers(struct btrfs_workers *workers) | ||
| 528 | { | ||
| 529 | spin_lock_irq(&workers->lock); | ||
| 530 | workers->num_workers_starting++; | ||
| 531 | spin_unlock_irq(&workers->lock); | ||
| 532 | return __btrfs_start_workers(workers); | ||
| 533 | } | ||
| 534 | |||
| 535 | /* | ||
| 536 | * run through the list and find a worker thread that doesn't have a lot | ||
| 537 | * to do right now. This can return null if we aren't yet at the thread | ||
| 538 | * count limit and all of the threads are busy. | ||
| 539 | */ | ||
| 540 | static struct btrfs_worker_thread *next_worker(struct btrfs_workers *workers) | ||
| 541 | { | ||
| 542 | struct btrfs_worker_thread *worker; | ||
| 543 | struct list_head *next; | ||
| 544 | int enforce_min; | ||
| 545 | |||
| 546 | enforce_min = (workers->num_workers + workers->num_workers_starting) < | ||
| 547 | workers->max_workers; | ||
| 548 | |||
| 549 | /* | ||
| 550 | * if we find an idle thread, don't move it to the end of the | ||
| 551 | * idle list. This improves the chance that the next submission | ||
| 552 | * will reuse the same thread, and maybe catch it while it is still | ||
| 553 | * working | ||
| 554 | */ | ||
| 555 | if (!list_empty(&workers->idle_list)) { | ||
| 556 | next = workers->idle_list.next; | ||
| 557 | worker = list_entry(next, struct btrfs_worker_thread, | ||
| 558 | worker_list); | ||
| 559 | return worker; | ||
| 560 | } | ||
| 561 | if (enforce_min || list_empty(&workers->worker_list)) | ||
| 562 | return NULL; | ||
| 563 | |||
| 564 | /* | ||
| 565 | * if we pick a busy task, move the task to the end of the list. | ||
| 566 | * hopefully this will keep things somewhat evenly balanced. | ||
| 567 | * Do the move in batches based on the sequence number. This groups | ||
| 568 | * requests submitted at roughly the same time onto the same worker. | ||
| 569 | */ | ||
| 570 | next = workers->worker_list.next; | ||
| 571 | worker = list_entry(next, struct btrfs_worker_thread, worker_list); | ||
| 572 | worker->sequence++; | ||
| 573 | |||
| 574 | if (worker->sequence % workers->idle_thresh == 0) | ||
| 575 | list_move_tail(next, &workers->worker_list); | ||
| 576 | return worker; | ||
| 577 | } | ||
| 578 | |||
| 579 | /* | ||
| 580 | * selects a worker thread to take the next job. This will either find | ||
| 581 | * an idle worker, start a new worker up to the max count, or just return | ||
| 582 | * one of the existing busy workers. | ||
| 583 | */ | ||
| 584 | static struct btrfs_worker_thread *find_worker(struct btrfs_workers *workers) | ||
| 585 | { | ||
| 586 | struct btrfs_worker_thread *worker; | ||
| 587 | unsigned long flags; | ||
| 588 | struct list_head *fallback; | ||
| 589 | int ret; | ||
| 590 | |||
| 591 | spin_lock_irqsave(&workers->lock, flags); | ||
| 592 | again: | ||
| 593 | worker = next_worker(workers); | ||
| 594 | |||
| 595 | if (!worker) { | ||
| 596 | if (workers->num_workers + workers->num_workers_starting >= | ||
| 597 | workers->max_workers) { | ||
| 598 | goto fallback; | ||
| 599 | } else if (workers->atomic_worker_start) { | ||
| 600 | workers->atomic_start_pending = 1; | ||
| 601 | goto fallback; | ||
| 602 | } else { | ||
| 603 | workers->num_workers_starting++; | ||
| 604 | spin_unlock_irqrestore(&workers->lock, flags); | ||
| 605 | /* we're below the limit, start another worker */ | ||
| 606 | ret = __btrfs_start_workers(workers); | ||
| 607 | spin_lock_irqsave(&workers->lock, flags); | ||
| 608 | if (ret) | ||
| 609 | goto fallback; | ||
| 610 | goto again; | ||
| 611 | } | ||
| 612 | } | ||
| 613 | goto found; | ||
| 614 | |||
| 615 | fallback: | ||
| 616 | fallback = NULL; | ||
| 617 | /* | ||
| 618 | * we have failed to find any workers, just | ||
| 619 | * return the first one we can find. | ||
| 620 | */ | ||
| 621 | if (!list_empty(&workers->worker_list)) | ||
| 622 | fallback = workers->worker_list.next; | ||
| 623 | if (!list_empty(&workers->idle_list)) | ||
| 624 | fallback = workers->idle_list.next; | ||
| 625 | BUG_ON(!fallback); | ||
| 626 | worker = list_entry(fallback, | ||
| 627 | struct btrfs_worker_thread, worker_list); | ||
| 628 | found: | ||
| 629 | /* | ||
| 630 | * this makes sure the worker doesn't exit before it is placed | ||
| 631 | * onto a busy/idle list | ||
| 632 | */ | ||
| 633 | atomic_inc(&worker->num_pending); | ||
| 634 | spin_unlock_irqrestore(&workers->lock, flags); | ||
| 635 | return worker; | ||
| 636 | } | ||
| 637 | |||
| 638 | /* | ||
| 639 | * btrfs_requeue_work just puts the work item back on the tail of the list | ||
| 640 | * it was taken from. It is intended for use with long running work functions | ||
| 641 | * that make some progress and want to give the cpu up for others. | ||
| 642 | */ | ||
| 643 | void btrfs_requeue_work(struct btrfs_work *work) | ||
| 644 | { | ||
| 645 | struct btrfs_worker_thread *worker = work->worker; | ||
| 646 | unsigned long flags; | ||
| 647 | int wake = 0; | ||
| 648 | |||
| 649 | if (test_and_set_bit(WORK_QUEUED_BIT, &work->flags)) | ||
| 650 | return; | ||
| 651 | |||
| 652 | spin_lock_irqsave(&worker->lock, flags); | ||
| 653 | if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags)) | ||
| 654 | list_add_tail(&work->list, &worker->prio_pending); | ||
| 655 | else | ||
| 656 | list_add_tail(&work->list, &worker->pending); | ||
| 657 | atomic_inc(&worker->num_pending); | ||
| 658 | |||
| 659 | /* by definition we're busy, take ourselves off the idle | ||
| 660 | * list | ||
| 661 | */ | ||
| 662 | if (worker->idle) { | ||
| 663 | spin_lock(&worker->workers->lock); | ||
| 664 | worker->idle = 0; | ||
| 665 | list_move_tail(&worker->worker_list, | ||
| 666 | &worker->workers->worker_list); | ||
| 667 | spin_unlock(&worker->workers->lock); | ||
| 668 | } | ||
| 669 | if (!worker->working) { | ||
| 670 | wake = 1; | ||
| 671 | worker->working = 1; | ||
| 672 | } | ||
| 673 | |||
| 674 | if (wake) | ||
| 675 | wake_up_process(worker->task); | ||
| 676 | spin_unlock_irqrestore(&worker->lock, flags); | ||
| 677 | } | ||
| 678 | |||
| 679 | void btrfs_set_work_high_prio(struct btrfs_work *work) | ||
| 680 | { | ||
| 681 | set_bit(WORK_HIGH_PRIO_BIT, &work->flags); | ||
| 682 | } | ||
| 683 | |||
| 684 | /* | ||
| 685 | * places a struct btrfs_work into the pending queue of one of the kthreads | ||
| 686 | */ | ||
| 687 | void btrfs_queue_worker(struct btrfs_workers *workers, struct btrfs_work *work) | ||
| 688 | { | ||
| 689 | struct btrfs_worker_thread *worker; | ||
| 690 | unsigned long flags; | ||
| 691 | int wake = 0; | ||
| 692 | |||
| 693 | /* don't requeue something already on a list */ | ||
| 694 | if (test_and_set_bit(WORK_QUEUED_BIT, &work->flags)) | ||
| 695 | return; | ||
| 696 | |||
| 697 | worker = find_worker(workers); | ||
| 698 | if (workers->ordered) { | ||
| 699 | /* | ||
| 700 | * you're not allowed to do ordered queues from an | ||
| 701 | * interrupt handler | ||
| 702 | */ | ||
| 703 | spin_lock(&workers->order_lock); | ||
| 704 | if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags)) { | ||
| 705 | list_add_tail(&work->order_list, | ||
| 706 | &workers->prio_order_list); | ||
| 707 | } else { | ||
| 708 | list_add_tail(&work->order_list, &workers->order_list); | ||
| 709 | } | ||
| 710 | spin_unlock(&workers->order_lock); | ||
| 711 | } else { | ||
| 712 | INIT_LIST_HEAD(&work->order_list); | ||
| 713 | } | ||
| 714 | |||
| 715 | spin_lock_irqsave(&worker->lock, flags); | ||
| 716 | |||
| 717 | if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags)) | ||
| 718 | list_add_tail(&work->list, &worker->prio_pending); | ||
| 719 | else | ||
| 720 | list_add_tail(&work->list, &worker->pending); | ||
| 721 | check_busy_worker(worker); | ||
| 722 | |||
| 723 | /* | ||
| 724 | * avoid calling into wake_up_process if this thread has already | ||
| 725 | * been kicked | ||
| 726 | */ | ||
| 727 | if (!worker->working) | ||
| 728 | wake = 1; | ||
| 729 | worker->working = 1; | ||
| 730 | |||
| 731 | if (wake) | ||
| 732 | wake_up_process(worker->task); | ||
| 733 | spin_unlock_irqrestore(&worker->lock, flags); | ||
| 734 | } | ||
| 735 | |||
| 736 | struct __btrfs_workqueue_struct { | 35 | struct __btrfs_workqueue_struct { |
| 737 | struct workqueue_struct *normal_wq; | 36 | struct workqueue_struct *normal_wq; |
| 738 | /* List head pointing to ordered work list */ | 37 | /* List head pointing to ordered work list */ |
diff --git a/fs/btrfs/async-thread.h b/fs/btrfs/async-thread.h index 3129d8a6128b..ab05904f791c 100644 --- a/fs/btrfs/async-thread.h +++ b/fs/btrfs/async-thread.h | |||
| @@ -20,106 +20,6 @@ | |||
| 20 | #ifndef __BTRFS_ASYNC_THREAD_ | 20 | #ifndef __BTRFS_ASYNC_THREAD_ |
| 21 | #define __BTRFS_ASYNC_THREAD_ | 21 | #define __BTRFS_ASYNC_THREAD_ |
| 22 | 22 | ||
| 23 | struct btrfs_worker_thread; | ||
| 24 | |||
| 25 | /* | ||
| 26 | * This is similar to a workqueue, but it is meant to spread the operations | ||
| 27 | * across all available cpus instead of just the CPU that was used to | ||
| 28 | * queue the work. There is also some batching introduced to try and | ||
| 29 | * cut down on context switches. | ||
| 30 | * | ||
| 31 | * By default threads are added on demand up to 2 * the number of cpus. | ||
| 32 | * Changing struct btrfs_workers->max_workers is one way to prevent | ||
| 33 | * demand creation of kthreads. | ||
| 34 | * | ||
| 35 | * the basic model of these worker threads is to embed a btrfs_work | ||
| 36 | * structure in your own data struct, and use container_of in a | ||
| 37 | * work function to get back to your data struct. | ||
| 38 | */ | ||
| 39 | struct btrfs_work { | ||
| 40 | /* | ||
| 41 | * func should be set to the function you want called | ||
| 42 | * your work struct is passed as the only arg | ||
| 43 | * | ||
| 44 | * ordered_func must be set for work sent to an ordered work queue, | ||
| 45 | * and it is called to complete a given work item in the same | ||
| 46 | * order they were sent to the queue. | ||
| 47 | */ | ||
| 48 | void (*func)(struct btrfs_work *work); | ||
| 49 | void (*ordered_func)(struct btrfs_work *work); | ||
| 50 | void (*ordered_free)(struct btrfs_work *work); | ||
| 51 | |||
| 52 | /* | ||
| 53 | * flags should be set to zero. It is used to make sure the | ||
| 54 | * struct is only inserted once into the list. | ||
| 55 | */ | ||
| 56 | unsigned long flags; | ||
| 57 | |||
| 58 | /* don't touch these */ | ||
| 59 | struct btrfs_worker_thread *worker; | ||
| 60 | struct list_head list; | ||
| 61 | struct list_head order_list; | ||
| 62 | }; | ||
| 63 | |||
| 64 | struct btrfs_workers { | ||
| 65 | /* current number of running workers */ | ||
| 66 | int num_workers; | ||
| 67 | |||
| 68 | int num_workers_starting; | ||
| 69 | |||
| 70 | /* max number of workers allowed. changed by btrfs_start_workers */ | ||
| 71 | int max_workers; | ||
| 72 | |||
| 73 | /* once a worker has this many requests or fewer, it is idle */ | ||
| 74 | int idle_thresh; | ||
| 75 | |||
| 76 | /* force completions in the order they were queued */ | ||
| 77 | int ordered; | ||
| 78 | |||
| 79 | /* more workers required, but in an interrupt handler */ | ||
| 80 | int atomic_start_pending; | ||
| 81 | |||
| 82 | /* | ||
| 83 | * are we allowed to sleep while starting workers or are we required | ||
| 84 | * to start them at a later time? If we can't sleep, this indicates | ||
| 85 | * which queue we need to use to schedule thread creation. | ||
| 86 | */ | ||
| 87 | struct btrfs_workers *atomic_worker_start; | ||
| 88 | |||
| 89 | /* list with all the work threads. The workers on the idle thread | ||
| 90 | * may be actively servicing jobs, but they haven't yet hit the | ||
| 91 | * idle thresh limit above. | ||
| 92 | */ | ||
| 93 | struct list_head worker_list; | ||
| 94 | struct list_head idle_list; | ||
| 95 | |||
| 96 | /* | ||
| 97 | * when operating in ordered mode, this maintains the list | ||
| 98 | * of work items waiting for completion | ||
| 99 | */ | ||
| 100 | struct list_head order_list; | ||
| 101 | struct list_head prio_order_list; | ||
| 102 | |||
| 103 | /* lock for finding the next worker thread to queue on */ | ||
| 104 | spinlock_t lock; | ||
| 105 | |||
| 106 | /* lock for the ordered lists */ | ||
| 107 | spinlock_t order_lock; | ||
| 108 | |||
| 109 | /* extra name for this worker, used for current->name */ | ||
| 110 | char *name; | ||
| 111 | |||
| 112 | int stopping; | ||
| 113 | }; | ||
| 114 | |||
| 115 | void btrfs_queue_worker(struct btrfs_workers *workers, struct btrfs_work *work); | ||
| 116 | int btrfs_start_workers(struct btrfs_workers *workers); | ||
| 117 | void btrfs_stop_workers(struct btrfs_workers *workers); | ||
| 118 | void btrfs_init_workers(struct btrfs_workers *workers, char *name, int max, | ||
| 119 | struct btrfs_workers *async_starter); | ||
| 120 | void btrfs_requeue_work(struct btrfs_work *work); | ||
| 121 | void btrfs_set_work_high_prio(struct btrfs_work *work); | ||
| 122 | |||
| 123 | struct btrfs_workqueue_struct; | 23 | struct btrfs_workqueue_struct; |
| 124 | /* Internal use only */ | 24 | /* Internal use only */ |
| 125 | struct __btrfs_workqueue_struct; | 25 | struct __btrfs_workqueue_struct; |
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index a98f86ac187e..5a8c77a441ba 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h | |||
| @@ -1504,7 +1504,6 @@ struct btrfs_fs_info { | |||
| 1504 | * A third pool does submit_bio to avoid deadlocking with the other | 1504 | * A third pool does submit_bio to avoid deadlocking with the other |
| 1505 | * two | 1505 | * two |
| 1506 | */ | 1506 | */ |
| 1507 | struct btrfs_workers generic_worker; | ||
| 1508 | struct btrfs_workqueue_struct *workers; | 1507 | struct btrfs_workqueue_struct *workers; |
| 1509 | struct btrfs_workqueue_struct *delalloc_workers; | 1508 | struct btrfs_workqueue_struct *delalloc_workers; |
| 1510 | struct btrfs_workqueue_struct *flush_workers; | 1509 | struct btrfs_workqueue_struct *flush_workers; |
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 9aaf9c309b54..c80d9507171c 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c | |||
| @@ -1994,7 +1994,6 @@ static noinline int next_root_backup(struct btrfs_fs_info *info, | |||
| 1994 | /* helper to cleanup workers */ | 1994 | /* helper to cleanup workers */ |
| 1995 | static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info) | 1995 | static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info) |
| 1996 | { | 1996 | { |
| 1997 | btrfs_stop_workers(&fs_info->generic_worker); | ||
| 1998 | btrfs_destroy_workqueue(fs_info->fixup_workers); | 1997 | btrfs_destroy_workqueue(fs_info->fixup_workers); |
| 1999 | btrfs_destroy_workqueue(fs_info->delalloc_workers); | 1998 | btrfs_destroy_workqueue(fs_info->delalloc_workers); |
| 2000 | btrfs_destroy_workqueue(fs_info->workers); | 1999 | btrfs_destroy_workqueue(fs_info->workers); |
| @@ -2472,8 +2471,6 @@ int open_ctree(struct super_block *sb, | |||
| 2472 | } | 2471 | } |
| 2473 | 2472 | ||
| 2474 | max_active = fs_info->thread_pool_size; | 2473 | max_active = fs_info->thread_pool_size; |
| 2475 | btrfs_init_workers(&fs_info->generic_worker, | ||
| 2476 | "genwork", 1, NULL); | ||
| 2477 | 2474 | ||
| 2478 | fs_info->workers = | 2475 | fs_info->workers = |
| 2479 | btrfs_alloc_workqueue("worker", flags | WQ_HIGHPRI, | 2476 | btrfs_alloc_workqueue("worker", flags | WQ_HIGHPRI, |
| @@ -2526,15 +2523,6 @@ int open_ctree(struct super_block *sb, | |||
| 2526 | fs_info->qgroup_rescan_workers = | 2523 | fs_info->qgroup_rescan_workers = |
| 2527 | btrfs_alloc_workqueue("qgroup-rescan", flags, 1, 0); | 2524 | btrfs_alloc_workqueue("qgroup-rescan", flags, 1, 0); |
| 2528 | 2525 | ||
| 2529 | /* | ||
| 2530 | * btrfs_start_workers can really only fail because of ENOMEM so just | ||
| 2531 | * return -ENOMEM if any of these fail. | ||
| 2532 | */ | ||
| 2533 | ret = btrfs_start_workers(&fs_info->generic_worker); | ||
| 2534 | if (ret) { | ||
| 2535 | err = -ENOMEM; | ||
| 2536 | goto fail_sb_buffer; | ||
| 2537 | } | ||
| 2538 | if (!(fs_info->workers && fs_info->delalloc_workers && | 2526 | if (!(fs_info->workers && fs_info->delalloc_workers && |
| 2539 | fs_info->submit_workers && fs_info->flush_workers && | 2527 | fs_info->submit_workers && fs_info->flush_workers && |
| 2540 | fs_info->endio_workers && fs_info->endio_meta_workers && | 2528 | fs_info->endio_workers && fs_info->endio_meta_workers && |
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index aed1e11060a0..d4878ddba87a 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c | |||
| @@ -1305,13 +1305,6 @@ error_fs_info: | |||
| 1305 | return ERR_PTR(error); | 1305 | return ERR_PTR(error); |
| 1306 | } | 1306 | } |
| 1307 | 1307 | ||
| 1308 | static void btrfs_set_max_workers(struct btrfs_workers *workers, int new_limit) | ||
| 1309 | { | ||
| 1310 | spin_lock_irq(&workers->lock); | ||
| 1311 | workers->max_workers = new_limit; | ||
| 1312 | spin_unlock_irq(&workers->lock); | ||
| 1313 | } | ||
| 1314 | |||
| 1315 | static void btrfs_resize_thread_pool(struct btrfs_fs_info *fs_info, | 1308 | static void btrfs_resize_thread_pool(struct btrfs_fs_info *fs_info, |
| 1316 | int new_pool_size, int old_pool_size) | 1309 | int new_pool_size, int old_pool_size) |
| 1317 | { | 1310 | { |
| @@ -1323,7 +1316,6 @@ static void btrfs_resize_thread_pool(struct btrfs_fs_info *fs_info, | |||
| 1323 | btrfs_info(fs_info, "resize thread pool %d -> %d", | 1316 | btrfs_info(fs_info, "resize thread pool %d -> %d", |
| 1324 | old_pool_size, new_pool_size); | 1317 | old_pool_size, new_pool_size); |
| 1325 | 1318 | ||
| 1326 | btrfs_set_max_workers(&fs_info->generic_worker, new_pool_size); | ||
| 1327 | btrfs_workqueue_set_max(fs_info->workers, new_pool_size); | 1319 | btrfs_workqueue_set_max(fs_info->workers, new_pool_size); |
| 1328 | btrfs_workqueue_set_max(fs_info->delalloc_workers, new_pool_size); | 1320 | btrfs_workqueue_set_max(fs_info->delalloc_workers, new_pool_size); |
| 1329 | btrfs_workqueue_set_max(fs_info->submit_workers, new_pool_size); | 1321 | btrfs_workqueue_set_max(fs_info->submit_workers, new_pool_size); |
