diff options
author | Qu Wenruo <quwenruo@cn.fujitsu.com> | 2014-02-27 21:46:18 -0500 |
---|---|---|
committer | Josef Bacik <jbacik@fb.com> | 2014-03-10 15:17:15 -0400 |
commit | a046e9c88b0f46677923864295eac7c92cd962cb (patch) | |
tree | 72cff91c3bb1426473d5d70a2b99684c86102594 /fs | |
parent | 0339ef2f42bcfbb2d4021ad6f38fe20580082c85 (diff) |
btrfs: Cleanup the old btrfs_worker.
Since all the btrfs_worker is replaced with the newly created
btrfs_workqueue, the old codes can be easily remove.
Signed-off-by: Quwenruo <quwenruo@cn.fujitsu.com>
Tested-by: David Sterba <dsterba@suse.cz>
Signed-off-by: Josef Bacik <jbacik@fb.com>
Diffstat (limited to 'fs')
-rw-r--r-- | fs/btrfs/async-thread.c | 707 | ||||
-rw-r--r-- | fs/btrfs/async-thread.h | 100 | ||||
-rw-r--r-- | fs/btrfs/ctree.h | 1 | ||||
-rw-r--r-- | fs/btrfs/disk-io.c | 12 | ||||
-rw-r--r-- | fs/btrfs/super.c | 8 |
5 files changed, 3 insertions, 825 deletions
diff --git a/fs/btrfs/async-thread.c b/fs/btrfs/async-thread.c index 977bce2ec887..2a5f383c3636 100644 --- a/fs/btrfs/async-thread.c +++ b/fs/btrfs/async-thread.c | |||
@@ -25,714 +25,13 @@ | |||
25 | #include <linux/workqueue.h> | 25 | #include <linux/workqueue.h> |
26 | #include "async-thread.h" | 26 | #include "async-thread.h" |
27 | 27 | ||
28 | #define WORK_QUEUED_BIT 0 | 28 | #define WORK_DONE_BIT 0 |
29 | #define WORK_DONE_BIT 1 | 29 | #define WORK_ORDER_DONE_BIT 1 |
30 | #define WORK_ORDER_DONE_BIT 2 | 30 | #define WORK_HIGH_PRIO_BIT 2 |
31 | #define WORK_HIGH_PRIO_BIT 3 | ||
32 | 31 | ||
33 | #define NO_THRESHOLD (-1) | 32 | #define NO_THRESHOLD (-1) |
34 | #define DFT_THRESHOLD (32) | 33 | #define DFT_THRESHOLD (32) |
35 | 34 | ||
36 | /* | ||
37 | * container for the kthread task pointer and the list of pending work | ||
38 | * One of these is allocated per thread. | ||
39 | */ | ||
40 | struct btrfs_worker_thread { | ||
41 | /* pool we belong to */ | ||
42 | struct btrfs_workers *workers; | ||
43 | |||
44 | /* list of struct btrfs_work that are waiting for service */ | ||
45 | struct list_head pending; | ||
46 | struct list_head prio_pending; | ||
47 | |||
48 | /* list of worker threads from struct btrfs_workers */ | ||
49 | struct list_head worker_list; | ||
50 | |||
51 | /* kthread */ | ||
52 | struct task_struct *task; | ||
53 | |||
54 | /* number of things on the pending list */ | ||
55 | atomic_t num_pending; | ||
56 | |||
57 | /* reference counter for this struct */ | ||
58 | atomic_t refs; | ||
59 | |||
60 | unsigned long sequence; | ||
61 | |||
62 | /* protects the pending list. */ | ||
63 | spinlock_t lock; | ||
64 | |||
65 | /* set to non-zero when this thread is already awake and kicking */ | ||
66 | int working; | ||
67 | |||
68 | /* are we currently idle */ | ||
69 | int idle; | ||
70 | }; | ||
71 | |||
72 | static int __btrfs_start_workers(struct btrfs_workers *workers); | ||
73 | |||
74 | /* | ||
75 | * btrfs_start_workers uses kthread_run, which can block waiting for memory | ||
76 | * for a very long time. It will actually throttle on page writeback, | ||
77 | * and so it may not make progress until after our btrfs worker threads | ||
78 | * process all of the pending work structs in their queue | ||
79 | * | ||
80 | * This means we can't use btrfs_start_workers from inside a btrfs worker | ||
81 | * thread that is used as part of cleaning dirty memory, which pretty much | ||
82 | * involves all of the worker threads. | ||
83 | * | ||
84 | * Instead we have a helper queue who never has more than one thread | ||
85 | * where we scheduler thread start operations. This worker_start struct | ||
86 | * is used to contain the work and hold a pointer to the queue that needs | ||
87 | * another worker. | ||
88 | */ | ||
89 | struct worker_start { | ||
90 | struct btrfs_work work; | ||
91 | struct btrfs_workers *queue; | ||
92 | }; | ||
93 | |||
94 | static void start_new_worker_func(struct btrfs_work *work) | ||
95 | { | ||
96 | struct worker_start *start; | ||
97 | start = container_of(work, struct worker_start, work); | ||
98 | __btrfs_start_workers(start->queue); | ||
99 | kfree(start); | ||
100 | } | ||
101 | |||
102 | /* | ||
103 | * helper function to move a thread onto the idle list after it | ||
104 | * has finished some requests. | ||
105 | */ | ||
106 | static void check_idle_worker(struct btrfs_worker_thread *worker) | ||
107 | { | ||
108 | if (!worker->idle && atomic_read(&worker->num_pending) < | ||
109 | worker->workers->idle_thresh / 2) { | ||
110 | unsigned long flags; | ||
111 | spin_lock_irqsave(&worker->workers->lock, flags); | ||
112 | worker->idle = 1; | ||
113 | |||
114 | /* the list may be empty if the worker is just starting */ | ||
115 | if (!list_empty(&worker->worker_list) && | ||
116 | !worker->workers->stopping) { | ||
117 | list_move(&worker->worker_list, | ||
118 | &worker->workers->idle_list); | ||
119 | } | ||
120 | spin_unlock_irqrestore(&worker->workers->lock, flags); | ||
121 | } | ||
122 | } | ||
123 | |||
124 | /* | ||
125 | * helper function to move a thread off the idle list after new | ||
126 | * pending work is added. | ||
127 | */ | ||
128 | static void check_busy_worker(struct btrfs_worker_thread *worker) | ||
129 | { | ||
130 | if (worker->idle && atomic_read(&worker->num_pending) >= | ||
131 | worker->workers->idle_thresh) { | ||
132 | unsigned long flags; | ||
133 | spin_lock_irqsave(&worker->workers->lock, flags); | ||
134 | worker->idle = 0; | ||
135 | |||
136 | if (!list_empty(&worker->worker_list) && | ||
137 | !worker->workers->stopping) { | ||
138 | list_move_tail(&worker->worker_list, | ||
139 | &worker->workers->worker_list); | ||
140 | } | ||
141 | spin_unlock_irqrestore(&worker->workers->lock, flags); | ||
142 | } | ||
143 | } | ||
144 | |||
145 | static void check_pending_worker_creates(struct btrfs_worker_thread *worker) | ||
146 | { | ||
147 | struct btrfs_workers *workers = worker->workers; | ||
148 | struct worker_start *start; | ||
149 | unsigned long flags; | ||
150 | |||
151 | rmb(); | ||
152 | if (!workers->atomic_start_pending) | ||
153 | return; | ||
154 | |||
155 | start = kzalloc(sizeof(*start), GFP_NOFS); | ||
156 | if (!start) | ||
157 | return; | ||
158 | |||
159 | start->work.func = start_new_worker_func; | ||
160 | start->queue = workers; | ||
161 | |||
162 | spin_lock_irqsave(&workers->lock, flags); | ||
163 | if (!workers->atomic_start_pending) | ||
164 | goto out; | ||
165 | |||
166 | workers->atomic_start_pending = 0; | ||
167 | if (workers->num_workers + workers->num_workers_starting >= | ||
168 | workers->max_workers) | ||
169 | goto out; | ||
170 | |||
171 | workers->num_workers_starting += 1; | ||
172 | spin_unlock_irqrestore(&workers->lock, flags); | ||
173 | btrfs_queue_worker(workers->atomic_worker_start, &start->work); | ||
174 | return; | ||
175 | |||
176 | out: | ||
177 | kfree(start); | ||
178 | spin_unlock_irqrestore(&workers->lock, flags); | ||
179 | } | ||
180 | |||
181 | static noinline void run_ordered_completions(struct btrfs_workers *workers, | ||
182 | struct btrfs_work *work) | ||
183 | { | ||
184 | if (!workers->ordered) | ||
185 | return; | ||
186 | |||
187 | set_bit(WORK_DONE_BIT, &work->flags); | ||
188 | |||
189 | spin_lock(&workers->order_lock); | ||
190 | |||
191 | while (1) { | ||
192 | if (!list_empty(&workers->prio_order_list)) { | ||
193 | work = list_entry(workers->prio_order_list.next, | ||
194 | struct btrfs_work, order_list); | ||
195 | } else if (!list_empty(&workers->order_list)) { | ||
196 | work = list_entry(workers->order_list.next, | ||
197 | struct btrfs_work, order_list); | ||
198 | } else { | ||
199 | break; | ||
200 | } | ||
201 | if (!test_bit(WORK_DONE_BIT, &work->flags)) | ||
202 | break; | ||
203 | |||
204 | /* we are going to call the ordered done function, but | ||
205 | * we leave the work item on the list as a barrier so | ||
206 | * that later work items that are done don't have their | ||
207 | * functions called before this one returns | ||
208 | */ | ||
209 | if (test_and_set_bit(WORK_ORDER_DONE_BIT, &work->flags)) | ||
210 | break; | ||
211 | |||
212 | spin_unlock(&workers->order_lock); | ||
213 | |||
214 | work->ordered_func(work); | ||
215 | |||
216 | /* now take the lock again and drop our item from the list */ | ||
217 | spin_lock(&workers->order_lock); | ||
218 | list_del(&work->order_list); | ||
219 | spin_unlock(&workers->order_lock); | ||
220 | |||
221 | /* | ||
222 | * we don't want to call the ordered free functions | ||
223 | * with the lock held though | ||
224 | */ | ||
225 | work->ordered_free(work); | ||
226 | spin_lock(&workers->order_lock); | ||
227 | } | ||
228 | |||
229 | spin_unlock(&workers->order_lock); | ||
230 | } | ||
231 | |||
232 | static void put_worker(struct btrfs_worker_thread *worker) | ||
233 | { | ||
234 | if (atomic_dec_and_test(&worker->refs)) | ||
235 | kfree(worker); | ||
236 | } | ||
237 | |||
238 | static int try_worker_shutdown(struct btrfs_worker_thread *worker) | ||
239 | { | ||
240 | int freeit = 0; | ||
241 | |||
242 | spin_lock_irq(&worker->lock); | ||
243 | spin_lock(&worker->workers->lock); | ||
244 | if (worker->workers->num_workers > 1 && | ||
245 | worker->idle && | ||
246 | !worker->working && | ||
247 | !list_empty(&worker->worker_list) && | ||
248 | list_empty(&worker->prio_pending) && | ||
249 | list_empty(&worker->pending) && | ||
250 | atomic_read(&worker->num_pending) == 0) { | ||
251 | freeit = 1; | ||
252 | list_del_init(&worker->worker_list); | ||
253 | worker->workers->num_workers--; | ||
254 | } | ||
255 | spin_unlock(&worker->workers->lock); | ||
256 | spin_unlock_irq(&worker->lock); | ||
257 | |||
258 | if (freeit) | ||
259 | put_worker(worker); | ||
260 | return freeit; | ||
261 | } | ||
262 | |||
263 | static struct btrfs_work *get_next_work(struct btrfs_worker_thread *worker, | ||
264 | struct list_head *prio_head, | ||
265 | struct list_head *head) | ||
266 | { | ||
267 | struct btrfs_work *work = NULL; | ||
268 | struct list_head *cur = NULL; | ||
269 | |||
270 | if (!list_empty(prio_head)) { | ||
271 | cur = prio_head->next; | ||
272 | goto out; | ||
273 | } | ||
274 | |||
275 | smp_mb(); | ||
276 | if (!list_empty(&worker->prio_pending)) | ||
277 | goto refill; | ||
278 | |||
279 | if (!list_empty(head)) { | ||
280 | cur = head->next; | ||
281 | goto out; | ||
282 | } | ||
283 | |||
284 | refill: | ||
285 | spin_lock_irq(&worker->lock); | ||
286 | list_splice_tail_init(&worker->prio_pending, prio_head); | ||
287 | list_splice_tail_init(&worker->pending, head); | ||
288 | |||
289 | if (!list_empty(prio_head)) | ||
290 | cur = prio_head->next; | ||
291 | else if (!list_empty(head)) | ||
292 | cur = head->next; | ||
293 | spin_unlock_irq(&worker->lock); | ||
294 | |||
295 | if (!cur) | ||
296 | goto out_fail; | ||
297 | |||
298 | out: | ||
299 | work = list_entry(cur, struct btrfs_work, list); | ||
300 | |||
301 | out_fail: | ||
302 | return work; | ||
303 | } | ||
304 | |||
305 | /* | ||
306 | * main loop for servicing work items | ||
307 | */ | ||
308 | static int worker_loop(void *arg) | ||
309 | { | ||
310 | struct btrfs_worker_thread *worker = arg; | ||
311 | struct list_head head; | ||
312 | struct list_head prio_head; | ||
313 | struct btrfs_work *work; | ||
314 | |||
315 | INIT_LIST_HEAD(&head); | ||
316 | INIT_LIST_HEAD(&prio_head); | ||
317 | |||
318 | do { | ||
319 | again: | ||
320 | while (1) { | ||
321 | |||
322 | |||
323 | work = get_next_work(worker, &prio_head, &head); | ||
324 | if (!work) | ||
325 | break; | ||
326 | |||
327 | list_del(&work->list); | ||
328 | clear_bit(WORK_QUEUED_BIT, &work->flags); | ||
329 | |||
330 | work->worker = worker; | ||
331 | |||
332 | work->func(work); | ||
333 | |||
334 | atomic_dec(&worker->num_pending); | ||
335 | /* | ||
336 | * unless this is an ordered work queue, | ||
337 | * 'work' was probably freed by func above. | ||
338 | */ | ||
339 | run_ordered_completions(worker->workers, work); | ||
340 | |||
341 | check_pending_worker_creates(worker); | ||
342 | cond_resched(); | ||
343 | } | ||
344 | |||
345 | spin_lock_irq(&worker->lock); | ||
346 | check_idle_worker(worker); | ||
347 | |||
348 | if (freezing(current)) { | ||
349 | worker->working = 0; | ||
350 | spin_unlock_irq(&worker->lock); | ||
351 | try_to_freeze(); | ||
352 | } else { | ||
353 | spin_unlock_irq(&worker->lock); | ||
354 | if (!kthread_should_stop()) { | ||
355 | cpu_relax(); | ||
356 | /* | ||
357 | * we've dropped the lock, did someone else | ||
358 | * jump_in? | ||
359 | */ | ||
360 | smp_mb(); | ||
361 | if (!list_empty(&worker->pending) || | ||
362 | !list_empty(&worker->prio_pending)) | ||
363 | continue; | ||
364 | |||
365 | /* | ||
366 | * this short schedule allows more work to | ||
367 | * come in without the queue functions | ||
368 | * needing to go through wake_up_process() | ||
369 | * | ||
370 | * worker->working is still 1, so nobody | ||
371 | * is going to try and wake us up | ||
372 | */ | ||
373 | schedule_timeout(1); | ||
374 | smp_mb(); | ||
375 | if (!list_empty(&worker->pending) || | ||
376 | !list_empty(&worker->prio_pending)) | ||
377 | continue; | ||
378 | |||
379 | if (kthread_should_stop()) | ||
380 | break; | ||
381 | |||
382 | /* still no more work?, sleep for real */ | ||
383 | spin_lock_irq(&worker->lock); | ||
384 | set_current_state(TASK_INTERRUPTIBLE); | ||
385 | if (!list_empty(&worker->pending) || | ||
386 | !list_empty(&worker->prio_pending)) { | ||
387 | spin_unlock_irq(&worker->lock); | ||
388 | set_current_state(TASK_RUNNING); | ||
389 | goto again; | ||
390 | } | ||
391 | |||
392 | /* | ||
393 | * this makes sure we get a wakeup when someone | ||
394 | * adds something new to the queue | ||
395 | */ | ||
396 | worker->working = 0; | ||
397 | spin_unlock_irq(&worker->lock); | ||
398 | |||
399 | if (!kthread_should_stop()) { | ||
400 | schedule_timeout(HZ * 120); | ||
401 | if (!worker->working && | ||
402 | try_worker_shutdown(worker)) { | ||
403 | return 0; | ||
404 | } | ||
405 | } | ||
406 | } | ||
407 | __set_current_state(TASK_RUNNING); | ||
408 | } | ||
409 | } while (!kthread_should_stop()); | ||
410 | return 0; | ||
411 | } | ||
412 | |||
413 | /* | ||
414 | * this will wait for all the worker threads to shutdown | ||
415 | */ | ||
416 | void btrfs_stop_workers(struct btrfs_workers *workers) | ||
417 | { | ||
418 | struct list_head *cur; | ||
419 | struct btrfs_worker_thread *worker; | ||
420 | int can_stop; | ||
421 | |||
422 | spin_lock_irq(&workers->lock); | ||
423 | workers->stopping = 1; | ||
424 | list_splice_init(&workers->idle_list, &workers->worker_list); | ||
425 | while (!list_empty(&workers->worker_list)) { | ||
426 | cur = workers->worker_list.next; | ||
427 | worker = list_entry(cur, struct btrfs_worker_thread, | ||
428 | worker_list); | ||
429 | |||
430 | atomic_inc(&worker->refs); | ||
431 | workers->num_workers -= 1; | ||
432 | if (!list_empty(&worker->worker_list)) { | ||
433 | list_del_init(&worker->worker_list); | ||
434 | put_worker(worker); | ||
435 | can_stop = 1; | ||
436 | } else | ||
437 | can_stop = 0; | ||
438 | spin_unlock_irq(&workers->lock); | ||
439 | if (can_stop) | ||
440 | kthread_stop(worker->task); | ||
441 | spin_lock_irq(&workers->lock); | ||
442 | put_worker(worker); | ||
443 | } | ||
444 | spin_unlock_irq(&workers->lock); | ||
445 | } | ||
446 | |||
447 | /* | ||
448 | * simple init on struct btrfs_workers | ||
449 | */ | ||
450 | void btrfs_init_workers(struct btrfs_workers *workers, char *name, int max, | ||
451 | struct btrfs_workers *async_helper) | ||
452 | { | ||
453 | workers->num_workers = 0; | ||
454 | workers->num_workers_starting = 0; | ||
455 | INIT_LIST_HEAD(&workers->worker_list); | ||
456 | INIT_LIST_HEAD(&workers->idle_list); | ||
457 | INIT_LIST_HEAD(&workers->order_list); | ||
458 | INIT_LIST_HEAD(&workers->prio_order_list); | ||
459 | spin_lock_init(&workers->lock); | ||
460 | spin_lock_init(&workers->order_lock); | ||
461 | workers->max_workers = max; | ||
462 | workers->idle_thresh = 32; | ||
463 | workers->name = name; | ||
464 | workers->ordered = 0; | ||
465 | workers->atomic_start_pending = 0; | ||
466 | workers->atomic_worker_start = async_helper; | ||
467 | workers->stopping = 0; | ||
468 | } | ||
469 | |||
470 | /* | ||
471 | * starts new worker threads. This does not enforce the max worker | ||
472 | * count in case you need to temporarily go past it. | ||
473 | */ | ||
474 | static int __btrfs_start_workers(struct btrfs_workers *workers) | ||
475 | { | ||
476 | struct btrfs_worker_thread *worker; | ||
477 | int ret = 0; | ||
478 | |||
479 | worker = kzalloc(sizeof(*worker), GFP_NOFS); | ||
480 | if (!worker) { | ||
481 | ret = -ENOMEM; | ||
482 | goto fail; | ||
483 | } | ||
484 | |||
485 | INIT_LIST_HEAD(&worker->pending); | ||
486 | INIT_LIST_HEAD(&worker->prio_pending); | ||
487 | INIT_LIST_HEAD(&worker->worker_list); | ||
488 | spin_lock_init(&worker->lock); | ||
489 | |||
490 | atomic_set(&worker->num_pending, 0); | ||
491 | atomic_set(&worker->refs, 1); | ||
492 | worker->workers = workers; | ||
493 | worker->task = kthread_create(worker_loop, worker, | ||
494 | "btrfs-%s-%d", workers->name, | ||
495 | workers->num_workers + 1); | ||
496 | if (IS_ERR(worker->task)) { | ||
497 | ret = PTR_ERR(worker->task); | ||
498 | goto fail; | ||
499 | } | ||
500 | |||
501 | spin_lock_irq(&workers->lock); | ||
502 | if (workers->stopping) { | ||
503 | spin_unlock_irq(&workers->lock); | ||
504 | ret = -EINVAL; | ||
505 | goto fail_kthread; | ||
506 | } | ||
507 | list_add_tail(&worker->worker_list, &workers->idle_list); | ||
508 | worker->idle = 1; | ||
509 | workers->num_workers++; | ||
510 | workers->num_workers_starting--; | ||
511 | WARN_ON(workers->num_workers_starting < 0); | ||
512 | spin_unlock_irq(&workers->lock); | ||
513 | |||
514 | wake_up_process(worker->task); | ||
515 | return 0; | ||
516 | |||
517 | fail_kthread: | ||
518 | kthread_stop(worker->task); | ||
519 | fail: | ||
520 | kfree(worker); | ||
521 | spin_lock_irq(&workers->lock); | ||
522 | workers->num_workers_starting--; | ||
523 | spin_unlock_irq(&workers->lock); | ||
524 | return ret; | ||
525 | } | ||
526 | |||
527 | int btrfs_start_workers(struct btrfs_workers *workers) | ||
528 | { | ||
529 | spin_lock_irq(&workers->lock); | ||
530 | workers->num_workers_starting++; | ||
531 | spin_unlock_irq(&workers->lock); | ||
532 | return __btrfs_start_workers(workers); | ||
533 | } | ||
534 | |||
535 | /* | ||
536 | * run through the list and find a worker thread that doesn't have a lot | ||
537 | * to do right now. This can return null if we aren't yet at the thread | ||
538 | * count limit and all of the threads are busy. | ||
539 | */ | ||
540 | static struct btrfs_worker_thread *next_worker(struct btrfs_workers *workers) | ||
541 | { | ||
542 | struct btrfs_worker_thread *worker; | ||
543 | struct list_head *next; | ||
544 | int enforce_min; | ||
545 | |||
546 | enforce_min = (workers->num_workers + workers->num_workers_starting) < | ||
547 | workers->max_workers; | ||
548 | |||
549 | /* | ||
550 | * if we find an idle thread, don't move it to the end of the | ||
551 | * idle list. This improves the chance that the next submission | ||
552 | * will reuse the same thread, and maybe catch it while it is still | ||
553 | * working | ||
554 | */ | ||
555 | if (!list_empty(&workers->idle_list)) { | ||
556 | next = workers->idle_list.next; | ||
557 | worker = list_entry(next, struct btrfs_worker_thread, | ||
558 | worker_list); | ||
559 | return worker; | ||
560 | } | ||
561 | if (enforce_min || list_empty(&workers->worker_list)) | ||
562 | return NULL; | ||
563 | |||
564 | /* | ||
565 | * if we pick a busy task, move the task to the end of the list. | ||
566 | * hopefully this will keep things somewhat evenly balanced. | ||
567 | * Do the move in batches based on the sequence number. This groups | ||
568 | * requests submitted at roughly the same time onto the same worker. | ||
569 | */ | ||
570 | next = workers->worker_list.next; | ||
571 | worker = list_entry(next, struct btrfs_worker_thread, worker_list); | ||
572 | worker->sequence++; | ||
573 | |||
574 | if (worker->sequence % workers->idle_thresh == 0) | ||
575 | list_move_tail(next, &workers->worker_list); | ||
576 | return worker; | ||
577 | } | ||
578 | |||
579 | /* | ||
580 | * selects a worker thread to take the next job. This will either find | ||
581 | * an idle worker, start a new worker up to the max count, or just return | ||
582 | * one of the existing busy workers. | ||
583 | */ | ||
584 | static struct btrfs_worker_thread *find_worker(struct btrfs_workers *workers) | ||
585 | { | ||
586 | struct btrfs_worker_thread *worker; | ||
587 | unsigned long flags; | ||
588 | struct list_head *fallback; | ||
589 | int ret; | ||
590 | |||
591 | spin_lock_irqsave(&workers->lock, flags); | ||
592 | again: | ||
593 | worker = next_worker(workers); | ||
594 | |||
595 | if (!worker) { | ||
596 | if (workers->num_workers + workers->num_workers_starting >= | ||
597 | workers->max_workers) { | ||
598 | goto fallback; | ||
599 | } else if (workers->atomic_worker_start) { | ||
600 | workers->atomic_start_pending = 1; | ||
601 | goto fallback; | ||
602 | } else { | ||
603 | workers->num_workers_starting++; | ||
604 | spin_unlock_irqrestore(&workers->lock, flags); | ||
605 | /* we're below the limit, start another worker */ | ||
606 | ret = __btrfs_start_workers(workers); | ||
607 | spin_lock_irqsave(&workers->lock, flags); | ||
608 | if (ret) | ||
609 | goto fallback; | ||
610 | goto again; | ||
611 | } | ||
612 | } | ||
613 | goto found; | ||
614 | |||
615 | fallback: | ||
616 | fallback = NULL; | ||
617 | /* | ||
618 | * we have failed to find any workers, just | ||
619 | * return the first one we can find. | ||
620 | */ | ||
621 | if (!list_empty(&workers->worker_list)) | ||
622 | fallback = workers->worker_list.next; | ||
623 | if (!list_empty(&workers->idle_list)) | ||
624 | fallback = workers->idle_list.next; | ||
625 | BUG_ON(!fallback); | ||
626 | worker = list_entry(fallback, | ||
627 | struct btrfs_worker_thread, worker_list); | ||
628 | found: | ||
629 | /* | ||
630 | * this makes sure the worker doesn't exit before it is placed | ||
631 | * onto a busy/idle list | ||
632 | */ | ||
633 | atomic_inc(&worker->num_pending); | ||
634 | spin_unlock_irqrestore(&workers->lock, flags); | ||
635 | return worker; | ||
636 | } | ||
637 | |||
638 | /* | ||
639 | * btrfs_requeue_work just puts the work item back on the tail of the list | ||
640 | * it was taken from. It is intended for use with long running work functions | ||
641 | * that make some progress and want to give the cpu up for others. | ||
642 | */ | ||
643 | void btrfs_requeue_work(struct btrfs_work *work) | ||
644 | { | ||
645 | struct btrfs_worker_thread *worker = work->worker; | ||
646 | unsigned long flags; | ||
647 | int wake = 0; | ||
648 | |||
649 | if (test_and_set_bit(WORK_QUEUED_BIT, &work->flags)) | ||
650 | return; | ||
651 | |||
652 | spin_lock_irqsave(&worker->lock, flags); | ||
653 | if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags)) | ||
654 | list_add_tail(&work->list, &worker->prio_pending); | ||
655 | else | ||
656 | list_add_tail(&work->list, &worker->pending); | ||
657 | atomic_inc(&worker->num_pending); | ||
658 | |||
659 | /* by definition we're busy, take ourselves off the idle | ||
660 | * list | ||
661 | */ | ||
662 | if (worker->idle) { | ||
663 | spin_lock(&worker->workers->lock); | ||
664 | worker->idle = 0; | ||
665 | list_move_tail(&worker->worker_list, | ||
666 | &worker->workers->worker_list); | ||
667 | spin_unlock(&worker->workers->lock); | ||
668 | } | ||
669 | if (!worker->working) { | ||
670 | wake = 1; | ||
671 | worker->working = 1; | ||
672 | } | ||
673 | |||
674 | if (wake) | ||
675 | wake_up_process(worker->task); | ||
676 | spin_unlock_irqrestore(&worker->lock, flags); | ||
677 | } | ||
678 | |||
679 | void btrfs_set_work_high_prio(struct btrfs_work *work) | ||
680 | { | ||
681 | set_bit(WORK_HIGH_PRIO_BIT, &work->flags); | ||
682 | } | ||
683 | |||
684 | /* | ||
685 | * places a struct btrfs_work into the pending queue of one of the kthreads | ||
686 | */ | ||
687 | void btrfs_queue_worker(struct btrfs_workers *workers, struct btrfs_work *work) | ||
688 | { | ||
689 | struct btrfs_worker_thread *worker; | ||
690 | unsigned long flags; | ||
691 | int wake = 0; | ||
692 | |||
693 | /* don't requeue something already on a list */ | ||
694 | if (test_and_set_bit(WORK_QUEUED_BIT, &work->flags)) | ||
695 | return; | ||
696 | |||
697 | worker = find_worker(workers); | ||
698 | if (workers->ordered) { | ||
699 | /* | ||
700 | * you're not allowed to do ordered queues from an | ||
701 | * interrupt handler | ||
702 | */ | ||
703 | spin_lock(&workers->order_lock); | ||
704 | if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags)) { | ||
705 | list_add_tail(&work->order_list, | ||
706 | &workers->prio_order_list); | ||
707 | } else { | ||
708 | list_add_tail(&work->order_list, &workers->order_list); | ||
709 | } | ||
710 | spin_unlock(&workers->order_lock); | ||
711 | } else { | ||
712 | INIT_LIST_HEAD(&work->order_list); | ||
713 | } | ||
714 | |||
715 | spin_lock_irqsave(&worker->lock, flags); | ||
716 | |||
717 | if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags)) | ||
718 | list_add_tail(&work->list, &worker->prio_pending); | ||
719 | else | ||
720 | list_add_tail(&work->list, &worker->pending); | ||
721 | check_busy_worker(worker); | ||
722 | |||
723 | /* | ||
724 | * avoid calling into wake_up_process if this thread has already | ||
725 | * been kicked | ||
726 | */ | ||
727 | if (!worker->working) | ||
728 | wake = 1; | ||
729 | worker->working = 1; | ||
730 | |||
731 | if (wake) | ||
732 | wake_up_process(worker->task); | ||
733 | spin_unlock_irqrestore(&worker->lock, flags); | ||
734 | } | ||
735 | |||
736 | struct __btrfs_workqueue_struct { | 35 | struct __btrfs_workqueue_struct { |
737 | struct workqueue_struct *normal_wq; | 36 | struct workqueue_struct *normal_wq; |
738 | /* List head pointing to ordered work list */ | 37 | /* List head pointing to ordered work list */ |
diff --git a/fs/btrfs/async-thread.h b/fs/btrfs/async-thread.h index 3129d8a6128b..ab05904f791c 100644 --- a/fs/btrfs/async-thread.h +++ b/fs/btrfs/async-thread.h | |||
@@ -20,106 +20,6 @@ | |||
20 | #ifndef __BTRFS_ASYNC_THREAD_ | 20 | #ifndef __BTRFS_ASYNC_THREAD_ |
21 | #define __BTRFS_ASYNC_THREAD_ | 21 | #define __BTRFS_ASYNC_THREAD_ |
22 | 22 | ||
23 | struct btrfs_worker_thread; | ||
24 | |||
25 | /* | ||
26 | * This is similar to a workqueue, but it is meant to spread the operations | ||
27 | * across all available cpus instead of just the CPU that was used to | ||
28 | * queue the work. There is also some batching introduced to try and | ||
29 | * cut down on context switches. | ||
30 | * | ||
31 | * By default threads are added on demand up to 2 * the number of cpus. | ||
32 | * Changing struct btrfs_workers->max_workers is one way to prevent | ||
33 | * demand creation of kthreads. | ||
34 | * | ||
35 | * the basic model of these worker threads is to embed a btrfs_work | ||
36 | * structure in your own data struct, and use container_of in a | ||
37 | * work function to get back to your data struct. | ||
38 | */ | ||
39 | struct btrfs_work { | ||
40 | /* | ||
41 | * func should be set to the function you want called | ||
42 | * your work struct is passed as the only arg | ||
43 | * | ||
44 | * ordered_func must be set for work sent to an ordered work queue, | ||
45 | * and it is called to complete a given work item in the same | ||
46 | * order they were sent to the queue. | ||
47 | */ | ||
48 | void (*func)(struct btrfs_work *work); | ||
49 | void (*ordered_func)(struct btrfs_work *work); | ||
50 | void (*ordered_free)(struct btrfs_work *work); | ||
51 | |||
52 | /* | ||
53 | * flags should be set to zero. It is used to make sure the | ||
54 | * struct is only inserted once into the list. | ||
55 | */ | ||
56 | unsigned long flags; | ||
57 | |||
58 | /* don't touch these */ | ||
59 | struct btrfs_worker_thread *worker; | ||
60 | struct list_head list; | ||
61 | struct list_head order_list; | ||
62 | }; | ||
63 | |||
64 | struct btrfs_workers { | ||
65 | /* current number of running workers */ | ||
66 | int num_workers; | ||
67 | |||
68 | int num_workers_starting; | ||
69 | |||
70 | /* max number of workers allowed. changed by btrfs_start_workers */ | ||
71 | int max_workers; | ||
72 | |||
73 | /* once a worker has this many requests or fewer, it is idle */ | ||
74 | int idle_thresh; | ||
75 | |||
76 | /* force completions in the order they were queued */ | ||
77 | int ordered; | ||
78 | |||
79 | /* more workers required, but in an interrupt handler */ | ||
80 | int atomic_start_pending; | ||
81 | |||
82 | /* | ||
83 | * are we allowed to sleep while starting workers or are we required | ||
84 | * to start them at a later time? If we can't sleep, this indicates | ||
85 | * which queue we need to use to schedule thread creation. | ||
86 | */ | ||
87 | struct btrfs_workers *atomic_worker_start; | ||
88 | |||
89 | /* list with all the work threads. The workers on the idle thread | ||
90 | * may be actively servicing jobs, but they haven't yet hit the | ||
91 | * idle thresh limit above. | ||
92 | */ | ||
93 | struct list_head worker_list; | ||
94 | struct list_head idle_list; | ||
95 | |||
96 | /* | ||
97 | * when operating in ordered mode, this maintains the list | ||
98 | * of work items waiting for completion | ||
99 | */ | ||
100 | struct list_head order_list; | ||
101 | struct list_head prio_order_list; | ||
102 | |||
103 | /* lock for finding the next worker thread to queue on */ | ||
104 | spinlock_t lock; | ||
105 | |||
106 | /* lock for the ordered lists */ | ||
107 | spinlock_t order_lock; | ||
108 | |||
109 | /* extra name for this worker, used for current->name */ | ||
110 | char *name; | ||
111 | |||
112 | int stopping; | ||
113 | }; | ||
114 | |||
115 | void btrfs_queue_worker(struct btrfs_workers *workers, struct btrfs_work *work); | ||
116 | int btrfs_start_workers(struct btrfs_workers *workers); | ||
117 | void btrfs_stop_workers(struct btrfs_workers *workers); | ||
118 | void btrfs_init_workers(struct btrfs_workers *workers, char *name, int max, | ||
119 | struct btrfs_workers *async_starter); | ||
120 | void btrfs_requeue_work(struct btrfs_work *work); | ||
121 | void btrfs_set_work_high_prio(struct btrfs_work *work); | ||
122 | |||
123 | struct btrfs_workqueue_struct; | 23 | struct btrfs_workqueue_struct; |
124 | /* Internal use only */ | 24 | /* Internal use only */ |
125 | struct __btrfs_workqueue_struct; | 25 | struct __btrfs_workqueue_struct; |
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index a98f86ac187e..5a8c77a441ba 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h | |||
@@ -1504,7 +1504,6 @@ struct btrfs_fs_info { | |||
1504 | * A third pool does submit_bio to avoid deadlocking with the other | 1504 | * A third pool does submit_bio to avoid deadlocking with the other |
1505 | * two | 1505 | * two |
1506 | */ | 1506 | */ |
1507 | struct btrfs_workers generic_worker; | ||
1508 | struct btrfs_workqueue_struct *workers; | 1507 | struct btrfs_workqueue_struct *workers; |
1509 | struct btrfs_workqueue_struct *delalloc_workers; | 1508 | struct btrfs_workqueue_struct *delalloc_workers; |
1510 | struct btrfs_workqueue_struct *flush_workers; | 1509 | struct btrfs_workqueue_struct *flush_workers; |
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 9aaf9c309b54..c80d9507171c 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c | |||
@@ -1994,7 +1994,6 @@ static noinline int next_root_backup(struct btrfs_fs_info *info, | |||
1994 | /* helper to cleanup workers */ | 1994 | /* helper to cleanup workers */ |
1995 | static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info) | 1995 | static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info) |
1996 | { | 1996 | { |
1997 | btrfs_stop_workers(&fs_info->generic_worker); | ||
1998 | btrfs_destroy_workqueue(fs_info->fixup_workers); | 1997 | btrfs_destroy_workqueue(fs_info->fixup_workers); |
1999 | btrfs_destroy_workqueue(fs_info->delalloc_workers); | 1998 | btrfs_destroy_workqueue(fs_info->delalloc_workers); |
2000 | btrfs_destroy_workqueue(fs_info->workers); | 1999 | btrfs_destroy_workqueue(fs_info->workers); |
@@ -2472,8 +2471,6 @@ int open_ctree(struct super_block *sb, | |||
2472 | } | 2471 | } |
2473 | 2472 | ||
2474 | max_active = fs_info->thread_pool_size; | 2473 | max_active = fs_info->thread_pool_size; |
2475 | btrfs_init_workers(&fs_info->generic_worker, | ||
2476 | "genwork", 1, NULL); | ||
2477 | 2474 | ||
2478 | fs_info->workers = | 2475 | fs_info->workers = |
2479 | btrfs_alloc_workqueue("worker", flags | WQ_HIGHPRI, | 2476 | btrfs_alloc_workqueue("worker", flags | WQ_HIGHPRI, |
@@ -2526,15 +2523,6 @@ int open_ctree(struct super_block *sb, | |||
2526 | fs_info->qgroup_rescan_workers = | 2523 | fs_info->qgroup_rescan_workers = |
2527 | btrfs_alloc_workqueue("qgroup-rescan", flags, 1, 0); | 2524 | btrfs_alloc_workqueue("qgroup-rescan", flags, 1, 0); |
2528 | 2525 | ||
2529 | /* | ||
2530 | * btrfs_start_workers can really only fail because of ENOMEM so just | ||
2531 | * return -ENOMEM if any of these fail. | ||
2532 | */ | ||
2533 | ret = btrfs_start_workers(&fs_info->generic_worker); | ||
2534 | if (ret) { | ||
2535 | err = -ENOMEM; | ||
2536 | goto fail_sb_buffer; | ||
2537 | } | ||
2538 | if (!(fs_info->workers && fs_info->delalloc_workers && | 2526 | if (!(fs_info->workers && fs_info->delalloc_workers && |
2539 | fs_info->submit_workers && fs_info->flush_workers && | 2527 | fs_info->submit_workers && fs_info->flush_workers && |
2540 | fs_info->endio_workers && fs_info->endio_meta_workers && | 2528 | fs_info->endio_workers && fs_info->endio_meta_workers && |
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index aed1e11060a0..d4878ddba87a 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c | |||
@@ -1305,13 +1305,6 @@ error_fs_info: | |||
1305 | return ERR_PTR(error); | 1305 | return ERR_PTR(error); |
1306 | } | 1306 | } |
1307 | 1307 | ||
1308 | static void btrfs_set_max_workers(struct btrfs_workers *workers, int new_limit) | ||
1309 | { | ||
1310 | spin_lock_irq(&workers->lock); | ||
1311 | workers->max_workers = new_limit; | ||
1312 | spin_unlock_irq(&workers->lock); | ||
1313 | } | ||
1314 | |||
1315 | static void btrfs_resize_thread_pool(struct btrfs_fs_info *fs_info, | 1308 | static void btrfs_resize_thread_pool(struct btrfs_fs_info *fs_info, |
1316 | int new_pool_size, int old_pool_size) | 1309 | int new_pool_size, int old_pool_size) |
1317 | { | 1310 | { |
@@ -1323,7 +1316,6 @@ static void btrfs_resize_thread_pool(struct btrfs_fs_info *fs_info, | |||
1323 | btrfs_info(fs_info, "resize thread pool %d -> %d", | 1316 | btrfs_info(fs_info, "resize thread pool %d -> %d", |
1324 | old_pool_size, new_pool_size); | 1317 | old_pool_size, new_pool_size); |
1325 | 1318 | ||
1326 | btrfs_set_max_workers(&fs_info->generic_worker, new_pool_size); | ||
1327 | btrfs_workqueue_set_max(fs_info->workers, new_pool_size); | 1319 | btrfs_workqueue_set_max(fs_info->workers, new_pool_size); |
1328 | btrfs_workqueue_set_max(fs_info->delalloc_workers, new_pool_size); | 1320 | btrfs_workqueue_set_max(fs_info->delalloc_workers, new_pool_size); |
1329 | btrfs_workqueue_set_max(fs_info->submit_workers, new_pool_size); | 1321 | btrfs_workqueue_set_max(fs_info->submit_workers, new_pool_size); |