aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/virtio
diff options
context:
space:
mode:
authorRafael Aquini <aquini@redhat.com>2012-12-11 19:02:45 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2012-12-11 20:22:27 -0500
commite22504296d4f64fbbbd741602ab47ee874649c18 (patch)
treea21e72dfae575091beb338aaedc939a3777e8828 /drivers/virtio
parentbf6bddf1924eaebf2beb85e4249a89dd16d4eed6 (diff)
virtio_balloon: introduce migration primitives to balloon pages
Memory fragmentation introduced by ballooning might reduce significantly the number of 2MB contiguous memory blocks that can be used within a guest, thus imposing performance penalties associated with the reduced number of transparent huge pages that could be used by the guest workload. Besides making balloon pages movable at allocation time and introducing the necessary primitives to perform balloon page migration/compaction, this patch also introduces the following locking scheme, in order to enhance the syncronization methods for accessing elements of struct virtio_balloon, thus providing protection against concurrent access introduced by parallel memory migration threads. - balloon_lock (mutex) : synchronizes the access demand to elements of struct virtio_balloon and its queue operations; [yongjun_wei@trendmicro.com.cn: fix missing unlock on error in fill_balloon()] [akpm@linux-foundation.org: avoid having multiple return points in fill_balloon()] [akpm@linux-foundation.org: fix printk warning]Signed-off-by: Rafael Aquini <aquini@redhat.com> Acked-by: Michael S. Tsirkin <mst@redhat.com> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Rik van Riel <riel@redhat.com> Cc: Mel Gorman <mel@csn.ul.ie> Cc: Andi Kleen <andi@firstfloor.org> Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> Cc: Minchan Kim <minchan@kernel.org> Signed-off-by: Wei Yongjun <yongjun_wei@trendmicro.com.cn> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'drivers/virtio')
-rw-r--r--drivers/virtio/virtio_balloon.c151
1 files changed, 132 insertions, 19 deletions
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 0908e6044333..2a70558b36ea 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -27,13 +27,15 @@
27#include <linux/delay.h> 27#include <linux/delay.h>
28#include <linux/slab.h> 28#include <linux/slab.h>
29#include <linux/module.h> 29#include <linux/module.h>
30#include <linux/balloon_compaction.h>
30 31
31/* 32/*
32 * Balloon device works in 4K page units. So each page is pointed to by 33 * Balloon device works in 4K page units. So each page is pointed to by
33 * multiple balloon pages. All memory counters in this driver are in balloon 34 * multiple balloon pages. All memory counters in this driver are in balloon
34 * page units. 35 * page units.
35 */ 36 */
36#define VIRTIO_BALLOON_PAGES_PER_PAGE (PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT) 37#define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
38#define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
37 39
38struct virtio_balloon 40struct virtio_balloon
39{ 41{
@@ -52,15 +54,19 @@ struct virtio_balloon
52 /* Number of balloon pages we've told the Host we're not using. */ 54 /* Number of balloon pages we've told the Host we're not using. */
53 unsigned int num_pages; 55 unsigned int num_pages;
54 /* 56 /*
55 * The pages we've told the Host we're not using. 57 * The pages we've told the Host we're not using are enqueued
58 * at vb_dev_info->pages list.
56 * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE 59 * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
57 * to num_pages above. 60 * to num_pages above.
58 */ 61 */
59 struct list_head pages; 62 struct balloon_dev_info *vb_dev_info;
63
64 /* Synchronize access/update to this struct virtio_balloon elements */
65 struct mutex balloon_lock;
60 66
61 /* The array of pfns we tell the Host about. */ 67 /* The array of pfns we tell the Host about. */
62 unsigned int num_pfns; 68 unsigned int num_pfns;
63 u32 pfns[256]; 69 u32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
64 70
65 /* Memory statistics */ 71 /* Memory statistics */
66 int need_stats_update; 72 int need_stats_update;
@@ -122,18 +128,21 @@ static void set_page_pfns(u32 pfns[], struct page *page)
122 128
123static void fill_balloon(struct virtio_balloon *vb, size_t num) 129static void fill_balloon(struct virtio_balloon *vb, size_t num)
124{ 130{
131 struct balloon_dev_info *vb_dev_info = vb->vb_dev_info;
132
125 /* We can only do one array worth at a time. */ 133 /* We can only do one array worth at a time. */
126 num = min(num, ARRAY_SIZE(vb->pfns)); 134 num = min(num, ARRAY_SIZE(vb->pfns));
127 135
136 mutex_lock(&vb->balloon_lock);
128 for (vb->num_pfns = 0; vb->num_pfns < num; 137 for (vb->num_pfns = 0; vb->num_pfns < num;
129 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { 138 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
130 struct page *page = alloc_page(GFP_HIGHUSER | __GFP_NORETRY | 139 struct page *page = balloon_page_enqueue(vb_dev_info);
131 __GFP_NOMEMALLOC | __GFP_NOWARN); 140
132 if (!page) { 141 if (!page) {
133 if (printk_ratelimit()) 142 if (printk_ratelimit())
134 dev_printk(KERN_INFO, &vb->vdev->dev, 143 dev_printk(KERN_INFO, &vb->vdev->dev,
135 "Out of puff! Can't get %zu pages\n", 144 "Out of puff! Can't get %u pages\n",
136 num); 145 VIRTIO_BALLOON_PAGES_PER_PAGE);
137 /* Sleep for at least 1/5 of a second before retry. */ 146 /* Sleep for at least 1/5 of a second before retry. */
138 msleep(200); 147 msleep(200);
139 break; 148 break;
@@ -141,14 +150,12 @@ static void fill_balloon(struct virtio_balloon *vb, size_t num)
141 set_page_pfns(vb->pfns + vb->num_pfns, page); 150 set_page_pfns(vb->pfns + vb->num_pfns, page);
142 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE; 151 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
143 totalram_pages--; 152 totalram_pages--;
144 list_add(&page->lru, &vb->pages);
145 } 153 }
146 154
147 /* Didn't get any? Oh well. */ 155 /* Did we get any? */
148 if (vb->num_pfns == 0) 156 if (vb->num_pfns != 0)
149 return; 157 tell_host(vb, vb->inflate_vq);
150 158 mutex_unlock(&vb->balloon_lock);
151 tell_host(vb, vb->inflate_vq);
152} 159}
153 160
154static void release_pages_by_pfn(const u32 pfns[], unsigned int num) 161static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
@@ -157,7 +164,7 @@ static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
157 164
158 /* Find pfns pointing at start of each page, get pages and free them. */ 165 /* Find pfns pointing at start of each page, get pages and free them. */
159 for (i = 0; i < num; i += VIRTIO_BALLOON_PAGES_PER_PAGE) { 166 for (i = 0; i < num; i += VIRTIO_BALLOON_PAGES_PER_PAGE) {
160 __free_page(balloon_pfn_to_page(pfns[i])); 167 balloon_page_free(balloon_pfn_to_page(pfns[i]));
161 totalram_pages++; 168 totalram_pages++;
162 } 169 }
163} 170}
@@ -165,14 +172,17 @@ static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
165static void leak_balloon(struct virtio_balloon *vb, size_t num) 172static void leak_balloon(struct virtio_balloon *vb, size_t num)
166{ 173{
167 struct page *page; 174 struct page *page;
175 struct balloon_dev_info *vb_dev_info = vb->vb_dev_info;
168 176
169 /* We can only do one array worth at a time. */ 177 /* We can only do one array worth at a time. */
170 num = min(num, ARRAY_SIZE(vb->pfns)); 178 num = min(num, ARRAY_SIZE(vb->pfns));
171 179
180 mutex_lock(&vb->balloon_lock);
172 for (vb->num_pfns = 0; vb->num_pfns < num; 181 for (vb->num_pfns = 0; vb->num_pfns < num;
173 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { 182 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
174 page = list_first_entry(&vb->pages, struct page, lru); 183 page = balloon_page_dequeue(vb_dev_info);
175 list_del(&page->lru); 184 if (!page)
185 break;
176 set_page_pfns(vb->pfns + vb->num_pfns, page); 186 set_page_pfns(vb->pfns + vb->num_pfns, page);
177 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE; 187 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
178 } 188 }
@@ -183,6 +193,7 @@ static void leak_balloon(struct virtio_balloon *vb, size_t num)
183 * is true, we *have* to do it in this order 193 * is true, we *have* to do it in this order
184 */ 194 */
185 tell_host(vb, vb->deflate_vq); 195 tell_host(vb, vb->deflate_vq);
196 mutex_unlock(&vb->balloon_lock);
186 release_pages_by_pfn(vb->pfns, vb->num_pfns); 197 release_pages_by_pfn(vb->pfns, vb->num_pfns);
187} 198}
188 199
@@ -339,9 +350,84 @@ static int init_vqs(struct virtio_balloon *vb)
339 return 0; 350 return 0;
340} 351}
341 352
353static const struct address_space_operations virtio_balloon_aops;
354#ifdef CONFIG_BALLOON_COMPACTION
355/*
356 * virtballoon_migratepage - perform the balloon page migration on behalf of
357 * a compation thread. (called under page lock)
358 * @mapping: the page->mapping which will be assigned to the new migrated page.
359 * @newpage: page that will replace the isolated page after migration finishes.
360 * @page : the isolated (old) page that is about to be migrated to newpage.
361 * @mode : compaction mode -- not used for balloon page migration.
362 *
363 * After a ballooned page gets isolated by compaction procedures, this is the
364 * function that performs the page migration on behalf of a compaction thread
365 * The page migration for virtio balloon is done in a simple swap fashion which
366 * follows these two macro steps:
367 * 1) insert newpage into vb->pages list and update the host about it;
368 * 2) update the host about the old page removed from vb->pages list;
369 *
370 * This function preforms the balloon page migration task.
371 * Called through balloon_mapping->a_ops->migratepage
372 */
373int virtballoon_migratepage(struct address_space *mapping,
374 struct page *newpage, struct page *page, enum migrate_mode mode)
375{
376 struct balloon_dev_info *vb_dev_info = balloon_page_device(page);
377 struct virtio_balloon *vb;
378 unsigned long flags;
379
380 BUG_ON(!vb_dev_info);
381
382 vb = vb_dev_info->balloon_device;
383
384 /*
385 * In order to avoid lock contention while migrating pages concurrently
386 * to leak_balloon() or fill_balloon() we just give up the balloon_lock
387 * this turn, as it is easier to retry the page migration later.
388 * This also prevents fill_balloon() getting stuck into a mutex
389 * recursion in the case it ends up triggering memory compaction
390 * while it is attempting to inflate the ballon.
391 */
392 if (!mutex_trylock(&vb->balloon_lock))
393 return -EAGAIN;
394
395 /* balloon's page migration 1st step -- inflate "newpage" */
396 spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
397 balloon_page_insert(newpage, mapping, &vb_dev_info->pages);
398 vb_dev_info->isolated_pages--;
399 spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
400 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
401 set_page_pfns(vb->pfns, newpage);
402 tell_host(vb, vb->inflate_vq);
403
404 /*
405 * balloon's page migration 2nd step -- deflate "page"
406 *
407 * It's safe to delete page->lru here because this page is at
408 * an isolated migration list, and this step is expected to happen here
409 */
410 balloon_page_delete(page);
411 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
412 set_page_pfns(vb->pfns, page);
413 tell_host(vb, vb->deflate_vq);
414
415 mutex_unlock(&vb->balloon_lock);
416
417 return MIGRATEPAGE_BALLOON_SUCCESS;
418}
419
420/* define the balloon_mapping->a_ops callback to allow balloon page migration */
421static const struct address_space_operations virtio_balloon_aops = {
422 .migratepage = virtballoon_migratepage,
423};
424#endif /* CONFIG_BALLOON_COMPACTION */
425
342static int virtballoon_probe(struct virtio_device *vdev) 426static int virtballoon_probe(struct virtio_device *vdev)
343{ 427{
344 struct virtio_balloon *vb; 428 struct virtio_balloon *vb;
429 struct address_space *vb_mapping;
430 struct balloon_dev_info *vb_devinfo;
345 int err; 431 int err;
346 432
347 vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL); 433 vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
@@ -350,16 +436,37 @@ static int virtballoon_probe(struct virtio_device *vdev)
350 goto out; 436 goto out;
351 } 437 }
352 438
353 INIT_LIST_HEAD(&vb->pages);
354 vb->num_pages = 0; 439 vb->num_pages = 0;
440 mutex_init(&vb->balloon_lock);
355 init_waitqueue_head(&vb->config_change); 441 init_waitqueue_head(&vb->config_change);
356 init_waitqueue_head(&vb->acked); 442 init_waitqueue_head(&vb->acked);
357 vb->vdev = vdev; 443 vb->vdev = vdev;
358 vb->need_stats_update = 0; 444 vb->need_stats_update = 0;
359 445
446 vb_devinfo = balloon_devinfo_alloc(vb);
447 if (IS_ERR(vb_devinfo)) {
448 err = PTR_ERR(vb_devinfo);
449 goto out_free_vb;
450 }
451
452 vb_mapping = balloon_mapping_alloc(vb_devinfo,
453 (balloon_compaction_check()) ?
454 &virtio_balloon_aops : NULL);
455 if (IS_ERR(vb_mapping)) {
456 /*
457 * IS_ERR(vb_mapping) && PTR_ERR(vb_mapping) == -EOPNOTSUPP
458 * This means !CONFIG_BALLOON_COMPACTION, otherwise we get off.
459 */
460 err = PTR_ERR(vb_mapping);
461 if (err != -EOPNOTSUPP)
462 goto out_free_vb_devinfo;
463 }
464
465 vb->vb_dev_info = vb_devinfo;
466
360 err = init_vqs(vb); 467 err = init_vqs(vb);
361 if (err) 468 if (err)
362 goto out_free_vb; 469 goto out_free_vb_mapping;
363 470
364 vb->thread = kthread_run(balloon, vb, "vballoon"); 471 vb->thread = kthread_run(balloon, vb, "vballoon");
365 if (IS_ERR(vb->thread)) { 472 if (IS_ERR(vb->thread)) {
@@ -371,6 +478,10 @@ static int virtballoon_probe(struct virtio_device *vdev)
371 478
372out_del_vqs: 479out_del_vqs:
373 vdev->config->del_vqs(vdev); 480 vdev->config->del_vqs(vdev);
481out_free_vb_mapping:
482 balloon_mapping_free(vb_mapping);
483out_free_vb_devinfo:
484 balloon_devinfo_free(vb_devinfo);
374out_free_vb: 485out_free_vb:
375 kfree(vb); 486 kfree(vb);
376out: 487out:
@@ -396,6 +507,8 @@ static void __devexit virtballoon_remove(struct virtio_device *vdev)
396 507
397 kthread_stop(vb->thread); 508 kthread_stop(vb->thread);
398 remove_common(vb); 509 remove_common(vb);
510 balloon_mapping_free(vb->vb_dev_info->mapping);
511 balloon_devinfo_free(vb->vb_dev_info);
399 kfree(vb); 512 kfree(vb);
400} 513}
401 514