aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/virtio
diff options
context:
space:
mode:
authorJonathan Herman <hermanjl@cs.unc.edu>2013-01-17 16:15:55 -0500
committerJonathan Herman <hermanjl@cs.unc.edu>2013-01-17 16:15:55 -0500
commit8dea78da5cee153b8af9c07a2745f6c55057fe12 (patch)
treea8f4d49d63b1ecc92f2fddceba0655b2472c5bd9 /drivers/virtio
parent406089d01562f1e2bf9f089fd7637009ebaad589 (diff)
Patched in Tegra support.
Diffstat (limited to 'drivers/virtio')
-rw-r--r--drivers/virtio/Kconfig37
-rw-r--r--drivers/virtio/Makefile4
-rw-r--r--drivers/virtio/config.c1
-rw-r--r--drivers/virtio/virtio.c51
-rw-r--r--drivers/virtio/virtio_balloon.c342
-rw-r--r--drivers/virtio/virtio_mmio.c665
-rw-r--r--drivers/virtio/virtio_pci.c186
-rw-r--r--drivers/virtio/virtio_ring.c288
8 files changed, 198 insertions, 1376 deletions
diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
index 8d5bddb56cb..57e493b1bd2 100644
--- a/drivers/virtio/Kconfig
+++ b/drivers/virtio/Kconfig
@@ -1,9 +1,11 @@
1# Virtio always gets selected by whoever wants it.
1config VIRTIO 2config VIRTIO
2 tristate 3 tristate
3 ---help--- 4
4 This option is selected by any driver which implements the virtio 5# Similarly the virtio ring implementation.
5 bus, such as CONFIG_VIRTIO_PCI, CONFIG_VIRTIO_MMIO, CONFIG_LGUEST, 6config VIRTIO_RING
6 CONFIG_RPMSG or CONFIG_S390_GUEST. 7 tristate
8 depends on VIRTIO
7 9
8menu "Virtio drivers" 10menu "Virtio drivers"
9 11
@@ -11,6 +13,7 @@ config VIRTIO_PCI
11 tristate "PCI driver for virtio devices (EXPERIMENTAL)" 13 tristate "PCI driver for virtio devices (EXPERIMENTAL)"
12 depends on PCI && EXPERIMENTAL 14 depends on PCI && EXPERIMENTAL
13 select VIRTIO 15 select VIRTIO
16 select VIRTIO_RING
14 ---help--- 17 ---help---
15 This drivers provides support for virtio based paravirtual device 18 This drivers provides support for virtio based paravirtual device
16 drivers over PCI. This requires that your VMM has appropriate PCI 19 drivers over PCI. This requires that your VMM has appropriate PCI
@@ -23,33 +26,13 @@ config VIRTIO_PCI
23 If unsure, say M. 26 If unsure, say M.
24 27
25config VIRTIO_BALLOON 28config VIRTIO_BALLOON
26 tristate "Virtio balloon driver" 29 tristate "Virtio balloon driver (EXPERIMENTAL)"
27 depends on VIRTIO 30 select VIRTIO
31 select VIRTIO_RING
28 ---help--- 32 ---help---
29 This driver supports increasing and decreasing the amount 33 This driver supports increasing and decreasing the amount
30 of memory within a KVM guest. 34 of memory within a KVM guest.
31 35
32 If unsure, say M. 36 If unsure, say M.
33 37
34 config VIRTIO_MMIO
35 tristate "Platform bus driver for memory mapped virtio devices (EXPERIMENTAL)"
36 depends on HAS_IOMEM && EXPERIMENTAL
37 select VIRTIO
38 ---help---
39 This drivers provides support for memory mapped virtio
40 platform device driver.
41
42 If unsure, say N.
43
44config VIRTIO_MMIO_CMDLINE_DEVICES
45 bool "Memory mapped virtio devices parameter parsing"
46 depends on VIRTIO_MMIO
47 ---help---
48 Allow virtio-mmio devices instantiation via the kernel command line
49 or module parameters. Be aware that using incorrect parameters (base
50 address in particular) can crash your system - you have been warned.
51 See Documentation/kernel-parameters.txt for details.
52
53 If unsure, say 'N'.
54
55endmenu 38endmenu
diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile
index 9076635697b..6738c446c19 100644
--- a/drivers/virtio/Makefile
+++ b/drivers/virtio/Makefile
@@ -1,4 +1,4 @@
1obj-$(CONFIG_VIRTIO) += virtio.o virtio_ring.o 1obj-$(CONFIG_VIRTIO) += virtio.o
2obj-$(CONFIG_VIRTIO_MMIO) += virtio_mmio.o 2obj-$(CONFIG_VIRTIO_RING) += virtio_ring.o
3obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o 3obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
4obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o 4obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o
diff --git a/drivers/virtio/config.c b/drivers/virtio/config.c
index f70bcd2ff98..983d482fba4 100644
--- a/drivers/virtio/config.c
+++ b/drivers/virtio/config.c
@@ -9,4 +9,5 @@
9#include <linux/virtio.h> 9#include <linux/virtio.h>
10#include <linux/virtio_config.h> 10#include <linux/virtio_config.h>
11#include <linux/bug.h> 11#include <linux/bug.h>
12#include <asm/system.h>
12 13
diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
index ee59b74768d..efb35aa8309 100644
--- a/drivers/virtio/virtio.c
+++ b/drivers/virtio/virtio.c
@@ -1,41 +1,40 @@
1#include <linux/virtio.h> 1#include <linux/virtio.h>
2#include <linux/spinlock.h> 2#include <linux/spinlock.h>
3#include <linux/virtio_config.h> 3#include <linux/virtio_config.h>
4#include <linux/module.h>
5#include <linux/idr.h>
6 4
7/* Unique numbering for virtio devices. */ 5/* Unique numbering for virtio devices. */
8static DEFINE_IDA(virtio_index_ida); 6static unsigned int dev_index;
9 7
10static ssize_t device_show(struct device *_d, 8static ssize_t device_show(struct device *_d,
11 struct device_attribute *attr, char *buf) 9 struct device_attribute *attr, char *buf)
12{ 10{
13 struct virtio_device *dev = dev_to_virtio(_d); 11 struct virtio_device *dev = container_of(_d,struct virtio_device,dev);
14 return sprintf(buf, "0x%04x\n", dev->id.device); 12 return sprintf(buf, "0x%04x\n", dev->id.device);
15} 13}
16static ssize_t vendor_show(struct device *_d, 14static ssize_t vendor_show(struct device *_d,
17 struct device_attribute *attr, char *buf) 15 struct device_attribute *attr, char *buf)
18{ 16{
19 struct virtio_device *dev = dev_to_virtio(_d); 17 struct virtio_device *dev = container_of(_d,struct virtio_device,dev);
20 return sprintf(buf, "0x%04x\n", dev->id.vendor); 18 return sprintf(buf, "0x%04x\n", dev->id.vendor);
21} 19}
22static ssize_t status_show(struct device *_d, 20static ssize_t status_show(struct device *_d,
23 struct device_attribute *attr, char *buf) 21 struct device_attribute *attr, char *buf)
24{ 22{
25 struct virtio_device *dev = dev_to_virtio(_d); 23 struct virtio_device *dev = container_of(_d,struct virtio_device,dev);
26 return sprintf(buf, "0x%08x\n", dev->config->get_status(dev)); 24 return sprintf(buf, "0x%08x\n", dev->config->get_status(dev));
27} 25}
28static ssize_t modalias_show(struct device *_d, 26static ssize_t modalias_show(struct device *_d,
29 struct device_attribute *attr, char *buf) 27 struct device_attribute *attr, char *buf)
30{ 28{
31 struct virtio_device *dev = dev_to_virtio(_d); 29 struct virtio_device *dev = container_of(_d,struct virtio_device,dev);
30
32 return sprintf(buf, "virtio:d%08Xv%08X\n", 31 return sprintf(buf, "virtio:d%08Xv%08X\n",
33 dev->id.device, dev->id.vendor); 32 dev->id.device, dev->id.vendor);
34} 33}
35static ssize_t features_show(struct device *_d, 34static ssize_t features_show(struct device *_d,
36 struct device_attribute *attr, char *buf) 35 struct device_attribute *attr, char *buf)
37{ 36{
38 struct virtio_device *dev = dev_to_virtio(_d); 37 struct virtio_device *dev = container_of(_d, struct virtio_device, dev);
39 unsigned int i; 38 unsigned int i;
40 ssize_t len = 0; 39 ssize_t len = 0;
41 40
@@ -70,10 +69,10 @@ static inline int virtio_id_match(const struct virtio_device *dev,
70static int virtio_dev_match(struct device *_dv, struct device_driver *_dr) 69static int virtio_dev_match(struct device *_dv, struct device_driver *_dr)
71{ 70{
72 unsigned int i; 71 unsigned int i;
73 struct virtio_device *dev = dev_to_virtio(_dv); 72 struct virtio_device *dev = container_of(_dv,struct virtio_device,dev);
74 const struct virtio_device_id *ids; 73 const struct virtio_device_id *ids;
75 74
76 ids = drv_to_virtio(_dr)->id_table; 75 ids = container_of(_dr, struct virtio_driver, driver)->id_table;
77 for (i = 0; ids[i].device; i++) 76 for (i = 0; ids[i].device; i++)
78 if (virtio_id_match(dev, &ids[i])) 77 if (virtio_id_match(dev, &ids[i]))
79 return 1; 78 return 1;
@@ -82,7 +81,7 @@ static int virtio_dev_match(struct device *_dv, struct device_driver *_dr)
82 81
83static int virtio_uevent(struct device *_dv, struct kobj_uevent_env *env) 82static int virtio_uevent(struct device *_dv, struct kobj_uevent_env *env)
84{ 83{
85 struct virtio_device *dev = dev_to_virtio(_dv); 84 struct virtio_device *dev = container_of(_dv,struct virtio_device,dev);
86 85
87 return add_uevent_var(env, "MODALIAS=virtio:d%08Xv%08X", 86 return add_uevent_var(env, "MODALIAS=virtio:d%08Xv%08X",
88 dev->id.device, dev->id.vendor); 87 dev->id.device, dev->id.vendor);
@@ -97,7 +96,8 @@ void virtio_check_driver_offered_feature(const struct virtio_device *vdev,
97 unsigned int fbit) 96 unsigned int fbit)
98{ 97{
99 unsigned int i; 98 unsigned int i;
100 struct virtio_driver *drv = drv_to_virtio(vdev->dev.driver); 99 struct virtio_driver *drv = container_of(vdev->dev.driver,
100 struct virtio_driver, driver);
101 101
102 for (i = 0; i < drv->feature_table_size; i++) 102 for (i = 0; i < drv->feature_table_size; i++)
103 if (drv->feature_table[i] == fbit) 103 if (drv->feature_table[i] == fbit)
@@ -109,8 +109,9 @@ EXPORT_SYMBOL_GPL(virtio_check_driver_offered_feature);
109static int virtio_dev_probe(struct device *_d) 109static int virtio_dev_probe(struct device *_d)
110{ 110{
111 int err, i; 111 int err, i;
112 struct virtio_device *dev = dev_to_virtio(_d); 112 struct virtio_device *dev = container_of(_d,struct virtio_device,dev);
113 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); 113 struct virtio_driver *drv = container_of(dev->dev.driver,
114 struct virtio_driver, driver);
114 u32 device_features; 115 u32 device_features;
115 116
116 /* We have a driver! */ 117 /* We have a driver! */
@@ -138,24 +139,22 @@ static int virtio_dev_probe(struct device *_d)
138 err = drv->probe(dev); 139 err = drv->probe(dev);
139 if (err) 140 if (err)
140 add_status(dev, VIRTIO_CONFIG_S_FAILED); 141 add_status(dev, VIRTIO_CONFIG_S_FAILED);
141 else { 142 else
142 add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); 143 add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
143 if (drv->scan)
144 drv->scan(dev);
145 }
146 144
147 return err; 145 return err;
148} 146}
149 147
150static int virtio_dev_remove(struct device *_d) 148static int virtio_dev_remove(struct device *_d)
151{ 149{
152 struct virtio_device *dev = dev_to_virtio(_d); 150 struct virtio_device *dev = container_of(_d,struct virtio_device,dev);
153 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); 151 struct virtio_driver *drv = container_of(dev->dev.driver,
152 struct virtio_driver, driver);
154 153
155 drv->remove(dev); 154 drv->remove(dev);
156 155
157 /* Driver should have reset device. */ 156 /* Driver should have reset device. */
158 WARN_ON_ONCE(dev->config->get_status(dev)); 157 BUG_ON(dev->config->get_status(dev));
159 158
160 /* Acknowledge the device's existence again. */ 159 /* Acknowledge the device's existence again. */
161 add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE); 160 add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
@@ -193,11 +192,7 @@ int register_virtio_device(struct virtio_device *dev)
193 dev->dev.bus = &virtio_bus; 192 dev->dev.bus = &virtio_bus;
194 193
195 /* Assign a unique device index and hence name. */ 194 /* Assign a unique device index and hence name. */
196 err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL); 195 dev->index = dev_index++;
197 if (err < 0)
198 goto out;
199
200 dev->index = err;
201 dev_set_name(&dev->dev, "virtio%u", dev->index); 196 dev_set_name(&dev->dev, "virtio%u", dev->index);
202 197
203 /* We always start by resetting the device, in case a previous 198 /* We always start by resetting the device, in case a previous
@@ -212,7 +207,6 @@ int register_virtio_device(struct virtio_device *dev)
212 /* device_register() causes the bus infrastructure to look for a 207 /* device_register() causes the bus infrastructure to look for a
213 * matching driver. */ 208 * matching driver. */
214 err = device_register(&dev->dev); 209 err = device_register(&dev->dev);
215out:
216 if (err) 210 if (err)
217 add_status(dev, VIRTIO_CONFIG_S_FAILED); 211 add_status(dev, VIRTIO_CONFIG_S_FAILED);
218 return err; 212 return err;
@@ -221,10 +215,7 @@ EXPORT_SYMBOL_GPL(register_virtio_device);
221 215
222void unregister_virtio_device(struct virtio_device *dev) 216void unregister_virtio_device(struct virtio_device *dev)
223{ 217{
224 int index = dev->index; /* save for after device release */
225
226 device_unregister(&dev->dev); 218 device_unregister(&dev->dev);
227 ida_simple_remove(&virtio_index_ida, index);
228} 219}
229EXPORT_SYMBOL_GPL(unregister_virtio_device); 220EXPORT_SYMBOL_GPL(unregister_virtio_device);
230 221
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 797e1c79a10..e058ace2a4a 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -1,5 +1,4 @@
1/* 1/* Virtio balloon implementation, inspired by Dor Loar and Marcelo
2 * Virtio balloon implementation, inspired by Dor Laor and Marcelo
3 * Tosatti's implementations. 2 * Tosatti's implementations.
4 * 3 *
5 * Copyright 2008 Rusty Russell IBM Corporation 4 * Copyright 2008 Rusty Russell IBM Corporation
@@ -18,7 +17,7 @@
18 * along with this program; if not, write to the Free Software 17 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 */ 19 */
21 20//#define DEBUG
22#include <linux/virtio.h> 21#include <linux/virtio.h>
23#include <linux/virtio_balloon.h> 22#include <linux/virtio_balloon.h>
24#include <linux/swap.h> 23#include <linux/swap.h>
@@ -26,16 +25,6 @@
26#include <linux/freezer.h> 25#include <linux/freezer.h>
27#include <linux/delay.h> 26#include <linux/delay.h>
28#include <linux/slab.h> 27#include <linux/slab.h>
29#include <linux/module.h>
30#include <linux/balloon_compaction.h>
31
32/*
33 * Balloon device works in 4K page units. So each page is pointed to by
34 * multiple balloon pages. All memory counters in this driver are in balloon
35 * page units.
36 */
37#define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
38#define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
39 28
40struct virtio_balloon 29struct virtio_balloon
41{ 30{
@@ -49,24 +38,15 @@ struct virtio_balloon
49 struct task_struct *thread; 38 struct task_struct *thread;
50 39
51 /* Waiting for host to ack the pages we released. */ 40 /* Waiting for host to ack the pages we released. */
52 wait_queue_head_t acked; 41 struct completion acked;
53 42
54 /* Number of balloon pages we've told the Host we're not using. */ 43 /* The pages we've told the Host we're not using. */
55 unsigned int num_pages; 44 unsigned int num_pages;
56 /* 45 struct list_head pages;
57 * The pages we've told the Host we're not using are enqueued
58 * at vb_dev_info->pages list.
59 * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
60 * to num_pages above.
61 */
62 struct balloon_dev_info *vb_dev_info;
63
64 /* Synchronize access/update to this struct virtio_balloon elements */
65 struct mutex balloon_lock;
66 46
67 /* The array of pfns we tell the Host about. */ 47 /* The array of pfns we tell the Host about. */
68 unsigned int num_pfns; 48 unsigned int num_pfns;
69 u32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX]; 49 u32 pfns[256];
70 50
71 /* Memory statistics */ 51 /* Memory statistics */
72 int need_stats_update; 52 int need_stats_update;
@@ -84,86 +64,72 @@ static u32 page_to_balloon_pfn(struct page *page)
84 64
85 BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT); 65 BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT);
86 /* Convert pfn from Linux page size to balloon page size. */ 66 /* Convert pfn from Linux page size to balloon page size. */
87 return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE; 67 return pfn >> (PAGE_SHIFT - VIRTIO_BALLOON_PFN_SHIFT);
88}
89
90static struct page *balloon_pfn_to_page(u32 pfn)
91{
92 BUG_ON(pfn % VIRTIO_BALLOON_PAGES_PER_PAGE);
93 return pfn_to_page(pfn / VIRTIO_BALLOON_PAGES_PER_PAGE);
94} 68}
95 69
96static void balloon_ack(struct virtqueue *vq) 70static void balloon_ack(struct virtqueue *vq)
97{ 71{
98 struct virtio_balloon *vb = vq->vdev->priv; 72 struct virtio_balloon *vb;
73 unsigned int len;
99 74
100 wake_up(&vb->acked); 75 vb = virtqueue_get_buf(vq, &len);
76 if (vb)
77 complete(&vb->acked);
101} 78}
102 79
103static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq) 80static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
104{ 81{
105 struct scatterlist sg; 82 struct scatterlist sg;
106 unsigned int len;
107 83
108 sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns); 84 sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
109 85
86 init_completion(&vb->acked);
87
110 /* We should always be able to add one buffer to an empty queue. */ 88 /* We should always be able to add one buffer to an empty queue. */
111 if (virtqueue_add_buf(vq, &sg, 1, 0, vb, GFP_KERNEL) < 0) 89 if (virtqueue_add_buf(vq, &sg, 1, 0, vb) < 0)
112 BUG(); 90 BUG();
113 virtqueue_kick(vq); 91 virtqueue_kick(vq);
114 92
115 /* When host has read buffer, this completes via balloon_ack */ 93 /* When host has read buffer, this completes via balloon_ack */
116 wait_event(vb->acked, virtqueue_get_buf(vq, &len)); 94 wait_for_completion(&vb->acked);
117}
118
119static void set_page_pfns(u32 pfns[], struct page *page)
120{
121 unsigned int i;
122
123 /* Set balloon pfns pointing at this page.
124 * Note that the first pfn points at start of the page. */
125 for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
126 pfns[i] = page_to_balloon_pfn(page) + i;
127} 95}
128 96
129static void fill_balloon(struct virtio_balloon *vb, size_t num) 97static void fill_balloon(struct virtio_balloon *vb, size_t num)
130{ 98{
131 struct balloon_dev_info *vb_dev_info = vb->vb_dev_info;
132
133 /* We can only do one array worth at a time. */ 99 /* We can only do one array worth at a time. */
134 num = min(num, ARRAY_SIZE(vb->pfns)); 100 num = min(num, ARRAY_SIZE(vb->pfns));
135 101
136 mutex_lock(&vb->balloon_lock); 102 for (vb->num_pfns = 0; vb->num_pfns < num; vb->num_pfns++) {
137 for (vb->num_pfns = 0; vb->num_pfns < num; 103 struct page *page = alloc_page(GFP_HIGHUSER | __GFP_NORETRY |
138 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { 104 __GFP_NOMEMALLOC | __GFP_NOWARN);
139 struct page *page = balloon_page_enqueue(vb_dev_info);
140
141 if (!page) { 105 if (!page) {
142 dev_info_ratelimited(&vb->vdev->dev, 106 if (printk_ratelimit())
143 "Out of puff! Can't get %u pages\n", 107 dev_printk(KERN_INFO, &vb->vdev->dev,
144 VIRTIO_BALLOON_PAGES_PER_PAGE); 108 "Out of puff! Can't get %zu pages\n",
109 num);
145 /* Sleep for at least 1/5 of a second before retry. */ 110 /* Sleep for at least 1/5 of a second before retry. */
146 msleep(200); 111 msleep(200);
147 break; 112 break;
148 } 113 }
149 set_page_pfns(vb->pfns + vb->num_pfns, page); 114 vb->pfns[vb->num_pfns] = page_to_balloon_pfn(page);
150 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
151 totalram_pages--; 115 totalram_pages--;
116 vb->num_pages++;
117 list_add(&page->lru, &vb->pages);
152 } 118 }
153 119
154 /* Did we get any? */ 120 /* Didn't get any? Oh well. */
155 if (vb->num_pfns != 0) 121 if (vb->num_pfns == 0)
156 tell_host(vb, vb->inflate_vq); 122 return;
157 mutex_unlock(&vb->balloon_lock); 123
124 tell_host(vb, vb->inflate_vq);
158} 125}
159 126
160static void release_pages_by_pfn(const u32 pfns[], unsigned int num) 127static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
161{ 128{
162 unsigned int i; 129 unsigned int i;
163 130
164 /* Find pfns pointing at start of each page, get pages and free them. */ 131 for (i = 0; i < num; i++) {
165 for (i = 0; i < num; i += VIRTIO_BALLOON_PAGES_PER_PAGE) { 132 __free_page(pfn_to_page(pfns[i]));
166 balloon_page_free(balloon_pfn_to_page(pfns[i]));
167 totalram_pages++; 133 totalram_pages++;
168 } 134 }
169} 135}
@@ -171,28 +137,24 @@ static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
171static void leak_balloon(struct virtio_balloon *vb, size_t num) 137static void leak_balloon(struct virtio_balloon *vb, size_t num)
172{ 138{
173 struct page *page; 139 struct page *page;
174 struct balloon_dev_info *vb_dev_info = vb->vb_dev_info;
175 140
176 /* We can only do one array worth at a time. */ 141 /* We can only do one array worth at a time. */
177 num = min(num, ARRAY_SIZE(vb->pfns)); 142 num = min(num, ARRAY_SIZE(vb->pfns));
178 143
179 mutex_lock(&vb->balloon_lock); 144 for (vb->num_pfns = 0; vb->num_pfns < num; vb->num_pfns++) {
180 for (vb->num_pfns = 0; vb->num_pfns < num; 145 page = list_first_entry(&vb->pages, struct page, lru);
181 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { 146 list_del(&page->lru);
182 page = balloon_page_dequeue(vb_dev_info); 147 vb->pfns[vb->num_pfns] = page_to_balloon_pfn(page);
183 if (!page) 148 vb->num_pages--;
184 break;
185 set_page_pfns(vb->pfns + vb->num_pfns, page);
186 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
187 } 149 }
188 150
151
189 /* 152 /*
190 * Note that if 153 * Note that if
191 * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST); 154 * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
192 * is true, we *have* to do it in this order 155 * is true, we *have* to do it in this order
193 */ 156 */
194 tell_host(vb, vb->deflate_vq); 157 tell_host(vb, vb->deflate_vq);
195 mutex_unlock(&vb->balloon_lock);
196 release_pages_by_pfn(vb->pfns, vb->num_pfns); 158 release_pages_by_pfn(vb->pfns, vb->num_pfns);
197} 159}
198 160
@@ -237,8 +199,12 @@ static void update_balloon_stats(struct virtio_balloon *vb)
237 */ 199 */
238static void stats_request(struct virtqueue *vq) 200static void stats_request(struct virtqueue *vq)
239{ 201{
240 struct virtio_balloon *vb = vq->vdev->priv; 202 struct virtio_balloon *vb;
203 unsigned int len;
241 204
205 vb = virtqueue_get_buf(vq, &len);
206 if (!vb)
207 return;
242 vb->need_stats_update = 1; 208 vb->need_stats_update = 1;
243 wake_up(&vb->config_change); 209 wake_up(&vb->config_change);
244} 210}
@@ -247,16 +213,13 @@ static void stats_handle_request(struct virtio_balloon *vb)
247{ 213{
248 struct virtqueue *vq; 214 struct virtqueue *vq;
249 struct scatterlist sg; 215 struct scatterlist sg;
250 unsigned int len;
251 216
252 vb->need_stats_update = 0; 217 vb->need_stats_update = 0;
253 update_balloon_stats(vb); 218 update_balloon_stats(vb);
254 219
255 vq = vb->stats_vq; 220 vq = vb->stats_vq;
256 if (!virtqueue_get_buf(vq, &len))
257 return;
258 sg_init_one(&sg, vb->stats, sizeof(vb->stats)); 221 sg_init_one(&sg, vb->stats, sizeof(vb->stats));
259 if (virtqueue_add_buf(vq, &sg, 1, 0, vb, GFP_KERNEL) < 0) 222 if (virtqueue_add_buf(vq, &sg, 1, 0, vb) < 0)
260 BUG(); 223 BUG();
261 virtqueue_kick(vq); 224 virtqueue_kick(vq);
262} 225}
@@ -270,14 +233,11 @@ static void virtballoon_changed(struct virtio_device *vdev)
270 233
271static inline s64 towards_target(struct virtio_balloon *vb) 234static inline s64 towards_target(struct virtio_balloon *vb)
272{ 235{
273 __le32 v; 236 u32 v;
274 s64 target;
275
276 vb->vdev->config->get(vb->vdev, 237 vb->vdev->config->get(vb->vdev,
277 offsetof(struct virtio_balloon_config, num_pages), 238 offsetof(struct virtio_balloon_config, num_pages),
278 &v, sizeof(v)); 239 &v, sizeof(v));
279 target = le32_to_cpu(v); 240 return (s64)v - vb->num_pages;
280 return target - vb->num_pages;
281} 241}
282 242
283static void update_balloon_size(struct virtio_balloon *vb) 243static void update_balloon_size(struct virtio_balloon *vb)
@@ -314,21 +274,32 @@ static int balloon(void *_vballoon)
314 return 0; 274 return 0;
315} 275}
316 276
317static int init_vqs(struct virtio_balloon *vb) 277static int virtballoon_probe(struct virtio_device *vdev)
318{ 278{
279 struct virtio_balloon *vb;
319 struct virtqueue *vqs[3]; 280 struct virtqueue *vqs[3];
320 vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request }; 281 vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
321 const char *names[] = { "inflate", "deflate", "stats" }; 282 const char *names[] = { "inflate", "deflate", "stats" };
322 int err, nvqs; 283 int err, nvqs;
323 284
324 /* 285 vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
325 * We expect two virtqueues: inflate and deflate, and 286 if (!vb) {
326 * optionally stat. 287 err = -ENOMEM;
327 */ 288 goto out;
289 }
290
291 INIT_LIST_HEAD(&vb->pages);
292 vb->num_pages = 0;
293 init_waitqueue_head(&vb->config_change);
294 vb->vdev = vdev;
295 vb->need_stats_update = 0;
296
297 /* We expect two virtqueues: inflate and deflate,
298 * and optionally stat. */
328 nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2; 299 nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
329 err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names); 300 err = vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names);
330 if (err) 301 if (err)
331 return err; 302 goto out_free_vb;
332 303
333 vb->inflate_vq = vqs[0]; 304 vb->inflate_vq = vqs[0];
334 vb->deflate_vq = vqs[1]; 305 vb->deflate_vq = vqs[1];
@@ -341,131 +312,10 @@ static int init_vqs(struct virtio_balloon *vb)
341 * use it to signal us later. 312 * use it to signal us later.
342 */ 313 */
343 sg_init_one(&sg, vb->stats, sizeof vb->stats); 314 sg_init_one(&sg, vb->stats, sizeof vb->stats);
344 if (virtqueue_add_buf(vb->stats_vq, &sg, 1, 0, vb, GFP_KERNEL) 315 if (virtqueue_add_buf(vb->stats_vq, &sg, 1, 0, vb) < 0)
345 < 0)
346 BUG(); 316 BUG();
347 virtqueue_kick(vb->stats_vq); 317 virtqueue_kick(vb->stats_vq);
348 } 318 }
349 return 0;
350}
351
352static const struct address_space_operations virtio_balloon_aops;
353#ifdef CONFIG_BALLOON_COMPACTION
354/*
355 * virtballoon_migratepage - perform the balloon page migration on behalf of
356 * a compation thread. (called under page lock)
357 * @mapping: the page->mapping which will be assigned to the new migrated page.
358 * @newpage: page that will replace the isolated page after migration finishes.
359 * @page : the isolated (old) page that is about to be migrated to newpage.
360 * @mode : compaction mode -- not used for balloon page migration.
361 *
362 * After a ballooned page gets isolated by compaction procedures, this is the
363 * function that performs the page migration on behalf of a compaction thread
364 * The page migration for virtio balloon is done in a simple swap fashion which
365 * follows these two macro steps:
366 * 1) insert newpage into vb->pages list and update the host about it;
367 * 2) update the host about the old page removed from vb->pages list;
368 *
369 * This function preforms the balloon page migration task.
370 * Called through balloon_mapping->a_ops->migratepage
371 */
372int virtballoon_migratepage(struct address_space *mapping,
373 struct page *newpage, struct page *page, enum migrate_mode mode)
374{
375 struct balloon_dev_info *vb_dev_info = balloon_page_device(page);
376 struct virtio_balloon *vb;
377 unsigned long flags;
378
379 BUG_ON(!vb_dev_info);
380
381 vb = vb_dev_info->balloon_device;
382
383 /*
384 * In order to avoid lock contention while migrating pages concurrently
385 * to leak_balloon() or fill_balloon() we just give up the balloon_lock
386 * this turn, as it is easier to retry the page migration later.
387 * This also prevents fill_balloon() getting stuck into a mutex
388 * recursion in the case it ends up triggering memory compaction
389 * while it is attempting to inflate the ballon.
390 */
391 if (!mutex_trylock(&vb->balloon_lock))
392 return -EAGAIN;
393
394 /* balloon's page migration 1st step -- inflate "newpage" */
395 spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
396 balloon_page_insert(newpage, mapping, &vb_dev_info->pages);
397 vb_dev_info->isolated_pages--;
398 spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
399 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
400 set_page_pfns(vb->pfns, newpage);
401 tell_host(vb, vb->inflate_vq);
402
403 /*
404 * balloon's page migration 2nd step -- deflate "page"
405 *
406 * It's safe to delete page->lru here because this page is at
407 * an isolated migration list, and this step is expected to happen here
408 */
409 balloon_page_delete(page);
410 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
411 set_page_pfns(vb->pfns, page);
412 tell_host(vb, vb->deflate_vq);
413
414 mutex_unlock(&vb->balloon_lock);
415
416 return MIGRATEPAGE_BALLOON_SUCCESS;
417}
418
419/* define the balloon_mapping->a_ops callback to allow balloon page migration */
420static const struct address_space_operations virtio_balloon_aops = {
421 .migratepage = virtballoon_migratepage,
422};
423#endif /* CONFIG_BALLOON_COMPACTION */
424
425static int virtballoon_probe(struct virtio_device *vdev)
426{
427 struct virtio_balloon *vb;
428 struct address_space *vb_mapping;
429 struct balloon_dev_info *vb_devinfo;
430 int err;
431
432 vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
433 if (!vb) {
434 err = -ENOMEM;
435 goto out;
436 }
437
438 vb->num_pages = 0;
439 mutex_init(&vb->balloon_lock);
440 init_waitqueue_head(&vb->config_change);
441 init_waitqueue_head(&vb->acked);
442 vb->vdev = vdev;
443 vb->need_stats_update = 0;
444
445 vb_devinfo = balloon_devinfo_alloc(vb);
446 if (IS_ERR(vb_devinfo)) {
447 err = PTR_ERR(vb_devinfo);
448 goto out_free_vb;
449 }
450
451 vb_mapping = balloon_mapping_alloc(vb_devinfo,
452 (balloon_compaction_check()) ?
453 &virtio_balloon_aops : NULL);
454 if (IS_ERR(vb_mapping)) {
455 /*
456 * IS_ERR(vb_mapping) && PTR_ERR(vb_mapping) == -EOPNOTSUPP
457 * This means !CONFIG_BALLOON_COMPACTION, otherwise we get off.
458 */
459 err = PTR_ERR(vb_mapping);
460 if (err != -EOPNOTSUPP)
461 goto out_free_vb_devinfo;
462 }
463
464 vb->vb_dev_info = vb_devinfo;
465
466 err = init_vqs(vb);
467 if (err)
468 goto out_free_vb_mapping;
469 319
470 vb->thread = kthread_run(balloon, vb, "vballoon"); 320 vb->thread = kthread_run(balloon, vb, "vballoon");
471 if (IS_ERR(vb->thread)) { 321 if (IS_ERR(vb->thread)) {
@@ -477,69 +327,29 @@ static int virtballoon_probe(struct virtio_device *vdev)
477 327
478out_del_vqs: 328out_del_vqs:
479 vdev->config->del_vqs(vdev); 329 vdev->config->del_vqs(vdev);
480out_free_vb_mapping:
481 balloon_mapping_free(vb_mapping);
482out_free_vb_devinfo:
483 balloon_devinfo_free(vb_devinfo);
484out_free_vb: 330out_free_vb:
485 kfree(vb); 331 kfree(vb);
486out: 332out:
487 return err; 333 return err;
488} 334}
489 335
490static void remove_common(struct virtio_balloon *vb) 336static void __devexit virtballoon_remove(struct virtio_device *vdev)
491{ 337{
338 struct virtio_balloon *vb = vdev->priv;
339
340 kthread_stop(vb->thread);
341
492 /* There might be pages left in the balloon: free them. */ 342 /* There might be pages left in the balloon: free them. */
493 while (vb->num_pages) 343 while (vb->num_pages)
494 leak_balloon(vb, vb->num_pages); 344 leak_balloon(vb, vb->num_pages);
495 update_balloon_size(vb);
496 345
497 /* Now we reset the device so we can clean up the queues. */ 346 /* Now we reset the device so we can clean up the queues. */
498 vb->vdev->config->reset(vb->vdev); 347 vdev->config->reset(vdev);
499 348
500 vb->vdev->config->del_vqs(vb->vdev); 349 vdev->config->del_vqs(vdev);
501}
502
503static void virtballoon_remove(struct virtio_device *vdev)
504{
505 struct virtio_balloon *vb = vdev->priv;
506
507 kthread_stop(vb->thread);
508 remove_common(vb);
509 balloon_mapping_free(vb->vb_dev_info->mapping);
510 balloon_devinfo_free(vb->vb_dev_info);
511 kfree(vb); 350 kfree(vb);
512} 351}
513 352
514#ifdef CONFIG_PM
515static int virtballoon_freeze(struct virtio_device *vdev)
516{
517 struct virtio_balloon *vb = vdev->priv;
518
519 /*
520 * The kthread is already frozen by the PM core before this
521 * function is called.
522 */
523
524 remove_common(vb);
525 return 0;
526}
527
528static int virtballoon_restore(struct virtio_device *vdev)
529{
530 struct virtio_balloon *vb = vdev->priv;
531 int ret;
532
533 ret = init_vqs(vdev->priv);
534 if (ret)
535 return ret;
536
537 fill_balloon(vb, towards_target(vb));
538 update_balloon_size(vb);
539 return 0;
540}
541#endif
542
543static unsigned int features[] = { 353static unsigned int features[] = {
544 VIRTIO_BALLOON_F_MUST_TELL_HOST, 354 VIRTIO_BALLOON_F_MUST_TELL_HOST,
545 VIRTIO_BALLOON_F_STATS_VQ, 355 VIRTIO_BALLOON_F_STATS_VQ,
@@ -552,12 +362,8 @@ static struct virtio_driver virtio_balloon_driver = {
552 .driver.owner = THIS_MODULE, 362 .driver.owner = THIS_MODULE,
553 .id_table = id_table, 363 .id_table = id_table,
554 .probe = virtballoon_probe, 364 .probe = virtballoon_probe,
555 .remove = virtballoon_remove, 365 .remove = __devexit_p(virtballoon_remove),
556 .config_changed = virtballoon_changed, 366 .config_changed = virtballoon_changed,
557#ifdef CONFIG_PM
558 .freeze = virtballoon_freeze,
559 .restore = virtballoon_restore,
560#endif
561}; 367};
562 368
563static int __init init(void) 369static int __init init(void)
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
deleted file mode 100644
index 31f966f4d27..00000000000
--- a/drivers/virtio/virtio_mmio.c
+++ /dev/null
@@ -1,665 +0,0 @@
1/*
2 * Virtio memory mapped device driver
3 *
4 * Copyright 2011, ARM Ltd.
5 *
6 * This module allows virtio devices to be used over a virtual, memory mapped
7 * platform device.
8 *
9 * The guest device(s) may be instantiated in one of three equivalent ways:
10 *
11 * 1. Static platform device in board's code, eg.:
12 *
13 * static struct platform_device v2m_virtio_device = {
14 * .name = "virtio-mmio",
15 * .id = -1,
16 * .num_resources = 2,
17 * .resource = (struct resource []) {
18 * {
19 * .start = 0x1001e000,
20 * .end = 0x1001e0ff,
21 * .flags = IORESOURCE_MEM,
22 * }, {
23 * .start = 42 + 32,
24 * .end = 42 + 32,
25 * .flags = IORESOURCE_IRQ,
26 * },
27 * }
28 * };
29 *
30 * 2. Device Tree node, eg.:
31 *
32 * virtio_block@1e000 {
33 * compatible = "virtio,mmio";
34 * reg = <0x1e000 0x100>;
35 * interrupts = <42>;
36 * }
37 *
38 * 3. Kernel module (or command line) parameter. Can be used more than once -
39 * one device will be created for each one. Syntax:
40 *
41 * [virtio_mmio.]device=<size>@<baseaddr>:<irq>[:<id>]
42 * where:
43 * <size> := size (can use standard suffixes like K, M or G)
44 * <baseaddr> := physical base address
45 * <irq> := interrupt number (as passed to request_irq())
46 * <id> := (optional) platform device id
47 * eg.:
48 * virtio_mmio.device=0x100@0x100b0000:48 \
49 * virtio_mmio.device=1K@0x1001e000:74
50 *
51 *
52 *
53 * Registers layout (all 32-bit wide):
54 *
55 * offset d. name description
56 * ------ -- ---------------- -----------------
57 *
58 * 0x000 R MagicValue Magic value "virt"
59 * 0x004 R Version Device version (current max. 1)
60 * 0x008 R DeviceID Virtio device ID
61 * 0x00c R VendorID Virtio vendor ID
62 *
63 * 0x010 R HostFeatures Features supported by the host
64 * 0x014 W HostFeaturesSel Set of host features to access via HostFeatures
65 *
66 * 0x020 W GuestFeatures Features activated by the guest
67 * 0x024 W GuestFeaturesSel Set of activated features to set via GuestFeatures
68 * 0x028 W GuestPageSize Size of guest's memory page in bytes
69 *
70 * 0x030 W QueueSel Queue selector
71 * 0x034 R QueueNumMax Maximum size of the currently selected queue
72 * 0x038 W QueueNum Queue size for the currently selected queue
73 * 0x03c W QueueAlign Used Ring alignment for the current queue
74 * 0x040 RW QueuePFN PFN for the currently selected queue
75 *
76 * 0x050 W QueueNotify Queue notifier
77 * 0x060 R InterruptStatus Interrupt status register
78 * 0x060 W InterruptACK Interrupt acknowledge register
79 * 0x070 RW Status Device status register
80 *
81 * 0x100+ RW Device-specific configuration space
82 *
83 * Based on Virtio PCI driver by Anthony Liguori, copyright IBM Corp. 2007
84 *
85 * This work is licensed under the terms of the GNU GPL, version 2 or later.
86 * See the COPYING file in the top-level directory.
87 */
88
89#define pr_fmt(fmt) "virtio-mmio: " fmt
90
91#include <linux/highmem.h>
92#include <linux/interrupt.h>
93#include <linux/io.h>
94#include <linux/list.h>
95#include <linux/module.h>
96#include <linux/platform_device.h>
97#include <linux/slab.h>
98#include <linux/spinlock.h>
99#include <linux/virtio.h>
100#include <linux/virtio_config.h>
101#include <linux/virtio_mmio.h>
102#include <linux/virtio_ring.h>
103
104
105
106/* The alignment to use between consumer and producer parts of vring.
107 * Currently hardcoded to the page size. */
108#define VIRTIO_MMIO_VRING_ALIGN PAGE_SIZE
109
110
111
112#define to_virtio_mmio_device(_plat_dev) \
113 container_of(_plat_dev, struct virtio_mmio_device, vdev)
114
115struct virtio_mmio_device {
116 struct virtio_device vdev;
117 struct platform_device *pdev;
118
119 void __iomem *base;
120 unsigned long version;
121
122 /* a list of queues so we can dispatch IRQs */
123 spinlock_t lock;
124 struct list_head virtqueues;
125};
126
127struct virtio_mmio_vq_info {
128 /* the actual virtqueue */
129 struct virtqueue *vq;
130
131 /* the number of entries in the queue */
132 unsigned int num;
133
134 /* the virtual address of the ring queue */
135 void *queue;
136
137 /* the list node for the virtqueues list */
138 struct list_head node;
139};
140
141
142
143/* Configuration interface */
144
145static u32 vm_get_features(struct virtio_device *vdev)
146{
147 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
148
149 /* TODO: Features > 32 bits */
150 writel(0, vm_dev->base + VIRTIO_MMIO_HOST_FEATURES_SEL);
151
152 return readl(vm_dev->base + VIRTIO_MMIO_HOST_FEATURES);
153}
154
155static void vm_finalize_features(struct virtio_device *vdev)
156{
157 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
158 int i;
159
160 /* Give virtio_ring a chance to accept features. */
161 vring_transport_features(vdev);
162
163 for (i = 0; i < ARRAY_SIZE(vdev->features); i++) {
164 writel(i, vm_dev->base + VIRTIO_MMIO_GUEST_FEATURES_SEL);
165 writel(vdev->features[i],
166 vm_dev->base + VIRTIO_MMIO_GUEST_FEATURES);
167 }
168}
169
170static void vm_get(struct virtio_device *vdev, unsigned offset,
171 void *buf, unsigned len)
172{
173 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
174 u8 *ptr = buf;
175 int i;
176
177 for (i = 0; i < len; i++)
178 ptr[i] = readb(vm_dev->base + VIRTIO_MMIO_CONFIG + offset + i);
179}
180
181static void vm_set(struct virtio_device *vdev, unsigned offset,
182 const void *buf, unsigned len)
183{
184 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
185 const u8 *ptr = buf;
186 int i;
187
188 for (i = 0; i < len; i++)
189 writeb(ptr[i], vm_dev->base + VIRTIO_MMIO_CONFIG + offset + i);
190}
191
192static u8 vm_get_status(struct virtio_device *vdev)
193{
194 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
195
196 return readl(vm_dev->base + VIRTIO_MMIO_STATUS) & 0xff;
197}
198
199static void vm_set_status(struct virtio_device *vdev, u8 status)
200{
201 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
202
203 /* We should never be setting status to 0. */
204 BUG_ON(status == 0);
205
206 writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
207}
208
209static void vm_reset(struct virtio_device *vdev)
210{
211 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
212
213 /* 0 status means a reset. */
214 writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
215}
216
217
218
219/* Transport interface */
220
221/* the notify function used when creating a virt queue */
222static void vm_notify(struct virtqueue *vq)
223{
224 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);
225
226 /* We write the queue's selector into the notification register to
227 * signal the other end */
228 writel(vq->index, vm_dev->base + VIRTIO_MMIO_QUEUE_NOTIFY);
229}
230
231/* Notify all virtqueues on an interrupt. */
232static irqreturn_t vm_interrupt(int irq, void *opaque)
233{
234 struct virtio_mmio_device *vm_dev = opaque;
235 struct virtio_mmio_vq_info *info;
236 struct virtio_driver *vdrv = container_of(vm_dev->vdev.dev.driver,
237 struct virtio_driver, driver);
238 unsigned long status;
239 unsigned long flags;
240 irqreturn_t ret = IRQ_NONE;
241
242 /* Read and acknowledge interrupts */
243 status = readl(vm_dev->base + VIRTIO_MMIO_INTERRUPT_STATUS);
244 writel(status, vm_dev->base + VIRTIO_MMIO_INTERRUPT_ACK);
245
246 if (unlikely(status & VIRTIO_MMIO_INT_CONFIG)
247 && vdrv && vdrv->config_changed) {
248 vdrv->config_changed(&vm_dev->vdev);
249 ret = IRQ_HANDLED;
250 }
251
252 if (likely(status & VIRTIO_MMIO_INT_VRING)) {
253 spin_lock_irqsave(&vm_dev->lock, flags);
254 list_for_each_entry(info, &vm_dev->virtqueues, node)
255 ret |= vring_interrupt(irq, info->vq);
256 spin_unlock_irqrestore(&vm_dev->lock, flags);
257 }
258
259 return ret;
260}
261
262
263
264static void vm_del_vq(struct virtqueue *vq)
265{
266 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);
267 struct virtio_mmio_vq_info *info = vq->priv;
268 unsigned long flags, size;
269 unsigned int index = vq->index;
270
271 spin_lock_irqsave(&vm_dev->lock, flags);
272 list_del(&info->node);
273 spin_unlock_irqrestore(&vm_dev->lock, flags);
274
275 vring_del_virtqueue(vq);
276
277 /* Select and deactivate the queue */
278 writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL);
279 writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
280
281 size = PAGE_ALIGN(vring_size(info->num, VIRTIO_MMIO_VRING_ALIGN));
282 free_pages_exact(info->queue, size);
283 kfree(info);
284}
285
286static void vm_del_vqs(struct virtio_device *vdev)
287{
288 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
289 struct virtqueue *vq, *n;
290
291 list_for_each_entry_safe(vq, n, &vdev->vqs, list)
292 vm_del_vq(vq);
293
294 free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev);
295}
296
297
298
299static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index,
300 void (*callback)(struct virtqueue *vq),
301 const char *name)
302{
303 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
304 struct virtio_mmio_vq_info *info;
305 struct virtqueue *vq;
306 unsigned long flags, size;
307 int err;
308
309 if (!name)
310 return NULL;
311
312 /* Select the queue we're interested in */
313 writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL);
314
315 /* Queue shouldn't already be set up. */
316 if (readl(vm_dev->base + VIRTIO_MMIO_QUEUE_PFN)) {
317 err = -ENOENT;
318 goto error_available;
319 }
320
321 /* Allocate and fill out our active queue description */
322 info = kmalloc(sizeof(*info), GFP_KERNEL);
323 if (!info) {
324 err = -ENOMEM;
325 goto error_kmalloc;
326 }
327
328 /* Allocate pages for the queue - start with a queue as big as
329 * possible (limited by maximum size allowed by device), drop down
330 * to a minimal size, just big enough to fit descriptor table
331 * and two rings (which makes it "alignment_size * 2")
332 */
333 info->num = readl(vm_dev->base + VIRTIO_MMIO_QUEUE_NUM_MAX);
334
335 /* If the device reports a 0 entry queue, we won't be able to
336 * use it to perform I/O, and vring_new_virtqueue() can't create
337 * empty queues anyway, so don't bother to set up the device.
338 */
339 if (info->num == 0) {
340 err = -ENOENT;
341 goto error_alloc_pages;
342 }
343
344 while (1) {
345 size = PAGE_ALIGN(vring_size(info->num,
346 VIRTIO_MMIO_VRING_ALIGN));
347 /* Did the last iter shrink the queue below minimum size? */
348 if (size < VIRTIO_MMIO_VRING_ALIGN * 2) {
349 err = -ENOMEM;
350 goto error_alloc_pages;
351 }
352
353 info->queue = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
354 if (info->queue)
355 break;
356
357 info->num /= 2;
358 }
359
360 /* Activate the queue */
361 writel(info->num, vm_dev->base + VIRTIO_MMIO_QUEUE_NUM);
362 writel(VIRTIO_MMIO_VRING_ALIGN,
363 vm_dev->base + VIRTIO_MMIO_QUEUE_ALIGN);
364 writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
365 vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
366
367 /* Create the vring */
368 vq = vring_new_virtqueue(index, info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
369 true, info->queue, vm_notify, callback, name);
370 if (!vq) {
371 err = -ENOMEM;
372 goto error_new_virtqueue;
373 }
374
375 vq->priv = info;
376 info->vq = vq;
377
378 spin_lock_irqsave(&vm_dev->lock, flags);
379 list_add(&info->node, &vm_dev->virtqueues);
380 spin_unlock_irqrestore(&vm_dev->lock, flags);
381
382 return vq;
383
384error_new_virtqueue:
385 writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
386 free_pages_exact(info->queue, size);
387error_alloc_pages:
388 kfree(info);
389error_kmalloc:
390error_available:
391 return ERR_PTR(err);
392}
393
394static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
395 struct virtqueue *vqs[],
396 vq_callback_t *callbacks[],
397 const char *names[])
398{
399 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
400 unsigned int irq = platform_get_irq(vm_dev->pdev, 0);
401 int i, err;
402
403 err = request_irq(irq, vm_interrupt, IRQF_SHARED,
404 dev_name(&vdev->dev), vm_dev);
405 if (err)
406 return err;
407
408 for (i = 0; i < nvqs; ++i) {
409 vqs[i] = vm_setup_vq(vdev, i, callbacks[i], names[i]);
410 if (IS_ERR(vqs[i])) {
411 vm_del_vqs(vdev);
412 return PTR_ERR(vqs[i]);
413 }
414 }
415
416 return 0;
417}
418
419static const char *vm_bus_name(struct virtio_device *vdev)
420{
421 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
422
423 return vm_dev->pdev->name;
424}
425
426static struct virtio_config_ops virtio_mmio_config_ops = {
427 .get = vm_get,
428 .set = vm_set,
429 .get_status = vm_get_status,
430 .set_status = vm_set_status,
431 .reset = vm_reset,
432 .find_vqs = vm_find_vqs,
433 .del_vqs = vm_del_vqs,
434 .get_features = vm_get_features,
435 .finalize_features = vm_finalize_features,
436 .bus_name = vm_bus_name,
437};
438
439
440
441/* Platform device */
442
443static int virtio_mmio_probe(struct platform_device *pdev)
444{
445 struct virtio_mmio_device *vm_dev;
446 struct resource *mem;
447 unsigned long magic;
448
449 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
450 if (!mem)
451 return -EINVAL;
452
453 if (!devm_request_mem_region(&pdev->dev, mem->start,
454 resource_size(mem), pdev->name))
455 return -EBUSY;
456
457 vm_dev = devm_kzalloc(&pdev->dev, sizeof(*vm_dev), GFP_KERNEL);
458 if (!vm_dev)
459 return -ENOMEM;
460
461 vm_dev->vdev.dev.parent = &pdev->dev;
462 vm_dev->vdev.config = &virtio_mmio_config_ops;
463 vm_dev->pdev = pdev;
464 INIT_LIST_HEAD(&vm_dev->virtqueues);
465 spin_lock_init(&vm_dev->lock);
466
467 vm_dev->base = devm_ioremap(&pdev->dev, mem->start, resource_size(mem));
468 if (vm_dev->base == NULL)
469 return -EFAULT;
470
471 /* Check magic value */
472 magic = readl(vm_dev->base + VIRTIO_MMIO_MAGIC_VALUE);
473 if (memcmp(&magic, "virt", 4) != 0) {
474 dev_warn(&pdev->dev, "Wrong magic value 0x%08lx!\n", magic);
475 return -ENODEV;
476 }
477
478 /* Check device version */
479 vm_dev->version = readl(vm_dev->base + VIRTIO_MMIO_VERSION);
480 if (vm_dev->version != 1) {
481 dev_err(&pdev->dev, "Version %ld not supported!\n",
482 vm_dev->version);
483 return -ENXIO;
484 }
485
486 vm_dev->vdev.id.device = readl(vm_dev->base + VIRTIO_MMIO_DEVICE_ID);
487 vm_dev->vdev.id.vendor = readl(vm_dev->base + VIRTIO_MMIO_VENDOR_ID);
488
489 writel(PAGE_SIZE, vm_dev->base + VIRTIO_MMIO_GUEST_PAGE_SIZE);
490
491 platform_set_drvdata(pdev, vm_dev);
492
493 return register_virtio_device(&vm_dev->vdev);
494}
495
496static int virtio_mmio_remove(struct platform_device *pdev)
497{
498 struct virtio_mmio_device *vm_dev = platform_get_drvdata(pdev);
499
500 unregister_virtio_device(&vm_dev->vdev);
501
502 return 0;
503}
504
505
506
507/* Devices list parameter */
508
509#if defined(CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES)
510
511static struct device vm_cmdline_parent = {
512 .init_name = "virtio-mmio-cmdline",
513};
514
515static int vm_cmdline_parent_registered;
516static int vm_cmdline_id;
517
518static int vm_cmdline_set(const char *device,
519 const struct kernel_param *kp)
520{
521 int err;
522 struct resource resources[2] = {};
523 char *str;
524 long long int base, size;
525 unsigned int irq;
526 int processed, consumed = 0;
527 struct platform_device *pdev;
528
529 /* Consume "size" part of the command line parameter */
530 size = memparse(device, &str);
531
532 /* Get "@<base>:<irq>[:<id>]" chunks */
533 processed = sscanf(str, "@%lli:%u%n:%d%n",
534 &base, &irq, &consumed,
535 &vm_cmdline_id, &consumed);
536
537 /*
538 * sscanf() must processes at least 2 chunks; also there
539 * must be no extra characters after the last chunk, so
540 * str[consumed] must be '\0'
541 */
542 if (processed < 2 || str[consumed])
543 return -EINVAL;
544
545 resources[0].flags = IORESOURCE_MEM;
546 resources[0].start = base;
547 resources[0].end = base + size - 1;
548
549 resources[1].flags = IORESOURCE_IRQ;
550 resources[1].start = resources[1].end = irq;
551
552 if (!vm_cmdline_parent_registered) {
553 err = device_register(&vm_cmdline_parent);
554 if (err) {
555 pr_err("Failed to register parent device!\n");
556 return err;
557 }
558 vm_cmdline_parent_registered = 1;
559 }
560
561 pr_info("Registering device virtio-mmio.%d at 0x%llx-0x%llx, IRQ %d.\n",
562 vm_cmdline_id,
563 (unsigned long long)resources[0].start,
564 (unsigned long long)resources[0].end,
565 (int)resources[1].start);
566
567 pdev = platform_device_register_resndata(&vm_cmdline_parent,
568 "virtio-mmio", vm_cmdline_id++,
569 resources, ARRAY_SIZE(resources), NULL, 0);
570 if (IS_ERR(pdev))
571 return PTR_ERR(pdev);
572
573 return 0;
574}
575
576static int vm_cmdline_get_device(struct device *dev, void *data)
577{
578 char *buffer = data;
579 unsigned int len = strlen(buffer);
580 struct platform_device *pdev = to_platform_device(dev);
581
582 snprintf(buffer + len, PAGE_SIZE - len, "0x%llx@0x%llx:%llu:%d\n",
583 pdev->resource[0].end - pdev->resource[0].start + 1ULL,
584 (unsigned long long)pdev->resource[0].start,
585 (unsigned long long)pdev->resource[1].start,
586 pdev->id);
587 return 0;
588}
589
590static int vm_cmdline_get(char *buffer, const struct kernel_param *kp)
591{
592 buffer[0] = '\0';
593 device_for_each_child(&vm_cmdline_parent, buffer,
594 vm_cmdline_get_device);
595 return strlen(buffer) + 1;
596}
597
598static struct kernel_param_ops vm_cmdline_param_ops = {
599 .set = vm_cmdline_set,
600 .get = vm_cmdline_get,
601};
602
603device_param_cb(device, &vm_cmdline_param_ops, NULL, S_IRUSR);
604
605static int vm_unregister_cmdline_device(struct device *dev,
606 void *data)
607{
608 platform_device_unregister(to_platform_device(dev));
609
610 return 0;
611}
612
613static void vm_unregister_cmdline_devices(void)
614{
615 if (vm_cmdline_parent_registered) {
616 device_for_each_child(&vm_cmdline_parent, NULL,
617 vm_unregister_cmdline_device);
618 device_unregister(&vm_cmdline_parent);
619 vm_cmdline_parent_registered = 0;
620 }
621}
622
623#else
624
625static void vm_unregister_cmdline_devices(void)
626{
627}
628
629#endif
630
631/* Platform driver */
632
633static struct of_device_id virtio_mmio_match[] = {
634 { .compatible = "virtio,mmio", },
635 {},
636};
637MODULE_DEVICE_TABLE(of, virtio_mmio_match);
638
639static struct platform_driver virtio_mmio_driver = {
640 .probe = virtio_mmio_probe,
641 .remove = virtio_mmio_remove,
642 .driver = {
643 .name = "virtio-mmio",
644 .owner = THIS_MODULE,
645 .of_match_table = virtio_mmio_match,
646 },
647};
648
649static int __init virtio_mmio_init(void)
650{
651 return platform_driver_register(&virtio_mmio_driver);
652}
653
654static void __exit virtio_mmio_exit(void)
655{
656 platform_driver_unregister(&virtio_mmio_driver);
657 vm_unregister_cmdline_devices();
658}
659
660module_init(virtio_mmio_init);
661module_exit(virtio_mmio_exit);
662
663MODULE_AUTHOR("Pawel Moll <pawel.moll@arm.com>");
664MODULE_DESCRIPTION("Platform bus driver for memory mapped virtio devices");
665MODULE_LICENSE("GPL");
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
index 0c142892c10..ecb925411e0 100644
--- a/drivers/virtio/virtio_pci.c
+++ b/drivers/virtio/virtio_pci.c
@@ -48,7 +48,6 @@ struct virtio_pci_device
48 int msix_enabled; 48 int msix_enabled;
49 int intx_enabled; 49 int intx_enabled;
50 struct msix_entry *msix_entries; 50 struct msix_entry *msix_entries;
51 cpumask_var_t *msix_affinity_masks;
52 /* Name strings for interrupts. This size should be enough, 51 /* Name strings for interrupts. This size should be enough,
53 * and I'm too lazy to allocate each name separately. */ 52 * and I'm too lazy to allocate each name separately. */
54 char (*msix_names)[256]; 53 char (*msix_names)[256];
@@ -56,10 +55,6 @@ struct virtio_pci_device
56 unsigned msix_vectors; 55 unsigned msix_vectors;
57 /* Vectors allocated, excluding per-vq vectors if any */ 56 /* Vectors allocated, excluding per-vq vectors if any */
58 unsigned msix_used_vectors; 57 unsigned msix_used_vectors;
59
60 /* Status saved during hibernate/restore */
61 u8 saved_status;
62
63 /* Whether we have vector per vq */ 58 /* Whether we have vector per vq */
64 bool per_vq_vectors; 59 bool per_vq_vectors;
65}; 60};
@@ -80,6 +75,9 @@ struct virtio_pci_vq_info
80 /* the number of entries in the queue */ 75 /* the number of entries in the queue */
81 int num; 76 int num;
82 77
78 /* the index of the queue */
79 int queue_index;
80
83 /* the virtual address of the ring queue */ 81 /* the virtual address of the ring queue */
84 void *queue; 82 void *queue;
85 83
@@ -171,39 +169,22 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
171 iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS); 169 iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
172} 170}
173 171
174/* wait for pending irq handlers */
175static void vp_synchronize_vectors(struct virtio_device *vdev)
176{
177 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
178 int i;
179
180 if (vp_dev->intx_enabled)
181 synchronize_irq(vp_dev->pci_dev->irq);
182
183 for (i = 0; i < vp_dev->msix_vectors; ++i)
184 synchronize_irq(vp_dev->msix_entries[i].vector);
185}
186
187static void vp_reset(struct virtio_device *vdev) 172static void vp_reset(struct virtio_device *vdev)
188{ 173{
189 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 174 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
190 /* 0 status means a reset. */ 175 /* 0 status means a reset. */
191 iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS); 176 iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
192 /* Flush out the status write, and flush in device writes,
193 * including MSi-X interrupts, if any. */
194 ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
195 /* Flush pending VQ/configuration callbacks. */
196 vp_synchronize_vectors(vdev);
197} 177}
198 178
199/* the notify function used when creating a virt queue */ 179/* the notify function used when creating a virt queue */
200static void vp_notify(struct virtqueue *vq) 180static void vp_notify(struct virtqueue *vq)
201{ 181{
202 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 182 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
183 struct virtio_pci_vq_info *info = vq->priv;
203 184
204 /* we write the queue's selector into the notification register to 185 /* we write the queue's selector into the notification register to
205 * signal the other end */ 186 * signal the other end */
206 iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY); 187 iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY);
207} 188}
208 189
209/* Handle a configuration change: Tell driver if it wants to know. */ 190/* Handle a configuration change: Tell driver if it wants to know. */
@@ -276,10 +257,6 @@ static void vp_free_vectors(struct virtio_device *vdev)
276 for (i = 0; i < vp_dev->msix_used_vectors; ++i) 257 for (i = 0; i < vp_dev->msix_used_vectors; ++i)
277 free_irq(vp_dev->msix_entries[i].vector, vp_dev); 258 free_irq(vp_dev->msix_entries[i].vector, vp_dev);
278 259
279 for (i = 0; i < vp_dev->msix_vectors; i++)
280 if (vp_dev->msix_affinity_masks[i])
281 free_cpumask_var(vp_dev->msix_affinity_masks[i]);
282
283 if (vp_dev->msix_enabled) { 260 if (vp_dev->msix_enabled) {
284 /* Disable the vector used for configuration */ 261 /* Disable the vector used for configuration */
285 iowrite16(VIRTIO_MSI_NO_VECTOR, 262 iowrite16(VIRTIO_MSI_NO_VECTOR,
@@ -297,8 +274,6 @@ static void vp_free_vectors(struct virtio_device *vdev)
297 vp_dev->msix_names = NULL; 274 vp_dev->msix_names = NULL;
298 kfree(vp_dev->msix_entries); 275 kfree(vp_dev->msix_entries);
299 vp_dev->msix_entries = NULL; 276 vp_dev->msix_entries = NULL;
300 kfree(vp_dev->msix_affinity_masks);
301 vp_dev->msix_affinity_masks = NULL;
302} 277}
303 278
304static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, 279static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
@@ -317,15 +292,6 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
317 GFP_KERNEL); 292 GFP_KERNEL);
318 if (!vp_dev->msix_names) 293 if (!vp_dev->msix_names)
319 goto error; 294 goto error;
320 vp_dev->msix_affinity_masks
321 = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
322 GFP_KERNEL);
323 if (!vp_dev->msix_affinity_masks)
324 goto error;
325 for (i = 0; i < nvectors; ++i)
326 if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
327 GFP_KERNEL))
328 goto error;
329 295
330 for (i = 0; i < nvectors; ++i) 296 for (i = 0; i < nvectors; ++i)
331 vp_dev->msix_entries[i].entry = i; 297 vp_dev->msix_entries[i].entry = i;
@@ -414,6 +380,7 @@ static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
414 if (!info) 380 if (!info)
415 return ERR_PTR(-ENOMEM); 381 return ERR_PTR(-ENOMEM);
416 382
383 info->queue_index = index;
417 info->num = num; 384 info->num = num;
418 info->msix_vector = msix_vec; 385 info->msix_vector = msix_vec;
419 386
@@ -429,8 +396,8 @@ static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
429 vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); 396 vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
430 397
431 /* create the vring */ 398 /* create the vring */
432 vq = vring_new_virtqueue(index, info->num, VIRTIO_PCI_VRING_ALIGN, vdev, 399 vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
433 true, info->queue, vp_notify, callback, name); 400 vdev, info->queue, vp_notify, callback, name);
434 if (!vq) { 401 if (!vq) {
435 err = -ENOMEM; 402 err = -ENOMEM;
436 goto out_activate_queue; 403 goto out_activate_queue;
@@ -448,13 +415,9 @@ static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
448 } 415 }
449 } 416 }
450 417
451 if (callback) { 418 spin_lock_irqsave(&vp_dev->lock, flags);
452 spin_lock_irqsave(&vp_dev->lock, flags); 419 list_add(&info->node, &vp_dev->virtqueues);
453 list_add(&info->node, &vp_dev->virtqueues); 420 spin_unlock_irqrestore(&vp_dev->lock, flags);
454 spin_unlock_irqrestore(&vp_dev->lock, flags);
455 } else {
456 INIT_LIST_HEAD(&info->node);
457 }
458 421
459 return vq; 422 return vq;
460 423
@@ -478,7 +441,7 @@ static void vp_del_vq(struct virtqueue *vq)
478 list_del(&info->node); 441 list_del(&info->node);
479 spin_unlock_irqrestore(&vp_dev->lock, flags); 442 spin_unlock_irqrestore(&vp_dev->lock, flags);
480 443
481 iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL); 444 iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
482 445
483 if (vp_dev->msix_enabled) { 446 if (vp_dev->msix_enabled) {
484 iowrite16(VIRTIO_MSI_NO_VECTOR, 447 iowrite16(VIRTIO_MSI_NO_VECTOR,
@@ -553,10 +516,7 @@ static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
553 vp_dev->per_vq_vectors = per_vq_vectors; 516 vp_dev->per_vq_vectors = per_vq_vectors;
554 allocated_vectors = vp_dev->msix_used_vectors; 517 allocated_vectors = vp_dev->msix_used_vectors;
555 for (i = 0; i < nvqs; ++i) { 518 for (i = 0; i < nvqs; ++i) {
556 if (!names[i]) { 519 if (!callbacks[i] || !vp_dev->msix_enabled)
557 vqs[i] = NULL;
558 continue;
559 } else if (!callbacks[i] || !vp_dev->msix_enabled)
560 msix_vec = VIRTIO_MSI_NO_VECTOR; 520 msix_vec = VIRTIO_MSI_NO_VECTOR;
561 else if (vp_dev->per_vq_vectors) 521 else if (vp_dev->per_vq_vectors)
562 msix_vec = allocated_vectors++; 522 msix_vec = allocated_vectors++;
@@ -616,42 +576,6 @@ static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
616 false, false); 576 false, false);
617} 577}
618 578
619static const char *vp_bus_name(struct virtio_device *vdev)
620{
621 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
622
623 return pci_name(vp_dev->pci_dev);
624}
625
626/* Setup the affinity for a virtqueue:
627 * - force the affinity for per vq vector
628 * - OR over all affinities for shared MSI
629 * - ignore the affinity request if we're using INTX
630 */
631static int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
632{
633 struct virtio_device *vdev = vq->vdev;
634 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
635 struct virtio_pci_vq_info *info = vq->priv;
636 struct cpumask *mask;
637 unsigned int irq;
638
639 if (!vq->callback)
640 return -EINVAL;
641
642 if (vp_dev->msix_enabled) {
643 mask = vp_dev->msix_affinity_masks[info->msix_vector];
644 irq = vp_dev->msix_entries[info->msix_vector].vector;
645 if (cpu == -1)
646 irq_set_affinity_hint(irq, NULL);
647 else {
648 cpumask_set_cpu(cpu, mask);
649 irq_set_affinity_hint(irq, mask);
650 }
651 }
652 return 0;
653}
654
655static struct virtio_config_ops virtio_pci_config_ops = { 579static struct virtio_config_ops virtio_pci_config_ops = {
656 .get = vp_get, 580 .get = vp_get,
657 .set = vp_set, 581 .set = vp_set,
@@ -662,8 +586,6 @@ static struct virtio_config_ops virtio_pci_config_ops = {
662 .del_vqs = vp_del_vqs, 586 .del_vqs = vp_del_vqs,
663 .get_features = vp_get_features, 587 .get_features = vp_get_features,
664 .finalize_features = vp_finalize_features, 588 .finalize_features = vp_finalize_features,
665 .bus_name = vp_bus_name,
666 .set_vq_affinity = vp_set_vq_affinity,
667}; 589};
668 590
669static void virtio_pci_release_dev(struct device *_d) 591static void virtio_pci_release_dev(struct device *_d)
@@ -676,8 +598,8 @@ static void virtio_pci_release_dev(struct device *_d)
676} 598}
677 599
678/* the PCI probing function */ 600/* the PCI probing function */
679static int virtio_pci_probe(struct pci_dev *pci_dev, 601static int __devinit virtio_pci_probe(struct pci_dev *pci_dev,
680 const struct pci_device_id *id) 602 const struct pci_device_id *id)
681{ 603{
682 struct virtio_pci_device *vp_dev; 604 struct virtio_pci_device *vp_dev;
683 int err; 605 int err;
@@ -717,10 +639,8 @@ static int virtio_pci_probe(struct pci_dev *pci_dev,
717 goto out_enable_device; 639 goto out_enable_device;
718 640
719 vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0); 641 vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);
720 if (vp_dev->ioaddr == NULL) { 642 if (vp_dev->ioaddr == NULL)
721 err = -ENOMEM;
722 goto out_req_regions; 643 goto out_req_regions;
723 }
724 644
725 pci_set_drvdata(pci_dev, vp_dev); 645 pci_set_drvdata(pci_dev, vp_dev);
726 pci_set_master(pci_dev); 646 pci_set_master(pci_dev);
@@ -751,7 +671,7 @@ out:
751 return err; 671 return err;
752} 672}
753 673
754static void virtio_pci_remove(struct pci_dev *pci_dev) 674static void __devexit virtio_pci_remove(struct pci_dev *pci_dev)
755{ 675{
756 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); 676 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
757 677
@@ -766,66 +686,42 @@ static void virtio_pci_remove(struct pci_dev *pci_dev)
766} 686}
767 687
768#ifdef CONFIG_PM 688#ifdef CONFIG_PM
769static int virtio_pci_freeze(struct device *dev) 689static int virtio_pci_suspend(struct pci_dev *pci_dev, pm_message_t state)
770{ 690{
771 struct pci_dev *pci_dev = to_pci_dev(dev); 691 pci_save_state(pci_dev);
772 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); 692 pci_set_power_state(pci_dev, PCI_D3hot);
773 struct virtio_driver *drv; 693 return 0;
774 int ret;
775
776 drv = container_of(vp_dev->vdev.dev.driver,
777 struct virtio_driver, driver);
778
779 ret = 0;
780 vp_dev->saved_status = vp_get_status(&vp_dev->vdev);
781 if (drv && drv->freeze)
782 ret = drv->freeze(&vp_dev->vdev);
783
784 if (!ret)
785 pci_disable_device(pci_dev);
786 return ret;
787} 694}
788 695
789static int virtio_pci_restore(struct device *dev) 696static int virtio_pci_resume(struct pci_dev *pci_dev)
790{ 697{
791 struct pci_dev *pci_dev = to_pci_dev(dev); 698 pci_restore_state(pci_dev);
792 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); 699 pci_set_power_state(pci_dev, PCI_D0);
793 struct virtio_driver *drv; 700 return 0;
794 int ret;
795
796 drv = container_of(vp_dev->vdev.dev.driver,
797 struct virtio_driver, driver);
798
799 ret = pci_enable_device(pci_dev);
800 if (ret)
801 return ret;
802
803 pci_set_master(pci_dev);
804 vp_finalize_features(&vp_dev->vdev);
805
806 if (drv && drv->restore)
807 ret = drv->restore(&vp_dev->vdev);
808
809 /* Finally, tell the device we're all set */
810 if (!ret)
811 vp_set_status(&vp_dev->vdev, vp_dev->saved_status);
812
813 return ret;
814} 701}
815
816static const struct dev_pm_ops virtio_pci_pm_ops = {
817 SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
818};
819#endif 702#endif
820 703
821static struct pci_driver virtio_pci_driver = { 704static struct pci_driver virtio_pci_driver = {
822 .name = "virtio-pci", 705 .name = "virtio-pci",
823 .id_table = virtio_pci_id_table, 706 .id_table = virtio_pci_id_table,
824 .probe = virtio_pci_probe, 707 .probe = virtio_pci_probe,
825 .remove = virtio_pci_remove, 708 .remove = __devexit_p(virtio_pci_remove),
826#ifdef CONFIG_PM 709#ifdef CONFIG_PM
827 .driver.pm = &virtio_pci_pm_ops, 710 .suspend = virtio_pci_suspend,
711 .resume = virtio_pci_resume,
828#endif 712#endif
829}; 713};
830 714
831module_pci_driver(virtio_pci_driver); 715static int __init virtio_pci_init(void)
716{
717 return pci_register_driver(&virtio_pci_driver);
718}
719
720module_init(virtio_pci_init);
721
722static void __exit virtio_pci_exit(void)
723{
724 pci_unregister_driver(&virtio_pci_driver);
725}
726
727module_exit(virtio_pci_exit);
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index ffd7e7da5d3..68b9136847a 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -21,28 +21,23 @@
21#include <linux/virtio_config.h> 21#include <linux/virtio_config.h>
22#include <linux/device.h> 22#include <linux/device.h>
23#include <linux/slab.h> 23#include <linux/slab.h>
24#include <linux/module.h>
25#include <linux/hrtimer.h>
26 24
27/* virtio guest is communicating with a virtual "device" that actually runs on 25/* virtio guest is communicating with a virtual "device" that actually runs on
28 * a host processor. Memory barriers are used to control SMP effects. */ 26 * a host processor. Memory barriers are used to control SMP effects. */
29#ifdef CONFIG_SMP 27#ifdef CONFIG_SMP
30/* Where possible, use SMP barriers which are more lightweight than mandatory 28/* Where possible, use SMP barriers which are more lightweight than mandatory
31 * barriers, because mandatory barriers control MMIO effects on accesses 29 * barriers, because mandatory barriers control MMIO effects on accesses
32 * through relaxed memory I/O windows (which virtio-pci does not use). */ 30 * through relaxed memory I/O windows (which virtio does not use). */
33#define virtio_mb(vq) \ 31#define virtio_mb() smp_mb()
34 do { if ((vq)->weak_barriers) smp_mb(); else mb(); } while(0) 32#define virtio_rmb() smp_rmb()
35#define virtio_rmb(vq) \ 33#define virtio_wmb() smp_wmb()
36 do { if ((vq)->weak_barriers) smp_rmb(); else rmb(); } while(0)
37#define virtio_wmb(vq) \
38 do { if ((vq)->weak_barriers) smp_wmb(); else wmb(); } while(0)
39#else 34#else
40/* We must force memory ordering even if guest is UP since host could be 35/* We must force memory ordering even if guest is UP since host could be
41 * running on another CPU, but SMP barriers are defined to barrier() in that 36 * running on another CPU, but SMP barriers are defined to barrier() in that
42 * configuration. So fall back to mandatory barriers instead. */ 37 * configuration. So fall back to mandatory barriers instead. */
43#define virtio_mb(vq) mb() 38#define virtio_mb() mb()
44#define virtio_rmb(vq) rmb() 39#define virtio_rmb() rmb()
45#define virtio_wmb(vq) wmb() 40#define virtio_wmb() wmb()
46#endif 41#endif
47 42
48#ifdef DEBUG 43#ifdef DEBUG
@@ -81,9 +76,6 @@ struct vring_virtqueue
81 /* Actual memory layout for this queue */ 76 /* Actual memory layout for this queue */
82 struct vring vring; 77 struct vring vring;
83 78
84 /* Can we use weak barriers? */
85 bool weak_barriers;
86
87 /* Other side has made a mess, don't try any more. */ 79 /* Other side has made a mess, don't try any more. */
88 bool broken; 80 bool broken;
89 81
@@ -93,6 +85,8 @@ struct vring_virtqueue
93 /* Host publishes avail event idx */ 85 /* Host publishes avail event idx */
94 bool event; 86 bool event;
95 87
88 /* Number of free buffers */
89 unsigned int num_free;
96 /* Head of free buffer list. */ 90 /* Head of free buffer list. */
97 unsigned int free_head; 91 unsigned int free_head;
98 /* Number we've added since last sync. */ 92 /* Number we've added since last sync. */
@@ -107,10 +101,6 @@ struct vring_virtqueue
107#ifdef DEBUG 101#ifdef DEBUG
108 /* They're supposed to lock for us. */ 102 /* They're supposed to lock for us. */
109 unsigned int in_use; 103 unsigned int in_use;
110
111 /* Figure out if their kicks are too delayed. */
112 bool last_add_time_valid;
113 ktime_t last_add_time;
114#endif 104#endif
115 105
116 /* Tokens for callbacks. */ 106 /* Tokens for callbacks. */
@@ -130,13 +120,6 @@ static int vring_add_indirect(struct vring_virtqueue *vq,
130 unsigned head; 120 unsigned head;
131 int i; 121 int i;
132 122
133 /*
134 * We require lowmem mappings for the descriptors because
135 * otherwise virt_to_phys will give us bogus addresses in the
136 * virtqueue.
137 */
138 gfp &= ~(__GFP_HIGHMEM | __GFP_HIGH);
139
140 desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp); 123 desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp);
141 if (!desc) 124 if (!desc)
142 return -ENOMEM; 125 return -ENOMEM;
@@ -162,7 +145,7 @@ static int vring_add_indirect(struct vring_virtqueue *vq,
162 desc[i-1].next = 0; 145 desc[i-1].next = 0;
163 146
164 /* We're about to use a buffer */ 147 /* We're about to use a buffer */
165 vq->vq.num_free--; 148 vq->num_free--;
166 149
167 /* Use a single buffer which doesn't continue */ 150 /* Use a single buffer which doesn't continue */
168 head = vq->free_head; 151 head = vq->free_head;
@@ -176,26 +159,12 @@ static int vring_add_indirect(struct vring_virtqueue *vq,
176 return head; 159 return head;
177} 160}
178 161
179/** 162int virtqueue_add_buf_gfp(struct virtqueue *_vq,
180 * virtqueue_add_buf - expose buffer to other end 163 struct scatterlist sg[],
181 * @vq: the struct virtqueue we're talking about. 164 unsigned int out,
182 * @sg: the description of the buffer(s). 165 unsigned int in,
183 * @out_num: the number of sg readable by other side 166 void *data,
184 * @in_num: the number of sg which are writable (after readable ones) 167 gfp_t gfp)
185 * @data: the token identifying the buffer.
186 * @gfp: how to do memory allocations (if necessary).
187 *
188 * Caller must ensure we don't call this with other virtqueue operations
189 * at the same time (except where noted).
190 *
191 * Returns zero or a negative error (ie. ENOSPC, ENOMEM).
192 */
193int virtqueue_add_buf(struct virtqueue *_vq,
194 struct scatterlist sg[],
195 unsigned int out,
196 unsigned int in,
197 void *data,
198 gfp_t gfp)
199{ 168{
200 struct vring_virtqueue *vq = to_vvq(_vq); 169 struct vring_virtqueue *vq = to_vvq(_vq);
201 unsigned int i, avail, uninitialized_var(prev); 170 unsigned int i, avail, uninitialized_var(prev);
@@ -205,22 +174,9 @@ int virtqueue_add_buf(struct virtqueue *_vq,
205 174
206 BUG_ON(data == NULL); 175 BUG_ON(data == NULL);
207 176
208#ifdef DEBUG
209 {
210 ktime_t now = ktime_get();
211
212 /* No kick or get, with .1 second between? Warn. */
213 if (vq->last_add_time_valid)
214 WARN_ON(ktime_to_ms(ktime_sub(now, vq->last_add_time))
215 > 100);
216 vq->last_add_time = now;
217 vq->last_add_time_valid = true;
218 }
219#endif
220
221 /* If the host supports indirect descriptor tables, and we have multiple 177 /* If the host supports indirect descriptor tables, and we have multiple
222 * buffers, then go indirect. FIXME: tune this threshold */ 178 * buffers, then go indirect. FIXME: tune this threshold */
223 if (vq->indirect && (out + in) > 1 && vq->vq.num_free) { 179 if (vq->indirect && (out + in) > 1 && vq->num_free) {
224 head = vring_add_indirect(vq, sg, out, in, gfp); 180 head = vring_add_indirect(vq, sg, out, in, gfp);
225 if (likely(head >= 0)) 181 if (likely(head >= 0))
226 goto add_head; 182 goto add_head;
@@ -229,9 +185,9 @@ int virtqueue_add_buf(struct virtqueue *_vq,
229 BUG_ON(out + in > vq->vring.num); 185 BUG_ON(out + in > vq->vring.num);
230 BUG_ON(out + in == 0); 186 BUG_ON(out + in == 0);
231 187
232 if (vq->vq.num_free < out + in) { 188 if (vq->num_free < out + in) {
233 pr_debug("Can't add buf len %i - avail = %i\n", 189 pr_debug("Can't add buf len %i - avail = %i\n",
234 out + in, vq->vq.num_free); 190 out + in, vq->num_free);
235 /* FIXME: for historical reasons, we force a notify here if 191 /* FIXME: for historical reasons, we force a notify here if
236 * there are outgoing parts to the buffer. Presumably the 192 * there are outgoing parts to the buffer. Presumably the
237 * host should service the ring ASAP. */ 193 * host should service the ring ASAP. */
@@ -242,7 +198,7 @@ int virtqueue_add_buf(struct virtqueue *_vq,
242 } 198 }
243 199
244 /* We're about to use some buffers from the free list. */ 200 /* We're about to use some buffers from the free list. */
245 vq->vq.num_free -= out + in; 201 vq->num_free -= out + in;
246 202
247 head = vq->free_head; 203 head = vq->free_head;
248 for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) { 204 for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) {
@@ -270,102 +226,40 @@ add_head:
270 vq->data[head] = data; 226 vq->data[head] = data;
271 227
272 /* Put entry in available array (but don't update avail->idx until they 228 /* Put entry in available array (but don't update avail->idx until they
273 * do sync). */ 229 * do sync). FIXME: avoid modulus here? */
274 avail = (vq->vring.avail->idx & (vq->vring.num-1)); 230 avail = (vq->vring.avail->idx + vq->num_added++) % vq->vring.num;
275 vq->vring.avail->ring[avail] = head; 231 vq->vring.avail->ring[avail] = head;
276 232
277 /* Descriptors and available array need to be set before we expose the
278 * new available array entries. */
279 virtio_wmb(vq);
280 vq->vring.avail->idx++;
281 vq->num_added++;
282
283 /* This is very unlikely, but theoretically possible. Kick
284 * just in case. */
285 if (unlikely(vq->num_added == (1 << 16) - 1))
286 virtqueue_kick(_vq);
287
288 pr_debug("Added buffer head %i to %p\n", head, vq); 233 pr_debug("Added buffer head %i to %p\n", head, vq);
289 END_USE(vq); 234 END_USE(vq);
290 235
291 return 0; 236 return vq->num_free;
292} 237}
293EXPORT_SYMBOL_GPL(virtqueue_add_buf); 238EXPORT_SYMBOL_GPL(virtqueue_add_buf_gfp);
294 239
295/** 240void virtqueue_kick(struct virtqueue *_vq)
296 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
297 * @vq: the struct virtqueue
298 *
299 * Instead of virtqueue_kick(), you can do:
300 * if (virtqueue_kick_prepare(vq))
301 * virtqueue_notify(vq);
302 *
303 * This is sometimes useful because the virtqueue_kick_prepare() needs
304 * to be serialized, but the actual virtqueue_notify() call does not.
305 */
306bool virtqueue_kick_prepare(struct virtqueue *_vq)
307{ 241{
308 struct vring_virtqueue *vq = to_vvq(_vq); 242 struct vring_virtqueue *vq = to_vvq(_vq);
309 u16 new, old; 243 u16 new, old;
310 bool needs_kick;
311
312 START_USE(vq); 244 START_USE(vq);
313 /* We need to expose available array entries before checking avail 245 /* Descriptors and available array need to be set before we expose the
314 * event. */ 246 * new available array entries. */
315 virtio_mb(vq); 247 virtio_wmb();
316 248
317 old = vq->vring.avail->idx - vq->num_added; 249 old = vq->vring.avail->idx;
318 new = vq->vring.avail->idx; 250 new = vq->vring.avail->idx = old + vq->num_added;
319 vq->num_added = 0; 251 vq->num_added = 0;
320 252
321#ifdef DEBUG 253 /* Need to update avail index before checking if we should notify */
322 if (vq->last_add_time_valid) { 254 virtio_mb();
323 WARN_ON(ktime_to_ms(ktime_sub(ktime_get(),
324 vq->last_add_time)) > 100);
325 }
326 vq->last_add_time_valid = false;
327#endif
328
329 if (vq->event) {
330 needs_kick = vring_need_event(vring_avail_event(&vq->vring),
331 new, old);
332 } else {
333 needs_kick = !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY);
334 }
335 END_USE(vq);
336 return needs_kick;
337}
338EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
339
340/**
341 * virtqueue_notify - second half of split virtqueue_kick call.
342 * @vq: the struct virtqueue
343 *
344 * This does not need to be serialized.
345 */
346void virtqueue_notify(struct virtqueue *_vq)
347{
348 struct vring_virtqueue *vq = to_vvq(_vq);
349 255
350 /* Prod other side to tell it about changes. */ 256 if (vq->event ?
351 vq->notify(_vq); 257 vring_need_event(vring_avail_event(&vq->vring), new, old) :
352} 258 !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY))
353EXPORT_SYMBOL_GPL(virtqueue_notify); 259 /* Prod other side to tell it about changes. */
260 vq->notify(&vq->vq);
354 261
355/** 262 END_USE(vq);
356 * virtqueue_kick - update after add_buf
357 * @vq: the struct virtqueue
358 *
359 * After one or more virtqueue_add_buf calls, invoke this to kick
360 * the other side.
361 *
362 * Caller must ensure we don't call this with other virtqueue
363 * operations at the same time (except where noted).
364 */
365void virtqueue_kick(struct virtqueue *vq)
366{
367 if (virtqueue_kick_prepare(vq))
368 virtqueue_notify(vq);
369} 263}
370EXPORT_SYMBOL_GPL(virtqueue_kick); 264EXPORT_SYMBOL_GPL(virtqueue_kick);
371 265
@@ -385,13 +279,13 @@ static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
385 279
386 while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) { 280 while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) {
387 i = vq->vring.desc[i].next; 281 i = vq->vring.desc[i].next;
388 vq->vq.num_free++; 282 vq->num_free++;
389 } 283 }
390 284
391 vq->vring.desc[i].next = vq->free_head; 285 vq->vring.desc[i].next = vq->free_head;
392 vq->free_head = head; 286 vq->free_head = head;
393 /* Plus final descriptor */ 287 /* Plus final descriptor */
394 vq->vq.num_free++; 288 vq->num_free++;
395} 289}
396 290
397static inline bool more_used(const struct vring_virtqueue *vq) 291static inline bool more_used(const struct vring_virtqueue *vq)
@@ -399,28 +293,11 @@ static inline bool more_used(const struct vring_virtqueue *vq)
399 return vq->last_used_idx != vq->vring.used->idx; 293 return vq->last_used_idx != vq->vring.used->idx;
400} 294}
401 295
402/**
403 * virtqueue_get_buf - get the next used buffer
404 * @vq: the struct virtqueue we're talking about.
405 * @len: the length written into the buffer
406 *
407 * If the driver wrote data into the buffer, @len will be set to the
408 * amount written. This means you don't need to clear the buffer
409 * beforehand to ensure there's no data leakage in the case of short
410 * writes.
411 *
412 * Caller must ensure we don't call this with other virtqueue
413 * operations at the same time (except where noted).
414 *
415 * Returns NULL if there are no used buffers, or the "data" token
416 * handed to virtqueue_add_buf().
417 */
418void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len) 296void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
419{ 297{
420 struct vring_virtqueue *vq = to_vvq(_vq); 298 struct vring_virtqueue *vq = to_vvq(_vq);
421 void *ret; 299 void *ret;
422 unsigned int i; 300 unsigned int i;
423 u16 last_used;
424 301
425 START_USE(vq); 302 START_USE(vq);
426 303
@@ -436,11 +313,10 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
436 } 313 }
437 314
438 /* Only get used array entries after they have been exposed by host. */ 315 /* Only get used array entries after they have been exposed by host. */
439 virtio_rmb(vq); 316 virtio_rmb();
440 317
441 last_used = (vq->last_used_idx & (vq->vring.num - 1)); 318 i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
442 i = vq->vring.used->ring[last_used].id; 319 *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
443 *len = vq->vring.used->ring[last_used].len;
444 320
445 if (unlikely(i >= vq->vring.num)) { 321 if (unlikely(i >= vq->vring.num)) {
446 BAD_RING(vq, "id %u out of range\n", i); 322 BAD_RING(vq, "id %u out of range\n", i);
@@ -460,27 +336,14 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
460 * the read in the next get_buf call. */ 336 * the read in the next get_buf call. */
461 if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) { 337 if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
462 vring_used_event(&vq->vring) = vq->last_used_idx; 338 vring_used_event(&vq->vring) = vq->last_used_idx;
463 virtio_mb(vq); 339 virtio_mb();
464 } 340 }
465 341
466#ifdef DEBUG
467 vq->last_add_time_valid = false;
468#endif
469
470 END_USE(vq); 342 END_USE(vq);
471 return ret; 343 return ret;
472} 344}
473EXPORT_SYMBOL_GPL(virtqueue_get_buf); 345EXPORT_SYMBOL_GPL(virtqueue_get_buf);
474 346
475/**
476 * virtqueue_disable_cb - disable callbacks
477 * @vq: the struct virtqueue we're talking about.
478 *
479 * Note that this is not necessarily synchronous, hence unreliable and only
480 * useful as an optimization.
481 *
482 * Unlike other operations, this need not be serialized.
483 */
484void virtqueue_disable_cb(struct virtqueue *_vq) 347void virtqueue_disable_cb(struct virtqueue *_vq)
485{ 348{
486 struct vring_virtqueue *vq = to_vvq(_vq); 349 struct vring_virtqueue *vq = to_vvq(_vq);
@@ -489,17 +352,6 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
489} 352}
490EXPORT_SYMBOL_GPL(virtqueue_disable_cb); 353EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
491 354
492/**
493 * virtqueue_enable_cb - restart callbacks after disable_cb.
494 * @vq: the struct virtqueue we're talking about.
495 *
496 * This re-enables callbacks; it returns "false" if there are pending
497 * buffers in the queue, to detect a possible race between the driver
498 * checking for more work, and enabling callbacks.
499 *
500 * Caller must ensure we don't call this with other virtqueue
501 * operations at the same time (except where noted).
502 */
503bool virtqueue_enable_cb(struct virtqueue *_vq) 355bool virtqueue_enable_cb(struct virtqueue *_vq)
504{ 356{
505 struct vring_virtqueue *vq = to_vvq(_vq); 357 struct vring_virtqueue *vq = to_vvq(_vq);
@@ -513,7 +365,7 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
513 * entry. Always do both to keep code simple. */ 365 * entry. Always do both to keep code simple. */
514 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 366 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
515 vring_used_event(&vq->vring) = vq->last_used_idx; 367 vring_used_event(&vq->vring) = vq->last_used_idx;
516 virtio_mb(vq); 368 virtio_mb();
517 if (unlikely(more_used(vq))) { 369 if (unlikely(more_used(vq))) {
518 END_USE(vq); 370 END_USE(vq);
519 return false; 371 return false;
@@ -524,19 +376,6 @@ bool virtqueue_enable_cb(struct virtqueue *_vq)
524} 376}
525EXPORT_SYMBOL_GPL(virtqueue_enable_cb); 377EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
526 378
527/**
528 * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
529 * @vq: the struct virtqueue we're talking about.
530 *
531 * This re-enables callbacks but hints to the other side to delay
532 * interrupts until most of the available buffers have been processed;
533 * it returns "false" if there are many pending buffers in the queue,
534 * to detect a possible race between the driver checking for more work,
535 * and enabling callbacks.
536 *
537 * Caller must ensure we don't call this with other virtqueue
538 * operations at the same time (except where noted).
539 */
540bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) 379bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
541{ 380{
542 struct vring_virtqueue *vq = to_vvq(_vq); 381 struct vring_virtqueue *vq = to_vvq(_vq);
@@ -553,7 +392,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
553 /* TODO: tune this threshold */ 392 /* TODO: tune this threshold */
554 bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4; 393 bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4;
555 vring_used_event(&vq->vring) = vq->last_used_idx + bufs; 394 vring_used_event(&vq->vring) = vq->last_used_idx + bufs;
556 virtio_mb(vq); 395 virtio_mb();
557 if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) { 396 if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) {
558 END_USE(vq); 397 END_USE(vq);
559 return false; 398 return false;
@@ -564,14 +403,6 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
564} 403}
565EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); 404EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
566 405
567/**
568 * virtqueue_detach_unused_buf - detach first unused buffer
569 * @vq: the struct virtqueue we're talking about.
570 *
571 * Returns NULL or the "data" token handed to virtqueue_add_buf().
572 * This is not valid on an active queue; it is useful only for device
573 * shutdown.
574 */
575void *virtqueue_detach_unused_buf(struct virtqueue *_vq) 406void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
576{ 407{
577 struct vring_virtqueue *vq = to_vvq(_vq); 408 struct vring_virtqueue *vq = to_vvq(_vq);
@@ -591,7 +422,7 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
591 return buf; 422 return buf;
592 } 423 }
593 /* That should have freed everything. */ 424 /* That should have freed everything. */
594 BUG_ON(vq->vq.num_free != vq->vring.num); 425 BUG_ON(vq->num_free != vq->vring.num);
595 426
596 END_USE(vq); 427 END_USE(vq);
597 return NULL; 428 return NULL;
@@ -618,11 +449,9 @@ irqreturn_t vring_interrupt(int irq, void *_vq)
618} 449}
619EXPORT_SYMBOL_GPL(vring_interrupt); 450EXPORT_SYMBOL_GPL(vring_interrupt);
620 451
621struct virtqueue *vring_new_virtqueue(unsigned int index, 452struct virtqueue *vring_new_virtqueue(unsigned int num,
622 unsigned int num,
623 unsigned int vring_align, 453 unsigned int vring_align,
624 struct virtio_device *vdev, 454 struct virtio_device *vdev,
625 bool weak_barriers,
626 void *pages, 455 void *pages,
627 void (*notify)(struct virtqueue *), 456 void (*notify)(struct virtqueue *),
628 void (*callback)(struct virtqueue *), 457 void (*callback)(struct virtqueue *),
@@ -645,17 +474,13 @@ struct virtqueue *vring_new_virtqueue(unsigned int index,
645 vq->vq.callback = callback; 474 vq->vq.callback = callback;
646 vq->vq.vdev = vdev; 475 vq->vq.vdev = vdev;
647 vq->vq.name = name; 476 vq->vq.name = name;
648 vq->vq.num_free = num;
649 vq->vq.index = index;
650 vq->notify = notify; 477 vq->notify = notify;
651 vq->weak_barriers = weak_barriers;
652 vq->broken = false; 478 vq->broken = false;
653 vq->last_used_idx = 0; 479 vq->last_used_idx = 0;
654 vq->num_added = 0; 480 vq->num_added = 0;
655 list_add_tail(&vq->vq.list, &vdev->vqs); 481 list_add_tail(&vq->vq.list, &vdev->vqs);
656#ifdef DEBUG 482#ifdef DEBUG
657 vq->in_use = false; 483 vq->in_use = false;
658 vq->last_add_time_valid = false;
659#endif 484#endif
660 485
661 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC); 486 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
@@ -666,6 +491,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int index,
666 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 491 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
667 492
668 /* Put everything in free lists. */ 493 /* Put everything in free lists. */
494 vq->num_free = num;
669 vq->free_head = 0; 495 vq->free_head = 0;
670 for (i = 0; i < num-1; i++) { 496 for (i = 0; i < num-1; i++) {
671 vq->vring.desc[i].next = i+1; 497 vq->vring.desc[i].next = i+1;
@@ -703,20 +529,4 @@ void vring_transport_features(struct virtio_device *vdev)
703} 529}
704EXPORT_SYMBOL_GPL(vring_transport_features); 530EXPORT_SYMBOL_GPL(vring_transport_features);
705 531
706/**
707 * virtqueue_get_vring_size - return the size of the virtqueue's vring
708 * @vq: the struct virtqueue containing the vring of interest.
709 *
710 * Returns the size of the vring. This is mainly used for boasting to
711 * userspace. Unlike other operations, this need not be serialized.
712 */
713unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
714{
715
716 struct vring_virtqueue *vq = to_vvq(_vq);
717
718 return vq->vring.num;
719}
720EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
721
722MODULE_LICENSE("GPL"); 532MODULE_LICENSE("GPL");
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-10 02:42:31 -0500