aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/block/virtio_blk.c74
-rw-r--r--drivers/char/virtio_console.c39
-rw-r--r--drivers/lguest/lguest_device.c17
-rw-r--r--drivers/misc/mic/card/mic_virtio.c14
-rw-r--r--drivers/net/macvtap.c68
-rw-r--r--drivers/net/tun.c168
-rw-r--r--drivers/net/virtio_net.c161
-rw-r--r--drivers/remoteproc/remoteproc_virtio.c11
-rw-r--r--drivers/s390/kvm/kvm_virtio.c11
-rw-r--r--drivers/s390/kvm/virtio_ccw.c203
-rw-r--r--drivers/scsi/virtio_scsi.c50
-rw-r--r--drivers/vhost/net.c31
-rw-r--r--drivers/vhost/scsi.c22
-rw-r--r--drivers/vhost/vhost.c93
-rw-r--r--drivers/vhost/vhost.h41
-rw-r--r--drivers/virtio/Makefile1
-rw-r--r--drivers/virtio/virtio.c102
-rw-r--r--drivers/virtio/virtio_mmio.c17
-rw-r--r--drivers/virtio/virtio_pci.c802
-rw-r--r--drivers/virtio/virtio_pci_common.c464
-rw-r--r--drivers/virtio/virtio_pci_common.h136
-rw-r--r--drivers/virtio/virtio_pci_legacy.c326
-rw-r--r--drivers/virtio/virtio_ring.c109
-rw-r--r--include/linux/virtio.h12
-rw-r--r--include/linux/virtio_byteorder.h59
-rw-r--r--include/linux/virtio_config.h103
-rw-r--r--include/uapi/linux/Kbuild2
-rw-r--r--include/uapi/linux/if_tun.h17
-rw-r--r--include/uapi/linux/virtio_blk.h15
-rw-r--r--include/uapi/linux/virtio_config.h9
-rw-r--r--include/uapi/linux/virtio_console.h7
-rw-r--r--include/uapi/linux/virtio_net.h15
-rw-r--r--include/uapi/linux/virtio_ring.h45
-rw-r--r--include/uapi/linux/virtio_scsi.h (renamed from include/linux/virtio_scsi.h)106
-rw-r--r--include/uapi/linux/virtio_types.h46
-rw-r--r--net/packet/af_packet.c35
-rw-r--r--tools/virtio/linux/virtio.h22
-rw-r--r--tools/virtio/linux/virtio_config.h2
-rw-r--r--tools/virtio/virtio_test.c5
-rw-r--r--tools/virtio/vringh_test.c16
40 files changed, 2048 insertions, 1428 deletions
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index c6a27d54ad62..1fb9e09fbbc5 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -80,7 +80,7 @@ static int __virtblk_add_req(struct virtqueue *vq,
80{ 80{
81 struct scatterlist hdr, status, cmd, sense, inhdr, *sgs[6]; 81 struct scatterlist hdr, status, cmd, sense, inhdr, *sgs[6];
82 unsigned int num_out = 0, num_in = 0; 82 unsigned int num_out = 0, num_in = 0;
83 int type = vbr->out_hdr.type & ~VIRTIO_BLK_T_OUT; 83 __virtio32 type = vbr->out_hdr.type & ~cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT);
84 84
85 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); 85 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr));
86 sgs[num_out++] = &hdr; 86 sgs[num_out++] = &hdr;
@@ -91,19 +91,19 @@ static int __virtblk_add_req(struct virtqueue *vq,
91 * block, and before the normal inhdr we put the sense data and the 91 * block, and before the normal inhdr we put the sense data and the
92 * inhdr with additional status information. 92 * inhdr with additional status information.
93 */ 93 */
94 if (type == VIRTIO_BLK_T_SCSI_CMD) { 94 if (type == cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_SCSI_CMD)) {
95 sg_init_one(&cmd, vbr->req->cmd, vbr->req->cmd_len); 95 sg_init_one(&cmd, vbr->req->cmd, vbr->req->cmd_len);
96 sgs[num_out++] = &cmd; 96 sgs[num_out++] = &cmd;
97 } 97 }
98 98
99 if (have_data) { 99 if (have_data) {
100 if (vbr->out_hdr.type & VIRTIO_BLK_T_OUT) 100 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT))
101 sgs[num_out++] = data_sg; 101 sgs[num_out++] = data_sg;
102 else 102 else
103 sgs[num_out + num_in++] = data_sg; 103 sgs[num_out + num_in++] = data_sg;
104 } 104 }
105 105
106 if (type == VIRTIO_BLK_T_SCSI_CMD) { 106 if (type == cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_SCSI_CMD)) {
107 sg_init_one(&sense, vbr->req->sense, SCSI_SENSE_BUFFERSIZE); 107 sg_init_one(&sense, vbr->req->sense, SCSI_SENSE_BUFFERSIZE);
108 sgs[num_out + num_in++] = &sense; 108 sgs[num_out + num_in++] = &sense;
109 sg_init_one(&inhdr, &vbr->in_hdr, sizeof(vbr->in_hdr)); 109 sg_init_one(&inhdr, &vbr->in_hdr, sizeof(vbr->in_hdr));
@@ -119,12 +119,13 @@ static int __virtblk_add_req(struct virtqueue *vq,
119static inline void virtblk_request_done(struct request *req) 119static inline void virtblk_request_done(struct request *req)
120{ 120{
121 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); 121 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
122 struct virtio_blk *vblk = req->q->queuedata;
122 int error = virtblk_result(vbr); 123 int error = virtblk_result(vbr);
123 124
124 if (req->cmd_type == REQ_TYPE_BLOCK_PC) { 125 if (req->cmd_type == REQ_TYPE_BLOCK_PC) {
125 req->resid_len = vbr->in_hdr.residual; 126 req->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual);
126 req->sense_len = vbr->in_hdr.sense_len; 127 req->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len);
127 req->errors = vbr->in_hdr.errors; 128 req->errors = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors);
128 } else if (req->cmd_type == REQ_TYPE_SPECIAL) { 129 } else if (req->cmd_type == REQ_TYPE_SPECIAL) {
129 req->errors = (error != 0); 130 req->errors = (error != 0);
130 } 131 }
@@ -173,25 +174,25 @@ static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req,
173 174
174 vbr->req = req; 175 vbr->req = req;
175 if (req->cmd_flags & REQ_FLUSH) { 176 if (req->cmd_flags & REQ_FLUSH) {
176 vbr->out_hdr.type = VIRTIO_BLK_T_FLUSH; 177 vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_FLUSH);
177 vbr->out_hdr.sector = 0; 178 vbr->out_hdr.sector = 0;
178 vbr->out_hdr.ioprio = req_get_ioprio(vbr->req); 179 vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));
179 } else { 180 } else {
180 switch (req->cmd_type) { 181 switch (req->cmd_type) {
181 case REQ_TYPE_FS: 182 case REQ_TYPE_FS:
182 vbr->out_hdr.type = 0; 183 vbr->out_hdr.type = 0;
183 vbr->out_hdr.sector = blk_rq_pos(vbr->req); 184 vbr->out_hdr.sector = cpu_to_virtio64(vblk->vdev, blk_rq_pos(vbr->req));
184 vbr->out_hdr.ioprio = req_get_ioprio(vbr->req); 185 vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));
185 break; 186 break;
186 case REQ_TYPE_BLOCK_PC: 187 case REQ_TYPE_BLOCK_PC:
187 vbr->out_hdr.type = VIRTIO_BLK_T_SCSI_CMD; 188 vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_SCSI_CMD);
188 vbr->out_hdr.sector = 0; 189 vbr->out_hdr.sector = 0;
189 vbr->out_hdr.ioprio = req_get_ioprio(vbr->req); 190 vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));
190 break; 191 break;
191 case REQ_TYPE_SPECIAL: 192 case REQ_TYPE_SPECIAL:
192 vbr->out_hdr.type = VIRTIO_BLK_T_GET_ID; 193 vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_GET_ID);
193 vbr->out_hdr.sector = 0; 194 vbr->out_hdr.sector = 0;
194 vbr->out_hdr.ioprio = req_get_ioprio(vbr->req); 195 vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));
195 break; 196 break;
196 default: 197 default:
197 /* We don't put anything else in the queue. */ 198 /* We don't put anything else in the queue. */
@@ -204,9 +205,9 @@ static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *req,
204 num = blk_rq_map_sg(hctx->queue, vbr->req, vbr->sg); 205 num = blk_rq_map_sg(hctx->queue, vbr->req, vbr->sg);
205 if (num) { 206 if (num) {
206 if (rq_data_dir(vbr->req) == WRITE) 207 if (rq_data_dir(vbr->req) == WRITE)
207 vbr->out_hdr.type |= VIRTIO_BLK_T_OUT; 208 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT);
208 else 209 else
209 vbr->out_hdr.type |= VIRTIO_BLK_T_IN; 210 vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN);
210 } 211 }
211 212
212 spin_lock_irqsave(&vblk->vqs[qid].lock, flags); 213 spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
@@ -331,7 +332,8 @@ static ssize_t virtblk_serial_show(struct device *dev,
331 332
332 return err; 333 return err;
333} 334}
334DEVICE_ATTR(serial, S_IRUGO, virtblk_serial_show, NULL); 335
336static DEVICE_ATTR(serial, S_IRUGO, virtblk_serial_show, NULL);
335 337
336static void virtblk_config_changed_work(struct work_struct *work) 338static void virtblk_config_changed_work(struct work_struct *work)
337{ 339{
@@ -476,7 +478,8 @@ static int virtblk_get_cache_mode(struct virtio_device *vdev)
476 struct virtio_blk_config, wce, 478 struct virtio_blk_config, wce,
477 &writeback); 479 &writeback);
478 if (err) 480 if (err)
479 writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_WCE); 481 writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_WCE) ||
482 virtio_has_feature(vdev, VIRTIO_F_VERSION_1);
480 483
481 return writeback; 484 return writeback;
482} 485}
@@ -821,25 +824,34 @@ static const struct virtio_device_id id_table[] = {
821 { 0 }, 824 { 0 },
822}; 825};
823 826
824static unsigned int features[] = { 827static unsigned int features_legacy[] = {
825 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, 828 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
826 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, VIRTIO_BLK_F_SCSI, 829 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, VIRTIO_BLK_F_SCSI,
827 VIRTIO_BLK_F_WCE, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, 830 VIRTIO_BLK_F_WCE, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
828 VIRTIO_BLK_F_MQ, 831 VIRTIO_BLK_F_MQ,
832}
833;
834static unsigned int features[] = {
835 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
836 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
837 VIRTIO_BLK_F_TOPOLOGY,
838 VIRTIO_BLK_F_MQ,
829}; 839};
830 840
831static struct virtio_driver virtio_blk = { 841static struct virtio_driver virtio_blk = {
832 .feature_table = features, 842 .feature_table = features,
833 .feature_table_size = ARRAY_SIZE(features), 843 .feature_table_size = ARRAY_SIZE(features),
834 .driver.name = KBUILD_MODNAME, 844 .feature_table_legacy = features_legacy,
835 .driver.owner = THIS_MODULE, 845 .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
836 .id_table = id_table, 846 .driver.name = KBUILD_MODNAME,
837 .probe = virtblk_probe, 847 .driver.owner = THIS_MODULE,
838 .remove = virtblk_remove, 848 .id_table = id_table,
839 .config_changed = virtblk_config_changed, 849 .probe = virtblk_probe,
850 .remove = virtblk_remove,
851 .config_changed = virtblk_config_changed,
840#ifdef CONFIG_PM_SLEEP 852#ifdef CONFIG_PM_SLEEP
841 .freeze = virtblk_freeze, 853 .freeze = virtblk_freeze,
842 .restore = virtblk_restore, 854 .restore = virtblk_restore,
843#endif 855#endif
844}; 856};
845 857
@@ -871,8 +883,8 @@ out_destroy_workqueue:
871 883
872static void __exit fini(void) 884static void __exit fini(void)
873{ 885{
874 unregister_blkdev(major, "virtblk");
875 unregister_virtio_driver(&virtio_blk); 886 unregister_virtio_driver(&virtio_blk);
887 unregister_blkdev(major, "virtblk");
876 destroy_workqueue(virtblk_wq); 888 destroy_workqueue(virtblk_wq);
877} 889}
878module_init(init); 890module_init(init);
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index cf7a561fad7c..de03df9dd7c9 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -355,7 +355,7 @@ static inline bool use_multiport(struct ports_device *portdev)
355 */ 355 */
356 if (!portdev->vdev) 356 if (!portdev->vdev)
357 return 0; 357 return 0;
358 return portdev->vdev->features[0] & (1 << VIRTIO_CONSOLE_F_MULTIPORT); 358 return __virtio_test_bit(portdev->vdev, VIRTIO_CONSOLE_F_MULTIPORT);
359} 359}
360 360
361static DEFINE_SPINLOCK(dma_bufs_lock); 361static DEFINE_SPINLOCK(dma_bufs_lock);
@@ -566,9 +566,9 @@ static ssize_t __send_control_msg(struct ports_device *portdev, u32 port_id,
566 if (!use_multiport(portdev)) 566 if (!use_multiport(portdev))
567 return 0; 567 return 0;
568 568
569 cpkt.id = port_id; 569 cpkt.id = cpu_to_virtio32(portdev->vdev, port_id);
570 cpkt.event = event; 570 cpkt.event = cpu_to_virtio16(portdev->vdev, event);
571 cpkt.value = value; 571 cpkt.value = cpu_to_virtio16(portdev->vdev, value);
572 572
573 vq = portdev->c_ovq; 573 vq = portdev->c_ovq;
574 574
@@ -669,8 +669,8 @@ done:
669 * Give out the data that's requested from the buffer that we have 669 * Give out the data that's requested from the buffer that we have
670 * queued up. 670 * queued up.
671 */ 671 */
672static ssize_t fill_readbuf(struct port *port, char *out_buf, size_t out_count, 672static ssize_t fill_readbuf(struct port *port, char __user *out_buf,
673 bool to_user) 673 size_t out_count, bool to_user)
674{ 674{
675 struct port_buffer *buf; 675 struct port_buffer *buf;
676 unsigned long flags; 676 unsigned long flags;
@@ -688,7 +688,8 @@ static ssize_t fill_readbuf(struct port *port, char *out_buf, size_t out_count,
688 if (ret) 688 if (ret)
689 return -EFAULT; 689 return -EFAULT;
690 } else { 690 } else {
691 memcpy(out_buf, buf->buf + buf->offset, out_count); 691 memcpy((__force char *)out_buf, buf->buf + buf->offset,
692 out_count);
692 } 693 }
693 694
694 buf->offset += out_count; 695 buf->offset += out_count;
@@ -1162,7 +1163,7 @@ static int get_chars(u32 vtermno, char *buf, int count)
1162 /* If we don't have an input queue yet, we can't get input. */ 1163 /* If we don't have an input queue yet, we can't get input. */
1163 BUG_ON(!port->in_vq); 1164 BUG_ON(!port->in_vq);
1164 1165
1165 return fill_readbuf(port, buf, count, false); 1166 return fill_readbuf(port, (__force char __user *)buf, count, false);
1166} 1167}
1167 1168
1168static void resize_console(struct port *port) 1169static void resize_console(struct port *port)
@@ -1602,7 +1603,8 @@ static void unplug_port(struct port *port)
1602} 1603}
1603 1604
1604/* Any private messages that the Host and Guest want to share */ 1605/* Any private messages that the Host and Guest want to share */
1605static void handle_control_message(struct ports_device *portdev, 1606static void handle_control_message(struct virtio_device *vdev,
1607 struct ports_device *portdev,
1606 struct port_buffer *buf) 1608 struct port_buffer *buf)
1607{ 1609{
1608 struct virtio_console_control *cpkt; 1610 struct virtio_console_control *cpkt;
@@ -1612,15 +1614,16 @@ static void handle_control_message(struct ports_device *portdev,
1612 1614
1613 cpkt = (struct virtio_console_control *)(buf->buf + buf->offset); 1615 cpkt = (struct virtio_console_control *)(buf->buf + buf->offset);
1614 1616
1615 port = find_port_by_id(portdev, cpkt->id); 1617 port = find_port_by_id(portdev, virtio32_to_cpu(vdev, cpkt->id));
1616 if (!port && cpkt->event != VIRTIO_CONSOLE_PORT_ADD) { 1618 if (!port &&
1619 cpkt->event != cpu_to_virtio16(vdev, VIRTIO_CONSOLE_PORT_ADD)) {
1617 /* No valid header at start of buffer. Drop it. */ 1620 /* No valid header at start of buffer. Drop it. */
1618 dev_dbg(&portdev->vdev->dev, 1621 dev_dbg(&portdev->vdev->dev,
1619 "Invalid index %u in control packet\n", cpkt->id); 1622 "Invalid index %u in control packet\n", cpkt->id);
1620 return; 1623 return;
1621 } 1624 }
1622 1625
1623 switch (cpkt->event) { 1626 switch (virtio16_to_cpu(vdev, cpkt->event)) {
1624 case VIRTIO_CONSOLE_PORT_ADD: 1627 case VIRTIO_CONSOLE_PORT_ADD:
1625 if (port) { 1628 if (port) {
1626 dev_dbg(&portdev->vdev->dev, 1629 dev_dbg(&portdev->vdev->dev,
@@ -1628,13 +1631,15 @@ static void handle_control_message(struct ports_device *portdev,
1628 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, 1); 1631 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, 1);
1629 break; 1632 break;
1630 } 1633 }
1631 if (cpkt->id >= portdev->config.max_nr_ports) { 1634 if (virtio32_to_cpu(vdev, cpkt->id) >=
1635 portdev->config.max_nr_ports) {
1632 dev_warn(&portdev->vdev->dev, 1636 dev_warn(&portdev->vdev->dev,
1633 "Request for adding port with out-of-bound id %u, max. supported id: %u\n", 1637 "Request for adding port with "
1638 "out-of-bound id %u, max. supported id: %u\n",
1634 cpkt->id, portdev->config.max_nr_ports - 1); 1639 cpkt->id, portdev->config.max_nr_ports - 1);
1635 break; 1640 break;
1636 } 1641 }
1637 add_port(portdev, cpkt->id); 1642 add_port(portdev, virtio32_to_cpu(vdev, cpkt->id));
1638 break; 1643 break;
1639 case VIRTIO_CONSOLE_PORT_REMOVE: 1644 case VIRTIO_CONSOLE_PORT_REMOVE:
1640 unplug_port(port); 1645 unplug_port(port);
@@ -1670,7 +1675,7 @@ static void handle_control_message(struct ports_device *portdev,
1670 break; 1675 break;
1671 } 1676 }
1672 case VIRTIO_CONSOLE_PORT_OPEN: 1677 case VIRTIO_CONSOLE_PORT_OPEN:
1673 port->host_connected = cpkt->value; 1678 port->host_connected = virtio16_to_cpu(vdev, cpkt->value);
1674 wake_up_interruptible(&port->waitqueue); 1679 wake_up_interruptible(&port->waitqueue);
1675 /* 1680 /*
1676 * If the host port got closed and the host had any 1681 * If the host port got closed and the host had any
@@ -1752,7 +1757,7 @@ static void control_work_handler(struct work_struct *work)
1752 buf->len = len; 1757 buf->len = len;
1753 buf->offset = 0; 1758 buf->offset = 0;
1754 1759
1755 handle_control_message(portdev, buf); 1760 handle_control_message(vq->vdev, portdev, buf);
1756 1761
1757 spin_lock(&portdev->c_ivq_lock); 1762 spin_lock(&portdev->c_ivq_lock);
1758 if (add_inbuf(portdev->c_ivq, buf) < 0) { 1763 if (add_inbuf(portdev->c_ivq, buf) < 0) {
diff --git a/drivers/lguest/lguest_device.c b/drivers/lguest/lguest_device.c
index d0a1d8a45c81..89088d6538fd 100644
--- a/drivers/lguest/lguest_device.c
+++ b/drivers/lguest/lguest_device.c
@@ -94,7 +94,7 @@ static unsigned desc_size(const struct lguest_device_desc *desc)
94} 94}
95 95
96/* This gets the device's feature bits. */ 96/* This gets the device's feature bits. */
97static u32 lg_get_features(struct virtio_device *vdev) 97static u64 lg_get_features(struct virtio_device *vdev)
98{ 98{
99 unsigned int i; 99 unsigned int i;
100 u32 features = 0; 100 u32 features = 0;
@@ -126,7 +126,7 @@ static void status_notify(struct virtio_device *vdev)
126 * sorted out, this routine is called so we can tell the Host which features we 126 * sorted out, this routine is called so we can tell the Host which features we
127 * understand and accept. 127 * understand and accept.
128 */ 128 */
129static void lg_finalize_features(struct virtio_device *vdev) 129static int lg_finalize_features(struct virtio_device *vdev)
130{ 130{
131 unsigned int i, bits; 131 unsigned int i, bits;
132 struct lguest_device_desc *desc = to_lgdev(vdev)->desc; 132 struct lguest_device_desc *desc = to_lgdev(vdev)->desc;
@@ -136,20 +136,25 @@ static void lg_finalize_features(struct virtio_device *vdev)
136 /* Give virtio_ring a chance to accept features. */ 136 /* Give virtio_ring a chance to accept features. */
137 vring_transport_features(vdev); 137 vring_transport_features(vdev);
138 138
139 /* Make sure we don't have any features > 32 bits! */
140 BUG_ON((u32)vdev->features != vdev->features);
141
139 /* 142 /*
140 * The vdev->feature array is a Linux bitmask: this isn't the same as a 143 * Since lguest is currently x86-only, we're little-endian. That
141 * the simple array of bits used by lguest devices for features. So we 144 * means we could just memcpy. But it's not time critical, and in
142 * do this slow, manual conversion which is completely general. 145 * case someone copies this code, we do it the slow, obvious way.
143 */ 146 */
144 memset(out_features, 0, desc->feature_len); 147 memset(out_features, 0, desc->feature_len);
145 bits = min_t(unsigned, desc->feature_len, sizeof(vdev->features)) * 8; 148 bits = min_t(unsigned, desc->feature_len, sizeof(vdev->features)) * 8;
146 for (i = 0; i < bits; i++) { 149 for (i = 0; i < bits; i++) {
147 if (test_bit(i, vdev->features)) 150 if (__virtio_test_bit(vdev, i))
148 out_features[i / 8] |= (1 << (i % 8)); 151 out_features[i / 8] |= (1 << (i % 8));
149 } 152 }
150 153
151 /* Tell Host we've finished with this device's feature negotiation */ 154 /* Tell Host we've finished with this device's feature negotiation */
152 status_notify(vdev); 155 status_notify(vdev);
156
157 return 0;
153} 158}
154 159
155/* Once they've found a field, getting a copy of it is easy. */ 160/* Once they've found a field, getting a copy of it is easy. */
diff --git a/drivers/misc/mic/card/mic_virtio.c b/drivers/misc/mic/card/mic_virtio.c
index e64794730e21..e486a0c26267 100644
--- a/drivers/misc/mic/card/mic_virtio.c
+++ b/drivers/misc/mic/card/mic_virtio.c
@@ -68,7 +68,7 @@ static inline struct device *mic_dev(struct mic_vdev *mvdev)
68} 68}
69 69
70/* This gets the device's feature bits. */ 70/* This gets the device's feature bits. */
71static u32 mic_get_features(struct virtio_device *vdev) 71static u64 mic_get_features(struct virtio_device *vdev)
72{ 72{
73 unsigned int i, bits; 73 unsigned int i, bits;
74 u32 features = 0; 74 u32 features = 0;
@@ -76,8 +76,7 @@ static u32 mic_get_features(struct virtio_device *vdev)
76 u8 __iomem *in_features = mic_vq_features(desc); 76 u8 __iomem *in_features = mic_vq_features(desc);
77 int feature_len = ioread8(&desc->feature_len); 77 int feature_len = ioread8(&desc->feature_len);
78 78
79 bits = min_t(unsigned, feature_len, 79 bits = min_t(unsigned, feature_len, sizeof(features)) * 8;
80 sizeof(vdev->features)) * 8;
81 for (i = 0; i < bits; i++) 80 for (i = 0; i < bits; i++)
82 if (ioread8(&in_features[i / 8]) & (BIT(i % 8))) 81 if (ioread8(&in_features[i / 8]) & (BIT(i % 8)))
83 features |= BIT(i); 82 features |= BIT(i);
@@ -85,7 +84,7 @@ static u32 mic_get_features(struct virtio_device *vdev)
85 return features; 84 return features;
86} 85}
87 86
88static void mic_finalize_features(struct virtio_device *vdev) 87static int mic_finalize_features(struct virtio_device *vdev)
89{ 88{
90 unsigned int i, bits; 89 unsigned int i, bits;
91 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc; 90 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc;
@@ -97,14 +96,19 @@ static void mic_finalize_features(struct virtio_device *vdev)
97 /* Give virtio_ring a chance to accept features. */ 96 /* Give virtio_ring a chance to accept features. */
98 vring_transport_features(vdev); 97 vring_transport_features(vdev);
99 98
99 /* Make sure we don't have any features > 32 bits! */
100 BUG_ON((u32)vdev->features != vdev->features);
101
100 memset_io(out_features, 0, feature_len); 102 memset_io(out_features, 0, feature_len);
101 bits = min_t(unsigned, feature_len, 103 bits = min_t(unsigned, feature_len,
102 sizeof(vdev->features)) * 8; 104 sizeof(vdev->features)) * 8;
103 for (i = 0; i < bits; i++) { 105 for (i = 0; i < bits; i++) {
104 if (test_bit(i, vdev->features)) 106 if (__virtio_test_bit(vdev, i))
105 iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)), 107 iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)),
106 &out_features[i / 8]); 108 &out_features[i / 8]);
107 } 109 }
110
111 return 0;
108} 112}
109 113
110/* 114/*
diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c
index 880cc090dc44..af90ab5e5768 100644
--- a/drivers/net/macvtap.c
+++ b/drivers/net/macvtap.c
@@ -45,6 +45,18 @@ struct macvtap_queue {
45 struct list_head next; 45 struct list_head next;
46}; 46};
47 47
48#define MACVTAP_FEATURES (IFF_VNET_HDR | IFF_VNET_LE | IFF_MULTI_QUEUE)
49
50static inline u16 macvtap16_to_cpu(struct macvtap_queue *q, __virtio16 val)
51{
52 return __virtio16_to_cpu(q->flags & IFF_VNET_LE, val);
53}
54
55static inline __virtio16 cpu_to_macvtap16(struct macvtap_queue *q, u16 val)
56{
57 return __cpu_to_virtio16(q->flags & IFF_VNET_LE, val);
58}
59
48static struct proto macvtap_proto = { 60static struct proto macvtap_proto = {
49 .name = "macvtap", 61 .name = "macvtap",
50 .owner = THIS_MODULE, 62 .owner = THIS_MODULE,
@@ -557,7 +569,8 @@ static inline struct sk_buff *macvtap_alloc_skb(struct sock *sk, size_t prepad,
557 * macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should 569 * macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should
558 * be shared with the tun/tap driver. 570 * be shared with the tun/tap driver.
559 */ 571 */
560static int macvtap_skb_from_vnet_hdr(struct sk_buff *skb, 572static int macvtap_skb_from_vnet_hdr(struct macvtap_queue *q,
573 struct sk_buff *skb,
561 struct virtio_net_hdr *vnet_hdr) 574 struct virtio_net_hdr *vnet_hdr)
562{ 575{
563 unsigned short gso_type = 0; 576 unsigned short gso_type = 0;
@@ -588,13 +601,13 @@ static int macvtap_skb_from_vnet_hdr(struct sk_buff *skb,
588 } 601 }
589 602
590 if (vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 603 if (vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
591 if (!skb_partial_csum_set(skb, vnet_hdr->csum_start, 604 if (!skb_partial_csum_set(skb, macvtap16_to_cpu(q, vnet_hdr->csum_start),
592 vnet_hdr->csum_offset)) 605 macvtap16_to_cpu(q, vnet_hdr->csum_offset)))
593 return -EINVAL; 606 return -EINVAL;
594 } 607 }
595 608
596 if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 609 if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
597 skb_shinfo(skb)->gso_size = vnet_hdr->gso_size; 610 skb_shinfo(skb)->gso_size = macvtap16_to_cpu(q, vnet_hdr->gso_size);
598 skb_shinfo(skb)->gso_type = gso_type; 611 skb_shinfo(skb)->gso_type = gso_type;
599 612
600 /* Header must be checked, and gso_segs computed. */ 613 /* Header must be checked, and gso_segs computed. */
@@ -604,8 +617,9 @@ static int macvtap_skb_from_vnet_hdr(struct sk_buff *skb,
604 return 0; 617 return 0;
605} 618}
606 619
607static void macvtap_skb_to_vnet_hdr(const struct sk_buff *skb, 620static void macvtap_skb_to_vnet_hdr(struct macvtap_queue *q,
608 struct virtio_net_hdr *vnet_hdr) 621 const struct sk_buff *skb,
622 struct virtio_net_hdr *vnet_hdr)
609{ 623{
610 memset(vnet_hdr, 0, sizeof(*vnet_hdr)); 624 memset(vnet_hdr, 0, sizeof(*vnet_hdr));
611 625
@@ -613,8 +627,8 @@ static void macvtap_skb_to_vnet_hdr(const struct sk_buff *skb,
613 struct skb_shared_info *sinfo = skb_shinfo(skb); 627 struct skb_shared_info *sinfo = skb_shinfo(skb);
614 628
615 /* This is a hint as to how much should be linear. */ 629 /* This is a hint as to how much should be linear. */
616 vnet_hdr->hdr_len = skb_headlen(skb); 630 vnet_hdr->hdr_len = cpu_to_macvtap16(q, skb_headlen(skb));
617 vnet_hdr->gso_size = sinfo->gso_size; 631 vnet_hdr->gso_size = cpu_to_macvtap16(q, sinfo->gso_size);
618 if (sinfo->gso_type & SKB_GSO_TCPV4) 632 if (sinfo->gso_type & SKB_GSO_TCPV4)
619 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 633 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
620 else if (sinfo->gso_type & SKB_GSO_TCPV6) 634 else if (sinfo->gso_type & SKB_GSO_TCPV6)
@@ -628,10 +642,13 @@ static void macvtap_skb_to_vnet_hdr(const struct sk_buff *skb,
628 642
629 if (skb->ip_summed == CHECKSUM_PARTIAL) { 643 if (skb->ip_summed == CHECKSUM_PARTIAL) {
630 vnet_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 644 vnet_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
631 vnet_hdr->csum_start = skb_checksum_start_offset(skb);
632 if (vlan_tx_tag_present(skb)) 645 if (vlan_tx_tag_present(skb))
633 vnet_hdr->csum_start += VLAN_HLEN; 646 vnet_hdr->csum_start = cpu_to_macvtap16(q,
634 vnet_hdr->csum_offset = skb->csum_offset; 647 skb_checksum_start_offset(skb) + VLAN_HLEN);
648 else
649 vnet_hdr->csum_start = cpu_to_macvtap16(q,
650 skb_checksum_start_offset(skb));
651 vnet_hdr->csum_offset = cpu_to_macvtap16(q, skb->csum_offset);
635 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 652 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
636 vnet_hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID; 653 vnet_hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID;
637 } /* else everything is zero */ 654 } /* else everything is zero */
@@ -666,12 +683,14 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
666 if (err < 0) 683 if (err < 0)
667 goto err; 684 goto err;
668 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 685 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
669 vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 > 686 macvtap16_to_cpu(q, vnet_hdr.csum_start) +
670 vnet_hdr.hdr_len) 687 macvtap16_to_cpu(q, vnet_hdr.csum_offset) + 2 >
671 vnet_hdr.hdr_len = vnet_hdr.csum_start + 688 macvtap16_to_cpu(q, vnet_hdr.hdr_len))
672 vnet_hdr.csum_offset + 2; 689 vnet_hdr.hdr_len = cpu_to_macvtap16(q,
690 macvtap16_to_cpu(q, vnet_hdr.csum_start) +
691 macvtap16_to_cpu(q, vnet_hdr.csum_offset) + 2);
673 err = -EINVAL; 692 err = -EINVAL;
674 if (vnet_hdr.hdr_len > len) 693 if (macvtap16_to_cpu(q, vnet_hdr.hdr_len) > len)
675 goto err; 694 goto err;
676 } 695 }
677 696
@@ -684,7 +703,8 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
684 goto err; 703 goto err;
685 704
686 if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) { 705 if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) {
687 copylen = vnet_hdr.hdr_len ? vnet_hdr.hdr_len : GOODCOPY_LEN; 706 copylen = vnet_hdr.hdr_len ?
707 macvtap16_to_cpu(q, vnet_hdr.hdr_len) : GOODCOPY_LEN;
688 if (copylen > good_linear) 708 if (copylen > good_linear)
689 copylen = good_linear; 709 copylen = good_linear;
690 linear = copylen; 710 linear = copylen;
@@ -695,10 +715,10 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
695 715
696 if (!zerocopy) { 716 if (!zerocopy) {
697 copylen = len; 717 copylen = len;
698 if (vnet_hdr.hdr_len > good_linear) 718 if (macvtap16_to_cpu(q, vnet_hdr.hdr_len) > good_linear)
699 linear = good_linear; 719 linear = good_linear;
700 else 720 else
701 linear = vnet_hdr.hdr_len; 721 linear = macvtap16_to_cpu(q, vnet_hdr.hdr_len);
702 } 722 }
703 723
704 skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, copylen, 724 skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, copylen,
@@ -725,7 +745,7 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
725 skb->protocol = eth_hdr(skb)->h_proto; 745 skb->protocol = eth_hdr(skb)->h_proto;
726 746
727 if (vnet_hdr_len) { 747 if (vnet_hdr_len) {
728 err = macvtap_skb_from_vnet_hdr(skb, &vnet_hdr); 748 err = macvtap_skb_from_vnet_hdr(q, skb, &vnet_hdr);
729 if (err) 749 if (err)
730 goto err_kfree; 750 goto err_kfree;
731 } 751 }
@@ -791,7 +811,7 @@ static ssize_t macvtap_put_user(struct macvtap_queue *q,
791 if ((len -= vnet_hdr_len) < 0) 811 if ((len -= vnet_hdr_len) < 0)
792 return -EINVAL; 812 return -EINVAL;
793 813
794 macvtap_skb_to_vnet_hdr(skb, &vnet_hdr); 814 macvtap_skb_to_vnet_hdr(q, skb, &vnet_hdr);
795 815
796 if (memcpy_toiovecend(iv, (void *)&vnet_hdr, 0, sizeof(vnet_hdr))) 816 if (memcpy_toiovecend(iv, (void *)&vnet_hdr, 0, sizeof(vnet_hdr)))
797 return -EFAULT; 817 return -EFAULT;
@@ -1003,8 +1023,7 @@ static long macvtap_ioctl(struct file *file, unsigned int cmd,
1003 return -EFAULT; 1023 return -EFAULT;
1004 1024
1005 ret = 0; 1025 ret = 0;
1006 if ((u & ~(IFF_VNET_HDR | IFF_MULTI_QUEUE)) != 1026 if ((u & ~MACVTAP_FEATURES) != (IFF_NO_PI | IFF_TAP))
1007 (IFF_NO_PI | IFF_TAP))
1008 ret = -EINVAL; 1027 ret = -EINVAL;
1009 else 1028 else
1010 q->flags = u; 1029 q->flags = u;
@@ -1036,8 +1055,7 @@ static long macvtap_ioctl(struct file *file, unsigned int cmd,
1036 return ret; 1055 return ret;
1037 1056
1038 case TUNGETFEATURES: 1057 case TUNGETFEATURES:
1039 if (put_user(IFF_TAP | IFF_NO_PI | IFF_VNET_HDR | 1058 if (put_user(IFF_TAP | IFF_NO_PI | MACVTAP_FEATURES, up))
1040 IFF_MULTI_QUEUE, up))
1041 return -EFAULT; 1059 return -EFAULT;
1042 return 0; 1060 return 0;
1043 1061
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index 4d332dc93b70..798ce70e3d61 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -103,6 +103,15 @@ do { \
103} while (0) 103} while (0)
104#endif 104#endif
105 105
106/* TUN device flags */
107
108/* IFF_ATTACH_QUEUE is never stored in device flags,
109 * overload it to mean fasync when stored there.
110 */
111#define TUN_FASYNC IFF_ATTACH_QUEUE
112
113#define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \
114 IFF_VNET_LE | IFF_MULTI_QUEUE)
106#define GOODCOPY_LEN 128 115#define GOODCOPY_LEN 128
107 116
108#define FLT_EXACT_COUNT 8 117#define FLT_EXACT_COUNT 8
@@ -196,6 +205,16 @@ struct tun_struct {
196 u32 flow_count; 205 u32 flow_count;
197}; 206};
198 207
208static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val)
209{
210 return __virtio16_to_cpu(tun->flags & IFF_VNET_LE, val);
211}
212
213static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val)
214{
215 return __cpu_to_virtio16(tun->flags & IFF_VNET_LE, val);
216}
217
199static inline u32 tun_hashfn(u32 rxhash) 218static inline u32 tun_hashfn(u32 rxhash)
200{ 219{
201 return rxhash & 0x3ff; 220 return rxhash & 0x3ff;
@@ -472,7 +491,7 @@ static void __tun_detach(struct tun_file *tfile, bool clean)
472 if (tun && tun->numqueues == 0 && tun->numdisabled == 0) { 491 if (tun && tun->numqueues == 0 && tun->numdisabled == 0) {
473 netif_carrier_off(tun->dev); 492 netif_carrier_off(tun->dev);
474 493
475 if (!(tun->flags & TUN_PERSIST) && 494 if (!(tun->flags & IFF_PERSIST) &&
476 tun->dev->reg_state == NETREG_REGISTERED) 495 tun->dev->reg_state == NETREG_REGISTERED)
477 unregister_netdevice(tun->dev); 496 unregister_netdevice(tun->dev);
478 } 497 }
@@ -523,7 +542,7 @@ static void tun_detach_all(struct net_device *dev)
523 } 542 }
524 BUG_ON(tun->numdisabled != 0); 543 BUG_ON(tun->numdisabled != 0);
525 544
526 if (tun->flags & TUN_PERSIST) 545 if (tun->flags & IFF_PERSIST)
527 module_put(THIS_MODULE); 546 module_put(THIS_MODULE);
528} 547}
529 548
@@ -541,7 +560,7 @@ static int tun_attach(struct tun_struct *tun, struct file *file, bool skip_filte
541 goto out; 560 goto out;
542 561
543 err = -EBUSY; 562 err = -EBUSY;
544 if (!(tun->flags & TUN_TAP_MQ) && tun->numqueues == 1) 563 if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1)
545 goto out; 564 goto out;
546 565
547 err = -E2BIG; 566 err = -E2BIG;
@@ -920,7 +939,7 @@ static void tun_net_init(struct net_device *dev)
920 struct tun_struct *tun = netdev_priv(dev); 939 struct tun_struct *tun = netdev_priv(dev);
921 940
922 switch (tun->flags & TUN_TYPE_MASK) { 941 switch (tun->flags & TUN_TYPE_MASK) {
923 case TUN_TUN_DEV: 942 case IFF_TUN:
924 dev->netdev_ops = &tun_netdev_ops; 943 dev->netdev_ops = &tun_netdev_ops;
925 944
926 /* Point-to-Point TUN Device */ 945 /* Point-to-Point TUN Device */
@@ -934,7 +953,7 @@ static void tun_net_init(struct net_device *dev)
934 dev->tx_queue_len = TUN_READQ_SIZE; /* We prefer our own queue length */ 953 dev->tx_queue_len = TUN_READQ_SIZE; /* We prefer our own queue length */
935 break; 954 break;
936 955
937 case TUN_TAP_DEV: 956 case IFF_TAP:
938 dev->netdev_ops = &tap_netdev_ops; 957 dev->netdev_ops = &tap_netdev_ops;
939 /* Ethernet TAP Device */ 958 /* Ethernet TAP Device */
940 ether_setup(dev); 959 ether_setup(dev);
@@ -1025,7 +1044,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1025 int err; 1044 int err;
1026 u32 rxhash; 1045 u32 rxhash;
1027 1046
1028 if (!(tun->flags & TUN_NO_PI)) { 1047 if (!(tun->flags & IFF_NO_PI)) {
1029 if (len < sizeof(pi)) 1048 if (len < sizeof(pi))
1030 return -EINVAL; 1049 return -EINVAL;
1031 len -= sizeof(pi); 1050 len -= sizeof(pi);
@@ -1035,7 +1054,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1035 offset += sizeof(pi); 1054 offset += sizeof(pi);
1036 } 1055 }
1037 1056
1038 if (tun->flags & TUN_VNET_HDR) { 1057 if (tun->flags & IFF_VNET_HDR) {
1039 if (len < tun->vnet_hdr_sz) 1058 if (len < tun->vnet_hdr_sz)
1040 return -EINVAL; 1059 return -EINVAL;
1041 len -= tun->vnet_hdr_sz; 1060 len -= tun->vnet_hdr_sz;
@@ -1044,18 +1063,18 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1044 return -EFAULT; 1063 return -EFAULT;
1045 1064
1046 if ((gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 1065 if ((gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
1047 gso.csum_start + gso.csum_offset + 2 > gso.hdr_len) 1066 tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2 > tun16_to_cpu(tun, gso.hdr_len))
1048 gso.hdr_len = gso.csum_start + gso.csum_offset + 2; 1067 gso.hdr_len = cpu_to_tun16(tun, tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2);
1049 1068
1050 if (gso.hdr_len > len) 1069 if (tun16_to_cpu(tun, gso.hdr_len) > len)
1051 return -EINVAL; 1070 return -EINVAL;
1052 offset += tun->vnet_hdr_sz; 1071 offset += tun->vnet_hdr_sz;
1053 } 1072 }
1054 1073
1055 if ((tun->flags & TUN_TYPE_MASK) == TUN_TAP_DEV) { 1074 if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) {
1056 align += NET_IP_ALIGN; 1075 align += NET_IP_ALIGN;
1057 if (unlikely(len < ETH_HLEN || 1076 if (unlikely(len < ETH_HLEN ||
1058 (gso.hdr_len && gso.hdr_len < ETH_HLEN))) 1077 (gso.hdr_len && tun16_to_cpu(tun, gso.hdr_len) < ETH_HLEN)))
1059 return -EINVAL; 1078 return -EINVAL;
1060 } 1079 }
1061 1080
@@ -1066,7 +1085,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1066 * enough room for skb expand head in case it is used. 1085 * enough room for skb expand head in case it is used.
1067 * The rest of the buffer is mapped from userspace. 1086 * The rest of the buffer is mapped from userspace.
1068 */ 1087 */
1069 copylen = gso.hdr_len ? gso.hdr_len : GOODCOPY_LEN; 1088 copylen = gso.hdr_len ? tun16_to_cpu(tun, gso.hdr_len) : GOODCOPY_LEN;
1070 if (copylen > good_linear) 1089 if (copylen > good_linear)
1071 copylen = good_linear; 1090 copylen = good_linear;
1072 linear = copylen; 1091 linear = copylen;
@@ -1076,10 +1095,10 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1076 1095
1077 if (!zerocopy) { 1096 if (!zerocopy) {
1078 copylen = len; 1097 copylen = len;
1079 if (gso.hdr_len > good_linear) 1098 if (tun16_to_cpu(tun, gso.hdr_len) > good_linear)
1080 linear = good_linear; 1099 linear = good_linear;
1081 else 1100 else
1082 linear = gso.hdr_len; 1101 linear = tun16_to_cpu(tun, gso.hdr_len);
1083 } 1102 }
1084 1103
1085 skb = tun_alloc_skb(tfile, align, copylen, linear, noblock); 1104 skb = tun_alloc_skb(tfile, align, copylen, linear, noblock);
@@ -1106,8 +1125,8 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1106 } 1125 }
1107 1126
1108 if (gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 1127 if (gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1109 if (!skb_partial_csum_set(skb, gso.csum_start, 1128 if (!skb_partial_csum_set(skb, tun16_to_cpu(tun, gso.csum_start),
1110 gso.csum_offset)) { 1129 tun16_to_cpu(tun, gso.csum_offset))) {
1111 tun->dev->stats.rx_frame_errors++; 1130 tun->dev->stats.rx_frame_errors++;
1112 kfree_skb(skb); 1131 kfree_skb(skb);
1113 return -EINVAL; 1132 return -EINVAL;
@@ -1115,8 +1134,8 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1115 } 1134 }
1116 1135
1117 switch (tun->flags & TUN_TYPE_MASK) { 1136 switch (tun->flags & TUN_TYPE_MASK) {
1118 case TUN_TUN_DEV: 1137 case IFF_TUN:
1119 if (tun->flags & TUN_NO_PI) { 1138 if (tun->flags & IFF_NO_PI) {
1120 switch (skb->data[0] & 0xf0) { 1139 switch (skb->data[0] & 0xf0) {
1121 case 0x40: 1140 case 0x40:
1122 pi.proto = htons(ETH_P_IP); 1141 pi.proto = htons(ETH_P_IP);
@@ -1135,7 +1154,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1135 skb->protocol = pi.proto; 1154 skb->protocol = pi.proto;
1136 skb->dev = tun->dev; 1155 skb->dev = tun->dev;
1137 break; 1156 break;
1138 case TUN_TAP_DEV: 1157 case IFF_TAP:
1139 skb->protocol = eth_type_trans(skb, tun->dev); 1158 skb->protocol = eth_type_trans(skb, tun->dev);
1140 break; 1159 break;
1141 } 1160 }
@@ -1175,7 +1194,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1175 if (gso.gso_type & VIRTIO_NET_HDR_GSO_ECN) 1194 if (gso.gso_type & VIRTIO_NET_HDR_GSO_ECN)
1176 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; 1195 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
1177 1196
1178 skb_shinfo(skb)->gso_size = gso.gso_size; 1197 skb_shinfo(skb)->gso_size = tun16_to_cpu(tun, gso.gso_size);
1179 if (skb_shinfo(skb)->gso_size == 0) { 1198 if (skb_shinfo(skb)->gso_size == 0) {
1180 tun->dev->stats.rx_frame_errors++; 1199 tun->dev->stats.rx_frame_errors++;
1181 kfree_skb(skb); 1200 kfree_skb(skb);
@@ -1241,10 +1260,10 @@ static ssize_t tun_put_user(struct tun_struct *tun,
1241 if (vlan_tx_tag_present(skb)) 1260 if (vlan_tx_tag_present(skb))
1242 vlan_hlen = VLAN_HLEN; 1261 vlan_hlen = VLAN_HLEN;
1243 1262
1244 if (tun->flags & TUN_VNET_HDR) 1263 if (tun->flags & IFF_VNET_HDR)
1245 vnet_hdr_sz = tun->vnet_hdr_sz; 1264 vnet_hdr_sz = tun->vnet_hdr_sz;
1246 1265
1247 if (!(tun->flags & TUN_NO_PI)) { 1266 if (!(tun->flags & IFF_NO_PI)) {
1248 if ((len -= sizeof(pi)) < 0) 1267 if ((len -= sizeof(pi)) < 0)
1249 return -EINVAL; 1268 return -EINVAL;
1250 1269
@@ -1267,8 +1286,8 @@ static ssize_t tun_put_user(struct tun_struct *tun,
1267 struct skb_shared_info *sinfo = skb_shinfo(skb); 1286 struct skb_shared_info *sinfo = skb_shinfo(skb);
1268 1287
1269 /* This is a hint as to how much should be linear. */ 1288 /* This is a hint as to how much should be linear. */
1270 gso.hdr_len = skb_headlen(skb); 1289 gso.hdr_len = cpu_to_tun16(tun, skb_headlen(skb));
1271 gso.gso_size = sinfo->gso_size; 1290 gso.gso_size = cpu_to_tun16(tun, sinfo->gso_size);
1272 if (sinfo->gso_type & SKB_GSO_TCPV4) 1291 if (sinfo->gso_type & SKB_GSO_TCPV4)
1273 gso.gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 1292 gso.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1274 else if (sinfo->gso_type & SKB_GSO_TCPV6) 1293 else if (sinfo->gso_type & SKB_GSO_TCPV6)
@@ -1276,12 +1295,12 @@ static ssize_t tun_put_user(struct tun_struct *tun,
1276 else { 1295 else {
1277 pr_err("unexpected GSO type: " 1296 pr_err("unexpected GSO type: "
1278 "0x%x, gso_size %d, hdr_len %d\n", 1297 "0x%x, gso_size %d, hdr_len %d\n",
1279 sinfo->gso_type, gso.gso_size, 1298 sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size),
1280 gso.hdr_len); 1299 tun16_to_cpu(tun, gso.hdr_len));
1281 print_hex_dump(KERN_ERR, "tun: ", 1300 print_hex_dump(KERN_ERR, "tun: ",
1282 DUMP_PREFIX_NONE, 1301 DUMP_PREFIX_NONE,
1283 16, 1, skb->head, 1302 16, 1, skb->head,
1284 min((int)gso.hdr_len, 64), true); 1303 min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true);
1285 WARN_ON_ONCE(1); 1304 WARN_ON_ONCE(1);
1286 return -EINVAL; 1305 return -EINVAL;
1287 } 1306 }
@@ -1292,9 +1311,9 @@ static ssize_t tun_put_user(struct tun_struct *tun,
1292 1311
1293 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1312 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1294 gso.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 1313 gso.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
1295 gso.csum_start = skb_checksum_start_offset(skb) + 1314 gso.csum_start = cpu_to_tun16(tun, skb_checksum_start_offset(skb) +
1296 vlan_hlen; 1315 vlan_hlen);
1297 gso.csum_offset = skb->csum_offset; 1316 gso.csum_offset = cpu_to_tun16(tun, skb->csum_offset);
1298 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 1317 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
1299 gso.flags = VIRTIO_NET_HDR_F_DATA_VALID; 1318 gso.flags = VIRTIO_NET_HDR_F_DATA_VALID;
1300 } /* else everything is zero */ 1319 } /* else everything is zero */
@@ -1521,32 +1540,7 @@ static struct proto tun_proto = {
1521 1540
1522static int tun_flags(struct tun_struct *tun) 1541static int tun_flags(struct tun_struct *tun)
1523{ 1542{
1524 int flags = 0; 1543 return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP);
1525
1526 if (tun->flags & TUN_TUN_DEV)
1527 flags |= IFF_TUN;
1528 else
1529 flags |= IFF_TAP;
1530
1531 if (tun->flags & TUN_NO_PI)
1532 flags |= IFF_NO_PI;
1533
1534 /* This flag has no real effect. We track the value for backwards
1535 * compatibility.
1536 */
1537 if (tun->flags & TUN_ONE_QUEUE)
1538 flags |= IFF_ONE_QUEUE;
1539
1540 if (tun->flags & TUN_VNET_HDR)
1541 flags |= IFF_VNET_HDR;
1542
1543 if (tun->flags & TUN_TAP_MQ)
1544 flags |= IFF_MULTI_QUEUE;
1545
1546 if (tun->flags & TUN_PERSIST)
1547 flags |= IFF_PERSIST;
1548
1549 return flags;
1550} 1544}
1551 1545
1552static ssize_t tun_show_flags(struct device *dev, struct device_attribute *attr, 1546static ssize_t tun_show_flags(struct device *dev, struct device_attribute *attr,
@@ -1602,7 +1596,7 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
1602 return -EINVAL; 1596 return -EINVAL;
1603 1597
1604 if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) != 1598 if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) !=
1605 !!(tun->flags & TUN_TAP_MQ)) 1599 !!(tun->flags & IFF_MULTI_QUEUE))
1606 return -EINVAL; 1600 return -EINVAL;
1607 1601
1608 if (tun_not_capable(tun)) 1602 if (tun_not_capable(tun))
@@ -1615,7 +1609,7 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
1615 if (err < 0) 1609 if (err < 0)
1616 return err; 1610 return err;
1617 1611
1618 if (tun->flags & TUN_TAP_MQ && 1612 if (tun->flags & IFF_MULTI_QUEUE &&
1619 (tun->numqueues + tun->numdisabled > 1)) { 1613 (tun->numqueues + tun->numdisabled > 1)) {
1620 /* One or more queue has already been attached, no need 1614 /* One or more queue has already been attached, no need
1621 * to initialize the device again. 1615 * to initialize the device again.
@@ -1638,11 +1632,11 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
1638 /* Set dev type */ 1632 /* Set dev type */
1639 if (ifr->ifr_flags & IFF_TUN) { 1633 if (ifr->ifr_flags & IFF_TUN) {
1640 /* TUN device */ 1634 /* TUN device */
1641 flags |= TUN_TUN_DEV; 1635 flags |= IFF_TUN;
1642 name = "tun%d"; 1636 name = "tun%d";
1643 } else if (ifr->ifr_flags & IFF_TAP) { 1637 } else if (ifr->ifr_flags & IFF_TAP) {
1644 /* TAP device */ 1638 /* TAP device */
1645 flags |= TUN_TAP_DEV; 1639 flags |= IFF_TAP;
1646 name = "tap%d"; 1640 name = "tap%d";
1647 } else 1641 } else
1648 return -EINVAL; 1642 return -EINVAL;
@@ -1706,28 +1700,8 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
1706 1700
1707 tun_debug(KERN_INFO, tun, "tun_set_iff\n"); 1701 tun_debug(KERN_INFO, tun, "tun_set_iff\n");
1708 1702
1709 if (ifr->ifr_flags & IFF_NO_PI) 1703 tun->flags = (tun->flags & ~TUN_FEATURES) |
1710 tun->flags |= TUN_NO_PI; 1704 (ifr->ifr_flags & TUN_FEATURES);
1711 else
1712 tun->flags &= ~TUN_NO_PI;
1713
1714 /* This flag has no real effect. We track the value for backwards
1715 * compatibility.
1716 */
1717 if (ifr->ifr_flags & IFF_ONE_QUEUE)
1718 tun->flags |= TUN_ONE_QUEUE;
1719 else
1720 tun->flags &= ~TUN_ONE_QUEUE;
1721
1722 if (ifr->ifr_flags & IFF_VNET_HDR)
1723 tun->flags |= TUN_VNET_HDR;
1724 else
1725 tun->flags &= ~TUN_VNET_HDR;
1726
1727 if (ifr->ifr_flags & IFF_MULTI_QUEUE)
1728 tun->flags |= TUN_TAP_MQ;
1729 else
1730 tun->flags &= ~TUN_TAP_MQ;
1731 1705
1732 /* Make sure persistent devices do not get stuck in 1706 /* Make sure persistent devices do not get stuck in
1733 * xoff state. 1707 * xoff state.
@@ -1855,7 +1829,7 @@ static int tun_set_queue(struct file *file, struct ifreq *ifr)
1855 ret = tun_attach(tun, file, false); 1829 ret = tun_attach(tun, file, false);
1856 } else if (ifr->ifr_flags & IFF_DETACH_QUEUE) { 1830 } else if (ifr->ifr_flags & IFF_DETACH_QUEUE) {
1857 tun = rtnl_dereference(tfile->tun); 1831 tun = rtnl_dereference(tfile->tun);
1858 if (!tun || !(tun->flags & TUN_TAP_MQ) || tfile->detached) 1832 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)
1859 ret = -EINVAL; 1833 ret = -EINVAL;
1860 else 1834 else
1861 __tun_detach(tfile, false); 1835 __tun_detach(tfile, false);
@@ -1890,9 +1864,9 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
1890 if (cmd == TUNGETFEATURES) { 1864 if (cmd == TUNGETFEATURES) {
1891 /* Currently this just means: "what IFF flags are valid?". 1865 /* Currently this just means: "what IFF flags are valid?".
1892 * This is needed because we never checked for invalid flags on 1866 * This is needed because we never checked for invalid flags on
1893 * TUNSETIFF. */ 1867 * TUNSETIFF.
1894 return put_user(IFF_TUN | IFF_TAP | IFF_NO_PI | IFF_ONE_QUEUE | 1868 */
1895 IFF_VNET_HDR | IFF_MULTI_QUEUE, 1869 return put_user(IFF_TUN | IFF_TAP | TUN_FEATURES,
1896 (unsigned int __user*)argp); 1870 (unsigned int __user*)argp);
1897 } else if (cmd == TUNSETQUEUE) 1871 } else if (cmd == TUNSETQUEUE)
1898 return tun_set_queue(file, &ifr); 1872 return tun_set_queue(file, &ifr);
@@ -1959,12 +1933,12 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
1959 /* Disable/Enable persist mode. Keep an extra reference to the 1933 /* Disable/Enable persist mode. Keep an extra reference to the
1960 * module to prevent the module being unprobed. 1934 * module to prevent the module being unprobed.
1961 */ 1935 */
1962 if (arg && !(tun->flags & TUN_PERSIST)) { 1936 if (arg && !(tun->flags & IFF_PERSIST)) {
1963 tun->flags |= TUN_PERSIST; 1937 tun->flags |= IFF_PERSIST;
1964 __module_get(THIS_MODULE); 1938 __module_get(THIS_MODULE);
1965 } 1939 }
1966 if (!arg && (tun->flags & TUN_PERSIST)) { 1940 if (!arg && (tun->flags & IFF_PERSIST)) {
1967 tun->flags &= ~TUN_PERSIST; 1941 tun->flags &= ~IFF_PERSIST;
1968 module_put(THIS_MODULE); 1942 module_put(THIS_MODULE);
1969 } 1943 }
1970 1944
@@ -2022,7 +1996,7 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
2022 case TUNSETTXFILTER: 1996 case TUNSETTXFILTER:
2023 /* Can be set only for TAPs */ 1997 /* Can be set only for TAPs */
2024 ret = -EINVAL; 1998 ret = -EINVAL;
2025 if ((tun->flags & TUN_TYPE_MASK) != TUN_TAP_DEV) 1999 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
2026 break; 2000 break;
2027 ret = update_filter(&tun->txflt, (void __user *)arg); 2001 ret = update_filter(&tun->txflt, (void __user *)arg);
2028 break; 2002 break;
@@ -2081,7 +2055,7 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
2081 case TUNATTACHFILTER: 2055 case TUNATTACHFILTER:
2082 /* Can be set only for TAPs */ 2056 /* Can be set only for TAPs */
2083 ret = -EINVAL; 2057 ret = -EINVAL;
2084 if ((tun->flags & TUN_TYPE_MASK) != TUN_TAP_DEV) 2058 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
2085 break; 2059 break;
2086 ret = -EFAULT; 2060 ret = -EFAULT;
2087 if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog))) 2061 if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog)))
@@ -2093,7 +2067,7 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
2093 case TUNDETACHFILTER: 2067 case TUNDETACHFILTER:
2094 /* Can be set only for TAPs */ 2068 /* Can be set only for TAPs */
2095 ret = -EINVAL; 2069 ret = -EINVAL;
2096 if ((tun->flags & TUN_TYPE_MASK) != TUN_TAP_DEV) 2070 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
2097 break; 2071 break;
2098 ret = 0; 2072 ret = 0;
2099 tun_detach_filter(tun, tun->numqueues); 2073 tun_detach_filter(tun, tun->numqueues);
@@ -2101,7 +2075,7 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
2101 2075
2102 case TUNGETFILTER: 2076 case TUNGETFILTER:
2103 ret = -EINVAL; 2077 ret = -EINVAL;
2104 if ((tun->flags & TUN_TYPE_MASK) != TUN_TAP_DEV) 2078 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
2105 break; 2079 break;
2106 ret = -EFAULT; 2080 ret = -EFAULT;
2107 if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog))) 2081 if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog)))
@@ -2294,10 +2268,10 @@ static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info
2294 strlcpy(info->version, DRV_VERSION, sizeof(info->version)); 2268 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
2295 2269
2296 switch (tun->flags & TUN_TYPE_MASK) { 2270 switch (tun->flags & TUN_TYPE_MASK) {
2297 case TUN_TUN_DEV: 2271 case IFF_TUN:
2298 strlcpy(info->bus_info, "tun", sizeof(info->bus_info)); 2272 strlcpy(info->bus_info, "tun", sizeof(info->bus_info));
2299 break; 2273 break;
2300 case TUN_TAP_DEV: 2274 case IFF_TAP:
2301 strlcpy(info->bus_info, "tap", sizeof(info->bus_info)); 2275 strlcpy(info->bus_info, "tap", sizeof(info->bus_info));
2302 break; 2276 break;
2303 } 2277 }
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index b0bc8ead47de..b8bd7191572d 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -123,6 +123,9 @@ struct virtnet_info {
123 /* Host can handle any s/g split between our header and packet data */ 123 /* Host can handle any s/g split between our header and packet data */
124 bool any_header_sg; 124 bool any_header_sg;
125 125
126 /* Packet virtio header size */
127 u8 hdr_len;
128
126 /* Active statistics */ 129 /* Active statistics */
127 struct virtnet_stats __percpu *stats; 130 struct virtnet_stats __percpu *stats;
128 131
@@ -139,21 +142,14 @@ struct virtnet_info {
139 struct notifier_block nb; 142 struct notifier_block nb;
140}; 143};
141 144
142struct skb_vnet_hdr {
143 union {
144 struct virtio_net_hdr hdr;
145 struct virtio_net_hdr_mrg_rxbuf mhdr;
146 };
147};
148
149struct padded_vnet_hdr { 145struct padded_vnet_hdr {
150 struct virtio_net_hdr hdr; 146 struct virtio_net_hdr_mrg_rxbuf hdr;
151 /* 147 /*
152 * virtio_net_hdr should be in a separated sg buffer because of a 148 * hdr is in a separate sg buffer, and data sg buffer shares same page
153 * QEMU bug, and data sg buffer shares same page with this header sg. 149 * with this header sg. This padding makes next sg 16 byte aligned
154 * This padding makes next sg 16 byte aligned after virtio_net_hdr. 150 * after the header.
155 */ 151 */
156 char padding[6]; 152 char padding[4];
157}; 153};
158 154
159/* Converting between virtqueue no. and kernel tx/rx queue no. 155/* Converting between virtqueue no. and kernel tx/rx queue no.
@@ -179,9 +175,9 @@ static int rxq2vq(int rxq)
179 return rxq * 2; 175 return rxq * 2;
180} 176}
181 177
182static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb) 178static inline struct virtio_net_hdr_mrg_rxbuf *skb_vnet_hdr(struct sk_buff *skb)
183{ 179{
184 return (struct skb_vnet_hdr *)skb->cb; 180 return (struct virtio_net_hdr_mrg_rxbuf *)skb->cb;
185} 181}
186 182
187/* 183/*
@@ -241,13 +237,13 @@ static unsigned long mergeable_buf_to_ctx(void *buf, unsigned int truesize)
241} 237}
242 238
243/* Called from bottom half context */ 239/* Called from bottom half context */
244static struct sk_buff *page_to_skb(struct receive_queue *rq, 240static struct sk_buff *page_to_skb(struct virtnet_info *vi,
241 struct receive_queue *rq,
245 struct page *page, unsigned int offset, 242 struct page *page, unsigned int offset,
246 unsigned int len, unsigned int truesize) 243 unsigned int len, unsigned int truesize)
247{ 244{
248 struct virtnet_info *vi = rq->vq->vdev->priv;
249 struct sk_buff *skb; 245 struct sk_buff *skb;
250 struct skb_vnet_hdr *hdr; 246 struct virtio_net_hdr_mrg_rxbuf *hdr;
251 unsigned int copy, hdr_len, hdr_padded_len; 247 unsigned int copy, hdr_len, hdr_padded_len;
252 char *p; 248 char *p;
253 249
@@ -260,13 +256,11 @@ static struct sk_buff *page_to_skb(struct receive_queue *rq,
260 256
261 hdr = skb_vnet_hdr(skb); 257 hdr = skb_vnet_hdr(skb);
262 258
263 if (vi->mergeable_rx_bufs) { 259 hdr_len = vi->hdr_len;
264 hdr_len = sizeof hdr->mhdr; 260 if (vi->mergeable_rx_bufs)
265 hdr_padded_len = sizeof hdr->mhdr; 261 hdr_padded_len = sizeof *hdr;
266 } else { 262 else
267 hdr_len = sizeof hdr->hdr;
268 hdr_padded_len = sizeof(struct padded_vnet_hdr); 263 hdr_padded_len = sizeof(struct padded_vnet_hdr);
269 }
270 264
271 memcpy(hdr, p, hdr_len); 265 memcpy(hdr, p, hdr_len);
272 266
@@ -317,23 +311,24 @@ static struct sk_buff *page_to_skb(struct receive_queue *rq,
317 return skb; 311 return skb;
318} 312}
319 313
320static struct sk_buff *receive_small(void *buf, unsigned int len) 314static struct sk_buff *receive_small(struct virtnet_info *vi, void *buf, unsigned int len)
321{ 315{
322 struct sk_buff * skb = buf; 316 struct sk_buff * skb = buf;
323 317
324 len -= sizeof(struct virtio_net_hdr); 318 len -= vi->hdr_len;
325 skb_trim(skb, len); 319 skb_trim(skb, len);
326 320
327 return skb; 321 return skb;
328} 322}
329 323
330static struct sk_buff *receive_big(struct net_device *dev, 324static struct sk_buff *receive_big(struct net_device *dev,
325 struct virtnet_info *vi,
331 struct receive_queue *rq, 326 struct receive_queue *rq,
332 void *buf, 327 void *buf,
333 unsigned int len) 328 unsigned int len)
334{ 329{
335 struct page *page = buf; 330 struct page *page = buf;
336 struct sk_buff *skb = page_to_skb(rq, page, 0, len, PAGE_SIZE); 331 struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE);
337 332
338 if (unlikely(!skb)) 333 if (unlikely(!skb))
339 goto err; 334 goto err;
@@ -347,18 +342,20 @@ err:
347} 342}
348 343
349static struct sk_buff *receive_mergeable(struct net_device *dev, 344static struct sk_buff *receive_mergeable(struct net_device *dev,
345 struct virtnet_info *vi,
350 struct receive_queue *rq, 346 struct receive_queue *rq,
351 unsigned long ctx, 347 unsigned long ctx,
352 unsigned int len) 348 unsigned int len)
353{ 349{
354 void *buf = mergeable_ctx_to_buf_address(ctx); 350 void *buf = mergeable_ctx_to_buf_address(ctx);
355 struct skb_vnet_hdr *hdr = buf; 351 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
356 int num_buf = hdr->mhdr.num_buffers; 352 u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
357 struct page *page = virt_to_head_page(buf); 353 struct page *page = virt_to_head_page(buf);
358 int offset = buf - page_address(page); 354 int offset = buf - page_address(page);
359 unsigned int truesize = max(len, mergeable_ctx_to_buf_truesize(ctx)); 355 unsigned int truesize = max(len, mergeable_ctx_to_buf_truesize(ctx));
360 356
361 struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize); 357 struct sk_buff *head_skb = page_to_skb(vi, rq, page, offset, len,
358 truesize);
362 struct sk_buff *curr_skb = head_skb; 359 struct sk_buff *curr_skb = head_skb;
363 360
364 if (unlikely(!curr_skb)) 361 if (unlikely(!curr_skb))
@@ -369,7 +366,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
369 ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len); 366 ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len);
370 if (unlikely(!ctx)) { 367 if (unlikely(!ctx)) {
371 pr_debug("%s: rx error: %d buffers out of %d missing\n", 368 pr_debug("%s: rx error: %d buffers out of %d missing\n",
372 dev->name, num_buf, hdr->mhdr.num_buffers); 369 dev->name, num_buf,
370 virtio16_to_cpu(vi->vdev,
371 hdr->num_buffers));
373 dev->stats.rx_length_errors++; 372 dev->stats.rx_length_errors++;
374 goto err_buf; 373 goto err_buf;
375 } 374 }
@@ -430,15 +429,15 @@ err_buf:
430 return NULL; 429 return NULL;
431} 430}
432 431
433static void receive_buf(struct receive_queue *rq, void *buf, unsigned int len) 432static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
433 void *buf, unsigned int len)
434{ 434{
435 struct virtnet_info *vi = rq->vq->vdev->priv;
436 struct net_device *dev = vi->dev; 435 struct net_device *dev = vi->dev;
437 struct virtnet_stats *stats = this_cpu_ptr(vi->stats); 436 struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
438 struct sk_buff *skb; 437 struct sk_buff *skb;
439 struct skb_vnet_hdr *hdr; 438 struct virtio_net_hdr_mrg_rxbuf *hdr;
440 439
441 if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) { 440 if (unlikely(len < vi->hdr_len + ETH_HLEN)) {
442 pr_debug("%s: short packet %i\n", dev->name, len); 441 pr_debug("%s: short packet %i\n", dev->name, len);
443 dev->stats.rx_length_errors++; 442 dev->stats.rx_length_errors++;
444 if (vi->mergeable_rx_bufs) { 443 if (vi->mergeable_rx_bufs) {
@@ -454,11 +453,11 @@ static void receive_buf(struct receive_queue *rq, void *buf, unsigned int len)
454 } 453 }
455 454
456 if (vi->mergeable_rx_bufs) 455 if (vi->mergeable_rx_bufs)
457 skb = receive_mergeable(dev, rq, (unsigned long)buf, len); 456 skb = receive_mergeable(dev, vi, rq, (unsigned long)buf, len);
458 else if (vi->big_packets) 457 else if (vi->big_packets)
459 skb = receive_big(dev, rq, buf, len); 458 skb = receive_big(dev, vi, rq, buf, len);
460 else 459 else
461 skb = receive_small(buf, len); 460 skb = receive_small(vi, buf, len);
462 461
463 if (unlikely(!skb)) 462 if (unlikely(!skb))
464 return; 463 return;
@@ -473,8 +472,8 @@ static void receive_buf(struct receive_queue *rq, void *buf, unsigned int len)
473 if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 472 if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
474 pr_debug("Needs csum!\n"); 473 pr_debug("Needs csum!\n");
475 if (!skb_partial_csum_set(skb, 474 if (!skb_partial_csum_set(skb,
476 hdr->hdr.csum_start, 475 virtio16_to_cpu(vi->vdev, hdr->hdr.csum_start),
477 hdr->hdr.csum_offset)) 476 virtio16_to_cpu(vi->vdev, hdr->hdr.csum_offset)))
478 goto frame_err; 477 goto frame_err;
479 } else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) { 478 } else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
480 skb->ip_summed = CHECKSUM_UNNECESSARY; 479 skb->ip_summed = CHECKSUM_UNNECESSARY;
@@ -514,7 +513,8 @@ static void receive_buf(struct receive_queue *rq, void *buf, unsigned int len)
514 if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN) 513 if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
515 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; 514 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
516 515
517 skb_shinfo(skb)->gso_size = hdr->hdr.gso_size; 516 skb_shinfo(skb)->gso_size = virtio16_to_cpu(vi->vdev,
517 hdr->hdr.gso_size);
518 if (skb_shinfo(skb)->gso_size == 0) { 518 if (skb_shinfo(skb)->gso_size == 0) {
519 net_warn_ratelimited("%s: zero gso size.\n", dev->name); 519 net_warn_ratelimited("%s: zero gso size.\n", dev->name);
520 goto frame_err; 520 goto frame_err;
@@ -535,11 +535,11 @@ frame_err:
535 dev_kfree_skb(skb); 535 dev_kfree_skb(skb);
536} 536}
537 537
538static int add_recvbuf_small(struct receive_queue *rq, gfp_t gfp) 538static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
539 gfp_t gfp)
539{ 540{
540 struct virtnet_info *vi = rq->vq->vdev->priv;
541 struct sk_buff *skb; 541 struct sk_buff *skb;
542 struct skb_vnet_hdr *hdr; 542 struct virtio_net_hdr_mrg_rxbuf *hdr;
543 int err; 543 int err;
544 544
545 skb = __netdev_alloc_skb_ip_align(vi->dev, GOOD_PACKET_LEN, gfp); 545 skb = __netdev_alloc_skb_ip_align(vi->dev, GOOD_PACKET_LEN, gfp);
@@ -550,7 +550,7 @@ static int add_recvbuf_small(struct receive_queue *rq, gfp_t gfp)
550 550
551 hdr = skb_vnet_hdr(skb); 551 hdr = skb_vnet_hdr(skb);
552 sg_init_table(rq->sg, MAX_SKB_FRAGS + 2); 552 sg_init_table(rq->sg, MAX_SKB_FRAGS + 2);
553 sg_set_buf(rq->sg, &hdr->hdr, sizeof hdr->hdr); 553 sg_set_buf(rq->sg, hdr, vi->hdr_len);
554 skb_to_sgvec(skb, rq->sg + 1, 0, skb->len); 554 skb_to_sgvec(skb, rq->sg + 1, 0, skb->len);
555 555
556 err = virtqueue_add_inbuf(rq->vq, rq->sg, 2, skb, gfp); 556 err = virtqueue_add_inbuf(rq->vq, rq->sg, 2, skb, gfp);
@@ -560,7 +560,8 @@ static int add_recvbuf_small(struct receive_queue *rq, gfp_t gfp)
560 return err; 560 return err;
561} 561}
562 562
563static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp) 563static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
564 gfp_t gfp)
564{ 565{
565 struct page *first, *list = NULL; 566 struct page *first, *list = NULL;
566 char *p; 567 char *p;
@@ -591,8 +592,8 @@ static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
591 p = page_address(first); 592 p = page_address(first);
592 593
593 /* rq->sg[0], rq->sg[1] share the same page */ 594 /* rq->sg[0], rq->sg[1] share the same page */
594 /* a separated rq->sg[0] for virtio_net_hdr only due to QEMU bug */ 595 /* a separated rq->sg[0] for header - required in case !any_header_sg */
595 sg_set_buf(&rq->sg[0], p, sizeof(struct virtio_net_hdr)); 596 sg_set_buf(&rq->sg[0], p, vi->hdr_len);
596 597
597 /* rq->sg[1] for data packet, from offset */ 598 /* rq->sg[1] for data packet, from offset */
598 offset = sizeof(struct padded_vnet_hdr); 599 offset = sizeof(struct padded_vnet_hdr);
@@ -660,9 +661,9 @@ static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
660 * before we're receiving packets, or from refill_work which is 661 * before we're receiving packets, or from refill_work which is
661 * careful to disable receiving (using napi_disable). 662 * careful to disable receiving (using napi_disable).
662 */ 663 */
663static bool try_fill_recv(struct receive_queue *rq, gfp_t gfp) 664static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
665 gfp_t gfp)
664{ 666{
665 struct virtnet_info *vi = rq->vq->vdev->priv;
666 int err; 667 int err;
667 bool oom; 668 bool oom;
668 669
@@ -671,9 +672,9 @@ static bool try_fill_recv(struct receive_queue *rq, gfp_t gfp)
671 if (vi->mergeable_rx_bufs) 672 if (vi->mergeable_rx_bufs)
672 err = add_recvbuf_mergeable(rq, gfp); 673 err = add_recvbuf_mergeable(rq, gfp);
673 else if (vi->big_packets) 674 else if (vi->big_packets)
674 err = add_recvbuf_big(rq, gfp); 675 err = add_recvbuf_big(vi, rq, gfp);
675 else 676 else
676 err = add_recvbuf_small(rq, gfp); 677 err = add_recvbuf_small(vi, rq, gfp);
677 678
678 oom = err == -ENOMEM; 679 oom = err == -ENOMEM;
679 if (err) 680 if (err)
@@ -722,7 +723,7 @@ static void refill_work(struct work_struct *work)
722 struct receive_queue *rq = &vi->rq[i]; 723 struct receive_queue *rq = &vi->rq[i];
723 724
724 napi_disable(&rq->napi); 725 napi_disable(&rq->napi);
725 still_empty = !try_fill_recv(rq, GFP_KERNEL); 726 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
726 virtnet_napi_enable(rq); 727 virtnet_napi_enable(rq);
727 728
728 /* In theory, this can happen: if we don't get any buffers in 729 /* In theory, this can happen: if we don't get any buffers in
@@ -741,12 +742,12 @@ static int virtnet_receive(struct receive_queue *rq, int budget)
741 742
742 while (received < budget && 743 while (received < budget &&
743 (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) { 744 (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
744 receive_buf(rq, buf, len); 745 receive_buf(vi, rq, buf, len);
745 received++; 746 received++;
746 } 747 }
747 748
748 if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) { 749 if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
749 if (!try_fill_recv(rq, GFP_ATOMIC)) 750 if (!try_fill_recv(vi, rq, GFP_ATOMIC))
750 schedule_delayed_work(&vi->refill, 0); 751 schedule_delayed_work(&vi->refill, 0);
751 } 752 }
752 753
@@ -822,7 +823,7 @@ static int virtnet_open(struct net_device *dev)
822 for (i = 0; i < vi->max_queue_pairs; i++) { 823 for (i = 0; i < vi->max_queue_pairs; i++) {
823 if (i < vi->curr_queue_pairs) 824 if (i < vi->curr_queue_pairs)
824 /* Make sure we have some buffers: if oom use wq. */ 825 /* Make sure we have some buffers: if oom use wq. */
825 if (!try_fill_recv(&vi->rq[i], GFP_KERNEL)) 826 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
826 schedule_delayed_work(&vi->refill, 0); 827 schedule_delayed_work(&vi->refill, 0);
827 virtnet_napi_enable(&vi->rq[i]); 828 virtnet_napi_enable(&vi->rq[i]);
828 } 829 }
@@ -851,18 +852,14 @@ static void free_old_xmit_skbs(struct send_queue *sq)
851 852
852static int xmit_skb(struct send_queue *sq, struct sk_buff *skb) 853static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
853{ 854{
854 struct skb_vnet_hdr *hdr; 855 struct virtio_net_hdr_mrg_rxbuf *hdr;
855 const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest; 856 const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
856 struct virtnet_info *vi = sq->vq->vdev->priv; 857 struct virtnet_info *vi = sq->vq->vdev->priv;
857 unsigned num_sg; 858 unsigned num_sg;
858 unsigned hdr_len; 859 unsigned hdr_len = vi->hdr_len;
859 bool can_push; 860 bool can_push;
860 861
861 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); 862 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
862 if (vi->mergeable_rx_bufs)
863 hdr_len = sizeof hdr->mhdr;
864 else
865 hdr_len = sizeof hdr->hdr;
866 863
867 can_push = vi->any_header_sg && 864 can_push = vi->any_header_sg &&
868 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) && 865 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
@@ -870,22 +867,25 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
870 /* Even if we can, don't push here yet as this would skew 867 /* Even if we can, don't push here yet as this would skew
871 * csum_start offset below. */ 868 * csum_start offset below. */
872 if (can_push) 869 if (can_push)
873 hdr = (struct skb_vnet_hdr *)(skb->data - hdr_len); 870 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len);
874 else 871 else
875 hdr = skb_vnet_hdr(skb); 872 hdr = skb_vnet_hdr(skb);
876 873
877 if (skb->ip_summed == CHECKSUM_PARTIAL) { 874 if (skb->ip_summed == CHECKSUM_PARTIAL) {
878 hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 875 hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
879 hdr->hdr.csum_start = skb_checksum_start_offset(skb); 876 hdr->hdr.csum_start = cpu_to_virtio16(vi->vdev,
880 hdr->hdr.csum_offset = skb->csum_offset; 877 skb_checksum_start_offset(skb));
878 hdr->hdr.csum_offset = cpu_to_virtio16(vi->vdev,
879 skb->csum_offset);
881 } else { 880 } else {
882 hdr->hdr.flags = 0; 881 hdr->hdr.flags = 0;
883 hdr->hdr.csum_offset = hdr->hdr.csum_start = 0; 882 hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
884 } 883 }
885 884
886 if (skb_is_gso(skb)) { 885 if (skb_is_gso(skb)) {
887 hdr->hdr.hdr_len = skb_headlen(skb); 886 hdr->hdr.hdr_len = cpu_to_virtio16(vi->vdev, skb_headlen(skb));
888 hdr->hdr.gso_size = skb_shinfo(skb)->gso_size; 887 hdr->hdr.gso_size = cpu_to_virtio16(vi->vdev,
888 skb_shinfo(skb)->gso_size);
889 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) 889 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
890 hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 890 hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
891 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) 891 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
@@ -900,7 +900,7 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
900 } 900 }
901 901
902 if (vi->mergeable_rx_bufs) 902 if (vi->mergeable_rx_bufs)
903 hdr->mhdr.num_buffers = 0; 903 hdr->num_buffers = 0;
904 904
905 sg_init_table(sq->sg, MAX_SKB_FRAGS + 2); 905 sg_init_table(sq->sg, MAX_SKB_FRAGS + 2);
906 if (can_push) { 906 if (can_push) {
@@ -1030,7 +1030,8 @@ static int virtnet_set_mac_address(struct net_device *dev, void *p)
1030 "Failed to set mac address by vq command.\n"); 1030 "Failed to set mac address by vq command.\n");
1031 return -EINVAL; 1031 return -EINVAL;
1032 } 1032 }
1033 } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) { 1033 } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) &&
1034 !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1034 unsigned int i; 1035 unsigned int i;
1035 1036
1036 /* Naturally, this has an atomicity problem. */ 1037 /* Naturally, this has an atomicity problem. */
@@ -1112,7 +1113,7 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
1112 if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ)) 1113 if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
1113 return 0; 1114 return 0;
1114 1115
1115 s.virtqueue_pairs = queue_pairs; 1116 s.virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs);
1116 sg_init_one(&sg, &s, sizeof(s)); 1117 sg_init_one(&sg, &s, sizeof(s));
1117 1118
1118 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 1119 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
@@ -1189,7 +1190,7 @@ static void virtnet_set_rx_mode(struct net_device *dev)
1189 sg_init_table(sg, 2); 1190 sg_init_table(sg, 2);
1190 1191
1191 /* Store the unicast list and count in the front of the buffer */ 1192 /* Store the unicast list and count in the front of the buffer */
1192 mac_data->entries = uc_count; 1193 mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count);
1193 i = 0; 1194 i = 0;
1194 netdev_for_each_uc_addr(ha, dev) 1195 netdev_for_each_uc_addr(ha, dev)
1195 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 1196 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
@@ -1200,7 +1201,7 @@ static void virtnet_set_rx_mode(struct net_device *dev)
1200 /* multicast list and count fill the end */ 1201 /* multicast list and count fill the end */
1201 mac_data = (void *)&mac_data->macs[uc_count][0]; 1202 mac_data = (void *)&mac_data->macs[uc_count][0];
1202 1203
1203 mac_data->entries = mc_count; 1204 mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count);
1204 i = 0; 1205 i = 0;
1205 netdev_for_each_mc_addr(ha, dev) 1206 netdev_for_each_mc_addr(ha, dev)
1206 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 1207 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
@@ -1805,18 +1806,20 @@ static int virtnet_probe(struct virtio_device *vdev)
1805 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) 1806 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
1806 vi->mergeable_rx_bufs = true; 1807 vi->mergeable_rx_bufs = true;
1807 1808
1809 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
1810 virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
1811 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1812 else
1813 vi->hdr_len = sizeof(struct virtio_net_hdr);
1814
1808 if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT)) 1815 if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT))
1809 vi->any_header_sg = true; 1816 vi->any_header_sg = true;
1810 1817
1811 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 1818 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
1812 vi->has_cvq = true; 1819 vi->has_cvq = true;
1813 1820
1814 if (vi->any_header_sg) { 1821 if (vi->any_header_sg)
1815 if (vi->mergeable_rx_bufs) 1822 dev->needed_headroom = vi->hdr_len;
1816 dev->needed_headroom = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1817 else
1818 dev->needed_headroom = sizeof(struct virtio_net_hdr);
1819 }
1820 1823
1821 /* Use single tx/rx queue pair as default */ 1824 /* Use single tx/rx queue pair as default */
1822 vi->curr_queue_pairs = 1; 1825 vi->curr_queue_pairs = 1;
@@ -1844,7 +1847,7 @@ static int virtnet_probe(struct virtio_device *vdev)
1844 1847
1845 /* Last of all, set up some receive buffers. */ 1848 /* Last of all, set up some receive buffers. */
1846 for (i = 0; i < vi->curr_queue_pairs; i++) { 1849 for (i = 0; i < vi->curr_queue_pairs; i++) {
1847 try_fill_recv(&vi->rq[i], GFP_KERNEL); 1850 try_fill_recv(vi, &vi->rq[i], GFP_KERNEL);
1848 1851
1849 /* If we didn't even get one input buffer, we're useless. */ 1852 /* If we didn't even get one input buffer, we're useless. */
1850 if (vi->rq[i].vq->num_free == 1853 if (vi->rq[i].vq->num_free ==
@@ -1964,7 +1967,7 @@ static int virtnet_restore(struct virtio_device *vdev)
1964 1967
1965 if (netif_running(vi->dev)) { 1968 if (netif_running(vi->dev)) {
1966 for (i = 0; i < vi->curr_queue_pairs; i++) 1969 for (i = 0; i < vi->curr_queue_pairs; i++)
1967 if (!try_fill_recv(&vi->rq[i], GFP_KERNEL)) 1970 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
1968 schedule_delayed_work(&vi->refill, 0); 1971 schedule_delayed_work(&vi->refill, 0);
1969 1972
1970 for (i = 0; i < vi->max_queue_pairs; i++) 1973 for (i = 0; i < vi->max_queue_pairs; i++)
diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
index a34b50690b4e..e1a10232a943 100644
--- a/drivers/remoteproc/remoteproc_virtio.c
+++ b/drivers/remoteproc/remoteproc_virtio.c
@@ -207,7 +207,7 @@ static void rproc_virtio_reset(struct virtio_device *vdev)
207} 207}
208 208
209/* provide the vdev features as retrieved from the firmware */ 209/* provide the vdev features as retrieved from the firmware */
210static u32 rproc_virtio_get_features(struct virtio_device *vdev) 210static u64 rproc_virtio_get_features(struct virtio_device *vdev)
211{ 211{
212 struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); 212 struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
213 struct fw_rsc_vdev *rsc; 213 struct fw_rsc_vdev *rsc;
@@ -217,7 +217,7 @@ static u32 rproc_virtio_get_features(struct virtio_device *vdev)
217 return rsc->dfeatures; 217 return rsc->dfeatures;
218} 218}
219 219
220static void rproc_virtio_finalize_features(struct virtio_device *vdev) 220static int rproc_virtio_finalize_features(struct virtio_device *vdev)
221{ 221{
222 struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); 222 struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
223 struct fw_rsc_vdev *rsc; 223 struct fw_rsc_vdev *rsc;
@@ -227,11 +227,16 @@ static void rproc_virtio_finalize_features(struct virtio_device *vdev)
227 /* Give virtio_ring a chance to accept features */ 227 /* Give virtio_ring a chance to accept features */
228 vring_transport_features(vdev); 228 vring_transport_features(vdev);
229 229
230 /* Make sure we don't have any features > 32 bits! */
231 BUG_ON((u32)vdev->features != vdev->features);
232
230 /* 233 /*
231 * Remember the finalized features of our vdev, and provide it 234 * Remember the finalized features of our vdev, and provide it
232 * to the remote processor once it is powered on. 235 * to the remote processor once it is powered on.
233 */ 236 */
234 rsc->gfeatures = vdev->features[0]; 237 rsc->gfeatures = vdev->features;
238
239 return 0;
235} 240}
236 241
237static void rproc_virtio_get(struct virtio_device *vdev, unsigned offset, 242static void rproc_virtio_get(struct virtio_device *vdev, unsigned offset,
diff --git a/drivers/s390/kvm/kvm_virtio.c b/drivers/s390/kvm/kvm_virtio.c
index 643129070c51..dd65c8b4c7fe 100644
--- a/drivers/s390/kvm/kvm_virtio.c
+++ b/drivers/s390/kvm/kvm_virtio.c
@@ -80,7 +80,7 @@ static unsigned desc_size(const struct kvm_device_desc *desc)
80} 80}
81 81
82/* This gets the device's feature bits. */ 82/* This gets the device's feature bits. */
83static u32 kvm_get_features(struct virtio_device *vdev) 83static u64 kvm_get_features(struct virtio_device *vdev)
84{ 84{
85 unsigned int i; 85 unsigned int i;
86 u32 features = 0; 86 u32 features = 0;
@@ -93,7 +93,7 @@ static u32 kvm_get_features(struct virtio_device *vdev)
93 return features; 93 return features;
94} 94}
95 95
96static void kvm_finalize_features(struct virtio_device *vdev) 96static int kvm_finalize_features(struct virtio_device *vdev)
97{ 97{
98 unsigned int i, bits; 98 unsigned int i, bits;
99 struct kvm_device_desc *desc = to_kvmdev(vdev)->desc; 99 struct kvm_device_desc *desc = to_kvmdev(vdev)->desc;
@@ -103,12 +103,17 @@ static void kvm_finalize_features(struct virtio_device *vdev)
103 /* Give virtio_ring a chance to accept features. */ 103 /* Give virtio_ring a chance to accept features. */
104 vring_transport_features(vdev); 104 vring_transport_features(vdev);
105 105
106 /* Make sure we don't have any features > 32 bits! */
107 BUG_ON((u32)vdev->features != vdev->features);
108
106 memset(out_features, 0, desc->feature_len); 109 memset(out_features, 0, desc->feature_len);
107 bits = min_t(unsigned, desc->feature_len, sizeof(vdev->features)) * 8; 110 bits = min_t(unsigned, desc->feature_len, sizeof(vdev->features)) * 8;
108 for (i = 0; i < bits; i++) { 111 for (i = 0; i < bits; i++) {
109 if (test_bit(i, vdev->features)) 112 if (__virtio_test_bit(vdev, i))
110 out_features[i / 8] |= (1 << (i % 8)); 113 out_features[i / 8] |= (1 << (i % 8));
111 } 114 }
115
116 return 0;
112} 117}
113 118
114/* 119/*
diff --git a/drivers/s390/kvm/virtio_ccw.c b/drivers/s390/kvm/virtio_ccw.c
index bda52f18e967..71d7802aa8b4 100644
--- a/drivers/s390/kvm/virtio_ccw.c
+++ b/drivers/s390/kvm/virtio_ccw.c
@@ -55,6 +55,7 @@ struct virtio_ccw_device {
55 struct ccw_device *cdev; 55 struct ccw_device *cdev;
56 __u32 curr_io; 56 __u32 curr_io;
57 int err; 57 int err;
58 unsigned int revision; /* Transport revision */
58 wait_queue_head_t wait_q; 59 wait_queue_head_t wait_q;
59 spinlock_t lock; 60 spinlock_t lock;
60 struct list_head virtqueues; 61 struct list_head virtqueues;
@@ -67,13 +68,22 @@ struct virtio_ccw_device {
67 void *airq_info; 68 void *airq_info;
68}; 69};
69 70
70struct vq_info_block { 71struct vq_info_block_legacy {
71 __u64 queue; 72 __u64 queue;
72 __u32 align; 73 __u32 align;
73 __u16 index; 74 __u16 index;
74 __u16 num; 75 __u16 num;
75} __packed; 76} __packed;
76 77
78struct vq_info_block {
79 __u64 desc;
80 __u32 res0;
81 __u16 index;
82 __u16 num;
83 __u64 avail;
84 __u64 used;
85} __packed;
86
77struct virtio_feature_desc { 87struct virtio_feature_desc {
78 __u32 features; 88 __u32 features;
79 __u8 index; 89 __u8 index;
@@ -86,11 +96,23 @@ struct virtio_thinint_area {
86 u8 isc; 96 u8 isc;
87} __packed; 97} __packed;
88 98
99struct virtio_rev_info {
100 __u16 revision;
101 __u16 length;
102 __u8 data[];
103};
104
105/* the highest virtio-ccw revision we support */
106#define VIRTIO_CCW_REV_MAX 1
107
89struct virtio_ccw_vq_info { 108struct virtio_ccw_vq_info {
90 struct virtqueue *vq; 109 struct virtqueue *vq;
91 int num; 110 int num;
92 void *queue; 111 void *queue;
93 struct vq_info_block *info_block; 112 union {
113 struct vq_info_block s;
114 struct vq_info_block_legacy l;
115 } *info_block;
94 int bit_nr; 116 int bit_nr;
95 struct list_head node; 117 struct list_head node;
96 long cookie; 118 long cookie;
@@ -122,6 +144,7 @@ static struct airq_info *airq_areas[MAX_AIRQ_AREAS];
122#define CCW_CMD_WRITE_STATUS 0x31 144#define CCW_CMD_WRITE_STATUS 0x31
123#define CCW_CMD_READ_VQ_CONF 0x32 145#define CCW_CMD_READ_VQ_CONF 0x32
124#define CCW_CMD_SET_IND_ADAPTER 0x73 146#define CCW_CMD_SET_IND_ADAPTER 0x73
147#define CCW_CMD_SET_VIRTIO_REV 0x83
125 148
126#define VIRTIO_CCW_DOING_SET_VQ 0x00010000 149#define VIRTIO_CCW_DOING_SET_VQ 0x00010000
127#define VIRTIO_CCW_DOING_RESET 0x00040000 150#define VIRTIO_CCW_DOING_RESET 0x00040000
@@ -134,6 +157,7 @@ static struct airq_info *airq_areas[MAX_AIRQ_AREAS];
134#define VIRTIO_CCW_DOING_READ_VQ_CONF 0x02000000 157#define VIRTIO_CCW_DOING_READ_VQ_CONF 0x02000000
135#define VIRTIO_CCW_DOING_SET_CONF_IND 0x04000000 158#define VIRTIO_CCW_DOING_SET_CONF_IND 0x04000000
136#define VIRTIO_CCW_DOING_SET_IND_ADAPTER 0x08000000 159#define VIRTIO_CCW_DOING_SET_IND_ADAPTER 0x08000000
160#define VIRTIO_CCW_DOING_SET_VIRTIO_REV 0x10000000
137#define VIRTIO_CCW_INTPARM_MASK 0xffff0000 161#define VIRTIO_CCW_INTPARM_MASK 0xffff0000
138 162
139static struct virtio_ccw_device *to_vc_device(struct virtio_device *vdev) 163static struct virtio_ccw_device *to_vc_device(struct virtio_device *vdev)
@@ -399,13 +423,22 @@ static void virtio_ccw_del_vq(struct virtqueue *vq, struct ccw1 *ccw)
399 spin_unlock_irqrestore(&vcdev->lock, flags); 423 spin_unlock_irqrestore(&vcdev->lock, flags);
400 424
401 /* Release from host. */ 425 /* Release from host. */
402 info->info_block->queue = 0; 426 if (vcdev->revision == 0) {
403 info->info_block->align = 0; 427 info->info_block->l.queue = 0;
404 info->info_block->index = index; 428 info->info_block->l.align = 0;
405 info->info_block->num = 0; 429 info->info_block->l.index = index;
430 info->info_block->l.num = 0;
431 ccw->count = sizeof(info->info_block->l);
432 } else {
433 info->info_block->s.desc = 0;
434 info->info_block->s.index = index;
435 info->info_block->s.num = 0;
436 info->info_block->s.avail = 0;
437 info->info_block->s.used = 0;
438 ccw->count = sizeof(info->info_block->s);
439 }
406 ccw->cmd_code = CCW_CMD_SET_VQ; 440 ccw->cmd_code = CCW_CMD_SET_VQ;
407 ccw->flags = 0; 441 ccw->flags = 0;
408 ccw->count = sizeof(*info->info_block);
409 ccw->cda = (__u32)(unsigned long)(info->info_block); 442 ccw->cda = (__u32)(unsigned long)(info->info_block);
410 ret = ccw_io_helper(vcdev, ccw, 443 ret = ccw_io_helper(vcdev, ccw,
411 VIRTIO_CCW_DOING_SET_VQ | index); 444 VIRTIO_CCW_DOING_SET_VQ | index);
@@ -488,13 +521,22 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev,
488 } 521 }
489 522
490 /* Register it with the host. */ 523 /* Register it with the host. */
491 info->info_block->queue = (__u64)info->queue; 524 if (vcdev->revision == 0) {
492 info->info_block->align = KVM_VIRTIO_CCW_RING_ALIGN; 525 info->info_block->l.queue = (__u64)info->queue;
493 info->info_block->index = i; 526 info->info_block->l.align = KVM_VIRTIO_CCW_RING_ALIGN;
494 info->info_block->num = info->num; 527 info->info_block->l.index = i;
528 info->info_block->l.num = info->num;
529 ccw->count = sizeof(info->info_block->l);
530 } else {
531 info->info_block->s.desc = (__u64)info->queue;
532 info->info_block->s.index = i;
533 info->info_block->s.num = info->num;
534 info->info_block->s.avail = (__u64)virtqueue_get_avail(vq);
535 info->info_block->s.used = (__u64)virtqueue_get_used(vq);
536 ccw->count = sizeof(info->info_block->s);
537 }
495 ccw->cmd_code = CCW_CMD_SET_VQ; 538 ccw->cmd_code = CCW_CMD_SET_VQ;
496 ccw->flags = 0; 539 ccw->flags = 0;
497 ccw->count = sizeof(*info->info_block);
498 ccw->cda = (__u32)(unsigned long)(info->info_block); 540 ccw->cda = (__u32)(unsigned long)(info->info_block);
499 err = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_SET_VQ | i); 541 err = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_SET_VQ | i);
500 if (err) { 542 if (err) {
@@ -660,11 +702,12 @@ static void virtio_ccw_reset(struct virtio_device *vdev)
660 kfree(ccw); 702 kfree(ccw);
661} 703}
662 704
663static u32 virtio_ccw_get_features(struct virtio_device *vdev) 705static u64 virtio_ccw_get_features(struct virtio_device *vdev)
664{ 706{
665 struct virtio_ccw_device *vcdev = to_vc_device(vdev); 707 struct virtio_ccw_device *vcdev = to_vc_device(vdev);
666 struct virtio_feature_desc *features; 708 struct virtio_feature_desc *features;
667 int ret, rc; 709 int ret;
710 u64 rc;
668 struct ccw1 *ccw; 711 struct ccw1 *ccw;
669 712
670 ccw = kzalloc(sizeof(*ccw), GFP_DMA | GFP_KERNEL); 713 ccw = kzalloc(sizeof(*ccw), GFP_DMA | GFP_KERNEL);
@@ -677,7 +720,6 @@ static u32 virtio_ccw_get_features(struct virtio_device *vdev)
677 goto out_free; 720 goto out_free;
678 } 721 }
679 /* Read the feature bits from the host. */ 722 /* Read the feature bits from the host. */
680 /* TODO: Features > 32 bits */
681 features->index = 0; 723 features->index = 0;
682 ccw->cmd_code = CCW_CMD_READ_FEAT; 724 ccw->cmd_code = CCW_CMD_READ_FEAT;
683 ccw->flags = 0; 725 ccw->flags = 0;
@@ -691,46 +733,79 @@ static u32 virtio_ccw_get_features(struct virtio_device *vdev)
691 733
692 rc = le32_to_cpu(features->features); 734 rc = le32_to_cpu(features->features);
693 735
736 if (vcdev->revision == 0)
737 goto out_free;
738
739 /* Read second half of the feature bits from the host. */
740 features->index = 1;
741 ccw->cmd_code = CCW_CMD_READ_FEAT;
742 ccw->flags = 0;
743 ccw->count = sizeof(*features);
744 ccw->cda = (__u32)(unsigned long)features;
745 ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_READ_FEAT);
746 if (ret == 0)
747 rc |= (u64)le32_to_cpu(features->features) << 32;
748
694out_free: 749out_free:
695 kfree(features); 750 kfree(features);
696 kfree(ccw); 751 kfree(ccw);
697 return rc; 752 return rc;
698} 753}
699 754
700static void virtio_ccw_finalize_features(struct virtio_device *vdev) 755static int virtio_ccw_finalize_features(struct virtio_device *vdev)
701{ 756{
702 struct virtio_ccw_device *vcdev = to_vc_device(vdev); 757 struct virtio_ccw_device *vcdev = to_vc_device(vdev);
703 struct virtio_feature_desc *features; 758 struct virtio_feature_desc *features;
704 int i;
705 struct ccw1 *ccw; 759 struct ccw1 *ccw;
760 int ret;
761
762 if (vcdev->revision >= 1 &&
763 !__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) {
764 dev_err(&vdev->dev, "virtio: device uses revision 1 "
765 "but does not have VIRTIO_F_VERSION_1\n");
766 return -EINVAL;
767 }
706 768
707 ccw = kzalloc(sizeof(*ccw), GFP_DMA | GFP_KERNEL); 769 ccw = kzalloc(sizeof(*ccw), GFP_DMA | GFP_KERNEL);
708 if (!ccw) 770 if (!ccw)
709 return; 771 return -ENOMEM;
710 772
711 features = kzalloc(sizeof(*features), GFP_DMA | GFP_KERNEL); 773 features = kzalloc(sizeof(*features), GFP_DMA | GFP_KERNEL);
712 if (!features) 774 if (!features) {
775 ret = -ENOMEM;
713 goto out_free; 776 goto out_free;
714 777 }
715 /* Give virtio_ring a chance to accept features. */ 778 /* Give virtio_ring a chance to accept features. */
716 vring_transport_features(vdev); 779 vring_transport_features(vdev);
717 780
718 for (i = 0; i < sizeof(*vdev->features) / sizeof(features->features); 781 features->index = 0;
719 i++) { 782 features->features = cpu_to_le32((u32)vdev->features);
720 int highbits = i % 2 ? 32 : 0; 783 /* Write the first half of the feature bits to the host. */
721 features->index = i; 784 ccw->cmd_code = CCW_CMD_WRITE_FEAT;
722 features->features = cpu_to_le32(vdev->features[i / 2] 785 ccw->flags = 0;
723 >> highbits); 786 ccw->count = sizeof(*features);
724 /* Write the feature bits to the host. */ 787 ccw->cda = (__u32)(unsigned long)features;
725 ccw->cmd_code = CCW_CMD_WRITE_FEAT; 788 ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_WRITE_FEAT);
726 ccw->flags = 0; 789 if (ret)
727 ccw->count = sizeof(*features); 790 goto out_free;
728 ccw->cda = (__u32)(unsigned long)features; 791
729 ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_WRITE_FEAT); 792 if (vcdev->revision == 0)
730 } 793 goto out_free;
794
795 features->index = 1;
796 features->features = cpu_to_le32(vdev->features >> 32);
797 /* Write the second half of the feature bits to the host. */
798 ccw->cmd_code = CCW_CMD_WRITE_FEAT;
799 ccw->flags = 0;
800 ccw->count = sizeof(*features);
801 ccw->cda = (__u32)(unsigned long)features;
802 ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_WRITE_FEAT);
803
731out_free: 804out_free:
732 kfree(features); 805 kfree(features);
733 kfree(ccw); 806 kfree(ccw);
807
808 return ret;
734} 809}
735 810
736static void virtio_ccw_get_config(struct virtio_device *vdev, 811static void virtio_ccw_get_config(struct virtio_device *vdev,
@@ -806,7 +881,9 @@ static u8 virtio_ccw_get_status(struct virtio_device *vdev)
806static void virtio_ccw_set_status(struct virtio_device *vdev, u8 status) 881static void virtio_ccw_set_status(struct virtio_device *vdev, u8 status)
807{ 882{
808 struct virtio_ccw_device *vcdev = to_vc_device(vdev); 883 struct virtio_ccw_device *vcdev = to_vc_device(vdev);
884 u8 old_status = *vcdev->status;
809 struct ccw1 *ccw; 885 struct ccw1 *ccw;
886 int ret;
810 887
811 ccw = kzalloc(sizeof(*ccw), GFP_DMA | GFP_KERNEL); 888 ccw = kzalloc(sizeof(*ccw), GFP_DMA | GFP_KERNEL);
812 if (!ccw) 889 if (!ccw)
@@ -818,7 +895,10 @@ static void virtio_ccw_set_status(struct virtio_device *vdev, u8 status)
818 ccw->flags = 0; 895 ccw->flags = 0;
819 ccw->count = sizeof(status); 896 ccw->count = sizeof(status);
820 ccw->cda = (__u32)(unsigned long)vcdev->status; 897 ccw->cda = (__u32)(unsigned long)vcdev->status;
821 ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_WRITE_STATUS); 898 ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_WRITE_STATUS);
899 /* Write failed? We assume status is unchanged. */
900 if (ret)
901 *vcdev->status = old_status;
822 kfree(ccw); 902 kfree(ccw);
823} 903}
824 904
@@ -919,6 +999,7 @@ static void virtio_ccw_int_handler(struct ccw_device *cdev,
919 case VIRTIO_CCW_DOING_RESET: 999 case VIRTIO_CCW_DOING_RESET:
920 case VIRTIO_CCW_DOING_READ_VQ_CONF: 1000 case VIRTIO_CCW_DOING_READ_VQ_CONF:
921 case VIRTIO_CCW_DOING_SET_IND_ADAPTER: 1001 case VIRTIO_CCW_DOING_SET_IND_ADAPTER:
1002 case VIRTIO_CCW_DOING_SET_VIRTIO_REV:
922 vcdev->curr_io &= ~activity; 1003 vcdev->curr_io &= ~activity;
923 wake_up(&vcdev->wait_q); 1004 wake_up(&vcdev->wait_q);
924 break; 1005 break;
@@ -1034,6 +1115,51 @@ static int virtio_ccw_offline(struct ccw_device *cdev)
1034 return 0; 1115 return 0;
1035} 1116}
1036 1117
1118static int virtio_ccw_set_transport_rev(struct virtio_ccw_device *vcdev)
1119{
1120 struct virtio_rev_info *rev;
1121 struct ccw1 *ccw;
1122 int ret;
1123
1124 ccw = kzalloc(sizeof(*ccw), GFP_DMA | GFP_KERNEL);
1125 if (!ccw)
1126 return -ENOMEM;
1127 rev = kzalloc(sizeof(*rev), GFP_DMA | GFP_KERNEL);
1128 if (!rev) {
1129 kfree(ccw);
1130 return -ENOMEM;
1131 }
1132
1133 /* Set transport revision */
1134 ccw->cmd_code = CCW_CMD_SET_VIRTIO_REV;
1135 ccw->flags = 0;
1136 ccw->count = sizeof(*rev);
1137 ccw->cda = (__u32)(unsigned long)rev;
1138
1139 vcdev->revision = VIRTIO_CCW_REV_MAX;
1140 do {
1141 rev->revision = vcdev->revision;
1142 /* none of our supported revisions carry payload */
1143 rev->length = 0;
1144 ret = ccw_io_helper(vcdev, ccw,
1145 VIRTIO_CCW_DOING_SET_VIRTIO_REV);
1146 if (ret == -EOPNOTSUPP) {
1147 if (vcdev->revision == 0)
1148 /*
1149 * The host device does not support setting
1150 * the revision: let's operate it in legacy
1151 * mode.
1152 */
1153 ret = 0;
1154 else
1155 vcdev->revision--;
1156 }
1157 } while (ret == -EOPNOTSUPP);
1158
1159 kfree(ccw);
1160 kfree(rev);
1161 return ret;
1162}
1037 1163
1038static int virtio_ccw_online(struct ccw_device *cdev) 1164static int virtio_ccw_online(struct ccw_device *cdev)
1039{ 1165{
@@ -1074,6 +1200,15 @@ static int virtio_ccw_online(struct ccw_device *cdev)
1074 spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags); 1200 spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
1075 vcdev->vdev.id.vendor = cdev->id.cu_type; 1201 vcdev->vdev.id.vendor = cdev->id.cu_type;
1076 vcdev->vdev.id.device = cdev->id.cu_model; 1202 vcdev->vdev.id.device = cdev->id.cu_model;
1203
1204 if (virtio_device_is_legacy_only(vcdev->vdev.id)) {
1205 vcdev->revision = 0;
1206 } else {
1207 ret = virtio_ccw_set_transport_rev(vcdev);
1208 if (ret)
1209 goto out_free;
1210 }
1211
1077 ret = register_virtio_device(&vcdev->vdev); 1212 ret = register_virtio_device(&vcdev->vdev);
1078 if (ret) { 1213 if (ret) {
1079 dev_warn(&cdev->dev, "Failed to register virtio device: %d\n", 1214 dev_warn(&cdev->dev, "Failed to register virtio device: %d\n",
diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
index 22e70126425b..c52bb5dfaedb 100644
--- a/drivers/scsi/virtio_scsi.c
+++ b/drivers/scsi/virtio_scsi.c
@@ -158,7 +158,7 @@ static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf)
158 sc, resp->response, resp->status, resp->sense_len); 158 sc, resp->response, resp->status, resp->sense_len);
159 159
160 sc->result = resp->status; 160 sc->result = resp->status;
161 virtscsi_compute_resid(sc, resp->resid); 161 virtscsi_compute_resid(sc, virtio32_to_cpu(vscsi->vdev, resp->resid));
162 switch (resp->response) { 162 switch (resp->response) {
163 case VIRTIO_SCSI_S_OK: 163 case VIRTIO_SCSI_S_OK:
164 set_host_byte(sc, DID_OK); 164 set_host_byte(sc, DID_OK);
@@ -196,10 +196,13 @@ static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf)
196 break; 196 break;
197 } 197 }
198 198
199 WARN_ON(resp->sense_len > VIRTIO_SCSI_SENSE_SIZE); 199 WARN_ON(virtio32_to_cpu(vscsi->vdev, resp->sense_len) >
200 VIRTIO_SCSI_SENSE_SIZE);
200 if (sc->sense_buffer) { 201 if (sc->sense_buffer) {
201 memcpy(sc->sense_buffer, resp->sense, 202 memcpy(sc->sense_buffer, resp->sense,
202 min_t(u32, resp->sense_len, VIRTIO_SCSI_SENSE_SIZE)); 203 min_t(u32,
204 virtio32_to_cpu(vscsi->vdev, resp->sense_len),
205 VIRTIO_SCSI_SENSE_SIZE));
203 if (resp->sense_len) 206 if (resp->sense_len)
204 set_driver_byte(sc, DRIVER_SENSE); 207 set_driver_byte(sc, DRIVER_SENSE);
205 } 208 }
@@ -323,7 +326,7 @@ static void virtscsi_handle_transport_reset(struct virtio_scsi *vscsi,
323 unsigned int target = event->lun[1]; 326 unsigned int target = event->lun[1];
324 unsigned int lun = (event->lun[2] << 8) | event->lun[3]; 327 unsigned int lun = (event->lun[2] << 8) | event->lun[3];
325 328
326 switch (event->reason) { 329 switch (virtio32_to_cpu(vscsi->vdev, event->reason)) {
327 case VIRTIO_SCSI_EVT_RESET_RESCAN: 330 case VIRTIO_SCSI_EVT_RESET_RESCAN:
328 scsi_add_device(shost, 0, target, lun); 331 scsi_add_device(shost, 0, target, lun);
329 break; 332 break;
@@ -349,8 +352,8 @@ static void virtscsi_handle_param_change(struct virtio_scsi *vscsi,
349 struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev); 352 struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
350 unsigned int target = event->lun[1]; 353 unsigned int target = event->lun[1];
351 unsigned int lun = (event->lun[2] << 8) | event->lun[3]; 354 unsigned int lun = (event->lun[2] << 8) | event->lun[3];
352 u8 asc = event->reason & 255; 355 u8 asc = virtio32_to_cpu(vscsi->vdev, event->reason) & 255;
353 u8 ascq = event->reason >> 8; 356 u8 ascq = virtio32_to_cpu(vscsi->vdev, event->reason) >> 8;
354 357
355 sdev = scsi_device_lookup(shost, 0, target, lun); 358 sdev = scsi_device_lookup(shost, 0, target, lun);
356 if (!sdev) { 359 if (!sdev) {
@@ -374,12 +377,14 @@ static void virtscsi_handle_event(struct work_struct *work)
374 struct virtio_scsi *vscsi = event_node->vscsi; 377 struct virtio_scsi *vscsi = event_node->vscsi;
375 struct virtio_scsi_event *event = &event_node->event; 378 struct virtio_scsi_event *event = &event_node->event;
376 379
377 if (event->event & VIRTIO_SCSI_T_EVENTS_MISSED) { 380 if (event->event &
378 event->event &= ~VIRTIO_SCSI_T_EVENTS_MISSED; 381 cpu_to_virtio32(vscsi->vdev, VIRTIO_SCSI_T_EVENTS_MISSED)) {
382 event->event &= ~cpu_to_virtio32(vscsi->vdev,
383 VIRTIO_SCSI_T_EVENTS_MISSED);
379 scsi_scan_host(virtio_scsi_host(vscsi->vdev)); 384 scsi_scan_host(virtio_scsi_host(vscsi->vdev));
380 } 385 }
381 386
382 switch (event->event) { 387 switch (virtio32_to_cpu(vscsi->vdev, event->event)) {
383 case VIRTIO_SCSI_T_NO_EVENT: 388 case VIRTIO_SCSI_T_NO_EVENT:
384 break; 389 break;
385 case VIRTIO_SCSI_T_TRANSPORT_RESET: 390 case VIRTIO_SCSI_T_TRANSPORT_RESET:
@@ -482,26 +487,28 @@ static int virtscsi_kick_cmd(struct virtio_scsi_vq *vq,
482 return err; 487 return err;
483} 488}
484 489
485static void virtio_scsi_init_hdr(struct virtio_scsi_cmd_req *cmd, 490static void virtio_scsi_init_hdr(struct virtio_device *vdev,
491 struct virtio_scsi_cmd_req *cmd,
486 struct scsi_cmnd *sc) 492 struct scsi_cmnd *sc)
487{ 493{
488 cmd->lun[0] = 1; 494 cmd->lun[0] = 1;
489 cmd->lun[1] = sc->device->id; 495 cmd->lun[1] = sc->device->id;
490 cmd->lun[2] = (sc->device->lun >> 8) | 0x40; 496 cmd->lun[2] = (sc->device->lun >> 8) | 0x40;
491 cmd->lun[3] = sc->device->lun & 0xff; 497 cmd->lun[3] = sc->device->lun & 0xff;
492 cmd->tag = (unsigned long)sc; 498 cmd->tag = cpu_to_virtio64(vdev, (unsigned long)sc);
493 cmd->task_attr = VIRTIO_SCSI_S_SIMPLE; 499 cmd->task_attr = VIRTIO_SCSI_S_SIMPLE;
494 cmd->prio = 0; 500 cmd->prio = 0;
495 cmd->crn = 0; 501 cmd->crn = 0;
496} 502}
497 503
498static void virtio_scsi_init_hdr_pi(struct virtio_scsi_cmd_req_pi *cmd_pi, 504static void virtio_scsi_init_hdr_pi(struct virtio_device *vdev,
505 struct virtio_scsi_cmd_req_pi *cmd_pi,
499 struct scsi_cmnd *sc) 506 struct scsi_cmnd *sc)
500{ 507{
501 struct request *rq = sc->request; 508 struct request *rq = sc->request;
502 struct blk_integrity *bi; 509 struct blk_integrity *bi;
503 510
504 virtio_scsi_init_hdr((struct virtio_scsi_cmd_req *)cmd_pi, sc); 511 virtio_scsi_init_hdr(vdev, (struct virtio_scsi_cmd_req *)cmd_pi, sc);
505 512
506 if (!rq || !scsi_prot_sg_count(sc)) 513 if (!rq || !scsi_prot_sg_count(sc))
507 return; 514 return;
@@ -509,9 +516,13 @@ static void virtio_scsi_init_hdr_pi(struct virtio_scsi_cmd_req_pi *cmd_pi,
509 bi = blk_get_integrity(rq->rq_disk); 516 bi = blk_get_integrity(rq->rq_disk);
510 517
511 if (sc->sc_data_direction == DMA_TO_DEVICE) 518 if (sc->sc_data_direction == DMA_TO_DEVICE)
512 cmd_pi->pi_bytesout = blk_rq_sectors(rq) * bi->tuple_size; 519 cmd_pi->pi_bytesout = cpu_to_virtio32(vdev,
520 blk_rq_sectors(rq) *
521 bi->tuple_size);
513 else if (sc->sc_data_direction == DMA_FROM_DEVICE) 522 else if (sc->sc_data_direction == DMA_FROM_DEVICE)
514 cmd_pi->pi_bytesin = blk_rq_sectors(rq) * bi->tuple_size; 523 cmd_pi->pi_bytesin = cpu_to_virtio32(vdev,
524 blk_rq_sectors(rq) *
525 bi->tuple_size);
515} 526}
516 527
517static int virtscsi_queuecommand(struct virtio_scsi *vscsi, 528static int virtscsi_queuecommand(struct virtio_scsi *vscsi,
@@ -536,11 +547,11 @@ static int virtscsi_queuecommand(struct virtio_scsi *vscsi,
536 BUG_ON(sc->cmd_len > VIRTIO_SCSI_CDB_SIZE); 547 BUG_ON(sc->cmd_len > VIRTIO_SCSI_CDB_SIZE);
537 548
538 if (virtio_has_feature(vscsi->vdev, VIRTIO_SCSI_F_T10_PI)) { 549 if (virtio_has_feature(vscsi->vdev, VIRTIO_SCSI_F_T10_PI)) {
539 virtio_scsi_init_hdr_pi(&cmd->req.cmd_pi, sc); 550 virtio_scsi_init_hdr_pi(vscsi->vdev, &cmd->req.cmd_pi, sc);
540 memcpy(cmd->req.cmd_pi.cdb, sc->cmnd, sc->cmd_len); 551 memcpy(cmd->req.cmd_pi.cdb, sc->cmnd, sc->cmd_len);
541 req_size = sizeof(cmd->req.cmd_pi); 552 req_size = sizeof(cmd->req.cmd_pi);
542 } else { 553 } else {
543 virtio_scsi_init_hdr(&cmd->req.cmd, sc); 554 virtio_scsi_init_hdr(vscsi->vdev, &cmd->req.cmd, sc);
544 memcpy(cmd->req.cmd.cdb, sc->cmnd, sc->cmd_len); 555 memcpy(cmd->req.cmd.cdb, sc->cmnd, sc->cmd_len);
545 req_size = sizeof(cmd->req.cmd); 556 req_size = sizeof(cmd->req.cmd);
546 } 557 }
@@ -669,7 +680,8 @@ static int virtscsi_device_reset(struct scsi_cmnd *sc)
669 cmd->sc = sc; 680 cmd->sc = sc;
670 cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){ 681 cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){
671 .type = VIRTIO_SCSI_T_TMF, 682 .type = VIRTIO_SCSI_T_TMF,
672 .subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET, 683 .subtype = cpu_to_virtio32(vscsi->vdev,
684 VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET),
673 .lun[0] = 1, 685 .lun[0] = 1,
674 .lun[1] = sc->device->id, 686 .lun[1] = sc->device->id,
675 .lun[2] = (sc->device->lun >> 8) | 0x40, 687 .lun[2] = (sc->device->lun >> 8) | 0x40,
@@ -710,7 +722,7 @@ static int virtscsi_abort(struct scsi_cmnd *sc)
710 .lun[1] = sc->device->id, 722 .lun[1] = sc->device->id,
711 .lun[2] = (sc->device->lun >> 8) | 0x40, 723 .lun[2] = (sc->device->lun >> 8) | 0x40,
712 .lun[3] = sc->device->lun & 0xff, 724 .lun[3] = sc->device->lun & 0xff,
713 .tag = (unsigned long)sc, 725 .tag = cpu_to_virtio64(vscsi->vdev, (unsigned long)sc),
714 }; 726 };
715 return virtscsi_tmf(vscsi, cmd); 727 return virtscsi_tmf(vscsi, cmd);
716} 728}
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 8dae2f724a35..a935c254749e 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -48,20 +48,21 @@ MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
48 * status internally; used for zerocopy tx only. 48 * status internally; used for zerocopy tx only.
49 */ 49 */
50/* Lower device DMA failed */ 50/* Lower device DMA failed */
51#define VHOST_DMA_FAILED_LEN 3 51#define VHOST_DMA_FAILED_LEN ((__force __virtio32)3)
52/* Lower device DMA done */ 52/* Lower device DMA done */
53#define VHOST_DMA_DONE_LEN 2 53#define VHOST_DMA_DONE_LEN ((__force __virtio32)2)
54/* Lower device DMA in progress */ 54/* Lower device DMA in progress */
55#define VHOST_DMA_IN_PROGRESS 1 55#define VHOST_DMA_IN_PROGRESS ((__force __virtio32)1)
56/* Buffer unused */ 56/* Buffer unused */
57#define VHOST_DMA_CLEAR_LEN 0 57#define VHOST_DMA_CLEAR_LEN ((__force __virtio32)0)
58 58
59#define VHOST_DMA_IS_DONE(len) ((len) >= VHOST_DMA_DONE_LEN) 59#define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN)
60 60
61enum { 61enum {
62 VHOST_NET_FEATURES = VHOST_FEATURES | 62 VHOST_NET_FEATURES = VHOST_FEATURES |
63 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) | 63 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) |
64 (1ULL << VIRTIO_NET_F_MRG_RXBUF), 64 (1ULL << VIRTIO_NET_F_MRG_RXBUF) |
65 (1ULL << VIRTIO_F_VERSION_1),
65}; 66};
66 67
67enum { 68enum {
@@ -416,7 +417,7 @@ static void handle_tx(struct vhost_net *net)
416 struct ubuf_info *ubuf; 417 struct ubuf_info *ubuf;
417 ubuf = nvq->ubuf_info + nvq->upend_idx; 418 ubuf = nvq->ubuf_info + nvq->upend_idx;
418 419
419 vq->heads[nvq->upend_idx].id = head; 420 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
420 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS; 421 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
421 ubuf->callback = vhost_zerocopy_callback; 422 ubuf->callback = vhost_zerocopy_callback;
422 ubuf->ctx = nvq->ubufs; 423 ubuf->ctx = nvq->ubufs;
@@ -500,6 +501,10 @@ static int get_rx_bufs(struct vhost_virtqueue *vq,
500 int headcount = 0; 501 int headcount = 0;
501 unsigned d; 502 unsigned d;
502 int r, nlogs = 0; 503 int r, nlogs = 0;
504 /* len is always initialized before use since we are always called with
505 * datalen > 0.
506 */
507 u32 uninitialized_var(len);
503 508
504 while (datalen > 0 && headcount < quota) { 509 while (datalen > 0 && headcount < quota) {
505 if (unlikely(seg >= UIO_MAXIOV)) { 510 if (unlikely(seg >= UIO_MAXIOV)) {
@@ -527,13 +532,14 @@ static int get_rx_bufs(struct vhost_virtqueue *vq,
527 nlogs += *log_num; 532 nlogs += *log_num;
528 log += *log_num; 533 log += *log_num;
529 } 534 }
530 heads[headcount].id = d; 535 heads[headcount].id = cpu_to_vhost32(vq, d);
531 heads[headcount].len = iov_length(vq->iov + seg, in); 536 len = iov_length(vq->iov + seg, in);
532 datalen -= heads[headcount].len; 537 heads[headcount].len = cpu_to_vhost32(vq, len);
538 datalen -= len;
533 ++headcount; 539 ++headcount;
534 seg += in; 540 seg += in;
535 } 541 }
536 heads[headcount - 1].len += datalen; 542 heads[headcount - 1].len = cpu_to_vhost32(vq, len - datalen);
537 *iovcount = seg; 543 *iovcount = seg;
538 if (unlikely(log)) 544 if (unlikely(log))
539 *log_num = nlogs; 545 *log_num = nlogs;
@@ -1025,7 +1031,8 @@ static int vhost_net_set_features(struct vhost_net *n, u64 features)
1025 size_t vhost_hlen, sock_hlen, hdr_len; 1031 size_t vhost_hlen, sock_hlen, hdr_len;
1026 int i; 1032 int i;
1027 1033
1028 hdr_len = (features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? 1034 hdr_len = (features & ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
1035 (1ULL << VIRTIO_F_VERSION_1))) ?
1029 sizeof(struct virtio_net_hdr_mrg_rxbuf) : 1036 sizeof(struct virtio_net_hdr_mrg_rxbuf) :
1030 sizeof(struct virtio_net_hdr); 1037 sizeof(struct virtio_net_hdr);
1031 if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) { 1038 if (features & (1 << VHOST_NET_F_VIRTIO_NET_HDR)) {
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index a17f11850669..01c01cb3933f 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -168,6 +168,7 @@ enum {
168 VHOST_SCSI_VQ_IO = 2, 168 VHOST_SCSI_VQ_IO = 2,
169}; 169};
170 170
171/* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */
171enum { 172enum {
172 VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) | 173 VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) |
173 (1ULL << VIRTIO_SCSI_F_T10_PI) 174 (1ULL << VIRTIO_SCSI_F_T10_PI)
@@ -577,8 +578,8 @@ tcm_vhost_allocate_evt(struct vhost_scsi *vs,
577 return NULL; 578 return NULL;
578 } 579 }
579 580
580 evt->event.event = event; 581 evt->event.event = cpu_to_vhost32(vq, event);
581 evt->event.reason = reason; 582 evt->event.reason = cpu_to_vhost32(vq, reason);
582 vs->vs_events_nr++; 583 vs->vs_events_nr++;
583 584
584 return evt; 585 return evt;
@@ -636,7 +637,7 @@ again:
636 } 637 }
637 638
638 if (vs->vs_events_missed) { 639 if (vs->vs_events_missed) {
639 event->event |= VIRTIO_SCSI_T_EVENTS_MISSED; 640 event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);
640 vs->vs_events_missed = false; 641 vs->vs_events_missed = false;
641 } 642 }
642 643
@@ -695,12 +696,13 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
695 cmd, se_cmd->residual_count, se_cmd->scsi_status); 696 cmd, se_cmd->residual_count, se_cmd->scsi_status);
696 697
697 memset(&v_rsp, 0, sizeof(v_rsp)); 698 memset(&v_rsp, 0, sizeof(v_rsp));
698 v_rsp.resid = se_cmd->residual_count; 699 v_rsp.resid = cpu_to_vhost32(cmd->tvc_vq, se_cmd->residual_count);
699 /* TODO is status_qualifier field needed? */ 700 /* TODO is status_qualifier field needed? */
700 v_rsp.status = se_cmd->scsi_status; 701 v_rsp.status = se_cmd->scsi_status;
701 v_rsp.sense_len = se_cmd->scsi_sense_length; 702 v_rsp.sense_len = cpu_to_vhost32(cmd->tvc_vq,
703 se_cmd->scsi_sense_length);
702 memcpy(v_rsp.sense, cmd->tvc_sense_buf, 704 memcpy(v_rsp.sense, cmd->tvc_sense_buf,
703 v_rsp.sense_len); 705 se_cmd->scsi_sense_length);
704 ret = copy_to_user(cmd->tvc_resp, &v_rsp, sizeof(v_rsp)); 706 ret = copy_to_user(cmd->tvc_resp, &v_rsp, sizeof(v_rsp));
705 if (likely(ret == 0)) { 707 if (likely(ret == 0)) {
706 struct vhost_scsi_virtqueue *q; 708 struct vhost_scsi_virtqueue *q;
@@ -1095,14 +1097,14 @@ vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1095 ", but wrong data_direction\n"); 1097 ", but wrong data_direction\n");
1096 goto err_cmd; 1098 goto err_cmd;
1097 } 1099 }
1098 prot_bytes = v_req_pi.pi_bytesout; 1100 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);
1099 } else if (v_req_pi.pi_bytesin) { 1101 } else if (v_req_pi.pi_bytesin) {
1100 if (data_direction != DMA_FROM_DEVICE) { 1102 if (data_direction != DMA_FROM_DEVICE) {
1101 vq_err(vq, "Received non zero di_pi_niov" 1103 vq_err(vq, "Received non zero di_pi_niov"
1102 ", but wrong data_direction\n"); 1104 ", but wrong data_direction\n");
1103 goto err_cmd; 1105 goto err_cmd;
1104 } 1106 }
1105 prot_bytes = v_req_pi.pi_bytesin; 1107 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);
1106 } 1108 }
1107 if (prot_bytes) { 1109 if (prot_bytes) {
1108 int tmp = 0; 1110 int tmp = 0;
@@ -1117,12 +1119,12 @@ vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1117 data_first += prot_niov; 1119 data_first += prot_niov;
1118 data_niov = data_num - prot_niov; 1120 data_niov = data_num - prot_niov;
1119 } 1121 }
1120 tag = v_req_pi.tag; 1122 tag = vhost64_to_cpu(vq, v_req_pi.tag);
1121 task_attr = v_req_pi.task_attr; 1123 task_attr = v_req_pi.task_attr;
1122 cdb = &v_req_pi.cdb[0]; 1124 cdb = &v_req_pi.cdb[0];
1123 lun = ((v_req_pi.lun[2] << 8) | v_req_pi.lun[3]) & 0x3FFF; 1125 lun = ((v_req_pi.lun[2] << 8) | v_req_pi.lun[3]) & 0x3FFF;
1124 } else { 1126 } else {
1125 tag = v_req.tag; 1127 tag = vhost64_to_cpu(vq, v_req.tag);
1126 task_attr = v_req.task_attr; 1128 task_attr = v_req.task_attr;
1127 cdb = &v_req.cdb[0]; 1129 cdb = &v_req.cdb[0];
1128 lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF; 1130 lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF;
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index c90f4374442a..ed71b5347a76 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -33,8 +33,8 @@ enum {
33 VHOST_MEMORY_F_LOG = 0x1, 33 VHOST_MEMORY_F_LOG = 0x1,
34}; 34};
35 35
36#define vhost_used_event(vq) ((u16 __user *)&vq->avail->ring[vq->num]) 36#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
37#define vhost_avail_event(vq) ((u16 __user *)&vq->used->ring[vq->num]) 37#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
38 38
39static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh, 39static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
40 poll_table *pt) 40 poll_table *pt)
@@ -1001,7 +1001,7 @@ EXPORT_SYMBOL_GPL(vhost_log_write);
1001static int vhost_update_used_flags(struct vhost_virtqueue *vq) 1001static int vhost_update_used_flags(struct vhost_virtqueue *vq)
1002{ 1002{
1003 void __user *used; 1003 void __user *used;
1004 if (__put_user(vq->used_flags, &vq->used->flags) < 0) 1004 if (__put_user(cpu_to_vhost16(vq, vq->used_flags), &vq->used->flags) < 0)
1005 return -EFAULT; 1005 return -EFAULT;
1006 if (unlikely(vq->log_used)) { 1006 if (unlikely(vq->log_used)) {
1007 /* Make sure the flag is seen before log. */ 1007 /* Make sure the flag is seen before log. */
@@ -1019,7 +1019,7 @@ static int vhost_update_used_flags(struct vhost_virtqueue *vq)
1019 1019
1020static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event) 1020static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)
1021{ 1021{
1022 if (__put_user(vq->avail_idx, vhost_avail_event(vq))) 1022 if (__put_user(cpu_to_vhost16(vq, vq->avail_idx), vhost_avail_event(vq)))
1023 return -EFAULT; 1023 return -EFAULT;
1024 if (unlikely(vq->log_used)) { 1024 if (unlikely(vq->log_used)) {
1025 void __user *used; 1025 void __user *used;
@@ -1038,6 +1038,7 @@ static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)
1038 1038
1039int vhost_init_used(struct vhost_virtqueue *vq) 1039int vhost_init_used(struct vhost_virtqueue *vq)
1040{ 1040{
1041 __virtio16 last_used_idx;
1041 int r; 1042 int r;
1042 if (!vq->private_data) 1043 if (!vq->private_data)
1043 return 0; 1044 return 0;
@@ -1046,7 +1047,13 @@ int vhost_init_used(struct vhost_virtqueue *vq)
1046 if (r) 1047 if (r)
1047 return r; 1048 return r;
1048 vq->signalled_used_valid = false; 1049 vq->signalled_used_valid = false;
1049 return get_user(vq->last_used_idx, &vq->used->idx); 1050 if (!access_ok(VERIFY_READ, &vq->used->idx, sizeof vq->used->idx))
1051 return -EFAULT;
1052 r = __get_user(last_used_idx, &vq->used->idx);
1053 if (r)
1054 return r;
1055 vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx);
1056 return 0;
1050} 1057}
1051EXPORT_SYMBOL_GPL(vhost_init_used); 1058EXPORT_SYMBOL_GPL(vhost_init_used);
1052 1059
@@ -1087,16 +1094,16 @@ static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
1087/* Each buffer in the virtqueues is actually a chain of descriptors. This 1094/* Each buffer in the virtqueues is actually a chain of descriptors. This
1088 * function returns the next descriptor in the chain, 1095 * function returns the next descriptor in the chain,
1089 * or -1U if we're at the end. */ 1096 * or -1U if we're at the end. */
1090static unsigned next_desc(struct vring_desc *desc) 1097static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc)
1091{ 1098{
1092 unsigned int next; 1099 unsigned int next;
1093 1100
1094 /* If this descriptor says it doesn't chain, we're done. */ 1101 /* If this descriptor says it doesn't chain, we're done. */
1095 if (!(desc->flags & VRING_DESC_F_NEXT)) 1102 if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT)))
1096 return -1U; 1103 return -1U;
1097 1104
1098 /* Check they're not leading us off end of descriptors. */ 1105 /* Check they're not leading us off end of descriptors. */
1099 next = desc->next; 1106 next = vhost16_to_cpu(vq, desc->next);
1100 /* Make sure compiler knows to grab that: we don't want it changing! */ 1107 /* Make sure compiler knows to grab that: we don't want it changing! */
1101 /* We will use the result as an index in an array, so most 1108 /* We will use the result as an index in an array, so most
1102 * architectures only need a compiler barrier here. */ 1109 * architectures only need a compiler barrier here. */
@@ -1113,18 +1120,19 @@ static int get_indirect(struct vhost_virtqueue *vq,
1113{ 1120{
1114 struct vring_desc desc; 1121 struct vring_desc desc;
1115 unsigned int i = 0, count, found = 0; 1122 unsigned int i = 0, count, found = 0;
1123 u32 len = vhost32_to_cpu(vq, indirect->len);
1116 int ret; 1124 int ret;
1117 1125
1118 /* Sanity check */ 1126 /* Sanity check */
1119 if (unlikely(indirect->len % sizeof desc)) { 1127 if (unlikely(len % sizeof desc)) {
1120 vq_err(vq, "Invalid length in indirect descriptor: " 1128 vq_err(vq, "Invalid length in indirect descriptor: "
1121 "len 0x%llx not multiple of 0x%zx\n", 1129 "len 0x%llx not multiple of 0x%zx\n",
1122 (unsigned long long)indirect->len, 1130 (unsigned long long)len,
1123 sizeof desc); 1131 sizeof desc);
1124 return -EINVAL; 1132 return -EINVAL;
1125 } 1133 }
1126 1134
1127 ret = translate_desc(vq, indirect->addr, indirect->len, vq->indirect, 1135 ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect,
1128 UIO_MAXIOV); 1136 UIO_MAXIOV);
1129 if (unlikely(ret < 0)) { 1137 if (unlikely(ret < 0)) {
1130 vq_err(vq, "Translation failure %d in indirect.\n", ret); 1138 vq_err(vq, "Translation failure %d in indirect.\n", ret);
@@ -1135,7 +1143,7 @@ static int get_indirect(struct vhost_virtqueue *vq,
1135 * architectures only need a compiler barrier here. */ 1143 * architectures only need a compiler barrier here. */
1136 read_barrier_depends(); 1144 read_barrier_depends();
1137 1145
1138 count = indirect->len / sizeof desc; 1146 count = len / sizeof desc;
1139 /* Buffers are chained via a 16 bit next field, so 1147 /* Buffers are chained via a 16 bit next field, so
1140 * we can have at most 2^16 of these. */ 1148 * we can have at most 2^16 of these. */
1141 if (unlikely(count > USHRT_MAX + 1)) { 1149 if (unlikely(count > USHRT_MAX + 1)) {
@@ -1155,16 +1163,17 @@ static int get_indirect(struct vhost_virtqueue *vq,
1155 if (unlikely(memcpy_fromiovec((unsigned char *)&desc, 1163 if (unlikely(memcpy_fromiovec((unsigned char *)&desc,
1156 vq->indirect, sizeof desc))) { 1164 vq->indirect, sizeof desc))) {
1157 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n", 1165 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
1158 i, (size_t)indirect->addr + i * sizeof desc); 1166 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
1159 return -EINVAL; 1167 return -EINVAL;
1160 } 1168 }
1161 if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) { 1169 if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) {
1162 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n", 1170 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
1163 i, (size_t)indirect->addr + i * sizeof desc); 1171 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
1164 return -EINVAL; 1172 return -EINVAL;
1165 } 1173 }
1166 1174
1167 ret = translate_desc(vq, desc.addr, desc.len, iov + iov_count, 1175 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
1176 vhost32_to_cpu(vq, desc.len), iov + iov_count,
1168 iov_size - iov_count); 1177 iov_size - iov_count);
1169 if (unlikely(ret < 0)) { 1178 if (unlikely(ret < 0)) {
1170 vq_err(vq, "Translation failure %d indirect idx %d\n", 1179 vq_err(vq, "Translation failure %d indirect idx %d\n",
@@ -1172,11 +1181,11 @@ static int get_indirect(struct vhost_virtqueue *vq,
1172 return ret; 1181 return ret;
1173 } 1182 }
1174 /* If this is an input descriptor, increment that count. */ 1183 /* If this is an input descriptor, increment that count. */
1175 if (desc.flags & VRING_DESC_F_WRITE) { 1184 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) {
1176 *in_num += ret; 1185 *in_num += ret;
1177 if (unlikely(log)) { 1186 if (unlikely(log)) {
1178 log[*log_num].addr = desc.addr; 1187 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
1179 log[*log_num].len = desc.len; 1188 log[*log_num].len = vhost32_to_cpu(vq, desc.len);
1180 ++*log_num; 1189 ++*log_num;
1181 } 1190 }
1182 } else { 1191 } else {
@@ -1189,7 +1198,7 @@ static int get_indirect(struct vhost_virtqueue *vq,
1189 } 1198 }
1190 *out_num += ret; 1199 *out_num += ret;
1191 } 1200 }
1192 } while ((i = next_desc(&desc)) != -1); 1201 } while ((i = next_desc(vq, &desc)) != -1);
1193 return 0; 1202 return 0;
1194} 1203}
1195 1204
@@ -1209,15 +1218,18 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq,
1209 struct vring_desc desc; 1218 struct vring_desc desc;
1210 unsigned int i, head, found = 0; 1219 unsigned int i, head, found = 0;
1211 u16 last_avail_idx; 1220 u16 last_avail_idx;
1221 __virtio16 avail_idx;
1222 __virtio16 ring_head;
1212 int ret; 1223 int ret;
1213 1224
1214 /* Check it isn't doing very strange things with descriptor numbers. */ 1225 /* Check it isn't doing very strange things with descriptor numbers. */
1215 last_avail_idx = vq->last_avail_idx; 1226 last_avail_idx = vq->last_avail_idx;
1216 if (unlikely(__get_user(vq->avail_idx, &vq->avail->idx))) { 1227 if (unlikely(__get_user(avail_idx, &vq->avail->idx))) {
1217 vq_err(vq, "Failed to access avail idx at %p\n", 1228 vq_err(vq, "Failed to access avail idx at %p\n",
1218 &vq->avail->idx); 1229 &vq->avail->idx);
1219 return -EFAULT; 1230 return -EFAULT;
1220 } 1231 }
1232 vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
1221 1233
1222 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { 1234 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
1223 vq_err(vq, "Guest moved used index from %u to %u", 1235 vq_err(vq, "Guest moved used index from %u to %u",
@@ -1234,7 +1246,7 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq,
1234 1246
1235 /* Grab the next descriptor number they're advertising, and increment 1247 /* Grab the next descriptor number they're advertising, and increment
1236 * the index we've seen. */ 1248 * the index we've seen. */
1237 if (unlikely(__get_user(head, 1249 if (unlikely(__get_user(ring_head,
1238 &vq->avail->ring[last_avail_idx % vq->num]))) { 1250 &vq->avail->ring[last_avail_idx % vq->num]))) {
1239 vq_err(vq, "Failed to read head: idx %d address %p\n", 1251 vq_err(vq, "Failed to read head: idx %d address %p\n",
1240 last_avail_idx, 1252 last_avail_idx,
@@ -1242,6 +1254,8 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq,
1242 return -EFAULT; 1254 return -EFAULT;
1243 } 1255 }
1244 1256
1257 head = vhost16_to_cpu(vq, ring_head);
1258
1245 /* If their number is silly, that's an error. */ 1259 /* If their number is silly, that's an error. */
1246 if (unlikely(head >= vq->num)) { 1260 if (unlikely(head >= vq->num)) {
1247 vq_err(vq, "Guest says index %u > %u is available", 1261 vq_err(vq, "Guest says index %u > %u is available",
@@ -1274,7 +1288,7 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq,
1274 i, vq->desc + i); 1288 i, vq->desc + i);
1275 return -EFAULT; 1289 return -EFAULT;
1276 } 1290 }
1277 if (desc.flags & VRING_DESC_F_INDIRECT) { 1291 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) {
1278 ret = get_indirect(vq, iov, iov_size, 1292 ret = get_indirect(vq, iov, iov_size,
1279 out_num, in_num, 1293 out_num, in_num,
1280 log, log_num, &desc); 1294 log, log_num, &desc);
@@ -1286,20 +1300,21 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq,
1286 continue; 1300 continue;
1287 } 1301 }
1288 1302
1289 ret = translate_desc(vq, desc.addr, desc.len, iov + iov_count, 1303 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
1304 vhost32_to_cpu(vq, desc.len), iov + iov_count,
1290 iov_size - iov_count); 1305 iov_size - iov_count);
1291 if (unlikely(ret < 0)) { 1306 if (unlikely(ret < 0)) {
1292 vq_err(vq, "Translation failure %d descriptor idx %d\n", 1307 vq_err(vq, "Translation failure %d descriptor idx %d\n",
1293 ret, i); 1308 ret, i);
1294 return ret; 1309 return ret;
1295 } 1310 }
1296 if (desc.flags & VRING_DESC_F_WRITE) { 1311 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) {
1297 /* If this is an input descriptor, 1312 /* If this is an input descriptor,
1298 * increment that count. */ 1313 * increment that count. */
1299 *in_num += ret; 1314 *in_num += ret;
1300 if (unlikely(log)) { 1315 if (unlikely(log)) {
1301 log[*log_num].addr = desc.addr; 1316 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
1302 log[*log_num].len = desc.len; 1317 log[*log_num].len = vhost32_to_cpu(vq, desc.len);
1303 ++*log_num; 1318 ++*log_num;
1304 } 1319 }
1305 } else { 1320 } else {
@@ -1312,7 +1327,7 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq,
1312 } 1327 }
1313 *out_num += ret; 1328 *out_num += ret;
1314 } 1329 }
1315 } while ((i = next_desc(&desc)) != -1); 1330 } while ((i = next_desc(vq, &desc)) != -1);
1316 1331
1317 /* On success, increment avail index. */ 1332 /* On success, increment avail index. */
1318 vq->last_avail_idx++; 1333 vq->last_avail_idx++;
@@ -1335,7 +1350,10 @@ EXPORT_SYMBOL_GPL(vhost_discard_vq_desc);
1335 * want to notify the guest, using eventfd. */ 1350 * want to notify the guest, using eventfd. */
1336int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len) 1351int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
1337{ 1352{
1338 struct vring_used_elem heads = { head, len }; 1353 struct vring_used_elem heads = {
1354 cpu_to_vhost32(vq, head),
1355 cpu_to_vhost32(vq, len)
1356 };
1339 1357
1340 return vhost_add_used_n(vq, &heads, 1); 1358 return vhost_add_used_n(vq, &heads, 1);
1341} 1359}
@@ -1404,7 +1422,7 @@ int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
1404 1422
1405 /* Make sure buffer is written before we update index. */ 1423 /* Make sure buffer is written before we update index. */
1406 smp_wmb(); 1424 smp_wmb();
1407 if (put_user(vq->last_used_idx, &vq->used->idx)) { 1425 if (__put_user(cpu_to_vhost16(vq, vq->last_used_idx), &vq->used->idx)) {
1408 vq_err(vq, "Failed to increment used idx"); 1426 vq_err(vq, "Failed to increment used idx");
1409 return -EFAULT; 1427 return -EFAULT;
1410 } 1428 }
@@ -1422,7 +1440,8 @@ EXPORT_SYMBOL_GPL(vhost_add_used_n);
1422 1440
1423static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 1441static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1424{ 1442{
1425 __u16 old, new, event; 1443 __u16 old, new;
1444 __virtio16 event;
1426 bool v; 1445 bool v;
1427 /* Flush out used index updates. This is paired 1446 /* Flush out used index updates. This is paired
1428 * with the barrier that the Guest executes when enabling 1447 * with the barrier that the Guest executes when enabling
@@ -1434,12 +1453,12 @@ static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1434 return true; 1453 return true;
1435 1454
1436 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 1455 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
1437 __u16 flags; 1456 __virtio16 flags;
1438 if (__get_user(flags, &vq->avail->flags)) { 1457 if (__get_user(flags, &vq->avail->flags)) {
1439 vq_err(vq, "Failed to get flags"); 1458 vq_err(vq, "Failed to get flags");
1440 return true; 1459 return true;
1441 } 1460 }
1442 return !(flags & VRING_AVAIL_F_NO_INTERRUPT); 1461 return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT));
1443 } 1462 }
1444 old = vq->signalled_used; 1463 old = vq->signalled_used;
1445 v = vq->signalled_used_valid; 1464 v = vq->signalled_used_valid;
@@ -1449,11 +1468,11 @@ static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1449 if (unlikely(!v)) 1468 if (unlikely(!v))
1450 return true; 1469 return true;
1451 1470
1452 if (get_user(event, vhost_used_event(vq))) { 1471 if (__get_user(event, vhost_used_event(vq))) {
1453 vq_err(vq, "Failed to get used event idx"); 1472 vq_err(vq, "Failed to get used event idx");
1454 return true; 1473 return true;
1455 } 1474 }
1456 return vring_need_event(event, new, old); 1475 return vring_need_event(vhost16_to_cpu(vq, event), new, old);
1457} 1476}
1458 1477
1459/* This actually signals the guest, using eventfd. */ 1478/* This actually signals the guest, using eventfd. */
@@ -1488,7 +1507,7 @@ EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
1488/* OK, now we need to know about added descriptors. */ 1507/* OK, now we need to know about added descriptors. */
1489bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 1508bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1490{ 1509{
1491 u16 avail_idx; 1510 __virtio16 avail_idx;
1492 int r; 1511 int r;
1493 1512
1494 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) 1513 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
@@ -1519,7 +1538,7 @@ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1519 return false; 1538 return false;
1520 } 1539 }
1521 1540
1522 return avail_idx != vq->avail_idx; 1541 return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx;
1523} 1542}
1524EXPORT_SYMBOL_GPL(vhost_enable_notify); 1543EXPORT_SYMBOL_GPL(vhost_enable_notify);
1525 1544
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 3eda654b8f5a..8c1c792900ba 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -12,8 +12,6 @@
12#include <linux/virtio_ring.h> 12#include <linux/virtio_ring.h>
13#include <linux/atomic.h> 13#include <linux/atomic.h>
14 14
15struct vhost_device;
16
17struct vhost_work; 15struct vhost_work;
18typedef void (*vhost_work_fn_t)(struct vhost_work *work); 16typedef void (*vhost_work_fn_t)(struct vhost_work *work);
19 17
@@ -54,8 +52,6 @@ struct vhost_log {
54 u64 len; 52 u64 len;
55}; 53};
56 54
57struct vhost_virtqueue;
58
59/* The virtqueue structure describes a queue attached to a device. */ 55/* The virtqueue structure describes a queue attached to a device. */
60struct vhost_virtqueue { 56struct vhost_virtqueue {
61 struct vhost_dev *dev; 57 struct vhost_dev *dev;
@@ -106,7 +102,7 @@ struct vhost_virtqueue {
106 /* Protected by virtqueue mutex. */ 102 /* Protected by virtqueue mutex. */
107 struct vhost_memory *memory; 103 struct vhost_memory *memory;
108 void *private_data; 104 void *private_data;
109 unsigned acked_features; 105 u64 acked_features;
110 /* Log write descriptors */ 106 /* Log write descriptors */
111 void __user *log_base; 107 void __user *log_base;
112 struct vhost_log *log; 108 struct vhost_log *log;
@@ -172,8 +168,39 @@ enum {
172 (1ULL << VHOST_F_LOG_ALL), 168 (1ULL << VHOST_F_LOG_ALL),
173}; 169};
174 170
175static inline int vhost_has_feature(struct vhost_virtqueue *vq, int bit) 171static inline bool vhost_has_feature(struct vhost_virtqueue *vq, int bit)
172{
173 return vq->acked_features & (1ULL << bit);
174}
175
176/* Memory accessors */
177static inline u16 vhost16_to_cpu(struct vhost_virtqueue *vq, __virtio16 val)
178{
179 return __virtio16_to_cpu(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val);
180}
181
182static inline __virtio16 cpu_to_vhost16(struct vhost_virtqueue *vq, u16 val)
183{
184 return __cpu_to_virtio16(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val);
185}
186
187static inline u32 vhost32_to_cpu(struct vhost_virtqueue *vq, __virtio32 val)
188{
189 return __virtio32_to_cpu(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val);
190}
191
192static inline __virtio32 cpu_to_vhost32(struct vhost_virtqueue *vq, u32 val)
193{
194 return __cpu_to_virtio32(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val);
195}
196
197static inline u64 vhost64_to_cpu(struct vhost_virtqueue *vq, __virtio64 val)
198{
199 return __virtio64_to_cpu(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val);
200}
201
202static inline __virtio64 cpu_to_vhost64(struct vhost_virtqueue *vq, u64 val)
176{ 203{
177 return vq->acked_features & (1 << bit); 204 return __cpu_to_virtio64(vhost_has_feature(vq, VIRTIO_F_VERSION_1), val);
178} 205}
179#endif 206#endif
diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile
index 9076635697bb..bf5104b56894 100644
--- a/drivers/virtio/Makefile
+++ b/drivers/virtio/Makefile
@@ -1,4 +1,5 @@
1obj-$(CONFIG_VIRTIO) += virtio.o virtio_ring.o 1obj-$(CONFIG_VIRTIO) += virtio.o virtio_ring.o
2obj-$(CONFIG_VIRTIO_MMIO) += virtio_mmio.o 2obj-$(CONFIG_VIRTIO_MMIO) += virtio_mmio.o
3obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o 3obj-$(CONFIG_VIRTIO_PCI) += virtio_pci.o
4virtio_pci-y := virtio_pci_legacy.o virtio_pci_common.o
4obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o 5obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o
diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
index df598dd8c5c8..f22665868781 100644
--- a/drivers/virtio/virtio.c
+++ b/drivers/virtio/virtio.c
@@ -3,6 +3,7 @@
3#include <linux/virtio_config.h> 3#include <linux/virtio_config.h>
4#include <linux/module.h> 4#include <linux/module.h>
5#include <linux/idr.h> 5#include <linux/idr.h>
6#include <uapi/linux/virtio_ids.h>
6 7
7/* Unique numbering for virtio devices. */ 8/* Unique numbering for virtio devices. */
8static DEFINE_IDA(virtio_index_ida); 9static DEFINE_IDA(virtio_index_ida);
@@ -49,9 +50,9 @@ static ssize_t features_show(struct device *_d,
49 50
50 /* We actually represent this as a bitstring, as it could be 51 /* We actually represent this as a bitstring, as it could be
51 * arbitrary length in future. */ 52 * arbitrary length in future. */
52 for (i = 0; i < ARRAY_SIZE(dev->features)*BITS_PER_LONG; i++) 53 for (i = 0; i < sizeof(dev->features)*8; i++)
53 len += sprintf(buf+len, "%c", 54 len += sprintf(buf+len, "%c",
54 test_bit(i, dev->features) ? '1' : '0'); 55 __virtio_test_bit(dev, i) ? '1' : '0');
55 len += sprintf(buf+len, "\n"); 56 len += sprintf(buf+len, "\n");
56 return len; 57 return len;
57} 58}
@@ -113,6 +114,13 @@ void virtio_check_driver_offered_feature(const struct virtio_device *vdev,
113 for (i = 0; i < drv->feature_table_size; i++) 114 for (i = 0; i < drv->feature_table_size; i++)
114 if (drv->feature_table[i] == fbit) 115 if (drv->feature_table[i] == fbit)
115 return; 116 return;
117
118 if (drv->feature_table_legacy) {
119 for (i = 0; i < drv->feature_table_size_legacy; i++)
120 if (drv->feature_table_legacy[i] == fbit)
121 return;
122 }
123
116 BUG(); 124 BUG();
117} 125}
118EXPORT_SYMBOL_GPL(virtio_check_driver_offered_feature); 126EXPORT_SYMBOL_GPL(virtio_check_driver_offered_feature);
@@ -159,7 +167,10 @@ static int virtio_dev_probe(struct device *_d)
159 int err, i; 167 int err, i;
160 struct virtio_device *dev = dev_to_virtio(_d); 168 struct virtio_device *dev = dev_to_virtio(_d);
161 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); 169 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
162 u32 device_features; 170 u64 device_features;
171 u64 driver_features;
172 u64 driver_features_legacy;
173 unsigned status;
163 174
164 /* We have a driver! */ 175 /* We have a driver! */
165 add_status(dev, VIRTIO_CONFIG_S_DRIVER); 176 add_status(dev, VIRTIO_CONFIG_S_DRIVER);
@@ -167,34 +178,66 @@ static int virtio_dev_probe(struct device *_d)
167 /* Figure out what features the device supports. */ 178 /* Figure out what features the device supports. */
168 device_features = dev->config->get_features(dev); 179 device_features = dev->config->get_features(dev);
169 180
170 /* Features supported by both device and driver into dev->features. */ 181 /* Figure out what features the driver supports. */
171 memset(dev->features, 0, sizeof(dev->features)); 182 driver_features = 0;
172 for (i = 0; i < drv->feature_table_size; i++) { 183 for (i = 0; i < drv->feature_table_size; i++) {
173 unsigned int f = drv->feature_table[i]; 184 unsigned int f = drv->feature_table[i];
174 BUG_ON(f >= 32); 185 BUG_ON(f >= 64);
175 if (device_features & (1 << f)) 186 driver_features |= (1ULL << f);
176 set_bit(f, dev->features); 187 }
188
189 /* Some drivers have a separate feature table for virtio v1.0 */
190 if (drv->feature_table_legacy) {
191 driver_features_legacy = 0;
192 for (i = 0; i < drv->feature_table_size_legacy; i++) {
193 unsigned int f = drv->feature_table_legacy[i];
194 BUG_ON(f >= 64);
195 driver_features_legacy |= (1ULL << f);
196 }
197 } else {
198 driver_features_legacy = driver_features;
177 } 199 }
178 200
201 if (device_features & (1ULL << VIRTIO_F_VERSION_1))
202 dev->features = driver_features & device_features;
203 else
204 dev->features = driver_features_legacy & device_features;
205
179 /* Transport features always preserved to pass to finalize_features. */ 206 /* Transport features always preserved to pass to finalize_features. */
180 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) 207 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++)
181 if (device_features & (1 << i)) 208 if (device_features & (1ULL << i))
182 set_bit(i, dev->features); 209 __virtio_set_bit(dev, i);
183 210
184 dev->config->finalize_features(dev); 211 err = dev->config->finalize_features(dev);
212 if (err)
213 goto err;
214
215 if (virtio_has_feature(dev, VIRTIO_F_VERSION_1)) {
216 add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
217 status = dev->config->get_status(dev);
218 if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
219 dev_err(_d, "virtio: device refuses features: %x\n",
220 status);
221 err = -ENODEV;
222 goto err;
223 }
224 }
185 225
186 err = drv->probe(dev); 226 err = drv->probe(dev);
187 if (err) 227 if (err)
188 add_status(dev, VIRTIO_CONFIG_S_FAILED); 228 goto err;
189 else {
190 add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
191 if (drv->scan)
192 drv->scan(dev);
193 229
194 virtio_config_enable(dev); 230 add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
195 } 231 if (drv->scan)
232 drv->scan(dev);
233
234 virtio_config_enable(dev);
196 235
236 return 0;
237err:
238 add_status(dev, VIRTIO_CONFIG_S_FAILED);
197 return err; 239 return err;
240
198} 241}
199 242
200static int virtio_dev_remove(struct device *_d) 243static int virtio_dev_remove(struct device *_d)
@@ -223,6 +266,12 @@ static struct bus_type virtio_bus = {
223 .remove = virtio_dev_remove, 266 .remove = virtio_dev_remove,
224}; 267};
225 268
269bool virtio_device_is_legacy_only(struct virtio_device_id id)
270{
271 return id.device == VIRTIO_ID_BALLOON;
272}
273EXPORT_SYMBOL_GPL(virtio_device_is_legacy_only);
274
226int register_virtio_driver(struct virtio_driver *driver) 275int register_virtio_driver(struct virtio_driver *driver)
227{ 276{
228 /* Catch this early. */ 277 /* Catch this early. */
@@ -303,6 +352,7 @@ EXPORT_SYMBOL_GPL(virtio_device_freeze);
303int virtio_device_restore(struct virtio_device *dev) 352int virtio_device_restore(struct virtio_device *dev)
304{ 353{
305 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); 354 struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
355 int ret;
306 356
307 /* We always start by resetting the device, in case a previous 357 /* We always start by resetting the device, in case a previous
308 * driver messed it up. */ 358 * driver messed it up. */
@@ -322,14 +372,14 @@ int virtio_device_restore(struct virtio_device *dev)
322 /* We have a driver! */ 372 /* We have a driver! */
323 add_status(dev, VIRTIO_CONFIG_S_DRIVER); 373 add_status(dev, VIRTIO_CONFIG_S_DRIVER);
324 374
325 dev->config->finalize_features(dev); 375 ret = dev->config->finalize_features(dev);
376 if (ret)
377 goto err;
326 378
327 if (drv->restore) { 379 if (drv->restore) {
328 int ret = drv->restore(dev); 380 ret = drv->restore(dev);
329 if (ret) { 381 if (ret)
330 add_status(dev, VIRTIO_CONFIG_S_FAILED); 382 goto err;
331 return ret;
332 }
333 } 383 }
334 384
335 /* Finally, tell the device we're all set */ 385 /* Finally, tell the device we're all set */
@@ -338,6 +388,10 @@ int virtio_device_restore(struct virtio_device *dev)
338 virtio_config_enable(dev); 388 virtio_config_enable(dev);
339 389
340 return 0; 390 return 0;
391
392err:
393 add_status(dev, VIRTIO_CONFIG_S_FAILED);
394 return ret;
341} 395}
342EXPORT_SYMBOL_GPL(virtio_device_restore); 396EXPORT_SYMBOL_GPL(virtio_device_restore);
343#endif 397#endif
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index ef9a1650bb80..5219210d31ce 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -142,7 +142,7 @@ struct virtio_mmio_vq_info {
142 142
143/* Configuration interface */ 143/* Configuration interface */
144 144
145static u32 vm_get_features(struct virtio_device *vdev) 145static u64 vm_get_features(struct virtio_device *vdev)
146{ 146{
147 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); 147 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
148 148
@@ -152,19 +152,20 @@ static u32 vm_get_features(struct virtio_device *vdev)
152 return readl(vm_dev->base + VIRTIO_MMIO_HOST_FEATURES); 152 return readl(vm_dev->base + VIRTIO_MMIO_HOST_FEATURES);
153} 153}
154 154
155static void vm_finalize_features(struct virtio_device *vdev) 155static int vm_finalize_features(struct virtio_device *vdev)
156{ 156{
157 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); 157 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
158 int i;
159 158
160 /* Give virtio_ring a chance to accept features. */ 159 /* Give virtio_ring a chance to accept features. */
161 vring_transport_features(vdev); 160 vring_transport_features(vdev);
162 161
163 for (i = 0; i < ARRAY_SIZE(vdev->features); i++) { 162 /* Make sure we don't have any features > 32 bits! */
164 writel(i, vm_dev->base + VIRTIO_MMIO_GUEST_FEATURES_SEL); 163 BUG_ON((u32)vdev->features != vdev->features);
165 writel(vdev->features[i], 164
166 vm_dev->base + VIRTIO_MMIO_GUEST_FEATURES); 165 writel(0, vm_dev->base + VIRTIO_MMIO_GUEST_FEATURES_SEL);
167 } 166 writel(vdev->features, vm_dev->base + VIRTIO_MMIO_GUEST_FEATURES);
167
168 return 0;
168} 169}
169 170
170static void vm_get(struct virtio_device *vdev, unsigned offset, 171static void vm_get(struct virtio_device *vdev, unsigned offset,
diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c
deleted file mode 100644
index d34ebfa604f3..000000000000
--- a/drivers/virtio/virtio_pci.c
+++ /dev/null
@@ -1,802 +0,0 @@
1/*
2 * Virtio PCI driver
3 *
4 * This module allows virtio devices to be used over a virtual PCI device.
5 * This can be used with QEMU based VMMs like KVM or Xen.
6 *
7 * Copyright IBM Corp. 2007
8 *
9 * Authors:
10 * Anthony Liguori <aliguori@us.ibm.com>
11 *
12 * This work is licensed under the terms of the GNU GPL, version 2 or later.
13 * See the COPYING file in the top-level directory.
14 *
15 */
16
17#include <linux/module.h>
18#include <linux/list.h>
19#include <linux/pci.h>
20#include <linux/slab.h>
21#include <linux/interrupt.h>
22#include <linux/virtio.h>
23#include <linux/virtio_config.h>
24#include <linux/virtio_ring.h>
25#include <linux/virtio_pci.h>
26#include <linux/highmem.h>
27#include <linux/spinlock.h>
28
29MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
30MODULE_DESCRIPTION("virtio-pci");
31MODULE_LICENSE("GPL");
32MODULE_VERSION("1");
33
34/* Our device structure */
35struct virtio_pci_device
36{
37 struct virtio_device vdev;
38 struct pci_dev *pci_dev;
39
40 /* the IO mapping for the PCI config space */
41 void __iomem *ioaddr;
42
43 /* a list of queues so we can dispatch IRQs */
44 spinlock_t lock;
45 struct list_head virtqueues;
46
47 /* MSI-X support */
48 int msix_enabled;
49 int intx_enabled;
50 struct msix_entry *msix_entries;
51 cpumask_var_t *msix_affinity_masks;
52 /* Name strings for interrupts. This size should be enough,
53 * and I'm too lazy to allocate each name separately. */
54 char (*msix_names)[256];
55 /* Number of available vectors */
56 unsigned msix_vectors;
57 /* Vectors allocated, excluding per-vq vectors if any */
58 unsigned msix_used_vectors;
59
60 /* Whether we have vector per vq */
61 bool per_vq_vectors;
62};
63
64/* Constants for MSI-X */
65/* Use first vector for configuration changes, second and the rest for
66 * virtqueues Thus, we need at least 2 vectors for MSI. */
67enum {
68 VP_MSIX_CONFIG_VECTOR = 0,
69 VP_MSIX_VQ_VECTOR = 1,
70};
71
72struct virtio_pci_vq_info
73{
74 /* the actual virtqueue */
75 struct virtqueue *vq;
76
77 /* the number of entries in the queue */
78 int num;
79
80 /* the virtual address of the ring queue */
81 void *queue;
82
83 /* the list node for the virtqueues list */
84 struct list_head node;
85
86 /* MSI-X vector (or none) */
87 unsigned msix_vector;
88};
89
90/* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
91static const struct pci_device_id virtio_pci_id_table[] = {
92 { PCI_DEVICE(0x1af4, PCI_ANY_ID) },
93 { 0 }
94};
95
96MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
97
98/* Convert a generic virtio device to our structure */
99static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
100{
101 return container_of(vdev, struct virtio_pci_device, vdev);
102}
103
104/* virtio config->get_features() implementation */
105static u32 vp_get_features(struct virtio_device *vdev)
106{
107 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
108
109 /* When someone needs more than 32 feature bits, we'll need to
110 * steal a bit to indicate that the rest are somewhere else. */
111 return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES);
112}
113
114/* virtio config->finalize_features() implementation */
115static void vp_finalize_features(struct virtio_device *vdev)
116{
117 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
118
119 /* Give virtio_ring a chance to accept features. */
120 vring_transport_features(vdev);
121
122 /* We only support 32 feature bits. */
123 BUILD_BUG_ON(ARRAY_SIZE(vdev->features) != 1);
124 iowrite32(vdev->features[0], vp_dev->ioaddr+VIRTIO_PCI_GUEST_FEATURES);
125}
126
127/* virtio config->get() implementation */
128static void vp_get(struct virtio_device *vdev, unsigned offset,
129 void *buf, unsigned len)
130{
131 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
132 void __iomem *ioaddr = vp_dev->ioaddr +
133 VIRTIO_PCI_CONFIG(vp_dev) + offset;
134 u8 *ptr = buf;
135 int i;
136
137 for (i = 0; i < len; i++)
138 ptr[i] = ioread8(ioaddr + i);
139}
140
141/* the config->set() implementation. it's symmetric to the config->get()
142 * implementation */
143static void vp_set(struct virtio_device *vdev, unsigned offset,
144 const void *buf, unsigned len)
145{
146 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
147 void __iomem *ioaddr = vp_dev->ioaddr +
148 VIRTIO_PCI_CONFIG(vp_dev) + offset;
149 const u8 *ptr = buf;
150 int i;
151
152 for (i = 0; i < len; i++)
153 iowrite8(ptr[i], ioaddr + i);
154}
155
156/* config->{get,set}_status() implementations */
157static u8 vp_get_status(struct virtio_device *vdev)
158{
159 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
160 return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
161}
162
163static void vp_set_status(struct virtio_device *vdev, u8 status)
164{
165 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
166 /* We should never be setting status to 0. */
167 BUG_ON(status == 0);
168 iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
169}
170
171/* wait for pending irq handlers */
172static void vp_synchronize_vectors(struct virtio_device *vdev)
173{
174 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
175 int i;
176
177 if (vp_dev->intx_enabled)
178 synchronize_irq(vp_dev->pci_dev->irq);
179
180 for (i = 0; i < vp_dev->msix_vectors; ++i)
181 synchronize_irq(vp_dev->msix_entries[i].vector);
182}
183
184static void vp_reset(struct virtio_device *vdev)
185{
186 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
187 /* 0 status means a reset. */
188 iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
189 /* Flush out the status write, and flush in device writes,
190 * including MSi-X interrupts, if any. */
191 ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
192 /* Flush pending VQ/configuration callbacks. */
193 vp_synchronize_vectors(vdev);
194}
195
196/* the notify function used when creating a virt queue */
197static bool vp_notify(struct virtqueue *vq)
198{
199 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
200
201 /* we write the queue's selector into the notification register to
202 * signal the other end */
203 iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY);
204 return true;
205}
206
207/* Handle a configuration change: Tell driver if it wants to know. */
208static irqreturn_t vp_config_changed(int irq, void *opaque)
209{
210 struct virtio_pci_device *vp_dev = opaque;
211
212 virtio_config_changed(&vp_dev->vdev);
213 return IRQ_HANDLED;
214}
215
216/* Notify all virtqueues on an interrupt. */
217static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
218{
219 struct virtio_pci_device *vp_dev = opaque;
220 struct virtio_pci_vq_info *info;
221 irqreturn_t ret = IRQ_NONE;
222 unsigned long flags;
223
224 spin_lock_irqsave(&vp_dev->lock, flags);
225 list_for_each_entry(info, &vp_dev->virtqueues, node) {
226 if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
227 ret = IRQ_HANDLED;
228 }
229 spin_unlock_irqrestore(&vp_dev->lock, flags);
230
231 return ret;
232}
233
234/* A small wrapper to also acknowledge the interrupt when it's handled.
235 * I really need an EIO hook for the vring so I can ack the interrupt once we
236 * know that we'll be handling the IRQ but before we invoke the callback since
237 * the callback may notify the host which results in the host attempting to
238 * raise an interrupt that we would then mask once we acknowledged the
239 * interrupt. */
240static irqreturn_t vp_interrupt(int irq, void *opaque)
241{
242 struct virtio_pci_device *vp_dev = opaque;
243 u8 isr;
244
245 /* reading the ISR has the effect of also clearing it so it's very
246 * important to save off the value. */
247 isr = ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
248
249 /* It's definitely not us if the ISR was not high */
250 if (!isr)
251 return IRQ_NONE;
252
253 /* Configuration change? Tell driver if it wants to know. */
254 if (isr & VIRTIO_PCI_ISR_CONFIG)
255 vp_config_changed(irq, opaque);
256
257 return vp_vring_interrupt(irq, opaque);
258}
259
260static void vp_free_vectors(struct virtio_device *vdev)
261{
262 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
263 int i;
264
265 if (vp_dev->intx_enabled) {
266 free_irq(vp_dev->pci_dev->irq, vp_dev);
267 vp_dev->intx_enabled = 0;
268 }
269
270 for (i = 0; i < vp_dev->msix_used_vectors; ++i)
271 free_irq(vp_dev->msix_entries[i].vector, vp_dev);
272
273 for (i = 0; i < vp_dev->msix_vectors; i++)
274 if (vp_dev->msix_affinity_masks[i])
275 free_cpumask_var(vp_dev->msix_affinity_masks[i]);
276
277 if (vp_dev->msix_enabled) {
278 /* Disable the vector used for configuration */
279 iowrite16(VIRTIO_MSI_NO_VECTOR,
280 vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
281 /* Flush the write out to device */
282 ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
283
284 pci_disable_msix(vp_dev->pci_dev);
285 vp_dev->msix_enabled = 0;
286 }
287
288 vp_dev->msix_vectors = 0;
289 vp_dev->msix_used_vectors = 0;
290 kfree(vp_dev->msix_names);
291 vp_dev->msix_names = NULL;
292 kfree(vp_dev->msix_entries);
293 vp_dev->msix_entries = NULL;
294 kfree(vp_dev->msix_affinity_masks);
295 vp_dev->msix_affinity_masks = NULL;
296}
297
298static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
299 bool per_vq_vectors)
300{
301 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
302 const char *name = dev_name(&vp_dev->vdev.dev);
303 unsigned i, v;
304 int err = -ENOMEM;
305
306 vp_dev->msix_vectors = nvectors;
307
308 vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
309 GFP_KERNEL);
310 if (!vp_dev->msix_entries)
311 goto error;
312 vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
313 GFP_KERNEL);
314 if (!vp_dev->msix_names)
315 goto error;
316 vp_dev->msix_affinity_masks
317 = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
318 GFP_KERNEL);
319 if (!vp_dev->msix_affinity_masks)
320 goto error;
321 for (i = 0; i < nvectors; ++i)
322 if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
323 GFP_KERNEL))
324 goto error;
325
326 for (i = 0; i < nvectors; ++i)
327 vp_dev->msix_entries[i].entry = i;
328
329 err = pci_enable_msix_exact(vp_dev->pci_dev,
330 vp_dev->msix_entries, nvectors);
331 if (err)
332 goto error;
333 vp_dev->msix_enabled = 1;
334
335 /* Set the vector used for configuration */
336 v = vp_dev->msix_used_vectors;
337 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
338 "%s-config", name);
339 err = request_irq(vp_dev->msix_entries[v].vector,
340 vp_config_changed, 0, vp_dev->msix_names[v],
341 vp_dev);
342 if (err)
343 goto error;
344 ++vp_dev->msix_used_vectors;
345
346 iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
347 /* Verify we had enough resources to assign the vector */
348 v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
349 if (v == VIRTIO_MSI_NO_VECTOR) {
350 err = -EBUSY;
351 goto error;
352 }
353
354 if (!per_vq_vectors) {
355 /* Shared vector for all VQs */
356 v = vp_dev->msix_used_vectors;
357 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
358 "%s-virtqueues", name);
359 err = request_irq(vp_dev->msix_entries[v].vector,
360 vp_vring_interrupt, 0, vp_dev->msix_names[v],
361 vp_dev);
362 if (err)
363 goto error;
364 ++vp_dev->msix_used_vectors;
365 }
366 return 0;
367error:
368 vp_free_vectors(vdev);
369 return err;
370}
371
372static int vp_request_intx(struct virtio_device *vdev)
373{
374 int err;
375 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
376
377 err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
378 IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
379 if (!err)
380 vp_dev->intx_enabled = 1;
381 return err;
382}
383
384static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
385 void (*callback)(struct virtqueue *vq),
386 const char *name,
387 u16 msix_vec)
388{
389 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
390 struct virtio_pci_vq_info *info;
391 struct virtqueue *vq;
392 unsigned long flags, size;
393 u16 num;
394 int err;
395
396 /* Select the queue we're interested in */
397 iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
398
399 /* Check if queue is either not available or already active. */
400 num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM);
401 if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
402 return ERR_PTR(-ENOENT);
403
404 /* allocate and fill out our structure the represents an active
405 * queue */
406 info = kmalloc(sizeof(struct virtio_pci_vq_info), GFP_KERNEL);
407 if (!info)
408 return ERR_PTR(-ENOMEM);
409
410 info->num = num;
411 info->msix_vector = msix_vec;
412
413 size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
414 info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
415 if (info->queue == NULL) {
416 err = -ENOMEM;
417 goto out_info;
418 }
419
420 /* activate the queue */
421 iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT,
422 vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
423
424 /* create the vring */
425 vq = vring_new_virtqueue(index, info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
426 true, info->queue, vp_notify, callback, name);
427 if (!vq) {
428 err = -ENOMEM;
429 goto out_activate_queue;
430 }
431
432 vq->priv = info;
433 info->vq = vq;
434
435 if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
436 iowrite16(msix_vec, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
437 msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
438 if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
439 err = -EBUSY;
440 goto out_assign;
441 }
442 }
443
444 if (callback) {
445 spin_lock_irqsave(&vp_dev->lock, flags);
446 list_add(&info->node, &vp_dev->virtqueues);
447 spin_unlock_irqrestore(&vp_dev->lock, flags);
448 } else {
449 INIT_LIST_HEAD(&info->node);
450 }
451
452 return vq;
453
454out_assign:
455 vring_del_virtqueue(vq);
456out_activate_queue:
457 iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
458 free_pages_exact(info->queue, size);
459out_info:
460 kfree(info);
461 return ERR_PTR(err);
462}
463
464static void vp_del_vq(struct virtqueue *vq)
465{
466 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
467 struct virtio_pci_vq_info *info = vq->priv;
468 unsigned long flags, size;
469
470 spin_lock_irqsave(&vp_dev->lock, flags);
471 list_del(&info->node);
472 spin_unlock_irqrestore(&vp_dev->lock, flags);
473
474 iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
475
476 if (vp_dev->msix_enabled) {
477 iowrite16(VIRTIO_MSI_NO_VECTOR,
478 vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
479 /* Flush the write out to device */
480 ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
481 }
482
483 vring_del_virtqueue(vq);
484
485 /* Select and deactivate the queue */
486 iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
487
488 size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN));
489 free_pages_exact(info->queue, size);
490 kfree(info);
491}
492
493/* the config->del_vqs() implementation */
494static void vp_del_vqs(struct virtio_device *vdev)
495{
496 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
497 struct virtqueue *vq, *n;
498 struct virtio_pci_vq_info *info;
499
500 list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
501 info = vq->priv;
502 if (vp_dev->per_vq_vectors &&
503 info->msix_vector != VIRTIO_MSI_NO_VECTOR)
504 free_irq(vp_dev->msix_entries[info->msix_vector].vector,
505 vq);
506 vp_del_vq(vq);
507 }
508 vp_dev->per_vq_vectors = false;
509
510 vp_free_vectors(vdev);
511}
512
513static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
514 struct virtqueue *vqs[],
515 vq_callback_t *callbacks[],
516 const char *names[],
517 bool use_msix,
518 bool per_vq_vectors)
519{
520 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
521 u16 msix_vec;
522 int i, err, nvectors, allocated_vectors;
523
524 if (!use_msix) {
525 /* Old style: one normal interrupt for change and all vqs. */
526 err = vp_request_intx(vdev);
527 if (err)
528 goto error_request;
529 } else {
530 if (per_vq_vectors) {
531 /* Best option: one for change interrupt, one per vq. */
532 nvectors = 1;
533 for (i = 0; i < nvqs; ++i)
534 if (callbacks[i])
535 ++nvectors;
536 } else {
537 /* Second best: one for change, shared for all vqs. */
538 nvectors = 2;
539 }
540
541 err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
542 if (err)
543 goto error_request;
544 }
545
546 vp_dev->per_vq_vectors = per_vq_vectors;
547 allocated_vectors = vp_dev->msix_used_vectors;
548 for (i = 0; i < nvqs; ++i) {
549 if (!names[i]) {
550 vqs[i] = NULL;
551 continue;
552 } else if (!callbacks[i] || !vp_dev->msix_enabled)
553 msix_vec = VIRTIO_MSI_NO_VECTOR;
554 else if (vp_dev->per_vq_vectors)
555 msix_vec = allocated_vectors++;
556 else
557 msix_vec = VP_MSIX_VQ_VECTOR;
558 vqs[i] = setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
559 if (IS_ERR(vqs[i])) {
560 err = PTR_ERR(vqs[i]);
561 goto error_find;
562 }
563
564 if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
565 continue;
566
567 /* allocate per-vq irq if available and necessary */
568 snprintf(vp_dev->msix_names[msix_vec],
569 sizeof *vp_dev->msix_names,
570 "%s-%s",
571 dev_name(&vp_dev->vdev.dev), names[i]);
572 err = request_irq(vp_dev->msix_entries[msix_vec].vector,
573 vring_interrupt, 0,
574 vp_dev->msix_names[msix_vec],
575 vqs[i]);
576 if (err) {
577 vp_del_vq(vqs[i]);
578 goto error_find;
579 }
580 }
581 return 0;
582
583error_find:
584 vp_del_vqs(vdev);
585
586error_request:
587 return err;
588}
589
590/* the config->find_vqs() implementation */
591static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
592 struct virtqueue *vqs[],
593 vq_callback_t *callbacks[],
594 const char *names[])
595{
596 int err;
597
598 /* Try MSI-X with one vector per queue. */
599 err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
600 if (!err)
601 return 0;
602 /* Fallback: MSI-X with one vector for config, one shared for queues. */
603 err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
604 true, false);
605 if (!err)
606 return 0;
607 /* Finally fall back to regular interrupts. */
608 return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
609 false, false);
610}
611
612static const char *vp_bus_name(struct virtio_device *vdev)
613{
614 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
615
616 return pci_name(vp_dev->pci_dev);
617}
618
619/* Setup the affinity for a virtqueue:
620 * - force the affinity for per vq vector
621 * - OR over all affinities for shared MSI
622 * - ignore the affinity request if we're using INTX
623 */
624static int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
625{
626 struct virtio_device *vdev = vq->vdev;
627 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
628 struct virtio_pci_vq_info *info = vq->priv;
629 struct cpumask *mask;
630 unsigned int irq;
631
632 if (!vq->callback)
633 return -EINVAL;
634
635 if (vp_dev->msix_enabled) {
636 mask = vp_dev->msix_affinity_masks[info->msix_vector];
637 irq = vp_dev->msix_entries[info->msix_vector].vector;
638 if (cpu == -1)
639 irq_set_affinity_hint(irq, NULL);
640 else {
641 cpumask_set_cpu(cpu, mask);
642 irq_set_affinity_hint(irq, mask);
643 }
644 }
645 return 0;
646}
647
648static const struct virtio_config_ops virtio_pci_config_ops = {
649 .get = vp_get,
650 .set = vp_set,
651 .get_status = vp_get_status,
652 .set_status = vp_set_status,
653 .reset = vp_reset,
654 .find_vqs = vp_find_vqs,
655 .del_vqs = vp_del_vqs,
656 .get_features = vp_get_features,
657 .finalize_features = vp_finalize_features,
658 .bus_name = vp_bus_name,
659 .set_vq_affinity = vp_set_vq_affinity,
660};
661
662static void virtio_pci_release_dev(struct device *_d)
663{
664 /*
665 * No need for a release method as we allocate/free
666 * all devices together with the pci devices.
667 * Provide an empty one to avoid getting a warning from core.
668 */
669}
670
671/* the PCI probing function */
672static int virtio_pci_probe(struct pci_dev *pci_dev,
673 const struct pci_device_id *id)
674{
675 struct virtio_pci_device *vp_dev;
676 int err;
677
678 /* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */
679 if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f)
680 return -ENODEV;
681
682 if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) {
683 printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n",
684 VIRTIO_PCI_ABI_VERSION, pci_dev->revision);
685 return -ENODEV;
686 }
687
688 /* allocate our structure and fill it out */
689 vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
690 if (vp_dev == NULL)
691 return -ENOMEM;
692
693 vp_dev->vdev.dev.parent = &pci_dev->dev;
694 vp_dev->vdev.dev.release = virtio_pci_release_dev;
695 vp_dev->vdev.config = &virtio_pci_config_ops;
696 vp_dev->pci_dev = pci_dev;
697 INIT_LIST_HEAD(&vp_dev->virtqueues);
698 spin_lock_init(&vp_dev->lock);
699
700 /* Disable MSI/MSIX to bring device to a known good state. */
701 pci_msi_off(pci_dev);
702
703 /* enable the device */
704 err = pci_enable_device(pci_dev);
705 if (err)
706 goto out;
707
708 err = pci_request_regions(pci_dev, "virtio-pci");
709 if (err)
710 goto out_enable_device;
711
712 vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);
713 if (vp_dev->ioaddr == NULL) {
714 err = -ENOMEM;
715 goto out_req_regions;
716 }
717
718 pci_set_drvdata(pci_dev, vp_dev);
719 pci_set_master(pci_dev);
720
721 /* we use the subsystem vendor/device id as the virtio vendor/device
722 * id. this allows us to use the same PCI vendor/device id for all
723 * virtio devices and to identify the particular virtio driver by
724 * the subsystem ids */
725 vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
726 vp_dev->vdev.id.device = pci_dev->subsystem_device;
727
728 /* finally register the virtio device */
729 err = register_virtio_device(&vp_dev->vdev);
730 if (err)
731 goto out_set_drvdata;
732
733 return 0;
734
735out_set_drvdata:
736 pci_iounmap(pci_dev, vp_dev->ioaddr);
737out_req_regions:
738 pci_release_regions(pci_dev);
739out_enable_device:
740 pci_disable_device(pci_dev);
741out:
742 kfree(vp_dev);
743 return err;
744}
745
746static void virtio_pci_remove(struct pci_dev *pci_dev)
747{
748 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
749
750 unregister_virtio_device(&vp_dev->vdev);
751
752 vp_del_vqs(&vp_dev->vdev);
753 pci_iounmap(pci_dev, vp_dev->ioaddr);
754 pci_release_regions(pci_dev);
755 pci_disable_device(pci_dev);
756 kfree(vp_dev);
757}
758
759#ifdef CONFIG_PM_SLEEP
760static int virtio_pci_freeze(struct device *dev)
761{
762 struct pci_dev *pci_dev = to_pci_dev(dev);
763 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
764 int ret;
765
766 ret = virtio_device_freeze(&vp_dev->vdev);
767
768 if (!ret)
769 pci_disable_device(pci_dev);
770 return ret;
771}
772
773static int virtio_pci_restore(struct device *dev)
774{
775 struct pci_dev *pci_dev = to_pci_dev(dev);
776 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
777 int ret;
778
779 ret = pci_enable_device(pci_dev);
780 if (ret)
781 return ret;
782
783 pci_set_master(pci_dev);
784 return virtio_device_restore(&vp_dev->vdev);
785}
786
787static const struct dev_pm_ops virtio_pci_pm_ops = {
788 SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
789};
790#endif
791
792static struct pci_driver virtio_pci_driver = {
793 .name = "virtio-pci",
794 .id_table = virtio_pci_id_table,
795 .probe = virtio_pci_probe,
796 .remove = virtio_pci_remove,
797#ifdef CONFIG_PM_SLEEP
798 .driver.pm = &virtio_pci_pm_ops,
799#endif
800};
801
802module_pci_driver(virtio_pci_driver);
diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c
new file mode 100644
index 000000000000..953057d84185
--- /dev/null
+++ b/drivers/virtio/virtio_pci_common.c
@@ -0,0 +1,464 @@
1/*
2 * Virtio PCI driver - common functionality for all device versions
3 *
4 * This module allows virtio devices to be used over a virtual PCI device.
5 * This can be used with QEMU based VMMs like KVM or Xen.
6 *
7 * Copyright IBM Corp. 2007
8 * Copyright Red Hat, Inc. 2014
9 *
10 * Authors:
11 * Anthony Liguori <aliguori@us.ibm.com>
12 * Rusty Russell <rusty@rustcorp.com.au>
13 * Michael S. Tsirkin <mst@redhat.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2 or later.
16 * See the COPYING file in the top-level directory.
17 *
18 */
19
20#include "virtio_pci_common.h"
21
22/* wait for pending irq handlers */
23void vp_synchronize_vectors(struct virtio_device *vdev)
24{
25 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
26 int i;
27
28 if (vp_dev->intx_enabled)
29 synchronize_irq(vp_dev->pci_dev->irq);
30
31 for (i = 0; i < vp_dev->msix_vectors; ++i)
32 synchronize_irq(vp_dev->msix_entries[i].vector);
33}
34
35/* the notify function used when creating a virt queue */
36bool vp_notify(struct virtqueue *vq)
37{
38 /* we write the queue's selector into the notification register to
39 * signal the other end */
40 iowrite16(vq->index, (void __iomem *)vq->priv);
41 return true;
42}
43
44/* Handle a configuration change: Tell driver if it wants to know. */
45static irqreturn_t vp_config_changed(int irq, void *opaque)
46{
47 struct virtio_pci_device *vp_dev = opaque;
48
49 virtio_config_changed(&vp_dev->vdev);
50 return IRQ_HANDLED;
51}
52
53/* Notify all virtqueues on an interrupt. */
54static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
55{
56 struct virtio_pci_device *vp_dev = opaque;
57 struct virtio_pci_vq_info *info;
58 irqreturn_t ret = IRQ_NONE;
59 unsigned long flags;
60
61 spin_lock_irqsave(&vp_dev->lock, flags);
62 list_for_each_entry(info, &vp_dev->virtqueues, node) {
63 if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
64 ret = IRQ_HANDLED;
65 }
66 spin_unlock_irqrestore(&vp_dev->lock, flags);
67
68 return ret;
69}
70
71/* A small wrapper to also acknowledge the interrupt when it's handled.
72 * I really need an EIO hook for the vring so I can ack the interrupt once we
73 * know that we'll be handling the IRQ but before we invoke the callback since
74 * the callback may notify the host which results in the host attempting to
75 * raise an interrupt that we would then mask once we acknowledged the
76 * interrupt. */
77static irqreturn_t vp_interrupt(int irq, void *opaque)
78{
79 struct virtio_pci_device *vp_dev = opaque;
80 u8 isr;
81
82 /* reading the ISR has the effect of also clearing it so it's very
83 * important to save off the value. */
84 isr = ioread8(vp_dev->isr);
85
86 /* It's definitely not us if the ISR was not high */
87 if (!isr)
88 return IRQ_NONE;
89
90 /* Configuration change? Tell driver if it wants to know. */
91 if (isr & VIRTIO_PCI_ISR_CONFIG)
92 vp_config_changed(irq, opaque);
93
94 return vp_vring_interrupt(irq, opaque);
95}
96
97static void vp_free_vectors(struct virtio_device *vdev)
98{
99 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
100 int i;
101
102 if (vp_dev->intx_enabled) {
103 free_irq(vp_dev->pci_dev->irq, vp_dev);
104 vp_dev->intx_enabled = 0;
105 }
106
107 for (i = 0; i < vp_dev->msix_used_vectors; ++i)
108 free_irq(vp_dev->msix_entries[i].vector, vp_dev);
109
110 for (i = 0; i < vp_dev->msix_vectors; i++)
111 if (vp_dev->msix_affinity_masks[i])
112 free_cpumask_var(vp_dev->msix_affinity_masks[i]);
113
114 if (vp_dev->msix_enabled) {
115 /* Disable the vector used for configuration */
116 vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
117
118 pci_disable_msix(vp_dev->pci_dev);
119 vp_dev->msix_enabled = 0;
120 }
121
122 vp_dev->msix_vectors = 0;
123 vp_dev->msix_used_vectors = 0;
124 kfree(vp_dev->msix_names);
125 vp_dev->msix_names = NULL;
126 kfree(vp_dev->msix_entries);
127 vp_dev->msix_entries = NULL;
128 kfree(vp_dev->msix_affinity_masks);
129 vp_dev->msix_affinity_masks = NULL;
130}
131
132static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
133 bool per_vq_vectors)
134{
135 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
136 const char *name = dev_name(&vp_dev->vdev.dev);
137 unsigned i, v;
138 int err = -ENOMEM;
139
140 vp_dev->msix_vectors = nvectors;
141
142 vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
143 GFP_KERNEL);
144 if (!vp_dev->msix_entries)
145 goto error;
146 vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
147 GFP_KERNEL);
148 if (!vp_dev->msix_names)
149 goto error;
150 vp_dev->msix_affinity_masks
151 = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
152 GFP_KERNEL);
153 if (!vp_dev->msix_affinity_masks)
154 goto error;
155 for (i = 0; i < nvectors; ++i)
156 if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
157 GFP_KERNEL))
158 goto error;
159
160 for (i = 0; i < nvectors; ++i)
161 vp_dev->msix_entries[i].entry = i;
162
163 err = pci_enable_msix_exact(vp_dev->pci_dev,
164 vp_dev->msix_entries, nvectors);
165 if (err)
166 goto error;
167 vp_dev->msix_enabled = 1;
168
169 /* Set the vector used for configuration */
170 v = vp_dev->msix_used_vectors;
171 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
172 "%s-config", name);
173 err = request_irq(vp_dev->msix_entries[v].vector,
174 vp_config_changed, 0, vp_dev->msix_names[v],
175 vp_dev);
176 if (err)
177 goto error;
178 ++vp_dev->msix_used_vectors;
179
180 v = vp_dev->config_vector(vp_dev, v);
181 /* Verify we had enough resources to assign the vector */
182 if (v == VIRTIO_MSI_NO_VECTOR) {
183 err = -EBUSY;
184 goto error;
185 }
186
187 if (!per_vq_vectors) {
188 /* Shared vector for all VQs */
189 v = vp_dev->msix_used_vectors;
190 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
191 "%s-virtqueues", name);
192 err = request_irq(vp_dev->msix_entries[v].vector,
193 vp_vring_interrupt, 0, vp_dev->msix_names[v],
194 vp_dev);
195 if (err)
196 goto error;
197 ++vp_dev->msix_used_vectors;
198 }
199 return 0;
200error:
201 vp_free_vectors(vdev);
202 return err;
203}
204
205static int vp_request_intx(struct virtio_device *vdev)
206{
207 int err;
208 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
209
210 err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
211 IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
212 if (!err)
213 vp_dev->intx_enabled = 1;
214 return err;
215}
216
217static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned index,
218 void (*callback)(struct virtqueue *vq),
219 const char *name,
220 u16 msix_vec)
221{
222 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
223 struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL);
224 struct virtqueue *vq;
225 unsigned long flags;
226
227 /* fill out our structure that represents an active queue */
228 if (!info)
229 return ERR_PTR(-ENOMEM);
230
231 vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, msix_vec);
232 if (IS_ERR(vq))
233 goto out_info;
234
235 info->vq = vq;
236 if (callback) {
237 spin_lock_irqsave(&vp_dev->lock, flags);
238 list_add(&info->node, &vp_dev->virtqueues);
239 spin_unlock_irqrestore(&vp_dev->lock, flags);
240 } else {
241 INIT_LIST_HEAD(&info->node);
242 }
243
244 vp_dev->vqs[index] = info;
245 return vq;
246
247out_info:
248 kfree(info);
249 return vq;
250}
251
252static void vp_del_vq(struct virtqueue *vq)
253{
254 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
255 struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
256 unsigned long flags;
257
258 spin_lock_irqsave(&vp_dev->lock, flags);
259 list_del(&info->node);
260 spin_unlock_irqrestore(&vp_dev->lock, flags);
261
262 vp_dev->del_vq(info);
263 kfree(info);
264}
265
266/* the config->del_vqs() implementation */
267void vp_del_vqs(struct virtio_device *vdev)
268{
269 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
270 struct virtqueue *vq, *n;
271 struct virtio_pci_vq_info *info;
272
273 list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
274 info = vp_dev->vqs[vq->index];
275 if (vp_dev->per_vq_vectors &&
276 info->msix_vector != VIRTIO_MSI_NO_VECTOR)
277 free_irq(vp_dev->msix_entries[info->msix_vector].vector,
278 vq);
279 vp_del_vq(vq);
280 }
281 vp_dev->per_vq_vectors = false;
282
283 vp_free_vectors(vdev);
284 kfree(vp_dev->vqs);
285}
286
287static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
288 struct virtqueue *vqs[],
289 vq_callback_t *callbacks[],
290 const char *names[],
291 bool use_msix,
292 bool per_vq_vectors)
293{
294 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
295 u16 msix_vec;
296 int i, err, nvectors, allocated_vectors;
297
298 vp_dev->vqs = kmalloc(nvqs * sizeof *vp_dev->vqs, GFP_KERNEL);
299 if (!vp_dev->vqs)
300 return -ENOMEM;
301
302 if (!use_msix) {
303 /* Old style: one normal interrupt for change and all vqs. */
304 err = vp_request_intx(vdev);
305 if (err)
306 goto error_find;
307 } else {
308 if (per_vq_vectors) {
309 /* Best option: one for change interrupt, one per vq. */
310 nvectors = 1;
311 for (i = 0; i < nvqs; ++i)
312 if (callbacks[i])
313 ++nvectors;
314 } else {
315 /* Second best: one for change, shared for all vqs. */
316 nvectors = 2;
317 }
318
319 err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
320 if (err)
321 goto error_find;
322 }
323
324 vp_dev->per_vq_vectors = per_vq_vectors;
325 allocated_vectors = vp_dev->msix_used_vectors;
326 for (i = 0; i < nvqs; ++i) {
327 if (!names[i]) {
328 vqs[i] = NULL;
329 continue;
330 } else if (!callbacks[i] || !vp_dev->msix_enabled)
331 msix_vec = VIRTIO_MSI_NO_VECTOR;
332 else if (vp_dev->per_vq_vectors)
333 msix_vec = allocated_vectors++;
334 else
335 msix_vec = VP_MSIX_VQ_VECTOR;
336 vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
337 if (IS_ERR(vqs[i])) {
338 err = PTR_ERR(vqs[i]);
339 goto error_find;
340 }
341
342 if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
343 continue;
344
345 /* allocate per-vq irq if available and necessary */
346 snprintf(vp_dev->msix_names[msix_vec],
347 sizeof *vp_dev->msix_names,
348 "%s-%s",
349 dev_name(&vp_dev->vdev.dev), names[i]);
350 err = request_irq(vp_dev->msix_entries[msix_vec].vector,
351 vring_interrupt, 0,
352 vp_dev->msix_names[msix_vec],
353 vqs[i]);
354 if (err) {
355 vp_del_vq(vqs[i]);
356 goto error_find;
357 }
358 }
359 return 0;
360
361error_find:
362 vp_del_vqs(vdev);
363 return err;
364}
365
366/* the config->find_vqs() implementation */
367int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
368 struct virtqueue *vqs[],
369 vq_callback_t *callbacks[],
370 const char *names[])
371{
372 int err;
373
374 /* Try MSI-X with one vector per queue. */
375 err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
376 if (!err)
377 return 0;
378 /* Fallback: MSI-X with one vector for config, one shared for queues. */
379 err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
380 true, false);
381 if (!err)
382 return 0;
383 /* Finally fall back to regular interrupts. */
384 return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
385 false, false);
386}
387
388const char *vp_bus_name(struct virtio_device *vdev)
389{
390 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
391
392 return pci_name(vp_dev->pci_dev);
393}
394
395/* Setup the affinity for a virtqueue:
396 * - force the affinity for per vq vector
397 * - OR over all affinities for shared MSI
398 * - ignore the affinity request if we're using INTX
399 */
400int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
401{
402 struct virtio_device *vdev = vq->vdev;
403 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
404 struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
405 struct cpumask *mask;
406 unsigned int irq;
407
408 if (!vq->callback)
409 return -EINVAL;
410
411 if (vp_dev->msix_enabled) {
412 mask = vp_dev->msix_affinity_masks[info->msix_vector];
413 irq = vp_dev->msix_entries[info->msix_vector].vector;
414 if (cpu == -1)
415 irq_set_affinity_hint(irq, NULL);
416 else {
417 cpumask_set_cpu(cpu, mask);
418 irq_set_affinity_hint(irq, mask);
419 }
420 }
421 return 0;
422}
423
424void virtio_pci_release_dev(struct device *_d)
425{
426 /*
427 * No need for a release method as we allocate/free
428 * all devices together with the pci devices.
429 * Provide an empty one to avoid getting a warning from core.
430 */
431}
432
433#ifdef CONFIG_PM_SLEEP
434static int virtio_pci_freeze(struct device *dev)
435{
436 struct pci_dev *pci_dev = to_pci_dev(dev);
437 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
438 int ret;
439
440 ret = virtio_device_freeze(&vp_dev->vdev);
441
442 if (!ret)
443 pci_disable_device(pci_dev);
444 return ret;
445}
446
447static int virtio_pci_restore(struct device *dev)
448{
449 struct pci_dev *pci_dev = to_pci_dev(dev);
450 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
451 int ret;
452
453 ret = pci_enable_device(pci_dev);
454 if (ret)
455 return ret;
456
457 pci_set_master(pci_dev);
458 return virtio_device_restore(&vp_dev->vdev);
459}
460
461const struct dev_pm_ops virtio_pci_pm_ops = {
462 SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
463};
464#endif
diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h
new file mode 100644
index 000000000000..d840dad4149d
--- /dev/null
+++ b/drivers/virtio/virtio_pci_common.h
@@ -0,0 +1,136 @@
1#ifndef _DRIVERS_VIRTIO_VIRTIO_PCI_COMMON_H
2#define _DRIVERS_VIRTIO_VIRTIO_PCI_COMMON_H
3/*
4 * Virtio PCI driver - APIs for common functionality for all device versions
5 *
6 * This module allows virtio devices to be used over a virtual PCI device.
7 * This can be used with QEMU based VMMs like KVM or Xen.
8 *
9 * Copyright IBM Corp. 2007
10 * Copyright Red Hat, Inc. 2014
11 *
12 * Authors:
13 * Anthony Liguori <aliguori@us.ibm.com>
14 * Rusty Russell <rusty@rustcorp.com.au>
15 * Michael S. Tsirkin <mst@redhat.com>
16 *
17 * This work is licensed under the terms of the GNU GPL, version 2 or later.
18 * See the COPYING file in the top-level directory.
19 *
20 */
21
22#include <linux/module.h>
23#include <linux/list.h>
24#include <linux/pci.h>
25#include <linux/slab.h>
26#include <linux/interrupt.h>
27#include <linux/virtio.h>
28#include <linux/virtio_config.h>
29#include <linux/virtio_ring.h>
30#define VIRTIO_PCI_NO_LEGACY
31#include <linux/virtio_pci.h>
32#include <linux/highmem.h>
33#include <linux/spinlock.h>
34
35struct virtio_pci_vq_info {
36 /* the actual virtqueue */
37 struct virtqueue *vq;
38
39 /* the number of entries in the queue */
40 int num;
41
42 /* the virtual address of the ring queue */
43 void *queue;
44
45 /* the list node for the virtqueues list */
46 struct list_head node;
47
48 /* MSI-X vector (or none) */
49 unsigned msix_vector;
50};
51
52/* Our device structure */
53struct virtio_pci_device {
54 struct virtio_device vdev;
55 struct pci_dev *pci_dev;
56
57 /* the IO mapping for the PCI config space */
58 void __iomem *ioaddr;
59
60 /* the IO mapping for ISR operation */
61 void __iomem *isr;
62
63 /* a list of queues so we can dispatch IRQs */
64 spinlock_t lock;
65 struct list_head virtqueues;
66
67 /* array of all queues for house-keeping */
68 struct virtio_pci_vq_info **vqs;
69
70 /* MSI-X support */
71 int msix_enabled;
72 int intx_enabled;
73 struct msix_entry *msix_entries;
74 cpumask_var_t *msix_affinity_masks;
75 /* Name strings for interrupts. This size should be enough,
76 * and I'm too lazy to allocate each name separately. */
77 char (*msix_names)[256];
78 /* Number of available vectors */
79 unsigned msix_vectors;
80 /* Vectors allocated, excluding per-vq vectors if any */
81 unsigned msix_used_vectors;
82
83 /* Whether we have vector per vq */
84 bool per_vq_vectors;
85
86 struct virtqueue *(*setup_vq)(struct virtio_pci_device *vp_dev,
87 struct virtio_pci_vq_info *info,
88 unsigned idx,
89 void (*callback)(struct virtqueue *vq),
90 const char *name,
91 u16 msix_vec);
92 void (*del_vq)(struct virtio_pci_vq_info *info);
93
94 u16 (*config_vector)(struct virtio_pci_device *vp_dev, u16 vector);
95};
96
97/* Constants for MSI-X */
98/* Use first vector for configuration changes, second and the rest for
99 * virtqueues Thus, we need at least 2 vectors for MSI. */
100enum {
101 VP_MSIX_CONFIG_VECTOR = 0,
102 VP_MSIX_VQ_VECTOR = 1,
103};
104
105/* Convert a generic virtio device to our structure */
106static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
107{
108 return container_of(vdev, struct virtio_pci_device, vdev);
109}
110
111/* wait for pending irq handlers */
112void vp_synchronize_vectors(struct virtio_device *vdev);
113/* the notify function used when creating a virt queue */
114bool vp_notify(struct virtqueue *vq);
115/* the config->del_vqs() implementation */
116void vp_del_vqs(struct virtio_device *vdev);
117/* the config->find_vqs() implementation */
118int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
119 struct virtqueue *vqs[],
120 vq_callback_t *callbacks[],
121 const char *names[]);
122const char *vp_bus_name(struct virtio_device *vdev);
123
124/* Setup the affinity for a virtqueue:
125 * - force the affinity for per vq vector
126 * - OR over all affinities for shared MSI
127 * - ignore the affinity request if we're using INTX
128 */
129int vp_set_vq_affinity(struct virtqueue *vq, int cpu);
130void virtio_pci_release_dev(struct device *);
131
132#ifdef CONFIG_PM_SLEEP
133extern const struct dev_pm_ops virtio_pci_pm_ops;
134#endif
135
136#endif
diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c
new file mode 100644
index 000000000000..2588252e5c1c
--- /dev/null
+++ b/drivers/virtio/virtio_pci_legacy.c
@@ -0,0 +1,326 @@
1/*
2 * Virtio PCI driver - legacy device support
3 *
4 * This module allows virtio devices to be used over a virtual PCI device.
5 * This can be used with QEMU based VMMs like KVM or Xen.
6 *
7 * Copyright IBM Corp. 2007
8 * Copyright Red Hat, Inc. 2014
9 *
10 * Authors:
11 * Anthony Liguori <aliguori@us.ibm.com>
12 * Rusty Russell <rusty@rustcorp.com.au>
13 * Michael S. Tsirkin <mst@redhat.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2 or later.
16 * See the COPYING file in the top-level directory.
17 *
18 */
19
20#include "virtio_pci_common.h"
21
22/* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
23static const struct pci_device_id virtio_pci_id_table[] = {
24 { PCI_DEVICE(0x1af4, PCI_ANY_ID) },
25 { 0 }
26};
27
28MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
29
30/* virtio config->get_features() implementation */
31static u64 vp_get_features(struct virtio_device *vdev)
32{
33 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
34
35 /* When someone needs more than 32 feature bits, we'll need to
36 * steal a bit to indicate that the rest are somewhere else. */
37 return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES);
38}
39
40/* virtio config->finalize_features() implementation */
41static int vp_finalize_features(struct virtio_device *vdev)
42{
43 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
44
45 /* Give virtio_ring a chance to accept features. */
46 vring_transport_features(vdev);
47
48 /* Make sure we don't have any features > 32 bits! */
49 BUG_ON((u32)vdev->features != vdev->features);
50
51 /* We only support 32 feature bits. */
52 iowrite32(vdev->features, vp_dev->ioaddr + VIRTIO_PCI_GUEST_FEATURES);
53
54 return 0;
55}
56
57/* virtio config->get() implementation */
58static void vp_get(struct virtio_device *vdev, unsigned offset,
59 void *buf, unsigned len)
60{
61 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
62 void __iomem *ioaddr = vp_dev->ioaddr +
63 VIRTIO_PCI_CONFIG(vp_dev) + offset;
64 u8 *ptr = buf;
65 int i;
66
67 for (i = 0; i < len; i++)
68 ptr[i] = ioread8(ioaddr + i);
69}
70
71/* the config->set() implementation. it's symmetric to the config->get()
72 * implementation */
73static void vp_set(struct virtio_device *vdev, unsigned offset,
74 const void *buf, unsigned len)
75{
76 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
77 void __iomem *ioaddr = vp_dev->ioaddr +
78 VIRTIO_PCI_CONFIG(vp_dev) + offset;
79 const u8 *ptr = buf;
80 int i;
81
82 for (i = 0; i < len; i++)
83 iowrite8(ptr[i], ioaddr + i);
84}
85
86/* config->{get,set}_status() implementations */
87static u8 vp_get_status(struct virtio_device *vdev)
88{
89 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
90 return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
91}
92
93static void vp_set_status(struct virtio_device *vdev, u8 status)
94{
95 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
96 /* We should never be setting status to 0. */
97 BUG_ON(status == 0);
98 iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
99}
100
101static void vp_reset(struct virtio_device *vdev)
102{
103 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
104 /* 0 status means a reset. */
105 iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
106 /* Flush out the status write, and flush in device writes,
107 * including MSi-X interrupts, if any. */
108 ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
109 /* Flush pending VQ/configuration callbacks. */
110 vp_synchronize_vectors(vdev);
111}
112
113static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
114{
115 /* Setup the vector used for configuration events */
116 iowrite16(vector, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
117 /* Verify we had enough resources to assign the vector */
118 /* Will also flush the write out to device */
119 return ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
120}
121
122static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
123 struct virtio_pci_vq_info *info,
124 unsigned index,
125 void (*callback)(struct virtqueue *vq),
126 const char *name,
127 u16 msix_vec)
128{
129 struct virtqueue *vq;
130 unsigned long size;
131 u16 num;
132 int err;
133
134 /* Select the queue we're interested in */
135 iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
136
137 /* Check if queue is either not available or already active. */
138 num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM);
139 if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
140 return ERR_PTR(-ENOENT);
141
142 info->num = num;
143 info->msix_vector = msix_vec;
144
145 size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
146 info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
147 if (info->queue == NULL)
148 return ERR_PTR(-ENOMEM);
149
150 /* activate the queue */
151 iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT,
152 vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
153
154 /* create the vring */
155 vq = vring_new_virtqueue(index, info->num,
156 VIRTIO_PCI_VRING_ALIGN, &vp_dev->vdev,
157 true, info->queue, vp_notify, callback, name);
158 if (!vq) {
159 err = -ENOMEM;
160 goto out_activate_queue;
161 }
162
163 vq->priv = (void __force *)vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY;
164
165 if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
166 iowrite16(msix_vec, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
167 msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
168 if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
169 err = -EBUSY;
170 goto out_assign;
171 }
172 }
173
174 return vq;
175
176out_assign:
177 vring_del_virtqueue(vq);
178out_activate_queue:
179 iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
180 free_pages_exact(info->queue, size);
181 return ERR_PTR(err);
182}
183
184static void del_vq(struct virtio_pci_vq_info *info)
185{
186 struct virtqueue *vq = info->vq;
187 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
188 unsigned long size;
189
190 iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
191
192 if (vp_dev->msix_enabled) {
193 iowrite16(VIRTIO_MSI_NO_VECTOR,
194 vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
195 /* Flush the write out to device */
196 ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
197 }
198
199 vring_del_virtqueue(vq);
200
201 /* Select and deactivate the queue */
202 iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
203
204 size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN));
205 free_pages_exact(info->queue, size);
206}
207
208static const struct virtio_config_ops virtio_pci_config_ops = {
209 .get = vp_get,
210 .set = vp_set,
211 .get_status = vp_get_status,
212 .set_status = vp_set_status,
213 .reset = vp_reset,
214 .find_vqs = vp_find_vqs,
215 .del_vqs = vp_del_vqs,
216 .get_features = vp_get_features,
217 .finalize_features = vp_finalize_features,
218 .bus_name = vp_bus_name,
219 .set_vq_affinity = vp_set_vq_affinity,
220};
221
222/* the PCI probing function */
223static int virtio_pci_probe(struct pci_dev *pci_dev,
224 const struct pci_device_id *id)
225{
226 struct virtio_pci_device *vp_dev;
227 int err;
228
229 /* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */
230 if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f)
231 return -ENODEV;
232
233 if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) {
234 printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n",
235 VIRTIO_PCI_ABI_VERSION, pci_dev->revision);
236 return -ENODEV;
237 }
238
239 /* allocate our structure and fill it out */
240 vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
241 if (vp_dev == NULL)
242 return -ENOMEM;
243
244 vp_dev->vdev.dev.parent = &pci_dev->dev;
245 vp_dev->vdev.dev.release = virtio_pci_release_dev;
246 vp_dev->vdev.config = &virtio_pci_config_ops;
247 vp_dev->pci_dev = pci_dev;
248 INIT_LIST_HEAD(&vp_dev->virtqueues);
249 spin_lock_init(&vp_dev->lock);
250
251 /* Disable MSI/MSIX to bring device to a known good state. */
252 pci_msi_off(pci_dev);
253
254 /* enable the device */
255 err = pci_enable_device(pci_dev);
256 if (err)
257 goto out;
258
259 err = pci_request_regions(pci_dev, "virtio-pci");
260 if (err)
261 goto out_enable_device;
262
263 vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);
264 if (vp_dev->ioaddr == NULL) {
265 err = -ENOMEM;
266 goto out_req_regions;
267 }
268
269 vp_dev->isr = vp_dev->ioaddr + VIRTIO_PCI_ISR;
270
271 pci_set_drvdata(pci_dev, vp_dev);
272 pci_set_master(pci_dev);
273
274 /* we use the subsystem vendor/device id as the virtio vendor/device
275 * id. this allows us to use the same PCI vendor/device id for all
276 * virtio devices and to identify the particular virtio driver by
277 * the subsystem ids */
278 vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
279 vp_dev->vdev.id.device = pci_dev->subsystem_device;
280
281 vp_dev->config_vector = vp_config_vector;
282 vp_dev->setup_vq = setup_vq;
283 vp_dev->del_vq = del_vq;
284
285 /* finally register the virtio device */
286 err = register_virtio_device(&vp_dev->vdev);
287 if (err)
288 goto out_set_drvdata;
289
290 return 0;
291
292out_set_drvdata:
293 pci_iounmap(pci_dev, vp_dev->ioaddr);
294out_req_regions:
295 pci_release_regions(pci_dev);
296out_enable_device:
297 pci_disable_device(pci_dev);
298out:
299 kfree(vp_dev);
300 return err;
301}
302
303static void virtio_pci_remove(struct pci_dev *pci_dev)
304{
305 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
306
307 unregister_virtio_device(&vp_dev->vdev);
308
309 vp_del_vqs(&vp_dev->vdev);
310 pci_iounmap(pci_dev, vp_dev->ioaddr);
311 pci_release_regions(pci_dev);
312 pci_disable_device(pci_dev);
313 kfree(vp_dev);
314}
315
316static struct pci_driver virtio_pci_driver = {
317 .name = "virtio-pci",
318 .id_table = virtio_pci_id_table,
319 .probe = virtio_pci_probe,
320 .remove = virtio_pci_remove,
321#ifdef CONFIG_PM_SLEEP
322 .driver.pm = &virtio_pci_pm_ops,
323#endif
324};
325
326module_pci_driver(virtio_pci_driver);
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 3b1f89b6e743..00ec6b3f96b2 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -99,7 +99,8 @@ struct vring_virtqueue
99 99
100#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) 100#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
101 101
102static struct vring_desc *alloc_indirect(unsigned int total_sg, gfp_t gfp) 102static struct vring_desc *alloc_indirect(struct virtqueue *_vq,
103 unsigned int total_sg, gfp_t gfp)
103{ 104{
104 struct vring_desc *desc; 105 struct vring_desc *desc;
105 unsigned int i; 106 unsigned int i;
@@ -116,7 +117,7 @@ static struct vring_desc *alloc_indirect(unsigned int total_sg, gfp_t gfp)
116 return NULL; 117 return NULL;
117 118
118 for (i = 0; i < total_sg; i++) 119 for (i = 0; i < total_sg; i++)
119 desc[i].next = i+1; 120 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
120 return desc; 121 return desc;
121} 122}
122 123
@@ -165,17 +166,17 @@ static inline int virtqueue_add(struct virtqueue *_vq,
165 /* If the host supports indirect descriptor tables, and we have multiple 166 /* If the host supports indirect descriptor tables, and we have multiple
166 * buffers, then go indirect. FIXME: tune this threshold */ 167 * buffers, then go indirect. FIXME: tune this threshold */
167 if (vq->indirect && total_sg > 1 && vq->vq.num_free) 168 if (vq->indirect && total_sg > 1 && vq->vq.num_free)
168 desc = alloc_indirect(total_sg, gfp); 169 desc = alloc_indirect(_vq, total_sg, gfp);
169 else 170 else
170 desc = NULL; 171 desc = NULL;
171 172
172 if (desc) { 173 if (desc) {
173 /* Use a single buffer which doesn't continue */ 174 /* Use a single buffer which doesn't continue */
174 vq->vring.desc[head].flags = VRING_DESC_F_INDIRECT; 175 vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);
175 vq->vring.desc[head].addr = virt_to_phys(desc); 176 vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, virt_to_phys(desc));
176 /* avoid kmemleak false positive (hidden by virt_to_phys) */ 177 /* avoid kmemleak false positive (hidden by virt_to_phys) */
177 kmemleak_ignore(desc); 178 kmemleak_ignore(desc);
178 vq->vring.desc[head].len = total_sg * sizeof(struct vring_desc); 179 vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));
179 180
180 /* Set up rest to use this indirect table. */ 181 /* Set up rest to use this indirect table. */
181 i = 0; 182 i = 0;
@@ -205,28 +206,28 @@ static inline int virtqueue_add(struct virtqueue *_vq,
205 206
206 for (n = 0; n < out_sgs; n++) { 207 for (n = 0; n < out_sgs; n++) {
207 for (sg = sgs[n]; sg; sg = sg_next(sg)) { 208 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
208 desc[i].flags = VRING_DESC_F_NEXT; 209 desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);
209 desc[i].addr = sg_phys(sg); 210 desc[i].addr = cpu_to_virtio64(_vq->vdev, sg_phys(sg));
210 desc[i].len = sg->length; 211 desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
211 prev = i; 212 prev = i;
212 i = desc[i].next; 213 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
213 } 214 }
214 } 215 }
215 for (; n < (out_sgs + in_sgs); n++) { 216 for (; n < (out_sgs + in_sgs); n++) {
216 for (sg = sgs[n]; sg; sg = sg_next(sg)) { 217 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
217 desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE; 218 desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);
218 desc[i].addr = sg_phys(sg); 219 desc[i].addr = cpu_to_virtio64(_vq->vdev, sg_phys(sg));
219 desc[i].len = sg->length; 220 desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
220 prev = i; 221 prev = i;
221 i = desc[i].next; 222 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
222 } 223 }
223 } 224 }
224 /* Last one doesn't continue. */ 225 /* Last one doesn't continue. */
225 desc[prev].flags &= ~VRING_DESC_F_NEXT; 226 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
226 227
227 /* Update free pointer */ 228 /* Update free pointer */
228 if (indirect) 229 if (indirect)
229 vq->free_head = vq->vring.desc[head].next; 230 vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);
230 else 231 else
231 vq->free_head = i; 232 vq->free_head = i;
232 233
@@ -235,13 +236,13 @@ static inline int virtqueue_add(struct virtqueue *_vq,
235 236
236 /* Put entry in available array (but don't update avail->idx until they 237 /* Put entry in available array (but don't update avail->idx until they
237 * do sync). */ 238 * do sync). */
238 avail = (vq->vring.avail->idx & (vq->vring.num-1)); 239 avail = virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) & (vq->vring.num - 1);
239 vq->vring.avail->ring[avail] = head; 240 vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
240 241
241 /* Descriptors and available array need to be set before we expose the 242 /* Descriptors and available array need to be set before we expose the
242 * new available array entries. */ 243 * new available array entries. */
243 virtio_wmb(vq->weak_barriers); 244 virtio_wmb(vq->weak_barriers);
244 vq->vring.avail->idx++; 245 vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) + 1);
245 vq->num_added++; 246 vq->num_added++;
246 247
247 /* This is very unlikely, but theoretically possible. Kick 248 /* This is very unlikely, but theoretically possible. Kick
@@ -354,8 +355,8 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
354 * event. */ 355 * event. */
355 virtio_mb(vq->weak_barriers); 356 virtio_mb(vq->weak_barriers);
356 357
357 old = vq->vring.avail->idx - vq->num_added; 358 old = virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) - vq->num_added;
358 new = vq->vring.avail->idx; 359 new = virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx);
359 vq->num_added = 0; 360 vq->num_added = 0;
360 361
361#ifdef DEBUG 362#ifdef DEBUG
@@ -367,10 +368,10 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
367#endif 368#endif
368 369
369 if (vq->event) { 370 if (vq->event) {
370 needs_kick = vring_need_event(vring_avail_event(&vq->vring), 371 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),
371 new, old); 372 new, old);
372 } else { 373 } else {
373 needs_kick = !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY); 374 needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));
374 } 375 }
375 END_USE(vq); 376 END_USE(vq);
376 return needs_kick; 377 return needs_kick;
@@ -432,15 +433,15 @@ static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
432 i = head; 433 i = head;
433 434
434 /* Free the indirect table */ 435 /* Free the indirect table */
435 if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT) 436 if (vq->vring.desc[i].flags & cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT))
436 kfree(phys_to_virt(vq->vring.desc[i].addr)); 437 kfree(phys_to_virt(virtio64_to_cpu(vq->vq.vdev, vq->vring.desc[i].addr)));
437 438
438 while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) { 439 while (vq->vring.desc[i].flags & cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT)) {
439 i = vq->vring.desc[i].next; 440 i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next);
440 vq->vq.num_free++; 441 vq->vq.num_free++;
441 } 442 }
442 443
443 vq->vring.desc[i].next = vq->free_head; 444 vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head);
444 vq->free_head = head; 445 vq->free_head = head;
445 /* Plus final descriptor */ 446 /* Plus final descriptor */
446 vq->vq.num_free++; 447 vq->vq.num_free++;
@@ -448,7 +449,7 @@ static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
448 449
449static inline bool more_used(const struct vring_virtqueue *vq) 450static inline bool more_used(const struct vring_virtqueue *vq)
450{ 451{
451 return vq->last_used_idx != vq->vring.used->idx; 452 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx);
452} 453}
453 454
454/** 455/**
@@ -491,8 +492,8 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
491 virtio_rmb(vq->weak_barriers); 492 virtio_rmb(vq->weak_barriers);
492 493
493 last_used = (vq->last_used_idx & (vq->vring.num - 1)); 494 last_used = (vq->last_used_idx & (vq->vring.num - 1));
494 i = vq->vring.used->ring[last_used].id; 495 i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);
495 *len = vq->vring.used->ring[last_used].len; 496 *len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);
496 497
497 if (unlikely(i >= vq->vring.num)) { 498 if (unlikely(i >= vq->vring.num)) {
498 BAD_RING(vq, "id %u out of range\n", i); 499 BAD_RING(vq, "id %u out of range\n", i);
@@ -510,8 +511,8 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
510 /* If we expect an interrupt for the next entry, tell host 511 /* If we expect an interrupt for the next entry, tell host
511 * by writing event index and flush out the write before 512 * by writing event index and flush out the write before
512 * the read in the next get_buf call. */ 513 * the read in the next get_buf call. */
513 if (!(vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) { 514 if (!(vq->vring.avail->flags & cpu_to_virtio16(_vq->vdev, VRING_AVAIL_F_NO_INTERRUPT))) {
514 vring_used_event(&vq->vring) = vq->last_used_idx; 515 vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx);
515 virtio_mb(vq->weak_barriers); 516 virtio_mb(vq->weak_barriers);
516 } 517 }
517 518
@@ -537,7 +538,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
537{ 538{
538 struct vring_virtqueue *vq = to_vvq(_vq); 539 struct vring_virtqueue *vq = to_vvq(_vq);
539 540
540 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 541 vq->vring.avail->flags |= cpu_to_virtio16(_vq->vdev, VRING_AVAIL_F_NO_INTERRUPT);
541} 542}
542EXPORT_SYMBOL_GPL(virtqueue_disable_cb); 543EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
543 544
@@ -565,8 +566,8 @@ unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
565 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to 566 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
566 * either clear the flags bit or point the event index at the next 567 * either clear the flags bit or point the event index at the next
567 * entry. Always do both to keep code simple. */ 568 * entry. Always do both to keep code simple. */
568 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 569 vq->vring.avail->flags &= cpu_to_virtio16(_vq->vdev, ~VRING_AVAIL_F_NO_INTERRUPT);
569 vring_used_event(&vq->vring) = last_used_idx = vq->last_used_idx; 570 vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);
570 END_USE(vq); 571 END_USE(vq);
571 return last_used_idx; 572 return last_used_idx;
572} 573}
@@ -586,7 +587,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)
586 struct vring_virtqueue *vq = to_vvq(_vq); 587 struct vring_virtqueue *vq = to_vvq(_vq);
587 588
588 virtio_mb(vq->weak_barriers); 589 virtio_mb(vq->weak_barriers);
589 return (u16)last_used_idx != vq->vring.used->idx; 590 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);
590} 591}
591EXPORT_SYMBOL_GPL(virtqueue_poll); 592EXPORT_SYMBOL_GPL(virtqueue_poll);
592 593
@@ -633,12 +634,12 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
633 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to 634 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
634 * either clear the flags bit or point the event index at the next 635 * either clear the flags bit or point the event index at the next
635 * entry. Always do both to keep code simple. */ 636 * entry. Always do both to keep code simple. */
636 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 637 vq->vring.avail->flags &= cpu_to_virtio16(_vq->vdev, ~VRING_AVAIL_F_NO_INTERRUPT);
637 /* TODO: tune this threshold */ 638 /* TODO: tune this threshold */
638 bufs = (u16)(vq->vring.avail->idx - vq->last_used_idx) * 3 / 4; 639 bufs = (u16)(virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) - vq->last_used_idx) * 3 / 4;
639 vring_used_event(&vq->vring) = vq->last_used_idx + bufs; 640 vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs);
640 virtio_mb(vq->weak_barriers); 641 virtio_mb(vq->weak_barriers);
641 if (unlikely((u16)(vq->vring.used->idx - vq->last_used_idx) > bufs)) { 642 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {
642 END_USE(vq); 643 END_USE(vq);
643 return false; 644 return false;
644 } 645 }
@@ -670,7 +671,7 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
670 /* detach_buf clears data, so grab it now. */ 671 /* detach_buf clears data, so grab it now. */
671 buf = vq->data[i]; 672 buf = vq->data[i];
672 detach_buf(vq, i); 673 detach_buf(vq, i);
673 vq->vring.avail->idx--; 674 vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, virtio16_to_cpu(_vq->vdev, vq->vring.avail->idx) - 1);
674 END_USE(vq); 675 END_USE(vq);
675 return buf; 676 return buf;
676 } 677 }
@@ -747,12 +748,12 @@ struct virtqueue *vring_new_virtqueue(unsigned int index,
747 748
748 /* No callback? Tell other side not to bother us. */ 749 /* No callback? Tell other side not to bother us. */
749 if (!callback) 750 if (!callback)
750 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 751 vq->vring.avail->flags |= cpu_to_virtio16(vdev, VRING_AVAIL_F_NO_INTERRUPT);
751 752
752 /* Put everything in free lists. */ 753 /* Put everything in free lists. */
753 vq->free_head = 0; 754 vq->free_head = 0;
754 for (i = 0; i < num-1; i++) { 755 for (i = 0; i < num-1; i++) {
755 vq->vring.desc[i].next = i+1; 756 vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
756 vq->data[i] = NULL; 757 vq->data[i] = NULL;
757 } 758 }
758 vq->data[i] = NULL; 759 vq->data[i] = NULL;
@@ -779,9 +780,11 @@ void vring_transport_features(struct virtio_device *vdev)
779 break; 780 break;
780 case VIRTIO_RING_F_EVENT_IDX: 781 case VIRTIO_RING_F_EVENT_IDX:
781 break; 782 break;
783 case VIRTIO_F_VERSION_1:
784 break;
782 default: 785 default:
783 /* We don't understand this bit. */ 786 /* We don't understand this bit. */
784 clear_bit(i, vdev->features); 787 __virtio_clear_bit(vdev, i);
785 } 788 }
786 } 789 }
787} 790}
@@ -826,4 +829,20 @@ void virtio_break_device(struct virtio_device *dev)
826} 829}
827EXPORT_SYMBOL_GPL(virtio_break_device); 830EXPORT_SYMBOL_GPL(virtio_break_device);
828 831
832void *virtqueue_get_avail(struct virtqueue *_vq)
833{
834 struct vring_virtqueue *vq = to_vvq(_vq);
835
836 return vq->vring.avail;
837}
838EXPORT_SYMBOL_GPL(virtqueue_get_avail);
839
840void *virtqueue_get_used(struct virtqueue *_vq)
841{
842 struct vring_virtqueue *vq = to_vvq(_vq);
843
844 return vq->vring.used;
845}
846EXPORT_SYMBOL_GPL(virtqueue_get_used);
847
829MODULE_LICENSE("GPL"); 848MODULE_LICENSE("GPL");
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index 65261a7244fc..d09e0938fd60 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -75,6 +75,9 @@ unsigned int virtqueue_get_vring_size(struct virtqueue *vq);
75 75
76bool virtqueue_is_broken(struct virtqueue *vq); 76bool virtqueue_is_broken(struct virtqueue *vq);
77 77
78void *virtqueue_get_avail(struct virtqueue *vq);
79void *virtqueue_get_used(struct virtqueue *vq);
80
78/** 81/**
79 * virtio_device - representation of a device using virtio 82 * virtio_device - representation of a device using virtio
80 * @index: unique position on the virtio bus 83 * @index: unique position on the virtio bus
@@ -101,11 +104,12 @@ struct virtio_device {
101 const struct virtio_config_ops *config; 104 const struct virtio_config_ops *config;
102 const struct vringh_config_ops *vringh_config; 105 const struct vringh_config_ops *vringh_config;
103 struct list_head vqs; 106 struct list_head vqs;
104 /* Note that this is a Linux set_bit-style bitmap. */ 107 u64 features;
105 unsigned long features[1];
106 void *priv; 108 void *priv;
107}; 109};
108 110
111bool virtio_device_is_legacy_only(struct virtio_device_id id);
112
109static inline struct virtio_device *dev_to_virtio(struct device *_dev) 113static inline struct virtio_device *dev_to_virtio(struct device *_dev)
110{ 114{
111 return container_of(_dev, struct virtio_device, dev); 115 return container_of(_dev, struct virtio_device, dev);
@@ -128,6 +132,8 @@ int virtio_device_restore(struct virtio_device *dev);
128 * @id_table: the ids serviced by this driver. 132 * @id_table: the ids serviced by this driver.
129 * @feature_table: an array of feature numbers supported by this driver. 133 * @feature_table: an array of feature numbers supported by this driver.
130 * @feature_table_size: number of entries in the feature table array. 134 * @feature_table_size: number of entries in the feature table array.
135 * @feature_table_legacy: same as feature_table but when working in legacy mode.
136 * @feature_table_size_legacy: number of entries in feature table legacy array.
131 * @probe: the function to call when a device is found. Returns 0 or -errno. 137 * @probe: the function to call when a device is found. Returns 0 or -errno.
132 * @remove: the function to call when a device is removed. 138 * @remove: the function to call when a device is removed.
133 * @config_changed: optional function to call when the device configuration 139 * @config_changed: optional function to call when the device configuration
@@ -138,6 +144,8 @@ struct virtio_driver {
138 const struct virtio_device_id *id_table; 144 const struct virtio_device_id *id_table;
139 const unsigned int *feature_table; 145 const unsigned int *feature_table;
140 unsigned int feature_table_size; 146 unsigned int feature_table_size;
147 const unsigned int *feature_table_legacy;
148 unsigned int feature_table_size_legacy;
141 int (*probe)(struct virtio_device *dev); 149 int (*probe)(struct virtio_device *dev);
142 void (*scan)(struct virtio_device *dev); 150 void (*scan)(struct virtio_device *dev);
143 void (*remove)(struct virtio_device *dev); 151 void (*remove)(struct virtio_device *dev);
diff --git a/include/linux/virtio_byteorder.h b/include/linux/virtio_byteorder.h
new file mode 100644
index 000000000000..51865d05b267
--- /dev/null
+++ b/include/linux/virtio_byteorder.h
@@ -0,0 +1,59 @@
1#ifndef _LINUX_VIRTIO_BYTEORDER_H
2#define _LINUX_VIRTIO_BYTEORDER_H
3#include <linux/types.h>
4#include <uapi/linux/virtio_types.h>
5
6/*
7 * Low-level memory accessors for handling virtio in modern little endian and in
8 * compatibility native endian format.
9 */
10
11static inline u16 __virtio16_to_cpu(bool little_endian, __virtio16 val)
12{
13 if (little_endian)
14 return le16_to_cpu((__force __le16)val);
15 else
16 return (__force u16)val;
17}
18
19static inline __virtio16 __cpu_to_virtio16(bool little_endian, u16 val)
20{
21 if (little_endian)
22 return (__force __virtio16)cpu_to_le16(val);
23 else
24 return (__force __virtio16)val;
25}
26
27static inline u32 __virtio32_to_cpu(bool little_endian, __virtio32 val)
28{
29 if (little_endian)
30 return le32_to_cpu((__force __le32)val);
31 else
32 return (__force u32)val;
33}
34
35static inline __virtio32 __cpu_to_virtio32(bool little_endian, u32 val)
36{
37 if (little_endian)
38 return (__force __virtio32)cpu_to_le32(val);
39 else
40 return (__force __virtio32)val;
41}
42
43static inline u64 __virtio64_to_cpu(bool little_endian, __virtio64 val)
44{
45 if (little_endian)
46 return le64_to_cpu((__force __le64)val);
47 else
48 return (__force u64)val;
49}
50
51static inline __virtio64 __cpu_to_virtio64(bool little_endian, u64 val)
52{
53 if (little_endian)
54 return (__force __virtio64)cpu_to_le64(val);
55 else
56 return (__force __virtio64)val;
57}
58
59#endif /* _LINUX_VIRTIO_BYTEORDER */
diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
index 7f4ef66873ef..7979f850e7ac 100644
--- a/include/linux/virtio_config.h
+++ b/include/linux/virtio_config.h
@@ -4,6 +4,7 @@
4#include <linux/err.h> 4#include <linux/err.h>
5#include <linux/bug.h> 5#include <linux/bug.h>
6#include <linux/virtio.h> 6#include <linux/virtio.h>
7#include <linux/virtio_byteorder.h>
7#include <uapi/linux/virtio_config.h> 8#include <uapi/linux/virtio_config.h>
8 9
9/** 10/**
@@ -46,6 +47,7 @@
46 * vdev: the virtio_device 47 * vdev: the virtio_device
47 * This gives the final feature bits for the device: it can change 48 * This gives the final feature bits for the device: it can change
48 * the dev->feature bits if it wants. 49 * the dev->feature bits if it wants.
50 * Returns 0 on success or error status
49 * @bus_name: return the bus name associated with the device 51 * @bus_name: return the bus name associated with the device
50 * vdev: the virtio_device 52 * vdev: the virtio_device
51 * This returns a pointer to the bus name a la pci_name from which 53 * This returns a pointer to the bus name a la pci_name from which
@@ -66,8 +68,8 @@ struct virtio_config_ops {
66 vq_callback_t *callbacks[], 68 vq_callback_t *callbacks[],
67 const char *names[]); 69 const char *names[]);
68 void (*del_vqs)(struct virtio_device *); 70 void (*del_vqs)(struct virtio_device *);
69 u32 (*get_features)(struct virtio_device *vdev); 71 u64 (*get_features)(struct virtio_device *vdev);
70 void (*finalize_features)(struct virtio_device *vdev); 72 int (*finalize_features)(struct virtio_device *vdev);
71 const char *(*bus_name)(struct virtio_device *vdev); 73 const char *(*bus_name)(struct virtio_device *vdev);
72 int (*set_vq_affinity)(struct virtqueue *vq, int cpu); 74 int (*set_vq_affinity)(struct virtqueue *vq, int cpu);
73}; 75};
@@ -77,23 +79,70 @@ void virtio_check_driver_offered_feature(const struct virtio_device *vdev,
77 unsigned int fbit); 79 unsigned int fbit);
78 80
79/** 81/**
80 * virtio_has_feature - helper to determine if this device has this feature. 82 * __virtio_test_bit - helper to test feature bits. For use by transports.
83 * Devices should normally use virtio_has_feature,
84 * which includes more checks.
81 * @vdev: the device 85 * @vdev: the device
82 * @fbit: the feature bit 86 * @fbit: the feature bit
83 */ 87 */
84static inline bool virtio_has_feature(const struct virtio_device *vdev, 88static inline bool __virtio_test_bit(const struct virtio_device *vdev,
89 unsigned int fbit)
90{
91 /* Did you forget to fix assumptions on max features? */
92 if (__builtin_constant_p(fbit))
93 BUILD_BUG_ON(fbit >= 64);
94 else
95 BUG_ON(fbit >= 64);
96
97 return vdev->features & BIT_ULL(fbit);
98}
99
100/**
101 * __virtio_set_bit - helper to set feature bits. For use by transports.
102 * @vdev: the device
103 * @fbit: the feature bit
104 */
105static inline void __virtio_set_bit(struct virtio_device *vdev,
106 unsigned int fbit)
107{
108 /* Did you forget to fix assumptions on max features? */
109 if (__builtin_constant_p(fbit))
110 BUILD_BUG_ON(fbit >= 64);
111 else
112 BUG_ON(fbit >= 64);
113
114 vdev->features |= BIT_ULL(fbit);
115}
116
117/**
118 * __virtio_clear_bit - helper to clear feature bits. For use by transports.
119 * @vdev: the device
120 * @fbit: the feature bit
121 */
122static inline void __virtio_clear_bit(struct virtio_device *vdev,
85 unsigned int fbit) 123 unsigned int fbit)
86{ 124{
87 /* Did you forget to fix assumptions on max features? */ 125 /* Did you forget to fix assumptions on max features? */
88 if (__builtin_constant_p(fbit)) 126 if (__builtin_constant_p(fbit))
89 BUILD_BUG_ON(fbit >= 32); 127 BUILD_BUG_ON(fbit >= 64);
90 else 128 else
91 BUG_ON(fbit >= 32); 129 BUG_ON(fbit >= 64);
92 130
131 vdev->features &= ~BIT_ULL(fbit);
132}
133
134/**
135 * virtio_has_feature - helper to determine if this device has this feature.
136 * @vdev: the device
137 * @fbit: the feature bit
138 */
139static inline bool virtio_has_feature(const struct virtio_device *vdev,
140 unsigned int fbit)
141{
93 if (fbit < VIRTIO_TRANSPORT_F_START) 142 if (fbit < VIRTIO_TRANSPORT_F_START)
94 virtio_check_driver_offered_feature(vdev, fbit); 143 virtio_check_driver_offered_feature(vdev, fbit);
95 144
96 return test_bit(fbit, vdev->features); 145 return __virtio_test_bit(vdev, fbit);
97} 146}
98 147
99static inline 148static inline
@@ -152,6 +201,37 @@ int virtqueue_set_affinity(struct virtqueue *vq, int cpu)
152 return 0; 201 return 0;
153} 202}
154 203
204/* Memory accessors */
205static inline u16 virtio16_to_cpu(struct virtio_device *vdev, __virtio16 val)
206{
207 return __virtio16_to_cpu(virtio_has_feature(vdev, VIRTIO_F_VERSION_1), val);
208}
209
210static inline __virtio16 cpu_to_virtio16(struct virtio_device *vdev, u16 val)
211{
212 return __cpu_to_virtio16(virtio_has_feature(vdev, VIRTIO_F_VERSION_1), val);
213}
214
215static inline u32 virtio32_to_cpu(struct virtio_device *vdev, __virtio32 val)
216{
217 return __virtio32_to_cpu(virtio_has_feature(vdev, VIRTIO_F_VERSION_1), val);
218}
219
220static inline __virtio32 cpu_to_virtio32(struct virtio_device *vdev, u32 val)
221{
222 return __cpu_to_virtio32(virtio_has_feature(vdev, VIRTIO_F_VERSION_1), val);
223}
224
225static inline u64 virtio64_to_cpu(struct virtio_device *vdev, __virtio64 val)
226{
227 return __virtio64_to_cpu(virtio_has_feature(vdev, VIRTIO_F_VERSION_1), val);
228}
229
230static inline __virtio64 cpu_to_virtio64(struct virtio_device *vdev, u64 val)
231{
232 return __cpu_to_virtio64(virtio_has_feature(vdev, VIRTIO_F_VERSION_1), val);
233}
234
155/* Config space accessors. */ 235/* Config space accessors. */
156#define virtio_cread(vdev, structname, member, ptr) \ 236#define virtio_cread(vdev, structname, member, ptr) \
157 do { \ 237 do { \
@@ -239,12 +319,13 @@ static inline u16 virtio_cread16(struct virtio_device *vdev,
239{ 319{
240 u16 ret; 320 u16 ret;
241 vdev->config->get(vdev, offset, &ret, sizeof(ret)); 321 vdev->config->get(vdev, offset, &ret, sizeof(ret));
242 return ret; 322 return virtio16_to_cpu(vdev, (__force __virtio16)ret);
243} 323}
244 324
245static inline void virtio_cwrite16(struct virtio_device *vdev, 325static inline void virtio_cwrite16(struct virtio_device *vdev,
246 unsigned int offset, u16 val) 326 unsigned int offset, u16 val)
247{ 327{
328 val = (__force u16)cpu_to_virtio16(vdev, val);
248 vdev->config->set(vdev, offset, &val, sizeof(val)); 329 vdev->config->set(vdev, offset, &val, sizeof(val));
249} 330}
250 331
@@ -253,12 +334,13 @@ static inline u32 virtio_cread32(struct virtio_device *vdev,
253{ 334{
254 u32 ret; 335 u32 ret;
255 vdev->config->get(vdev, offset, &ret, sizeof(ret)); 336 vdev->config->get(vdev, offset, &ret, sizeof(ret));
256 return ret; 337 return virtio32_to_cpu(vdev, (__force __virtio32)ret);
257} 338}
258 339
259static inline void virtio_cwrite32(struct virtio_device *vdev, 340static inline void virtio_cwrite32(struct virtio_device *vdev,
260 unsigned int offset, u32 val) 341 unsigned int offset, u32 val)
261{ 342{
343 val = (__force u32)cpu_to_virtio32(vdev, val);
262 vdev->config->set(vdev, offset, &val, sizeof(val)); 344 vdev->config->set(vdev, offset, &val, sizeof(val));
263} 345}
264 346
@@ -267,12 +349,13 @@ static inline u64 virtio_cread64(struct virtio_device *vdev,
267{ 349{
268 u64 ret; 350 u64 ret;
269 vdev->config->get(vdev, offset, &ret, sizeof(ret)); 351 vdev->config->get(vdev, offset, &ret, sizeof(ret));
270 return ret; 352 return virtio64_to_cpu(vdev, (__force __virtio64)ret);
271} 353}
272 354
273static inline void virtio_cwrite64(struct virtio_device *vdev, 355static inline void virtio_cwrite64(struct virtio_device *vdev,
274 unsigned int offset, u64 val) 356 unsigned int offset, u64 val)
275{ 357{
358 val = (__force u64)cpu_to_virtio64(vdev, val);
276 vdev->config->set(vdev, offset, &val, sizeof(val)); 359 vdev->config->set(vdev, offset, &val, sizeof(val));
277} 360}
278 361
diff --git a/include/uapi/linux/Kbuild b/include/uapi/linux/Kbuild
index 401c4c3ec285..c54fcb5993c3 100644
--- a/include/uapi/linux/Kbuild
+++ b/include/uapi/linux/Kbuild
@@ -424,10 +424,12 @@ header-y += virtio_blk.h
424header-y += virtio_config.h 424header-y += virtio_config.h
425header-y += virtio_console.h 425header-y += virtio_console.h
426header-y += virtio_ids.h 426header-y += virtio_ids.h
427header-y += virtio_types.h
427header-y += virtio_net.h 428header-y += virtio_net.h
428header-y += virtio_pci.h 429header-y += virtio_pci.h
429header-y += virtio_ring.h 430header-y += virtio_ring.h
430header-y += virtio_rng.h 431header-y += virtio_rng.h
432header-y += virtio_scsi.h
431header-y += vm_sockets.h 433header-y += vm_sockets.h
432header-y += vt.h 434header-y += vt.h
433header-y += wait.h 435header-y += wait.h
diff --git a/include/uapi/linux/if_tun.h b/include/uapi/linux/if_tun.h
index e9502dd1ee2c..18b2403982f9 100644
--- a/include/uapi/linux/if_tun.h
+++ b/include/uapi/linux/if_tun.h
@@ -22,21 +22,11 @@
22 22
23/* Read queue size */ 23/* Read queue size */
24#define TUN_READQ_SIZE 500 24#define TUN_READQ_SIZE 500
25 25/* TUN device type flags: deprecated. Use IFF_TUN/IFF_TAP instead. */
26/* TUN device flags */ 26#define TUN_TUN_DEV IFF_TUN
27#define TUN_TUN_DEV 0x0001 27#define TUN_TAP_DEV IFF_TAP
28#define TUN_TAP_DEV 0x0002
29#define TUN_TYPE_MASK 0x000f 28#define TUN_TYPE_MASK 0x000f
30 29
31#define TUN_FASYNC 0x0010
32#define TUN_NOCHECKSUM 0x0020
33#define TUN_NO_PI 0x0040
34/* This flag has no real effect */
35#define TUN_ONE_QUEUE 0x0080
36#define TUN_PERSIST 0x0100
37#define TUN_VNET_HDR 0x0200
38#define TUN_TAP_MQ 0x0400
39
40/* Ioctl defines */ 30/* Ioctl defines */
41#define TUNSETNOCSUM _IOW('T', 200, int) 31#define TUNSETNOCSUM _IOW('T', 200, int)
42#define TUNSETDEBUG _IOW('T', 201, int) 32#define TUNSETDEBUG _IOW('T', 201, int)
@@ -67,6 +57,7 @@
67#define IFF_ONE_QUEUE 0x2000 57#define IFF_ONE_QUEUE 0x2000
68#define IFF_VNET_HDR 0x4000 58#define IFF_VNET_HDR 0x4000
69#define IFF_TUN_EXCL 0x8000 59#define IFF_TUN_EXCL 0x8000
60#define IFF_VNET_LE 0x10000
70#define IFF_MULTI_QUEUE 0x0100 61#define IFF_MULTI_QUEUE 0x0100
71#define IFF_ATTACH_QUEUE 0x0200 62#define IFF_ATTACH_QUEUE 0x0200
72#define IFF_DETACH_QUEUE 0x0400 63#define IFF_DETACH_QUEUE 0x0400
diff --git a/include/uapi/linux/virtio_blk.h b/include/uapi/linux/virtio_blk.h
index 9ad67b267584..247c8ba8544a 100644
--- a/include/uapi/linux/virtio_blk.h
+++ b/include/uapi/linux/virtio_blk.h
@@ -28,6 +28,7 @@
28#include <linux/types.h> 28#include <linux/types.h>
29#include <linux/virtio_ids.h> 29#include <linux/virtio_ids.h>
30#include <linux/virtio_config.h> 30#include <linux/virtio_config.h>
31#include <linux/virtio_types.h>
31 32
32/* Feature bits */ 33/* Feature bits */
33#define VIRTIO_BLK_F_BARRIER 0 /* Does host support barriers? */ 34#define VIRTIO_BLK_F_BARRIER 0 /* Does host support barriers? */
@@ -114,18 +115,18 @@ struct virtio_blk_config {
114/* This is the first element of the read scatter-gather list. */ 115/* This is the first element of the read scatter-gather list. */
115struct virtio_blk_outhdr { 116struct virtio_blk_outhdr {
116 /* VIRTIO_BLK_T* */ 117 /* VIRTIO_BLK_T* */
117 __u32 type; 118 __virtio32 type;
118 /* io priority. */ 119 /* io priority. */
119 __u32 ioprio; 120 __virtio32 ioprio;
120 /* Sector (ie. 512 byte offset) */ 121 /* Sector (ie. 512 byte offset) */
121 __u64 sector; 122 __virtio64 sector;
122}; 123};
123 124
124struct virtio_scsi_inhdr { 125struct virtio_scsi_inhdr {
125 __u32 errors; 126 __virtio32 errors;
126 __u32 data_len; 127 __virtio32 data_len;
127 __u32 sense_len; 128 __virtio32 sense_len;
128 __u32 residual; 129 __virtio32 residual;
129}; 130};
130 131
131/* And this is the final byte of the write scatter-gather list. */ 132/* And this is the final byte of the write scatter-gather list. */
diff --git a/include/uapi/linux/virtio_config.h b/include/uapi/linux/virtio_config.h
index 3ce768c6910d..a6d0cdeaacd4 100644
--- a/include/uapi/linux/virtio_config.h
+++ b/include/uapi/linux/virtio_config.h
@@ -38,14 +38,16 @@
38#define VIRTIO_CONFIG_S_DRIVER 2 38#define VIRTIO_CONFIG_S_DRIVER 2
39/* Driver has used its parts of the config, and is happy */ 39/* Driver has used its parts of the config, and is happy */
40#define VIRTIO_CONFIG_S_DRIVER_OK 4 40#define VIRTIO_CONFIG_S_DRIVER_OK 4
41/* Driver has finished configuring features */
42#define VIRTIO_CONFIG_S_FEATURES_OK 8
41/* We've given up on this device. */ 43/* We've given up on this device. */
42#define VIRTIO_CONFIG_S_FAILED 0x80 44#define VIRTIO_CONFIG_S_FAILED 0x80
43 45
44/* Some virtio feature bits (currently bits 28 through 31) are reserved for the 46/* Some virtio feature bits (currently bits 28 through 32) are reserved for the
45 * transport being used (eg. virtio_ring), the rest are per-device feature 47 * transport being used (eg. virtio_ring), the rest are per-device feature
46 * bits. */ 48 * bits. */
47#define VIRTIO_TRANSPORT_F_START 28 49#define VIRTIO_TRANSPORT_F_START 28
48#define VIRTIO_TRANSPORT_F_END 32 50#define VIRTIO_TRANSPORT_F_END 33
49 51
50/* Do we get callbacks when the ring is completely used, even if we've 52/* Do we get callbacks when the ring is completely used, even if we've
51 * suppressed them? */ 53 * suppressed them? */
@@ -54,4 +56,7 @@
54/* Can the device handle any descriptor layout? */ 56/* Can the device handle any descriptor layout? */
55#define VIRTIO_F_ANY_LAYOUT 27 57#define VIRTIO_F_ANY_LAYOUT 27
56 58
59/* v1.0 compliant. */
60#define VIRTIO_F_VERSION_1 32
61
57#endif /* _UAPI_LINUX_VIRTIO_CONFIG_H */ 62#endif /* _UAPI_LINUX_VIRTIO_CONFIG_H */
diff --git a/include/uapi/linux/virtio_console.h b/include/uapi/linux/virtio_console.h
index ba260dd0b33a..b7fb108c9310 100644
--- a/include/uapi/linux/virtio_console.h
+++ b/include/uapi/linux/virtio_console.h
@@ -32,6 +32,7 @@
32#ifndef _UAPI_LINUX_VIRTIO_CONSOLE_H 32#ifndef _UAPI_LINUX_VIRTIO_CONSOLE_H
33#define _UAPI_LINUX_VIRTIO_CONSOLE_H 33#define _UAPI_LINUX_VIRTIO_CONSOLE_H
34#include <linux/types.h> 34#include <linux/types.h>
35#include <linux/virtio_types.h>
35#include <linux/virtio_ids.h> 36#include <linux/virtio_ids.h>
36#include <linux/virtio_config.h> 37#include <linux/virtio_config.h>
37 38
@@ -58,9 +59,9 @@ struct virtio_console_config {
58 * particular port. 59 * particular port.
59 */ 60 */
60struct virtio_console_control { 61struct virtio_console_control {
61 __u32 id; /* Port number */ 62 __virtio32 id; /* Port number */
62 __u16 event; /* The kind of control event (see below) */ 63 __virtio16 event; /* The kind of control event (see below) */
63 __u16 value; /* Extra information for the key */ 64 __virtio16 value; /* Extra information for the key */
64}; 65};
65 66
66/* Some events for control messages */ 67/* Some events for control messages */
diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h
index 172a7f00780c..b5f1677b291c 100644
--- a/include/uapi/linux/virtio_net.h
+++ b/include/uapi/linux/virtio_net.h
@@ -28,6 +28,7 @@
28#include <linux/types.h> 28#include <linux/types.h>
29#include <linux/virtio_ids.h> 29#include <linux/virtio_ids.h>
30#include <linux/virtio_config.h> 30#include <linux/virtio_config.h>
31#include <linux/virtio_types.h>
31#include <linux/if_ether.h> 32#include <linux/if_ether.h>
32 33
33/* The feature bitmap for virtio net */ 34/* The feature bitmap for virtio net */
@@ -84,17 +85,17 @@ struct virtio_net_hdr {
84#define VIRTIO_NET_HDR_GSO_TCPV6 4 // GSO frame, IPv6 TCP 85#define VIRTIO_NET_HDR_GSO_TCPV6 4 // GSO frame, IPv6 TCP
85#define VIRTIO_NET_HDR_GSO_ECN 0x80 // TCP has ECN set 86#define VIRTIO_NET_HDR_GSO_ECN 0x80 // TCP has ECN set
86 __u8 gso_type; 87 __u8 gso_type;
87 __u16 hdr_len; /* Ethernet + IP + tcp/udp hdrs */ 88 __virtio16 hdr_len; /* Ethernet + IP + tcp/udp hdrs */
88 __u16 gso_size; /* Bytes to append to hdr_len per frame */ 89 __virtio16 gso_size; /* Bytes to append to hdr_len per frame */
89 __u16 csum_start; /* Position to start checksumming from */ 90 __virtio16 csum_start; /* Position to start checksumming from */
90 __u16 csum_offset; /* Offset after that to place checksum */ 91 __virtio16 csum_offset; /* Offset after that to place checksum */
91}; 92};
92 93
93/* This is the version of the header to use when the MRG_RXBUF 94/* This is the version of the header to use when the MRG_RXBUF
94 * feature has been negotiated. */ 95 * feature has been negotiated. */
95struct virtio_net_hdr_mrg_rxbuf { 96struct virtio_net_hdr_mrg_rxbuf {
96 struct virtio_net_hdr hdr; 97 struct virtio_net_hdr hdr;
97 __u16 num_buffers; /* Number of merged rx buffers */ 98 __virtio16 num_buffers; /* Number of merged rx buffers */
98}; 99};
99 100
100/* 101/*
@@ -149,7 +150,7 @@ typedef __u8 virtio_net_ctrl_ack;
149 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available. 150 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available.
150 */ 151 */
151struct virtio_net_ctrl_mac { 152struct virtio_net_ctrl_mac {
152 __u32 entries; 153 __virtio32 entries;
153 __u8 macs[][ETH_ALEN]; 154 __u8 macs[][ETH_ALEN];
154} __attribute__((packed)); 155} __attribute__((packed));
155 156
@@ -193,7 +194,7 @@ struct virtio_net_ctrl_mac {
193 * specified. 194 * specified.
194 */ 195 */
195struct virtio_net_ctrl_mq { 196struct virtio_net_ctrl_mq {
196 __u16 virtqueue_pairs; 197 __virtio16 virtqueue_pairs;
197}; 198};
198 199
199#define VIRTIO_NET_CTRL_MQ 4 200#define VIRTIO_NET_CTRL_MQ 4
diff --git a/include/uapi/linux/virtio_ring.h b/include/uapi/linux/virtio_ring.h
index a99f9b7caa67..61c818a7fe70 100644
--- a/include/uapi/linux/virtio_ring.h
+++ b/include/uapi/linux/virtio_ring.h
@@ -32,6 +32,7 @@
32 * 32 *
33 * Copyright Rusty Russell IBM Corporation 2007. */ 33 * Copyright Rusty Russell IBM Corporation 2007. */
34#include <linux/types.h> 34#include <linux/types.h>
35#include <linux/virtio_types.h>
35 36
36/* This marks a buffer as continuing via the next field. */ 37/* This marks a buffer as continuing via the next field. */
37#define VRING_DESC_F_NEXT 1 38#define VRING_DESC_F_NEXT 1
@@ -61,32 +62,32 @@
61/* Virtio ring descriptors: 16 bytes. These can chain together via "next". */ 62/* Virtio ring descriptors: 16 bytes. These can chain together via "next". */
62struct vring_desc { 63struct vring_desc {
63 /* Address (guest-physical). */ 64 /* Address (guest-physical). */
64 __u64 addr; 65 __virtio64 addr;
65 /* Length. */ 66 /* Length. */
66 __u32 len; 67 __virtio32 len;
67 /* The flags as indicated above. */ 68 /* The flags as indicated above. */
68 __u16 flags; 69 __virtio16 flags;
69 /* We chain unused descriptors via this, too */ 70 /* We chain unused descriptors via this, too */
70 __u16 next; 71 __virtio16 next;
71}; 72};
72 73
73struct vring_avail { 74struct vring_avail {
74 __u16 flags; 75 __virtio16 flags;
75 __u16 idx; 76 __virtio16 idx;
76 __u16 ring[]; 77 __virtio16 ring[];
77}; 78};
78 79
79/* u32 is used here for ids for padding reasons. */ 80/* u32 is used here for ids for padding reasons. */
80struct vring_used_elem { 81struct vring_used_elem {
81 /* Index of start of used descriptor chain. */ 82 /* Index of start of used descriptor chain. */
82 __u32 id; 83 __virtio32 id;
83 /* Total length of the descriptor chain which was used (written to) */ 84 /* Total length of the descriptor chain which was used (written to) */
84 __u32 len; 85 __virtio32 len;
85}; 86};
86 87
87struct vring_used { 88struct vring_used {
88 __u16 flags; 89 __virtio16 flags;
89 __u16 idx; 90 __virtio16 idx;
90 struct vring_used_elem ring[]; 91 struct vring_used_elem ring[];
91}; 92};
92 93
@@ -109,25 +110,25 @@ struct vring {
109 * struct vring_desc desc[num]; 110 * struct vring_desc desc[num];
110 * 111 *
111 * // A ring of available descriptor heads with free-running index. 112 * // A ring of available descriptor heads with free-running index.
112 * __u16 avail_flags; 113 * __virtio16 avail_flags;
113 * __u16 avail_idx; 114 * __virtio16 avail_idx;
114 * __u16 available[num]; 115 * __virtio16 available[num];
115 * __u16 used_event_idx; 116 * __virtio16 used_event_idx;
116 * 117 *
117 * // Padding to the next align boundary. 118 * // Padding to the next align boundary.
118 * char pad[]; 119 * char pad[];
119 * 120 *
120 * // A ring of used descriptor heads with free-running index. 121 * // A ring of used descriptor heads with free-running index.
121 * __u16 used_flags; 122 * __virtio16 used_flags;
122 * __u16 used_idx; 123 * __virtio16 used_idx;
123 * struct vring_used_elem used[num]; 124 * struct vring_used_elem used[num];
124 * __u16 avail_event_idx; 125 * __virtio16 avail_event_idx;
125 * }; 126 * };
126 */ 127 */
127/* We publish the used event index at the end of the available ring, and vice 128/* We publish the used event index at the end of the available ring, and vice
128 * versa. They are at the end for backwards compatibility. */ 129 * versa. They are at the end for backwards compatibility. */
129#define vring_used_event(vr) ((vr)->avail->ring[(vr)->num]) 130#define vring_used_event(vr) ((vr)->avail->ring[(vr)->num])
130#define vring_avail_event(vr) (*(__u16 *)&(vr)->used->ring[(vr)->num]) 131#define vring_avail_event(vr) (*(__virtio16 *)&(vr)->used->ring[(vr)->num])
131 132
132static inline void vring_init(struct vring *vr, unsigned int num, void *p, 133static inline void vring_init(struct vring *vr, unsigned int num, void *p,
133 unsigned long align) 134 unsigned long align)
@@ -135,15 +136,15 @@ static inline void vring_init(struct vring *vr, unsigned int num, void *p,
135 vr->num = num; 136 vr->num = num;
136 vr->desc = p; 137 vr->desc = p;
137 vr->avail = p + num*sizeof(struct vring_desc); 138 vr->avail = p + num*sizeof(struct vring_desc);
138 vr->used = (void *)(((unsigned long)&vr->avail->ring[num] + sizeof(__u16) 139 vr->used = (void *)(((unsigned long)&vr->avail->ring[num] + sizeof(__virtio16)
139 + align-1) & ~(align - 1)); 140 + align-1) & ~(align - 1));
140} 141}
141 142
142static inline unsigned vring_size(unsigned int num, unsigned long align) 143static inline unsigned vring_size(unsigned int num, unsigned long align)
143{ 144{
144 return ((sizeof(struct vring_desc) * num + sizeof(__u16) * (3 + num) 145 return ((sizeof(struct vring_desc) * num + sizeof(__virtio16) * (3 + num)
145 + align - 1) & ~(align - 1)) 146 + align - 1) & ~(align - 1))
146 + sizeof(__u16) * 3 + sizeof(struct vring_used_elem) * num; 147 + sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num;
147} 148}
148 149
149/* The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX */ 150/* The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX */
diff --git a/include/linux/virtio_scsi.h b/include/uapi/linux/virtio_scsi.h
index de429d1f4357..42b9370771b0 100644
--- a/include/linux/virtio_scsi.h
+++ b/include/uapi/linux/virtio_scsi.h
@@ -27,83 +27,85 @@
27#ifndef _LINUX_VIRTIO_SCSI_H 27#ifndef _LINUX_VIRTIO_SCSI_H
28#define _LINUX_VIRTIO_SCSI_H 28#define _LINUX_VIRTIO_SCSI_H
29 29
30#include <linux/virtio_types.h>
31
30#define VIRTIO_SCSI_CDB_SIZE 32 32#define VIRTIO_SCSI_CDB_SIZE 32
31#define VIRTIO_SCSI_SENSE_SIZE 96 33#define VIRTIO_SCSI_SENSE_SIZE 96
32 34
33/* SCSI command request, followed by data-out */ 35/* SCSI command request, followed by data-out */
34struct virtio_scsi_cmd_req { 36struct virtio_scsi_cmd_req {
35 u8 lun[8]; /* Logical Unit Number */ 37 __u8 lun[8]; /* Logical Unit Number */
36 u64 tag; /* Command identifier */ 38 __virtio64 tag; /* Command identifier */
37 u8 task_attr; /* Task attribute */ 39 __u8 task_attr; /* Task attribute */
38 u8 prio; /* SAM command priority field */ 40 __u8 prio; /* SAM command priority field */
39 u8 crn; 41 __u8 crn;
40 u8 cdb[VIRTIO_SCSI_CDB_SIZE]; 42 __u8 cdb[VIRTIO_SCSI_CDB_SIZE];
41} __packed; 43} __attribute__((packed));
42 44
43/* SCSI command request, followed by protection information */ 45/* SCSI command request, followed by protection information */
44struct virtio_scsi_cmd_req_pi { 46struct virtio_scsi_cmd_req_pi {
45 u8 lun[8]; /* Logical Unit Number */ 47 __u8 lun[8]; /* Logical Unit Number */
46 u64 tag; /* Command identifier */ 48 __virtio64 tag; /* Command identifier */
47 u8 task_attr; /* Task attribute */ 49 __u8 task_attr; /* Task attribute */
48 u8 prio; /* SAM command priority field */ 50 __u8 prio; /* SAM command priority field */
49 u8 crn; 51 __u8 crn;
50 u32 pi_bytesout; /* DataOUT PI Number of bytes */ 52 __virtio32 pi_bytesout; /* DataOUT PI Number of bytes */
51 u32 pi_bytesin; /* DataIN PI Number of bytes */ 53 __virtio32 pi_bytesin; /* DataIN PI Number of bytes */
52 u8 cdb[VIRTIO_SCSI_CDB_SIZE]; 54 __u8 cdb[VIRTIO_SCSI_CDB_SIZE];
53} __packed; 55} __attribute__((packed));
54 56
55/* Response, followed by sense data and data-in */ 57/* Response, followed by sense data and data-in */
56struct virtio_scsi_cmd_resp { 58struct virtio_scsi_cmd_resp {
57 u32 sense_len; /* Sense data length */ 59 __virtio32 sense_len; /* Sense data length */
58 u32 resid; /* Residual bytes in data buffer */ 60 __virtio32 resid; /* Residual bytes in data buffer */
59 u16 status_qualifier; /* Status qualifier */ 61 __virtio16 status_qualifier; /* Status qualifier */
60 u8 status; /* Command completion status */ 62 __u8 status; /* Command completion status */
61 u8 response; /* Response values */ 63 __u8 response; /* Response values */
62 u8 sense[VIRTIO_SCSI_SENSE_SIZE]; 64 __u8 sense[VIRTIO_SCSI_SENSE_SIZE];
63} __packed; 65} __attribute__((packed));
64 66
65/* Task Management Request */ 67/* Task Management Request */
66struct virtio_scsi_ctrl_tmf_req { 68struct virtio_scsi_ctrl_tmf_req {
67 u32 type; 69 __virtio32 type;
68 u32 subtype; 70 __virtio32 subtype;
69 u8 lun[8]; 71 __u8 lun[8];
70 u64 tag; 72 __virtio64 tag;
71} __packed; 73} __attribute__((packed));
72 74
73struct virtio_scsi_ctrl_tmf_resp { 75struct virtio_scsi_ctrl_tmf_resp {
74 u8 response; 76 __u8 response;
75} __packed; 77} __attribute__((packed));
76 78
77/* Asynchronous notification query/subscription */ 79/* Asynchronous notification query/subscription */
78struct virtio_scsi_ctrl_an_req { 80struct virtio_scsi_ctrl_an_req {
79 u32 type; 81 __virtio32 type;
80 u8 lun[8]; 82 __u8 lun[8];
81 u32 event_requested; 83 __virtio32 event_requested;
82} __packed; 84} __attribute__((packed));
83 85
84struct virtio_scsi_ctrl_an_resp { 86struct virtio_scsi_ctrl_an_resp {
85 u32 event_actual; 87 __virtio32 event_actual;
86 u8 response; 88 __u8 response;
87} __packed; 89} __attribute__((packed));
88 90
89struct virtio_scsi_event { 91struct virtio_scsi_event {
90 u32 event; 92 __virtio32 event;
91 u8 lun[8]; 93 __u8 lun[8];
92 u32 reason; 94 __virtio32 reason;
93} __packed; 95} __attribute__((packed));
94 96
95struct virtio_scsi_config { 97struct virtio_scsi_config {
96 u32 num_queues; 98 __u32 num_queues;
97 u32 seg_max; 99 __u32 seg_max;
98 u32 max_sectors; 100 __u32 max_sectors;
99 u32 cmd_per_lun; 101 __u32 cmd_per_lun;
100 u32 event_info_size; 102 __u32 event_info_size;
101 u32 sense_size; 103 __u32 sense_size;
102 u32 cdb_size; 104 __u32 cdb_size;
103 u16 max_channel; 105 __u16 max_channel;
104 u16 max_target; 106 __u16 max_target;
105 u32 max_lun; 107 __u32 max_lun;
106} __packed; 108} __attribute__((packed));
107 109
108/* Feature Bits */ 110/* Feature Bits */
109#define VIRTIO_SCSI_F_INOUT 0 111#define VIRTIO_SCSI_F_INOUT 0
diff --git a/include/uapi/linux/virtio_types.h b/include/uapi/linux/virtio_types.h
new file mode 100644
index 000000000000..e845e8c4cbee
--- /dev/null
+++ b/include/uapi/linux/virtio_types.h
@@ -0,0 +1,46 @@
1#ifndef _UAPI_LINUX_VIRTIO_TYPES_H
2#define _UAPI_LINUX_VIRTIO_TYPES_H
3/* Type definitions for virtio implementations.
4 *
5 * This header is BSD licensed so anyone can use the definitions to implement
6 * compatible drivers/servers.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of IBM nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 * Copyright (C) 2014 Red Hat, Inc.
32 * Author: Michael S. Tsirkin <mst@redhat.com>
33 */
34#include <linux/types.h>
35
36/*
37 * __virtio{16,32,64} have the following meaning:
38 * - __u{16,32,64} for virtio devices in legacy mode, accessed in native endian
39 * - __le{16,32,64} for standard-compliant virtio devices
40 */
41
42typedef __u16 __bitwise__ __virtio16;
43typedef __u32 __bitwise__ __virtio32;
44typedef __u64 __bitwise__ __virtio64;
45
46#endif /* _UAPI_LINUX_VIRTIO_TYPES_H */
diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index 07c04a841ba0..586229a14ad3 100644
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -2444,13 +2444,15 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2444 goto out_unlock; 2444 goto out_unlock;
2445 2445
2446 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 2446 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2447 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 > 2447 (__virtio16_to_cpu(false, vnet_hdr.csum_start) +
2448 vnet_hdr.hdr_len)) 2448 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2 >
2449 vnet_hdr.hdr_len = vnet_hdr.csum_start + 2449 __virtio16_to_cpu(false, vnet_hdr.hdr_len)))
2450 vnet_hdr.csum_offset + 2; 2450 vnet_hdr.hdr_len = __cpu_to_virtio16(false,
2451 __virtio16_to_cpu(false, vnet_hdr.csum_start) +
2452 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2);
2451 2453
2452 err = -EINVAL; 2454 err = -EINVAL;
2453 if (vnet_hdr.hdr_len > len) 2455 if (__virtio16_to_cpu(false, vnet_hdr.hdr_len) > len)
2454 goto out_unlock; 2456 goto out_unlock;
2455 2457
2456 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) { 2458 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
@@ -2492,7 +2494,8 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2492 err = -ENOBUFS; 2494 err = -ENOBUFS;
2493 hlen = LL_RESERVED_SPACE(dev); 2495 hlen = LL_RESERVED_SPACE(dev);
2494 tlen = dev->needed_tailroom; 2496 tlen = dev->needed_tailroom;
2495 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len, 2497 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len,
2498 __virtio16_to_cpu(false, vnet_hdr.hdr_len),
2496 msg->msg_flags & MSG_DONTWAIT, &err); 2499 msg->msg_flags & MSG_DONTWAIT, &err);
2497 if (skb == NULL) 2500 if (skb == NULL)
2498 goto out_unlock; 2501 goto out_unlock;
@@ -2534,14 +2537,16 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2534 2537
2535 if (po->has_vnet_hdr) { 2538 if (po->has_vnet_hdr) {
2536 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 2539 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2537 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start, 2540 u16 s = __virtio16_to_cpu(false, vnet_hdr.csum_start);
2538 vnet_hdr.csum_offset)) { 2541 u16 o = __virtio16_to_cpu(false, vnet_hdr.csum_offset);
2542 if (!skb_partial_csum_set(skb, s, o)) {
2539 err = -EINVAL; 2543 err = -EINVAL;
2540 goto out_free; 2544 goto out_free;
2541 } 2545 }
2542 } 2546 }
2543 2547
2544 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size; 2548 skb_shinfo(skb)->gso_size =
2549 __virtio16_to_cpu(false, vnet_hdr.gso_size);
2545 skb_shinfo(skb)->gso_type = gso_type; 2550 skb_shinfo(skb)->gso_type = gso_type;
2546 2551
2547 /* Header must be checked, and gso_segs computed. */ 2552 /* Header must be checked, and gso_segs computed. */
@@ -2912,8 +2917,10 @@ static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2912 struct skb_shared_info *sinfo = skb_shinfo(skb); 2917 struct skb_shared_info *sinfo = skb_shinfo(skb);
2913 2918
2914 /* This is a hint as to how much should be linear. */ 2919 /* This is a hint as to how much should be linear. */
2915 vnet_hdr.hdr_len = skb_headlen(skb); 2920 vnet_hdr.hdr_len =
2916 vnet_hdr.gso_size = sinfo->gso_size; 2921 __cpu_to_virtio16(false, skb_headlen(skb));
2922 vnet_hdr.gso_size =
2923 __cpu_to_virtio16(false, sinfo->gso_size);
2917 if (sinfo->gso_type & SKB_GSO_TCPV4) 2924 if (sinfo->gso_type & SKB_GSO_TCPV4)
2918 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 2925 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2919 else if (sinfo->gso_type & SKB_GSO_TCPV6) 2926 else if (sinfo->gso_type & SKB_GSO_TCPV6)
@@ -2931,8 +2938,10 @@ static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2931 2938
2932 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2939 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2933 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 2940 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
2934 vnet_hdr.csum_start = skb_checksum_start_offset(skb); 2941 vnet_hdr.csum_start = __cpu_to_virtio16(false,
2935 vnet_hdr.csum_offset = skb->csum_offset; 2942 skb_checksum_start_offset(skb));
2943 vnet_hdr.csum_offset = __cpu_to_virtio16(false,
2944 skb->csum_offset);
2936 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 2945 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2937 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID; 2946 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
2938 } /* else everything is zero */ 2947 } /* else everything is zero */
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
index 5a2d1f0f6bc7..8eb6421761cd 100644
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -6,31 +6,11 @@
6/* TODO: empty stubs for now. Broken but enough for virtio_ring.c */ 6/* TODO: empty stubs for now. Broken but enough for virtio_ring.c */
7#define list_add_tail(a, b) do {} while (0) 7#define list_add_tail(a, b) do {} while (0)
8#define list_del(a) do {} while (0) 8#define list_del(a) do {} while (0)
9
10#define BIT_WORD(nr) ((nr) / BITS_PER_LONG)
11#define BITS_PER_BYTE 8
12#define BITS_PER_LONG (sizeof(long) * BITS_PER_BYTE)
13#define BIT_MASK(nr) (1UL << ((nr) % BITS_PER_LONG))
14
15/* TODO: Not atomic as it should be:
16 * we don't use this for anything important. */
17static inline void clear_bit(int nr, volatile unsigned long *addr)
18{
19 unsigned long mask = BIT_MASK(nr);
20 unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
21
22 *p &= ~mask;
23}
24
25static inline int test_bit(int nr, const volatile unsigned long *addr)
26{
27 return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
28}
29/* end of stubs */ 9/* end of stubs */
30 10
31struct virtio_device { 11struct virtio_device {
32 void *dev; 12 void *dev;
33 unsigned long features[1]; 13 u64 features;
34}; 14};
35 15
36struct virtqueue { 16struct virtqueue {
diff --git a/tools/virtio/linux/virtio_config.h b/tools/virtio/linux/virtio_config.h
index 5049967f99f7..83b27e8e9d72 100644
--- a/tools/virtio/linux/virtio_config.h
+++ b/tools/virtio/linux/virtio_config.h
@@ -2,5 +2,5 @@
2#define VIRTIO_TRANSPORT_F_END 32 2#define VIRTIO_TRANSPORT_F_END 32
3 3
4#define virtio_has_feature(dev, feature) \ 4#define virtio_has_feature(dev, feature) \
5 test_bit((feature), (dev)->features) 5 (__virtio_test_bit((dev), feature))
6 6
diff --git a/tools/virtio/virtio_test.c b/tools/virtio/virtio_test.c
index 00ea679b3826..db3437c641a6 100644
--- a/tools/virtio/virtio_test.c
+++ b/tools/virtio/virtio_test.c
@@ -60,7 +60,7 @@ void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info)
60{ 60{
61 struct vhost_vring_state state = { .index = info->idx }; 61 struct vhost_vring_state state = { .index = info->idx };
62 struct vhost_vring_file file = { .index = info->idx }; 62 struct vhost_vring_file file = { .index = info->idx };
63 unsigned long long features = dev->vdev.features[0]; 63 unsigned long long features = dev->vdev.features;
64 struct vhost_vring_addr addr = { 64 struct vhost_vring_addr addr = {
65 .index = info->idx, 65 .index = info->idx,
66 .desc_user_addr = (uint64_t)(unsigned long)info->vring.desc, 66 .desc_user_addr = (uint64_t)(unsigned long)info->vring.desc,
@@ -113,8 +113,7 @@ static void vdev_info_init(struct vdev_info* dev, unsigned long long features)
113{ 113{
114 int r; 114 int r;
115 memset(dev, 0, sizeof *dev); 115 memset(dev, 0, sizeof *dev);
116 dev->vdev.features[0] = features; 116 dev->vdev.features = features;
117 dev->vdev.features[1] = features >> 32;
118 dev->buf_size = 1024; 117 dev->buf_size = 1024;
119 dev->buf = malloc(dev->buf_size); 118 dev->buf = malloc(dev->buf_size);
120 assert(dev->buf); 119 assert(dev->buf);
diff --git a/tools/virtio/vringh_test.c b/tools/virtio/vringh_test.c
index 14a4f4cab5b9..9d4b1bca54be 100644
--- a/tools/virtio/vringh_test.c
+++ b/tools/virtio/vringh_test.c
@@ -304,7 +304,7 @@ static int parallel_test(unsigned long features,
304 close(to_guest[1]); 304 close(to_guest[1]);
305 close(to_host[0]); 305 close(to_host[0]);
306 306
307 gvdev.vdev.features[0] = features; 307 gvdev.vdev.features = features;
308 gvdev.to_host_fd = to_host[1]; 308 gvdev.to_host_fd = to_host[1];
309 gvdev.notifies = 0; 309 gvdev.notifies = 0;
310 310
@@ -449,13 +449,13 @@ int main(int argc, char *argv[])
449 bool fast_vringh = false, parallel = false; 449 bool fast_vringh = false, parallel = false;
450 450
451 getrange = getrange_iov; 451 getrange = getrange_iov;
452 vdev.features[0] = 0; 452 vdev.features = 0;
453 453
454 while (argv[1]) { 454 while (argv[1]) {
455 if (strcmp(argv[1], "--indirect") == 0) 455 if (strcmp(argv[1], "--indirect") == 0)
456 vdev.features[0] |= (1 << VIRTIO_RING_F_INDIRECT_DESC); 456 __virtio_set_bit(&vdev, VIRTIO_RING_F_INDIRECT_DESC);
457 else if (strcmp(argv[1], "--eventidx") == 0) 457 else if (strcmp(argv[1], "--eventidx") == 0)
458 vdev.features[0] |= (1 << VIRTIO_RING_F_EVENT_IDX); 458 __virtio_set_bit(&vdev, VIRTIO_RING_F_EVENT_IDX);
459 else if (strcmp(argv[1], "--slow-range") == 0) 459 else if (strcmp(argv[1], "--slow-range") == 0)
460 getrange = getrange_slow; 460 getrange = getrange_slow;
461 else if (strcmp(argv[1], "--fast-vringh") == 0) 461 else if (strcmp(argv[1], "--fast-vringh") == 0)
@@ -468,7 +468,7 @@ int main(int argc, char *argv[])
468 } 468 }
469 469
470 if (parallel) 470 if (parallel)
471 return parallel_test(vdev.features[0], getrange, fast_vringh); 471 return parallel_test(vdev.features, getrange, fast_vringh);
472 472
473 if (posix_memalign(&__user_addr_min, PAGE_SIZE, USER_MEM) != 0) 473 if (posix_memalign(&__user_addr_min, PAGE_SIZE, USER_MEM) != 0)
474 abort(); 474 abort();
@@ -483,7 +483,7 @@ int main(int argc, char *argv[])
483 483
484 /* Set up host side. */ 484 /* Set up host side. */
485 vring_init(&vrh.vring, RINGSIZE, __user_addr_min, ALIGN); 485 vring_init(&vrh.vring, RINGSIZE, __user_addr_min, ALIGN);
486 vringh_init_user(&vrh, vdev.features[0], RINGSIZE, true, 486 vringh_init_user(&vrh, vdev.features, RINGSIZE, true,
487 vrh.vring.desc, vrh.vring.avail, vrh.vring.used); 487 vrh.vring.desc, vrh.vring.avail, vrh.vring.used);
488 488
489 /* No descriptor to get yet... */ 489 /* No descriptor to get yet... */
@@ -652,13 +652,13 @@ int main(int argc, char *argv[])
652 } 652 }
653 653
654 /* Test weird (but legal!) indirect. */ 654 /* Test weird (but legal!) indirect. */
655 if (vdev.features[0] & (1 << VIRTIO_RING_F_INDIRECT_DESC)) { 655 if (__virtio_test_bit(&vdev, VIRTIO_RING_F_INDIRECT_DESC)) {
656 char *data = __user_addr_max - USER_MEM/4; 656 char *data = __user_addr_max - USER_MEM/4;
657 struct vring_desc *d = __user_addr_max - USER_MEM/2; 657 struct vring_desc *d = __user_addr_max - USER_MEM/2;
658 struct vring vring; 658 struct vring vring;
659 659
660 /* Force creation of direct, which we modify. */ 660 /* Force creation of direct, which we modify. */
661 vdev.features[0] &= ~(1 << VIRTIO_RING_F_INDIRECT_DESC); 661 __virtio_clear_bit(&vdev, VIRTIO_RING_F_INDIRECT_DESC);
662 vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &vdev, true, 662 vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &vdev, true,
663 __user_addr_min, 663 __user_addr_min,
664 never_notify_host, 664 never_notify_host,
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-04 14:03:30 -0500