aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/vhost/Kconfig8
-rw-r--r--drivers/vhost/Makefile3
-rw-r--r--drivers/vhost/net.c4
-rw-r--r--drivers/vhost/scsi.c472
-rw-r--r--drivers/vhost/test.c33
-rw-r--r--drivers/vhost/vhost.c86
-rw-r--r--drivers/vhost/vhost.h2
7 files changed, 349 insertions, 259 deletions
diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig
index 8b9226da3f54..017a1e8a8f6f 100644
--- a/drivers/vhost/Kconfig
+++ b/drivers/vhost/Kconfig
@@ -1,6 +1,7 @@
1config VHOST_NET 1config VHOST_NET
2 tristate "Host kernel accelerator for virtio net" 2 tristate "Host kernel accelerator for virtio net"
3 depends on NET && EVENTFD && (TUN || !TUN) && (MACVTAP || !MACVTAP) 3 depends on NET && EVENTFD && (TUN || !TUN) && (MACVTAP || !MACVTAP)
4 select VHOST
4 select VHOST_RING 5 select VHOST_RING
5 ---help--- 6 ---help---
6 This kernel module can be loaded in host kernel to accelerate 7 This kernel module can be loaded in host kernel to accelerate
@@ -13,6 +14,7 @@ config VHOST_NET
13config VHOST_SCSI 14config VHOST_SCSI
14 tristate "VHOST_SCSI TCM fabric driver" 15 tristate "VHOST_SCSI TCM fabric driver"
15 depends on TARGET_CORE && EVENTFD && m 16 depends on TARGET_CORE && EVENTFD && m
17 select VHOST
16 select VHOST_RING 18 select VHOST_RING
17 default n 19 default n
18 ---help--- 20 ---help---
@@ -24,3 +26,9 @@ config VHOST_RING
24 ---help--- 26 ---help---
25 This option is selected by any driver which needs to access 27 This option is selected by any driver which needs to access
26 the host side of a virtio ring. 28 the host side of a virtio ring.
29
30config VHOST
31 tristate
32 ---help---
33 This option is selected by any driver which needs to access
34 the core of vhost.
diff --git a/drivers/vhost/Makefile b/drivers/vhost/Makefile
index 654e9afb11f5..e0441c34db1c 100644
--- a/drivers/vhost/Makefile
+++ b/drivers/vhost/Makefile
@@ -1,7 +1,8 @@
1obj-$(CONFIG_VHOST_NET) += vhost_net.o 1obj-$(CONFIG_VHOST_NET) += vhost_net.o
2vhost_net-y := vhost.o net.o 2vhost_net-y := net.o
3 3
4obj-$(CONFIG_VHOST_SCSI) += vhost_scsi.o 4obj-$(CONFIG_VHOST_SCSI) += vhost_scsi.o
5vhost_scsi-y := scsi.o 5vhost_scsi-y := scsi.o
6 6
7obj-$(CONFIG_VHOST_RING) += vringh.o 7obj-$(CONFIG_VHOST_RING) += vringh.o
8obj-$(CONFIG_VHOST) += vhost.o
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 8ca5ac71b845..027be91db139 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -168,7 +168,7 @@ static void vhost_net_clear_ubuf_info(struct vhost_net *n)
168 } 168 }
169} 169}
170 170
171int vhost_net_set_ubuf_info(struct vhost_net *n) 171static int vhost_net_set_ubuf_info(struct vhost_net *n)
172{ 172{
173 bool zcopy; 173 bool zcopy;
174 int i; 174 int i;
@@ -189,7 +189,7 @@ err:
189 return -ENOMEM; 189 return -ENOMEM;
190} 190}
191 191
192void vhost_net_vq_reset(struct vhost_net *n) 192static void vhost_net_vq_reset(struct vhost_net *n)
193{ 193{
194 int i; 194 int i;
195 195
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index 701420297225..4264840ef7dc 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -49,7 +49,6 @@
49#include <linux/llist.h> 49#include <linux/llist.h>
50#include <linux/bitmap.h> 50#include <linux/bitmap.h>
51 51
52#include "vhost.c"
53#include "vhost.h" 52#include "vhost.h"
54 53
55#define TCM_VHOST_VERSION "v0.1" 54#define TCM_VHOST_VERSION "v0.1"
@@ -116,7 +115,6 @@ struct tcm_vhost_nacl {
116 struct se_node_acl se_node_acl; 115 struct se_node_acl se_node_acl;
117}; 116};
118 117
119struct vhost_scsi;
120struct tcm_vhost_tpg { 118struct tcm_vhost_tpg {
121 /* Vhost port target portal group tag for TCM */ 119 /* Vhost port target portal group tag for TCM */
122 u16 tport_tpgt; 120 u16 tport_tpgt;
@@ -218,7 +216,7 @@ static int iov_num_pages(struct iovec *iov)
218 ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT; 216 ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT;
219} 217}
220 218
221void tcm_vhost_done_inflight(struct kref *kref) 219static void tcm_vhost_done_inflight(struct kref *kref)
222{ 220{
223 struct vhost_scsi_inflight *inflight; 221 struct vhost_scsi_inflight *inflight;
224 222
@@ -329,11 +327,12 @@ static u32 tcm_vhost_get_default_depth(struct se_portal_group *se_tpg)
329 return 1; 327 return 1;
330} 328}
331 329
332static u32 tcm_vhost_get_pr_transport_id(struct se_portal_group *se_tpg, 330static u32
333 struct se_node_acl *se_nacl, 331tcm_vhost_get_pr_transport_id(struct se_portal_group *se_tpg,
334 struct t10_pr_registration *pr_reg, 332 struct se_node_acl *se_nacl,
335 int *format_code, 333 struct t10_pr_registration *pr_reg,
336 unsigned char *buf) 334 int *format_code,
335 unsigned char *buf)
337{ 336{
338 struct tcm_vhost_tpg *tpg = container_of(se_tpg, 337 struct tcm_vhost_tpg *tpg = container_of(se_tpg,
339 struct tcm_vhost_tpg, se_tpg); 338 struct tcm_vhost_tpg, se_tpg);
@@ -359,10 +358,11 @@ static u32 tcm_vhost_get_pr_transport_id(struct se_portal_group *se_tpg,
359 format_code, buf); 358 format_code, buf);
360} 359}
361 360
362static u32 tcm_vhost_get_pr_transport_id_len(struct se_portal_group *se_tpg, 361static u32
363 struct se_node_acl *se_nacl, 362tcm_vhost_get_pr_transport_id_len(struct se_portal_group *se_tpg,
364 struct t10_pr_registration *pr_reg, 363 struct se_node_acl *se_nacl,
365 int *format_code) 364 struct t10_pr_registration *pr_reg,
365 int *format_code)
366{ 366{
367 struct tcm_vhost_tpg *tpg = container_of(se_tpg, 367 struct tcm_vhost_tpg *tpg = container_of(se_tpg,
368 struct tcm_vhost_tpg, se_tpg); 368 struct tcm_vhost_tpg, se_tpg);
@@ -388,10 +388,11 @@ static u32 tcm_vhost_get_pr_transport_id_len(struct se_portal_group *se_tpg,
388 format_code); 388 format_code);
389} 389}
390 390
391static char *tcm_vhost_parse_pr_out_transport_id(struct se_portal_group *se_tpg, 391static char *
392 const char *buf, 392tcm_vhost_parse_pr_out_transport_id(struct se_portal_group *se_tpg,
393 u32 *out_tid_len, 393 const char *buf,
394 char **port_nexus_ptr) 394 u32 *out_tid_len,
395 char **port_nexus_ptr)
395{ 396{
396 struct tcm_vhost_tpg *tpg = container_of(se_tpg, 397 struct tcm_vhost_tpg *tpg = container_of(se_tpg,
397 struct tcm_vhost_tpg, se_tpg); 398 struct tcm_vhost_tpg, se_tpg);
@@ -417,8 +418,8 @@ static char *tcm_vhost_parse_pr_out_transport_id(struct se_portal_group *se_tpg,
417 port_nexus_ptr); 418 port_nexus_ptr);
418} 419}
419 420
420static struct se_node_acl *tcm_vhost_alloc_fabric_acl( 421static struct se_node_acl *
421 struct se_portal_group *se_tpg) 422tcm_vhost_alloc_fabric_acl(struct se_portal_group *se_tpg)
422{ 423{
423 struct tcm_vhost_nacl *nacl; 424 struct tcm_vhost_nacl *nacl;
424 425
@@ -431,8 +432,9 @@ static struct se_node_acl *tcm_vhost_alloc_fabric_acl(
431 return &nacl->se_node_acl; 432 return &nacl->se_node_acl;
432} 433}
433 434
434static void tcm_vhost_release_fabric_acl(struct se_portal_group *se_tpg, 435static void
435 struct se_node_acl *se_nacl) 436tcm_vhost_release_fabric_acl(struct se_portal_group *se_tpg,
437 struct se_node_acl *se_nacl)
436{ 438{
437 struct tcm_vhost_nacl *nacl = container_of(se_nacl, 439 struct tcm_vhost_nacl *nacl = container_of(se_nacl,
438 struct tcm_vhost_nacl, se_node_acl); 440 struct tcm_vhost_nacl, se_node_acl);
@@ -491,28 +493,28 @@ static int tcm_vhost_get_cmd_state(struct se_cmd *se_cmd)
491 return 0; 493 return 0;
492} 494}
493 495
494static void vhost_scsi_complete_cmd(struct tcm_vhost_cmd *tv_cmd) 496static void vhost_scsi_complete_cmd(struct tcm_vhost_cmd *cmd)
495{ 497{
496 struct vhost_scsi *vs = tv_cmd->tvc_vhost; 498 struct vhost_scsi *vs = cmd->tvc_vhost;
497 499
498 llist_add(&tv_cmd->tvc_completion_list, &vs->vs_completion_list); 500 llist_add(&cmd->tvc_completion_list, &vs->vs_completion_list);
499 501
500 vhost_work_queue(&vs->dev, &vs->vs_completion_work); 502 vhost_work_queue(&vs->dev, &vs->vs_completion_work);
501} 503}
502 504
503static int tcm_vhost_queue_data_in(struct se_cmd *se_cmd) 505static int tcm_vhost_queue_data_in(struct se_cmd *se_cmd)
504{ 506{
505 struct tcm_vhost_cmd *tv_cmd = container_of(se_cmd, 507 struct tcm_vhost_cmd *cmd = container_of(se_cmd,
506 struct tcm_vhost_cmd, tvc_se_cmd); 508 struct tcm_vhost_cmd, tvc_se_cmd);
507 vhost_scsi_complete_cmd(tv_cmd); 509 vhost_scsi_complete_cmd(cmd);
508 return 0; 510 return 0;
509} 511}
510 512
511static int tcm_vhost_queue_status(struct se_cmd *se_cmd) 513static int tcm_vhost_queue_status(struct se_cmd *se_cmd)
512{ 514{
513 struct tcm_vhost_cmd *tv_cmd = container_of(se_cmd, 515 struct tcm_vhost_cmd *cmd = container_of(se_cmd,
514 struct tcm_vhost_cmd, tvc_se_cmd); 516 struct tcm_vhost_cmd, tvc_se_cmd);
515 vhost_scsi_complete_cmd(tv_cmd); 517 vhost_scsi_complete_cmd(cmd);
516 return 0; 518 return 0;
517} 519}
518 520
@@ -527,8 +529,9 @@ static void tcm_vhost_free_evt(struct vhost_scsi *vs, struct tcm_vhost_evt *evt)
527 kfree(evt); 529 kfree(evt);
528} 530}
529 531
530static struct tcm_vhost_evt *tcm_vhost_allocate_evt(struct vhost_scsi *vs, 532static struct tcm_vhost_evt *
531 u32 event, u32 reason) 533tcm_vhost_allocate_evt(struct vhost_scsi *vs,
534 u32 event, u32 reason)
532{ 535{
533 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; 536 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
534 struct tcm_vhost_evt *evt; 537 struct tcm_vhost_evt *evt;
@@ -552,28 +555,28 @@ static struct tcm_vhost_evt *tcm_vhost_allocate_evt(struct vhost_scsi *vs,
552 return evt; 555 return evt;
553} 556}
554 557
555static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) 558static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *cmd)
556{ 559{
557 struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd; 560 struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
558 561
559 /* TODO locking against target/backend threads? */ 562 /* TODO locking against target/backend threads? */
560 transport_generic_free_cmd(se_cmd, 1); 563 transport_generic_free_cmd(se_cmd, 1);
561 564
562 if (tv_cmd->tvc_sgl_count) { 565 if (cmd->tvc_sgl_count) {
563 u32 i; 566 u32 i;
564 for (i = 0; i < tv_cmd->tvc_sgl_count; i++) 567 for (i = 0; i < cmd->tvc_sgl_count; i++)
565 put_page(sg_page(&tv_cmd->tvc_sgl[i])); 568 put_page(sg_page(&cmd->tvc_sgl[i]));
566 569
567 kfree(tv_cmd->tvc_sgl); 570 kfree(cmd->tvc_sgl);
568 } 571 }
569 572
570 tcm_vhost_put_inflight(tv_cmd->inflight); 573 tcm_vhost_put_inflight(cmd->inflight);
571 574
572 kfree(tv_cmd); 575 kfree(cmd);
573} 576}
574 577
575static void tcm_vhost_do_evt_work(struct vhost_scsi *vs, 578static void
576 struct tcm_vhost_evt *evt) 579tcm_vhost_do_evt_work(struct vhost_scsi *vs, struct tcm_vhost_evt *evt)
577{ 580{
578 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; 581 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
579 struct virtio_scsi_event *event = &evt->event; 582 struct virtio_scsi_event *event = &evt->event;
@@ -652,7 +655,7 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
652 vs_completion_work); 655 vs_completion_work);
653 DECLARE_BITMAP(signal, VHOST_SCSI_MAX_VQ); 656 DECLARE_BITMAP(signal, VHOST_SCSI_MAX_VQ);
654 struct virtio_scsi_cmd_resp v_rsp; 657 struct virtio_scsi_cmd_resp v_rsp;
655 struct tcm_vhost_cmd *tv_cmd; 658 struct tcm_vhost_cmd *cmd;
656 struct llist_node *llnode; 659 struct llist_node *llnode;
657 struct se_cmd *se_cmd; 660 struct se_cmd *se_cmd;
658 int ret, vq; 661 int ret, vq;
@@ -660,32 +663,32 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
660 bitmap_zero(signal, VHOST_SCSI_MAX_VQ); 663 bitmap_zero(signal, VHOST_SCSI_MAX_VQ);
661 llnode = llist_del_all(&vs->vs_completion_list); 664 llnode = llist_del_all(&vs->vs_completion_list);
662 while (llnode) { 665 while (llnode) {
663 tv_cmd = llist_entry(llnode, struct tcm_vhost_cmd, 666 cmd = llist_entry(llnode, struct tcm_vhost_cmd,
664 tvc_completion_list); 667 tvc_completion_list);
665 llnode = llist_next(llnode); 668 llnode = llist_next(llnode);
666 se_cmd = &tv_cmd->tvc_se_cmd; 669 se_cmd = &cmd->tvc_se_cmd;
667 670
668 pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__, 671 pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__,
669 tv_cmd, se_cmd->residual_count, se_cmd->scsi_status); 672 cmd, se_cmd->residual_count, se_cmd->scsi_status);
670 673
671 memset(&v_rsp, 0, sizeof(v_rsp)); 674 memset(&v_rsp, 0, sizeof(v_rsp));
672 v_rsp.resid = se_cmd->residual_count; 675 v_rsp.resid = se_cmd->residual_count;
673 /* TODO is status_qualifier field needed? */ 676 /* TODO is status_qualifier field needed? */
674 v_rsp.status = se_cmd->scsi_status; 677 v_rsp.status = se_cmd->scsi_status;
675 v_rsp.sense_len = se_cmd->scsi_sense_length; 678 v_rsp.sense_len = se_cmd->scsi_sense_length;
676 memcpy(v_rsp.sense, tv_cmd->tvc_sense_buf, 679 memcpy(v_rsp.sense, cmd->tvc_sense_buf,
677 v_rsp.sense_len); 680 v_rsp.sense_len);
678 ret = copy_to_user(tv_cmd->tvc_resp, &v_rsp, sizeof(v_rsp)); 681 ret = copy_to_user(cmd->tvc_resp, &v_rsp, sizeof(v_rsp));
679 if (likely(ret == 0)) { 682 if (likely(ret == 0)) {
680 struct vhost_scsi_virtqueue *q; 683 struct vhost_scsi_virtqueue *q;
681 vhost_add_used(tv_cmd->tvc_vq, tv_cmd->tvc_vq_desc, 0); 684 vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
682 q = container_of(tv_cmd->tvc_vq, struct vhost_scsi_virtqueue, vq); 685 q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq);
683 vq = q - vs->vqs; 686 vq = q - vs->vqs;
684 __set_bit(vq, signal); 687 __set_bit(vq, signal);
685 } else 688 } else
686 pr_err("Faulted on virtio_scsi_cmd_resp\n"); 689 pr_err("Faulted on virtio_scsi_cmd_resp\n");
687 690
688 vhost_scsi_free_cmd(tv_cmd); 691 vhost_scsi_free_cmd(cmd);
689 } 692 }
690 693
691 vq = -1; 694 vq = -1;
@@ -694,35 +697,35 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
694 vhost_signal(&vs->dev, &vs->vqs[vq].vq); 697 vhost_signal(&vs->dev, &vs->vqs[vq].vq);
695} 698}
696 699
697static struct tcm_vhost_cmd *vhost_scsi_allocate_cmd( 700static struct tcm_vhost_cmd *
698 struct vhost_virtqueue *vq, 701vhost_scsi_allocate_cmd(struct vhost_virtqueue *vq,
699 struct tcm_vhost_tpg *tv_tpg, 702 struct tcm_vhost_tpg *tpg,
700 struct virtio_scsi_cmd_req *v_req, 703 struct virtio_scsi_cmd_req *v_req,
701 u32 exp_data_len, 704 u32 exp_data_len,
702 int data_direction) 705 int data_direction)
703{ 706{
704 struct tcm_vhost_cmd *tv_cmd; 707 struct tcm_vhost_cmd *cmd;
705 struct tcm_vhost_nexus *tv_nexus; 708 struct tcm_vhost_nexus *tv_nexus;
706 709
707 tv_nexus = tv_tpg->tpg_nexus; 710 tv_nexus = tpg->tpg_nexus;
708 if (!tv_nexus) { 711 if (!tv_nexus) {
709 pr_err("Unable to locate active struct tcm_vhost_nexus\n"); 712 pr_err("Unable to locate active struct tcm_vhost_nexus\n");
710 return ERR_PTR(-EIO); 713 return ERR_PTR(-EIO);
711 } 714 }
712 715
713 tv_cmd = kzalloc(sizeof(struct tcm_vhost_cmd), GFP_ATOMIC); 716 cmd = kzalloc(sizeof(struct tcm_vhost_cmd), GFP_ATOMIC);
714 if (!tv_cmd) { 717 if (!cmd) {
715 pr_err("Unable to allocate struct tcm_vhost_cmd\n"); 718 pr_err("Unable to allocate struct tcm_vhost_cmd\n");
716 return ERR_PTR(-ENOMEM); 719 return ERR_PTR(-ENOMEM);
717 } 720 }
718 tv_cmd->tvc_tag = v_req->tag; 721 cmd->tvc_tag = v_req->tag;
719 tv_cmd->tvc_task_attr = v_req->task_attr; 722 cmd->tvc_task_attr = v_req->task_attr;
720 tv_cmd->tvc_exp_data_len = exp_data_len; 723 cmd->tvc_exp_data_len = exp_data_len;
721 tv_cmd->tvc_data_direction = data_direction; 724 cmd->tvc_data_direction = data_direction;
722 tv_cmd->tvc_nexus = tv_nexus; 725 cmd->tvc_nexus = tv_nexus;
723 tv_cmd->inflight = tcm_vhost_get_inflight(vq); 726 cmd->inflight = tcm_vhost_get_inflight(vq);
724 727
725 return tv_cmd; 728 return cmd;
726} 729}
727 730
728/* 731/*
@@ -730,8 +733,11 @@ static struct tcm_vhost_cmd *vhost_scsi_allocate_cmd(
730 * 733 *
731 * Returns the number of scatterlist entries used or -errno on error. 734 * Returns the number of scatterlist entries used or -errno on error.
732 */ 735 */
733static int vhost_scsi_map_to_sgl(struct scatterlist *sgl, 736static int
734 unsigned int sgl_count, struct iovec *iov, int write) 737vhost_scsi_map_to_sgl(struct scatterlist *sgl,
738 unsigned int sgl_count,
739 struct iovec *iov,
740 int write)
735{ 741{
736 unsigned int npages = 0, pages_nr, offset, nbytes; 742 unsigned int npages = 0, pages_nr, offset, nbytes;
737 struct scatterlist *sg = sgl; 743 struct scatterlist *sg = sgl;
@@ -775,8 +781,11 @@ out:
775 return ret; 781 return ret;
776} 782}
777 783
778static int vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd *tv_cmd, 784static int
779 struct iovec *iov, unsigned int niov, int write) 785vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd *cmd,
786 struct iovec *iov,
787 unsigned int niov,
788 int write)
780{ 789{
781 int ret; 790 int ret;
782 unsigned int i; 791 unsigned int i;
@@ -792,25 +801,25 @@ static int vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd *tv_cmd,
792 801
793 /* TODO overflow checking */ 802 /* TODO overflow checking */
794 803
795 sg = kmalloc(sizeof(tv_cmd->tvc_sgl[0]) * sgl_count, GFP_ATOMIC); 804 sg = kmalloc(sizeof(cmd->tvc_sgl[0]) * sgl_count, GFP_ATOMIC);
796 if (!sg) 805 if (!sg)
797 return -ENOMEM; 806 return -ENOMEM;
798 pr_debug("%s sg %p sgl_count %u is_err %d\n", __func__, 807 pr_debug("%s sg %p sgl_count %u is_err %d\n", __func__,
799 sg, sgl_count, !sg); 808 sg, sgl_count, !sg);
800 sg_init_table(sg, sgl_count); 809 sg_init_table(sg, sgl_count);
801 810
802 tv_cmd->tvc_sgl = sg; 811 cmd->tvc_sgl = sg;
803 tv_cmd->tvc_sgl_count = sgl_count; 812 cmd->tvc_sgl_count = sgl_count;
804 813
805 pr_debug("Mapping %u iovecs for %u pages\n", niov, sgl_count); 814 pr_debug("Mapping %u iovecs for %u pages\n", niov, sgl_count);
806 for (i = 0; i < niov; i++) { 815 for (i = 0; i < niov; i++) {
807 ret = vhost_scsi_map_to_sgl(sg, sgl_count, &iov[i], write); 816 ret = vhost_scsi_map_to_sgl(sg, sgl_count, &iov[i], write);
808 if (ret < 0) { 817 if (ret < 0) {
809 for (i = 0; i < tv_cmd->tvc_sgl_count; i++) 818 for (i = 0; i < cmd->tvc_sgl_count; i++)
810 put_page(sg_page(&tv_cmd->tvc_sgl[i])); 819 put_page(sg_page(&cmd->tvc_sgl[i]));
811 kfree(tv_cmd->tvc_sgl); 820 kfree(cmd->tvc_sgl);
812 tv_cmd->tvc_sgl = NULL; 821 cmd->tvc_sgl = NULL;
813 tv_cmd->tvc_sgl_count = 0; 822 cmd->tvc_sgl_count = 0;
814 return ret; 823 return ret;
815 } 824 }
816 825
@@ -822,15 +831,15 @@ static int vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd *tv_cmd,
822 831
823static void tcm_vhost_submission_work(struct work_struct *work) 832static void tcm_vhost_submission_work(struct work_struct *work)
824{ 833{
825 struct tcm_vhost_cmd *tv_cmd = 834 struct tcm_vhost_cmd *cmd =
826 container_of(work, struct tcm_vhost_cmd, work); 835 container_of(work, struct tcm_vhost_cmd, work);
827 struct tcm_vhost_nexus *tv_nexus; 836 struct tcm_vhost_nexus *tv_nexus;
828 struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd; 837 struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
829 struct scatterlist *sg_ptr, *sg_bidi_ptr = NULL; 838 struct scatterlist *sg_ptr, *sg_bidi_ptr = NULL;
830 int rc, sg_no_bidi = 0; 839 int rc, sg_no_bidi = 0;
831 840
832 if (tv_cmd->tvc_sgl_count) { 841 if (cmd->tvc_sgl_count) {
833 sg_ptr = tv_cmd->tvc_sgl; 842 sg_ptr = cmd->tvc_sgl;
834/* FIXME: Fix BIDI operation in tcm_vhost_submission_work() */ 843/* FIXME: Fix BIDI operation in tcm_vhost_submission_work() */
835#if 0 844#if 0
836 if (se_cmd->se_cmd_flags & SCF_BIDI) { 845 if (se_cmd->se_cmd_flags & SCF_BIDI) {
@@ -841,13 +850,13 @@ static void tcm_vhost_submission_work(struct work_struct *work)
841 } else { 850 } else {
842 sg_ptr = NULL; 851 sg_ptr = NULL;
843 } 852 }
844 tv_nexus = tv_cmd->tvc_nexus; 853 tv_nexus = cmd->tvc_nexus;
845 854
846 rc = target_submit_cmd_map_sgls(se_cmd, tv_nexus->tvn_se_sess, 855 rc = target_submit_cmd_map_sgls(se_cmd, tv_nexus->tvn_se_sess,
847 tv_cmd->tvc_cdb, &tv_cmd->tvc_sense_buf[0], 856 cmd->tvc_cdb, &cmd->tvc_sense_buf[0],
848 tv_cmd->tvc_lun, tv_cmd->tvc_exp_data_len, 857 cmd->tvc_lun, cmd->tvc_exp_data_len,
849 tv_cmd->tvc_task_attr, tv_cmd->tvc_data_direction, 858 cmd->tvc_task_attr, cmd->tvc_data_direction,
850 0, sg_ptr, tv_cmd->tvc_sgl_count, 859 0, sg_ptr, cmd->tvc_sgl_count,
851 sg_bidi_ptr, sg_no_bidi); 860 sg_bidi_ptr, sg_no_bidi);
852 if (rc < 0) { 861 if (rc < 0) {
853 transport_send_check_condition_and_sense(se_cmd, 862 transport_send_check_condition_and_sense(se_cmd,
@@ -856,8 +865,10 @@ static void tcm_vhost_submission_work(struct work_struct *work)
856 } 865 }
857} 866}
858 867
859static void vhost_scsi_send_bad_target(struct vhost_scsi *vs, 868static void
860 struct vhost_virtqueue *vq, int head, unsigned out) 869vhost_scsi_send_bad_target(struct vhost_scsi *vs,
870 struct vhost_virtqueue *vq,
871 int head, unsigned out)
861{ 872{
862 struct virtio_scsi_cmd_resp __user *resp; 873 struct virtio_scsi_cmd_resp __user *resp;
863 struct virtio_scsi_cmd_resp rsp; 874 struct virtio_scsi_cmd_resp rsp;
@@ -873,13 +884,13 @@ static void vhost_scsi_send_bad_target(struct vhost_scsi *vs,
873 pr_err("Faulted on virtio_scsi_cmd_resp\n"); 884 pr_err("Faulted on virtio_scsi_cmd_resp\n");
874} 885}
875 886
876static void vhost_scsi_handle_vq(struct vhost_scsi *vs, 887static void
877 struct vhost_virtqueue *vq) 888vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
878{ 889{
879 struct tcm_vhost_tpg **vs_tpg; 890 struct tcm_vhost_tpg **vs_tpg;
880 struct virtio_scsi_cmd_req v_req; 891 struct virtio_scsi_cmd_req v_req;
881 struct tcm_vhost_tpg *tv_tpg; 892 struct tcm_vhost_tpg *tpg;
882 struct tcm_vhost_cmd *tv_cmd; 893 struct tcm_vhost_cmd *cmd;
883 u32 exp_data_len, data_first, data_num, data_direction; 894 u32 exp_data_len, data_first, data_num, data_direction;
884 unsigned out, in, i; 895 unsigned out, in, i;
885 int head, ret; 896 int head, ret;
@@ -964,10 +975,10 @@ static void vhost_scsi_handle_vq(struct vhost_scsi *vs,
964 975
965 /* Extract the tpgt */ 976 /* Extract the tpgt */
966 target = v_req.lun[1]; 977 target = v_req.lun[1];
967 tv_tpg = ACCESS_ONCE(vs_tpg[target]); 978 tpg = ACCESS_ONCE(vs_tpg[target]);
968 979
969 /* Target does not exist, fail the request */ 980 /* Target does not exist, fail the request */
970 if (unlikely(!tv_tpg)) { 981 if (unlikely(!tpg)) {
971 vhost_scsi_send_bad_target(vs, vq, head, out); 982 vhost_scsi_send_bad_target(vs, vq, head, out);
972 continue; 983 continue;
973 } 984 }
@@ -976,46 +987,46 @@ static void vhost_scsi_handle_vq(struct vhost_scsi *vs,
976 for (i = 0; i < data_num; i++) 987 for (i = 0; i < data_num; i++)
977 exp_data_len += vq->iov[data_first + i].iov_len; 988 exp_data_len += vq->iov[data_first + i].iov_len;
978 989
979 tv_cmd = vhost_scsi_allocate_cmd(vq, tv_tpg, &v_req, 990 cmd = vhost_scsi_allocate_cmd(vq, tpg, &v_req,
980 exp_data_len, data_direction); 991 exp_data_len, data_direction);
981 if (IS_ERR(tv_cmd)) { 992 if (IS_ERR(cmd)) {
982 vq_err(vq, "vhost_scsi_allocate_cmd failed %ld\n", 993 vq_err(vq, "vhost_scsi_allocate_cmd failed %ld\n",
983 PTR_ERR(tv_cmd)); 994 PTR_ERR(cmd));
984 goto err_cmd; 995 goto err_cmd;
985 } 996 }
986 pr_debug("Allocated tv_cmd: %p exp_data_len: %d, data_direction" 997 pr_debug("Allocated tv_cmd: %p exp_data_len: %d, data_direction"
987 ": %d\n", tv_cmd, exp_data_len, data_direction); 998 ": %d\n", cmd, exp_data_len, data_direction);
988 999
989 tv_cmd->tvc_vhost = vs; 1000 cmd->tvc_vhost = vs;
990 tv_cmd->tvc_vq = vq; 1001 cmd->tvc_vq = vq;
991 tv_cmd->tvc_resp = vq->iov[out].iov_base; 1002 cmd->tvc_resp = vq->iov[out].iov_base;
992 1003
993 /* 1004 /*
994 * Copy in the recieved CDB descriptor into tv_cmd->tvc_cdb 1005 * Copy in the recieved CDB descriptor into cmd->tvc_cdb
995 * that will be used by tcm_vhost_new_cmd_map() and down into 1006 * that will be used by tcm_vhost_new_cmd_map() and down into
996 * target_setup_cmd_from_cdb() 1007 * target_setup_cmd_from_cdb()
997 */ 1008 */
998 memcpy(tv_cmd->tvc_cdb, v_req.cdb, TCM_VHOST_MAX_CDB_SIZE); 1009 memcpy(cmd->tvc_cdb, v_req.cdb, TCM_VHOST_MAX_CDB_SIZE);
999 /* 1010 /*
1000 * Check that the recieved CDB size does not exceeded our 1011 * Check that the recieved CDB size does not exceeded our
1001 * hardcoded max for tcm_vhost 1012 * hardcoded max for tcm_vhost
1002 */ 1013 */
1003 /* TODO what if cdb was too small for varlen cdb header? */ 1014 /* TODO what if cdb was too small for varlen cdb header? */
1004 if (unlikely(scsi_command_size(tv_cmd->tvc_cdb) > 1015 if (unlikely(scsi_command_size(cmd->tvc_cdb) >
1005 TCM_VHOST_MAX_CDB_SIZE)) { 1016 TCM_VHOST_MAX_CDB_SIZE)) {
1006 vq_err(vq, "Received SCSI CDB with command_size: %d that" 1017 vq_err(vq, "Received SCSI CDB with command_size: %d that"
1007 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", 1018 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1008 scsi_command_size(tv_cmd->tvc_cdb), 1019 scsi_command_size(cmd->tvc_cdb),
1009 TCM_VHOST_MAX_CDB_SIZE); 1020 TCM_VHOST_MAX_CDB_SIZE);
1010 goto err_free; 1021 goto err_free;
1011 } 1022 }
1012 tv_cmd->tvc_lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF; 1023 cmd->tvc_lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF;
1013 1024
1014 pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n", 1025 pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
1015 tv_cmd->tvc_cdb[0], tv_cmd->tvc_lun); 1026 cmd->tvc_cdb[0], cmd->tvc_lun);
1016 1027
1017 if (data_direction != DMA_NONE) { 1028 if (data_direction != DMA_NONE) {
1018 ret = vhost_scsi_map_iov_to_sgl(tv_cmd, 1029 ret = vhost_scsi_map_iov_to_sgl(cmd,
1019 &vq->iov[data_first], data_num, 1030 &vq->iov[data_first], data_num,
1020 data_direction == DMA_TO_DEVICE); 1031 data_direction == DMA_TO_DEVICE);
1021 if (unlikely(ret)) { 1032 if (unlikely(ret)) {
@@ -1029,22 +1040,22 @@ static void vhost_scsi_handle_vq(struct vhost_scsi *vs,
1029 * complete the virtio-scsi request in TCM callback context via 1040 * complete the virtio-scsi request in TCM callback context via
1030 * tcm_vhost_queue_data_in() and tcm_vhost_queue_status() 1041 * tcm_vhost_queue_data_in() and tcm_vhost_queue_status()
1031 */ 1042 */
1032 tv_cmd->tvc_vq_desc = head; 1043 cmd->tvc_vq_desc = head;
1033 /* 1044 /*
1034 * Dispatch tv_cmd descriptor for cmwq execution in process 1045 * Dispatch tv_cmd descriptor for cmwq execution in process
1035 * context provided by tcm_vhost_workqueue. This also ensures 1046 * context provided by tcm_vhost_workqueue. This also ensures
1036 * tv_cmd is executed on the same kworker CPU as this vhost 1047 * tv_cmd is executed on the same kworker CPU as this vhost
1037 * thread to gain positive L2 cache locality effects.. 1048 * thread to gain positive L2 cache locality effects..
1038 */ 1049 */
1039 INIT_WORK(&tv_cmd->work, tcm_vhost_submission_work); 1050 INIT_WORK(&cmd->work, tcm_vhost_submission_work);
1040 queue_work(tcm_vhost_workqueue, &tv_cmd->work); 1051 queue_work(tcm_vhost_workqueue, &cmd->work);
1041 } 1052 }
1042 1053
1043 mutex_unlock(&vq->mutex); 1054 mutex_unlock(&vq->mutex);
1044 return; 1055 return;
1045 1056
1046err_free: 1057err_free:
1047 vhost_scsi_free_cmd(tv_cmd); 1058 vhost_scsi_free_cmd(cmd);
1048err_cmd: 1059err_cmd:
1049 vhost_scsi_send_bad_target(vs, vq, head, out); 1060 vhost_scsi_send_bad_target(vs, vq, head, out);
1050 mutex_unlock(&vq->mutex); 1061 mutex_unlock(&vq->mutex);
@@ -1055,8 +1066,12 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)
1055 pr_debug("%s: The handling func for control queue.\n", __func__); 1066 pr_debug("%s: The handling func for control queue.\n", __func__);
1056} 1067}
1057 1068
1058static void tcm_vhost_send_evt(struct vhost_scsi *vs, struct tcm_vhost_tpg *tpg, 1069static void
1059 struct se_lun *lun, u32 event, u32 reason) 1070tcm_vhost_send_evt(struct vhost_scsi *vs,
1071 struct tcm_vhost_tpg *tpg,
1072 struct se_lun *lun,
1073 u32 event,
1074 u32 reason)
1060{ 1075{
1061 struct tcm_vhost_evt *evt; 1076 struct tcm_vhost_evt *evt;
1062 1077
@@ -1146,12 +1161,12 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)
1146 * The lock nesting rule is: 1161 * The lock nesting rule is:
1147 * tcm_vhost_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex 1162 * tcm_vhost_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex
1148 */ 1163 */
1149static int vhost_scsi_set_endpoint( 1164static int
1150 struct vhost_scsi *vs, 1165vhost_scsi_set_endpoint(struct vhost_scsi *vs,
1151 struct vhost_scsi_target *t) 1166 struct vhost_scsi_target *t)
1152{ 1167{
1153 struct tcm_vhost_tport *tv_tport; 1168 struct tcm_vhost_tport *tv_tport;
1154 struct tcm_vhost_tpg *tv_tpg; 1169 struct tcm_vhost_tpg *tpg;
1155 struct tcm_vhost_tpg **vs_tpg; 1170 struct tcm_vhost_tpg **vs_tpg;
1156 struct vhost_virtqueue *vq; 1171 struct vhost_virtqueue *vq;
1157 int index, ret, i, len; 1172 int index, ret, i, len;
@@ -1178,32 +1193,32 @@ static int vhost_scsi_set_endpoint(
1178 if (vs->vs_tpg) 1193 if (vs->vs_tpg)
1179 memcpy(vs_tpg, vs->vs_tpg, len); 1194 memcpy(vs_tpg, vs->vs_tpg, len);
1180 1195
1181 list_for_each_entry(tv_tpg, &tcm_vhost_list, tv_tpg_list) { 1196 list_for_each_entry(tpg, &tcm_vhost_list, tv_tpg_list) {
1182 mutex_lock(&tv_tpg->tv_tpg_mutex); 1197 mutex_lock(&tpg->tv_tpg_mutex);
1183 if (!tv_tpg->tpg_nexus) { 1198 if (!tpg->tpg_nexus) {
1184 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1199 mutex_unlock(&tpg->tv_tpg_mutex);
1185 continue; 1200 continue;
1186 } 1201 }
1187 if (tv_tpg->tv_tpg_vhost_count != 0) { 1202 if (tpg->tv_tpg_vhost_count != 0) {
1188 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1203 mutex_unlock(&tpg->tv_tpg_mutex);
1189 continue; 1204 continue;
1190 } 1205 }
1191 tv_tport = tv_tpg->tport; 1206 tv_tport = tpg->tport;
1192 1207
1193 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) { 1208 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1194 if (vs->vs_tpg && vs->vs_tpg[tv_tpg->tport_tpgt]) { 1209 if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {
1195 kfree(vs_tpg); 1210 kfree(vs_tpg);
1196 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1211 mutex_unlock(&tpg->tv_tpg_mutex);
1197 ret = -EEXIST; 1212 ret = -EEXIST;
1198 goto out; 1213 goto out;
1199 } 1214 }
1200 tv_tpg->tv_tpg_vhost_count++; 1215 tpg->tv_tpg_vhost_count++;
1201 tv_tpg->vhost_scsi = vs; 1216 tpg->vhost_scsi = vs;
1202 vs_tpg[tv_tpg->tport_tpgt] = tv_tpg; 1217 vs_tpg[tpg->tport_tpgt] = tpg;
1203 smp_mb__after_atomic_inc(); 1218 smp_mb__after_atomic_inc();
1204 match = true; 1219 match = true;
1205 } 1220 }
1206 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1221 mutex_unlock(&tpg->tv_tpg_mutex);
1207 } 1222 }
1208 1223
1209 if (match) { 1224 if (match) {
@@ -1236,12 +1251,12 @@ out:
1236 return ret; 1251 return ret;
1237} 1252}
1238 1253
1239static int vhost_scsi_clear_endpoint( 1254static int
1240 struct vhost_scsi *vs, 1255vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
1241 struct vhost_scsi_target *t) 1256 struct vhost_scsi_target *t)
1242{ 1257{
1243 struct tcm_vhost_tport *tv_tport; 1258 struct tcm_vhost_tport *tv_tport;
1244 struct tcm_vhost_tpg *tv_tpg; 1259 struct tcm_vhost_tpg *tpg;
1245 struct vhost_virtqueue *vq; 1260 struct vhost_virtqueue *vq;
1246 bool match = false; 1261 bool match = false;
1247 int index, ret, i; 1262 int index, ret, i;
@@ -1264,30 +1279,30 @@ static int vhost_scsi_clear_endpoint(
1264 1279
1265 for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) { 1280 for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1266 target = i; 1281 target = i;
1267 tv_tpg = vs->vs_tpg[target]; 1282 tpg = vs->vs_tpg[target];
1268 if (!tv_tpg) 1283 if (!tpg)
1269 continue; 1284 continue;
1270 1285
1271 mutex_lock(&tv_tpg->tv_tpg_mutex); 1286 mutex_lock(&tpg->tv_tpg_mutex);
1272 tv_tport = tv_tpg->tport; 1287 tv_tport = tpg->tport;
1273 if (!tv_tport) { 1288 if (!tv_tport) {
1274 ret = -ENODEV; 1289 ret = -ENODEV;
1275 goto err_tpg; 1290 goto err_tpg;
1276 } 1291 }
1277 1292
1278 if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) { 1293 if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1279 pr_warn("tv_tport->tport_name: %s, tv_tpg->tport_tpgt: %hu" 1294 pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu"
1280 " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n", 1295 " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n",
1281 tv_tport->tport_name, tv_tpg->tport_tpgt, 1296 tv_tport->tport_name, tpg->tport_tpgt,
1282 t->vhost_wwpn, t->vhost_tpgt); 1297 t->vhost_wwpn, t->vhost_tpgt);
1283 ret = -EINVAL; 1298 ret = -EINVAL;
1284 goto err_tpg; 1299 goto err_tpg;
1285 } 1300 }
1286 tv_tpg->tv_tpg_vhost_count--; 1301 tpg->tv_tpg_vhost_count--;
1287 tv_tpg->vhost_scsi = NULL; 1302 tpg->vhost_scsi = NULL;
1288 vs->vs_tpg[target] = NULL; 1303 vs->vs_tpg[target] = NULL;
1289 match = true; 1304 match = true;
1290 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1305 mutex_unlock(&tpg->tv_tpg_mutex);
1291 } 1306 }
1292 if (match) { 1307 if (match) {
1293 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { 1308 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
@@ -1311,7 +1326,7 @@ static int vhost_scsi_clear_endpoint(
1311 return 0; 1326 return 0;
1312 1327
1313err_tpg: 1328err_tpg:
1314 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1329 mutex_unlock(&tpg->tv_tpg_mutex);
1315err_dev: 1330err_dev:
1316 mutex_unlock(&vs->dev.mutex); 1331 mutex_unlock(&vs->dev.mutex);
1317 mutex_unlock(&tcm_vhost_mutex); 1332 mutex_unlock(&tcm_vhost_mutex);
@@ -1338,68 +1353,70 @@ static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
1338 1353
1339static int vhost_scsi_open(struct inode *inode, struct file *f) 1354static int vhost_scsi_open(struct inode *inode, struct file *f)
1340{ 1355{
1341 struct vhost_scsi *s; 1356 struct vhost_scsi *vs;
1342 struct vhost_virtqueue **vqs; 1357 struct vhost_virtqueue **vqs;
1343 int r, i; 1358 int r, i;
1344 1359
1345 s = kzalloc(sizeof(*s), GFP_KERNEL); 1360 vs = kzalloc(sizeof(*vs), GFP_KERNEL);
1346 if (!s) 1361 if (!vs)
1347 return -ENOMEM; 1362 return -ENOMEM;
1348 1363
1349 vqs = kmalloc(VHOST_SCSI_MAX_VQ * sizeof(*vqs), GFP_KERNEL); 1364 vqs = kmalloc(VHOST_SCSI_MAX_VQ * sizeof(*vqs), GFP_KERNEL);
1350 if (!vqs) { 1365 if (!vqs) {
1351 kfree(s); 1366 kfree(vs);
1352 return -ENOMEM; 1367 return -ENOMEM;
1353 } 1368 }
1354 1369
1355 vhost_work_init(&s->vs_completion_work, vhost_scsi_complete_cmd_work); 1370 vhost_work_init(&vs->vs_completion_work, vhost_scsi_complete_cmd_work);
1356 vhost_work_init(&s->vs_event_work, tcm_vhost_evt_work); 1371 vhost_work_init(&vs->vs_event_work, tcm_vhost_evt_work);
1357 1372
1358 s->vs_events_nr = 0; 1373 vs->vs_events_nr = 0;
1359 s->vs_events_missed = false; 1374 vs->vs_events_missed = false;
1360 1375
1361 vqs[VHOST_SCSI_VQ_CTL] = &s->vqs[VHOST_SCSI_VQ_CTL].vq; 1376 vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;
1362 vqs[VHOST_SCSI_VQ_EVT] = &s->vqs[VHOST_SCSI_VQ_EVT].vq; 1377 vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1363 s->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick; 1378 vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
1364 s->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick; 1379 vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
1365 for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) { 1380 for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) {
1366 vqs[i] = &s->vqs[i].vq; 1381 vqs[i] = &vs->vqs[i].vq;
1367 s->vqs[i].vq.handle_kick = vhost_scsi_handle_kick; 1382 vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;
1368 } 1383 }
1369 r = vhost_dev_init(&s->dev, vqs, VHOST_SCSI_MAX_VQ); 1384 r = vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ);
1370 1385
1371 tcm_vhost_init_inflight(s, NULL); 1386 tcm_vhost_init_inflight(vs, NULL);
1372 1387
1373 if (r < 0) { 1388 if (r < 0) {
1374 kfree(vqs); 1389 kfree(vqs);
1375 kfree(s); 1390 kfree(vs);
1376 return r; 1391 return r;
1377 } 1392 }
1378 1393
1379 f->private_data = s; 1394 f->private_data = vs;
1380 return 0; 1395 return 0;
1381} 1396}
1382 1397
1383static int vhost_scsi_release(struct inode *inode, struct file *f) 1398static int vhost_scsi_release(struct inode *inode, struct file *f)
1384{ 1399{
1385 struct vhost_scsi *s = f->private_data; 1400 struct vhost_scsi *vs = f->private_data;
1386 struct vhost_scsi_target t; 1401 struct vhost_scsi_target t;
1387 1402
1388 mutex_lock(&s->dev.mutex); 1403 mutex_lock(&vs->dev.mutex);
1389 memcpy(t.vhost_wwpn, s->vs_vhost_wwpn, sizeof(t.vhost_wwpn)); 1404 memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn));
1390 mutex_unlock(&s->dev.mutex); 1405 mutex_unlock(&vs->dev.mutex);
1391 vhost_scsi_clear_endpoint(s, &t); 1406 vhost_scsi_clear_endpoint(vs, &t);
1392 vhost_dev_stop(&s->dev); 1407 vhost_dev_stop(&vs->dev);
1393 vhost_dev_cleanup(&s->dev, false); 1408 vhost_dev_cleanup(&vs->dev, false);
1394 /* Jobs can re-queue themselves in evt kick handler. Do extra flush. */ 1409 /* Jobs can re-queue themselves in evt kick handler. Do extra flush. */
1395 vhost_scsi_flush(s); 1410 vhost_scsi_flush(vs);
1396 kfree(s->dev.vqs); 1411 kfree(vs->dev.vqs);
1397 kfree(s); 1412 kfree(vs);
1398 return 0; 1413 return 0;
1399} 1414}
1400 1415
1401static long vhost_scsi_ioctl(struct file *f, unsigned int ioctl, 1416static long
1402 unsigned long arg) 1417vhost_scsi_ioctl(struct file *f,
1418 unsigned int ioctl,
1419 unsigned long arg)
1403{ 1420{
1404 struct vhost_scsi *vs = f->private_data; 1421 struct vhost_scsi *vs = f->private_data;
1405 struct vhost_scsi_target backend; 1422 struct vhost_scsi_target backend;
@@ -1515,8 +1532,9 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)
1515 return "Unknown"; 1532 return "Unknown";
1516} 1533}
1517 1534
1518static void tcm_vhost_do_plug(struct tcm_vhost_tpg *tpg, 1535static void
1519 struct se_lun *lun, bool plug) 1536tcm_vhost_do_plug(struct tcm_vhost_tpg *tpg,
1537 struct se_lun *lun, bool plug)
1520{ 1538{
1521 1539
1522 struct vhost_scsi *vs = tpg->vhost_scsi; 1540 struct vhost_scsi *vs = tpg->vhost_scsi;
@@ -1556,18 +1574,18 @@ static void tcm_vhost_hotunplug(struct tcm_vhost_tpg *tpg, struct se_lun *lun)
1556} 1574}
1557 1575
1558static int tcm_vhost_port_link(struct se_portal_group *se_tpg, 1576static int tcm_vhost_port_link(struct se_portal_group *se_tpg,
1559 struct se_lun *lun) 1577 struct se_lun *lun)
1560{ 1578{
1561 struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg, 1579 struct tcm_vhost_tpg *tpg = container_of(se_tpg,
1562 struct tcm_vhost_tpg, se_tpg); 1580 struct tcm_vhost_tpg, se_tpg);
1563 1581
1564 mutex_lock(&tcm_vhost_mutex); 1582 mutex_lock(&tcm_vhost_mutex);
1565 1583
1566 mutex_lock(&tv_tpg->tv_tpg_mutex); 1584 mutex_lock(&tpg->tv_tpg_mutex);
1567 tv_tpg->tv_tpg_port_count++; 1585 tpg->tv_tpg_port_count++;
1568 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1586 mutex_unlock(&tpg->tv_tpg_mutex);
1569 1587
1570 tcm_vhost_hotplug(tv_tpg, lun); 1588 tcm_vhost_hotplug(tpg, lun);
1571 1589
1572 mutex_unlock(&tcm_vhost_mutex); 1590 mutex_unlock(&tcm_vhost_mutex);
1573 1591
@@ -1575,26 +1593,26 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg,
1575} 1593}
1576 1594
1577static void tcm_vhost_port_unlink(struct se_portal_group *se_tpg, 1595static void tcm_vhost_port_unlink(struct se_portal_group *se_tpg,
1578 struct se_lun *lun) 1596 struct se_lun *lun)
1579{ 1597{
1580 struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg, 1598 struct tcm_vhost_tpg *tpg = container_of(se_tpg,
1581 struct tcm_vhost_tpg, se_tpg); 1599 struct tcm_vhost_tpg, se_tpg);
1582 1600
1583 mutex_lock(&tcm_vhost_mutex); 1601 mutex_lock(&tcm_vhost_mutex);
1584 1602
1585 mutex_lock(&tv_tpg->tv_tpg_mutex); 1603 mutex_lock(&tpg->tv_tpg_mutex);
1586 tv_tpg->tv_tpg_port_count--; 1604 tpg->tv_tpg_port_count--;
1587 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1605 mutex_unlock(&tpg->tv_tpg_mutex);
1588 1606
1589 tcm_vhost_hotunplug(tv_tpg, lun); 1607 tcm_vhost_hotunplug(tpg, lun);
1590 1608
1591 mutex_unlock(&tcm_vhost_mutex); 1609 mutex_unlock(&tcm_vhost_mutex);
1592} 1610}
1593 1611
1594static struct se_node_acl *tcm_vhost_make_nodeacl( 1612static struct se_node_acl *
1595 struct se_portal_group *se_tpg, 1613tcm_vhost_make_nodeacl(struct se_portal_group *se_tpg,
1596 struct config_group *group, 1614 struct config_group *group,
1597 const char *name) 1615 const char *name)
1598{ 1616{
1599 struct se_node_acl *se_nacl, *se_nacl_new; 1617 struct se_node_acl *se_nacl, *se_nacl_new;
1600 struct tcm_vhost_nacl *nacl; 1618 struct tcm_vhost_nacl *nacl;
@@ -1635,23 +1653,23 @@ static void tcm_vhost_drop_nodeacl(struct se_node_acl *se_acl)
1635 kfree(nacl); 1653 kfree(nacl);
1636} 1654}
1637 1655
1638static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tv_tpg, 1656static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tpg,
1639 const char *name) 1657 const char *name)
1640{ 1658{
1641 struct se_portal_group *se_tpg; 1659 struct se_portal_group *se_tpg;
1642 struct tcm_vhost_nexus *tv_nexus; 1660 struct tcm_vhost_nexus *tv_nexus;
1643 1661
1644 mutex_lock(&tv_tpg->tv_tpg_mutex); 1662 mutex_lock(&tpg->tv_tpg_mutex);
1645 if (tv_tpg->tpg_nexus) { 1663 if (tpg->tpg_nexus) {
1646 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1664 mutex_unlock(&tpg->tv_tpg_mutex);
1647 pr_debug("tv_tpg->tpg_nexus already exists\n"); 1665 pr_debug("tpg->tpg_nexus already exists\n");
1648 return -EEXIST; 1666 return -EEXIST;
1649 } 1667 }
1650 se_tpg = &tv_tpg->se_tpg; 1668 se_tpg = &tpg->se_tpg;
1651 1669
1652 tv_nexus = kzalloc(sizeof(struct tcm_vhost_nexus), GFP_KERNEL); 1670 tv_nexus = kzalloc(sizeof(struct tcm_vhost_nexus), GFP_KERNEL);
1653 if (!tv_nexus) { 1671 if (!tv_nexus) {
1654 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1672 mutex_unlock(&tpg->tv_tpg_mutex);
1655 pr_err("Unable to allocate struct tcm_vhost_nexus\n"); 1673 pr_err("Unable to allocate struct tcm_vhost_nexus\n");
1656 return -ENOMEM; 1674 return -ENOMEM;
1657 } 1675 }
@@ -1660,7 +1678,7 @@ static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tv_tpg,
1660 */ 1678 */
1661 tv_nexus->tvn_se_sess = transport_init_session(); 1679 tv_nexus->tvn_se_sess = transport_init_session();
1662 if (IS_ERR(tv_nexus->tvn_se_sess)) { 1680 if (IS_ERR(tv_nexus->tvn_se_sess)) {
1663 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1681 mutex_unlock(&tpg->tv_tpg_mutex);
1664 kfree(tv_nexus); 1682 kfree(tv_nexus);
1665 return -ENOMEM; 1683 return -ENOMEM;
1666 } 1684 }
@@ -1672,7 +1690,7 @@ static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tv_tpg,
1672 tv_nexus->tvn_se_sess->se_node_acl = core_tpg_check_initiator_node_acl( 1690 tv_nexus->tvn_se_sess->se_node_acl = core_tpg_check_initiator_node_acl(
1673 se_tpg, (unsigned char *)name); 1691 se_tpg, (unsigned char *)name);
1674 if (!tv_nexus->tvn_se_sess->se_node_acl) { 1692 if (!tv_nexus->tvn_se_sess->se_node_acl) {
1675 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1693 mutex_unlock(&tpg->tv_tpg_mutex);
1676 pr_debug("core_tpg_check_initiator_node_acl() failed" 1694 pr_debug("core_tpg_check_initiator_node_acl() failed"
1677 " for %s\n", name); 1695 " for %s\n", name);
1678 transport_free_session(tv_nexus->tvn_se_sess); 1696 transport_free_session(tv_nexus->tvn_se_sess);
@@ -1685,9 +1703,9 @@ static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tv_tpg,
1685 */ 1703 */
1686 __transport_register_session(se_tpg, tv_nexus->tvn_se_sess->se_node_acl, 1704 __transport_register_session(se_tpg, tv_nexus->tvn_se_sess->se_node_acl,
1687 tv_nexus->tvn_se_sess, tv_nexus); 1705 tv_nexus->tvn_se_sess, tv_nexus);
1688 tv_tpg->tpg_nexus = tv_nexus; 1706 tpg->tpg_nexus = tv_nexus;
1689 1707
1690 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1708 mutex_unlock(&tpg->tv_tpg_mutex);
1691 return 0; 1709 return 0;
1692} 1710}
1693 1711
@@ -1740,40 +1758,40 @@ static int tcm_vhost_drop_nexus(struct tcm_vhost_tpg *tpg)
1740} 1758}
1741 1759
1742static ssize_t tcm_vhost_tpg_show_nexus(struct se_portal_group *se_tpg, 1760static ssize_t tcm_vhost_tpg_show_nexus(struct se_portal_group *se_tpg,
1743 char *page) 1761 char *page)
1744{ 1762{
1745 struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg, 1763 struct tcm_vhost_tpg *tpg = container_of(se_tpg,
1746 struct tcm_vhost_tpg, se_tpg); 1764 struct tcm_vhost_tpg, se_tpg);
1747 struct tcm_vhost_nexus *tv_nexus; 1765 struct tcm_vhost_nexus *tv_nexus;
1748 ssize_t ret; 1766 ssize_t ret;
1749 1767
1750 mutex_lock(&tv_tpg->tv_tpg_mutex); 1768 mutex_lock(&tpg->tv_tpg_mutex);
1751 tv_nexus = tv_tpg->tpg_nexus; 1769 tv_nexus = tpg->tpg_nexus;
1752 if (!tv_nexus) { 1770 if (!tv_nexus) {
1753 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1771 mutex_unlock(&tpg->tv_tpg_mutex);
1754 return -ENODEV; 1772 return -ENODEV;
1755 } 1773 }
1756 ret = snprintf(page, PAGE_SIZE, "%s\n", 1774 ret = snprintf(page, PAGE_SIZE, "%s\n",
1757 tv_nexus->tvn_se_sess->se_node_acl->initiatorname); 1775 tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
1758 mutex_unlock(&tv_tpg->tv_tpg_mutex); 1776 mutex_unlock(&tpg->tv_tpg_mutex);
1759 1777
1760 return ret; 1778 return ret;
1761} 1779}
1762 1780
1763static ssize_t tcm_vhost_tpg_store_nexus(struct se_portal_group *se_tpg, 1781static ssize_t tcm_vhost_tpg_store_nexus(struct se_portal_group *se_tpg,
1764 const char *page, 1782 const char *page,
1765 size_t count) 1783 size_t count)
1766{ 1784{
1767 struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg, 1785 struct tcm_vhost_tpg *tpg = container_of(se_tpg,
1768 struct tcm_vhost_tpg, se_tpg); 1786 struct tcm_vhost_tpg, se_tpg);
1769 struct tcm_vhost_tport *tport_wwn = tv_tpg->tport; 1787 struct tcm_vhost_tport *tport_wwn = tpg->tport;
1770 unsigned char i_port[TCM_VHOST_NAMELEN], *ptr, *port_ptr; 1788 unsigned char i_port[TCM_VHOST_NAMELEN], *ptr, *port_ptr;
1771 int ret; 1789 int ret;
1772 /* 1790 /*
1773 * Shutdown the active I_T nexus if 'NULL' is passed.. 1791 * Shutdown the active I_T nexus if 'NULL' is passed..
1774 */ 1792 */
1775 if (!strncmp(page, "NULL", 4)) { 1793 if (!strncmp(page, "NULL", 4)) {
1776 ret = tcm_vhost_drop_nexus(tv_tpg); 1794 ret = tcm_vhost_drop_nexus(tpg);
1777 return (!ret) ? count : ret; 1795 return (!ret) ? count : ret;
1778 } 1796 }
1779 /* 1797 /*
@@ -1831,7 +1849,7 @@ check_newline:
1831 if (i_port[strlen(i_port)-1] == '\n') 1849 if (i_port[strlen(i_port)-1] == '\n')
1832 i_port[strlen(i_port)-1] = '\0'; 1850 i_port[strlen(i_port)-1] = '\0';
1833 1851
1834 ret = tcm_vhost_make_nexus(tv_tpg, port_ptr); 1852 ret = tcm_vhost_make_nexus(tpg, port_ptr);
1835 if (ret < 0) 1853 if (ret < 0)
1836 return ret; 1854 return ret;
1837 1855
@@ -1845,9 +1863,10 @@ static struct configfs_attribute *tcm_vhost_tpg_attrs[] = {
1845 NULL, 1863 NULL,
1846}; 1864};
1847 1865
1848static struct se_portal_group *tcm_vhost_make_tpg(struct se_wwn *wwn, 1866static struct se_portal_group *
1849 struct config_group *group, 1867tcm_vhost_make_tpg(struct se_wwn *wwn,
1850 const char *name) 1868 struct config_group *group,
1869 const char *name)
1851{ 1870{
1852 struct tcm_vhost_tport *tport = container_of(wwn, 1871 struct tcm_vhost_tport *tport = container_of(wwn,
1853 struct tcm_vhost_tport, tport_wwn); 1872 struct tcm_vhost_tport, tport_wwn);
@@ -1903,9 +1922,10 @@ static void tcm_vhost_drop_tpg(struct se_portal_group *se_tpg)
1903 kfree(tpg); 1922 kfree(tpg);
1904} 1923}
1905 1924
1906static struct se_wwn *tcm_vhost_make_tport(struct target_fabric_configfs *tf, 1925static struct se_wwn *
1907 struct config_group *group, 1926tcm_vhost_make_tport(struct target_fabric_configfs *tf,
1908 const char *name) 1927 struct config_group *group,
1928 const char *name)
1909{ 1929{
1910 struct tcm_vhost_tport *tport; 1930 struct tcm_vhost_tport *tport;
1911 char *ptr; 1931 char *ptr;
@@ -1975,9 +1995,9 @@ static void tcm_vhost_drop_tport(struct se_wwn *wwn)
1975 kfree(tport); 1995 kfree(tport);
1976} 1996}
1977 1997
1978static ssize_t tcm_vhost_wwn_show_attr_version( 1998static ssize_t
1979 struct target_fabric_configfs *tf, 1999tcm_vhost_wwn_show_attr_version(struct target_fabric_configfs *tf,
1980 char *page) 2000 char *page)
1981{ 2001{
1982 return sprintf(page, "TCM_VHOST fabric module %s on %s/%s" 2002 return sprintf(page, "TCM_VHOST fabric module %s on %s/%s"
1983 "on "UTS_RELEASE"\n", TCM_VHOST_VERSION, utsname()->sysname, 2003 "on "UTS_RELEASE"\n", TCM_VHOST_VERSION, utsname()->sysname,
diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c
index 1ee45bc85f67..a73ea217f24d 100644
--- a/drivers/vhost/test.c
+++ b/drivers/vhost/test.c
@@ -18,7 +18,7 @@
18#include <linux/slab.h> 18#include <linux/slab.h>
19 19
20#include "test.h" 20#include "test.h"
21#include "vhost.c" 21#include "vhost.h"
22 22
23/* Max number of bytes transferred before requeueing the job. 23/* Max number of bytes transferred before requeueing the job.
24 * Using this limit prevents one virtqueue from starving others. */ 24 * Using this limit prevents one virtqueue from starving others. */
@@ -38,17 +38,19 @@ struct vhost_test {
38 * read-size critical section for our kind of RCU. */ 38 * read-size critical section for our kind of RCU. */
39static void handle_vq(struct vhost_test *n) 39static void handle_vq(struct vhost_test *n)
40{ 40{
41 struct vhost_virtqueue *vq = &n->dev.vqs[VHOST_TEST_VQ]; 41 struct vhost_virtqueue *vq = &n->vqs[VHOST_TEST_VQ];
42 unsigned out, in; 42 unsigned out, in;
43 int head; 43 int head;
44 size_t len, total_len = 0; 44 size_t len, total_len = 0;
45 void *private; 45 void *private;
46 46
47 private = rcu_dereference_check(vq->private_data, 1); 47 mutex_lock(&vq->mutex);
48 if (!private) 48 private = vq->private_data;
49 if (!private) {
50 mutex_unlock(&vq->mutex);
49 return; 51 return;
52 }
50 53
51 mutex_lock(&vq->mutex);
52 vhost_disable_notify(&n->dev, vq); 54 vhost_disable_notify(&n->dev, vq);
53 55
54 for (;;) { 56 for (;;) {
@@ -102,15 +104,23 @@ static int vhost_test_open(struct inode *inode, struct file *f)
102{ 104{
103 struct vhost_test *n = kmalloc(sizeof *n, GFP_KERNEL); 105 struct vhost_test *n = kmalloc(sizeof *n, GFP_KERNEL);
104 struct vhost_dev *dev; 106 struct vhost_dev *dev;
107 struct vhost_virtqueue **vqs;
105 int r; 108 int r;
106 109
107 if (!n) 110 if (!n)
108 return -ENOMEM; 111 return -ENOMEM;
112 vqs = kmalloc(VHOST_TEST_VQ_MAX * sizeof(*vqs), GFP_KERNEL);
113 if (!vqs) {
114 kfree(n);
115 return -ENOMEM;
116 }
109 117
110 dev = &n->dev; 118 dev = &n->dev;
119 vqs[VHOST_TEST_VQ] = &n->vqs[VHOST_TEST_VQ];
111 n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick; 120 n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick;
112 r = vhost_dev_init(dev, n->vqs, VHOST_TEST_VQ_MAX); 121 r = vhost_dev_init(dev, vqs, VHOST_TEST_VQ_MAX);
113 if (r < 0) { 122 if (r < 0) {
123 kfree(vqs);
114 kfree(n); 124 kfree(n);
115 return r; 125 return r;
116 } 126 }
@@ -126,9 +136,8 @@ static void *vhost_test_stop_vq(struct vhost_test *n,
126 void *private; 136 void *private;
127 137
128 mutex_lock(&vq->mutex); 138 mutex_lock(&vq->mutex);
129 private = rcu_dereference_protected(vq->private_data, 139 private = vq->private_data;
130 lockdep_is_held(&vq->mutex)); 140 vq->private_data = NULL;
131 rcu_assign_pointer(vq->private_data, NULL);
132 mutex_unlock(&vq->mutex); 141 mutex_unlock(&vq->mutex);
133 return private; 142 return private;
134} 143}
@@ -140,7 +149,7 @@ static void vhost_test_stop(struct vhost_test *n, void **privatep)
140 149
141static void vhost_test_flush_vq(struct vhost_test *n, int index) 150static void vhost_test_flush_vq(struct vhost_test *n, int index)
142{ 151{
143 vhost_poll_flush(&n->dev.vqs[index].poll); 152 vhost_poll_flush(&n->vqs[index].poll);
144} 153}
145 154
146static void vhost_test_flush(struct vhost_test *n) 155static void vhost_test_flush(struct vhost_test *n)
@@ -268,14 +277,14 @@ static long vhost_test_ioctl(struct file *f, unsigned int ioctl,
268 return -EFAULT; 277 return -EFAULT;
269 return vhost_test_run(n, test); 278 return vhost_test_run(n, test);
270 case VHOST_GET_FEATURES: 279 case VHOST_GET_FEATURES:
271 features = VHOST_NET_FEATURES; 280 features = VHOST_FEATURES;
272 if (copy_to_user(featurep, &features, sizeof features)) 281 if (copy_to_user(featurep, &features, sizeof features))
273 return -EFAULT; 282 return -EFAULT;
274 return 0; 283 return 0;
275 case VHOST_SET_FEATURES: 284 case VHOST_SET_FEATURES:
276 if (copy_from_user(&features, featurep, sizeof features)) 285 if (copy_from_user(&features, featurep, sizeof features))
277 return -EFAULT; 286 return -EFAULT;
278 if (features & ~VHOST_NET_FEATURES) 287 if (features & ~VHOST_FEATURES)
279 return -EOPNOTSUPP; 288 return -EOPNOTSUPP;
280 return vhost_test_set_features(n, features); 289 return vhost_test_set_features(n, features);
281 case VHOST_RESET_OWNER: 290 case VHOST_RESET_OWNER:
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 60aa5ad09a2f..e58cf0001cee 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -25,6 +25,7 @@
25#include <linux/slab.h> 25#include <linux/slab.h>
26#include <linux/kthread.h> 26#include <linux/kthread.h>
27#include <linux/cgroup.h> 27#include <linux/cgroup.h>
28#include <linux/module.h>
28 29
29#include "vhost.h" 30#include "vhost.h"
30 31
@@ -66,6 +67,7 @@ void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
66 work->flushing = 0; 67 work->flushing = 0;
67 work->queue_seq = work->done_seq = 0; 68 work->queue_seq = work->done_seq = 0;
68} 69}
70EXPORT_SYMBOL_GPL(vhost_work_init);
69 71
70/* Init poll structure */ 72/* Init poll structure */
71void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 73void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
@@ -79,6 +81,7 @@ void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
79 81
80 vhost_work_init(&poll->work, fn); 82 vhost_work_init(&poll->work, fn);
81} 83}
84EXPORT_SYMBOL_GPL(vhost_poll_init);
82 85
83/* Start polling a file. We add ourselves to file's wait queue. The caller must 86/* Start polling a file. We add ourselves to file's wait queue. The caller must
84 * keep a reference to a file until after vhost_poll_stop is called. */ 87 * keep a reference to a file until after vhost_poll_stop is called. */
@@ -101,6 +104,7 @@ int vhost_poll_start(struct vhost_poll *poll, struct file *file)
101 104
102 return ret; 105 return ret;
103} 106}
107EXPORT_SYMBOL_GPL(vhost_poll_start);
104 108
105/* Stop polling a file. After this function returns, it becomes safe to drop the 109/* Stop polling a file. After this function returns, it becomes safe to drop the
106 * file reference. You must also flush afterwards. */ 110 * file reference. You must also flush afterwards. */
@@ -111,6 +115,7 @@ void vhost_poll_stop(struct vhost_poll *poll)
111 poll->wqh = NULL; 115 poll->wqh = NULL;
112 } 116 }
113} 117}
118EXPORT_SYMBOL_GPL(vhost_poll_stop);
114 119
115static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work, 120static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work,
116 unsigned seq) 121 unsigned seq)
@@ -123,7 +128,7 @@ static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work,
123 return left <= 0; 128 return left <= 0;
124} 129}
125 130
126static void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work) 131void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
127{ 132{
128 unsigned seq; 133 unsigned seq;
129 int flushing; 134 int flushing;
@@ -138,6 +143,7 @@ static void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
138 spin_unlock_irq(&dev->work_lock); 143 spin_unlock_irq(&dev->work_lock);
139 BUG_ON(flushing < 0); 144 BUG_ON(flushing < 0);
140} 145}
146EXPORT_SYMBOL_GPL(vhost_work_flush);
141 147
142/* Flush any work that has been scheduled. When calling this, don't hold any 148/* Flush any work that has been scheduled. When calling this, don't hold any
143 * locks that are also used by the callback. */ 149 * locks that are also used by the callback. */
@@ -145,6 +151,7 @@ void vhost_poll_flush(struct vhost_poll *poll)
145{ 151{
146 vhost_work_flush(poll->dev, &poll->work); 152 vhost_work_flush(poll->dev, &poll->work);
147} 153}
154EXPORT_SYMBOL_GPL(vhost_poll_flush);
148 155
149void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) 156void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
150{ 157{
@@ -158,11 +165,13 @@ void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
158 } 165 }
159 spin_unlock_irqrestore(&dev->work_lock, flags); 166 spin_unlock_irqrestore(&dev->work_lock, flags);
160} 167}
168EXPORT_SYMBOL_GPL(vhost_work_queue);
161 169
162void vhost_poll_queue(struct vhost_poll *poll) 170void vhost_poll_queue(struct vhost_poll *poll)
163{ 171{
164 vhost_work_queue(poll->dev, &poll->work); 172 vhost_work_queue(poll->dev, &poll->work);
165} 173}
174EXPORT_SYMBOL_GPL(vhost_poll_queue);
166 175
167static void vhost_vq_reset(struct vhost_dev *dev, 176static void vhost_vq_reset(struct vhost_dev *dev,
168 struct vhost_virtqueue *vq) 177 struct vhost_virtqueue *vq)
@@ -251,17 +260,16 @@ static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
251/* Helper to allocate iovec buffers for all vqs. */ 260/* Helper to allocate iovec buffers for all vqs. */
252static long vhost_dev_alloc_iovecs(struct vhost_dev *dev) 261static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
253{ 262{
263 struct vhost_virtqueue *vq;
254 int i; 264 int i;
255 265
256 for (i = 0; i < dev->nvqs; ++i) { 266 for (i = 0; i < dev->nvqs; ++i) {
257 dev->vqs[i]->indirect = kmalloc(sizeof *dev->vqs[i]->indirect * 267 vq = dev->vqs[i];
258 UIO_MAXIOV, GFP_KERNEL); 268 vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV,
259 dev->vqs[i]->log = kmalloc(sizeof *dev->vqs[i]->log * UIO_MAXIOV, 269 GFP_KERNEL);
260 GFP_KERNEL); 270 vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL);
261 dev->vqs[i]->heads = kmalloc(sizeof *dev->vqs[i]->heads * 271 vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL);
262 UIO_MAXIOV, GFP_KERNEL); 272 if (!vq->indirect || !vq->log || !vq->heads)
263 if (!dev->vqs[i]->indirect || !dev->vqs[i]->log ||
264 !dev->vqs[i]->heads)
265 goto err_nomem; 273 goto err_nomem;
266 } 274 }
267 return 0; 275 return 0;
@@ -283,6 +291,7 @@ static void vhost_dev_free_iovecs(struct vhost_dev *dev)
283long vhost_dev_init(struct vhost_dev *dev, 291long vhost_dev_init(struct vhost_dev *dev,
284 struct vhost_virtqueue **vqs, int nvqs) 292 struct vhost_virtqueue **vqs, int nvqs)
285{ 293{
294 struct vhost_virtqueue *vq;
286 int i; 295 int i;
287 296
288 dev->vqs = vqs; 297 dev->vqs = vqs;
@@ -297,19 +306,21 @@ long vhost_dev_init(struct vhost_dev *dev,
297 dev->worker = NULL; 306 dev->worker = NULL;
298 307
299 for (i = 0; i < dev->nvqs; ++i) { 308 for (i = 0; i < dev->nvqs; ++i) {
300 dev->vqs[i]->log = NULL; 309 vq = dev->vqs[i];
301 dev->vqs[i]->indirect = NULL; 310 vq->log = NULL;
302 dev->vqs[i]->heads = NULL; 311 vq->indirect = NULL;
303 dev->vqs[i]->dev = dev; 312 vq->heads = NULL;
304 mutex_init(&dev->vqs[i]->mutex); 313 vq->dev = dev;
305 vhost_vq_reset(dev, dev->vqs[i]); 314 mutex_init(&vq->mutex);
306 if (dev->vqs[i]->handle_kick) 315 vhost_vq_reset(dev, vq);
307 vhost_poll_init(&dev->vqs[i]->poll, 316 if (vq->handle_kick)
308 dev->vqs[i]->handle_kick, POLLIN, dev); 317 vhost_poll_init(&vq->poll, vq->handle_kick,
318 POLLIN, dev);
309 } 319 }
310 320
311 return 0; 321 return 0;
312} 322}
323EXPORT_SYMBOL_GPL(vhost_dev_init);
313 324
314/* Caller should have device mutex */ 325/* Caller should have device mutex */
315long vhost_dev_check_owner(struct vhost_dev *dev) 326long vhost_dev_check_owner(struct vhost_dev *dev)
@@ -317,6 +328,7 @@ long vhost_dev_check_owner(struct vhost_dev *dev)
317 /* Are you the owner? If not, I don't think you mean to do that */ 328 /* Are you the owner? If not, I don't think you mean to do that */
318 return dev->mm == current->mm ? 0 : -EPERM; 329 return dev->mm == current->mm ? 0 : -EPERM;
319} 330}
331EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
320 332
321struct vhost_attach_cgroups_struct { 333struct vhost_attach_cgroups_struct {
322 struct vhost_work work; 334 struct vhost_work work;
@@ -348,6 +360,7 @@ bool vhost_dev_has_owner(struct vhost_dev *dev)
348{ 360{
349 return dev->mm; 361 return dev->mm;
350} 362}
363EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
351 364
352/* Caller should have device mutex */ 365/* Caller should have device mutex */
353long vhost_dev_set_owner(struct vhost_dev *dev) 366long vhost_dev_set_owner(struct vhost_dev *dev)
@@ -391,11 +404,13 @@ err_worker:
391err_mm: 404err_mm:
392 return err; 405 return err;
393} 406}
407EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
394 408
395struct vhost_memory *vhost_dev_reset_owner_prepare(void) 409struct vhost_memory *vhost_dev_reset_owner_prepare(void)
396{ 410{
397 return kmalloc(offsetof(struct vhost_memory, regions), GFP_KERNEL); 411 return kmalloc(offsetof(struct vhost_memory, regions), GFP_KERNEL);
398} 412}
413EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
399 414
400/* Caller should have device mutex */ 415/* Caller should have device mutex */
401void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_memory *memory) 416void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_memory *memory)
@@ -406,6 +421,7 @@ void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_memory *memory)
406 memory->nregions = 0; 421 memory->nregions = 0;
407 RCU_INIT_POINTER(dev->memory, memory); 422 RCU_INIT_POINTER(dev->memory, memory);
408} 423}
424EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
409 425
410void vhost_dev_stop(struct vhost_dev *dev) 426void vhost_dev_stop(struct vhost_dev *dev)
411{ 427{
@@ -418,6 +434,7 @@ void vhost_dev_stop(struct vhost_dev *dev)
418 } 434 }
419 } 435 }
420} 436}
437EXPORT_SYMBOL_GPL(vhost_dev_stop);
421 438
422/* Caller should have device mutex if and only if locked is set */ 439/* Caller should have device mutex if and only if locked is set */
423void vhost_dev_cleanup(struct vhost_dev *dev, bool locked) 440void vhost_dev_cleanup(struct vhost_dev *dev, bool locked)
@@ -458,6 +475,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev, bool locked)
458 mmput(dev->mm); 475 mmput(dev->mm);
459 dev->mm = NULL; 476 dev->mm = NULL;
460} 477}
478EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
461 479
462static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz) 480static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
463{ 481{
@@ -543,6 +561,7 @@ int vhost_log_access_ok(struct vhost_dev *dev)
543 lockdep_is_held(&dev->mutex)); 561 lockdep_is_held(&dev->mutex));
544 return memory_access_ok(dev, mp, 1); 562 return memory_access_ok(dev, mp, 1);
545} 563}
564EXPORT_SYMBOL_GPL(vhost_log_access_ok);
546 565
547/* Verify access for write logging. */ 566/* Verify access for write logging. */
548/* Caller should have vq mutex and device mutex */ 567/* Caller should have vq mutex and device mutex */
@@ -568,6 +587,7 @@ int vhost_vq_access_ok(struct vhost_virtqueue *vq)
568 return vq_access_ok(vq->dev, vq->num, vq->desc, vq->avail, vq->used) && 587 return vq_access_ok(vq->dev, vq->num, vq->desc, vq->avail, vq->used) &&
569 vq_log_access_ok(vq->dev, vq, vq->log_base); 588 vq_log_access_ok(vq->dev, vq, vq->log_base);
570} 589}
590EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
571 591
572static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) 592static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
573{ 593{
@@ -797,6 +817,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp)
797 vhost_poll_flush(&vq->poll); 817 vhost_poll_flush(&vq->poll);
798 return r; 818 return r;
799} 819}
820EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
800 821
801/* Caller must have device mutex */ 822/* Caller must have device mutex */
802long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) 823long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
@@ -877,6 +898,7 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
877done: 898done:
878 return r; 899 return r;
879} 900}
901EXPORT_SYMBOL_GPL(vhost_dev_ioctl);
880 902
881static const struct vhost_memory_region *find_region(struct vhost_memory *mem, 903static const struct vhost_memory_region *find_region(struct vhost_memory *mem,
882 __u64 addr, __u32 len) 904 __u64 addr, __u32 len)
@@ -968,6 +990,7 @@ int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
968 BUG(); 990 BUG();
969 return 0; 991 return 0;
970} 992}
993EXPORT_SYMBOL_GPL(vhost_log_write);
971 994
972static int vhost_update_used_flags(struct vhost_virtqueue *vq) 995static int vhost_update_used_flags(struct vhost_virtqueue *vq)
973{ 996{
@@ -1019,6 +1042,7 @@ int vhost_init_used(struct vhost_virtqueue *vq)
1019 vq->signalled_used_valid = false; 1042 vq->signalled_used_valid = false;
1020 return get_user(vq->last_used_idx, &vq->used->idx); 1043 return get_user(vq->last_used_idx, &vq->used->idx);
1021} 1044}
1045EXPORT_SYMBOL_GPL(vhost_init_used);
1022 1046
1023static int translate_desc(struct vhost_dev *dev, u64 addr, u32 len, 1047static int translate_desc(struct vhost_dev *dev, u64 addr, u32 len,
1024 struct iovec iov[], int iov_size) 1048 struct iovec iov[], int iov_size)
@@ -1295,12 +1319,14 @@ int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,
1295 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); 1319 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
1296 return head; 1320 return head;
1297} 1321}
1322EXPORT_SYMBOL_GPL(vhost_get_vq_desc);
1298 1323
1299/* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */ 1324/* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */
1300void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n) 1325void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n)
1301{ 1326{
1302 vq->last_avail_idx -= n; 1327 vq->last_avail_idx -= n;
1303} 1328}
1329EXPORT_SYMBOL_GPL(vhost_discard_vq_desc);
1304 1330
1305/* After we've used one of their buffers, we tell them about it. We'll then 1331/* After we've used one of their buffers, we tell them about it. We'll then
1306 * want to notify the guest, using eventfd. */ 1332 * want to notify the guest, using eventfd. */
@@ -1349,6 +1375,7 @@ int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
1349 vq->signalled_used_valid = false; 1375 vq->signalled_used_valid = false;
1350 return 0; 1376 return 0;
1351} 1377}
1378EXPORT_SYMBOL_GPL(vhost_add_used);
1352 1379
1353static int __vhost_add_used_n(struct vhost_virtqueue *vq, 1380static int __vhost_add_used_n(struct vhost_virtqueue *vq,
1354 struct vring_used_elem *heads, 1381 struct vring_used_elem *heads,
@@ -1418,6 +1445,7 @@ int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
1418 } 1445 }
1419 return r; 1446 return r;
1420} 1447}
1448EXPORT_SYMBOL_GPL(vhost_add_used_n);
1421 1449
1422static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 1450static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1423{ 1451{
@@ -1462,6 +1490,7 @@ void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1462 if (vq->call_ctx && vhost_notify(dev, vq)) 1490 if (vq->call_ctx && vhost_notify(dev, vq))
1463 eventfd_signal(vq->call_ctx, 1); 1491 eventfd_signal(vq->call_ctx, 1);
1464} 1492}
1493EXPORT_SYMBOL_GPL(vhost_signal);
1465 1494
1466/* And here's the combo meal deal. Supersize me! */ 1495/* And here's the combo meal deal. Supersize me! */
1467void vhost_add_used_and_signal(struct vhost_dev *dev, 1496void vhost_add_used_and_signal(struct vhost_dev *dev,
@@ -1471,6 +1500,7 @@ void vhost_add_used_and_signal(struct vhost_dev *dev,
1471 vhost_add_used(vq, head, len); 1500 vhost_add_used(vq, head, len);
1472 vhost_signal(dev, vq); 1501 vhost_signal(dev, vq);
1473} 1502}
1503EXPORT_SYMBOL_GPL(vhost_add_used_and_signal);
1474 1504
1475/* multi-buffer version of vhost_add_used_and_signal */ 1505/* multi-buffer version of vhost_add_used_and_signal */
1476void vhost_add_used_and_signal_n(struct vhost_dev *dev, 1506void vhost_add_used_and_signal_n(struct vhost_dev *dev,
@@ -1480,6 +1510,7 @@ void vhost_add_used_and_signal_n(struct vhost_dev *dev,
1480 vhost_add_used_n(vq, heads, count); 1510 vhost_add_used_n(vq, heads, count);
1481 vhost_signal(dev, vq); 1511 vhost_signal(dev, vq);
1482} 1512}
1513EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
1483 1514
1484/* OK, now we need to know about added descriptors. */ 1515/* OK, now we need to know about added descriptors. */
1485bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 1516bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
@@ -1517,6 +1548,7 @@ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1517 1548
1518 return avail_idx != vq->avail_idx; 1549 return avail_idx != vq->avail_idx;
1519} 1550}
1551EXPORT_SYMBOL_GPL(vhost_enable_notify);
1520 1552
1521/* We don't need to be notified again. */ 1553/* We don't need to be notified again. */
1522void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 1554void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
@@ -1533,3 +1565,21 @@ void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
1533 &vq->used->flags, r); 1565 &vq->used->flags, r);
1534 } 1566 }
1535} 1567}
1568EXPORT_SYMBOL_GPL(vhost_disable_notify);
1569
1570static int __init vhost_init(void)
1571{
1572 return 0;
1573}
1574
1575static void __exit vhost_exit(void)
1576{
1577}
1578
1579module_init(vhost_init);
1580module_exit(vhost_exit);
1581
1582MODULE_VERSION("0.0.1");
1583MODULE_LICENSE("GPL v2");
1584MODULE_AUTHOR("Michael S. Tsirkin");
1585MODULE_DESCRIPTION("Host kernel accelerator for virtio");
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 64adcf99ff33..42298cd23c73 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -46,6 +46,8 @@ int vhost_poll_start(struct vhost_poll *poll, struct file *file);
46void vhost_poll_stop(struct vhost_poll *poll); 46void vhost_poll_stop(struct vhost_poll *poll);
47void vhost_poll_flush(struct vhost_poll *poll); 47void vhost_poll_flush(struct vhost_poll *poll);
48void vhost_poll_queue(struct vhost_poll *poll); 48void vhost_poll_queue(struct vhost_poll *poll);
49void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work);
50long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp);
49 51
50struct vhost_log { 52struct vhost_log {
51 u64 addr; 53 u64 addr;
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-05 05:44:34 -0500