aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2017-05-12 14:44:13 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2017-05-12 14:44:13 -0400
commitbd1286f964041a2fe5eec8801a51bd79d905bd02 (patch)
tree6402e5fb3d5dab05cb6fe9c2065a6849fbeadb3e
parent050453295f808dd683b9a88f58a1b29540097394 (diff)
parent984a9d4c40bed351a92ed31f0723a710444295da (diff)
Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/nab/target-pending
Pull SCSI target updates from Nicholas Bellinger: "Things were a lot more calm than previously expected. It's primarily fixes in various areas, with most of the new functionality centering around TCMU backend driver work that Xiubo Li has been driving. Here's the summary on the feature side: - Make T10-PI verify configurable for emulated (FILEIO + RD) backends (Dmitry Monakhov) - Allow target-core/TCMU pass-through to use in-kernel SPC-PR logic (Bryant Ly + MNC) - Add TCMU support for growing ring buffer size (Xiubo Li + MNC) - Add TCMU support for global block data pool (Xiubo Li + MNC) and on the bug-fix side: - Fix COMPARE_AND_WRITE non GOOD status handling for READ phase failures (Gary Guo + nab) - Fix iscsi-target hang with explicitly changing per NodeACL CmdSN number depth with concurrent login driven session reinstatement. (Gary Guo + nab) - Fix ibmvscsis fabric driver ABORT task handling (Bryant Ly) - Fix target-core/FILEIO zero length handling (Bart Van Assche) Also, there was an OOPs introduced with the WRITE_VERIFY changes that I ended up reverting at the last minute, because as not unusual Bart and I could not agree on the fix in time for -rc1. Since it's specific to a conformance test, it's been reverted for now. There is a separate patch in the queue to address the underlying control CDB write overflow regression in >= v4.3 separate from the WRITE_VERIFY revert here, that will be pushed post -rc1" * 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/nab/target-pending: (30 commits) Revert "target: Fix VERIFY and WRITE VERIFY command parsing" IB/srpt: Avoid that aborting a command triggers a kernel warning IB/srpt: Fix abort handling target/fileio: Fix zero-length READ and WRITE handling ibmvscsis: Do not send aborted task response tcmu: fix module removal due to stuck thread target: Don't force session reset if queue_depth does not change iscsi-target: Set session_fall_back_to_erl0 when forcing reinstatement target: Fix compare_and_write_callback handling for non GOOD status tcmu: Recalculate the tcmu_cmd size to save cmd area memories tcmu: Add global data block pool support tcmu: Add dynamic growing data area feature support target: fixup error message in target_tg_pt_gp_tg_pt_gp_id_store() target: fixup error message in target_tg_pt_gp_alua_access_type_store() target/user: PGR Support target: Add WRITE_VERIFY_16 Documentation/target: add an example script to configure an iSCSI target target: Use kmalloc_array() in transport_kmap_data_sg() target: Use kmalloc_array() in compare_and_write_callback() target: Improve size determinations in two functions ...
Diffstat
-rwxr-xr-xDocumentation/target/target-export-device80
-rw-r--r--drivers/infiniband/ulp/srpt/ib_srpt.c9
-rw-r--r--drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c114
-rw-r--r--drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h2
-rw-r--r--drivers/target/iscsi/iscsi_target.c51
-rw-r--r--drivers/target/iscsi/iscsi_target_configfs.c1
-rw-r--r--drivers/target/iscsi/iscsi_target_login.c1
-rw-r--r--drivers/target/target_core_configfs.c54
-rw-r--r--drivers/target/target_core_device.c38
-rw-r--r--drivers/target/target_core_file.c32
-rw-r--r--drivers/target/target_core_iblock.c12
-rw-r--r--drivers/target/target_core_iblock.h3
-rw-r--r--drivers/target/target_core_pr.c2
-rw-r--r--drivers/target/target_core_pr.h9
-rw-r--r--drivers/target/target_core_pscsi.c3
-rw-r--r--drivers/target/target_core_rd.c50
-rw-r--r--drivers/target/target_core_sbc.c10
-rw-r--r--drivers/target/target_core_tpg.c7
-rw-r--r--drivers/target/target_core_transport.c2
-rw-r--r--drivers/target/target_core_user.c645
-rw-r--r--include/scsi/scsi_proto.h1
-rw-r--r--include/target/target_core_backend.h1
-rw-r--r--include/target/target_core_base.h1
23 files changed, 859 insertions, 269 deletions
diff --git a/Documentation/target/target-export-device b/Documentation/target/target-export-device
new file mode 100755
index 000000000000..b803f4f886b5
--- /dev/null
+++ b/Documentation/target/target-export-device
@@ -0,0 +1,80 @@
1#!/bin/sh
2#
3# This script illustrates the sequence of operations in configfs to
4# create a very simple LIO iSCSI target with a file or block device
5# backstore.
6#
7# (C) Copyright 2014 Christophe Vu-Brugier <cvubrugier@fastmail.fm>
8#
9
10print_usage() {
11 cat <<EOF
12Usage: $(basename $0) [-p PORTAL] DEVICE|FILE
13Export a block device or a file as an iSCSI target with a single LUN
14EOF
15}
16
17die() {
18 echo $1
19 exit 1
20}
21
22while getopts "hp:" arg; do
23 case $arg in
24 h) print_usage; exit 0;;
25 p) PORTAL=${OPTARG};;
26 esac
27done
28shift $(($OPTIND - 1))
29
30DEVICE=$1
31[ -n "$DEVICE" ] || die "Missing device or file argument"
32[ -b $DEVICE -o -f $DEVICE ] || die "Invalid device or file: ${DEVICE}"
33IQN="iqn.2003-01.org.linux-iscsi.$(hostname):$(basename $DEVICE)"
34[ -n "$PORTAL" ] || PORTAL="0.0.0.0:3260"
35
36CONFIGFS=/sys/kernel/config
37CORE_DIR=$CONFIGFS/target/core
38ISCSI_DIR=$CONFIGFS/target/iscsi
39
40# Load the target modules and mount the config file system
41lsmod | grep -q configfs || modprobe configfs
42lsmod | grep -q target_core_mod || modprobe target_core_mod
43mount | grep -q ^configfs || mount -t configfs none $CONFIGFS
44mkdir -p $ISCSI_DIR
45
46# Create a backstore
47if [ -b $DEVICE ]; then
48 BACKSTORE_DIR=$CORE_DIR/iblock_0/data
49 mkdir -p $BACKSTORE_DIR
50 echo "udev_path=${DEVICE}" > $BACKSTORE_DIR/control
51else
52 BACKSTORE_DIR=$CORE_DIR/fileio_0/data
53 mkdir -p $BACKSTORE_DIR
54 DEVICE_SIZE=$(du -b $DEVICE | cut -f1)
55 echo "fd_dev_name=${DEVICE}" > $BACKSTORE_DIR/control
56 echo "fd_dev_size=${DEVICE_SIZE}" > $BACKSTORE_DIR/control
57 echo 1 > $BACKSTORE_DIR/attrib/emulate_write_cache
58fi
59echo 1 > $BACKSTORE_DIR/enable
60
61# Create an iSCSI target and a target portal group (TPG)
62mkdir $ISCSI_DIR/$IQN
63mkdir $ISCSI_DIR/$IQN/tpgt_1/
64
65# Create a LUN
66mkdir $ISCSI_DIR/$IQN/tpgt_1/lun/lun_0
67ln -s $BACKSTORE_DIR $ISCSI_DIR/$IQN/tpgt_1/lun/lun_0/data
68echo 1 > $ISCSI_DIR/$IQN/tpgt_1/enable
69
70# Create a network portal
71mkdir $ISCSI_DIR/$IQN/tpgt_1/np/$PORTAL
72
73# Disable authentication
74echo 0 > $ISCSI_DIR/$IQN/tpgt_1/attrib/authentication
75echo 1 > $ISCSI_DIR/$IQN/tpgt_1/attrib/generate_node_acls
76
77# Allow write access for non authenticated initiators
78echo 0 > $ISCSI_DIR/$IQN/tpgt_1/attrib/demo_mode_write_protect
79
80echo "Target ${IQN}, portal ${PORTAL} has been created"
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 7d6c199de2d6..1ced0731c140 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -2302,12 +2302,8 @@ static void srpt_queue_response(struct se_cmd *cmd)
2302 } 2302 }
2303 spin_unlock_irqrestore(&ioctx->spinlock, flags); 2303 spin_unlock_irqrestore(&ioctx->spinlock, flags);
2304 2304
2305 if (unlikely(transport_check_aborted_status(&ioctx->cmd, false) 2305 if (unlikely(WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT)))
2306 || WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT))) {
2307 atomic_inc(&ch->req_lim_delta);
2308 srpt_abort_cmd(ioctx);
2309 return; 2306 return;
2310 }
2311 2307
2312 /* For read commands, transfer the data to the initiator. */ 2308 /* For read commands, transfer the data to the initiator. */
2313 if (ioctx->cmd.data_direction == DMA_FROM_DEVICE && 2309 if (ioctx->cmd.data_direction == DMA_FROM_DEVICE &&
@@ -2689,7 +2685,8 @@ static void srpt_release_cmd(struct se_cmd *se_cmd)
2689 struct srpt_rdma_ch *ch = ioctx->ch; 2685 struct srpt_rdma_ch *ch = ioctx->ch;
2690 unsigned long flags; 2686 unsigned long flags;
2691 2687
2692 WARN_ON(ioctx->state != SRPT_STATE_DONE); 2688 WARN_ON_ONCE(ioctx->state != SRPT_STATE_DONE &&
2689 !(ioctx->cmd.transport_state & CMD_T_ABORTED));
2693 2690
2694 if (ioctx->n_rw_ctx) { 2691 if (ioctx->n_rw_ctx) {
2695 srpt_free_rw_ctxs(ch, ioctx); 2692 srpt_free_rw_ctxs(ch, ioctx);
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
index 0f807798c624..d390325c99ec 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
@@ -1170,6 +1170,7 @@ static struct ibmvscsis_cmd *ibmvscsis_get_free_cmd(struct scsi_info *vscsi)
1170 cmd = list_first_entry_or_null(&vscsi->free_cmd, 1170 cmd = list_first_entry_or_null(&vscsi->free_cmd,
1171 struct ibmvscsis_cmd, list); 1171 struct ibmvscsis_cmd, list);
1172 if (cmd) { 1172 if (cmd) {
1173 cmd->flags &= ~(DELAY_SEND);
1173 list_del(&cmd->list); 1174 list_del(&cmd->list);
1174 cmd->iue = iue; 1175 cmd->iue = iue;
1175 cmd->type = UNSET_TYPE; 1176 cmd->type = UNSET_TYPE;
@@ -1749,45 +1750,79 @@ static void srp_snd_msg_failed(struct scsi_info *vscsi, long rc)
1749static void ibmvscsis_send_messages(struct scsi_info *vscsi) 1750static void ibmvscsis_send_messages(struct scsi_info *vscsi)
1750{ 1751{
1751 u64 msg_hi = 0; 1752 u64 msg_hi = 0;
1752 /* note do not attmempt to access the IU_data_ptr with this pointer 1753 /* note do not attempt to access the IU_data_ptr with this pointer
1753 * it is not valid 1754 * it is not valid
1754 */ 1755 */
1755 struct viosrp_crq *crq = (struct viosrp_crq *)&msg_hi; 1756 struct viosrp_crq *crq = (struct viosrp_crq *)&msg_hi;
1756 struct ibmvscsis_cmd *cmd, *nxt; 1757 struct ibmvscsis_cmd *cmd, *nxt;
1757 struct iu_entry *iue; 1758 struct iu_entry *iue;
1758 long rc = ADAPT_SUCCESS; 1759 long rc = ADAPT_SUCCESS;
1760 bool retry = false;
1759 1761
1760 if (!(vscsi->flags & RESPONSE_Q_DOWN)) { 1762 if (!(vscsi->flags & RESPONSE_Q_DOWN)) {
1761 list_for_each_entry_safe(cmd, nxt, &vscsi->waiting_rsp, list) { 1763 do {
1762 iue = cmd->iue; 1764 retry = false;
1765 list_for_each_entry_safe(cmd, nxt, &vscsi->waiting_rsp,
1766 list) {
1767 /*
1768 * Check to make sure abort cmd gets processed
1769 * prior to the abort tmr cmd
1770 */
1771 if (cmd->flags & DELAY_SEND)
1772 continue;
1763 1773
1764 crq->valid = VALID_CMD_RESP_EL; 1774 if (cmd->abort_cmd) {
1765 crq->format = cmd->rsp.format; 1775 retry = true;
1776 cmd->abort_cmd->flags &= ~(DELAY_SEND);
1777 }
1766 1778
1767 if (cmd->flags & CMD_FAST_FAIL) 1779 /*
1768 crq->status = VIOSRP_ADAPTER_FAIL; 1780 * If CMD_T_ABORTED w/o CMD_T_TAS scenarios and
1781 * the case where LIO issued a
1782 * ABORT_TASK: Sending TMR_TASK_DOES_NOT_EXIST
1783 * case then we dont send a response, since it
1784 * was already done.
1785 */
1786 if (cmd->se_cmd.transport_state & CMD_T_ABORTED &&
1787 !(cmd->se_cmd.transport_state & CMD_T_TAS)) {
1788 list_del(&cmd->list);
1789 ibmvscsis_free_cmd_resources(vscsi,
1790 cmd);
1791 } else {
1792 iue = cmd->iue;
1769 1793
1770 crq->IU_length = cpu_to_be16(cmd->rsp.len); 1794 crq->valid = VALID_CMD_RESP_EL;
1795 crq->format = cmd->rsp.format;
1771 1796
1772 rc = h_send_crq(vscsi->dma_dev->unit_address, 1797 if (cmd->flags & CMD_FAST_FAIL)
1773 be64_to_cpu(msg_hi), 1798 crq->status = VIOSRP_ADAPTER_FAIL;
1774 be64_to_cpu(cmd->rsp.tag));
1775 1799
1776 pr_debug("send_messages: cmd %p, tag 0x%llx, rc %ld\n", 1800 crq->IU_length = cpu_to_be16(cmd->rsp.len);
1777 cmd, be64_to_cpu(cmd->rsp.tag), rc);
1778 1801
1779 /* if all ok free up the command element resources */ 1802 rc = h_send_crq(vscsi->dma_dev->unit_address,
1780 if (rc == H_SUCCESS) { 1803 be64_to_cpu(msg_hi),
1781 /* some movement has occurred */ 1804 be64_to_cpu(cmd->rsp.tag));
1782 vscsi->rsp_q_timer.timer_pops = 0;
1783 list_del(&cmd->list);
1784 1805
1785 ibmvscsis_free_cmd_resources(vscsi, cmd); 1806 pr_debug("send_messages: cmd %p, tag 0x%llx, rc %ld\n",
1786 } else { 1807 cmd, be64_to_cpu(cmd->rsp.tag), rc);
1787 srp_snd_msg_failed(vscsi, rc); 1808
1788 break; 1809 /* if all ok free up the command
1810 * element resources
1811 */
1812 if (rc == H_SUCCESS) {
1813 /* some movement has occurred */
1814 vscsi->rsp_q_timer.timer_pops = 0;
1815 list_del(&cmd->list);
1816
1817 ibmvscsis_free_cmd_resources(vscsi,
1818 cmd);
1819 } else {
1820 srp_snd_msg_failed(vscsi, rc);
1821 break;
1822 }
1823 }
1789 } 1824 }
1790 } 1825 } while (retry);
1791 1826
1792 if (!rc) { 1827 if (!rc) {
1793 /* 1828 /*
@@ -2708,6 +2743,7 @@ static int ibmvscsis_alloc_cmds(struct scsi_info *vscsi, int num)
2708 2743
2709 for (i = 0, cmd = (struct ibmvscsis_cmd *)vscsi->cmd_pool; i < num; 2744 for (i = 0, cmd = (struct ibmvscsis_cmd *)vscsi->cmd_pool; i < num;
2710 i++, cmd++) { 2745 i++, cmd++) {
2746 cmd->abort_cmd = NULL;
2711 cmd->adapter = vscsi; 2747 cmd->adapter = vscsi;
2712 INIT_WORK(&cmd->work, ibmvscsis_scheduler); 2748 INIT_WORK(&cmd->work, ibmvscsis_scheduler);
2713 list_add_tail(&cmd->list, &vscsi->free_cmd); 2749 list_add_tail(&cmd->list, &vscsi->free_cmd);
@@ -3579,9 +3615,20 @@ static int ibmvscsis_write_pending(struct se_cmd *se_cmd)
3579{ 3615{
3580 struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd, 3616 struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3581 se_cmd); 3617 se_cmd);
3618 struct scsi_info *vscsi = cmd->adapter;
3582 struct iu_entry *iue = cmd->iue; 3619 struct iu_entry *iue = cmd->iue;
3583 int rc; 3620 int rc;
3584 3621
3622 /*
3623 * If CLIENT_FAILED OR RESPONSE_Q_DOWN, then just return success
3624 * since LIO can't do anything about it, and we dont want to
3625 * attempt an srp_transfer_data.
3626 */
3627 if ((vscsi->flags & (CLIENT_FAILED | RESPONSE_Q_DOWN))) {
3628 pr_err("write_pending failed since: %d\n", vscsi->flags);
3629 return 0;
3630 }
3631
3585 rc = srp_transfer_data(cmd, &vio_iu(iue)->srp.cmd, ibmvscsis_rdma, 3632 rc = srp_transfer_data(cmd, &vio_iu(iue)->srp.cmd, ibmvscsis_rdma,
3586 1, 1); 3633 1, 1);
3587 if (rc) { 3634 if (rc) {
@@ -3660,11 +3707,28 @@ static void ibmvscsis_queue_tm_rsp(struct se_cmd *se_cmd)
3660 struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd, 3707 struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3661 se_cmd); 3708 se_cmd);
3662 struct scsi_info *vscsi = cmd->adapter; 3709 struct scsi_info *vscsi = cmd->adapter;
3710 struct ibmvscsis_cmd *cmd_itr;
3711 struct iu_entry *iue = iue = cmd->iue;
3712 struct srp_tsk_mgmt *srp_tsk = &vio_iu(iue)->srp.tsk_mgmt;
3713 u64 tag_to_abort = be64_to_cpu(srp_tsk->task_tag);
3663 uint len; 3714 uint len;
3664 3715
3665 pr_debug("queue_tm_rsp %p, status %d\n", 3716 pr_debug("queue_tm_rsp %p, status %d\n",
3666 se_cmd, (int)se_cmd->se_tmr_req->response); 3717 se_cmd, (int)se_cmd->se_tmr_req->response);
3667 3718
3719 if (srp_tsk->tsk_mgmt_func == SRP_TSK_ABORT_TASK &&
3720 cmd->se_cmd.se_tmr_req->response == TMR_TASK_DOES_NOT_EXIST) {
3721 spin_lock_bh(&vscsi->intr_lock);
3722 list_for_each_entry(cmd_itr, &vscsi->active_q, list) {
3723 if (tag_to_abort == cmd_itr->se_cmd.tag) {
3724 cmd_itr->abort_cmd = cmd;
3725 cmd->flags |= DELAY_SEND;
3726 break;
3727 }
3728 }
3729 spin_unlock_bh(&vscsi->intr_lock);
3730 }
3731
3668 srp_build_response(vscsi, cmd, &len); 3732 srp_build_response(vscsi, cmd, &len);
3669 cmd->rsp.format = SRP_FORMAT; 3733 cmd->rsp.format = SRP_FORMAT;
3670 cmd->rsp.len = len; 3734 cmd->rsp.len = len;
@@ -3672,8 +3736,8 @@ static void ibmvscsis_queue_tm_rsp(struct se_cmd *se_cmd)
3672 3736
3673static void ibmvscsis_aborted_task(struct se_cmd *se_cmd) 3737static void ibmvscsis_aborted_task(struct se_cmd *se_cmd)
3674{ 3738{
3675 /* TBD: What (if anything) should we do here? */ 3739 pr_debug("ibmvscsis_aborted_task %p task_tag: %llu\n",
3676 pr_debug("ibmvscsis_aborted_task %p\n", se_cmd); 3740 se_cmd, se_cmd->tag);
3677} 3741}
3678 3742
3679static struct se_wwn *ibmvscsis_make_tport(struct target_fabric_configfs *tf, 3743static struct se_wwn *ibmvscsis_make_tport(struct target_fabric_configfs *tf,
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
index 65c6189885ab..b4391a8de456 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
@@ -168,10 +168,12 @@ struct ibmvscsis_cmd {
168 struct iu_rsp rsp; 168 struct iu_rsp rsp;
169 struct work_struct work; 169 struct work_struct work;
170 struct scsi_info *adapter; 170 struct scsi_info *adapter;
171 struct ibmvscsis_cmd *abort_cmd;
171 /* Sense buffer that will be mapped into outgoing status */ 172 /* Sense buffer that will be mapped into outgoing status */
172 unsigned char sense_buf[TRANSPORT_SENSE_BUFFER]; 173 unsigned char sense_buf[TRANSPORT_SENSE_BUFFER];
173 u64 init_time; 174 u64 init_time;
174#define CMD_FAST_FAIL BIT(0) 175#define CMD_FAST_FAIL BIT(0)
176#define DELAY_SEND BIT(1)
175 u32 flags; 177 u32 flags;
176 char type; 178 char type;
177}; 179};
diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c
index e3f9ed3690b7..26a9bcd5ee6a 100644
--- a/drivers/target/iscsi/iscsi_target.c
+++ b/drivers/target/iscsi/iscsi_target.c
@@ -128,11 +128,9 @@ struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf)
128 return ERR_PTR(-EINVAL); 128 return ERR_PTR(-EINVAL);
129 } 129 }
130 130
131 tiqn = kzalloc(sizeof(struct iscsi_tiqn), GFP_KERNEL); 131 tiqn = kzalloc(sizeof(*tiqn), GFP_KERNEL);
132 if (!tiqn) { 132 if (!tiqn)
133 pr_err("Unable to allocate struct iscsi_tiqn\n");
134 return ERR_PTR(-ENOMEM); 133 return ERR_PTR(-ENOMEM);
135 }
136 134
137 sprintf(tiqn->tiqn, "%s", buf); 135 sprintf(tiqn->tiqn, "%s", buf);
138 INIT_LIST_HEAD(&tiqn->tiqn_list); 136 INIT_LIST_HEAD(&tiqn->tiqn_list);
@@ -362,9 +360,8 @@ struct iscsi_np *iscsit_add_np(
362 return np; 360 return np;
363 } 361 }
364 362
365 np = kzalloc(sizeof(struct iscsi_np), GFP_KERNEL); 363 np = kzalloc(sizeof(*np), GFP_KERNEL);
366 if (!np) { 364 if (!np) {
367 pr_err("Unable to allocate memory for struct iscsi_np\n");
368 mutex_unlock(&np_lock); 365 mutex_unlock(&np_lock);
369 return ERR_PTR(-ENOMEM); 366 return ERR_PTR(-ENOMEM);
370 } 367 }
@@ -696,12 +693,10 @@ static int __init iscsi_target_init_module(void)
696 int ret = 0, size; 693 int ret = 0, size;
697 694
698 pr_debug("iSCSI-Target "ISCSIT_VERSION"\n"); 695 pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");
699 696 iscsit_global = kzalloc(sizeof(*iscsit_global), GFP_KERNEL);
700 iscsit_global = kzalloc(sizeof(struct iscsit_global), GFP_KERNEL); 697 if (!iscsit_global)
701 if (!iscsit_global) {
702 pr_err("Unable to allocate memory for iscsit_global\n");
703 return -1; 698 return -1;
704 } 699
705 spin_lock_init(&iscsit_global->ts_bitmap_lock); 700 spin_lock_init(&iscsit_global->ts_bitmap_lock);
706 mutex_init(&auth_id_lock); 701 mutex_init(&auth_id_lock);
707 spin_lock_init(&sess_idr_lock); 702 spin_lock_init(&sess_idr_lock);
@@ -714,10 +709,8 @@ static int __init iscsi_target_init_module(void)
714 709
715 size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long); 710 size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long);
716 iscsit_global->ts_bitmap = vzalloc(size); 711 iscsit_global->ts_bitmap = vzalloc(size);
717 if (!iscsit_global->ts_bitmap) { 712 if (!iscsit_global->ts_bitmap)
718 pr_err("Unable to allocate iscsit_global->ts_bitmap\n");
719 goto configfs_out; 713 goto configfs_out;
720 }
721 714
722 lio_qr_cache = kmem_cache_create("lio_qr_cache", 715 lio_qr_cache = kmem_cache_create("lio_qr_cache",
723 sizeof(struct iscsi_queue_req), 716 sizeof(struct iscsi_queue_req),
@@ -984,12 +977,9 @@ static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
984 u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE)); 977 u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE));
985 978
986 iov_count += ISCSI_IOV_DATA_BUFFER; 979 iov_count += ISCSI_IOV_DATA_BUFFER;
987 980 cmd->iov_data = kcalloc(iov_count, sizeof(*cmd->iov_data), GFP_KERNEL);
988 cmd->iov_data = kzalloc(iov_count * sizeof(struct kvec), GFP_KERNEL); 981 if (!cmd->iov_data)
989 if (!cmd->iov_data) {
990 pr_err("Unable to allocate cmd->iov_data\n");
991 return -ENOMEM; 982 return -ENOMEM;
992 }
993 983
994 cmd->orig_iov_data_count = iov_count; 984 cmd->orig_iov_data_count = iov_count;
995 return 0; 985 return 0;
@@ -1850,8 +1840,6 @@ static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1850 1840
1851 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1841 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1852 if (!ping_data) { 1842 if (!ping_data) {
1853 pr_err("Unable to allocate memory for"
1854 " NOPOUT ping data.\n");
1855 ret = -1; 1843 ret = -1;
1856 goto out; 1844 goto out;
1857 } 1845 }
@@ -1997,15 +1985,11 @@ iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1997 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG); 1985 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG);
1998 1986
1999 cmd->data_direction = DMA_NONE; 1987 cmd->data_direction = DMA_NONE;
2000 1988 cmd->tmr_req = kzalloc(sizeof(*cmd->tmr_req), GFP_KERNEL);
2001 cmd->tmr_req = kzalloc(sizeof(struct iscsi_tmr_req), GFP_KERNEL); 1989 if (!cmd->tmr_req)
2002 if (!cmd->tmr_req) {
2003 pr_err("Unable to allocate memory for"
2004 " Task Management command!\n");
2005 return iscsit_add_reject_cmd(cmd, 1990 return iscsit_add_reject_cmd(cmd,
2006 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1991 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
2007 buf); 1992 buf);
2008 }
2009 1993
2010 /* 1994 /*
2011 * TASK_REASSIGN for ERL=2 / connection stays inside of 1995 * TASK_REASSIGN for ERL=2 / connection stays inside of
@@ -2265,11 +2249,9 @@ iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2265 struct kvec iov[3]; 2249 struct kvec iov[3];
2266 2250
2267 text_in = kzalloc(payload_length, GFP_KERNEL); 2251 text_in = kzalloc(payload_length, GFP_KERNEL);
2268 if (!text_in) { 2252 if (!text_in)
2269 pr_err("Unable to allocate memory for"
2270 " incoming text parameters\n");
2271 goto reject; 2253 goto reject;
2272 } 2254
2273 cmd->text_in_ptr = text_in; 2255 cmd->text_in_ptr = text_in;
2274 2256
2275 memset(iov, 0, 3 * sizeof(struct kvec)); 2257 memset(iov, 0, 3 * sizeof(struct kvec));
@@ -3353,11 +3335,9 @@ iscsit_build_sendtargets_response(struct iscsi_cmd *cmd,
3353 SENDTARGETS_BUF_LIMIT); 3335 SENDTARGETS_BUF_LIMIT);
3354 3336
3355 payload = kzalloc(buffer_len, GFP_KERNEL); 3337 payload = kzalloc(buffer_len, GFP_KERNEL);
3356 if (!payload) { 3338 if (!payload)
3357 pr_err("Unable to allocate memory for sendtargets"
3358 " response.\n");
3359 return -ENOMEM; 3339 return -ENOMEM;
3360 } 3340
3361 /* 3341 /*
3362 * Locate pointer to iqn./eui. string for ICF_SENDTARGETS_SINGLE 3342 * Locate pointer to iqn./eui. string for ICF_SENDTARGETS_SINGLE
3363 * explicit case.. 3343 * explicit case..
@@ -4683,6 +4663,7 @@ int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force)
4683 continue; 4663 continue;
4684 } 4664 }
4685 atomic_set(&sess->session_reinstatement, 1); 4665 atomic_set(&sess->session_reinstatement, 1);
4666 atomic_set(&sess->session_fall_back_to_erl0, 1);
4686 spin_unlock(&sess->conn_lock); 4667 spin_unlock(&sess->conn_lock);
4687 4668
4688 list_move_tail(&se_sess->sess_list, &free_list); 4669 list_move_tail(&se_sess->sess_list, &free_list);
diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c
index 5798810197ec..535a8e06a401 100644
--- a/drivers/target/iscsi/iscsi_target_configfs.c
+++ b/drivers/target/iscsi/iscsi_target_configfs.c
@@ -1506,6 +1506,7 @@ static void lio_tpg_close_session(struct se_session *se_sess)
1506 return; 1506 return;
1507 } 1507 }
1508 atomic_set(&sess->session_reinstatement, 1); 1508 atomic_set(&sess->session_reinstatement, 1);
1509 atomic_set(&sess->session_fall_back_to_erl0, 1);
1509 spin_unlock(&sess->conn_lock); 1510 spin_unlock(&sess->conn_lock);
1510 1511
1511 iscsit_stop_time2retain_timer(sess); 1512 iscsit_stop_time2retain_timer(sess);
diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c
index ad8f3011bdc2..66238477137b 100644
--- a/drivers/target/iscsi/iscsi_target_login.c
+++ b/drivers/target/iscsi/iscsi_target_login.c
@@ -208,6 +208,7 @@ int iscsi_check_for_session_reinstatement(struct iscsi_conn *conn)
208 initiatorname_param->value) && 208 initiatorname_param->value) &&
209 (sess_p->sess_ops->SessionType == sessiontype))) { 209 (sess_p->sess_ops->SessionType == sessiontype))) {
210 atomic_set(&sess_p->session_reinstatement, 1); 210 atomic_set(&sess_p->session_reinstatement, 1);
211 atomic_set(&sess_p->session_fall_back_to_erl0, 1);
211 spin_unlock(&sess_p->conn_lock); 212 spin_unlock(&sess_p->conn_lock);
212 iscsit_inc_session_usage_count(sess_p); 213 iscsit_inc_session_usage_count(sess_p);
213 iscsit_stop_time2retain_timer(sess_p); 214 iscsit_stop_time2retain_timer(sess_p);
diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c
index 70657fd56440..0326607e5ab8 100644
--- a/drivers/target/target_core_configfs.c
+++ b/drivers/target/target_core_configfs.c
@@ -533,6 +533,7 @@ DEF_CONFIGFS_ATTRIB_SHOW(emulate_3pc);
533DEF_CONFIGFS_ATTRIB_SHOW(pi_prot_type); 533DEF_CONFIGFS_ATTRIB_SHOW(pi_prot_type);
534DEF_CONFIGFS_ATTRIB_SHOW(hw_pi_prot_type); 534DEF_CONFIGFS_ATTRIB_SHOW(hw_pi_prot_type);
535DEF_CONFIGFS_ATTRIB_SHOW(pi_prot_format); 535DEF_CONFIGFS_ATTRIB_SHOW(pi_prot_format);
536DEF_CONFIGFS_ATTRIB_SHOW(pi_prot_verify);
536DEF_CONFIGFS_ATTRIB_SHOW(enforce_pr_isids); 537DEF_CONFIGFS_ATTRIB_SHOW(enforce_pr_isids);
537DEF_CONFIGFS_ATTRIB_SHOW(is_nonrot); 538DEF_CONFIGFS_ATTRIB_SHOW(is_nonrot);
538DEF_CONFIGFS_ATTRIB_SHOW(emulate_rest_reord); 539DEF_CONFIGFS_ATTRIB_SHOW(emulate_rest_reord);
@@ -823,6 +824,7 @@ static ssize_t pi_prot_type_store(struct config_item *item,
823 ret = dev->transport->init_prot(dev); 824 ret = dev->transport->init_prot(dev);
824 if (ret) { 825 if (ret) {
825 da->pi_prot_type = old_prot; 826 da->pi_prot_type = old_prot;
827 da->pi_prot_verify = (bool) da->pi_prot_type;
826 return ret; 828 return ret;
827 } 829 }
828 830
@@ -830,6 +832,7 @@ static ssize_t pi_prot_type_store(struct config_item *item,
830 dev->transport->free_prot(dev); 832 dev->transport->free_prot(dev);
831 } 833 }
832 834
835 da->pi_prot_verify = (bool) da->pi_prot_type;
833 pr_debug("dev[%p]: SE Device Protection Type: %d\n", dev, flag); 836 pr_debug("dev[%p]: SE Device Protection Type: %d\n", dev, flag);
834 return count; 837 return count;
835} 838}
@@ -872,6 +875,35 @@ static ssize_t pi_prot_format_store(struct config_item *item,
872 return count; 875 return count;
873} 876}
874 877
878static ssize_t pi_prot_verify_store(struct config_item *item,
879 const char *page, size_t count)
880{
881 struct se_dev_attrib *da = to_attrib(item);
882 bool flag;
883 int ret;
884
885 ret = strtobool(page, &flag);
886 if (ret < 0)
887 return ret;
888
889 if (!flag) {
890 da->pi_prot_verify = flag;
891 return count;
892 }
893 if (da->hw_pi_prot_type) {
894 pr_warn("DIF protection enabled on underlying hardware,"
895 " ignoring\n");
896 return count;
897 }
898 if (!da->pi_prot_type) {
899 pr_warn("DIF protection not supported by backend, ignoring\n");
900 return count;
901 }
902 da->pi_prot_verify = flag;
903
904 return count;
905}
906
875static ssize_t force_pr_aptpl_store(struct config_item *item, 907static ssize_t force_pr_aptpl_store(struct config_item *item,
876 const char *page, size_t count) 908 const char *page, size_t count)
877{ 909{
@@ -1067,6 +1099,7 @@ CONFIGFS_ATTR(, emulate_3pc);
1067CONFIGFS_ATTR(, pi_prot_type); 1099CONFIGFS_ATTR(, pi_prot_type);
1068CONFIGFS_ATTR_RO(, hw_pi_prot_type); 1100CONFIGFS_ATTR_RO(, hw_pi_prot_type);
1069CONFIGFS_ATTR(, pi_prot_format); 1101CONFIGFS_ATTR(, pi_prot_format);
1102CONFIGFS_ATTR(, pi_prot_verify);
1070CONFIGFS_ATTR(, enforce_pr_isids); 1103CONFIGFS_ATTR(, enforce_pr_isids);
1071CONFIGFS_ATTR(, is_nonrot); 1104CONFIGFS_ATTR(, is_nonrot);
1072CONFIGFS_ATTR(, emulate_rest_reord); 1105CONFIGFS_ATTR(, emulate_rest_reord);
@@ -1104,6 +1137,7 @@ struct configfs_attribute *sbc_attrib_attrs[] = {
1104 &attr_pi_prot_type, 1137 &attr_pi_prot_type,
1105 &attr_hw_pi_prot_type, 1138 &attr_hw_pi_prot_type,
1106 &attr_pi_prot_format, 1139 &attr_pi_prot_format,
1140 &attr_pi_prot_verify,
1107 &attr_enforce_pr_isids, 1141 &attr_enforce_pr_isids,
1108 &attr_is_nonrot, 1142 &attr_is_nonrot,
1109 &attr_emulate_rest_reord, 1143 &attr_emulate_rest_reord,
@@ -1366,7 +1400,7 @@ static ssize_t target_pr_res_holder_show(struct config_item *item, char *page)
1366 struct se_device *dev = pr_to_dev(item); 1400 struct se_device *dev = pr_to_dev(item);
1367 int ret; 1401 int ret;
1368 1402
1369 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1403 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
1370 return sprintf(page, "Passthrough\n"); 1404 return sprintf(page, "Passthrough\n");
1371 1405
1372 spin_lock(&dev->dev_reservation_lock); 1406 spin_lock(&dev->dev_reservation_lock);
@@ -1506,7 +1540,7 @@ static ssize_t target_pr_res_type_show(struct config_item *item, char *page)
1506{ 1540{
1507 struct se_device *dev = pr_to_dev(item); 1541 struct se_device *dev = pr_to_dev(item);
1508 1542
1509 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1543 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
1510 return sprintf(page, "SPC_PASSTHROUGH\n"); 1544 return sprintf(page, "SPC_PASSTHROUGH\n");
1511 else if (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS) 1545 else if (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS)
1512 return sprintf(page, "SPC2_RESERVATIONS\n"); 1546 return sprintf(page, "SPC2_RESERVATIONS\n");
@@ -1519,7 +1553,7 @@ static ssize_t target_pr_res_aptpl_active_show(struct config_item *item,
1519{ 1553{
1520 struct se_device *dev = pr_to_dev(item); 1554 struct se_device *dev = pr_to_dev(item);
1521 1555
1522 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1556 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
1523 return 0; 1557 return 0;
1524 1558
1525 return sprintf(page, "APTPL Bit Status: %s\n", 1559 return sprintf(page, "APTPL Bit Status: %s\n",
@@ -1531,7 +1565,7 @@ static ssize_t target_pr_res_aptpl_metadata_show(struct config_item *item,
1531{ 1565{
1532 struct se_device *dev = pr_to_dev(item); 1566 struct se_device *dev = pr_to_dev(item);
1533 1567
1534 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1568 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
1535 return 0; 1569 return 0;
1536 1570
1537 return sprintf(page, "Ready to process PR APTPL metadata..\n"); 1571 return sprintf(page, "Ready to process PR APTPL metadata..\n");
@@ -1577,7 +1611,7 @@ static ssize_t target_pr_res_aptpl_metadata_store(struct config_item *item,
1577 u16 tpgt = 0; 1611 u16 tpgt = 0;
1578 u8 type = 0; 1612 u8 type = 0;
1579 1613
1580 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1614 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
1581 return count; 1615 return count;
1582 if (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS) 1616 if (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS)
1583 return count; 1617 return count;
@@ -2511,7 +2545,7 @@ static ssize_t target_tg_pt_gp_alua_support_##_name##_store( \
2511 int ret; \ 2545 int ret; \
2512 \ 2546 \
2513 if (!t->tg_pt_gp_valid_id) { \ 2547 if (!t->tg_pt_gp_valid_id) { \
2514 pr_err("Unable to do set ##_name ALUA state on non" \ 2548 pr_err("Unable to do set " #_name " ALUA state on non" \
2515 " valid tg_pt_gp ID: %hu\n", \ 2549 " valid tg_pt_gp ID: %hu\n", \
2516 t->tg_pt_gp_valid_id); \ 2550 t->tg_pt_gp_valid_id); \
2517 return -EINVAL; \ 2551 return -EINVAL; \
@@ -2643,13 +2677,13 @@ static ssize_t target_tg_pt_gp_tg_pt_gp_id_store(struct config_item *item,
2643 2677
2644 ret = kstrtoul(page, 0, &tg_pt_gp_id); 2678 ret = kstrtoul(page, 0, &tg_pt_gp_id);
2645 if (ret < 0) { 2679 if (ret < 0) {
2646 pr_err("kstrtoul() returned %d for" 2680 pr_err("ALUA tg_pt_gp_id: invalid value '%s' for tg_pt_gp_id\n",
2647 " tg_pt_gp_id\n", ret); 2681 page);
2648 return ret; 2682 return ret;
2649 } 2683 }
2650 if (tg_pt_gp_id > 0x0000ffff) { 2684 if (tg_pt_gp_id > 0x0000ffff) {
2651 pr_err("ALUA tg_pt_gp_id: %lu exceeds maximum:" 2685 pr_err("ALUA tg_pt_gp_id: %lu exceeds maximum: 0x0000ffff\n",
2652 " 0x0000ffff\n", tg_pt_gp_id); 2686 tg_pt_gp_id);
2653 return -EINVAL; 2687 return -EINVAL;
2654 } 2688 }
2655 2689
diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index d2f089cfa9ae..8add07f387f9 100644
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -1045,6 +1045,8 @@ passthrough_parse_cdb(struct se_cmd *cmd,
1045 sense_reason_t (*exec_cmd)(struct se_cmd *cmd)) 1045 sense_reason_t (*exec_cmd)(struct se_cmd *cmd))
1046{ 1046{
1047 unsigned char *cdb = cmd->t_task_cdb; 1047 unsigned char *cdb = cmd->t_task_cdb;
1048 struct se_device *dev = cmd->se_dev;
1049 unsigned int size;
1048 1050
1049 /* 1051 /*
1050 * Clear a lun set in the cdb if the initiator talking to use spoke 1052 * Clear a lun set in the cdb if the initiator talking to use spoke
@@ -1076,6 +1078,42 @@ passthrough_parse_cdb(struct se_cmd *cmd,
1076 return TCM_NO_SENSE; 1078 return TCM_NO_SENSE;
1077 } 1079 }
1078 1080
1081 /*
1082 * For PERSISTENT RESERVE IN/OUT, RELEASE, and RESERVE we need to
1083 * emulate the response, since tcmu does not have the information
1084 * required to process these commands.
1085 */
1086 if (!(dev->transport->transport_flags &
1087 TRANSPORT_FLAG_PASSTHROUGH_PGR)) {
1088 if (cdb[0] == PERSISTENT_RESERVE_IN) {
1089 cmd->execute_cmd = target_scsi3_emulate_pr_in;
1090 size = (cdb[7] << 8) + cdb[8];
1091 return target_cmd_size_check(cmd, size);
1092 }
1093 if (cdb[0] == PERSISTENT_RESERVE_OUT) {
1094 cmd->execute_cmd = target_scsi3_emulate_pr_out;
1095 size = (cdb[7] << 8) + cdb[8];
1096 return target_cmd_size_check(cmd, size);
1097 }
1098
1099 if (cdb[0] == RELEASE || cdb[0] == RELEASE_10) {
1100 cmd->execute_cmd = target_scsi2_reservation_release;
1101 if (cdb[0] == RELEASE_10)
1102 size = (cdb[7] << 8) | cdb[8];
1103 else
1104 size = cmd->data_length;
1105 return target_cmd_size_check(cmd, size);
1106 }
1107 if (cdb[0] == RESERVE || cdb[0] == RESERVE_10) {
1108 cmd->execute_cmd = target_scsi2_reservation_reserve;
1109 if (cdb[0] == RESERVE_10)
1110 size = (cdb[7] << 8) | cdb[8];
1111 else
1112 size = cmd->data_length;
1113 return target_cmd_size_check(cmd, size);
1114 }
1115 }
1116
1079 /* Set DATA_CDB flag for ops that should have it */ 1117 /* Set DATA_CDB flag for ops that should have it */
1080 switch (cdb[0]) { 1118 switch (cdb[0]) {
1081 case READ_6: 1119 case READ_6:
diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c
index 87aa376a1a1a..73b8f93a5fef 100644
--- a/drivers/target/target_core_file.c
+++ b/drivers/target/target_core_file.c
@@ -277,12 +277,11 @@ static int fd_do_rw(struct se_cmd *cmd, struct file *fd,
277 else 277 else
278 ret = vfs_iter_read(fd, &iter, &pos); 278 ret = vfs_iter_read(fd, &iter, &pos);
279 279
280 kfree(bvec);
281
282 if (is_write) { 280 if (is_write) {
283 if (ret < 0 || ret != data_length) { 281 if (ret < 0 || ret != data_length) {
284 pr_err("%s() write returned %d\n", __func__, ret); 282 pr_err("%s() write returned %d\n", __func__, ret);
285 return (ret < 0 ? ret : -EINVAL); 283 if (ret >= 0)
284 ret = -EINVAL;
286 } 285 }
287 } else { 286 } else {
288 /* 287 /*
@@ -295,17 +294,29 @@ static int fd_do_rw(struct se_cmd *cmd, struct file *fd,
295 pr_err("%s() returned %d, expecting %u for " 294 pr_err("%s() returned %d, expecting %u for "
296 "S_ISBLK\n", __func__, ret, 295 "S_ISBLK\n", __func__, ret,
297 data_length); 296 data_length);
298 return (ret < 0 ? ret : -EINVAL); 297 if (ret >= 0)
298 ret = -EINVAL;
299 } 299 }
300 } else { 300 } else {
301 if (ret < 0) { 301 if (ret < 0) {
302 pr_err("%s() returned %d for non S_ISBLK\n", 302 pr_err("%s() returned %d for non S_ISBLK\n",
303 __func__, ret); 303 __func__, ret);
304 return ret; 304 } else if (ret != data_length) {
305 /*
306 * Short read case:
307 * Probably some one truncate file under us.
308 * We must explicitly zero sg-pages to prevent
309 * expose uninizialized pages to userspace.
310 */
311 if (ret < data_length)
312 ret += iov_iter_zero(data_length - ret, &iter);
313 else
314 ret = -EINVAL;
305 } 315 }
306 } 316 }
307 } 317 }
308 return 1; 318 kfree(bvec);
319 return ret;
309} 320}
310 321
311static sense_reason_t 322static sense_reason_t
@@ -543,7 +554,8 @@ fd_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
543 ret = fd_do_rw(cmd, file, dev->dev_attrib.block_size, 554 ret = fd_do_rw(cmd, file, dev->dev_attrib.block_size,
544 sgl, sgl_nents, cmd->data_length, 0); 555 sgl, sgl_nents, cmd->data_length, 0);
545 556
546 if (ret > 0 && cmd->prot_type && dev->dev_attrib.pi_prot_type) { 557 if (ret > 0 && cmd->prot_type && dev->dev_attrib.pi_prot_type &&
558 dev->dev_attrib.pi_prot_verify) {
547 u32 sectors = cmd->data_length >> 559 u32 sectors = cmd->data_length >>
548 ilog2(dev->dev_attrib.block_size); 560 ilog2(dev->dev_attrib.block_size);
549 561
@@ -553,7 +565,8 @@ fd_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
553 return rc; 565 return rc;
554 } 566 }
555 } else { 567 } else {
556 if (cmd->prot_type && dev->dev_attrib.pi_prot_type) { 568 if (cmd->prot_type && dev->dev_attrib.pi_prot_type &&
569 dev->dev_attrib.pi_prot_verify) {
557 u32 sectors = cmd->data_length >> 570 u32 sectors = cmd->data_length >>
558 ilog2(dev->dev_attrib.block_size); 571 ilog2(dev->dev_attrib.block_size);
559 572
@@ -595,8 +608,7 @@ fd_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
595 if (ret < 0) 608 if (ret < 0)
596 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 609 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
597 610
598 if (ret) 611 target_complete_cmd(cmd, SAM_STAT_GOOD);
599 target_complete_cmd(cmd, SAM_STAT_GOOD);
600 return 0; 612 return 0;
601} 613}
602 614
diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c
index d316ed537d59..bb069ebe4aa6 100644
--- a/drivers/target/target_core_iblock.c
+++ b/drivers/target/target_core_iblock.c
@@ -279,7 +279,7 @@ static void iblock_complete_cmd(struct se_cmd *cmd)
279 struct iblock_req *ibr = cmd->priv; 279 struct iblock_req *ibr = cmd->priv;
280 u8 status; 280 u8 status;
281 281
282 if (!atomic_dec_and_test(&ibr->pending)) 282 if (!refcount_dec_and_test(&ibr->pending))
283 return; 283 return;
284 284
285 if (atomic_read(&ibr->ib_bio_err_cnt)) 285 if (atomic_read(&ibr->ib_bio_err_cnt))
@@ -487,7 +487,7 @@ iblock_execute_write_same(struct se_cmd *cmd)
487 bio_list_init(&list); 487 bio_list_init(&list);
488 bio_list_add(&list, bio); 488 bio_list_add(&list, bio);
489 489
490 atomic_set(&ibr->pending, 1); 490 refcount_set(&ibr->pending, 1);
491 491
492 while (sectors) { 492 while (sectors) {
493 while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset) 493 while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
@@ -498,7 +498,7 @@ iblock_execute_write_same(struct se_cmd *cmd)
498 if (!bio) 498 if (!bio)
499 goto fail_put_bios; 499 goto fail_put_bios;
500 500
501 atomic_inc(&ibr->pending); 501 refcount_inc(&ibr->pending);
502 bio_list_add(&list, bio); 502 bio_list_add(&list, bio);
503 } 503 }
504 504
@@ -706,7 +706,7 @@ iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
706 cmd->priv = ibr; 706 cmd->priv = ibr;
707 707
708 if (!sgl_nents) { 708 if (!sgl_nents) {
709 atomic_set(&ibr->pending, 1); 709 refcount_set(&ibr->pending, 1);
710 iblock_complete_cmd(cmd); 710 iblock_complete_cmd(cmd);
711 return 0; 711 return 0;
712 } 712 }
@@ -719,7 +719,7 @@ iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
719 bio_list_init(&list); 719 bio_list_init(&list);
720 bio_list_add(&list, bio); 720 bio_list_add(&list, bio);
721 721
722 atomic_set(&ibr->pending, 2); 722 refcount_set(&ibr->pending, 2);
723 bio_cnt = 1; 723 bio_cnt = 1;
724 724
725 for_each_sg(sgl, sg, sgl_nents, i) { 725 for_each_sg(sgl, sg, sgl_nents, i) {
@@ -740,7 +740,7 @@ iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
740 if (!bio) 740 if (!bio)
741 goto fail_put_bios; 741 goto fail_put_bios;
742 742
743 atomic_inc(&ibr->pending); 743 refcount_inc(&ibr->pending);
744 bio_list_add(&list, bio); 744 bio_list_add(&list, bio);
745 bio_cnt++; 745 bio_cnt++;
746 } 746 }
diff --git a/drivers/target/target_core_iblock.h b/drivers/target/target_core_iblock.h
index 718d3fcd3e7c..f2a5797217d4 100644
--- a/drivers/target/target_core_iblock.h
+++ b/drivers/target/target_core_iblock.h
@@ -2,6 +2,7 @@
2#define TARGET_CORE_IBLOCK_H 2#define TARGET_CORE_IBLOCK_H
3 3
4#include <linux/atomic.h> 4#include <linux/atomic.h>
5#include <linux/refcount.h>
5#include <target/target_core_base.h> 6#include <target/target_core_base.h>
6 7
7#define IBLOCK_VERSION "4.0" 8#define IBLOCK_VERSION "4.0"
@@ -10,7 +11,7 @@
10#define IBLOCK_LBA_SHIFT 9 11#define IBLOCK_LBA_SHIFT 9
11 12
12struct iblock_req { 13struct iblock_req {
13 atomic_t pending; 14 refcount_t pending;
14 atomic_t ib_bio_err_cnt; 15 atomic_t ib_bio_err_cnt;
15} ____cacheline_aligned; 16} ____cacheline_aligned;
16 17
diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c
index e18051185846..129ca572673c 100644
--- a/drivers/target/target_core_pr.c
+++ b/drivers/target/target_core_pr.c
@@ -4147,7 +4147,7 @@ target_check_reservation(struct se_cmd *cmd)
4147 return 0; 4147 return 0;
4148 if (dev->se_hba->hba_flags & HBA_FLAGS_INTERNAL_USE) 4148 if (dev->se_hba->hba_flags & HBA_FLAGS_INTERNAL_USE)
4149 return 0; 4149 return 0;
4150 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 4150 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
4151 return 0; 4151 return 0;
4152 4152
4153 spin_lock(&dev->dev_reservation_lock); 4153 spin_lock(&dev->dev_reservation_lock);
diff --git a/drivers/target/target_core_pr.h b/drivers/target/target_core_pr.h
index 847bd470339c..772f9148e75c 100644
--- a/drivers/target/target_core_pr.h
+++ b/drivers/target/target_core_pr.h
@@ -7,7 +7,7 @@
7/* 7/*
8 * PERSISTENT_RESERVE_OUT service action codes 8 * PERSISTENT_RESERVE_OUT service action codes
9 * 9 *
10 * spc4r17 section 6.14.2 Table 171 10 * spc5r04b section 6.15.2 Table 174
11 */ 11 */
12#define PRO_REGISTER 0x00 12#define PRO_REGISTER 0x00
13#define PRO_RESERVE 0x01 13#define PRO_RESERVE 0x01
@@ -17,10 +17,11 @@
17#define PRO_PREEMPT_AND_ABORT 0x05 17#define PRO_PREEMPT_AND_ABORT 0x05
18#define PRO_REGISTER_AND_IGNORE_EXISTING_KEY 0x06 18#define PRO_REGISTER_AND_IGNORE_EXISTING_KEY 0x06
19#define PRO_REGISTER_AND_MOVE 0x07 19#define PRO_REGISTER_AND_MOVE 0x07
20#define PRO_REPLACE_LOST_RESERVATION 0x08
20/* 21/*
21 * PERSISTENT_RESERVE_IN service action codes 22 * PERSISTENT_RESERVE_IN service action codes
22 * 23 *
23 * spc4r17 section 6.13.1 Table 159 24 * spc5r04b section 6.14.1 Table 162
24 */ 25 */
25#define PRI_READ_KEYS 0x00 26#define PRI_READ_KEYS 0x00
26#define PRI_READ_RESERVATION 0x01 27#define PRI_READ_RESERVATION 0x01
@@ -29,13 +30,13 @@
29/* 30/*
30 * PERSISTENT_RESERVE_ SCOPE field 31 * PERSISTENT_RESERVE_ SCOPE field
31 * 32 *
32 * spc4r17 section 6.13.3.3 Table 163 33 * spc5r04b section 6.14.3.2 Table 166
33 */ 34 */
34#define PR_SCOPE_LU_SCOPE 0x00 35#define PR_SCOPE_LU_SCOPE 0x00
35/* 36/*
36 * PERSISTENT_RESERVE_* TYPE field 37 * PERSISTENT_RESERVE_* TYPE field
37 * 38 *
38 * spc4r17 section 6.13.3.4 Table 164 39 * spc5r04b section 6.14.3.3 Table 167
39 */ 40 */
40#define PR_TYPE_WRITE_EXCLUSIVE 0x01 41#define PR_TYPE_WRITE_EXCLUSIVE 0x01
41#define PR_TYPE_EXCLUSIVE_ACCESS 0x03 42#define PR_TYPE_EXCLUSIVE_ACCESS 0x03
diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c
index a93d94e68ab5..3e4abb13f8ea 100644
--- a/drivers/target/target_core_pscsi.c
+++ b/drivers/target/target_core_pscsi.c
@@ -1081,7 +1081,8 @@ static const struct target_backend_ops pscsi_ops = {
1081 .name = "pscsi", 1081 .name = "pscsi",
1082 .owner = THIS_MODULE, 1082 .owner = THIS_MODULE,
1083 .transport_flags = TRANSPORT_FLAG_PASSTHROUGH | 1083 .transport_flags = TRANSPORT_FLAG_PASSTHROUGH |
1084 TRANSPORT_FLAG_PASSTHROUGH_ALUA, 1084 TRANSPORT_FLAG_PASSTHROUGH_ALUA |
1085 TRANSPORT_FLAG_PASSTHROUGH_PGR,
1085 .attach_hba = pscsi_attach_hba, 1086 .attach_hba = pscsi_attach_hba,
1086 .detach_hba = pscsi_detach_hba, 1087 .detach_hba = pscsi_detach_hba,
1087 .pmode_enable_hba = pscsi_pmode_enable_hba, 1088 .pmode_enable_hba = pscsi_pmode_enable_hba,
diff --git a/drivers/target/target_core_rd.c b/drivers/target/target_core_rd.c
index ddc216c9f1f6..20253d04103f 100644
--- a/drivers/target/target_core_rd.c
+++ b/drivers/target/target_core_rd.c
@@ -47,11 +47,9 @@ static int rd_attach_hba(struct se_hba *hba, u32 host_id)
47{ 47{
48 struct rd_host *rd_host; 48 struct rd_host *rd_host;
49 49
50 rd_host = kzalloc(sizeof(struct rd_host), GFP_KERNEL); 50 rd_host = kzalloc(sizeof(*rd_host), GFP_KERNEL);
51 if (!rd_host) { 51 if (!rd_host)
52 pr_err("Unable to allocate memory for struct rd_host\n");
53 return -ENOMEM; 52 return -ENOMEM;
54 }
55 53
56 rd_host->rd_host_id = host_id; 54 rd_host->rd_host_id = host_id;
57 55
@@ -148,11 +146,8 @@ static int rd_allocate_sgl_table(struct rd_dev *rd_dev, struct rd_dev_sg_table *
148 146
149 sg = kcalloc(sg_per_table + chain_entry, sizeof(*sg), 147 sg = kcalloc(sg_per_table + chain_entry, sizeof(*sg),
150 GFP_KERNEL); 148 GFP_KERNEL);
151 if (!sg) { 149 if (!sg)
152 pr_err("Unable to allocate scatterlist array"
153 " for struct rd_dev\n");
154 return -ENOMEM; 150 return -ENOMEM;
155 }
156 151
157 sg_init_table(sg, sg_per_table + chain_entry); 152 sg_init_table(sg, sg_per_table + chain_entry);
158 153
@@ -210,13 +205,9 @@ static int rd_build_device_space(struct rd_dev *rd_dev)
210 total_sg_needed = rd_dev->rd_page_count; 205 total_sg_needed = rd_dev->rd_page_count;
211 206
212 sg_tables = (total_sg_needed / max_sg_per_table) + 1; 207 sg_tables = (total_sg_needed / max_sg_per_table) + 1;
213 208 sg_table = kcalloc(sg_tables, sizeof(*sg_table), GFP_KERNEL);
214 sg_table = kzalloc(sg_tables * sizeof(struct rd_dev_sg_table), GFP_KERNEL); 209 if (!sg_table)
215 if (!sg_table) {
216 pr_err("Unable to allocate memory for Ramdisk"
217 " scatterlist tables\n");
218 return -ENOMEM; 210 return -ENOMEM;
219 }
220 211
221 rd_dev->sg_table_array = sg_table; 212 rd_dev->sg_table_array = sg_table;
222 rd_dev->sg_table_count = sg_tables; 213 rd_dev->sg_table_count = sg_tables;
@@ -271,13 +262,9 @@ static int rd_build_prot_space(struct rd_dev *rd_dev, int prot_length, int block
271 total_sg_needed = (rd_dev->rd_page_count * prot_length / block_size) + 1; 262 total_sg_needed = (rd_dev->rd_page_count * prot_length / block_size) + 1;
272 263
273 sg_tables = (total_sg_needed / max_sg_per_table) + 1; 264 sg_tables = (total_sg_needed / max_sg_per_table) + 1;
274 265 sg_table = kcalloc(sg_tables, sizeof(*sg_table), GFP_KERNEL);
275 sg_table = kzalloc(sg_tables * sizeof(struct rd_dev_sg_table), GFP_KERNEL); 266 if (!sg_table)
276 if (!sg_table) {
277 pr_err("Unable to allocate memory for Ramdisk protection"
278 " scatterlist tables\n");
279 return -ENOMEM; 267 return -ENOMEM;
280 }
281 268
282 rd_dev->sg_prot_array = sg_table; 269 rd_dev->sg_prot_array = sg_table;
283 rd_dev->sg_prot_count = sg_tables; 270 rd_dev->sg_prot_count = sg_tables;
@@ -298,11 +285,9 @@ static struct se_device *rd_alloc_device(struct se_hba *hba, const char *name)
298 struct rd_dev *rd_dev; 285 struct rd_dev *rd_dev;
299 struct rd_host *rd_host = hba->hba_ptr; 286 struct rd_host *rd_host = hba->hba_ptr;
300 287
301 rd_dev = kzalloc(sizeof(struct rd_dev), GFP_KERNEL); 288 rd_dev = kzalloc(sizeof(*rd_dev), GFP_KERNEL);
302 if (!rd_dev) { 289 if (!rd_dev)
303 pr_err("Unable to allocate memory for struct rd_dev\n");
304 return NULL; 290 return NULL;
305 }
306 291
307 rd_dev->rd_host = rd_host; 292 rd_dev->rd_host = rd_host;
308 293
@@ -410,7 +395,7 @@ static sense_reason_t rd_do_prot_rw(struct se_cmd *cmd, bool is_read)
410 u32 prot_offset, prot_page; 395 u32 prot_offset, prot_page;
411 u32 prot_npages __maybe_unused; 396 u32 prot_npages __maybe_unused;
412 u64 tmp; 397 u64 tmp;
413 sense_reason_t rc = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 398 sense_reason_t rc = 0;
414 399
415 tmp = cmd->t_task_lba * se_dev->prot_length; 400 tmp = cmd->t_task_lba * se_dev->prot_length;
416 prot_offset = do_div(tmp, PAGE_SIZE); 401 prot_offset = do_div(tmp, PAGE_SIZE);
@@ -423,13 +408,14 @@ static sense_reason_t rd_do_prot_rw(struct se_cmd *cmd, bool is_read)
423 prot_sg = &prot_table->sg_table[prot_page - 408 prot_sg = &prot_table->sg_table[prot_page -
424 prot_table->page_start_offset]; 409 prot_table->page_start_offset];
425 410
426 if (is_read) 411 if (se_dev->dev_attrib.pi_prot_verify) {
427 rc = sbc_dif_verify(cmd, cmd->t_task_lba, sectors, 0, 412 if (is_read)
428 prot_sg, prot_offset); 413 rc = sbc_dif_verify(cmd, cmd->t_task_lba, sectors, 0,
429 else 414 prot_sg, prot_offset);
430 rc = sbc_dif_verify(cmd, cmd->t_task_lba, sectors, 0, 415 else
431 cmd->t_prot_sg, 0); 416 rc = sbc_dif_verify(cmd, cmd->t_task_lba, sectors, 0,
432 417 cmd->t_prot_sg, 0);
418 }
433 if (!rc) 419 if (!rc)
434 sbc_dif_copy_prot(cmd, sectors, is_read, prot_sg, prot_offset); 420 sbc_dif_copy_prot(cmd, sectors, is_read, prot_sg, prot_offset);
435 421
diff --git a/drivers/target/target_core_sbc.c b/drivers/target/target_core_sbc.c
index c194063f169b..4316f7b65fb7 100644
--- a/drivers/target/target_core_sbc.c
+++ b/drivers/target/target_core_sbc.c
@@ -507,8 +507,11 @@ static sense_reason_t compare_and_write_callback(struct se_cmd *cmd, bool succes
507 * been failed with a non-zero SCSI status. 507 * been failed with a non-zero SCSI status.
508 */ 508 */
509 if (cmd->scsi_status) { 509 if (cmd->scsi_status) {
510 pr_err("compare_and_write_callback: non zero scsi_status:" 510 pr_debug("compare_and_write_callback: non zero scsi_status:"
511 " 0x%02x\n", cmd->scsi_status); 511 " 0x%02x\n", cmd->scsi_status);
512 *post_ret = 1;
513 if (cmd->scsi_status == SAM_STAT_CHECK_CONDITION)
514 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
512 goto out; 515 goto out;
513 } 516 }
514 517
@@ -519,8 +522,8 @@ static sense_reason_t compare_and_write_callback(struct se_cmd *cmd, bool succes
519 goto out; 522 goto out;
520 } 523 }
521 524
522 write_sg = kmalloc(sizeof(struct scatterlist) * cmd->t_data_nents, 525 write_sg = kmalloc_array(cmd->t_data_nents, sizeof(*write_sg),
523 GFP_KERNEL); 526 GFP_KERNEL);
524 if (!write_sg) { 527 if (!write_sg) {
525 pr_err("Unable to allocate compare_and_write sg\n"); 528 pr_err("Unable to allocate compare_and_write sg\n");
526 ret = TCM_OUT_OF_RESOURCES; 529 ret = TCM_OUT_OF_RESOURCES;
@@ -924,6 +927,7 @@ sbc_parse_cdb(struct se_cmd *cmd, struct sbc_ops *ops)
924 cmd->execute_cmd = sbc_execute_rw; 927 cmd->execute_cmd = sbc_execute_rw;
925 break; 928 break;
926 case WRITE_16: 929 case WRITE_16:
930 case WRITE_VERIFY_16:
927 sectors = transport_get_sectors_16(cdb); 931 sectors = transport_get_sectors_16(cdb);
928 cmd->t_task_lba = transport_lba_64(cdb); 932 cmd->t_task_lba = transport_lba_64(cdb);
929 933
diff --git a/drivers/target/target_core_tpg.c b/drivers/target/target_core_tpg.c
index dfaef4d3b2d2..310d9e55c6eb 100644
--- a/drivers/target/target_core_tpg.c
+++ b/drivers/target/target_core_tpg.c
@@ -398,6 +398,13 @@ int core_tpg_set_initiator_node_queue_depth(
398 struct se_portal_group *tpg = acl->se_tpg; 398 struct se_portal_group *tpg = acl->se_tpg;
399 399
400 /* 400 /*
401 * Allow the setting of se_node_acl queue_depth to be idempotent,
402 * and not force a session shutdown event if the value is not
403 * changing.
404 */
405 if (acl->queue_depth == queue_depth)
406 return 0;
407 /*
401 * User has requested to change the queue depth for a Initiator Node. 408 * User has requested to change the queue depth for a Initiator Node.
402 * Change the value in the Node's struct se_node_acl, and call 409 * Change the value in the Node's struct se_node_acl, and call
403 * target_set_nacl_queue_depth() to set the new queue depth. 410 * target_set_nacl_queue_depth() to set the new queue depth.
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index a0cd56ee5fe9..37f57357d4a0 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -2311,7 +2311,7 @@ void *transport_kmap_data_sg(struct se_cmd *cmd)
2311 return kmap(sg_page(sg)) + sg->offset; 2311 return kmap(sg_page(sg)) + sg->offset;
2312 2312
2313 /* >1 page. use vmap */ 2313 /* >1 page. use vmap */
2314 pages = kmalloc(sizeof(*pages) * cmd->t_data_nents, GFP_KERNEL); 2314 pages = kmalloc_array(cmd->t_data_nents, sizeof(*pages), GFP_KERNEL);
2315 if (!pages) 2315 if (!pages)
2316 return NULL; 2316 return NULL;
2317 2317
diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c
index f615c3bbb73e..9045837f748b 100644
--- a/drivers/target/target_core_user.c
+++ b/drivers/target/target_core_user.c
@@ -2,6 +2,7 @@
2 * Copyright (C) 2013 Shaohua Li <shli@kernel.org> 2 * Copyright (C) 2013 Shaohua Li <shli@kernel.org>
3 * Copyright (C) 2014 Red Hat, Inc. 3 * Copyright (C) 2014 Red Hat, Inc.
4 * Copyright (C) 2015 Arrikto, Inc. 4 * Copyright (C) 2015 Arrikto, Inc.
5 * Copyright (C) 2017 Chinamobile, Inc.
5 * 6 *
6 * This program is free software; you can redistribute it and/or modify it 7 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License, 8 * under the terms and conditions of the GNU General Public License,
@@ -25,10 +26,13 @@
25#include <linux/parser.h> 26#include <linux/parser.h>
26#include <linux/vmalloc.h> 27#include <linux/vmalloc.h>
27#include <linux/uio_driver.h> 28#include <linux/uio_driver.h>
29#include <linux/radix-tree.h>
28#include <linux/stringify.h> 30#include <linux/stringify.h>
29#include <linux/bitops.h> 31#include <linux/bitops.h>
30#include <linux/highmem.h> 32#include <linux/highmem.h>
31#include <linux/configfs.h> 33#include <linux/configfs.h>
34#include <linux/mutex.h>
35#include <linux/kthread.h>
32#include <net/genetlink.h> 36#include <net/genetlink.h>
33#include <scsi/scsi_common.h> 37#include <scsi/scsi_common.h>
34#include <scsi/scsi_proto.h> 38#include <scsi/scsi_proto.h>
@@ -63,17 +67,26 @@
63 * this may have a 'UAM' comment. 67 * this may have a 'UAM' comment.
64 */ 68 */
65 69
66
67#define TCMU_TIME_OUT (30 * MSEC_PER_SEC) 70#define TCMU_TIME_OUT (30 * MSEC_PER_SEC)
68 71
69#define DATA_BLOCK_BITS 256 72/* For cmd area, the size is fixed 8MB */
70#define DATA_BLOCK_SIZE 4096 73#define CMDR_SIZE (8 * 1024 * 1024)
71 74
72#define CMDR_SIZE (16 * 4096) 75/*
76 * For data area, the block size is PAGE_SIZE and
77 * the total size is 256K * PAGE_SIZE.
78 */
79#define DATA_BLOCK_SIZE PAGE_SIZE
80#define DATA_BLOCK_BITS (256 * 1024)
73#define DATA_SIZE (DATA_BLOCK_BITS * DATA_BLOCK_SIZE) 81#define DATA_SIZE (DATA_BLOCK_BITS * DATA_BLOCK_SIZE)
82#define DATA_BLOCK_INIT_BITS 128
74 83
84/* The total size of the ring is 8M + 256K * PAGE_SIZE */
75#define TCMU_RING_SIZE (CMDR_SIZE + DATA_SIZE) 85#define TCMU_RING_SIZE (CMDR_SIZE + DATA_SIZE)
76 86
87/* Default maximum of the global data blocks(512K * PAGE_SIZE) */
88#define TCMU_GLOBAL_MAX_BLOCKS (512 * 1024)
89
77static struct device *tcmu_root_device; 90static struct device *tcmu_root_device;
78 91
79struct tcmu_hba { 92struct tcmu_hba {
@@ -83,6 +96,8 @@ struct tcmu_hba {
83#define TCMU_CONFIG_LEN 256 96#define TCMU_CONFIG_LEN 256
84 97
85struct tcmu_dev { 98struct tcmu_dev {
99 struct list_head node;
100
86 struct se_device se_dev; 101 struct se_device se_dev;
87 102
88 char *name; 103 char *name;
@@ -94,6 +109,8 @@ struct tcmu_dev {
94 109
95 struct uio_info uio_info; 110 struct uio_info uio_info;
96 111
112 struct inode *inode;
113
97 struct tcmu_mailbox *mb_addr; 114 struct tcmu_mailbox *mb_addr;
98 size_t dev_size; 115 size_t dev_size;
99 u32 cmdr_size; 116 u32 cmdr_size;
@@ -103,11 +120,14 @@ struct tcmu_dev {
103 size_t data_off; 120 size_t data_off;
104 size_t data_size; 121 size_t data_size;
105 122
106 DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS);
107
108 wait_queue_head_t wait_cmdr; 123 wait_queue_head_t wait_cmdr;
109 /* TODO should this be a mutex? */ 124 struct mutex cmdr_lock;
110 spinlock_t cmdr_lock; 125
126 bool waiting_global;
127 uint32_t dbi_max;
128 uint32_t dbi_thresh;
129 DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS);
130 struct radix_tree_root data_blocks;
111 131
112 struct idr commands; 132 struct idr commands;
113 spinlock_t commands_lock; 133 spinlock_t commands_lock;
@@ -130,7 +150,9 @@ struct tcmu_cmd {
130 150
131 /* Can't use se_cmd when cleaning up expired cmds, because if 151 /* Can't use se_cmd when cleaning up expired cmds, because if
132 cmd has been completed then accessing se_cmd is off limits */ 152 cmd has been completed then accessing se_cmd is off limits */
133 DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS); 153 uint32_t dbi_cnt;
154 uint32_t dbi_cur;
155 uint32_t *dbi;
134 156
135 unsigned long deadline; 157 unsigned long deadline;
136 158
@@ -138,6 +160,13 @@ struct tcmu_cmd {
138 unsigned long flags; 160 unsigned long flags;
139}; 161};
140 162
163static struct task_struct *unmap_thread;
164static wait_queue_head_t unmap_wait;
165static DEFINE_MUTEX(root_udev_mutex);
166static LIST_HEAD(root_udev);
167
168static atomic_t global_db_count = ATOMIC_INIT(0);
169
141static struct kmem_cache *tcmu_cmd_cache; 170static struct kmem_cache *tcmu_cmd_cache;
142 171
143/* multicast group */ 172/* multicast group */
@@ -161,6 +190,114 @@ static struct genl_family tcmu_genl_family __ro_after_init = {
161 .netnsok = true, 190 .netnsok = true,
162}; 191};
163 192
193#define tcmu_cmd_set_dbi_cur(cmd, index) ((cmd)->dbi_cur = (index))
194#define tcmu_cmd_reset_dbi_cur(cmd) tcmu_cmd_set_dbi_cur(cmd, 0)
195#define tcmu_cmd_set_dbi(cmd, index) ((cmd)->dbi[(cmd)->dbi_cur++] = (index))
196#define tcmu_cmd_get_dbi(cmd) ((cmd)->dbi[(cmd)->dbi_cur++])
197
198static void tcmu_cmd_free_data(struct tcmu_cmd *tcmu_cmd, uint32_t len)
199{
200 struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
201 uint32_t i;
202
203 for (i = 0; i < len; i++)
204 clear_bit(tcmu_cmd->dbi[i], udev->data_bitmap);
205}
206
207static inline bool tcmu_get_empty_block(struct tcmu_dev *udev,
208 struct tcmu_cmd *tcmu_cmd)
209{
210 struct page *page;
211 int ret, dbi;
212
213 dbi = find_first_zero_bit(udev->data_bitmap, udev->dbi_thresh);
214 if (dbi == udev->dbi_thresh)
215 return false;
216
217 page = radix_tree_lookup(&udev->data_blocks, dbi);
218 if (!page) {
219
220 if (atomic_add_return(1, &global_db_count) >
221 TCMU_GLOBAL_MAX_BLOCKS) {
222 atomic_dec(&global_db_count);
223 return false;
224 }
225
226 /* try to get new page from the mm */
227 page = alloc_page(GFP_KERNEL);
228 if (!page)
229 return false;
230
231 ret = radix_tree_insert(&udev->data_blocks, dbi, page);
232 if (ret) {
233 __free_page(page);
234 return false;
235 }
236
237 }
238
239 if (dbi > udev->dbi_max)
240 udev->dbi_max = dbi;
241
242 set_bit(dbi, udev->data_bitmap);
243 tcmu_cmd_set_dbi(tcmu_cmd, dbi);
244
245 return true;
246}
247
248static bool tcmu_get_empty_blocks(struct tcmu_dev *udev,
249 struct tcmu_cmd *tcmu_cmd)
250{
251 int i;
252
253 udev->waiting_global = false;
254
255 for (i = tcmu_cmd->dbi_cur; i < tcmu_cmd->dbi_cnt; i++) {
256 if (!tcmu_get_empty_block(udev, tcmu_cmd))
257 goto err;
258 }
259 return true;
260
261err:
262 udev->waiting_global = true;
263 /* Try to wake up the unmap thread */
264 wake_up(&unmap_wait);
265 return false;
266}
267
268static inline struct page *
269tcmu_get_block_page(struct tcmu_dev *udev, uint32_t dbi)
270{
271 return radix_tree_lookup(&udev->data_blocks, dbi);
272}
273
274static inline void tcmu_free_cmd(struct tcmu_cmd *tcmu_cmd)
275{
276 kfree(tcmu_cmd->dbi);
277 kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
278}
279
280static inline size_t tcmu_cmd_get_data_length(struct tcmu_cmd *tcmu_cmd)
281{
282 struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
283 size_t data_length = round_up(se_cmd->data_length, DATA_BLOCK_SIZE);
284
285 if (se_cmd->se_cmd_flags & SCF_BIDI) {
286 BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents));
287 data_length += round_up(se_cmd->t_bidi_data_sg->length,
288 DATA_BLOCK_SIZE);
289 }
290
291 return data_length;
292}
293
294static inline uint32_t tcmu_cmd_get_block_cnt(struct tcmu_cmd *tcmu_cmd)
295{
296 size_t data_length = tcmu_cmd_get_data_length(tcmu_cmd);
297
298 return data_length / DATA_BLOCK_SIZE;
299}
300
164static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd) 301static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)
165{ 302{
166 struct se_device *se_dev = se_cmd->se_dev; 303 struct se_device *se_dev = se_cmd->se_dev;
@@ -178,6 +315,15 @@ static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)
178 tcmu_cmd->deadline = jiffies + 315 tcmu_cmd->deadline = jiffies +
179 msecs_to_jiffies(udev->cmd_time_out); 316 msecs_to_jiffies(udev->cmd_time_out);
180 317
318 tcmu_cmd_reset_dbi_cur(tcmu_cmd);
319 tcmu_cmd->dbi_cnt = tcmu_cmd_get_block_cnt(tcmu_cmd);
320 tcmu_cmd->dbi = kcalloc(tcmu_cmd->dbi_cnt, sizeof(uint32_t),
321 GFP_KERNEL);
322 if (!tcmu_cmd->dbi) {
323 kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
324 return NULL;
325 }
326
181 idr_preload(GFP_KERNEL); 327 idr_preload(GFP_KERNEL);
182 spin_lock_irq(&udev->commands_lock); 328 spin_lock_irq(&udev->commands_lock);
183 cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 0, 329 cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 0,
@@ -186,7 +332,7 @@ static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)
186 idr_preload_end(); 332 idr_preload_end();
187 333
188 if (cmd_id < 0) { 334 if (cmd_id < 0) {
189 kmem_cache_free(tcmu_cmd_cache, tcmu_cmd); 335 tcmu_free_cmd(tcmu_cmd);
190 return NULL; 336 return NULL;
191 } 337 }
192 tcmu_cmd->cmd_id = cmd_id; 338 tcmu_cmd->cmd_id = cmd_id;
@@ -248,10 +394,10 @@ static inline void new_iov(struct iovec **iov, int *iov_cnt,
248#define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size) 394#define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size)
249 395
250/* offset is relative to mb_addr */ 396/* offset is relative to mb_addr */
251static inline size_t get_block_offset(struct tcmu_dev *dev, 397static inline size_t get_block_offset_user(struct tcmu_dev *dev,
252 int block, int remaining) 398 int dbi, int remaining)
253{ 399{
254 return dev->data_off + block * DATA_BLOCK_SIZE + 400 return dev->data_off + dbi * DATA_BLOCK_SIZE +
255 DATA_BLOCK_SIZE - remaining; 401 DATA_BLOCK_SIZE - remaining;
256} 402}
257 403
@@ -260,37 +406,45 @@ static inline size_t iov_tail(struct tcmu_dev *udev, struct iovec *iov)
260 return (size_t)iov->iov_base + iov->iov_len; 406 return (size_t)iov->iov_base + iov->iov_len;
261} 407}
262 408
263static void alloc_and_scatter_data_area(struct tcmu_dev *udev, 409static int scatter_data_area(struct tcmu_dev *udev,
264 struct scatterlist *data_sg, unsigned int data_nents, 410 struct tcmu_cmd *tcmu_cmd, struct scatterlist *data_sg,
265 struct iovec **iov, int *iov_cnt, bool copy_data) 411 unsigned int data_nents, struct iovec **iov,
412 int *iov_cnt, bool copy_data)
266{ 413{
267 int i, block; 414 int i, dbi;
268 int block_remaining = 0; 415 int block_remaining = 0;
269 void *from, *to; 416 void *from, *to = NULL;
270 size_t copy_bytes, to_offset; 417 size_t copy_bytes, to_offset, offset;
271 struct scatterlist *sg; 418 struct scatterlist *sg;
419 struct page *page;
272 420
273 for_each_sg(data_sg, sg, data_nents, i) { 421 for_each_sg(data_sg, sg, data_nents, i) {
274 int sg_remaining = sg->length; 422 int sg_remaining = sg->length;
275 from = kmap_atomic(sg_page(sg)) + sg->offset; 423 from = kmap_atomic(sg_page(sg)) + sg->offset;
276 while (sg_remaining > 0) { 424 while (sg_remaining > 0) {
277 if (block_remaining == 0) { 425 if (block_remaining == 0) {
278 block = find_first_zero_bit(udev->data_bitmap, 426 if (to)
279 DATA_BLOCK_BITS); 427 kunmap_atomic(to);
428
280 block_remaining = DATA_BLOCK_SIZE; 429 block_remaining = DATA_BLOCK_SIZE;
281 set_bit(block, udev->data_bitmap); 430 dbi = tcmu_cmd_get_dbi(tcmu_cmd);
431 page = tcmu_get_block_page(udev, dbi);
432 to = kmap_atomic(page);
282 } 433 }
434
283 copy_bytes = min_t(size_t, sg_remaining, 435 copy_bytes = min_t(size_t, sg_remaining,
284 block_remaining); 436 block_remaining);
285 to_offset = get_block_offset(udev, block, 437 to_offset = get_block_offset_user(udev, dbi,
286 block_remaining); 438 block_remaining);
287 to = (void *)udev->mb_addr + to_offset; 439 offset = DATA_BLOCK_SIZE - block_remaining;
440 to = (void *)(unsigned long)to + offset;
441
288 if (*iov_cnt != 0 && 442 if (*iov_cnt != 0 &&
289 to_offset == iov_tail(udev, *iov)) { 443 to_offset == iov_tail(udev, *iov)) {
290 (*iov)->iov_len += copy_bytes; 444 (*iov)->iov_len += copy_bytes;
291 } else { 445 } else {
292 new_iov(iov, iov_cnt, udev); 446 new_iov(iov, iov_cnt, udev);
293 (*iov)->iov_base = (void __user *) to_offset; 447 (*iov)->iov_base = (void __user *)to_offset;
294 (*iov)->iov_len = copy_bytes; 448 (*iov)->iov_len = copy_bytes;
295 } 449 }
296 if (copy_data) { 450 if (copy_data) {
@@ -303,33 +457,29 @@ static void alloc_and_scatter_data_area(struct tcmu_dev *udev,
303 } 457 }
304 kunmap_atomic(from - sg->offset); 458 kunmap_atomic(from - sg->offset);
305 } 459 }
306} 460 if (to)
461 kunmap_atomic(to);
307 462
308static void free_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd) 463 return 0;
309{
310 bitmap_xor(udev->data_bitmap, udev->data_bitmap, cmd->data_bitmap,
311 DATA_BLOCK_BITS);
312} 464}
313 465
314static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd, 466static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
315 bool bidi) 467 bool bidi)
316{ 468{
317 struct se_cmd *se_cmd = cmd->se_cmd; 469 struct se_cmd *se_cmd = cmd->se_cmd;
318 int i, block; 470 int i, dbi;
319 int block_remaining = 0; 471 int block_remaining = 0;
320 void *from, *to; 472 void *from = NULL, *to;
321 size_t copy_bytes, from_offset; 473 size_t copy_bytes, offset;
322 struct scatterlist *sg, *data_sg; 474 struct scatterlist *sg, *data_sg;
475 struct page *page;
323 unsigned int data_nents; 476 unsigned int data_nents;
324 DECLARE_BITMAP(bitmap, DATA_BLOCK_BITS); 477 uint32_t count = 0;
325
326 bitmap_copy(bitmap, cmd->data_bitmap, DATA_BLOCK_BITS);
327 478
328 if (!bidi) { 479 if (!bidi) {
329 data_sg = se_cmd->t_data_sg; 480 data_sg = se_cmd->t_data_sg;
330 data_nents = se_cmd->t_data_nents; 481 data_nents = se_cmd->t_data_nents;
331 } else { 482 } else {
332 uint32_t count;
333 483
334 /* 484 /*
335 * For bidi case, the first count blocks are for Data-Out 485 * For bidi case, the first count blocks are for Data-Out
@@ -337,30 +487,30 @@ static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
337 * the Data-Out buffer blocks should be discarded. 487 * the Data-Out buffer blocks should be discarded.
338 */ 488 */
339 count = DIV_ROUND_UP(se_cmd->data_length, DATA_BLOCK_SIZE); 489 count = DIV_ROUND_UP(se_cmd->data_length, DATA_BLOCK_SIZE);
340 while (count--) {
341 block = find_first_bit(bitmap, DATA_BLOCK_BITS);
342 clear_bit(block, bitmap);
343 }
344 490
345 data_sg = se_cmd->t_bidi_data_sg; 491 data_sg = se_cmd->t_bidi_data_sg;
346 data_nents = se_cmd->t_bidi_data_nents; 492 data_nents = se_cmd->t_bidi_data_nents;
347 } 493 }
348 494
495 tcmu_cmd_set_dbi_cur(cmd, count);
496
349 for_each_sg(data_sg, sg, data_nents, i) { 497 for_each_sg(data_sg, sg, data_nents, i) {
350 int sg_remaining = sg->length; 498 int sg_remaining = sg->length;
351 to = kmap_atomic(sg_page(sg)) + sg->offset; 499 to = kmap_atomic(sg_page(sg)) + sg->offset;
352 while (sg_remaining > 0) { 500 while (sg_remaining > 0) {
353 if (block_remaining == 0) { 501 if (block_remaining == 0) {
354 block = find_first_bit(bitmap, 502 if (from)
355 DATA_BLOCK_BITS); 503 kunmap_atomic(from);
504
356 block_remaining = DATA_BLOCK_SIZE; 505 block_remaining = DATA_BLOCK_SIZE;
357 clear_bit(block, bitmap); 506 dbi = tcmu_cmd_get_dbi(cmd);
507 page = tcmu_get_block_page(udev, dbi);
508 from = kmap_atomic(page);
358 } 509 }
359 copy_bytes = min_t(size_t, sg_remaining, 510 copy_bytes = min_t(size_t, sg_remaining,
360 block_remaining); 511 block_remaining);
361 from_offset = get_block_offset(udev, block, 512 offset = DATA_BLOCK_SIZE - block_remaining;
362 block_remaining); 513 from = (void *)(unsigned long)from + offset;
363 from = (void *) udev->mb_addr + from_offset;
364 tcmu_flush_dcache_range(from, copy_bytes); 514 tcmu_flush_dcache_range(from, copy_bytes);
365 memcpy(to + sg->length - sg_remaining, from, 515 memcpy(to + sg->length - sg_remaining, from,
366 copy_bytes); 516 copy_bytes);
@@ -370,12 +520,13 @@ static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
370 } 520 }
371 kunmap_atomic(to - sg->offset); 521 kunmap_atomic(to - sg->offset);
372 } 522 }
523 if (from)
524 kunmap_atomic(from);
373} 525}
374 526
375static inline size_t spc_bitmap_free(unsigned long *bitmap) 527static inline size_t spc_bitmap_free(unsigned long *bitmap, uint32_t thresh)
376{ 528{
377 return DATA_BLOCK_SIZE * (DATA_BLOCK_BITS - 529 return DATA_BLOCK_SIZE * (thresh - bitmap_weight(bitmap, thresh));
378 bitmap_weight(bitmap, DATA_BLOCK_BITS));
379} 530}
380 531
381/* 532/*
@@ -384,9 +535,12 @@ static inline size_t spc_bitmap_free(unsigned long *bitmap)
384 * 535 *
385 * Called with ring lock held. 536 * Called with ring lock held.
386 */ 537 */
387static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size, size_t data_needed) 538static bool is_ring_space_avail(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
539 size_t cmd_size, size_t data_needed)
388{ 540{
389 struct tcmu_mailbox *mb = udev->mb_addr; 541 struct tcmu_mailbox *mb = udev->mb_addr;
542 uint32_t blocks_needed = (data_needed + DATA_BLOCK_SIZE - 1)
543 / DATA_BLOCK_SIZE;
390 size_t space, cmd_needed; 544 size_t space, cmd_needed;
391 u32 cmd_head; 545 u32 cmd_head;
392 546
@@ -410,35 +564,63 @@ static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size, size_t d
410 return false; 564 return false;
411 } 565 }
412 566
413 space = spc_bitmap_free(udev->data_bitmap); 567 /* try to check and get the data blocks as needed */
568 space = spc_bitmap_free(udev->data_bitmap, udev->dbi_thresh);
414 if (space < data_needed) { 569 if (space < data_needed) {
415 pr_debug("no data space: only %zu available, but ask for %zu\n", 570 unsigned long blocks_left = DATA_BLOCK_BITS - udev->dbi_thresh;
416 space, data_needed); 571 unsigned long grow;
417 return false; 572
573 if (blocks_left < blocks_needed) {
574 pr_debug("no data space: only %lu available, but ask for %zu\n",
575 blocks_left * DATA_BLOCK_SIZE,
576 data_needed);
577 return false;
578 }
579
580 /* Try to expand the thresh */
581 if (!udev->dbi_thresh) {
582 /* From idle state */
583 uint32_t init_thresh = DATA_BLOCK_INIT_BITS;
584
585 udev->dbi_thresh = max(blocks_needed, init_thresh);
586 } else {
587 /*
588 * Grow the data area by max(blocks needed,
589 * dbi_thresh / 2), but limited to the max
590 * DATA_BLOCK_BITS size.
591 */
592 grow = max(blocks_needed, udev->dbi_thresh / 2);
593 udev->dbi_thresh += grow;
594 if (udev->dbi_thresh > DATA_BLOCK_BITS)
595 udev->dbi_thresh = DATA_BLOCK_BITS;
596 }
418 } 597 }
419 598
599 if (!tcmu_get_empty_blocks(udev, cmd))
600 return false;
601
420 return true; 602 return true;
421} 603}
422 604
423static inline size_t tcmu_cmd_get_data_length(struct tcmu_cmd *tcmu_cmd) 605static inline size_t tcmu_cmd_get_base_cmd_size(size_t iov_cnt)
424{ 606{
425 struct se_cmd *se_cmd = tcmu_cmd->se_cmd; 607 return max(offsetof(struct tcmu_cmd_entry, req.iov[iov_cnt]),
426 size_t data_length = round_up(se_cmd->data_length, DATA_BLOCK_SIZE); 608 sizeof(struct tcmu_cmd_entry));
427
428 if (se_cmd->se_cmd_flags & SCF_BIDI) {
429 BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents));
430 data_length += round_up(se_cmd->t_bidi_data_sg->length,
431 DATA_BLOCK_SIZE);
432 }
433
434 return data_length;
435} 609}
436 610
437static inline uint32_t tcmu_cmd_get_block_cnt(struct tcmu_cmd *tcmu_cmd) 611static inline size_t tcmu_cmd_get_cmd_size(struct tcmu_cmd *tcmu_cmd,
612 size_t base_command_size)
438{ 613{
439 size_t data_length = tcmu_cmd_get_data_length(tcmu_cmd); 614 struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
615 size_t command_size;
440 616
441 return data_length / DATA_BLOCK_SIZE; 617 command_size = base_command_size +
618 round_up(scsi_command_size(se_cmd->t_task_cdb),
619 TCMU_OP_ALIGN_SIZE);
620
621 WARN_ON(command_size & (TCMU_OP_ALIGN_SIZE-1));
622
623 return command_size;
442} 624}
443 625
444static sense_reason_t 626static sense_reason_t
@@ -450,12 +632,11 @@ tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd)
450 struct tcmu_mailbox *mb; 632 struct tcmu_mailbox *mb;
451 struct tcmu_cmd_entry *entry; 633 struct tcmu_cmd_entry *entry;
452 struct iovec *iov; 634 struct iovec *iov;
453 int iov_cnt; 635 int iov_cnt, ret;
454 uint32_t cmd_head; 636 uint32_t cmd_head;
455 uint64_t cdb_off; 637 uint64_t cdb_off;
456 bool copy_to_data_area; 638 bool copy_to_data_area;
457 size_t data_length = tcmu_cmd_get_data_length(tcmu_cmd); 639 size_t data_length = tcmu_cmd_get_data_length(tcmu_cmd);
458 DECLARE_BITMAP(old_bitmap, DATA_BLOCK_BITS);
459 640
460 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) 641 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags))
461 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 642 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
@@ -464,18 +645,18 @@ tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd)
464 * Must be a certain minimum size for response sense info, but 645 * Must be a certain minimum size for response sense info, but
465 * also may be larger if the iov array is large. 646 * also may be larger if the iov array is large.
466 * 647 *
467 * We prepare way too many iovs for potential uses here, because it's 648 * We prepare as many iovs as possbile for potential uses here,
468 * expensive to tell how many regions are freed in the bitmap 649 * because it's expensive to tell how many regions are freed in
469 */ 650 * the bitmap & global data pool, as the size calculated here
470 base_command_size = max(offsetof(struct tcmu_cmd_entry, 651 * will only be used to do the checks.
471 req.iov[tcmu_cmd_get_block_cnt(tcmu_cmd)]), 652 *
472 sizeof(struct tcmu_cmd_entry)); 653 * The size will be recalculated later as actually needed to save
473 command_size = base_command_size 654 * cmd area memories.
474 + round_up(scsi_command_size(se_cmd->t_task_cdb), TCMU_OP_ALIGN_SIZE); 655 */
475 656 base_command_size = tcmu_cmd_get_base_cmd_size(tcmu_cmd->dbi_cnt);
476 WARN_ON(command_size & (TCMU_OP_ALIGN_SIZE-1)); 657 command_size = tcmu_cmd_get_cmd_size(tcmu_cmd, base_command_size);
477 658
478 spin_lock_irq(&udev->cmdr_lock); 659 mutex_lock(&udev->cmdr_lock);
479 660
480 mb = udev->mb_addr; 661 mb = udev->mb_addr;
481 cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ 662 cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
@@ -484,18 +665,18 @@ tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd)
484 pr_warn("TCMU: Request of size %zu/%zu is too big for %u/%zu " 665 pr_warn("TCMU: Request of size %zu/%zu is too big for %u/%zu "
485 "cmd ring/data area\n", command_size, data_length, 666 "cmd ring/data area\n", command_size, data_length,
486 udev->cmdr_size, udev->data_size); 667 udev->cmdr_size, udev->data_size);
487 spin_unlock_irq(&udev->cmdr_lock); 668 mutex_unlock(&udev->cmdr_lock);
488 return TCM_INVALID_CDB_FIELD; 669 return TCM_INVALID_CDB_FIELD;
489 } 670 }
490 671
491 while (!is_ring_space_avail(udev, command_size, data_length)) { 672 while (!is_ring_space_avail(udev, tcmu_cmd, command_size, data_length)) {
492 int ret; 673 int ret;
493 DEFINE_WAIT(__wait); 674 DEFINE_WAIT(__wait);
494 675
495 prepare_to_wait(&udev->wait_cmdr, &__wait, TASK_INTERRUPTIBLE); 676 prepare_to_wait(&udev->wait_cmdr, &__wait, TASK_INTERRUPTIBLE);
496 677
497 pr_debug("sleeping for ring space\n"); 678 pr_debug("sleeping for ring space\n");
498 spin_unlock_irq(&udev->cmdr_lock); 679 mutex_unlock(&udev->cmdr_lock);
499 if (udev->cmd_time_out) 680 if (udev->cmd_time_out)
500 ret = schedule_timeout( 681 ret = schedule_timeout(
501 msecs_to_jiffies(udev->cmd_time_out)); 682 msecs_to_jiffies(udev->cmd_time_out));
@@ -507,7 +688,7 @@ tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd)
507 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 688 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
508 } 689 }
509 690
510 spin_lock_irq(&udev->cmdr_lock); 691 mutex_lock(&udev->cmdr_lock);
511 692
512 /* We dropped cmdr_lock, cmd_head is stale */ 693 /* We dropped cmdr_lock, cmd_head is stale */
513 cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ 694 cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
@@ -534,20 +715,26 @@ tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd)
534 entry = (void *) mb + CMDR_OFF + cmd_head; 715 entry = (void *) mb + CMDR_OFF + cmd_head;
535 tcmu_flush_dcache_range(entry, sizeof(*entry)); 716 tcmu_flush_dcache_range(entry, sizeof(*entry));
536 tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_CMD); 717 tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_CMD);
537 tcmu_hdr_set_len(&entry->hdr.len_op, command_size);
538 entry->hdr.cmd_id = tcmu_cmd->cmd_id; 718 entry->hdr.cmd_id = tcmu_cmd->cmd_id;
539 entry->hdr.kflags = 0; 719 entry->hdr.kflags = 0;
540 entry->hdr.uflags = 0; 720 entry->hdr.uflags = 0;
541 721
542 bitmap_copy(old_bitmap, udev->data_bitmap, DATA_BLOCK_BITS);
543
544 /* Handle allocating space from the data area */ 722 /* Handle allocating space from the data area */
723 tcmu_cmd_reset_dbi_cur(tcmu_cmd);
545 iov = &entry->req.iov[0]; 724 iov = &entry->req.iov[0];
546 iov_cnt = 0; 725 iov_cnt = 0;
547 copy_to_data_area = (se_cmd->data_direction == DMA_TO_DEVICE 726 copy_to_data_area = (se_cmd->data_direction == DMA_TO_DEVICE
548 || se_cmd->se_cmd_flags & SCF_BIDI); 727 || se_cmd->se_cmd_flags & SCF_BIDI);
549 alloc_and_scatter_data_area(udev, se_cmd->t_data_sg, 728 ret = scatter_data_area(udev, tcmu_cmd, se_cmd->t_data_sg,
550 se_cmd->t_data_nents, &iov, &iov_cnt, copy_to_data_area); 729 se_cmd->t_data_nents, &iov, &iov_cnt,
730 copy_to_data_area);
731 if (ret) {
732 tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt);
733 mutex_unlock(&udev->cmdr_lock);
734
735 pr_err("tcmu: alloc and scatter data failed\n");
736 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
737 }
551 entry->req.iov_cnt = iov_cnt; 738 entry->req.iov_cnt = iov_cnt;
552 entry->req.iov_dif_cnt = 0; 739 entry->req.iov_dif_cnt = 0;
553 740
@@ -555,14 +742,29 @@ tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd)
555 if (se_cmd->se_cmd_flags & SCF_BIDI) { 742 if (se_cmd->se_cmd_flags & SCF_BIDI) {
556 iov_cnt = 0; 743 iov_cnt = 0;
557 iov++; 744 iov++;
558 alloc_and_scatter_data_area(udev, se_cmd->t_bidi_data_sg, 745 ret = scatter_data_area(udev, tcmu_cmd,
559 se_cmd->t_bidi_data_nents, &iov, &iov_cnt, 746 se_cmd->t_bidi_data_sg,
560 false); 747 se_cmd->t_bidi_data_nents,
748 &iov, &iov_cnt, false);
749 if (ret) {
750 tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt);
751 mutex_unlock(&udev->cmdr_lock);
752
753 pr_err("tcmu: alloc and scatter bidi data failed\n");
754 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
755 }
561 entry->req.iov_bidi_cnt = iov_cnt; 756 entry->req.iov_bidi_cnt = iov_cnt;
562 } 757 }
563 /* cmd's data_bitmap is what changed in process */ 758
564 bitmap_xor(tcmu_cmd->data_bitmap, old_bitmap, udev->data_bitmap, 759 /*
565 DATA_BLOCK_BITS); 760 * Recalaulate the command's base size and size according
761 * to the actual needs
762 */
763 base_command_size = tcmu_cmd_get_base_cmd_size(entry->req.iov_cnt +
764 entry->req.iov_bidi_cnt);
765 command_size = tcmu_cmd_get_cmd_size(tcmu_cmd, base_command_size);
766
767 tcmu_hdr_set_len(&entry->hdr.len_op, command_size);
566 768
567 /* All offsets relative to mb_addr, not start of entry! */ 769 /* All offsets relative to mb_addr, not start of entry! */
568 cdb_off = CMDR_OFF + cmd_head + base_command_size; 770 cdb_off = CMDR_OFF + cmd_head + base_command_size;
@@ -572,8 +774,7 @@ tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd)
572 774
573 UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size); 775 UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size);
574 tcmu_flush_dcache_range(mb, sizeof(*mb)); 776 tcmu_flush_dcache_range(mb, sizeof(*mb));
575 777 mutex_unlock(&udev->cmdr_lock);
576 spin_unlock_irq(&udev->cmdr_lock);
577 778
578 /* TODO: only if FLUSH and FUA? */ 779 /* TODO: only if FLUSH and FUA? */
579 uio_event_notify(&udev->uio_info); 780 uio_event_notify(&udev->uio_info);
@@ -604,7 +805,7 @@ tcmu_queue_cmd(struct se_cmd *se_cmd)
604 idr_remove(&udev->commands, tcmu_cmd->cmd_id); 805 idr_remove(&udev->commands, tcmu_cmd->cmd_id);
605 spin_unlock_irq(&udev->commands_lock); 806 spin_unlock_irq(&udev->commands_lock);
606 807
607 kmem_cache_free(tcmu_cmd_cache, tcmu_cmd); 808 tcmu_free_cmd(tcmu_cmd);
608 } 809 }
609 810
610 return ret; 811 return ret;
@@ -615,50 +816,45 @@ static void tcmu_handle_completion(struct tcmu_cmd *cmd, struct tcmu_cmd_entry *
615 struct se_cmd *se_cmd = cmd->se_cmd; 816 struct se_cmd *se_cmd = cmd->se_cmd;
616 struct tcmu_dev *udev = cmd->tcmu_dev; 817 struct tcmu_dev *udev = cmd->tcmu_dev;
617 818
618 if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) { 819 /*
619 /* 820 * cmd has been completed already from timeout, just reclaim
620 * cmd has been completed already from timeout, just reclaim 821 * data area space and free cmd
621 * data area space and free cmd 822 */
622 */ 823 if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags))
623 free_data_area(udev, cmd); 824 goto out;
624 825
625 kmem_cache_free(tcmu_cmd_cache, cmd); 826 tcmu_cmd_reset_dbi_cur(cmd);
626 return;
627 }
628 827
629 if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) { 828 if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) {
630 free_data_area(udev, cmd);
631 pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n", 829 pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n",
632 cmd->se_cmd); 830 cmd->se_cmd);
633 entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION; 831 entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION;
634 } else if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) { 832 } else if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) {
635 memcpy(se_cmd->sense_buffer, entry->rsp.sense_buffer, 833 memcpy(se_cmd->sense_buffer, entry->rsp.sense_buffer,
636 se_cmd->scsi_sense_length); 834 se_cmd->scsi_sense_length);
637 free_data_area(udev, cmd);
638 } else if (se_cmd->se_cmd_flags & SCF_BIDI) { 835 } else if (se_cmd->se_cmd_flags & SCF_BIDI) {
639 /* Get Data-In buffer before clean up */ 836 /* Get Data-In buffer before clean up */
640 gather_data_area(udev, cmd, true); 837 gather_data_area(udev, cmd, true);
641 free_data_area(udev, cmd);
642 } else if (se_cmd->data_direction == DMA_FROM_DEVICE) { 838 } else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
643 gather_data_area(udev, cmd, false); 839 gather_data_area(udev, cmd, false);
644 free_data_area(udev, cmd);
645 } else if (se_cmd->data_direction == DMA_TO_DEVICE) { 840 } else if (se_cmd->data_direction == DMA_TO_DEVICE) {
646 free_data_area(udev, cmd); 841 /* TODO: */
647 } else if (se_cmd->data_direction != DMA_NONE) { 842 } else if (se_cmd->data_direction != DMA_NONE) {
648 pr_warn("TCMU: data direction was %d!\n", 843 pr_warn("TCMU: data direction was %d!\n",
649 se_cmd->data_direction); 844 se_cmd->data_direction);
650 } 845 }
651 846
652 target_complete_cmd(cmd->se_cmd, entry->rsp.scsi_status); 847 target_complete_cmd(cmd->se_cmd, entry->rsp.scsi_status);
653 cmd->se_cmd = NULL;
654 848
655 kmem_cache_free(tcmu_cmd_cache, cmd); 849out:
850 cmd->se_cmd = NULL;
851 tcmu_cmd_free_data(cmd, cmd->dbi_cnt);
852 tcmu_free_cmd(cmd);
656} 853}
657 854
658static unsigned int tcmu_handle_completions(struct tcmu_dev *udev) 855static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)
659{ 856{
660 struct tcmu_mailbox *mb; 857 struct tcmu_mailbox *mb;
661 unsigned long flags;
662 int handled = 0; 858 int handled = 0;
663 859
664 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) { 860 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
@@ -666,8 +862,6 @@ static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)
666 return 0; 862 return 0;
667 } 863 }
668 864
669 spin_lock_irqsave(&udev->cmdr_lock, flags);
670
671 mb = udev->mb_addr; 865 mb = udev->mb_addr;
672 tcmu_flush_dcache_range(mb, sizeof(*mb)); 866 tcmu_flush_dcache_range(mb, sizeof(*mb));
673 867
@@ -708,8 +902,6 @@ static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)
708 if (mb->cmd_tail == mb->cmd_head) 902 if (mb->cmd_tail == mb->cmd_head)
709 del_timer(&udev->timeout); /* no more pending cmds */ 903 del_timer(&udev->timeout); /* no more pending cmds */
710 904
711 spin_unlock_irqrestore(&udev->cmdr_lock, flags);
712
713 wake_up(&udev->wait_cmdr); 905 wake_up(&udev->wait_cmdr);
714 906
715 return handled; 907 return handled;
@@ -736,16 +928,14 @@ static void tcmu_device_timedout(unsigned long data)
736{ 928{
737 struct tcmu_dev *udev = (struct tcmu_dev *)data; 929 struct tcmu_dev *udev = (struct tcmu_dev *)data;
738 unsigned long flags; 930 unsigned long flags;
739 int handled;
740
741 handled = tcmu_handle_completions(udev);
742
743 pr_warn("%d completions handled from timeout\n", handled);
744 931
745 spin_lock_irqsave(&udev->commands_lock, flags); 932 spin_lock_irqsave(&udev->commands_lock, flags);
746 idr_for_each(&udev->commands, tcmu_check_expired_cmd, NULL); 933 idr_for_each(&udev->commands, tcmu_check_expired_cmd, NULL);
747 spin_unlock_irqrestore(&udev->commands_lock, flags); 934 spin_unlock_irqrestore(&udev->commands_lock, flags);
748 935
936 /* Try to wake up the ummap thread */
937 wake_up(&unmap_wait);
938
749 /* 939 /*
750 * We don't need to wakeup threads on wait_cmdr since they have their 940 * We don't need to wakeup threads on wait_cmdr since they have their
751 * own timeout. 941 * own timeout.
@@ -790,7 +980,7 @@ static struct se_device *tcmu_alloc_device(struct se_hba *hba, const char *name)
790 udev->cmd_time_out = TCMU_TIME_OUT; 980 udev->cmd_time_out = TCMU_TIME_OUT;
791 981
792 init_waitqueue_head(&udev->wait_cmdr); 982 init_waitqueue_head(&udev->wait_cmdr);
793 spin_lock_init(&udev->cmdr_lock); 983 mutex_init(&udev->cmdr_lock);
794 984
795 idr_init(&udev->commands); 985 idr_init(&udev->commands);
796 spin_lock_init(&udev->commands_lock); 986 spin_lock_init(&udev->commands_lock);
@@ -805,7 +995,9 @@ static int tcmu_irqcontrol(struct uio_info *info, s32 irq_on)
805{ 995{
806 struct tcmu_dev *tcmu_dev = container_of(info, struct tcmu_dev, uio_info); 996 struct tcmu_dev *tcmu_dev = container_of(info, struct tcmu_dev, uio_info);
807 997
998 mutex_lock(&tcmu_dev->cmdr_lock);
808 tcmu_handle_completions(tcmu_dev); 999 tcmu_handle_completions(tcmu_dev);
1000 mutex_unlock(&tcmu_dev->cmdr_lock);
809 1001
810 return 0; 1002 return 0;
811} 1003}
@@ -827,6 +1019,60 @@ static int tcmu_find_mem_index(struct vm_area_struct *vma)
827 return -1; 1019 return -1;
828} 1020}
829 1021
1022static struct page *tcmu_try_get_block_page(struct tcmu_dev *udev, uint32_t dbi)
1023{
1024 struct page *page;
1025 int ret;
1026
1027 mutex_lock(&udev->cmdr_lock);
1028 page = tcmu_get_block_page(udev, dbi);
1029 if (likely(page)) {
1030 mutex_unlock(&udev->cmdr_lock);
1031 return page;
1032 }
1033
1034 /*
1035 * Normally it shouldn't be here:
1036 * Only when the userspace has touched the blocks which
1037 * are out of the tcmu_cmd's data iov[], and will return
1038 * one zeroed page.
1039 */
1040 pr_warn("Block(%u) out of cmd's iov[] has been touched!\n", dbi);
1041 pr_warn("Mostly it will be a bug of userspace, please have a check!\n");
1042
1043 if (dbi >= udev->dbi_thresh) {
1044 /* Extern the udev->dbi_thresh to dbi + 1 */
1045 udev->dbi_thresh = dbi + 1;
1046 udev->dbi_max = dbi;
1047 }
1048
1049 page = radix_tree_lookup(&udev->data_blocks, dbi);
1050 if (!page) {
1051 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1052 if (!page) {
1053 mutex_unlock(&udev->cmdr_lock);
1054 return NULL;
1055 }
1056
1057 ret = radix_tree_insert(&udev->data_blocks, dbi, page);
1058 if (ret) {
1059 mutex_unlock(&udev->cmdr_lock);
1060 __free_page(page);
1061 return NULL;
1062 }
1063
1064 /*
1065 * Since this case is rare in page fault routine, here we
1066 * will allow the global_db_count >= TCMU_GLOBAL_MAX_BLOCKS
1067 * to reduce possible page fault call trace.
1068 */
1069 atomic_inc(&global_db_count);
1070 }
1071 mutex_unlock(&udev->cmdr_lock);
1072
1073 return page;
1074}
1075
830static int tcmu_vma_fault(struct vm_fault *vmf) 1076static int tcmu_vma_fault(struct vm_fault *vmf)
831{ 1077{
832 struct tcmu_dev *udev = vmf->vma->vm_private_data; 1078 struct tcmu_dev *udev = vmf->vma->vm_private_data;
@@ -845,11 +1091,20 @@ static int tcmu_vma_fault(struct vm_fault *vmf)
845 */ 1091 */
846 offset = (vmf->pgoff - mi) << PAGE_SHIFT; 1092 offset = (vmf->pgoff - mi) << PAGE_SHIFT;
847 1093
848 addr = (void *)(unsigned long)info->mem[mi].addr + offset; 1094 if (offset < udev->data_off) {
849 if (info->mem[mi].memtype == UIO_MEM_LOGICAL) 1095 /* For the vmalloc()ed cmd area pages */
850 page = virt_to_page(addr); 1096 addr = (void *)(unsigned long)info->mem[mi].addr + offset;
851 else
852 page = vmalloc_to_page(addr); 1097 page = vmalloc_to_page(addr);
1098 } else {
1099 uint32_t dbi;
1100
1101 /* For the dynamically growing data area pages */
1102 dbi = (offset - udev->data_off) / DATA_BLOCK_SIZE;
1103 page = tcmu_try_get_block_page(udev, dbi);
1104 if (!page)
1105 return VM_FAULT_NOPAGE;
1106 }
1107
853 get_page(page); 1108 get_page(page);
854 vmf->page = page; 1109 vmf->page = page;
855 return 0; 1110 return 0;
@@ -883,6 +1138,8 @@ static int tcmu_open(struct uio_info *info, struct inode *inode)
883 if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags)) 1138 if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags))
884 return -EBUSY; 1139 return -EBUSY;
885 1140
1141 udev->inode = inode;
1142
886 pr_debug("open\n"); 1143 pr_debug("open\n");
887 1144
888 return 0; 1145 return 0;
@@ -963,7 +1220,7 @@ static int tcmu_configure_device(struct se_device *dev)
963 1220
964 info->name = str; 1221 info->name = str;
965 1222
966 udev->mb_addr = vzalloc(TCMU_RING_SIZE); 1223 udev->mb_addr = vzalloc(CMDR_SIZE);
967 if (!udev->mb_addr) { 1224 if (!udev->mb_addr) {
968 ret = -ENOMEM; 1225 ret = -ENOMEM;
969 goto err_vzalloc; 1226 goto err_vzalloc;
@@ -972,8 +1229,11 @@ static int tcmu_configure_device(struct se_device *dev)
972 /* mailbox fits in first part of CMDR space */ 1229 /* mailbox fits in first part of CMDR space */
973 udev->cmdr_size = CMDR_SIZE - CMDR_OFF; 1230 udev->cmdr_size = CMDR_SIZE - CMDR_OFF;
974 udev->data_off = CMDR_SIZE; 1231 udev->data_off = CMDR_SIZE;
975 udev->data_size = TCMU_RING_SIZE - CMDR_SIZE; 1232 udev->data_size = DATA_SIZE;
1233 udev->dbi_thresh = 0; /* Default in Idle state */
1234 udev->waiting_global = false;
976 1235
1236 /* Initialise the mailbox of the ring buffer */
977 mb = udev->mb_addr; 1237 mb = udev->mb_addr;
978 mb->version = TCMU_MAILBOX_VERSION; 1238 mb->version = TCMU_MAILBOX_VERSION;
979 mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC; 1239 mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC;
@@ -984,12 +1244,14 @@ static int tcmu_configure_device(struct se_device *dev)
984 WARN_ON(udev->data_size % PAGE_SIZE); 1244 WARN_ON(udev->data_size % PAGE_SIZE);
985 WARN_ON(udev->data_size % DATA_BLOCK_SIZE); 1245 WARN_ON(udev->data_size % DATA_BLOCK_SIZE);
986 1246
1247 INIT_RADIX_TREE(&udev->data_blocks, GFP_KERNEL);
1248
987 info->version = __stringify(TCMU_MAILBOX_VERSION); 1249 info->version = __stringify(TCMU_MAILBOX_VERSION);
988 1250
989 info->mem[0].name = "tcm-user command & data buffer"; 1251 info->mem[0].name = "tcm-user command & data buffer";
990 info->mem[0].addr = (phys_addr_t)(uintptr_t)udev->mb_addr; 1252 info->mem[0].addr = (phys_addr_t)(uintptr_t)udev->mb_addr;
991 info->mem[0].size = TCMU_RING_SIZE; 1253 info->mem[0].size = TCMU_RING_SIZE;
992 info->mem[0].memtype = UIO_MEM_VIRTUAL; 1254 info->mem[0].memtype = UIO_MEM_NONE;
993 1255
994 info->irqcontrol = tcmu_irqcontrol; 1256 info->irqcontrol = tcmu_irqcontrol;
995 info->irq = UIO_IRQ_CUSTOM; 1257 info->irq = UIO_IRQ_CUSTOM;
@@ -1015,6 +1277,10 @@ static int tcmu_configure_device(struct se_device *dev)
1015 if (ret) 1277 if (ret)
1016 goto err_netlink; 1278 goto err_netlink;
1017 1279
1280 mutex_lock(&root_udev_mutex);
1281 list_add(&udev->node, &root_udev);
1282 mutex_unlock(&root_udev_mutex);
1283
1018 return 0; 1284 return 0;
1019 1285
1020err_netlink: 1286err_netlink:
@@ -1049,6 +1315,23 @@ static bool tcmu_dev_configured(struct tcmu_dev *udev)
1049 return udev->uio_info.uio_dev ? true : false; 1315 return udev->uio_info.uio_dev ? true : false;
1050} 1316}
1051 1317
1318static void tcmu_blocks_release(struct tcmu_dev *udev)
1319{
1320 int i;
1321 struct page *page;
1322
1323 /* Try to release all block pages */
1324 mutex_lock(&udev->cmdr_lock);
1325 for (i = 0; i <= udev->dbi_max; i++) {
1326 page = radix_tree_delete(&udev->data_blocks, i);
1327 if (page) {
1328 __free_page(page);
1329 atomic_dec(&global_db_count);
1330 }
1331 }
1332 mutex_unlock(&udev->cmdr_lock);
1333}
1334
1052static void tcmu_free_device(struct se_device *dev) 1335static void tcmu_free_device(struct se_device *dev)
1053{ 1336{
1054 struct tcmu_dev *udev = TCMU_DEV(dev); 1337 struct tcmu_dev *udev = TCMU_DEV(dev);
@@ -1058,6 +1341,10 @@ static void tcmu_free_device(struct se_device *dev)
1058 1341
1059 del_timer_sync(&udev->timeout); 1342 del_timer_sync(&udev->timeout);
1060 1343
1344 mutex_lock(&root_udev_mutex);
1345 list_del(&udev->node);
1346 mutex_unlock(&root_udev_mutex);
1347
1061 vfree(udev->mb_addr); 1348 vfree(udev->mb_addr);
1062 1349
1063 /* Upper layer should drain all requests before calling this */ 1350 /* Upper layer should drain all requests before calling this */
@@ -1070,6 +1357,8 @@ static void tcmu_free_device(struct se_device *dev)
1070 spin_unlock_irq(&udev->commands_lock); 1357 spin_unlock_irq(&udev->commands_lock);
1071 WARN_ON(!all_expired); 1358 WARN_ON(!all_expired);
1072 1359
1360 tcmu_blocks_release(udev);
1361
1073 if (tcmu_dev_configured(udev)) { 1362 if (tcmu_dev_configured(udev)) {
1074 tcmu_netlink_event(TCMU_CMD_REMOVED_DEVICE, udev->uio_info.name, 1363 tcmu_netlink_event(TCMU_CMD_REMOVED_DEVICE, udev->uio_info.name,
1075 udev->uio_info.uio_dev->minor); 1364 udev->uio_info.uio_dev->minor);
@@ -1256,6 +1545,84 @@ static struct target_backend_ops tcmu_ops = {
1256 .tb_dev_attrib_attrs = NULL, 1545 .tb_dev_attrib_attrs = NULL,
1257}; 1546};
1258 1547
1548static int unmap_thread_fn(void *data)
1549{
1550 struct tcmu_dev *udev;
1551 loff_t off;
1552 uint32_t start, end, block;
1553 struct page *page;
1554 int i;
1555
1556 while (1) {
1557 DEFINE_WAIT(__wait);
1558
1559 prepare_to_wait(&unmap_wait, &__wait, TASK_INTERRUPTIBLE);
1560 schedule();
1561 finish_wait(&unmap_wait, &__wait);
1562
1563 if (kthread_should_stop())
1564 break;
1565
1566 mutex_lock(&root_udev_mutex);
1567 list_for_each_entry(udev, &root_udev, node) {
1568 mutex_lock(&udev->cmdr_lock);
1569
1570 /* Try to complete the finished commands first */
1571 tcmu_handle_completions(udev);
1572
1573 /* Skip the udevs waiting the global pool or in idle */
1574 if (udev->waiting_global || !udev->dbi_thresh) {
1575 mutex_unlock(&udev->cmdr_lock);
1576 continue;
1577 }
1578
1579 end = udev->dbi_max + 1;
1580 block = find_last_bit(udev->data_bitmap, end);
1581 if (block == udev->dbi_max) {
1582 /*
1583 * The last bit is dbi_max, so there is
1584 * no need to shrink any blocks.
1585 */
1586 mutex_unlock(&udev->cmdr_lock);
1587 continue;
1588 } else if (block == end) {
1589 /* The current udev will goto idle state */
1590 udev->dbi_thresh = start = 0;
1591 udev->dbi_max = 0;
1592 } else {
1593 udev->dbi_thresh = start = block + 1;
1594 udev->dbi_max = block;
1595 }
1596
1597 /* Here will truncate the data area from off */
1598 off = udev->data_off + start * DATA_BLOCK_SIZE;
1599 unmap_mapping_range(udev->inode->i_mapping, off, 0, 1);
1600
1601 /* Release the block pages */
1602 for (i = start; i < end; i++) {
1603 page = radix_tree_delete(&udev->data_blocks, i);
1604 if (page) {
1605 __free_page(page);
1606 atomic_dec(&global_db_count);
1607 }
1608 }
1609 mutex_unlock(&udev->cmdr_lock);
1610 }
1611
1612 /*
1613 * Try to wake up the udevs who are waiting
1614 * for the global data pool.
1615 */
1616 list_for_each_entry(udev, &root_udev, node) {
1617 if (udev->waiting_global)
1618 wake_up(&udev->wait_cmdr);
1619 }
1620 mutex_unlock(&root_udev_mutex);
1621 }
1622
1623 return 0;
1624}
1625
1259static int __init tcmu_module_init(void) 1626static int __init tcmu_module_init(void)
1260{ 1627{
1261 int ret, i, len = 0; 1628 int ret, i, len = 0;
@@ -1301,8 +1668,17 @@ static int __init tcmu_module_init(void)
1301 if (ret) 1668 if (ret)
1302 goto out_attrs; 1669 goto out_attrs;
1303 1670
1671 init_waitqueue_head(&unmap_wait);
1672 unmap_thread = kthread_run(unmap_thread_fn, NULL, "tcmu_unmap");
1673 if (IS_ERR(unmap_thread)) {
1674 ret = PTR_ERR(unmap_thread);
1675 goto out_unreg_transport;
1676 }
1677
1304 return 0; 1678 return 0;
1305 1679
1680out_unreg_transport:
1681 target_backend_unregister(&tcmu_ops);
1306out_attrs: 1682out_attrs:
1307 kfree(tcmu_attrs); 1683 kfree(tcmu_attrs);
1308out_unreg_genl: 1684out_unreg_genl:
@@ -1317,6 +1693,7 @@ out_free_cache:
1317 1693
1318static void __exit tcmu_module_exit(void) 1694static void __exit tcmu_module_exit(void)
1319{ 1695{
1696 kthread_stop(unmap_thread);
1320 target_backend_unregister(&tcmu_ops); 1697 target_backend_unregister(&tcmu_ops);
1321 kfree(tcmu_attrs); 1698 kfree(tcmu_attrs);
1322 genl_unregister_family(&tcmu_genl_family); 1699 genl_unregister_family(&tcmu_genl_family);
diff --git a/include/scsi/scsi_proto.h b/include/scsi/scsi_proto.h
index 6ba66e01f6df..ce78ec8e367d 100644
--- a/include/scsi/scsi_proto.h
+++ b/include/scsi/scsi_proto.h
@@ -112,6 +112,7 @@
112#define WRITE_16 0x8a 112#define WRITE_16 0x8a
113#define READ_ATTRIBUTE 0x8c 113#define READ_ATTRIBUTE 0x8c
114#define WRITE_ATTRIBUTE 0x8d 114#define WRITE_ATTRIBUTE 0x8d
115#define WRITE_VERIFY_16 0x8e
115#define VERIFY_16 0x8f 116#define VERIFY_16 0x8f
116#define SYNCHRONIZE_CACHE_16 0x91 117#define SYNCHRONIZE_CACHE_16 0x91
117#define WRITE_SAME_16 0x93 118#define WRITE_SAME_16 0x93
diff --git a/include/target/target_core_backend.h b/include/target/target_core_backend.h
index 1b0f447ce850..e475531565fd 100644
--- a/include/target/target_core_backend.h
+++ b/include/target/target_core_backend.h
@@ -10,6 +10,7 @@
10 * backend module. 10 * backend module.
11 */ 11 */
12#define TRANSPORT_FLAG_PASSTHROUGH_ALUA 0x2 12#define TRANSPORT_FLAG_PASSTHROUGH_ALUA 0x2
13#define TRANSPORT_FLAG_PASSTHROUGH_PGR 0x4
13 14
14struct request_queue; 15struct request_queue;
15struct scatterlist; 16struct scatterlist;
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index ccfad0e9c2cd..0c1dce2ac6f0 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -664,6 +664,7 @@ struct se_dev_attrib {
664 int pi_prot_format; 664 int pi_prot_format;
665 enum target_prot_type pi_prot_type; 665 enum target_prot_type pi_prot_type;
666 enum target_prot_type hw_pi_prot_type; 666 enum target_prot_type hw_pi_prot_type;
667 int pi_prot_verify;
667 int enforce_pr_isids; 668 int enforce_pr_isids;
668 int force_pr_aptpl; 669 int force_pr_aptpl;
669 int is_nonrot; 670 int is_nonrot;
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-12 20:36:53 -0400