|
@@ -24,11 +24,13 @@
|
|
|
|
|
|
#include "iscsi_target_core.h"
|
|
|
#include "iscsi_target_util.h"
|
|
|
+#include "iscsi_target_tpg.h"
|
|
|
#include "iscsi_target_seq_pdu_list.h"
|
|
|
|
|
|
#define OFFLOAD_BUF_SIZE 32768
|
|
|
|
|
|
-void iscsit_dump_seq_list(struct iscsi_cmd *cmd)
|
|
|
+#ifdef DEBUG
|
|
|
+static void iscsit_dump_seq_list(struct iscsi_cmd *cmd)
|
|
|
{
|
|
|
int i;
|
|
|
struct iscsi_seq *seq;
|
|
@@ -46,7 +48,7 @@ void iscsit_dump_seq_list(struct iscsi_cmd *cmd)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-void iscsit_dump_pdu_list(struct iscsi_cmd *cmd)
|
|
|
+static void iscsit_dump_pdu_list(struct iscsi_cmd *cmd)
|
|
|
{
|
|
|
int i;
|
|
|
struct iscsi_pdu *pdu;
|
|
@@ -61,6 +63,10 @@ void iscsit_dump_pdu_list(struct iscsi_cmd *cmd)
|
|
|
pdu->length, pdu->pdu_send_order, pdu->seq_no);
|
|
|
}
|
|
|
}
|
|
|
+#else
|
|
|
+static void iscsit_dump_seq_list(struct iscsi_cmd *cmd) {}
|
|
|
+static void iscsit_dump_pdu_list(struct iscsi_cmd *cmd) {}
|
|
|
+#endif
|
|
|
|
|
|
static void iscsit_ordered_seq_lists(
|
|
|
struct iscsi_cmd *cmd,
|
|
@@ -135,11 +141,11 @@ redo:
|
|
|
seq_count++;
|
|
|
continue;
|
|
|
}
|
|
|
- array = kzalloc(seq_count * sizeof(u32), GFP_KERNEL);
|
|
|
+ array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
|
|
|
if (!array) {
|
|
|
pr_err("Unable to allocate memory"
|
|
|
" for random array.\n");
|
|
|
- return -1;
|
|
|
+ return -ENOMEM;
|
|
|
}
|
|
|
iscsit_create_random_array(array, seq_count);
|
|
|
|
|
@@ -155,11 +161,11 @@ redo:
|
|
|
}
|
|
|
|
|
|
if (seq_count) {
|
|
|
- array = kzalloc(seq_count * sizeof(u32), GFP_KERNEL);
|
|
|
+ array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
|
|
|
if (!array) {
|
|
|
pr_err("Unable to allocate memory for"
|
|
|
" random array.\n");
|
|
|
- return -1;
|
|
|
+ return -ENOMEM;
|
|
|
}
|
|
|
iscsit_create_random_array(array, seq_count);
|
|
|
|
|
@@ -187,10 +193,10 @@ static int iscsit_randomize_seq_lists(
|
|
|
if (!seq_count)
|
|
|
return 0;
|
|
|
|
|
|
- array = kzalloc(seq_count * sizeof(u32), GFP_KERNEL);
|
|
|
+ array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
|
|
|
if (!array) {
|
|
|
pr_err("Unable to allocate memory for random array.\n");
|
|
|
- return -1;
|
|
|
+ return -ENOMEM;
|
|
|
}
|
|
|
iscsit_create_random_array(array, seq_count);
|
|
|
|
|
@@ -221,11 +227,10 @@ static void iscsit_determine_counts_for_list(
|
|
|
|
|
|
if ((bl->type == PDULIST_UNSOLICITED) ||
|
|
|
(bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
|
|
|
- unsolicited_data_length = (cmd->data_length >
|
|
|
- conn->sess->sess_ops->FirstBurstLength) ?
|
|
|
- conn->sess->sess_ops->FirstBurstLength : cmd->data_length;
|
|
|
+ unsolicited_data_length = min(cmd->se_cmd.data_length,
|
|
|
+ conn->sess->sess_ops->FirstBurstLength);
|
|
|
|
|
|
- while (offset < cmd->data_length) {
|
|
|
+ while (offset < cmd->se_cmd.data_length) {
|
|
|
*pdu_count += 1;
|
|
|
|
|
|
if (check_immediate) {
|
|
@@ -239,10 +244,10 @@ static void iscsit_determine_counts_for_list(
|
|
|
}
|
|
|
if (unsolicited_data_length > 0) {
|
|
|
if ((offset + conn->conn_ops->MaxRecvDataSegmentLength)
|
|
|
- >= cmd->data_length) {
|
|
|
+ >= cmd->se_cmd.data_length) {
|
|
|
unsolicited_data_length -=
|
|
|
- (cmd->data_length - offset);
|
|
|
- offset += (cmd->data_length - offset);
|
|
|
+ (cmd->se_cmd.data_length - offset);
|
|
|
+ offset += (cmd->se_cmd.data_length - offset);
|
|
|
continue;
|
|
|
}
|
|
|
if ((offset + conn->conn_ops->MaxRecvDataSegmentLength)
|
|
@@ -263,8 +268,8 @@ static void iscsit_determine_counts_for_list(
|
|
|
continue;
|
|
|
}
|
|
|
if ((offset + conn->conn_ops->MaxRecvDataSegmentLength) >=
|
|
|
- cmd->data_length) {
|
|
|
- offset += (cmd->data_length - offset);
|
|
|
+ cmd->se_cmd.data_length) {
|
|
|
+ offset += (cmd->se_cmd.data_length - offset);
|
|
|
continue;
|
|
|
}
|
|
|
if ((burstlength + conn->conn_ops->MaxRecvDataSegmentLength) >=
|
|
@@ -283,10 +288,10 @@ static void iscsit_determine_counts_for_list(
|
|
|
|
|
|
|
|
|
/*
|
|
|
- * Builds PDU and/or Sequence list, called while DataSequenceInOrder=No
|
|
|
- * and DataPDUInOrder=No.
|
|
|
+ * Builds PDU and/or Sequence list, called while DataSequenceInOrder=No
|
|
|
+ * or DataPDUInOrder=No.
|
|
|
*/
|
|
|
-static int iscsit_build_pdu_and_seq_list(
|
|
|
+static int iscsit_do_build_pdu_and_seq_lists(
|
|
|
struct iscsi_cmd *cmd,
|
|
|
struct iscsi_build_list *bl)
|
|
|
{
|
|
@@ -306,11 +311,10 @@ static int iscsit_build_pdu_and_seq_list(
|
|
|
|
|
|
if ((bl->type == PDULIST_UNSOLICITED) ||
|
|
|
(bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
|
|
|
- unsolicited_data_length = (cmd->data_length >
|
|
|
- conn->sess->sess_ops->FirstBurstLength) ?
|
|
|
- conn->sess->sess_ops->FirstBurstLength : cmd->data_length;
|
|
|
+ unsolicited_data_length = min(cmd->se_cmd.data_length,
|
|
|
+ conn->sess->sess_ops->FirstBurstLength);
|
|
|
|
|
|
- while (offset < cmd->data_length) {
|
|
|
+ while (offset < cmd->se_cmd.data_length) {
|
|
|
pdu_count++;
|
|
|
if (!datapduinorder) {
|
|
|
pdu[i].offset = offset;
|
|
@@ -346,21 +350,21 @@ static int iscsit_build_pdu_and_seq_list(
|
|
|
if (unsolicited_data_length > 0) {
|
|
|
if ((offset +
|
|
|
conn->conn_ops->MaxRecvDataSegmentLength) >=
|
|
|
- cmd->data_length) {
|
|
|
+ cmd->se_cmd.data_length) {
|
|
|
if (!datapduinorder) {
|
|
|
pdu[i].type = PDUTYPE_UNSOLICITED;
|
|
|
pdu[i].length =
|
|
|
- (cmd->data_length - offset);
|
|
|
+ (cmd->se_cmd.data_length - offset);
|
|
|
}
|
|
|
if (!datasequenceinorder) {
|
|
|
seq[seq_no].type = SEQTYPE_UNSOLICITED;
|
|
|
seq[seq_no].pdu_count = pdu_count;
|
|
|
seq[seq_no].xfer_len = (burstlength +
|
|
|
- (cmd->data_length - offset));
|
|
|
+ (cmd->se_cmd.data_length - offset));
|
|
|
}
|
|
|
unsolicited_data_length -=
|
|
|
- (cmd->data_length - offset);
|
|
|
- offset += (cmd->data_length - offset);
|
|
|
+ (cmd->se_cmd.data_length - offset);
|
|
|
+ offset += (cmd->se_cmd.data_length - offset);
|
|
|
continue;
|
|
|
}
|
|
|
if ((offset +
|
|
@@ -402,18 +406,18 @@ static int iscsit_build_pdu_and_seq_list(
|
|
|
continue;
|
|
|
}
|
|
|
if ((offset + conn->conn_ops->MaxRecvDataSegmentLength) >=
|
|
|
- cmd->data_length) {
|
|
|
+ cmd->se_cmd.data_length) {
|
|
|
if (!datapduinorder) {
|
|
|
pdu[i].type = PDUTYPE_NORMAL;
|
|
|
- pdu[i].length = (cmd->data_length - offset);
|
|
|
+ pdu[i].length = (cmd->se_cmd.data_length - offset);
|
|
|
}
|
|
|
if (!datasequenceinorder) {
|
|
|
seq[seq_no].type = SEQTYPE_NORMAL;
|
|
|
seq[seq_no].pdu_count = pdu_count;
|
|
|
seq[seq_no].xfer_len = (burstlength +
|
|
|
- (cmd->data_length - offset));
|
|
|
+ (cmd->se_cmd.data_length - offset));
|
|
|
}
|
|
|
- offset += (cmd->data_length - offset);
|
|
|
+ offset += (cmd->se_cmd.data_length - offset);
|
|
|
continue;
|
|
|
}
|
|
|
if ((burstlength + conn->conn_ops->MaxRecvDataSegmentLength) >=
|
|
@@ -464,9 +468,8 @@ static int iscsit_build_pdu_and_seq_list(
|
|
|
} else
|
|
|
iscsit_ordered_seq_lists(cmd, bl->type);
|
|
|
}
|
|
|
-#if 0
|
|
|
+
|
|
|
iscsit_dump_seq_list(cmd);
|
|
|
-#endif
|
|
|
}
|
|
|
if (!datapduinorder) {
|
|
|
if (bl->data_direction & ISCSI_PDU_WRITE) {
|
|
@@ -484,50 +487,86 @@ static int iscsit_build_pdu_and_seq_list(
|
|
|
} else
|
|
|
iscsit_ordered_pdu_lists(cmd, bl->type);
|
|
|
}
|
|
|
-#if 0
|
|
|
+
|
|
|
iscsit_dump_pdu_list(cmd);
|
|
|
-#endif
|
|
|
}
|
|
|
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-/*
|
|
|
- * Only called while DataSequenceInOrder=No or DataPDUInOrder=No.
|
|
|
- */
|
|
|
-int iscsit_do_build_list(
|
|
|
+int iscsit_build_pdu_and_seq_lists(
|
|
|
struct iscsi_cmd *cmd,
|
|
|
- struct iscsi_build_list *bl)
|
|
|
+ u32 immediate_data_length)
|
|
|
{
|
|
|
+ struct iscsi_build_list bl;
|
|
|
u32 pdu_count = 0, seq_count = 1;
|
|
|
struct iscsi_conn *conn = cmd->conn;
|
|
|
struct iscsi_pdu *pdu = NULL;
|
|
|
struct iscsi_seq *seq = NULL;
|
|
|
|
|
|
- iscsit_determine_counts_for_list(cmd, bl, &seq_count, &pdu_count);
|
|
|
+ struct iscsi_session *sess = conn->sess;
|
|
|
+ struct iscsi_node_attrib *na;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Do nothing if no OOO shenanigans
|
|
|
+ */
|
|
|
+ if (sess->sess_ops->DataSequenceInOrder &&
|
|
|
+ sess->sess_ops->DataPDUInOrder)
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ if (cmd->data_direction == DMA_NONE)
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ na = iscsit_tpg_get_node_attrib(sess);
|
|
|
+ memset(&bl, 0, sizeof(struct iscsi_build_list));
|
|
|
+
|
|
|
+ if (cmd->data_direction == DMA_FROM_DEVICE) {
|
|
|
+ bl.data_direction = ISCSI_PDU_READ;
|
|
|
+ bl.type = PDULIST_NORMAL;
|
|
|
+ if (na->random_datain_pdu_offsets)
|
|
|
+ bl.randomize |= RANDOM_DATAIN_PDU_OFFSETS;
|
|
|
+ if (na->random_datain_seq_offsets)
|
|
|
+ bl.randomize |= RANDOM_DATAIN_SEQ_OFFSETS;
|
|
|
+ } else {
|
|
|
+ bl.data_direction = ISCSI_PDU_WRITE;
|
|
|
+ bl.immediate_data_length = immediate_data_length;
|
|
|
+ if (na->random_r2t_offsets)
|
|
|
+ bl.randomize |= RANDOM_R2T_OFFSETS;
|
|
|
+
|
|
|
+ if (!cmd->immediate_data && !cmd->unsolicited_data)
|
|
|
+ bl.type = PDULIST_NORMAL;
|
|
|
+ else if (cmd->immediate_data && !cmd->unsolicited_data)
|
|
|
+ bl.type = PDULIST_IMMEDIATE;
|
|
|
+ else if (!cmd->immediate_data && cmd->unsolicited_data)
|
|
|
+ bl.type = PDULIST_UNSOLICITED;
|
|
|
+ else if (cmd->immediate_data && cmd->unsolicited_data)
|
|
|
+ bl.type = PDULIST_IMMEDIATE_AND_UNSOLICITED;
|
|
|
+ }
|
|
|
+
|
|
|
+ iscsit_determine_counts_for_list(cmd, &bl, &seq_count, &pdu_count);
|
|
|
|
|
|
if (!conn->sess->sess_ops->DataSequenceInOrder) {
|
|
|
- seq = kzalloc(seq_count * sizeof(struct iscsi_seq), GFP_ATOMIC);
|
|
|
+ seq = kcalloc(seq_count, sizeof(struct iscsi_seq), GFP_ATOMIC);
|
|
|
if (!seq) {
|
|
|
pr_err("Unable to allocate struct iscsi_seq list\n");
|
|
|
- return -1;
|
|
|
+ return -ENOMEM;
|
|
|
}
|
|
|
cmd->seq_list = seq;
|
|
|
cmd->seq_count = seq_count;
|
|
|
}
|
|
|
|
|
|
if (!conn->sess->sess_ops->DataPDUInOrder) {
|
|
|
- pdu = kzalloc(pdu_count * sizeof(struct iscsi_pdu), GFP_ATOMIC);
|
|
|
+ pdu = kcalloc(pdu_count, sizeof(struct iscsi_pdu), GFP_ATOMIC);
|
|
|
if (!pdu) {
|
|
|
pr_err("Unable to allocate struct iscsi_pdu list.\n");
|
|
|
kfree(seq);
|
|
|
- return -1;
|
|
|
+ return -ENOMEM;
|
|
|
}
|
|
|
cmd->pdu_list = pdu;
|
|
|
cmd->pdu_count = pdu_count;
|
|
|
}
|
|
|
|
|
|
- return iscsit_build_pdu_and_seq_list(cmd, bl);
|
|
|
+ return iscsit_do_build_pdu_and_seq_lists(cmd, &bl);
|
|
|
}
|
|
|
|
|
|
struct iscsi_pdu *iscsit_get_pdu_holder(
|
|
@@ -572,13 +611,12 @@ redo:
|
|
|
pdu = &cmd->pdu_list[cmd->pdu_start];
|
|
|
|
|
|
for (i = 0; pdu[i].seq_no != cmd->seq_no; i++) {
|
|
|
-#if 0
|
|
|
pr_debug("pdu[i].seq_no: %d, pdu[i].pdu"
|
|
|
"_send_order: %d, pdu[i].offset: %d,"
|
|
|
" pdu[i].length: %d\n", pdu[i].seq_no,
|
|
|
pdu[i].pdu_send_order, pdu[i].offset,
|
|
|
pdu[i].length);
|
|
|
-#endif
|
|
|
+
|
|
|
if (pdu[i].pdu_send_order == cmd->pdu_send_order) {
|
|
|
cmd->pdu_send_order++;
|
|
|
return &pdu[i];
|
|
@@ -601,11 +639,11 @@ redo:
|
|
|
pr_err("struct iscsi_seq is NULL!\n");
|
|
|
return NULL;
|
|
|
}
|
|
|
-#if 0
|
|
|
+
|
|
|
pr_debug("seq->pdu_start: %d, seq->pdu_count: %d,"
|
|
|
" seq->seq_no: %d\n", seq->pdu_start, seq->pdu_count,
|
|
|
seq->seq_no);
|
|
|
-#endif
|
|
|
+
|
|
|
pdu = &cmd->pdu_list[seq->pdu_start];
|
|
|
|
|
|
if (seq->pdu_send_order == seq->pdu_count) {
|
|
@@ -645,12 +683,11 @@ struct iscsi_seq *iscsit_get_seq_holder(
|
|
|
}
|
|
|
|
|
|
for (i = 0; i < cmd->seq_count; i++) {
|
|
|
-#if 0
|
|
|
pr_debug("seq_list[i].orig_offset: %d, seq_list[i]."
|
|
|
"xfer_len: %d, seq_list[i].seq_no %u\n",
|
|
|
cmd->seq_list[i].orig_offset, cmd->seq_list[i].xfer_len,
|
|
|
cmd->seq_list[i].seq_no);
|
|
|
-#endif
|
|
|
+
|
|
|
if ((cmd->seq_list[i].orig_offset +
|
|
|
cmd->seq_list[i].xfer_len) >=
|
|
|
(offset + length))
|