|
@@ -1578,6 +1578,45 @@ static inline int mthca_wq_overflow(struct mthca_wq *wq, int nreq,
|
|
|
return cur + nreq >= wq->max;
|
|
|
}
|
|
|
|
|
|
+static __always_inline void set_raddr_seg(struct mthca_raddr_seg *rseg,
|
|
|
+ u64 remote_addr, u32 rkey)
|
|
|
+{
|
|
|
+ rseg->raddr = cpu_to_be64(remote_addr);
|
|
|
+ rseg->rkey = cpu_to_be32(rkey);
|
|
|
+ rseg->reserved = 0;
|
|
|
+}
|
|
|
+
|
|
|
+static __always_inline void set_atomic_seg(struct mthca_atomic_seg *aseg,
|
|
|
+ struct ib_send_wr *wr)
|
|
|
+{
|
|
|
+ if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
|
|
|
+ aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap);
|
|
|
+ aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add);
|
|
|
+ } else {
|
|
|
+ aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add);
|
|
|
+ aseg->compare = 0;
|
|
|
+ }
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+static void set_tavor_ud_seg(struct mthca_tavor_ud_seg *useg,
|
|
|
+ struct ib_send_wr *wr)
|
|
|
+{
|
|
|
+ useg->lkey = cpu_to_be32(to_mah(wr->wr.ud.ah)->key);
|
|
|
+ useg->av_addr = cpu_to_be64(to_mah(wr->wr.ud.ah)->avdma);
|
|
|
+ useg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn);
|
|
|
+ useg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey);
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+static void set_arbel_ud_seg(struct mthca_arbel_ud_seg *useg,
|
|
|
+ struct ib_send_wr *wr)
|
|
|
+{
|
|
|
+ memcpy(useg->av, to_mah(wr->wr.ud.ah)->av, MTHCA_AV_SIZE);
|
|
|
+ useg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn);
|
|
|
+ useg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey);
|
|
|
+}
|
|
|
+
|
|
|
int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
struct ib_send_wr **bad_wr)
|
|
|
{
|
|
@@ -1590,8 +1629,15 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
int nreq;
|
|
|
int i;
|
|
|
int size;
|
|
|
- int size0 = 0;
|
|
|
- u32 f0 = 0;
|
|
|
+ /*
|
|
|
+ * f0 and size0 are only used if nreq != 0, and they will
|
|
|
+ * always be initialized the first time through the main loop
|
|
|
+ * before nreq is incremented. So nreq cannot become non-zero
|
|
|
+ * without initializing f0 and size0, and they are in fact
|
|
|
+ * never used uninitialized.
|
|
|
+ */
|
|
|
+ int uninitialized_var(size0);
|
|
|
+ u32 uninitialized_var(f0);
|
|
|
int ind;
|
|
|
u8 op0 = 0;
|
|
|
|
|
@@ -1636,25 +1682,11 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
switch (wr->opcode) {
|
|
|
case IB_WR_ATOMIC_CMP_AND_SWP:
|
|
|
case IB_WR_ATOMIC_FETCH_AND_ADD:
|
|
|
- ((struct mthca_raddr_seg *) wqe)->raddr =
|
|
|
- cpu_to_be64(wr->wr.atomic.remote_addr);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->rkey =
|
|
|
- cpu_to_be32(wr->wr.atomic.rkey);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->reserved = 0;
|
|
|
-
|
|
|
+ set_raddr_seg(wqe, wr->wr.atomic.remote_addr,
|
|
|
+ wr->wr.atomic.rkey);
|
|
|
wqe += sizeof (struct mthca_raddr_seg);
|
|
|
|
|
|
- if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
|
|
|
- ((struct mthca_atomic_seg *) wqe)->swap_add =
|
|
|
- cpu_to_be64(wr->wr.atomic.swap);
|
|
|
- ((struct mthca_atomic_seg *) wqe)->compare =
|
|
|
- cpu_to_be64(wr->wr.atomic.compare_add);
|
|
|
- } else {
|
|
|
- ((struct mthca_atomic_seg *) wqe)->swap_add =
|
|
|
- cpu_to_be64(wr->wr.atomic.compare_add);
|
|
|
- ((struct mthca_atomic_seg *) wqe)->compare = 0;
|
|
|
- }
|
|
|
-
|
|
|
+ set_atomic_seg(wqe, wr);
|
|
|
wqe += sizeof (struct mthca_atomic_seg);
|
|
|
size += (sizeof (struct mthca_raddr_seg) +
|
|
|
sizeof (struct mthca_atomic_seg)) / 16;
|
|
@@ -1663,12 +1695,9 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
case IB_WR_RDMA_WRITE:
|
|
|
case IB_WR_RDMA_WRITE_WITH_IMM:
|
|
|
case IB_WR_RDMA_READ:
|
|
|
- ((struct mthca_raddr_seg *) wqe)->raddr =
|
|
|
- cpu_to_be64(wr->wr.rdma.remote_addr);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->rkey =
|
|
|
- cpu_to_be32(wr->wr.rdma.rkey);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->reserved = 0;
|
|
|
- wqe += sizeof (struct mthca_raddr_seg);
|
|
|
+ set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
|
|
|
+ wr->wr.rdma.rkey);
|
|
|
+ wqe += sizeof (struct mthca_raddr_seg);
|
|
|
size += sizeof (struct mthca_raddr_seg) / 16;
|
|
|
break;
|
|
|
|
|
@@ -1683,12 +1712,9 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
switch (wr->opcode) {
|
|
|
case IB_WR_RDMA_WRITE:
|
|
|
case IB_WR_RDMA_WRITE_WITH_IMM:
|
|
|
- ((struct mthca_raddr_seg *) wqe)->raddr =
|
|
|
- cpu_to_be64(wr->wr.rdma.remote_addr);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->rkey =
|
|
|
- cpu_to_be32(wr->wr.rdma.rkey);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->reserved = 0;
|
|
|
- wqe += sizeof (struct mthca_raddr_seg);
|
|
|
+ set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
|
|
|
+ wr->wr.rdma.rkey);
|
|
|
+ wqe += sizeof (struct mthca_raddr_seg);
|
|
|
size += sizeof (struct mthca_raddr_seg) / 16;
|
|
|
break;
|
|
|
|
|
@@ -1700,16 +1726,8 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
break;
|
|
|
|
|
|
case UD:
|
|
|
- ((struct mthca_tavor_ud_seg *) wqe)->lkey =
|
|
|
- cpu_to_be32(to_mah(wr->wr.ud.ah)->key);
|
|
|
- ((struct mthca_tavor_ud_seg *) wqe)->av_addr =
|
|
|
- cpu_to_be64(to_mah(wr->wr.ud.ah)->avdma);
|
|
|
- ((struct mthca_tavor_ud_seg *) wqe)->dqpn =
|
|
|
- cpu_to_be32(wr->wr.ud.remote_qpn);
|
|
|
- ((struct mthca_tavor_ud_seg *) wqe)->qkey =
|
|
|
- cpu_to_be32(wr->wr.ud.remote_qkey);
|
|
|
-
|
|
|
- wqe += sizeof (struct mthca_tavor_ud_seg);
|
|
|
+ set_tavor_ud_seg(wqe, wr);
|
|
|
+ wqe += sizeof (struct mthca_tavor_ud_seg);
|
|
|
size += sizeof (struct mthca_tavor_ud_seg) / 16;
|
|
|
break;
|
|
|
|
|
@@ -1734,13 +1752,8 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
}
|
|
|
|
|
|
for (i = 0; i < wr->num_sge; ++i) {
|
|
|
- ((struct mthca_data_seg *) wqe)->byte_count =
|
|
|
- cpu_to_be32(wr->sg_list[i].length);
|
|
|
- ((struct mthca_data_seg *) wqe)->lkey =
|
|
|
- cpu_to_be32(wr->sg_list[i].lkey);
|
|
|
- ((struct mthca_data_seg *) wqe)->addr =
|
|
|
- cpu_to_be64(wr->sg_list[i].addr);
|
|
|
- wqe += sizeof (struct mthca_data_seg);
|
|
|
+ mthca_set_data_seg(wqe, wr->sg_list + i);
|
|
|
+ wqe += sizeof (struct mthca_data_seg);
|
|
|
size += sizeof (struct mthca_data_seg) / 16;
|
|
|
}
|
|
|
|
|
@@ -1768,11 +1781,11 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
mthca_opcode[wr->opcode]);
|
|
|
wmb();
|
|
|
((struct mthca_next_seg *) prev_wqe)->ee_nds =
|
|
|
- cpu_to_be32((size0 ? 0 : MTHCA_NEXT_DBD) | size |
|
|
|
+ cpu_to_be32((nreq ? 0 : MTHCA_NEXT_DBD) | size |
|
|
|
((wr->send_flags & IB_SEND_FENCE) ?
|
|
|
MTHCA_NEXT_FENCE : 0));
|
|
|
|
|
|
- if (!size0) {
|
|
|
+ if (!nreq) {
|
|
|
size0 = size;
|
|
|
op0 = mthca_opcode[wr->opcode];
|
|
|
f0 = wr->send_flags & IB_SEND_FENCE ?
|
|
@@ -1822,7 +1835,14 @@ int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
|
|
|
int nreq;
|
|
|
int i;
|
|
|
int size;
|
|
|
- int size0 = 0;
|
|
|
+ /*
|
|
|
+ * size0 is only used if nreq != 0, and it will always be
|
|
|
+ * initialized the first time through the main loop before
|
|
|
+ * nreq is incremented. So nreq cannot become non-zero
|
|
|
+ * without initializing size0, and it is in fact never used
|
|
|
+ * uninitialized.
|
|
|
+ */
|
|
|
+ int uninitialized_var(size0);
|
|
|
int ind;
|
|
|
void *wqe;
|
|
|
void *prev_wqe;
|
|
@@ -1863,13 +1883,8 @@ int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
|
|
|
}
|
|
|
|
|
|
for (i = 0; i < wr->num_sge; ++i) {
|
|
|
- ((struct mthca_data_seg *) wqe)->byte_count =
|
|
|
- cpu_to_be32(wr->sg_list[i].length);
|
|
|
- ((struct mthca_data_seg *) wqe)->lkey =
|
|
|
- cpu_to_be32(wr->sg_list[i].lkey);
|
|
|
- ((struct mthca_data_seg *) wqe)->addr =
|
|
|
- cpu_to_be64(wr->sg_list[i].addr);
|
|
|
- wqe += sizeof (struct mthca_data_seg);
|
|
|
+ mthca_set_data_seg(wqe, wr->sg_list + i);
|
|
|
+ wqe += sizeof (struct mthca_data_seg);
|
|
|
size += sizeof (struct mthca_data_seg) / 16;
|
|
|
}
|
|
|
|
|
@@ -1881,7 +1896,7 @@ int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
|
|
|
((struct mthca_next_seg *) prev_wqe)->ee_nds =
|
|
|
cpu_to_be32(MTHCA_NEXT_DBD | size);
|
|
|
|
|
|
- if (!size0)
|
|
|
+ if (!nreq)
|
|
|
size0 = size;
|
|
|
|
|
|
++ind;
|
|
@@ -1903,7 +1918,6 @@ int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
|
|
|
|
|
|
qp->rq.next_ind = ind;
|
|
|
qp->rq.head += MTHCA_TAVOR_MAX_WQES_PER_RECV_DB;
|
|
|
- size0 = 0;
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -1945,8 +1959,15 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
int nreq;
|
|
|
int i;
|
|
|
int size;
|
|
|
- int size0 = 0;
|
|
|
- u32 f0 = 0;
|
|
|
+ /*
|
|
|
+ * f0 and size0 are only used if nreq != 0, and they will
|
|
|
+ * always be initialized the first time through the main loop
|
|
|
+ * before nreq is incremented. So nreq cannot become non-zero
|
|
|
+ * without initializing f0 and size0, and they are in fact
|
|
|
+ * never used uninitialized.
|
|
|
+ */
|
|
|
+ int uninitialized_var(size0);
|
|
|
+ u32 uninitialized_var(f0);
|
|
|
int ind;
|
|
|
u8 op0 = 0;
|
|
|
|
|
@@ -1966,7 +1987,6 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
doorbell[1] = cpu_to_be32((qp->qpn << 8) | size0);
|
|
|
|
|
|
qp->sq.head += MTHCA_ARBEL_MAX_WQES_PER_SEND_DB;
|
|
|
- size0 = 0;
|
|
|
|
|
|
/*
|
|
|
* Make sure that descriptors are written before
|
|
@@ -2017,26 +2037,12 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
switch (wr->opcode) {
|
|
|
case IB_WR_ATOMIC_CMP_AND_SWP:
|
|
|
case IB_WR_ATOMIC_FETCH_AND_ADD:
|
|
|
- ((struct mthca_raddr_seg *) wqe)->raddr =
|
|
|
- cpu_to_be64(wr->wr.atomic.remote_addr);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->rkey =
|
|
|
- cpu_to_be32(wr->wr.atomic.rkey);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->reserved = 0;
|
|
|
-
|
|
|
+ set_raddr_seg(wqe, wr->wr.atomic.remote_addr,
|
|
|
+ wr->wr.atomic.rkey);
|
|
|
wqe += sizeof (struct mthca_raddr_seg);
|
|
|
|
|
|
- if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
|
|
|
- ((struct mthca_atomic_seg *) wqe)->swap_add =
|
|
|
- cpu_to_be64(wr->wr.atomic.swap);
|
|
|
- ((struct mthca_atomic_seg *) wqe)->compare =
|
|
|
- cpu_to_be64(wr->wr.atomic.compare_add);
|
|
|
- } else {
|
|
|
- ((struct mthca_atomic_seg *) wqe)->swap_add =
|
|
|
- cpu_to_be64(wr->wr.atomic.compare_add);
|
|
|
- ((struct mthca_atomic_seg *) wqe)->compare = 0;
|
|
|
- }
|
|
|
-
|
|
|
- wqe += sizeof (struct mthca_atomic_seg);
|
|
|
+ set_atomic_seg(wqe, wr);
|
|
|
+ wqe += sizeof (struct mthca_atomic_seg);
|
|
|
size += (sizeof (struct mthca_raddr_seg) +
|
|
|
sizeof (struct mthca_atomic_seg)) / 16;
|
|
|
break;
|
|
@@ -2044,12 +2050,9 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
case IB_WR_RDMA_READ:
|
|
|
case IB_WR_RDMA_WRITE:
|
|
|
case IB_WR_RDMA_WRITE_WITH_IMM:
|
|
|
- ((struct mthca_raddr_seg *) wqe)->raddr =
|
|
|
- cpu_to_be64(wr->wr.rdma.remote_addr);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->rkey =
|
|
|
- cpu_to_be32(wr->wr.rdma.rkey);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->reserved = 0;
|
|
|
- wqe += sizeof (struct mthca_raddr_seg);
|
|
|
+ set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
|
|
|
+ wr->wr.rdma.rkey);
|
|
|
+ wqe += sizeof (struct mthca_raddr_seg);
|
|
|
size += sizeof (struct mthca_raddr_seg) / 16;
|
|
|
break;
|
|
|
|
|
@@ -2064,12 +2067,9 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
switch (wr->opcode) {
|
|
|
case IB_WR_RDMA_WRITE:
|
|
|
case IB_WR_RDMA_WRITE_WITH_IMM:
|
|
|
- ((struct mthca_raddr_seg *) wqe)->raddr =
|
|
|
- cpu_to_be64(wr->wr.rdma.remote_addr);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->rkey =
|
|
|
- cpu_to_be32(wr->wr.rdma.rkey);
|
|
|
- ((struct mthca_raddr_seg *) wqe)->reserved = 0;
|
|
|
- wqe += sizeof (struct mthca_raddr_seg);
|
|
|
+ set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
|
|
|
+ wr->wr.rdma.rkey);
|
|
|
+ wqe += sizeof (struct mthca_raddr_seg);
|
|
|
size += sizeof (struct mthca_raddr_seg) / 16;
|
|
|
break;
|
|
|
|
|
@@ -2081,14 +2081,8 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
break;
|
|
|
|
|
|
case UD:
|
|
|
- memcpy(((struct mthca_arbel_ud_seg *) wqe)->av,
|
|
|
- to_mah(wr->wr.ud.ah)->av, MTHCA_AV_SIZE);
|
|
|
- ((struct mthca_arbel_ud_seg *) wqe)->dqpn =
|
|
|
- cpu_to_be32(wr->wr.ud.remote_qpn);
|
|
|
- ((struct mthca_arbel_ud_seg *) wqe)->qkey =
|
|
|
- cpu_to_be32(wr->wr.ud.remote_qkey);
|
|
|
-
|
|
|
- wqe += sizeof (struct mthca_arbel_ud_seg);
|
|
|
+ set_arbel_ud_seg(wqe, wr);
|
|
|
+ wqe += sizeof (struct mthca_arbel_ud_seg);
|
|
|
size += sizeof (struct mthca_arbel_ud_seg) / 16;
|
|
|
break;
|
|
|
|
|
@@ -2113,13 +2107,8 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
}
|
|
|
|
|
|
for (i = 0; i < wr->num_sge; ++i) {
|
|
|
- ((struct mthca_data_seg *) wqe)->byte_count =
|
|
|
- cpu_to_be32(wr->sg_list[i].length);
|
|
|
- ((struct mthca_data_seg *) wqe)->lkey =
|
|
|
- cpu_to_be32(wr->sg_list[i].lkey);
|
|
|
- ((struct mthca_data_seg *) wqe)->addr =
|
|
|
- cpu_to_be64(wr->sg_list[i].addr);
|
|
|
- wqe += sizeof (struct mthca_data_seg);
|
|
|
+ mthca_set_data_seg(wqe, wr->sg_list + i);
|
|
|
+ wqe += sizeof (struct mthca_data_seg);
|
|
|
size += sizeof (struct mthca_data_seg) / 16;
|
|
|
}
|
|
|
|
|
@@ -2151,7 +2140,7 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
|
|
|
((wr->send_flags & IB_SEND_FENCE) ?
|
|
|
MTHCA_NEXT_FENCE : 0));
|
|
|
|
|
|
- if (!size0) {
|
|
|
+ if (!nreq) {
|
|
|
size0 = size;
|
|
|
op0 = mthca_opcode[wr->opcode];
|
|
|
f0 = wr->send_flags & IB_SEND_FENCE ?
|
|
@@ -2241,20 +2230,12 @@ int mthca_arbel_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
|
|
|
}
|
|
|
|
|
|
for (i = 0; i < wr->num_sge; ++i) {
|
|
|
- ((struct mthca_data_seg *) wqe)->byte_count =
|
|
|
- cpu_to_be32(wr->sg_list[i].length);
|
|
|
- ((struct mthca_data_seg *) wqe)->lkey =
|
|
|
- cpu_to_be32(wr->sg_list[i].lkey);
|
|
|
- ((struct mthca_data_seg *) wqe)->addr =
|
|
|
- cpu_to_be64(wr->sg_list[i].addr);
|
|
|
+ mthca_set_data_seg(wqe, wr->sg_list + i);
|
|
|
wqe += sizeof (struct mthca_data_seg);
|
|
|
}
|
|
|
|
|
|
- if (i < qp->rq.max_gs) {
|
|
|
- ((struct mthca_data_seg *) wqe)->byte_count = 0;
|
|
|
- ((struct mthca_data_seg *) wqe)->lkey = cpu_to_be32(MTHCA_INVAL_LKEY);
|
|
|
- ((struct mthca_data_seg *) wqe)->addr = 0;
|
|
|
- }
|
|
|
+ if (i < qp->rq.max_gs)
|
|
|
+ mthca_set_data_seg_inval(wqe);
|
|
|
|
|
|
qp->wrid[ind] = wr->wr_id;
|
|
|
|