ipath_qp.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008
  1. /*
  2. * Copyright (c) 2006 QLogic, Inc. All rights reserved.
  3. * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
  4. *
  5. * This software is available to you under a choice of one of two
  6. * licenses. You may choose to be licensed under the terms of the GNU
  7. * General Public License (GPL) Version 2, available from the file
  8. * COPYING in the main directory of this source tree, or the
  9. * OpenIB.org BSD license below:
  10. *
  11. * Redistribution and use in source and binary forms, with or
  12. * without modification, are permitted provided that the following
  13. * conditions are met:
  14. *
  15. * - Redistributions of source code must retain the above
  16. * copyright notice, this list of conditions and the following
  17. * disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials
  22. * provided with the distribution.
  23. *
  24. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31. * SOFTWARE.
  32. */
  33. #include <linux/err.h>
  34. #include <linux/vmalloc.h>
  35. #include "ipath_verbs.h"
  36. #include "ipath_kernel.h"
  37. #define BITS_PER_PAGE (PAGE_SIZE*BITS_PER_BYTE)
  38. #define BITS_PER_PAGE_MASK (BITS_PER_PAGE-1)
  39. #define mk_qpn(qpt, map, off) (((map) - (qpt)->map) * BITS_PER_PAGE + \
  40. (off))
  41. #define find_next_offset(map, off) find_next_zero_bit((map)->page, \
  42. BITS_PER_PAGE, off)
  43. /*
  44. * Convert the AETH credit code into the number of credits.
  45. */
  46. static u32 credit_table[31] = {
  47. 0, /* 0 */
  48. 1, /* 1 */
  49. 2, /* 2 */
  50. 3, /* 3 */
  51. 4, /* 4 */
  52. 6, /* 5 */
  53. 8, /* 6 */
  54. 12, /* 7 */
  55. 16, /* 8 */
  56. 24, /* 9 */
  57. 32, /* A */
  58. 48, /* B */
  59. 64, /* C */
  60. 96, /* D */
  61. 128, /* E */
  62. 192, /* F */
  63. 256, /* 10 */
  64. 384, /* 11 */
  65. 512, /* 12 */
  66. 768, /* 13 */
  67. 1024, /* 14 */
  68. 1536, /* 15 */
  69. 2048, /* 16 */
  70. 3072, /* 17 */
  71. 4096, /* 18 */
  72. 6144, /* 19 */
  73. 8192, /* 1A */
  74. 12288, /* 1B */
  75. 16384, /* 1C */
  76. 24576, /* 1D */
  77. 32768 /* 1E */
  78. };
  79. static u32 alloc_qpn(struct ipath_qp_table *qpt)
  80. {
  81. u32 i, offset, max_scan, qpn;
  82. struct qpn_map *map;
  83. u32 ret;
  84. qpn = qpt->last + 1;
  85. if (qpn >= QPN_MAX)
  86. qpn = 2;
  87. offset = qpn & BITS_PER_PAGE_MASK;
  88. map = &qpt->map[qpn / BITS_PER_PAGE];
  89. max_scan = qpt->nmaps - !offset;
  90. for (i = 0;;) {
  91. if (unlikely(!map->page)) {
  92. unsigned long page = get_zeroed_page(GFP_KERNEL);
  93. unsigned long flags;
  94. /*
  95. * Free the page if someone raced with us
  96. * installing it:
  97. */
  98. spin_lock_irqsave(&qpt->lock, flags);
  99. if (map->page)
  100. free_page(page);
  101. else
  102. map->page = (void *)page;
  103. spin_unlock_irqrestore(&qpt->lock, flags);
  104. if (unlikely(!map->page))
  105. break;
  106. }
  107. if (likely(atomic_read(&map->n_free))) {
  108. do {
  109. if (!test_and_set_bit(offset, map->page)) {
  110. atomic_dec(&map->n_free);
  111. qpt->last = qpn;
  112. ret = qpn;
  113. goto bail;
  114. }
  115. offset = find_next_offset(map, offset);
  116. qpn = mk_qpn(qpt, map, offset);
  117. /*
  118. * This test differs from alloc_pidmap().
  119. * If find_next_offset() does find a zero
  120. * bit, we don't need to check for QPN
  121. * wrapping around past our starting QPN.
  122. * We just need to be sure we don't loop
  123. * forever.
  124. */
  125. } while (offset < BITS_PER_PAGE && qpn < QPN_MAX);
  126. }
  127. /*
  128. * In order to keep the number of pages allocated to a
  129. * minimum, we scan the all existing pages before increasing
  130. * the size of the bitmap table.
  131. */
  132. if (++i > max_scan) {
  133. if (qpt->nmaps == QPNMAP_ENTRIES)
  134. break;
  135. map = &qpt->map[qpt->nmaps++];
  136. offset = 0;
  137. } else if (map < &qpt->map[qpt->nmaps]) {
  138. ++map;
  139. offset = 0;
  140. } else {
  141. map = &qpt->map[0];
  142. offset = 2;
  143. }
  144. qpn = mk_qpn(qpt, map, offset);
  145. }
  146. ret = 0;
  147. bail:
  148. return ret;
  149. }
  150. static void free_qpn(struct ipath_qp_table *qpt, u32 qpn)
  151. {
  152. struct qpn_map *map;
  153. map = qpt->map + qpn / BITS_PER_PAGE;
  154. if (map->page)
  155. clear_bit(qpn & BITS_PER_PAGE_MASK, map->page);
  156. atomic_inc(&map->n_free);
  157. }
  158. /**
  159. * ipath_alloc_qpn - allocate a QP number
  160. * @qpt: the QP table
  161. * @qp: the QP
  162. * @type: the QP type (IB_QPT_SMI and IB_QPT_GSI are special)
  163. *
  164. * Allocate the next available QPN and put the QP into the hash table.
  165. * The hash table holds a reference to the QP.
  166. */
  167. static int ipath_alloc_qpn(struct ipath_qp_table *qpt, struct ipath_qp *qp,
  168. enum ib_qp_type type)
  169. {
  170. unsigned long flags;
  171. u32 qpn;
  172. int ret;
  173. if (type == IB_QPT_SMI)
  174. qpn = 0;
  175. else if (type == IB_QPT_GSI)
  176. qpn = 1;
  177. else {
  178. /* Allocate the next available QPN */
  179. qpn = alloc_qpn(qpt);
  180. if (qpn == 0) {
  181. ret = -ENOMEM;
  182. goto bail;
  183. }
  184. }
  185. qp->ibqp.qp_num = qpn;
  186. /* Add the QP to the hash table. */
  187. spin_lock_irqsave(&qpt->lock, flags);
  188. qpn %= qpt->max;
  189. qp->next = qpt->table[qpn];
  190. qpt->table[qpn] = qp;
  191. atomic_inc(&qp->refcount);
  192. spin_unlock_irqrestore(&qpt->lock, flags);
  193. ret = 0;
  194. bail:
  195. return ret;
  196. }
  197. /**
  198. * ipath_free_qp - remove a QP from the QP table
  199. * @qpt: the QP table
  200. * @qp: the QP to remove
  201. *
  202. * Remove the QP from the table so it can't be found asynchronously by
  203. * the receive interrupt routine.
  204. */
  205. static void ipath_free_qp(struct ipath_qp_table *qpt, struct ipath_qp *qp)
  206. {
  207. struct ipath_qp *q, **qpp;
  208. unsigned long flags;
  209. int fnd = 0;
  210. spin_lock_irqsave(&qpt->lock, flags);
  211. /* Remove QP from the hash table. */
  212. qpp = &qpt->table[qp->ibqp.qp_num % qpt->max];
  213. for (; (q = *qpp) != NULL; qpp = &q->next) {
  214. if (q == qp) {
  215. *qpp = qp->next;
  216. qp->next = NULL;
  217. atomic_dec(&qp->refcount);
  218. fnd = 1;
  219. break;
  220. }
  221. }
  222. spin_unlock_irqrestore(&qpt->lock, flags);
  223. if (!fnd)
  224. return;
  225. /* If QPN is not reserved, mark QPN free in the bitmap. */
  226. if (qp->ibqp.qp_num > 1)
  227. free_qpn(qpt, qp->ibqp.qp_num);
  228. wait_event(qp->wait, !atomic_read(&qp->refcount));
  229. }
  230. /**
  231. * ipath_free_all_qps - remove all QPs from the table
  232. * @qpt: the QP table to empty
  233. */
  234. void ipath_free_all_qps(struct ipath_qp_table *qpt)
  235. {
  236. unsigned long flags;
  237. struct ipath_qp *qp, *nqp;
  238. u32 n;
  239. for (n = 0; n < qpt->max; n++) {
  240. spin_lock_irqsave(&qpt->lock, flags);
  241. qp = qpt->table[n];
  242. qpt->table[n] = NULL;
  243. spin_unlock_irqrestore(&qpt->lock, flags);
  244. while (qp) {
  245. nqp = qp->next;
  246. if (qp->ibqp.qp_num > 1)
  247. free_qpn(qpt, qp->ibqp.qp_num);
  248. if (!atomic_dec_and_test(&qp->refcount) ||
  249. !ipath_destroy_qp(&qp->ibqp))
  250. _VERBS_INFO("QP memory leak!\n");
  251. qp = nqp;
  252. }
  253. }
  254. for (n = 0; n < ARRAY_SIZE(qpt->map); n++) {
  255. if (qpt->map[n].page)
  256. free_page((unsigned long)qpt->map[n].page);
  257. }
  258. }
  259. /**
  260. * ipath_lookup_qpn - return the QP with the given QPN
  261. * @qpt: the QP table
  262. * @qpn: the QP number to look up
  263. *
  264. * The caller is responsible for decrementing the QP reference count
  265. * when done.
  266. */
  267. struct ipath_qp *ipath_lookup_qpn(struct ipath_qp_table *qpt, u32 qpn)
  268. {
  269. unsigned long flags;
  270. struct ipath_qp *qp;
  271. spin_lock_irqsave(&qpt->lock, flags);
  272. for (qp = qpt->table[qpn % qpt->max]; qp; qp = qp->next) {
  273. if (qp->ibqp.qp_num == qpn) {
  274. atomic_inc(&qp->refcount);
  275. break;
  276. }
  277. }
  278. spin_unlock_irqrestore(&qpt->lock, flags);
  279. return qp;
  280. }
  281. /**
  282. * ipath_reset_qp - initialize the QP state to the reset state
  283. * @qp: the QP to reset
  284. */
  285. static void ipath_reset_qp(struct ipath_qp *qp)
  286. {
  287. qp->remote_qpn = 0;
  288. qp->qkey = 0;
  289. qp->qp_access_flags = 0;
  290. clear_bit(IPATH_S_BUSY, &qp->s_flags);
  291. qp->s_hdrwords = 0;
  292. qp->s_psn = 0;
  293. qp->r_psn = 0;
  294. qp->r_msn = 0;
  295. if (qp->ibqp.qp_type == IB_QPT_RC) {
  296. qp->s_state = IB_OPCODE_RC_SEND_LAST;
  297. qp->r_state = IB_OPCODE_RC_SEND_LAST;
  298. } else {
  299. qp->s_state = IB_OPCODE_UC_SEND_LAST;
  300. qp->r_state = IB_OPCODE_UC_SEND_LAST;
  301. }
  302. qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
  303. qp->r_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
  304. qp->r_nak_state = 0;
  305. qp->s_rnr_timeout = 0;
  306. qp->s_head = 0;
  307. qp->s_tail = 0;
  308. qp->s_cur = 0;
  309. qp->s_last = 0;
  310. qp->s_ssn = 1;
  311. qp->s_lsn = 0;
  312. if (qp->r_rq.wq) {
  313. qp->r_rq.wq->head = 0;
  314. qp->r_rq.wq->tail = 0;
  315. }
  316. qp->r_reuse_sge = 0;
  317. }
  318. /**
  319. * ipath_error_qp - put a QP into an error state
  320. * @qp: the QP to put into an error state
  321. *
  322. * Flushes both send and receive work queues.
  323. * QP s_lock should be held and interrupts disabled.
  324. */
  325. void ipath_error_qp(struct ipath_qp *qp)
  326. {
  327. struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
  328. struct ib_wc wc;
  329. _VERBS_INFO("QP%d/%d in error state\n",
  330. qp->ibqp.qp_num, qp->remote_qpn);
  331. spin_lock(&dev->pending_lock);
  332. /* XXX What if its already removed by the timeout code? */
  333. if (!list_empty(&qp->timerwait))
  334. list_del_init(&qp->timerwait);
  335. if (!list_empty(&qp->piowait))
  336. list_del_init(&qp->piowait);
  337. spin_unlock(&dev->pending_lock);
  338. wc.status = IB_WC_WR_FLUSH_ERR;
  339. wc.vendor_err = 0;
  340. wc.byte_len = 0;
  341. wc.imm_data = 0;
  342. wc.qp_num = qp->ibqp.qp_num;
  343. wc.src_qp = 0;
  344. wc.wc_flags = 0;
  345. wc.pkey_index = 0;
  346. wc.slid = 0;
  347. wc.sl = 0;
  348. wc.dlid_path_bits = 0;
  349. wc.port_num = 0;
  350. while (qp->s_last != qp->s_head) {
  351. struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
  352. wc.wr_id = wqe->wr.wr_id;
  353. wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
  354. if (++qp->s_last >= qp->s_size)
  355. qp->s_last = 0;
  356. ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 1);
  357. }
  358. qp->s_cur = qp->s_tail = qp->s_head;
  359. qp->s_hdrwords = 0;
  360. qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
  361. if (qp->r_rq.wq) {
  362. struct ipath_rwq *wq;
  363. u32 head;
  364. u32 tail;
  365. spin_lock(&qp->r_rq.lock);
  366. /* sanity check pointers before trusting them */
  367. wq = qp->r_rq.wq;
  368. head = wq->head;
  369. if (head >= qp->r_rq.size)
  370. head = 0;
  371. tail = wq->tail;
  372. if (tail >= qp->r_rq.size)
  373. tail = 0;
  374. wc.opcode = IB_WC_RECV;
  375. while (tail != head) {
  376. wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id;
  377. if (++tail >= qp->r_rq.size)
  378. tail = 0;
  379. ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
  380. }
  381. wq->tail = tail;
  382. spin_unlock(&qp->r_rq.lock);
  383. }
  384. }
  385. /**
  386. * ipath_modify_qp - modify the attributes of a queue pair
  387. * @ibqp: the queue pair who's attributes we're modifying
  388. * @attr: the new attributes
  389. * @attr_mask: the mask of attributes to modify
  390. * @udata: user data for ipathverbs.so
  391. *
  392. * Returns 0 on success, otherwise returns an errno.
  393. */
  394. int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
  395. int attr_mask, struct ib_udata *udata)
  396. {
  397. struct ipath_ibdev *dev = to_idev(ibqp->device);
  398. struct ipath_qp *qp = to_iqp(ibqp);
  399. enum ib_qp_state cur_state, new_state;
  400. unsigned long flags;
  401. int ret;
  402. spin_lock_irqsave(&qp->s_lock, flags);
  403. cur_state = attr_mask & IB_QP_CUR_STATE ?
  404. attr->cur_qp_state : qp->state;
  405. new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
  406. if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
  407. attr_mask))
  408. goto inval;
  409. if (attr_mask & IB_QP_AV)
  410. if (attr->ah_attr.dlid == 0 ||
  411. attr->ah_attr.dlid >= IPATH_MULTICAST_LID_BASE)
  412. goto inval;
  413. if (attr_mask & IB_QP_PKEY_INDEX)
  414. if (attr->pkey_index >= ipath_get_npkeys(dev->dd))
  415. goto inval;
  416. if (attr_mask & IB_QP_MIN_RNR_TIMER)
  417. if (attr->min_rnr_timer > 31)
  418. goto inval;
  419. switch (new_state) {
  420. case IB_QPS_RESET:
  421. ipath_reset_qp(qp);
  422. break;
  423. case IB_QPS_ERR:
  424. ipath_error_qp(qp);
  425. break;
  426. default:
  427. break;
  428. }
  429. if (attr_mask & IB_QP_PKEY_INDEX)
  430. qp->s_pkey_index = attr->pkey_index;
  431. if (attr_mask & IB_QP_DEST_QPN)
  432. qp->remote_qpn = attr->dest_qp_num;
  433. if (attr_mask & IB_QP_SQ_PSN) {
  434. qp->s_next_psn = attr->sq_psn;
  435. qp->s_last_psn = qp->s_next_psn - 1;
  436. }
  437. if (attr_mask & IB_QP_RQ_PSN)
  438. qp->r_psn = attr->rq_psn;
  439. if (attr_mask & IB_QP_ACCESS_FLAGS)
  440. qp->qp_access_flags = attr->qp_access_flags;
  441. if (attr_mask & IB_QP_AV)
  442. qp->remote_ah_attr = attr->ah_attr;
  443. if (attr_mask & IB_QP_PATH_MTU)
  444. qp->path_mtu = attr->path_mtu;
  445. if (attr_mask & IB_QP_RETRY_CNT)
  446. qp->s_retry = qp->s_retry_cnt = attr->retry_cnt;
  447. if (attr_mask & IB_QP_RNR_RETRY) {
  448. qp->s_rnr_retry = attr->rnr_retry;
  449. if (qp->s_rnr_retry > 7)
  450. qp->s_rnr_retry = 7;
  451. qp->s_rnr_retry_cnt = qp->s_rnr_retry;
  452. }
  453. if (attr_mask & IB_QP_MIN_RNR_TIMER)
  454. qp->r_min_rnr_timer = attr->min_rnr_timer;
  455. if (attr_mask & IB_QP_QKEY)
  456. qp->qkey = attr->qkey;
  457. qp->state = new_state;
  458. spin_unlock_irqrestore(&qp->s_lock, flags);
  459. ret = 0;
  460. goto bail;
  461. inval:
  462. spin_unlock_irqrestore(&qp->s_lock, flags);
  463. ret = -EINVAL;
  464. bail:
  465. return ret;
  466. }
  467. int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
  468. int attr_mask, struct ib_qp_init_attr *init_attr)
  469. {
  470. struct ipath_qp *qp = to_iqp(ibqp);
  471. attr->qp_state = qp->state;
  472. attr->cur_qp_state = attr->qp_state;
  473. attr->path_mtu = qp->path_mtu;
  474. attr->path_mig_state = 0;
  475. attr->qkey = qp->qkey;
  476. attr->rq_psn = qp->r_psn;
  477. attr->sq_psn = qp->s_next_psn;
  478. attr->dest_qp_num = qp->remote_qpn;
  479. attr->qp_access_flags = qp->qp_access_flags;
  480. attr->cap.max_send_wr = qp->s_size - 1;
  481. attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1;
  482. attr->cap.max_send_sge = qp->s_max_sge;
  483. attr->cap.max_recv_sge = qp->r_rq.max_sge;
  484. attr->cap.max_inline_data = 0;
  485. attr->ah_attr = qp->remote_ah_attr;
  486. memset(&attr->alt_ah_attr, 0, sizeof(attr->alt_ah_attr));
  487. attr->pkey_index = qp->s_pkey_index;
  488. attr->alt_pkey_index = 0;
  489. attr->en_sqd_async_notify = 0;
  490. attr->sq_draining = 0;
  491. attr->max_rd_atomic = 1;
  492. attr->max_dest_rd_atomic = 1;
  493. attr->min_rnr_timer = qp->r_min_rnr_timer;
  494. attr->port_num = 1;
  495. attr->timeout = 0;
  496. attr->retry_cnt = qp->s_retry_cnt;
  497. attr->rnr_retry = qp->s_rnr_retry;
  498. attr->alt_port_num = 0;
  499. attr->alt_timeout = 0;
  500. init_attr->event_handler = qp->ibqp.event_handler;
  501. init_attr->qp_context = qp->ibqp.qp_context;
  502. init_attr->send_cq = qp->ibqp.send_cq;
  503. init_attr->recv_cq = qp->ibqp.recv_cq;
  504. init_attr->srq = qp->ibqp.srq;
  505. init_attr->cap = attr->cap;
  506. init_attr->sq_sig_type =
  507. (qp->s_flags & (1 << IPATH_S_SIGNAL_REQ_WR))
  508. ? IB_SIGNAL_REQ_WR : 0;
  509. init_attr->qp_type = qp->ibqp.qp_type;
  510. init_attr->port_num = 1;
  511. return 0;
  512. }
  513. /**
  514. * ipath_compute_aeth - compute the AETH (syndrome + MSN)
  515. * @qp: the queue pair to compute the AETH for
  516. *
  517. * Returns the AETH.
  518. */
  519. __be32 ipath_compute_aeth(struct ipath_qp *qp)
  520. {
  521. u32 aeth = qp->r_msn & IPATH_MSN_MASK;
  522. if (qp->ibqp.srq) {
  523. /*
  524. * Shared receive queues don't generate credits.
  525. * Set the credit field to the invalid value.
  526. */
  527. aeth |= IPATH_AETH_CREDIT_INVAL << IPATH_AETH_CREDIT_SHIFT;
  528. } else {
  529. u32 min, max, x;
  530. u32 credits;
  531. struct ipath_rwq *wq = qp->r_rq.wq;
  532. u32 head;
  533. u32 tail;
  534. /* sanity check pointers before trusting them */
  535. head = wq->head;
  536. if (head >= qp->r_rq.size)
  537. head = 0;
  538. tail = wq->tail;
  539. if (tail >= qp->r_rq.size)
  540. tail = 0;
  541. /*
  542. * Compute the number of credits available (RWQEs).
  543. * XXX Not holding the r_rq.lock here so there is a small
  544. * chance that the pair of reads are not atomic.
  545. */
  546. credits = head - tail;
  547. if ((int)credits < 0)
  548. credits += qp->r_rq.size;
  549. /*
  550. * Binary search the credit table to find the code to
  551. * use.
  552. */
  553. min = 0;
  554. max = 31;
  555. for (;;) {
  556. x = (min + max) / 2;
  557. if (credit_table[x] == credits)
  558. break;
  559. if (credit_table[x] > credits)
  560. max = x;
  561. else if (min == x)
  562. break;
  563. else
  564. min = x;
  565. }
  566. aeth |= x << IPATH_AETH_CREDIT_SHIFT;
  567. }
  568. return cpu_to_be32(aeth);
  569. }
  570. /**
  571. * set_verbs_flags - set the verbs layer flags
  572. * @dd: the infinipath device
  573. * @flags: the flags to set
  574. */
  575. static int set_verbs_flags(struct ipath_devdata *dd, unsigned flags)
  576. {
  577. struct ipath_devdata *ss;
  578. unsigned long lflags;
  579. spin_lock_irqsave(&ipath_devs_lock, lflags);
  580. list_for_each_entry(ss, &ipath_dev_list, ipath_list) {
  581. if (!(ss->ipath_flags & IPATH_INITTED))
  582. continue;
  583. if ((flags & IPATH_VERBS_KERNEL_SMA) &&
  584. !(*ss->ipath_statusp & IPATH_STATUS_SMA))
  585. *ss->ipath_statusp |= IPATH_STATUS_OIB_SMA;
  586. else
  587. *ss->ipath_statusp &= ~IPATH_STATUS_OIB_SMA;
  588. }
  589. spin_unlock_irqrestore(&ipath_devs_lock, lflags);
  590. return 0;
  591. }
  592. /**
  593. * ipath_create_qp - create a queue pair for a device
  594. * @ibpd: the protection domain who's device we create the queue pair for
  595. * @init_attr: the attributes of the queue pair
  596. * @udata: unused by InfiniPath
  597. *
  598. * Returns the queue pair on success, otherwise returns an errno.
  599. *
  600. * Called by the ib_create_qp() core verbs function.
  601. */
  602. struct ib_qp *ipath_create_qp(struct ib_pd *ibpd,
  603. struct ib_qp_init_attr *init_attr,
  604. struct ib_udata *udata)
  605. {
  606. struct ipath_qp *qp;
  607. int err;
  608. struct ipath_swqe *swq = NULL;
  609. struct ipath_ibdev *dev;
  610. size_t sz;
  611. struct ib_qp *ret;
  612. if (init_attr->cap.max_send_sge > ib_ipath_max_sges ||
  613. init_attr->cap.max_recv_sge > ib_ipath_max_sges ||
  614. init_attr->cap.max_send_wr > ib_ipath_max_qp_wrs ||
  615. init_attr->cap.max_recv_wr > ib_ipath_max_qp_wrs) {
  616. ret = ERR_PTR(-ENOMEM);
  617. goto bail;
  618. }
  619. if (init_attr->cap.max_send_sge +
  620. init_attr->cap.max_recv_sge +
  621. init_attr->cap.max_send_wr +
  622. init_attr->cap.max_recv_wr == 0) {
  623. ret = ERR_PTR(-EINVAL);
  624. goto bail;
  625. }
  626. switch (init_attr->qp_type) {
  627. case IB_QPT_UC:
  628. case IB_QPT_RC:
  629. sz = sizeof(struct ipath_sge) *
  630. init_attr->cap.max_send_sge +
  631. sizeof(struct ipath_swqe);
  632. swq = vmalloc((init_attr->cap.max_send_wr + 1) * sz);
  633. if (swq == NULL) {
  634. ret = ERR_PTR(-ENOMEM);
  635. goto bail;
  636. }
  637. /* FALLTHROUGH */
  638. case IB_QPT_UD:
  639. case IB_QPT_SMI:
  640. case IB_QPT_GSI:
  641. sz = sizeof(*qp);
  642. if (init_attr->srq) {
  643. struct ipath_srq *srq = to_isrq(init_attr->srq);
  644. sz += sizeof(*qp->r_sg_list) *
  645. srq->rq.max_sge;
  646. } else
  647. sz += sizeof(*qp->r_sg_list) *
  648. init_attr->cap.max_recv_sge;
  649. qp = kmalloc(sz, GFP_KERNEL);
  650. if (!qp) {
  651. ret = ERR_PTR(-ENOMEM);
  652. goto bail_swq;
  653. }
  654. if (init_attr->srq) {
  655. sz = 0;
  656. qp->r_rq.size = 0;
  657. qp->r_rq.max_sge = 0;
  658. qp->r_rq.wq = NULL;
  659. init_attr->cap.max_recv_wr = 0;
  660. init_attr->cap.max_recv_sge = 0;
  661. } else {
  662. qp->r_rq.size = init_attr->cap.max_recv_wr + 1;
  663. qp->r_rq.max_sge = init_attr->cap.max_recv_sge;
  664. sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) +
  665. sizeof(struct ipath_rwqe);
  666. qp->r_rq.wq = vmalloc_user(sizeof(struct ipath_rwq) +
  667. qp->r_rq.size * sz);
  668. if (!qp->r_rq.wq) {
  669. ret = ERR_PTR(-ENOMEM);
  670. goto bail_qp;
  671. }
  672. }
  673. /*
  674. * ib_create_qp() will initialize qp->ibqp
  675. * except for qp->ibqp.qp_num.
  676. */
  677. spin_lock_init(&qp->s_lock);
  678. spin_lock_init(&qp->r_rq.lock);
  679. atomic_set(&qp->refcount, 0);
  680. init_waitqueue_head(&qp->wait);
  681. tasklet_init(&qp->s_task, ipath_do_ruc_send,
  682. (unsigned long)qp);
  683. INIT_LIST_HEAD(&qp->piowait);
  684. INIT_LIST_HEAD(&qp->timerwait);
  685. qp->state = IB_QPS_RESET;
  686. qp->s_wq = swq;
  687. qp->s_size = init_attr->cap.max_send_wr + 1;
  688. qp->s_max_sge = init_attr->cap.max_send_sge;
  689. qp->s_flags = init_attr->sq_sig_type == IB_SIGNAL_REQ_WR ?
  690. 1 << IPATH_S_SIGNAL_REQ_WR : 0;
  691. dev = to_idev(ibpd->device);
  692. err = ipath_alloc_qpn(&dev->qp_table, qp,
  693. init_attr->qp_type);
  694. if (err) {
  695. ret = ERR_PTR(err);
  696. goto bail_rwq;
  697. }
  698. qp->ip = NULL;
  699. ipath_reset_qp(qp);
  700. /* Tell the core driver that the kernel SMA is present. */
  701. if (init_attr->qp_type == IB_QPT_SMI)
  702. set_verbs_flags(dev->dd, IPATH_VERBS_KERNEL_SMA);
  703. break;
  704. default:
  705. /* Don't support raw QPs */
  706. ret = ERR_PTR(-ENOSYS);
  707. goto bail;
  708. }
  709. init_attr->cap.max_inline_data = 0;
  710. /*
  711. * Return the address of the RWQ as the offset to mmap.
  712. * See ipath_mmap() for details.
  713. */
  714. if (udata && udata->outlen >= sizeof(__u64)) {
  715. struct ipath_mmap_info *ip;
  716. __u64 offset = (__u64) qp->r_rq.wq;
  717. int err;
  718. err = ib_copy_to_udata(udata, &offset, sizeof(offset));
  719. if (err) {
  720. ret = ERR_PTR(err);
  721. goto bail_rwq;
  722. }
  723. if (qp->r_rq.wq) {
  724. /* Allocate info for ipath_mmap(). */
  725. ip = kmalloc(sizeof(*ip), GFP_KERNEL);
  726. if (!ip) {
  727. ret = ERR_PTR(-ENOMEM);
  728. goto bail_rwq;
  729. }
  730. qp->ip = ip;
  731. ip->context = ibpd->uobject->context;
  732. ip->obj = qp->r_rq.wq;
  733. kref_init(&ip->ref);
  734. ip->mmap_cnt = 0;
  735. ip->size = PAGE_ALIGN(sizeof(struct ipath_rwq) +
  736. qp->r_rq.size * sz);
  737. spin_lock_irq(&dev->pending_lock);
  738. ip->next = dev->pending_mmaps;
  739. dev->pending_mmaps = ip;
  740. spin_unlock_irq(&dev->pending_lock);
  741. }
  742. }
  743. ret = &qp->ibqp;
  744. goto bail;
  745. bail_rwq:
  746. vfree(qp->r_rq.wq);
  747. bail_qp:
  748. kfree(qp);
  749. bail_swq:
  750. vfree(swq);
  751. bail:
  752. return ret;
  753. }
  754. /**
  755. * ipath_destroy_qp - destroy a queue pair
  756. * @ibqp: the queue pair to destroy
  757. *
  758. * Returns 0 on success.
  759. *
  760. * Note that this can be called while the QP is actively sending or
  761. * receiving!
  762. */
  763. int ipath_destroy_qp(struct ib_qp *ibqp)
  764. {
  765. struct ipath_qp *qp = to_iqp(ibqp);
  766. struct ipath_ibdev *dev = to_idev(ibqp->device);
  767. unsigned long flags;
  768. /* Tell the core driver that the kernel SMA is gone. */
  769. if (qp->ibqp.qp_type == IB_QPT_SMI)
  770. set_verbs_flags(dev->dd, 0);
  771. spin_lock_irqsave(&qp->s_lock, flags);
  772. qp->state = IB_QPS_ERR;
  773. spin_unlock_irqrestore(&qp->s_lock, flags);
  774. /* Stop the sending tasklet. */
  775. tasklet_kill(&qp->s_task);
  776. /* Make sure the QP isn't on the timeout list. */
  777. spin_lock_irqsave(&dev->pending_lock, flags);
  778. if (!list_empty(&qp->timerwait))
  779. list_del_init(&qp->timerwait);
  780. if (!list_empty(&qp->piowait))
  781. list_del_init(&qp->piowait);
  782. spin_unlock_irqrestore(&dev->pending_lock, flags);
  783. /*
  784. * Make sure that the QP is not in the QPN table so receive
  785. * interrupts will discard packets for this QP. XXX Also remove QP
  786. * from multicast table.
  787. */
  788. if (atomic_read(&qp->refcount) != 0)
  789. ipath_free_qp(&dev->qp_table, qp);
  790. if (qp->ip)
  791. kref_put(&qp->ip->ref, ipath_release_mmap_info);
  792. else
  793. vfree(qp->r_rq.wq);
  794. vfree(qp->s_wq);
  795. kfree(qp);
  796. return 0;
  797. }
  798. /**
  799. * ipath_init_qp_table - initialize the QP table for a device
  800. * @idev: the device who's QP table we're initializing
  801. * @size: the size of the QP table
  802. *
  803. * Returns 0 on success, otherwise returns an errno.
  804. */
  805. int ipath_init_qp_table(struct ipath_ibdev *idev, int size)
  806. {
  807. int i;
  808. int ret;
  809. idev->qp_table.last = 1; /* QPN 0 and 1 are special. */
  810. idev->qp_table.max = size;
  811. idev->qp_table.nmaps = 1;
  812. idev->qp_table.table = kzalloc(size * sizeof(*idev->qp_table.table),
  813. GFP_KERNEL);
  814. if (idev->qp_table.table == NULL) {
  815. ret = -ENOMEM;
  816. goto bail;
  817. }
  818. for (i = 0; i < ARRAY_SIZE(idev->qp_table.map); i++) {
  819. atomic_set(&idev->qp_table.map[i].n_free, BITS_PER_PAGE);
  820. idev->qp_table.map[i].page = NULL;
  821. }
  822. ret = 0;
  823. bail:
  824. return ret;
  825. }
  826. /**
  827. * ipath_sqerror_qp - put a QP's send queue into an error state
  828. * @qp: QP who's send queue will be put into an error state
  829. * @wc: the WC responsible for putting the QP in this state
  830. *
  831. * Flushes the send work queue.
  832. * The QP s_lock should be held.
  833. */
  834. void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc)
  835. {
  836. struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
  837. struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
  838. _VERBS_INFO("Send queue error on QP%d/%d: err: %d\n",
  839. qp->ibqp.qp_num, qp->remote_qpn, wc->status);
  840. spin_lock(&dev->pending_lock);
  841. /* XXX What if its already removed by the timeout code? */
  842. if (!list_empty(&qp->timerwait))
  843. list_del_init(&qp->timerwait);
  844. if (!list_empty(&qp->piowait))
  845. list_del_init(&qp->piowait);
  846. spin_unlock(&dev->pending_lock);
  847. ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
  848. if (++qp->s_last >= qp->s_size)
  849. qp->s_last = 0;
  850. wc->status = IB_WC_WR_FLUSH_ERR;
  851. while (qp->s_last != qp->s_head) {
  852. wc->wr_id = wqe->wr.wr_id;
  853. wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
  854. ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
  855. if (++qp->s_last >= qp->s_size)
  856. qp->s_last = 0;
  857. wqe = get_swqe_ptr(qp, qp->s_last);
  858. }
  859. qp->s_cur = qp->s_tail = qp->s_head;
  860. qp->state = IB_QPS_SQE;
  861. }
  862. /**
  863. * ipath_get_credit - flush the send work queue of a QP
  864. * @qp: the qp who's send work queue to flush
  865. * @aeth: the Acknowledge Extended Transport Header
  866. *
  867. * The QP s_lock should be held.
  868. */
  869. void ipath_get_credit(struct ipath_qp *qp, u32 aeth)
  870. {
  871. u32 credit = (aeth >> IPATH_AETH_CREDIT_SHIFT) & IPATH_AETH_CREDIT_MASK;
  872. /*
  873. * If the credit is invalid, we can send
  874. * as many packets as we like. Otherwise, we have to
  875. * honor the credit field.
  876. */
  877. if (credit == IPATH_AETH_CREDIT_INVAL)
  878. qp->s_lsn = (u32) -1;
  879. else if (qp->s_lsn != (u32) -1) {
  880. /* Compute new LSN (i.e., MSN + credit) */
  881. credit = (aeth + credit_table[credit]) & IPATH_MSN_MASK;
  882. if (ipath_cmp24(credit, qp->s_lsn) > 0)
  883. qp->s_lsn = credit;
  884. }
  885. /* Restart sending if it was blocked due to lack of credits. */
  886. if (qp->s_cur != qp->s_head &&
  887. (qp->s_lsn == (u32) -1 ||
  888. ipath_cmp24(get_swqe_ptr(qp, qp->s_cur)->ssn,
  889. qp->s_lsn + 1) <= 0))
  890. tasklet_hi_schedule(&qp->s_task);
  891. }