mad.c 51 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875
  1. /*
  2. * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. */
  32. #include <rdma/ib_mad.h>
  33. #include <rdma/ib_smi.h>
  34. #include <rdma/ib_sa.h>
  35. #include <rdma/ib_cache.h>
  36. #include <linux/mlx4/cmd.h>
  37. #include <linux/gfp.h>
  38. #include <rdma/ib_pma.h>
  39. #include "mlx4_ib.h"
  40. enum {
  41. MLX4_IB_VENDOR_CLASS1 = 0x9,
  42. MLX4_IB_VENDOR_CLASS2 = 0xa
  43. };
  44. #define MLX4_TUN_SEND_WRID_SHIFT 34
  45. #define MLX4_TUN_QPN_SHIFT 32
  46. #define MLX4_TUN_WRID_RECV (((u64) 1) << MLX4_TUN_SEND_WRID_SHIFT)
  47. #define MLX4_TUN_SET_WRID_QPN(a) (((u64) ((a) & 0x3)) << MLX4_TUN_QPN_SHIFT)
  48. #define MLX4_TUN_IS_RECV(a) (((a) >> MLX4_TUN_SEND_WRID_SHIFT) & 0x1)
  49. #define MLX4_TUN_WRID_QPN(a) (((a) >> MLX4_TUN_QPN_SHIFT) & 0x3)
  50. struct mlx4_mad_rcv_buf {
  51. struct ib_grh grh;
  52. u8 payload[256];
  53. } __packed;
  54. struct mlx4_mad_snd_buf {
  55. u8 payload[256];
  56. } __packed;
  57. struct mlx4_tunnel_mad {
  58. struct ib_grh grh;
  59. struct mlx4_ib_tunnel_header hdr;
  60. struct ib_mad mad;
  61. } __packed;
  62. struct mlx4_rcv_tunnel_mad {
  63. struct mlx4_rcv_tunnel_hdr hdr;
  64. struct ib_grh grh;
  65. struct ib_mad mad;
  66. } __packed;
  67. static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num);
  68. __be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx)
  69. {
  70. return cpu_to_be64(atomic_inc_return(&ctx->tid)) |
  71. cpu_to_be64(0xff00000000000000LL);
  72. }
  73. int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags,
  74. int port, struct ib_wc *in_wc, struct ib_grh *in_grh,
  75. void *in_mad, void *response_mad)
  76. {
  77. struct mlx4_cmd_mailbox *inmailbox, *outmailbox;
  78. void *inbox;
  79. int err;
  80. u32 in_modifier = port;
  81. u8 op_modifier = 0;
  82. inmailbox = mlx4_alloc_cmd_mailbox(dev->dev);
  83. if (IS_ERR(inmailbox))
  84. return PTR_ERR(inmailbox);
  85. inbox = inmailbox->buf;
  86. outmailbox = mlx4_alloc_cmd_mailbox(dev->dev);
  87. if (IS_ERR(outmailbox)) {
  88. mlx4_free_cmd_mailbox(dev->dev, inmailbox);
  89. return PTR_ERR(outmailbox);
  90. }
  91. memcpy(inbox, in_mad, 256);
  92. /*
  93. * Key check traps can't be generated unless we have in_wc to
  94. * tell us where to send the trap.
  95. */
  96. if ((mad_ifc_flags & MLX4_MAD_IFC_IGNORE_MKEY) || !in_wc)
  97. op_modifier |= 0x1;
  98. if ((mad_ifc_flags & MLX4_MAD_IFC_IGNORE_BKEY) || !in_wc)
  99. op_modifier |= 0x2;
  100. if (mlx4_is_mfunc(dev->dev) &&
  101. (mad_ifc_flags & MLX4_MAD_IFC_NET_VIEW || in_wc))
  102. op_modifier |= 0x8;
  103. if (in_wc) {
  104. struct {
  105. __be32 my_qpn;
  106. u32 reserved1;
  107. __be32 rqpn;
  108. u8 sl;
  109. u8 g_path;
  110. u16 reserved2[2];
  111. __be16 pkey;
  112. u32 reserved3[11];
  113. u8 grh[40];
  114. } *ext_info;
  115. memset(inbox + 256, 0, 256);
  116. ext_info = inbox + 256;
  117. ext_info->my_qpn = cpu_to_be32(in_wc->qp->qp_num);
  118. ext_info->rqpn = cpu_to_be32(in_wc->src_qp);
  119. ext_info->sl = in_wc->sl << 4;
  120. ext_info->g_path = in_wc->dlid_path_bits |
  121. (in_wc->wc_flags & IB_WC_GRH ? 0x80 : 0);
  122. ext_info->pkey = cpu_to_be16(in_wc->pkey_index);
  123. if (in_grh)
  124. memcpy(ext_info->grh, in_grh, 40);
  125. op_modifier |= 0x4;
  126. in_modifier |= in_wc->slid << 16;
  127. }
  128. err = mlx4_cmd_box(dev->dev, inmailbox->dma, outmailbox->dma, in_modifier,
  129. mlx4_is_master(dev->dev) ? (op_modifier & ~0x8) : op_modifier,
  130. MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C,
  131. (op_modifier & 0x8) ? MLX4_CMD_NATIVE : MLX4_CMD_WRAPPED);
  132. if (!err)
  133. memcpy(response_mad, outmailbox->buf, 256);
  134. mlx4_free_cmd_mailbox(dev->dev, inmailbox);
  135. mlx4_free_cmd_mailbox(dev->dev, outmailbox);
  136. return err;
  137. }
  138. static void update_sm_ah(struct mlx4_ib_dev *dev, u8 port_num, u16 lid, u8 sl)
  139. {
  140. struct ib_ah *new_ah;
  141. struct ib_ah_attr ah_attr;
  142. unsigned long flags;
  143. if (!dev->send_agent[port_num - 1][0])
  144. return;
  145. memset(&ah_attr, 0, sizeof ah_attr);
  146. ah_attr.dlid = lid;
  147. ah_attr.sl = sl;
  148. ah_attr.port_num = port_num;
  149. new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd,
  150. &ah_attr);
  151. if (IS_ERR(new_ah))
  152. return;
  153. spin_lock_irqsave(&dev->sm_lock, flags);
  154. if (dev->sm_ah[port_num - 1])
  155. ib_destroy_ah(dev->sm_ah[port_num - 1]);
  156. dev->sm_ah[port_num - 1] = new_ah;
  157. spin_unlock_irqrestore(&dev->sm_lock, flags);
  158. }
  159. /*
  160. * Snoop SM MADs for port info, GUID info, and P_Key table sets, so we can
  161. * synthesize LID change, Client-Rereg, GID change, and P_Key change events.
  162. */
  163. static void smp_snoop(struct ib_device *ibdev, u8 port_num, struct ib_mad *mad,
  164. u16 prev_lid)
  165. {
  166. struct ib_port_info *pinfo;
  167. u16 lid;
  168. __be16 *base;
  169. u32 bn, pkey_change_bitmap;
  170. int i;
  171. struct mlx4_ib_dev *dev = to_mdev(ibdev);
  172. if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||
  173. mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) &&
  174. mad->mad_hdr.method == IB_MGMT_METHOD_SET)
  175. switch (mad->mad_hdr.attr_id) {
  176. case IB_SMP_ATTR_PORT_INFO:
  177. pinfo = (struct ib_port_info *) ((struct ib_smp *) mad)->data;
  178. lid = be16_to_cpu(pinfo->lid);
  179. update_sm_ah(dev, port_num,
  180. be16_to_cpu(pinfo->sm_lid),
  181. pinfo->neighbormtu_mastersmsl & 0xf);
  182. if (pinfo->clientrereg_resv_subnetto & 0x80)
  183. handle_client_rereg_event(dev, port_num);
  184. if (prev_lid != lid)
  185. mlx4_ib_dispatch_event(dev, port_num,
  186. IB_EVENT_LID_CHANGE);
  187. break;
  188. case IB_SMP_ATTR_PKEY_TABLE:
  189. if (!mlx4_is_mfunc(dev->dev)) {
  190. mlx4_ib_dispatch_event(dev, port_num,
  191. IB_EVENT_PKEY_CHANGE);
  192. break;
  193. }
  194. bn = be32_to_cpu(((struct ib_smp *)mad)->attr_mod) & 0xFFFF;
  195. base = (__be16 *) &(((struct ib_smp *)mad)->data[0]);
  196. pkey_change_bitmap = 0;
  197. for (i = 0; i < 32; i++) {
  198. pr_debug("PKEY[%d] = x%x\n",
  199. i + bn*32, be16_to_cpu(base[i]));
  200. if (be16_to_cpu(base[i]) !=
  201. dev->pkeys.phys_pkey_cache[port_num - 1][i + bn*32]) {
  202. pkey_change_bitmap |= (1 << i);
  203. dev->pkeys.phys_pkey_cache[port_num - 1][i + bn*32] =
  204. be16_to_cpu(base[i]);
  205. }
  206. }
  207. pr_debug("PKEY Change event: port=%d, "
  208. "block=0x%x, change_bitmap=0x%x\n",
  209. port_num, bn, pkey_change_bitmap);
  210. if (pkey_change_bitmap)
  211. mlx4_ib_dispatch_event(dev, port_num,
  212. IB_EVENT_PKEY_CHANGE);
  213. break;
  214. case IB_SMP_ATTR_GUID_INFO:
  215. /* paravirtualized master's guid is guid 0 -- does not change */
  216. if (!mlx4_is_master(dev->dev))
  217. mlx4_ib_dispatch_event(dev, port_num,
  218. IB_EVENT_GID_CHANGE);
  219. break;
  220. default:
  221. break;
  222. }
  223. }
  224. static void node_desc_override(struct ib_device *dev,
  225. struct ib_mad *mad)
  226. {
  227. unsigned long flags;
  228. if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||
  229. mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) &&
  230. mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP &&
  231. mad->mad_hdr.attr_id == IB_SMP_ATTR_NODE_DESC) {
  232. spin_lock_irqsave(&to_mdev(dev)->sm_lock, flags);
  233. memcpy(((struct ib_smp *) mad)->data, dev->node_desc, 64);
  234. spin_unlock_irqrestore(&to_mdev(dev)->sm_lock, flags);
  235. }
  236. }
  237. static void forward_trap(struct mlx4_ib_dev *dev, u8 port_num, struct ib_mad *mad)
  238. {
  239. int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED;
  240. struct ib_mad_send_buf *send_buf;
  241. struct ib_mad_agent *agent = dev->send_agent[port_num - 1][qpn];
  242. int ret;
  243. unsigned long flags;
  244. if (agent) {
  245. send_buf = ib_create_send_mad(agent, qpn, 0, 0, IB_MGMT_MAD_HDR,
  246. IB_MGMT_MAD_DATA, GFP_ATOMIC);
  247. if (IS_ERR(send_buf))
  248. return;
  249. /*
  250. * We rely here on the fact that MLX QPs don't use the
  251. * address handle after the send is posted (this is
  252. * wrong following the IB spec strictly, but we know
  253. * it's OK for our devices).
  254. */
  255. spin_lock_irqsave(&dev->sm_lock, flags);
  256. memcpy(send_buf->mad, mad, sizeof *mad);
  257. if ((send_buf->ah = dev->sm_ah[port_num - 1]))
  258. ret = ib_post_send_mad(send_buf, NULL);
  259. else
  260. ret = -EINVAL;
  261. spin_unlock_irqrestore(&dev->sm_lock, flags);
  262. if (ret)
  263. ib_free_send_mad(send_buf);
  264. }
  265. }
  266. static int mlx4_ib_demux_sa_handler(struct ib_device *ibdev, int port, int slave,
  267. struct ib_sa_mad *sa_mad)
  268. {
  269. int ret = 0;
  270. /* dispatch to different sa handlers */
  271. switch (be16_to_cpu(sa_mad->mad_hdr.attr_id)) {
  272. case IB_SA_ATTR_MC_MEMBER_REC:
  273. ret = mlx4_ib_mcg_demux_handler(ibdev, port, slave, sa_mad);
  274. break;
  275. default:
  276. break;
  277. }
  278. return ret;
  279. }
  280. int mlx4_ib_find_real_gid(struct ib_device *ibdev, u8 port, __be64 guid)
  281. {
  282. struct mlx4_ib_dev *dev = to_mdev(ibdev);
  283. int i;
  284. for (i = 0; i < dev->dev->caps.sqp_demux; i++) {
  285. if (dev->sriov.demux[port - 1].guid_cache[i] == guid)
  286. return i;
  287. }
  288. return -1;
  289. }
  290. static int get_pkey_phys_indices(struct mlx4_ib_dev *ibdev, u8 port, u8 ph_pkey_ix,
  291. u8 *full_pk_ix, u8 *partial_pk_ix,
  292. int *is_full_member)
  293. {
  294. u16 search_pkey;
  295. int fm;
  296. int err = 0;
  297. u16 pk;
  298. err = ib_get_cached_pkey(&ibdev->ib_dev, port, ph_pkey_ix, &search_pkey);
  299. if (err)
  300. return err;
  301. fm = (search_pkey & 0x8000) ? 1 : 0;
  302. if (fm) {
  303. *full_pk_ix = ph_pkey_ix;
  304. search_pkey &= 0x7FFF;
  305. } else {
  306. *partial_pk_ix = ph_pkey_ix;
  307. search_pkey |= 0x8000;
  308. }
  309. if (ib_find_exact_cached_pkey(&ibdev->ib_dev, port, search_pkey, &pk))
  310. pk = 0xFFFF;
  311. if (fm)
  312. *partial_pk_ix = (pk & 0xFF);
  313. else
  314. *full_pk_ix = (pk & 0xFF);
  315. *is_full_member = fm;
  316. return err;
  317. }
  318. int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,
  319. enum ib_qp_type dest_qpt, struct ib_wc *wc,
  320. struct ib_grh *grh, struct ib_mad *mad)
  321. {
  322. struct ib_sge list;
  323. struct ib_send_wr wr, *bad_wr;
  324. struct mlx4_ib_demux_pv_ctx *tun_ctx;
  325. struct mlx4_ib_demux_pv_qp *tun_qp;
  326. struct mlx4_rcv_tunnel_mad *tun_mad;
  327. struct ib_ah_attr attr;
  328. struct ib_ah *ah;
  329. struct ib_qp *src_qp = NULL;
  330. unsigned tun_tx_ix = 0;
  331. int dqpn;
  332. int ret = 0;
  333. int i;
  334. int is_full_member = 0;
  335. u16 tun_pkey_ix;
  336. u8 ph_pkey_ix, full_pk_ix = 0, partial_pk_ix = 0;
  337. if (dest_qpt > IB_QPT_GSI)
  338. return -EINVAL;
  339. tun_ctx = dev->sriov.demux[port-1].tun[slave];
  340. /* check if proxy qp created */
  341. if (!tun_ctx || tun_ctx->state != DEMUX_PV_STATE_ACTIVE)
  342. return -EAGAIN;
  343. /* QP0 forwarding only for Dom0 */
  344. if (!dest_qpt && (mlx4_master_func_num(dev->dev) != slave))
  345. return -EINVAL;
  346. if (!dest_qpt)
  347. tun_qp = &tun_ctx->qp[0];
  348. else
  349. tun_qp = &tun_ctx->qp[1];
  350. /* compute pkey index for slave */
  351. /* get physical pkey -- virtualized Dom0 pkey to phys*/
  352. if (dest_qpt) {
  353. ph_pkey_ix =
  354. dev->pkeys.virt2phys_pkey[mlx4_master_func_num(dev->dev)][port - 1][wc->pkey_index];
  355. /* now, translate this to the slave pkey index */
  356. ret = get_pkey_phys_indices(dev, port, ph_pkey_ix, &full_pk_ix,
  357. &partial_pk_ix, &is_full_member);
  358. if (ret)
  359. return -EINVAL;
  360. for (i = 0; i < dev->dev->caps.pkey_table_len[port]; i++) {
  361. if ((dev->pkeys.virt2phys_pkey[slave][port - 1][i] == full_pk_ix) ||
  362. (is_full_member &&
  363. (dev->pkeys.virt2phys_pkey[slave][port - 1][i] == partial_pk_ix)))
  364. break;
  365. }
  366. if (i == dev->dev->caps.pkey_table_len[port])
  367. return -EINVAL;
  368. tun_pkey_ix = i;
  369. } else
  370. tun_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][0];
  371. dqpn = dev->dev->caps.sqp_start + 8 * slave + port + (dest_qpt * 2) - 1;
  372. /* get tunnel tx data buf for slave */
  373. src_qp = tun_qp->qp;
  374. /* create ah. Just need an empty one with the port num for the post send.
  375. * The driver will set the force loopback bit in post_send */
  376. memset(&attr, 0, sizeof attr);
  377. attr.port_num = port;
  378. ah = ib_create_ah(tun_ctx->pd, &attr);
  379. if (IS_ERR(ah))
  380. return -ENOMEM;
  381. /* allocate tunnel tx buf after pass failure returns */
  382. spin_lock(&tun_qp->tx_lock);
  383. if (tun_qp->tx_ix_head - tun_qp->tx_ix_tail >=
  384. (MLX4_NUM_TUNNEL_BUFS - 1))
  385. ret = -EAGAIN;
  386. else
  387. tun_tx_ix = (++tun_qp->tx_ix_head) & (MLX4_NUM_TUNNEL_BUFS - 1);
  388. spin_unlock(&tun_qp->tx_lock);
  389. if (ret)
  390. goto out;
  391. tun_mad = (struct mlx4_rcv_tunnel_mad *) (tun_qp->tx_ring[tun_tx_ix].buf.addr);
  392. if (tun_qp->tx_ring[tun_tx_ix].ah)
  393. ib_destroy_ah(tun_qp->tx_ring[tun_tx_ix].ah);
  394. tun_qp->tx_ring[tun_tx_ix].ah = ah;
  395. ib_dma_sync_single_for_cpu(&dev->ib_dev,
  396. tun_qp->tx_ring[tun_tx_ix].buf.map,
  397. sizeof (struct mlx4_rcv_tunnel_mad),
  398. DMA_TO_DEVICE);
  399. /* copy over to tunnel buffer */
  400. if (grh)
  401. memcpy(&tun_mad->grh, grh, sizeof *grh);
  402. memcpy(&tun_mad->mad, mad, sizeof *mad);
  403. /* adjust tunnel data */
  404. tun_mad->hdr.pkey_index = cpu_to_be16(tun_pkey_ix);
  405. tun_mad->hdr.sl_vid = cpu_to_be16(((u16)(wc->sl)) << 12);
  406. tun_mad->hdr.slid_mac_47_32 = cpu_to_be16(wc->slid);
  407. tun_mad->hdr.flags_src_qp = cpu_to_be32(wc->src_qp & 0xFFFFFF);
  408. tun_mad->hdr.g_ml_path = (grh && (wc->wc_flags & IB_WC_GRH)) ? 0x80 : 0;
  409. ib_dma_sync_single_for_device(&dev->ib_dev,
  410. tun_qp->tx_ring[tun_tx_ix].buf.map,
  411. sizeof (struct mlx4_rcv_tunnel_mad),
  412. DMA_TO_DEVICE);
  413. list.addr = tun_qp->tx_ring[tun_tx_ix].buf.map;
  414. list.length = sizeof (struct mlx4_rcv_tunnel_mad);
  415. list.lkey = tun_ctx->mr->lkey;
  416. wr.wr.ud.ah = ah;
  417. wr.wr.ud.port_num = port;
  418. wr.wr.ud.remote_qkey = IB_QP_SET_QKEY;
  419. wr.wr.ud.remote_qpn = dqpn;
  420. wr.next = NULL;
  421. wr.wr_id = ((u64) tun_tx_ix) | MLX4_TUN_SET_WRID_QPN(dest_qpt);
  422. wr.sg_list = &list;
  423. wr.num_sge = 1;
  424. wr.opcode = IB_WR_SEND;
  425. wr.send_flags = IB_SEND_SIGNALED;
  426. ret = ib_post_send(src_qp, &wr, &bad_wr);
  427. out:
  428. if (ret)
  429. ib_destroy_ah(ah);
  430. return ret;
  431. }
  432. static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port,
  433. struct ib_wc *wc, struct ib_grh *grh,
  434. struct ib_mad *mad)
  435. {
  436. struct mlx4_ib_dev *dev = to_mdev(ibdev);
  437. int err;
  438. int slave;
  439. u8 *slave_id;
  440. /* Initially assume that this mad is for us */
  441. slave = mlx4_master_func_num(dev->dev);
  442. /* See if the slave id is encoded in a response mad */
  443. if (mad->mad_hdr.method & 0x80) {
  444. slave_id = (u8 *) &mad->mad_hdr.tid;
  445. slave = *slave_id;
  446. if (slave != 255) /*255 indicates the dom0*/
  447. *slave_id = 0; /* remap tid */
  448. }
  449. /* If a grh is present, we demux according to it */
  450. if (wc->wc_flags & IB_WC_GRH) {
  451. slave = mlx4_ib_find_real_gid(ibdev, port, grh->dgid.global.interface_id);
  452. if (slave < 0) {
  453. mlx4_ib_warn(ibdev, "failed matching grh\n");
  454. return -ENOENT;
  455. }
  456. }
  457. /* Class-specific handling */
  458. switch (mad->mad_hdr.mgmt_class) {
  459. case IB_MGMT_CLASS_SUBN_ADM:
  460. if (mlx4_ib_demux_sa_handler(ibdev, port, slave,
  461. (struct ib_sa_mad *) mad))
  462. return 0;
  463. break;
  464. case IB_MGMT_CLASS_CM:
  465. if (mlx4_ib_demux_cm_handler(ibdev, port, &slave, mad))
  466. return 0;
  467. break;
  468. case IB_MGMT_CLASS_DEVICE_MGMT:
  469. if (mad->mad_hdr.method != IB_MGMT_METHOD_GET_RESP)
  470. return 0;
  471. break;
  472. default:
  473. /* Drop unsupported classes for slaves in tunnel mode */
  474. if (slave != mlx4_master_func_num(dev->dev)) {
  475. pr_debug("dropping unsupported ingress mad from class:%d "
  476. "for slave:%d\n", mad->mad_hdr.mgmt_class, slave);
  477. return 0;
  478. }
  479. }
  480. /*make sure that no slave==255 was not handled yet.*/
  481. if (slave >= dev->dev->caps.sqp_demux) {
  482. mlx4_ib_warn(ibdev, "slave id: %d is bigger than allowed:%d\n",
  483. slave, dev->dev->caps.sqp_demux);
  484. return -ENOENT;
  485. }
  486. err = mlx4_ib_send_to_slave(dev, slave, port, wc->qp->qp_type, wc, grh, mad);
  487. if (err)
  488. pr_debug("failed sending to slave %d via tunnel qp (%d)\n",
  489. slave, err);
  490. return 0;
  491. }
  492. static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
  493. struct ib_wc *in_wc, struct ib_grh *in_grh,
  494. struct ib_mad *in_mad, struct ib_mad *out_mad)
  495. {
  496. u16 slid, prev_lid = 0;
  497. int err;
  498. struct ib_port_attr pattr;
  499. if (in_wc && in_wc->qp->qp_num) {
  500. pr_debug("received MAD: slid:%d sqpn:%d "
  501. "dlid_bits:%d dqpn:%d wc_flags:0x%x, cls %x, mtd %x, atr %x\n",
  502. in_wc->slid, in_wc->src_qp,
  503. in_wc->dlid_path_bits,
  504. in_wc->qp->qp_num,
  505. in_wc->wc_flags,
  506. in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.method,
  507. be16_to_cpu(in_mad->mad_hdr.attr_id));
  508. if (in_wc->wc_flags & IB_WC_GRH) {
  509. pr_debug("sgid_hi:0x%016llx sgid_lo:0x%016llx\n",
  510. be64_to_cpu(in_grh->sgid.global.subnet_prefix),
  511. be64_to_cpu(in_grh->sgid.global.interface_id));
  512. pr_debug("dgid_hi:0x%016llx dgid_lo:0x%016llx\n",
  513. be64_to_cpu(in_grh->dgid.global.subnet_prefix),
  514. be64_to_cpu(in_grh->dgid.global.interface_id));
  515. }
  516. }
  517. slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE);
  518. if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && slid == 0) {
  519. forward_trap(to_mdev(ibdev), port_num, in_mad);
  520. return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
  521. }
  522. if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||
  523. in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
  524. if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET &&
  525. in_mad->mad_hdr.method != IB_MGMT_METHOD_SET &&
  526. in_mad->mad_hdr.method != IB_MGMT_METHOD_TRAP_REPRESS)
  527. return IB_MAD_RESULT_SUCCESS;
  528. /*
  529. * Don't process SMInfo queries -- the SMA can't handle them.
  530. */
  531. if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO)
  532. return IB_MAD_RESULT_SUCCESS;
  533. } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT ||
  534. in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS1 ||
  535. in_mad->mad_hdr.mgmt_class == MLX4_IB_VENDOR_CLASS2 ||
  536. in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_CONG_MGMT) {
  537. if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET &&
  538. in_mad->mad_hdr.method != IB_MGMT_METHOD_SET)
  539. return IB_MAD_RESULT_SUCCESS;
  540. } else
  541. return IB_MAD_RESULT_SUCCESS;
  542. if ((in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||
  543. in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) &&
  544. in_mad->mad_hdr.method == IB_MGMT_METHOD_SET &&
  545. in_mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO &&
  546. !ib_query_port(ibdev, port_num, &pattr))
  547. prev_lid = pattr.lid;
  548. err = mlx4_MAD_IFC(to_mdev(ibdev),
  549. (mad_flags & IB_MAD_IGNORE_MKEY ? MLX4_MAD_IFC_IGNORE_MKEY : 0) |
  550. (mad_flags & IB_MAD_IGNORE_BKEY ? MLX4_MAD_IFC_IGNORE_BKEY : 0) |
  551. MLX4_MAD_IFC_NET_VIEW,
  552. port_num, in_wc, in_grh, in_mad, out_mad);
  553. if (err)
  554. return IB_MAD_RESULT_FAILURE;
  555. if (!out_mad->mad_hdr.status) {
  556. if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV))
  557. smp_snoop(ibdev, port_num, in_mad, prev_lid);
  558. node_desc_override(ibdev, out_mad);
  559. }
  560. /* set return bit in status of directed route responses */
  561. if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
  562. out_mad->mad_hdr.status |= cpu_to_be16(1 << 15);
  563. if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS)
  564. /* no response for trap repress */
  565. return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
  566. return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
  567. }
  568. static void edit_counter(struct mlx4_counter *cnt,
  569. struct ib_pma_portcounters *pma_cnt)
  570. {
  571. pma_cnt->port_xmit_data = cpu_to_be32((be64_to_cpu(cnt->tx_bytes)>>2));
  572. pma_cnt->port_rcv_data = cpu_to_be32((be64_to_cpu(cnt->rx_bytes)>>2));
  573. pma_cnt->port_xmit_packets = cpu_to_be32(be64_to_cpu(cnt->tx_frames));
  574. pma_cnt->port_rcv_packets = cpu_to_be32(be64_to_cpu(cnt->rx_frames));
  575. }
  576. static int iboe_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
  577. struct ib_wc *in_wc, struct ib_grh *in_grh,
  578. struct ib_mad *in_mad, struct ib_mad *out_mad)
  579. {
  580. struct mlx4_cmd_mailbox *mailbox;
  581. struct mlx4_ib_dev *dev = to_mdev(ibdev);
  582. int err;
  583. u32 inmod = dev->counters[port_num - 1] & 0xffff;
  584. u8 mode;
  585. if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT)
  586. return -EINVAL;
  587. mailbox = mlx4_alloc_cmd_mailbox(dev->dev);
  588. if (IS_ERR(mailbox))
  589. return IB_MAD_RESULT_FAILURE;
  590. err = mlx4_cmd_box(dev->dev, 0, mailbox->dma, inmod, 0,
  591. MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C,
  592. MLX4_CMD_WRAPPED);
  593. if (err)
  594. err = IB_MAD_RESULT_FAILURE;
  595. else {
  596. memset(out_mad->data, 0, sizeof out_mad->data);
  597. mode = ((struct mlx4_counter *)mailbox->buf)->counter_mode;
  598. switch (mode & 0xf) {
  599. case 0:
  600. edit_counter(mailbox->buf,
  601. (void *)(out_mad->data + 40));
  602. err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
  603. break;
  604. default:
  605. err = IB_MAD_RESULT_FAILURE;
  606. }
  607. }
  608. mlx4_free_cmd_mailbox(dev->dev, mailbox);
  609. return err;
  610. }
  611. int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
  612. struct ib_wc *in_wc, struct ib_grh *in_grh,
  613. struct ib_mad *in_mad, struct ib_mad *out_mad)
  614. {
  615. switch (rdma_port_get_link_layer(ibdev, port_num)) {
  616. case IB_LINK_LAYER_INFINIBAND:
  617. return ib_process_mad(ibdev, mad_flags, port_num, in_wc,
  618. in_grh, in_mad, out_mad);
  619. case IB_LINK_LAYER_ETHERNET:
  620. return iboe_process_mad(ibdev, mad_flags, port_num, in_wc,
  621. in_grh, in_mad, out_mad);
  622. default:
  623. return -EINVAL;
  624. }
  625. }
  626. static void send_handler(struct ib_mad_agent *agent,
  627. struct ib_mad_send_wc *mad_send_wc)
  628. {
  629. ib_free_send_mad(mad_send_wc->send_buf);
  630. }
  631. int mlx4_ib_mad_init(struct mlx4_ib_dev *dev)
  632. {
  633. struct ib_mad_agent *agent;
  634. int p, q;
  635. int ret;
  636. enum rdma_link_layer ll;
  637. for (p = 0; p < dev->num_ports; ++p) {
  638. ll = rdma_port_get_link_layer(&dev->ib_dev, p + 1);
  639. for (q = 0; q <= 1; ++q) {
  640. if (ll == IB_LINK_LAYER_INFINIBAND) {
  641. agent = ib_register_mad_agent(&dev->ib_dev, p + 1,
  642. q ? IB_QPT_GSI : IB_QPT_SMI,
  643. NULL, 0, send_handler,
  644. NULL, NULL);
  645. if (IS_ERR(agent)) {
  646. ret = PTR_ERR(agent);
  647. goto err;
  648. }
  649. dev->send_agent[p][q] = agent;
  650. } else
  651. dev->send_agent[p][q] = NULL;
  652. }
  653. }
  654. return 0;
  655. err:
  656. for (p = 0; p < dev->num_ports; ++p)
  657. for (q = 0; q <= 1; ++q)
  658. if (dev->send_agent[p][q])
  659. ib_unregister_mad_agent(dev->send_agent[p][q]);
  660. return ret;
  661. }
  662. void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev)
  663. {
  664. struct ib_mad_agent *agent;
  665. int p, q;
  666. for (p = 0; p < dev->num_ports; ++p) {
  667. for (q = 0; q <= 1; ++q) {
  668. agent = dev->send_agent[p][q];
  669. if (agent) {
  670. dev->send_agent[p][q] = NULL;
  671. ib_unregister_mad_agent(agent);
  672. }
  673. }
  674. if (dev->sm_ah[p])
  675. ib_destroy_ah(dev->sm_ah[p]);
  676. }
  677. }
  678. static void handle_client_rereg_event(struct mlx4_ib_dev *dev, u8 port_num)
  679. {
  680. /* re-configure the alias-guid and mcg's */
  681. if (mlx4_is_master(dev->dev)) {
  682. mlx4_ib_invalidate_all_guid_record(dev, port_num);
  683. if (!dev->sriov.is_going_down)
  684. mlx4_ib_mcg_port_cleanup(&dev->sriov.demux[port_num - 1], 0);
  685. }
  686. mlx4_ib_dispatch_event(dev, port_num, IB_EVENT_CLIENT_REREGISTER);
  687. }
  688. void handle_port_mgmt_change_event(struct work_struct *work)
  689. {
  690. struct ib_event_work *ew = container_of(work, struct ib_event_work, work);
  691. struct mlx4_ib_dev *dev = ew->ib_dev;
  692. struct mlx4_eqe *eqe = &(ew->ib_eqe);
  693. u8 port = eqe->event.port_mgmt_change.port;
  694. u32 changed_attr;
  695. switch (eqe->subtype) {
  696. case MLX4_DEV_PMC_SUBTYPE_PORT_INFO:
  697. changed_attr = be32_to_cpu(eqe->event.port_mgmt_change.params.port_info.changed_attr);
  698. /* Update the SM ah - This should be done before handling
  699. the other changed attributes so that MADs can be sent to the SM */
  700. if (changed_attr & MSTR_SM_CHANGE_MASK) {
  701. u16 lid = be16_to_cpu(eqe->event.port_mgmt_change.params.port_info.mstr_sm_lid);
  702. u8 sl = eqe->event.port_mgmt_change.params.port_info.mstr_sm_sl & 0xf;
  703. update_sm_ah(dev, port, lid, sl);
  704. }
  705. /* Check if it is a lid change event */
  706. if (changed_attr & MLX4_EQ_PORT_INFO_LID_CHANGE_MASK)
  707. mlx4_ib_dispatch_event(dev, port, IB_EVENT_LID_CHANGE);
  708. /* Generate GUID changed event */
  709. if (changed_attr & MLX4_EQ_PORT_INFO_GID_PFX_CHANGE_MASK)
  710. mlx4_ib_dispatch_event(dev, port, IB_EVENT_GID_CHANGE);
  711. if (changed_attr & MLX4_EQ_PORT_INFO_CLIENT_REREG_MASK)
  712. handle_client_rereg_event(dev, port);
  713. break;
  714. case MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE:
  715. mlx4_ib_dispatch_event(dev, port, IB_EVENT_PKEY_CHANGE);
  716. break;
  717. case MLX4_DEV_PMC_SUBTYPE_GUID_INFO:
  718. /* paravirtualized master's guid is guid 0 -- does not change */
  719. if (!mlx4_is_master(dev->dev))
  720. mlx4_ib_dispatch_event(dev, port, IB_EVENT_GID_CHANGE);
  721. break;
  722. default:
  723. pr_warn("Unsupported subtype 0x%x for "
  724. "Port Management Change event\n", eqe->subtype);
  725. }
  726. kfree(ew);
  727. }
  728. void mlx4_ib_dispatch_event(struct mlx4_ib_dev *dev, u8 port_num,
  729. enum ib_event_type type)
  730. {
  731. struct ib_event event;
  732. event.device = &dev->ib_dev;
  733. event.element.port_num = port_num;
  734. event.event = type;
  735. ib_dispatch_event(&event);
  736. }
  737. static void mlx4_ib_tunnel_comp_handler(struct ib_cq *cq, void *arg)
  738. {
  739. unsigned long flags;
  740. struct mlx4_ib_demux_pv_ctx *ctx = cq->cq_context;
  741. struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev);
  742. spin_lock_irqsave(&dev->sriov.going_down_lock, flags);
  743. if (!dev->sriov.is_going_down && ctx->state == DEMUX_PV_STATE_ACTIVE)
  744. queue_work(ctx->wq, &ctx->work);
  745. spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags);
  746. }
  747. static int mlx4_ib_post_pv_qp_buf(struct mlx4_ib_demux_pv_ctx *ctx,
  748. struct mlx4_ib_demux_pv_qp *tun_qp,
  749. int index)
  750. {
  751. struct ib_sge sg_list;
  752. struct ib_recv_wr recv_wr, *bad_recv_wr;
  753. int size;
  754. size = (tun_qp->qp->qp_type == IB_QPT_UD) ?
  755. sizeof (struct mlx4_tunnel_mad) : sizeof (struct mlx4_mad_rcv_buf);
  756. sg_list.addr = tun_qp->ring[index].map;
  757. sg_list.length = size;
  758. sg_list.lkey = ctx->mr->lkey;
  759. recv_wr.next = NULL;
  760. recv_wr.sg_list = &sg_list;
  761. recv_wr.num_sge = 1;
  762. recv_wr.wr_id = (u64) index | MLX4_TUN_WRID_RECV |
  763. MLX4_TUN_SET_WRID_QPN(tun_qp->proxy_qpt);
  764. ib_dma_sync_single_for_device(ctx->ib_dev, tun_qp->ring[index].map,
  765. size, DMA_FROM_DEVICE);
  766. return ib_post_recv(tun_qp->qp, &recv_wr, &bad_recv_wr);
  767. }
  768. static int mlx4_ib_multiplex_sa_handler(struct ib_device *ibdev, int port,
  769. int slave, struct ib_sa_mad *sa_mad)
  770. {
  771. int ret = 0;
  772. /* dispatch to different sa handlers */
  773. switch (be16_to_cpu(sa_mad->mad_hdr.attr_id)) {
  774. case IB_SA_ATTR_MC_MEMBER_REC:
  775. ret = mlx4_ib_mcg_multiplex_handler(ibdev, port, slave, sa_mad);
  776. break;
  777. default:
  778. break;
  779. }
  780. return ret;
  781. }
  782. static int is_proxy_qp0(struct mlx4_ib_dev *dev, int qpn, int slave)
  783. {
  784. int slave_start = dev->dev->caps.sqp_start + 8 * slave;
  785. return (qpn >= slave_start && qpn <= slave_start + 1);
  786. }
  787. int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
  788. enum ib_qp_type dest_qpt, u16 pkey_index, u32 remote_qpn,
  789. u32 qkey, struct ib_ah_attr *attr, struct ib_mad *mad)
  790. {
  791. struct ib_sge list;
  792. struct ib_send_wr wr, *bad_wr;
  793. struct mlx4_ib_demux_pv_ctx *sqp_ctx;
  794. struct mlx4_ib_demux_pv_qp *sqp;
  795. struct mlx4_mad_snd_buf *sqp_mad;
  796. struct ib_ah *ah;
  797. struct ib_qp *send_qp = NULL;
  798. unsigned wire_tx_ix = 0;
  799. int ret = 0;
  800. u16 wire_pkey_ix;
  801. int src_qpnum;
  802. u8 sgid_index;
  803. sqp_ctx = dev->sriov.sqps[port-1];
  804. /* check if proxy qp created */
  805. if (!sqp_ctx || sqp_ctx->state != DEMUX_PV_STATE_ACTIVE)
  806. return -EAGAIN;
  807. /* QP0 forwarding only for Dom0 */
  808. if (dest_qpt == IB_QPT_SMI && (mlx4_master_func_num(dev->dev) != slave))
  809. return -EINVAL;
  810. if (dest_qpt == IB_QPT_SMI) {
  811. src_qpnum = 0;
  812. sqp = &sqp_ctx->qp[0];
  813. wire_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][0];
  814. } else {
  815. src_qpnum = 1;
  816. sqp = &sqp_ctx->qp[1];
  817. wire_pkey_ix = dev->pkeys.virt2phys_pkey[slave][port - 1][pkey_index];
  818. }
  819. send_qp = sqp->qp;
  820. /* create ah */
  821. sgid_index = attr->grh.sgid_index;
  822. attr->grh.sgid_index = 0;
  823. ah = ib_create_ah(sqp_ctx->pd, attr);
  824. if (IS_ERR(ah))
  825. return -ENOMEM;
  826. attr->grh.sgid_index = sgid_index;
  827. to_mah(ah)->av.ib.gid_index = sgid_index;
  828. /* get rid of force-loopback bit */
  829. to_mah(ah)->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF);
  830. spin_lock(&sqp->tx_lock);
  831. if (sqp->tx_ix_head - sqp->tx_ix_tail >=
  832. (MLX4_NUM_TUNNEL_BUFS - 1))
  833. ret = -EAGAIN;
  834. else
  835. wire_tx_ix = (++sqp->tx_ix_head) & (MLX4_NUM_TUNNEL_BUFS - 1);
  836. spin_unlock(&sqp->tx_lock);
  837. if (ret)
  838. goto out;
  839. sqp_mad = (struct mlx4_mad_snd_buf *) (sqp->tx_ring[wire_tx_ix].buf.addr);
  840. if (sqp->tx_ring[wire_tx_ix].ah)
  841. ib_destroy_ah(sqp->tx_ring[wire_tx_ix].ah);
  842. sqp->tx_ring[wire_tx_ix].ah = ah;
  843. ib_dma_sync_single_for_cpu(&dev->ib_dev,
  844. sqp->tx_ring[wire_tx_ix].buf.map,
  845. sizeof (struct mlx4_mad_snd_buf),
  846. DMA_TO_DEVICE);
  847. memcpy(&sqp_mad->payload, mad, sizeof *mad);
  848. ib_dma_sync_single_for_device(&dev->ib_dev,
  849. sqp->tx_ring[wire_tx_ix].buf.map,
  850. sizeof (struct mlx4_mad_snd_buf),
  851. DMA_TO_DEVICE);
  852. list.addr = sqp->tx_ring[wire_tx_ix].buf.map;
  853. list.length = sizeof (struct mlx4_mad_snd_buf);
  854. list.lkey = sqp_ctx->mr->lkey;
  855. wr.wr.ud.ah = ah;
  856. wr.wr.ud.port_num = port;
  857. wr.wr.ud.pkey_index = wire_pkey_ix;
  858. wr.wr.ud.remote_qkey = qkey;
  859. wr.wr.ud.remote_qpn = remote_qpn;
  860. wr.next = NULL;
  861. wr.wr_id = ((u64) wire_tx_ix) | MLX4_TUN_SET_WRID_QPN(src_qpnum);
  862. wr.sg_list = &list;
  863. wr.num_sge = 1;
  864. wr.opcode = IB_WR_SEND;
  865. wr.send_flags = IB_SEND_SIGNALED;
  866. ret = ib_post_send(send_qp, &wr, &bad_wr);
  867. out:
  868. if (ret)
  869. ib_destroy_ah(ah);
  870. return ret;
  871. }
  872. static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc *wc)
  873. {
  874. struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev);
  875. struct mlx4_ib_demux_pv_qp *tun_qp = &ctx->qp[MLX4_TUN_WRID_QPN(wc->wr_id)];
  876. int wr_ix = wc->wr_id & (MLX4_NUM_TUNNEL_BUFS - 1);
  877. struct mlx4_tunnel_mad *tunnel = tun_qp->ring[wr_ix].addr;
  878. struct mlx4_ib_ah ah;
  879. struct ib_ah_attr ah_attr;
  880. u8 *slave_id;
  881. int slave;
  882. /* Get slave that sent this packet */
  883. if (wc->src_qp < dev->dev->caps.sqp_start ||
  884. wc->src_qp >= dev->dev->caps.base_tunnel_sqpn ||
  885. (wc->src_qp & 0x1) != ctx->port - 1 ||
  886. wc->src_qp & 0x4) {
  887. mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d\n", wc->src_qp);
  888. return;
  889. }
  890. slave = ((wc->src_qp & ~0x7) - dev->dev->caps.sqp_start) / 8;
  891. if (slave != ctx->slave) {
  892. mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d: "
  893. "belongs to another slave\n", wc->src_qp);
  894. return;
  895. }
  896. if (slave != mlx4_master_func_num(dev->dev) && !(wc->src_qp & 0x2)) {
  897. mlx4_ib_warn(ctx->ib_dev, "can't multiplex bad sqp:%d: "
  898. "non-master trying to send QP0 packets\n", wc->src_qp);
  899. return;
  900. }
  901. /* Map transaction ID */
  902. ib_dma_sync_single_for_cpu(ctx->ib_dev, tun_qp->ring[wr_ix].map,
  903. sizeof (struct mlx4_tunnel_mad),
  904. DMA_FROM_DEVICE);
  905. switch (tunnel->mad.mad_hdr.method) {
  906. case IB_MGMT_METHOD_SET:
  907. case IB_MGMT_METHOD_GET:
  908. case IB_MGMT_METHOD_REPORT:
  909. case IB_SA_METHOD_GET_TABLE:
  910. case IB_SA_METHOD_DELETE:
  911. case IB_SA_METHOD_GET_MULTI:
  912. case IB_SA_METHOD_GET_TRACE_TBL:
  913. slave_id = (u8 *) &tunnel->mad.mad_hdr.tid;
  914. if (*slave_id) {
  915. mlx4_ib_warn(ctx->ib_dev, "egress mad has non-null tid msb:%d "
  916. "class:%d slave:%d\n", *slave_id,
  917. tunnel->mad.mad_hdr.mgmt_class, slave);
  918. return;
  919. } else
  920. *slave_id = slave;
  921. default:
  922. /* nothing */;
  923. }
  924. /* Class-specific handling */
  925. switch (tunnel->mad.mad_hdr.mgmt_class) {
  926. case IB_MGMT_CLASS_SUBN_ADM:
  927. if (mlx4_ib_multiplex_sa_handler(ctx->ib_dev, ctx->port, slave,
  928. (struct ib_sa_mad *) &tunnel->mad))
  929. return;
  930. break;
  931. case IB_MGMT_CLASS_CM:
  932. if (mlx4_ib_multiplex_cm_handler(ctx->ib_dev, ctx->port, slave,
  933. (struct ib_mad *) &tunnel->mad))
  934. return;
  935. break;
  936. case IB_MGMT_CLASS_DEVICE_MGMT:
  937. if (tunnel->mad.mad_hdr.method != IB_MGMT_METHOD_GET &&
  938. tunnel->mad.mad_hdr.method != IB_MGMT_METHOD_SET)
  939. return;
  940. break;
  941. default:
  942. /* Drop unsupported classes for slaves in tunnel mode */
  943. if (slave != mlx4_master_func_num(dev->dev)) {
  944. mlx4_ib_warn(ctx->ib_dev, "dropping unsupported egress mad from class:%d "
  945. "for slave:%d\n", tunnel->mad.mad_hdr.mgmt_class, slave);
  946. return;
  947. }
  948. }
  949. /* We are using standard ib_core services to send the mad, so generate a
  950. * stadard address handle by decoding the tunnelled mlx4_ah fields */
  951. memcpy(&ah.av, &tunnel->hdr.av, sizeof (struct mlx4_av));
  952. ah.ibah.device = ctx->ib_dev;
  953. mlx4_ib_query_ah(&ah.ibah, &ah_attr);
  954. if ((ah_attr.ah_flags & IB_AH_GRH) &&
  955. (ah_attr.grh.sgid_index != slave)) {
  956. mlx4_ib_warn(ctx->ib_dev, "slave:%d accessed invalid sgid_index:%d\n",
  957. slave, ah_attr.grh.sgid_index);
  958. return;
  959. }
  960. mlx4_ib_send_to_wire(dev, slave, ctx->port,
  961. is_proxy_qp0(dev, wc->src_qp, slave) ?
  962. IB_QPT_SMI : IB_QPT_GSI,
  963. be16_to_cpu(tunnel->hdr.pkey_index),
  964. be32_to_cpu(tunnel->hdr.remote_qpn),
  965. be32_to_cpu(tunnel->hdr.qkey),
  966. &ah_attr, &tunnel->mad);
  967. }
  968. static int mlx4_ib_alloc_pv_bufs(struct mlx4_ib_demux_pv_ctx *ctx,
  969. enum ib_qp_type qp_type, int is_tun)
  970. {
  971. int i;
  972. struct mlx4_ib_demux_pv_qp *tun_qp;
  973. int rx_buf_size, tx_buf_size;
  974. if (qp_type > IB_QPT_GSI)
  975. return -EINVAL;
  976. tun_qp = &ctx->qp[qp_type];
  977. tun_qp->ring = kzalloc(sizeof (struct mlx4_ib_buf) * MLX4_NUM_TUNNEL_BUFS,
  978. GFP_KERNEL);
  979. if (!tun_qp->ring)
  980. return -ENOMEM;
  981. tun_qp->tx_ring = kcalloc(MLX4_NUM_TUNNEL_BUFS,
  982. sizeof (struct mlx4_ib_tun_tx_buf),
  983. GFP_KERNEL);
  984. if (!tun_qp->tx_ring) {
  985. kfree(tun_qp->ring);
  986. tun_qp->ring = NULL;
  987. return -ENOMEM;
  988. }
  989. if (is_tun) {
  990. rx_buf_size = sizeof (struct mlx4_tunnel_mad);
  991. tx_buf_size = sizeof (struct mlx4_rcv_tunnel_mad);
  992. } else {
  993. rx_buf_size = sizeof (struct mlx4_mad_rcv_buf);
  994. tx_buf_size = sizeof (struct mlx4_mad_snd_buf);
  995. }
  996. for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) {
  997. tun_qp->ring[i].addr = kmalloc(rx_buf_size, GFP_KERNEL);
  998. if (!tun_qp->ring[i].addr)
  999. goto err;
  1000. tun_qp->ring[i].map = ib_dma_map_single(ctx->ib_dev,
  1001. tun_qp->ring[i].addr,
  1002. rx_buf_size,
  1003. DMA_FROM_DEVICE);
  1004. }
  1005. for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) {
  1006. tun_qp->tx_ring[i].buf.addr =
  1007. kmalloc(tx_buf_size, GFP_KERNEL);
  1008. if (!tun_qp->tx_ring[i].buf.addr)
  1009. goto tx_err;
  1010. tun_qp->tx_ring[i].buf.map =
  1011. ib_dma_map_single(ctx->ib_dev,
  1012. tun_qp->tx_ring[i].buf.addr,
  1013. tx_buf_size,
  1014. DMA_TO_DEVICE);
  1015. tun_qp->tx_ring[i].ah = NULL;
  1016. }
  1017. spin_lock_init(&tun_qp->tx_lock);
  1018. tun_qp->tx_ix_head = 0;
  1019. tun_qp->tx_ix_tail = 0;
  1020. tun_qp->proxy_qpt = qp_type;
  1021. return 0;
  1022. tx_err:
  1023. while (i > 0) {
  1024. --i;
  1025. ib_dma_unmap_single(ctx->ib_dev, tun_qp->tx_ring[i].buf.map,
  1026. tx_buf_size, DMA_TO_DEVICE);
  1027. kfree(tun_qp->tx_ring[i].buf.addr);
  1028. }
  1029. kfree(tun_qp->tx_ring);
  1030. tun_qp->tx_ring = NULL;
  1031. i = MLX4_NUM_TUNNEL_BUFS;
  1032. err:
  1033. while (i > 0) {
  1034. --i;
  1035. ib_dma_unmap_single(ctx->ib_dev, tun_qp->ring[i].map,
  1036. rx_buf_size, DMA_FROM_DEVICE);
  1037. kfree(tun_qp->ring[i].addr);
  1038. }
  1039. kfree(tun_qp->ring);
  1040. tun_qp->ring = NULL;
  1041. return -ENOMEM;
  1042. }
  1043. static void mlx4_ib_free_pv_qp_bufs(struct mlx4_ib_demux_pv_ctx *ctx,
  1044. enum ib_qp_type qp_type, int is_tun)
  1045. {
  1046. int i;
  1047. struct mlx4_ib_demux_pv_qp *tun_qp;
  1048. int rx_buf_size, tx_buf_size;
  1049. if (qp_type > IB_QPT_GSI)
  1050. return;
  1051. tun_qp = &ctx->qp[qp_type];
  1052. if (is_tun) {
  1053. rx_buf_size = sizeof (struct mlx4_tunnel_mad);
  1054. tx_buf_size = sizeof (struct mlx4_rcv_tunnel_mad);
  1055. } else {
  1056. rx_buf_size = sizeof (struct mlx4_mad_rcv_buf);
  1057. tx_buf_size = sizeof (struct mlx4_mad_snd_buf);
  1058. }
  1059. for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) {
  1060. ib_dma_unmap_single(ctx->ib_dev, tun_qp->ring[i].map,
  1061. rx_buf_size, DMA_FROM_DEVICE);
  1062. kfree(tun_qp->ring[i].addr);
  1063. }
  1064. for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) {
  1065. ib_dma_unmap_single(ctx->ib_dev, tun_qp->tx_ring[i].buf.map,
  1066. tx_buf_size, DMA_TO_DEVICE);
  1067. kfree(tun_qp->tx_ring[i].buf.addr);
  1068. if (tun_qp->tx_ring[i].ah)
  1069. ib_destroy_ah(tun_qp->tx_ring[i].ah);
  1070. }
  1071. kfree(tun_qp->tx_ring);
  1072. kfree(tun_qp->ring);
  1073. }
  1074. static void mlx4_ib_tunnel_comp_worker(struct work_struct *work)
  1075. {
  1076. struct mlx4_ib_demux_pv_ctx *ctx;
  1077. struct mlx4_ib_demux_pv_qp *tun_qp;
  1078. struct ib_wc wc;
  1079. int ret;
  1080. ctx = container_of(work, struct mlx4_ib_demux_pv_ctx, work);
  1081. ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP);
  1082. while (ib_poll_cq(ctx->cq, 1, &wc) == 1) {
  1083. tun_qp = &ctx->qp[MLX4_TUN_WRID_QPN(wc.wr_id)];
  1084. if (wc.status == IB_WC_SUCCESS) {
  1085. switch (wc.opcode) {
  1086. case IB_WC_RECV:
  1087. mlx4_ib_multiplex_mad(ctx, &wc);
  1088. ret = mlx4_ib_post_pv_qp_buf(ctx, tun_qp,
  1089. wc.wr_id &
  1090. (MLX4_NUM_TUNNEL_BUFS - 1));
  1091. if (ret)
  1092. pr_err("Failed reposting tunnel "
  1093. "buf:%lld\n", wc.wr_id);
  1094. break;
  1095. case IB_WC_SEND:
  1096. pr_debug("received tunnel send completion:"
  1097. "wrid=0x%llx, status=0x%x\n",
  1098. wc.wr_id, wc.status);
  1099. ib_destroy_ah(tun_qp->tx_ring[wc.wr_id &
  1100. (MLX4_NUM_TUNNEL_BUFS - 1)].ah);
  1101. tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah
  1102. = NULL;
  1103. spin_lock(&tun_qp->tx_lock);
  1104. tun_qp->tx_ix_tail++;
  1105. spin_unlock(&tun_qp->tx_lock);
  1106. break;
  1107. default:
  1108. break;
  1109. }
  1110. } else {
  1111. pr_debug("mlx4_ib: completion error in tunnel: %d."
  1112. " status = %d, wrid = 0x%llx\n",
  1113. ctx->slave, wc.status, wc.wr_id);
  1114. if (!MLX4_TUN_IS_RECV(wc.wr_id)) {
  1115. ib_destroy_ah(tun_qp->tx_ring[wc.wr_id &
  1116. (MLX4_NUM_TUNNEL_BUFS - 1)].ah);
  1117. tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah
  1118. = NULL;
  1119. spin_lock(&tun_qp->tx_lock);
  1120. tun_qp->tx_ix_tail++;
  1121. spin_unlock(&tun_qp->tx_lock);
  1122. }
  1123. }
  1124. }
  1125. }
  1126. static void pv_qp_event_handler(struct ib_event *event, void *qp_context)
  1127. {
  1128. struct mlx4_ib_demux_pv_ctx *sqp = qp_context;
  1129. /* It's worse than that! He's dead, Jim! */
  1130. pr_err("Fatal error (%d) on a MAD QP on port %d\n",
  1131. event->event, sqp->port);
  1132. }
  1133. static int create_pv_sqp(struct mlx4_ib_demux_pv_ctx *ctx,
  1134. enum ib_qp_type qp_type, int create_tun)
  1135. {
  1136. int i, ret;
  1137. struct mlx4_ib_demux_pv_qp *tun_qp;
  1138. struct mlx4_ib_qp_tunnel_init_attr qp_init_attr;
  1139. struct ib_qp_attr attr;
  1140. int qp_attr_mask_INIT;
  1141. if (qp_type > IB_QPT_GSI)
  1142. return -EINVAL;
  1143. tun_qp = &ctx->qp[qp_type];
  1144. memset(&qp_init_attr, 0, sizeof qp_init_attr);
  1145. qp_init_attr.init_attr.send_cq = ctx->cq;
  1146. qp_init_attr.init_attr.recv_cq = ctx->cq;
  1147. qp_init_attr.init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
  1148. qp_init_attr.init_attr.cap.max_send_wr = MLX4_NUM_TUNNEL_BUFS;
  1149. qp_init_attr.init_attr.cap.max_recv_wr = MLX4_NUM_TUNNEL_BUFS;
  1150. qp_init_attr.init_attr.cap.max_send_sge = 1;
  1151. qp_init_attr.init_attr.cap.max_recv_sge = 1;
  1152. if (create_tun) {
  1153. qp_init_attr.init_attr.qp_type = IB_QPT_UD;
  1154. qp_init_attr.init_attr.create_flags = MLX4_IB_SRIOV_TUNNEL_QP;
  1155. qp_init_attr.port = ctx->port;
  1156. qp_init_attr.slave = ctx->slave;
  1157. qp_init_attr.proxy_qp_type = qp_type;
  1158. qp_attr_mask_INIT = IB_QP_STATE | IB_QP_PKEY_INDEX |
  1159. IB_QP_QKEY | IB_QP_PORT;
  1160. } else {
  1161. qp_init_attr.init_attr.qp_type = qp_type;
  1162. qp_init_attr.init_attr.create_flags = MLX4_IB_SRIOV_SQP;
  1163. qp_attr_mask_INIT = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_QKEY;
  1164. }
  1165. qp_init_attr.init_attr.port_num = ctx->port;
  1166. qp_init_attr.init_attr.qp_context = ctx;
  1167. qp_init_attr.init_attr.event_handler = pv_qp_event_handler;
  1168. tun_qp->qp = ib_create_qp(ctx->pd, &qp_init_attr.init_attr);
  1169. if (IS_ERR(tun_qp->qp)) {
  1170. ret = PTR_ERR(tun_qp->qp);
  1171. tun_qp->qp = NULL;
  1172. pr_err("Couldn't create %s QP (%d)\n",
  1173. create_tun ? "tunnel" : "special", ret);
  1174. return ret;
  1175. }
  1176. memset(&attr, 0, sizeof attr);
  1177. attr.qp_state = IB_QPS_INIT;
  1178. attr.pkey_index =
  1179. to_mdev(ctx->ib_dev)->pkeys.virt2phys_pkey[ctx->slave][ctx->port - 1][0];
  1180. attr.qkey = IB_QP1_QKEY;
  1181. attr.port_num = ctx->port;
  1182. ret = ib_modify_qp(tun_qp->qp, &attr, qp_attr_mask_INIT);
  1183. if (ret) {
  1184. pr_err("Couldn't change %s qp state to INIT (%d)\n",
  1185. create_tun ? "tunnel" : "special", ret);
  1186. goto err_qp;
  1187. }
  1188. attr.qp_state = IB_QPS_RTR;
  1189. ret = ib_modify_qp(tun_qp->qp, &attr, IB_QP_STATE);
  1190. if (ret) {
  1191. pr_err("Couldn't change %s qp state to RTR (%d)\n",
  1192. create_tun ? "tunnel" : "special", ret);
  1193. goto err_qp;
  1194. }
  1195. attr.qp_state = IB_QPS_RTS;
  1196. attr.sq_psn = 0;
  1197. ret = ib_modify_qp(tun_qp->qp, &attr, IB_QP_STATE | IB_QP_SQ_PSN);
  1198. if (ret) {
  1199. pr_err("Couldn't change %s qp state to RTS (%d)\n",
  1200. create_tun ? "tunnel" : "special", ret);
  1201. goto err_qp;
  1202. }
  1203. for (i = 0; i < MLX4_NUM_TUNNEL_BUFS; i++) {
  1204. ret = mlx4_ib_post_pv_qp_buf(ctx, tun_qp, i);
  1205. if (ret) {
  1206. pr_err(" mlx4_ib_post_pv_buf error"
  1207. " (err = %d, i = %d)\n", ret, i);
  1208. goto err_qp;
  1209. }
  1210. }
  1211. return 0;
  1212. err_qp:
  1213. ib_destroy_qp(tun_qp->qp);
  1214. tun_qp->qp = NULL;
  1215. return ret;
  1216. }
  1217. /*
  1218. * IB MAD completion callback for real SQPs
  1219. */
  1220. static void mlx4_ib_sqp_comp_worker(struct work_struct *work)
  1221. {
  1222. struct mlx4_ib_demux_pv_ctx *ctx;
  1223. struct mlx4_ib_demux_pv_qp *sqp;
  1224. struct ib_wc wc;
  1225. struct ib_grh *grh;
  1226. struct ib_mad *mad;
  1227. ctx = container_of(work, struct mlx4_ib_demux_pv_ctx, work);
  1228. ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP);
  1229. while (mlx4_ib_poll_cq(ctx->cq, 1, &wc) == 1) {
  1230. sqp = &ctx->qp[MLX4_TUN_WRID_QPN(wc.wr_id)];
  1231. if (wc.status == IB_WC_SUCCESS) {
  1232. switch (wc.opcode) {
  1233. case IB_WC_SEND:
  1234. ib_destroy_ah(sqp->tx_ring[wc.wr_id &
  1235. (MLX4_NUM_TUNNEL_BUFS - 1)].ah);
  1236. sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah
  1237. = NULL;
  1238. spin_lock(&sqp->tx_lock);
  1239. sqp->tx_ix_tail++;
  1240. spin_unlock(&sqp->tx_lock);
  1241. break;
  1242. case IB_WC_RECV:
  1243. mad = (struct ib_mad *) &(((struct mlx4_mad_rcv_buf *)
  1244. (sqp->ring[wc.wr_id &
  1245. (MLX4_NUM_TUNNEL_BUFS - 1)].addr))->payload);
  1246. grh = &(((struct mlx4_mad_rcv_buf *)
  1247. (sqp->ring[wc.wr_id &
  1248. (MLX4_NUM_TUNNEL_BUFS - 1)].addr))->grh);
  1249. mlx4_ib_demux_mad(ctx->ib_dev, ctx->port, &wc, grh, mad);
  1250. if (mlx4_ib_post_pv_qp_buf(ctx, sqp, wc.wr_id &
  1251. (MLX4_NUM_TUNNEL_BUFS - 1)))
  1252. pr_err("Failed reposting SQP "
  1253. "buf:%lld\n", wc.wr_id);
  1254. break;
  1255. default:
  1256. BUG_ON(1);
  1257. break;
  1258. }
  1259. } else {
  1260. pr_debug("mlx4_ib: completion error in tunnel: %d."
  1261. " status = %d, wrid = 0x%llx\n",
  1262. ctx->slave, wc.status, wc.wr_id);
  1263. if (!MLX4_TUN_IS_RECV(wc.wr_id)) {
  1264. ib_destroy_ah(sqp->tx_ring[wc.wr_id &
  1265. (MLX4_NUM_TUNNEL_BUFS - 1)].ah);
  1266. sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah
  1267. = NULL;
  1268. spin_lock(&sqp->tx_lock);
  1269. sqp->tx_ix_tail++;
  1270. spin_unlock(&sqp->tx_lock);
  1271. }
  1272. }
  1273. }
  1274. }
  1275. static int alloc_pv_object(struct mlx4_ib_dev *dev, int slave, int port,
  1276. struct mlx4_ib_demux_pv_ctx **ret_ctx)
  1277. {
  1278. struct mlx4_ib_demux_pv_ctx *ctx;
  1279. *ret_ctx = NULL;
  1280. ctx = kzalloc(sizeof (struct mlx4_ib_demux_pv_ctx), GFP_KERNEL);
  1281. if (!ctx) {
  1282. pr_err("failed allocating pv resource context "
  1283. "for port %d, slave %d\n", port, slave);
  1284. return -ENOMEM;
  1285. }
  1286. ctx->ib_dev = &dev->ib_dev;
  1287. ctx->port = port;
  1288. ctx->slave = slave;
  1289. *ret_ctx = ctx;
  1290. return 0;
  1291. }
  1292. static void free_pv_object(struct mlx4_ib_dev *dev, int slave, int port)
  1293. {
  1294. if (dev->sriov.demux[port - 1].tun[slave]) {
  1295. kfree(dev->sriov.demux[port - 1].tun[slave]);
  1296. dev->sriov.demux[port - 1].tun[slave] = NULL;
  1297. }
  1298. }
  1299. static int create_pv_resources(struct ib_device *ibdev, int slave, int port,
  1300. int create_tun, struct mlx4_ib_demux_pv_ctx *ctx)
  1301. {
  1302. int ret, cq_size;
  1303. ctx->state = DEMUX_PV_STATE_STARTING;
  1304. /* have QP0 only on port owner, and only if link layer is IB */
  1305. if (ctx->slave == mlx4_master_func_num(to_mdev(ctx->ib_dev)->dev) &&
  1306. rdma_port_get_link_layer(ibdev, ctx->port) == IB_LINK_LAYER_INFINIBAND)
  1307. ctx->has_smi = 1;
  1308. if (ctx->has_smi) {
  1309. ret = mlx4_ib_alloc_pv_bufs(ctx, IB_QPT_SMI, create_tun);
  1310. if (ret) {
  1311. pr_err("Failed allocating qp0 tunnel bufs (%d)\n", ret);
  1312. goto err_out;
  1313. }
  1314. }
  1315. ret = mlx4_ib_alloc_pv_bufs(ctx, IB_QPT_GSI, create_tun);
  1316. if (ret) {
  1317. pr_err("Failed allocating qp1 tunnel bufs (%d)\n", ret);
  1318. goto err_out_qp0;
  1319. }
  1320. cq_size = 2 * MLX4_NUM_TUNNEL_BUFS;
  1321. if (ctx->has_smi)
  1322. cq_size *= 2;
  1323. ctx->cq = ib_create_cq(ctx->ib_dev, mlx4_ib_tunnel_comp_handler,
  1324. NULL, ctx, cq_size, 0);
  1325. if (IS_ERR(ctx->cq)) {
  1326. ret = PTR_ERR(ctx->cq);
  1327. pr_err("Couldn't create tunnel CQ (%d)\n", ret);
  1328. goto err_buf;
  1329. }
  1330. ctx->pd = ib_alloc_pd(ctx->ib_dev);
  1331. if (IS_ERR(ctx->pd)) {
  1332. ret = PTR_ERR(ctx->pd);
  1333. pr_err("Couldn't create tunnel PD (%d)\n", ret);
  1334. goto err_cq;
  1335. }
  1336. ctx->mr = ib_get_dma_mr(ctx->pd, IB_ACCESS_LOCAL_WRITE);
  1337. if (IS_ERR(ctx->mr)) {
  1338. ret = PTR_ERR(ctx->mr);
  1339. pr_err("Couldn't get tunnel DMA MR (%d)\n", ret);
  1340. goto err_pd;
  1341. }
  1342. if (ctx->has_smi) {
  1343. ret = create_pv_sqp(ctx, IB_QPT_SMI, create_tun);
  1344. if (ret) {
  1345. pr_err("Couldn't create %s QP0 (%d)\n",
  1346. create_tun ? "tunnel for" : "", ret);
  1347. goto err_mr;
  1348. }
  1349. }
  1350. ret = create_pv_sqp(ctx, IB_QPT_GSI, create_tun);
  1351. if (ret) {
  1352. pr_err("Couldn't create %s QP1 (%d)\n",
  1353. create_tun ? "tunnel for" : "", ret);
  1354. goto err_qp0;
  1355. }
  1356. if (create_tun)
  1357. INIT_WORK(&ctx->work, mlx4_ib_tunnel_comp_worker);
  1358. else
  1359. INIT_WORK(&ctx->work, mlx4_ib_sqp_comp_worker);
  1360. ctx->wq = to_mdev(ibdev)->sriov.demux[port - 1].wq;
  1361. ret = ib_req_notify_cq(ctx->cq, IB_CQ_NEXT_COMP);
  1362. if (ret) {
  1363. pr_err("Couldn't arm tunnel cq (%d)\n", ret);
  1364. goto err_wq;
  1365. }
  1366. ctx->state = DEMUX_PV_STATE_ACTIVE;
  1367. return 0;
  1368. err_wq:
  1369. ctx->wq = NULL;
  1370. ib_destroy_qp(ctx->qp[1].qp);
  1371. ctx->qp[1].qp = NULL;
  1372. err_qp0:
  1373. if (ctx->has_smi)
  1374. ib_destroy_qp(ctx->qp[0].qp);
  1375. ctx->qp[0].qp = NULL;
  1376. err_mr:
  1377. ib_dereg_mr(ctx->mr);
  1378. ctx->mr = NULL;
  1379. err_pd:
  1380. ib_dealloc_pd(ctx->pd);
  1381. ctx->pd = NULL;
  1382. err_cq:
  1383. ib_destroy_cq(ctx->cq);
  1384. ctx->cq = NULL;
  1385. err_buf:
  1386. mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_GSI, create_tun);
  1387. err_out_qp0:
  1388. if (ctx->has_smi)
  1389. mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_SMI, create_tun);
  1390. err_out:
  1391. ctx->state = DEMUX_PV_STATE_DOWN;
  1392. return ret;
  1393. }
  1394. static void destroy_pv_resources(struct mlx4_ib_dev *dev, int slave, int port,
  1395. struct mlx4_ib_demux_pv_ctx *ctx, int flush)
  1396. {
  1397. if (!ctx)
  1398. return;
  1399. if (ctx->state > DEMUX_PV_STATE_DOWN) {
  1400. ctx->state = DEMUX_PV_STATE_DOWNING;
  1401. if (flush)
  1402. flush_workqueue(ctx->wq);
  1403. if (ctx->has_smi) {
  1404. ib_destroy_qp(ctx->qp[0].qp);
  1405. ctx->qp[0].qp = NULL;
  1406. mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_SMI, 1);
  1407. }
  1408. ib_destroy_qp(ctx->qp[1].qp);
  1409. ctx->qp[1].qp = NULL;
  1410. mlx4_ib_free_pv_qp_bufs(ctx, IB_QPT_GSI, 1);
  1411. ib_dereg_mr(ctx->mr);
  1412. ctx->mr = NULL;
  1413. ib_dealloc_pd(ctx->pd);
  1414. ctx->pd = NULL;
  1415. ib_destroy_cq(ctx->cq);
  1416. ctx->cq = NULL;
  1417. ctx->state = DEMUX_PV_STATE_DOWN;
  1418. }
  1419. }
  1420. static int mlx4_ib_tunnels_update(struct mlx4_ib_dev *dev, int slave,
  1421. int port, int do_init)
  1422. {
  1423. int ret = 0;
  1424. if (!do_init) {
  1425. clean_vf_mcast(&dev->sriov.demux[port - 1], slave);
  1426. /* for master, destroy real sqp resources */
  1427. if (slave == mlx4_master_func_num(dev->dev))
  1428. destroy_pv_resources(dev, slave, port,
  1429. dev->sriov.sqps[port - 1], 1);
  1430. /* destroy the tunnel qp resources */
  1431. destroy_pv_resources(dev, slave, port,
  1432. dev->sriov.demux[port - 1].tun[slave], 1);
  1433. return 0;
  1434. }
  1435. /* create the tunnel qp resources */
  1436. ret = create_pv_resources(&dev->ib_dev, slave, port, 1,
  1437. dev->sriov.demux[port - 1].tun[slave]);
  1438. /* for master, create the real sqp resources */
  1439. if (!ret && slave == mlx4_master_func_num(dev->dev))
  1440. ret = create_pv_resources(&dev->ib_dev, slave, port, 0,
  1441. dev->sriov.sqps[port - 1]);
  1442. return ret;
  1443. }
  1444. void mlx4_ib_tunnels_update_work(struct work_struct *work)
  1445. {
  1446. struct mlx4_ib_demux_work *dmxw;
  1447. dmxw = container_of(work, struct mlx4_ib_demux_work, work);
  1448. mlx4_ib_tunnels_update(dmxw->dev, dmxw->slave, (int) dmxw->port,
  1449. dmxw->do_init);
  1450. kfree(dmxw);
  1451. return;
  1452. }
  1453. static int mlx4_ib_alloc_demux_ctx(struct mlx4_ib_dev *dev,
  1454. struct mlx4_ib_demux_ctx *ctx,
  1455. int port)
  1456. {
  1457. char name[12];
  1458. int ret = 0;
  1459. int i;
  1460. ctx->tun = kcalloc(dev->dev->caps.sqp_demux,
  1461. sizeof (struct mlx4_ib_demux_pv_ctx *), GFP_KERNEL);
  1462. if (!ctx->tun)
  1463. return -ENOMEM;
  1464. ctx->dev = dev;
  1465. ctx->port = port;
  1466. ctx->ib_dev = &dev->ib_dev;
  1467. for (i = 0; i < dev->dev->caps.sqp_demux; i++) {
  1468. ret = alloc_pv_object(dev, i, port, &ctx->tun[i]);
  1469. if (ret) {
  1470. ret = -ENOMEM;
  1471. goto err_mcg;
  1472. }
  1473. }
  1474. ret = mlx4_ib_mcg_port_init(ctx);
  1475. if (ret) {
  1476. pr_err("Failed initializing mcg para-virt (%d)\n", ret);
  1477. goto err_mcg;
  1478. }
  1479. snprintf(name, sizeof name, "mlx4_ibt%d", port);
  1480. ctx->wq = create_singlethread_workqueue(name);
  1481. if (!ctx->wq) {
  1482. pr_err("Failed to create tunnelling WQ for port %d\n", port);
  1483. ret = -ENOMEM;
  1484. goto err_wq;
  1485. }
  1486. snprintf(name, sizeof name, "mlx4_ibud%d", port);
  1487. ctx->ud_wq = create_singlethread_workqueue(name);
  1488. if (!ctx->ud_wq) {
  1489. pr_err("Failed to create up/down WQ for port %d\n", port);
  1490. ret = -ENOMEM;
  1491. goto err_udwq;
  1492. }
  1493. return 0;
  1494. err_udwq:
  1495. destroy_workqueue(ctx->wq);
  1496. ctx->wq = NULL;
  1497. err_wq:
  1498. mlx4_ib_mcg_port_cleanup(ctx, 1);
  1499. err_mcg:
  1500. for (i = 0; i < dev->dev->caps.sqp_demux; i++)
  1501. free_pv_object(dev, i, port);
  1502. kfree(ctx->tun);
  1503. ctx->tun = NULL;
  1504. return ret;
  1505. }
  1506. static void mlx4_ib_free_sqp_ctx(struct mlx4_ib_demux_pv_ctx *sqp_ctx)
  1507. {
  1508. if (sqp_ctx->state > DEMUX_PV_STATE_DOWN) {
  1509. sqp_ctx->state = DEMUX_PV_STATE_DOWNING;
  1510. flush_workqueue(sqp_ctx->wq);
  1511. if (sqp_ctx->has_smi) {
  1512. ib_destroy_qp(sqp_ctx->qp[0].qp);
  1513. sqp_ctx->qp[0].qp = NULL;
  1514. mlx4_ib_free_pv_qp_bufs(sqp_ctx, IB_QPT_SMI, 0);
  1515. }
  1516. ib_destroy_qp(sqp_ctx->qp[1].qp);
  1517. sqp_ctx->qp[1].qp = NULL;
  1518. mlx4_ib_free_pv_qp_bufs(sqp_ctx, IB_QPT_GSI, 0);
  1519. ib_dereg_mr(sqp_ctx->mr);
  1520. sqp_ctx->mr = NULL;
  1521. ib_dealloc_pd(sqp_ctx->pd);
  1522. sqp_ctx->pd = NULL;
  1523. ib_destroy_cq(sqp_ctx->cq);
  1524. sqp_ctx->cq = NULL;
  1525. sqp_ctx->state = DEMUX_PV_STATE_DOWN;
  1526. }
  1527. }
  1528. static void mlx4_ib_free_demux_ctx(struct mlx4_ib_demux_ctx *ctx)
  1529. {
  1530. int i;
  1531. if (ctx) {
  1532. struct mlx4_ib_dev *dev = to_mdev(ctx->ib_dev);
  1533. mlx4_ib_mcg_port_cleanup(ctx, 1);
  1534. for (i = 0; i < dev->dev->caps.sqp_demux; i++) {
  1535. if (!ctx->tun[i])
  1536. continue;
  1537. if (ctx->tun[i]->state > DEMUX_PV_STATE_DOWN)
  1538. ctx->tun[i]->state = DEMUX_PV_STATE_DOWNING;
  1539. }
  1540. flush_workqueue(ctx->wq);
  1541. for (i = 0; i < dev->dev->caps.sqp_demux; i++) {
  1542. destroy_pv_resources(dev, i, ctx->port, ctx->tun[i], 0);
  1543. free_pv_object(dev, i, ctx->port);
  1544. }
  1545. kfree(ctx->tun);
  1546. destroy_workqueue(ctx->ud_wq);
  1547. destroy_workqueue(ctx->wq);
  1548. }
  1549. }
  1550. static void mlx4_ib_master_tunnels(struct mlx4_ib_dev *dev, int do_init)
  1551. {
  1552. int i;
  1553. if (!mlx4_is_master(dev->dev))
  1554. return;
  1555. /* initialize or tear down tunnel QPs for the master */
  1556. for (i = 0; i < dev->dev->caps.num_ports; i++)
  1557. mlx4_ib_tunnels_update(dev, mlx4_master_func_num(dev->dev), i + 1, do_init);
  1558. return;
  1559. }
  1560. int mlx4_ib_init_sriov(struct mlx4_ib_dev *dev)
  1561. {
  1562. int i = 0;
  1563. int err;
  1564. if (!mlx4_is_mfunc(dev->dev))
  1565. return 0;
  1566. dev->sriov.is_going_down = 0;
  1567. spin_lock_init(&dev->sriov.going_down_lock);
  1568. mlx4_ib_cm_paravirt_init(dev);
  1569. mlx4_ib_warn(&dev->ib_dev, "multi-function enabled\n");
  1570. if (mlx4_is_slave(dev->dev)) {
  1571. mlx4_ib_warn(&dev->ib_dev, "operating in qp1 tunnel mode\n");
  1572. return 0;
  1573. }
  1574. err = mlx4_ib_init_alias_guid_service(dev);
  1575. if (err) {
  1576. mlx4_ib_warn(&dev->ib_dev, "Failed init alias guid process.\n");
  1577. goto paravirt_err;
  1578. }
  1579. mlx4_ib_warn(&dev->ib_dev, "initializing demux service for %d qp1 clients\n",
  1580. dev->dev->caps.sqp_demux);
  1581. for (i = 0; i < dev->num_ports; i++) {
  1582. union ib_gid gid;
  1583. err = __mlx4_ib_query_gid(&dev->ib_dev, i + 1, 0, &gid, 1);
  1584. if (err)
  1585. goto demux_err;
  1586. dev->sriov.demux[i].guid_cache[0] = gid.global.interface_id;
  1587. err = alloc_pv_object(dev, mlx4_master_func_num(dev->dev), i + 1,
  1588. &dev->sriov.sqps[i]);
  1589. if (err)
  1590. goto demux_err;
  1591. err = mlx4_ib_alloc_demux_ctx(dev, &dev->sriov.demux[i], i + 1);
  1592. if (err)
  1593. goto demux_err;
  1594. }
  1595. mlx4_ib_master_tunnels(dev, 1);
  1596. return 0;
  1597. demux_err:
  1598. while (i > 0) {
  1599. free_pv_object(dev, mlx4_master_func_num(dev->dev), i + 1);
  1600. mlx4_ib_free_demux_ctx(&dev->sriov.demux[i]);
  1601. --i;
  1602. }
  1603. mlx4_ib_destroy_alias_guid_service(dev);
  1604. paravirt_err:
  1605. mlx4_ib_cm_paravirt_clean(dev, -1);
  1606. return err;
  1607. }
  1608. void mlx4_ib_close_sriov(struct mlx4_ib_dev *dev)
  1609. {
  1610. int i;
  1611. unsigned long flags;
  1612. if (!mlx4_is_mfunc(dev->dev))
  1613. return;
  1614. spin_lock_irqsave(&dev->sriov.going_down_lock, flags);
  1615. dev->sriov.is_going_down = 1;
  1616. spin_unlock_irqrestore(&dev->sriov.going_down_lock, flags);
  1617. if (mlx4_is_master(dev->dev)) {
  1618. for (i = 0; i < dev->num_ports; i++) {
  1619. flush_workqueue(dev->sriov.demux[i].ud_wq);
  1620. mlx4_ib_free_sqp_ctx(dev->sriov.sqps[i]);
  1621. kfree(dev->sriov.sqps[i]);
  1622. dev->sriov.sqps[i] = NULL;
  1623. mlx4_ib_free_demux_ctx(&dev->sriov.demux[i]);
  1624. }
  1625. mlx4_ib_cm_paravirt_clean(dev, -1);
  1626. mlx4_ib_destroy_alias_guid_service(dev);
  1627. }
  1628. }