eq.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696
  1. /*
  2. * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
  3. * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
  4. *
  5. * This software is available to you under a choice of one of two
  6. * licenses. You may choose to be licensed under the terms of the GNU
  7. * General Public License (GPL) Version 2, available from the file
  8. * COPYING in the main directory of this source tree, or the
  9. * OpenIB.org BSD license below:
  10. *
  11. * Redistribution and use in source and binary forms, with or
  12. * without modification, are permitted provided that the following
  13. * conditions are met:
  14. *
  15. * - Redistributions of source code must retain the above
  16. * copyright notice, this list of conditions and the following
  17. * disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials
  22. * provided with the distribution.
  23. *
  24. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31. * SOFTWARE.
  32. */
  33. #include <linux/init.h>
  34. #include <linux/interrupt.h>
  35. #include <linux/mlx4/cmd.h>
  36. #include "mlx4.h"
  37. #include "fw.h"
  38. enum {
  39. MLX4_NUM_ASYNC_EQE = 0x100,
  40. MLX4_NUM_SPARE_EQE = 0x80,
  41. MLX4_EQ_ENTRY_SIZE = 0x20
  42. };
  43. /*
  44. * Must be packed because start is 64 bits but only aligned to 32 bits.
  45. */
  46. struct mlx4_eq_context {
  47. __be32 flags;
  48. u16 reserved1[3];
  49. __be16 page_offset;
  50. u8 log_eq_size;
  51. u8 reserved2[4];
  52. u8 eq_period;
  53. u8 reserved3;
  54. u8 eq_max_count;
  55. u8 reserved4[3];
  56. u8 intr;
  57. u8 log_page_size;
  58. u8 reserved5[2];
  59. u8 mtt_base_addr_h;
  60. __be32 mtt_base_addr_l;
  61. u32 reserved6[2];
  62. __be32 consumer_index;
  63. __be32 producer_index;
  64. u32 reserved7[4];
  65. };
  66. #define MLX4_EQ_STATUS_OK ( 0 << 28)
  67. #define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28)
  68. #define MLX4_EQ_OWNER_SW ( 0 << 24)
  69. #define MLX4_EQ_OWNER_HW ( 1 << 24)
  70. #define MLX4_EQ_FLAG_EC ( 1 << 18)
  71. #define MLX4_EQ_FLAG_OI ( 1 << 17)
  72. #define MLX4_EQ_STATE_ARMED ( 9 << 8)
  73. #define MLX4_EQ_STATE_FIRED (10 << 8)
  74. #define MLX4_EQ_STATE_ALWAYS_ARMED (11 << 8)
  75. #define MLX4_ASYNC_EVENT_MASK ((1ull << MLX4_EVENT_TYPE_PATH_MIG) | \
  76. (1ull << MLX4_EVENT_TYPE_COMM_EST) | \
  77. (1ull << MLX4_EVENT_TYPE_SQ_DRAINED) | \
  78. (1ull << MLX4_EVENT_TYPE_CQ_ERROR) | \
  79. (1ull << MLX4_EVENT_TYPE_WQ_CATAS_ERROR) | \
  80. (1ull << MLX4_EVENT_TYPE_EEC_CATAS_ERROR) | \
  81. (1ull << MLX4_EVENT_TYPE_PATH_MIG_FAILED) | \
  82. (1ull << MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
  83. (1ull << MLX4_EVENT_TYPE_WQ_ACCESS_ERROR) | \
  84. (1ull << MLX4_EVENT_TYPE_LOCAL_CATAS_ERROR) | \
  85. (1ull << MLX4_EVENT_TYPE_PORT_CHANGE) | \
  86. (1ull << MLX4_EVENT_TYPE_ECC_DETECT) | \
  87. (1ull << MLX4_EVENT_TYPE_SRQ_CATAS_ERROR) | \
  88. (1ull << MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE) | \
  89. (1ull << MLX4_EVENT_TYPE_SRQ_LIMIT) | \
  90. (1ull << MLX4_EVENT_TYPE_CMD))
  91. #define MLX4_CATAS_EVENT_MASK (1ull << MLX4_EVENT_TYPE_LOCAL_CATAS_ERROR)
  92. struct mlx4_eqe {
  93. u8 reserved1;
  94. u8 type;
  95. u8 reserved2;
  96. u8 subtype;
  97. union {
  98. u32 raw[6];
  99. struct {
  100. __be32 cqn;
  101. } __attribute__((packed)) comp;
  102. struct {
  103. u16 reserved1;
  104. __be16 token;
  105. u32 reserved2;
  106. u8 reserved3[3];
  107. u8 status;
  108. __be64 out_param;
  109. } __attribute__((packed)) cmd;
  110. struct {
  111. __be32 qpn;
  112. } __attribute__((packed)) qp;
  113. struct {
  114. __be32 srqn;
  115. } __attribute__((packed)) srq;
  116. struct {
  117. __be32 cqn;
  118. u32 reserved1;
  119. u8 reserved2[3];
  120. u8 syndrome;
  121. } __attribute__((packed)) cq_err;
  122. struct {
  123. u32 reserved1[2];
  124. __be32 port;
  125. } __attribute__((packed)) port_change;
  126. } event;
  127. u8 reserved3[3];
  128. u8 owner;
  129. } __attribute__((packed));
  130. static void eq_set_ci(struct mlx4_eq *eq, int req_not)
  131. {
  132. __raw_writel((__force u32) cpu_to_be32((eq->cons_index & 0xffffff) |
  133. req_not << 31),
  134. eq->doorbell);
  135. /* We still want ordering, just not swabbing, so add a barrier */
  136. mb();
  137. }
  138. static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry)
  139. {
  140. unsigned long off = (entry & (eq->nent - 1)) * MLX4_EQ_ENTRY_SIZE;
  141. return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE;
  142. }
  143. static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq)
  144. {
  145. struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index);
  146. return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe;
  147. }
  148. static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
  149. {
  150. struct mlx4_eqe *eqe;
  151. int cqn;
  152. int eqes_found = 0;
  153. int set_ci = 0;
  154. while ((eqe = next_eqe_sw(eq))) {
  155. /*
  156. * Make sure we read EQ entry contents after we've
  157. * checked the ownership bit.
  158. */
  159. rmb();
  160. switch (eqe->type) {
  161. case MLX4_EVENT_TYPE_COMP:
  162. cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff;
  163. mlx4_cq_completion(dev, cqn);
  164. break;
  165. case MLX4_EVENT_TYPE_PATH_MIG:
  166. case MLX4_EVENT_TYPE_COMM_EST:
  167. case MLX4_EVENT_TYPE_SQ_DRAINED:
  168. case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
  169. case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
  170. case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
  171. case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
  172. case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
  173. mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
  174. eqe->type);
  175. break;
  176. case MLX4_EVENT_TYPE_SRQ_LIMIT:
  177. case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR:
  178. mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & 0xffffff,
  179. eqe->type);
  180. break;
  181. case MLX4_EVENT_TYPE_CMD:
  182. mlx4_cmd_event(dev,
  183. be16_to_cpu(eqe->event.cmd.token),
  184. eqe->event.cmd.status,
  185. be64_to_cpu(eqe->event.cmd.out_param));
  186. break;
  187. case MLX4_EVENT_TYPE_PORT_CHANGE:
  188. mlx4_dispatch_event(dev, eqe->type, eqe->subtype,
  189. be32_to_cpu(eqe->event.port_change.port) >> 28);
  190. break;
  191. case MLX4_EVENT_TYPE_CQ_ERROR:
  192. mlx4_warn(dev, "CQ %s on CQN %06x\n",
  193. eqe->event.cq_err.syndrome == 1 ?
  194. "overrun" : "access violation",
  195. be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff);
  196. mlx4_cq_event(dev, be32_to_cpu(eqe->event.cq_err.cqn),
  197. eqe->type);
  198. break;
  199. case MLX4_EVENT_TYPE_EQ_OVERFLOW:
  200. mlx4_warn(dev, "EQ overrun on EQN %d\n", eq->eqn);
  201. break;
  202. case MLX4_EVENT_TYPE_EEC_CATAS_ERROR:
  203. case MLX4_EVENT_TYPE_ECC_DETECT:
  204. default:
  205. mlx4_warn(dev, "Unhandled event %02x(%02x) on EQ %d at index %u\n",
  206. eqe->type, eqe->subtype, eq->eqn, eq->cons_index);
  207. break;
  208. };
  209. ++eq->cons_index;
  210. eqes_found = 1;
  211. ++set_ci;
  212. /*
  213. * The HCA will think the queue has overflowed if we
  214. * don't tell it we've been processing events. We
  215. * create our EQs with MLX4_NUM_SPARE_EQE extra
  216. * entries, so we must update our consumer index at
  217. * least that often.
  218. */
  219. if (unlikely(set_ci >= MLX4_NUM_SPARE_EQE)) {
  220. /*
  221. * Conditional on hca_type is OK here because
  222. * this is a rare case, not the fast path.
  223. */
  224. eq_set_ci(eq, 0);
  225. set_ci = 0;
  226. }
  227. }
  228. eq_set_ci(eq, 1);
  229. return eqes_found;
  230. }
  231. static irqreturn_t mlx4_interrupt(int irq, void *dev_ptr)
  232. {
  233. struct mlx4_dev *dev = dev_ptr;
  234. struct mlx4_priv *priv = mlx4_priv(dev);
  235. int work = 0;
  236. int i;
  237. writel(priv->eq_table.clr_mask, priv->eq_table.clr_int);
  238. for (i = 0; i < MLX4_EQ_CATAS; ++i)
  239. work |= mlx4_eq_int(dev, &priv->eq_table.eq[i]);
  240. return IRQ_RETVAL(work);
  241. }
  242. static irqreturn_t mlx4_msi_x_interrupt(int irq, void *eq_ptr)
  243. {
  244. struct mlx4_eq *eq = eq_ptr;
  245. struct mlx4_dev *dev = eq->dev;
  246. mlx4_eq_int(dev, eq);
  247. /* MSI-X vectors always belong to us */
  248. return IRQ_HANDLED;
  249. }
  250. static irqreturn_t mlx4_catas_interrupt(int irq, void *dev_ptr)
  251. {
  252. mlx4_handle_catas_err(dev_ptr);
  253. /* MSI-X vectors always belong to us */
  254. return IRQ_HANDLED;
  255. }
  256. static int mlx4_MAP_EQ(struct mlx4_dev *dev, u64 event_mask, int unmap,
  257. int eq_num)
  258. {
  259. return mlx4_cmd(dev, event_mask, (unmap << 31) | eq_num,
  260. 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B);
  261. }
  262. static int mlx4_SW2HW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
  263. int eq_num)
  264. {
  265. return mlx4_cmd(dev, mailbox->dma, eq_num, 0, MLX4_CMD_SW2HW_EQ,
  266. MLX4_CMD_TIME_CLASS_A);
  267. }
  268. static int mlx4_HW2SW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
  269. int eq_num)
  270. {
  271. return mlx4_cmd_box(dev, 0, mailbox->dma, eq_num, 0, MLX4_CMD_HW2SW_EQ,
  272. MLX4_CMD_TIME_CLASS_A);
  273. }
  274. static void __devinit __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev,
  275. struct mlx4_eq *eq)
  276. {
  277. struct mlx4_priv *priv = mlx4_priv(dev);
  278. int index;
  279. index = eq->eqn / 4 - dev->caps.reserved_eqs / 4;
  280. if (!priv->eq_table.uar_map[index]) {
  281. priv->eq_table.uar_map[index] =
  282. ioremap(pci_resource_start(dev->pdev, 2) +
  283. ((eq->eqn / 4) << PAGE_SHIFT),
  284. PAGE_SIZE);
  285. if (!priv->eq_table.uar_map[index]) {
  286. mlx4_err(dev, "Couldn't map EQ doorbell for EQN 0x%06x\n",
  287. eq->eqn);
  288. return NULL;
  289. }
  290. }
  291. return priv->eq_table.uar_map[index] + 0x800 + 8 * (eq->eqn % 4);
  292. }
  293. static int __devinit mlx4_create_eq(struct mlx4_dev *dev, int nent,
  294. u8 intr, struct mlx4_eq *eq)
  295. {
  296. struct mlx4_priv *priv = mlx4_priv(dev);
  297. struct mlx4_cmd_mailbox *mailbox;
  298. struct mlx4_eq_context *eq_context;
  299. int npages;
  300. u64 *dma_list = NULL;
  301. dma_addr_t t;
  302. u64 mtt_addr;
  303. int err = -ENOMEM;
  304. int i;
  305. eq->dev = dev;
  306. eq->nent = roundup_pow_of_two(max(nent, 2));
  307. npages = PAGE_ALIGN(eq->nent * MLX4_EQ_ENTRY_SIZE) / PAGE_SIZE;
  308. eq->page_list = kmalloc(npages * sizeof *eq->page_list,
  309. GFP_KERNEL);
  310. if (!eq->page_list)
  311. goto err_out;
  312. for (i = 0; i < npages; ++i)
  313. eq->page_list[i].buf = NULL;
  314. dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
  315. if (!dma_list)
  316. goto err_out_free;
  317. mailbox = mlx4_alloc_cmd_mailbox(dev);
  318. if (IS_ERR(mailbox))
  319. goto err_out_free;
  320. eq_context = mailbox->buf;
  321. for (i = 0; i < npages; ++i) {
  322. eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev,
  323. PAGE_SIZE, &t, GFP_KERNEL);
  324. if (!eq->page_list[i].buf)
  325. goto err_out_free_pages;
  326. dma_list[i] = t;
  327. eq->page_list[i].map = t;
  328. memset(eq->page_list[i].buf, 0, PAGE_SIZE);
  329. }
  330. eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap);
  331. if (eq->eqn == -1)
  332. goto err_out_free_pages;
  333. eq->doorbell = mlx4_get_eq_uar(dev, eq);
  334. if (!eq->doorbell) {
  335. err = -ENOMEM;
  336. goto err_out_free_eq;
  337. }
  338. err = mlx4_mtt_init(dev, npages, PAGE_SHIFT, &eq->mtt);
  339. if (err)
  340. goto err_out_free_eq;
  341. err = mlx4_write_mtt(dev, &eq->mtt, 0, npages, dma_list);
  342. if (err)
  343. goto err_out_free_mtt;
  344. memset(eq_context, 0, sizeof *eq_context);
  345. eq_context->flags = cpu_to_be32(MLX4_EQ_STATUS_OK |
  346. MLX4_EQ_STATE_ARMED);
  347. eq_context->log_eq_size = ilog2(eq->nent);
  348. eq_context->intr = intr;
  349. eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT;
  350. mtt_addr = mlx4_mtt_addr(dev, &eq->mtt);
  351. eq_context->mtt_base_addr_h = mtt_addr >> 32;
  352. eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
  353. err = mlx4_SW2HW_EQ(dev, mailbox, eq->eqn);
  354. if (err) {
  355. mlx4_warn(dev, "SW2HW_EQ failed (%d)\n", err);
  356. goto err_out_free_mtt;
  357. }
  358. kfree(dma_list);
  359. mlx4_free_cmd_mailbox(dev, mailbox);
  360. eq->cons_index = 0;
  361. return err;
  362. err_out_free_mtt:
  363. mlx4_mtt_cleanup(dev, &eq->mtt);
  364. err_out_free_eq:
  365. mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn);
  366. err_out_free_pages:
  367. for (i = 0; i < npages; ++i)
  368. if (eq->page_list[i].buf)
  369. dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
  370. eq->page_list[i].buf,
  371. eq->page_list[i].map);
  372. mlx4_free_cmd_mailbox(dev, mailbox);
  373. err_out_free:
  374. kfree(eq->page_list);
  375. kfree(dma_list);
  376. err_out:
  377. return err;
  378. }
  379. static void mlx4_free_eq(struct mlx4_dev *dev,
  380. struct mlx4_eq *eq)
  381. {
  382. struct mlx4_priv *priv = mlx4_priv(dev);
  383. struct mlx4_cmd_mailbox *mailbox;
  384. int err;
  385. int npages = PAGE_ALIGN(MLX4_EQ_ENTRY_SIZE * eq->nent) / PAGE_SIZE;
  386. int i;
  387. mailbox = mlx4_alloc_cmd_mailbox(dev);
  388. if (IS_ERR(mailbox))
  389. return;
  390. err = mlx4_HW2SW_EQ(dev, mailbox, eq->eqn);
  391. if (err)
  392. mlx4_warn(dev, "HW2SW_EQ failed (%d)\n", err);
  393. if (0) {
  394. mlx4_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn);
  395. for (i = 0; i < sizeof (struct mlx4_eq_context) / 4; ++i) {
  396. if (i % 4 == 0)
  397. printk("[%02x] ", i * 4);
  398. printk(" %08x", be32_to_cpup(mailbox->buf + i * 4));
  399. if ((i + 1) % 4 == 0)
  400. printk("\n");
  401. }
  402. }
  403. mlx4_mtt_cleanup(dev, &eq->mtt);
  404. for (i = 0; i < npages; ++i)
  405. pci_free_consistent(dev->pdev, PAGE_SIZE,
  406. eq->page_list[i].buf,
  407. eq->page_list[i].map);
  408. kfree(eq->page_list);
  409. mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn);
  410. mlx4_free_cmd_mailbox(dev, mailbox);
  411. }
  412. static void mlx4_free_irqs(struct mlx4_dev *dev)
  413. {
  414. struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table;
  415. int i;
  416. if (eq_table->have_irq)
  417. free_irq(dev->pdev->irq, dev);
  418. for (i = 0; i < MLX4_NUM_EQ; ++i)
  419. if (eq_table->eq[i].have_irq)
  420. free_irq(eq_table->eq[i].irq, eq_table->eq + i);
  421. }
  422. static int __devinit mlx4_map_clr_int(struct mlx4_dev *dev)
  423. {
  424. struct mlx4_priv *priv = mlx4_priv(dev);
  425. priv->clr_base = ioremap(pci_resource_start(dev->pdev, priv->fw.clr_int_bar) +
  426. priv->fw.clr_int_base, MLX4_CLR_INT_SIZE);
  427. if (!priv->clr_base) {
  428. mlx4_err(dev, "Couldn't map interrupt clear register, aborting.\n");
  429. return -ENOMEM;
  430. }
  431. return 0;
  432. }
  433. static void mlx4_unmap_clr_int(struct mlx4_dev *dev)
  434. {
  435. struct mlx4_priv *priv = mlx4_priv(dev);
  436. iounmap(priv->clr_base);
  437. }
  438. int __devinit mlx4_map_eq_icm(struct mlx4_dev *dev, u64 icm_virt)
  439. {
  440. struct mlx4_priv *priv = mlx4_priv(dev);
  441. int ret;
  442. /*
  443. * We assume that mapping one page is enough for the whole EQ
  444. * context table. This is fine with all current HCAs, because
  445. * we only use 32 EQs and each EQ uses 64 bytes of context
  446. * memory, or 1 KB total.
  447. */
  448. priv->eq_table.icm_virt = icm_virt;
  449. priv->eq_table.icm_page = alloc_page(GFP_HIGHUSER);
  450. if (!priv->eq_table.icm_page)
  451. return -ENOMEM;
  452. priv->eq_table.icm_dma = pci_map_page(dev->pdev, priv->eq_table.icm_page, 0,
  453. PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  454. if (pci_dma_mapping_error(priv->eq_table.icm_dma)) {
  455. __free_page(priv->eq_table.icm_page);
  456. return -ENOMEM;
  457. }
  458. ret = mlx4_MAP_ICM_page(dev, priv->eq_table.icm_dma, icm_virt);
  459. if (ret) {
  460. pci_unmap_page(dev->pdev, priv->eq_table.icm_dma, PAGE_SIZE,
  461. PCI_DMA_BIDIRECTIONAL);
  462. __free_page(priv->eq_table.icm_page);
  463. }
  464. return ret;
  465. }
  466. void mlx4_unmap_eq_icm(struct mlx4_dev *dev)
  467. {
  468. struct mlx4_priv *priv = mlx4_priv(dev);
  469. mlx4_UNMAP_ICM(dev, priv->eq_table.icm_virt, 1);
  470. pci_unmap_page(dev->pdev, priv->eq_table.icm_dma, PAGE_SIZE,
  471. PCI_DMA_BIDIRECTIONAL);
  472. __free_page(priv->eq_table.icm_page);
  473. }
  474. int __devinit mlx4_init_eq_table(struct mlx4_dev *dev)
  475. {
  476. struct mlx4_priv *priv = mlx4_priv(dev);
  477. int err;
  478. int i;
  479. err = mlx4_bitmap_init(&priv->eq_table.bitmap, dev->caps.num_eqs,
  480. dev->caps.num_eqs - 1, dev->caps.reserved_eqs);
  481. if (err)
  482. return err;
  483. for (i = 0; i < ARRAY_SIZE(priv->eq_table.uar_map); ++i)
  484. priv->eq_table.uar_map[i] = NULL;
  485. err = mlx4_map_clr_int(dev);
  486. if (err)
  487. goto err_out_free;
  488. priv->eq_table.clr_mask =
  489. swab32(1 << (priv->eq_table.inta_pin & 31));
  490. priv->eq_table.clr_int = priv->clr_base +
  491. (priv->eq_table.inta_pin < 32 ? 4 : 0);
  492. err = mlx4_create_eq(dev, dev->caps.num_cqs + MLX4_NUM_SPARE_EQE,
  493. (dev->flags & MLX4_FLAG_MSI_X) ? MLX4_EQ_COMP : 0,
  494. &priv->eq_table.eq[MLX4_EQ_COMP]);
  495. if (err)
  496. goto err_out_unmap;
  497. err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE,
  498. (dev->flags & MLX4_FLAG_MSI_X) ? MLX4_EQ_ASYNC : 0,
  499. &priv->eq_table.eq[MLX4_EQ_ASYNC]);
  500. if (err)
  501. goto err_out_comp;
  502. if (dev->flags & MLX4_FLAG_MSI_X) {
  503. static const char *eq_name[] = {
  504. [MLX4_EQ_COMP] = DRV_NAME " (comp)",
  505. [MLX4_EQ_ASYNC] = DRV_NAME " (async)",
  506. [MLX4_EQ_CATAS] = DRV_NAME " (catas)"
  507. };
  508. err = mlx4_create_eq(dev, 1, MLX4_EQ_CATAS,
  509. &priv->eq_table.eq[MLX4_EQ_CATAS]);
  510. if (err)
  511. goto err_out_async;
  512. for (i = 0; i < MLX4_EQ_CATAS; ++i) {
  513. err = request_irq(priv->eq_table.eq[i].irq,
  514. mlx4_msi_x_interrupt,
  515. 0, eq_name[i], priv->eq_table.eq + i);
  516. if (err)
  517. goto err_out_catas;
  518. priv->eq_table.eq[i].have_irq = 1;
  519. }
  520. err = request_irq(priv->eq_table.eq[MLX4_EQ_CATAS].irq,
  521. mlx4_catas_interrupt, 0,
  522. eq_name[MLX4_EQ_CATAS], dev);
  523. if (err)
  524. goto err_out_catas;
  525. priv->eq_table.eq[MLX4_EQ_CATAS].have_irq = 1;
  526. } else {
  527. err = request_irq(dev->pdev->irq, mlx4_interrupt,
  528. SA_SHIRQ, DRV_NAME, dev);
  529. if (err)
  530. goto err_out_async;
  531. priv->eq_table.have_irq = 1;
  532. }
  533. err = mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 0,
  534. priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);
  535. if (err)
  536. mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n",
  537. priv->eq_table.eq[MLX4_EQ_ASYNC].eqn, err);
  538. for (i = 0; i < MLX4_EQ_CATAS; ++i)
  539. eq_set_ci(&priv->eq_table.eq[i], 1);
  540. if (dev->flags & MLX4_FLAG_MSI_X) {
  541. err = mlx4_MAP_EQ(dev, MLX4_CATAS_EVENT_MASK, 0,
  542. priv->eq_table.eq[MLX4_EQ_CATAS].eqn);
  543. if (err)
  544. mlx4_warn(dev, "MAP_EQ for catas EQ %d failed (%d)\n",
  545. priv->eq_table.eq[MLX4_EQ_CATAS].eqn, err);
  546. }
  547. return 0;
  548. err_out_catas:
  549. mlx4_free_eq(dev, &priv->eq_table.eq[MLX4_EQ_CATAS]);
  550. err_out_async:
  551. mlx4_free_eq(dev, &priv->eq_table.eq[MLX4_EQ_ASYNC]);
  552. err_out_comp:
  553. mlx4_free_eq(dev, &priv->eq_table.eq[MLX4_EQ_COMP]);
  554. err_out_unmap:
  555. mlx4_unmap_clr_int(dev);
  556. mlx4_free_irqs(dev);
  557. err_out_free:
  558. mlx4_bitmap_cleanup(&priv->eq_table.bitmap);
  559. return err;
  560. }
  561. void mlx4_cleanup_eq_table(struct mlx4_dev *dev)
  562. {
  563. struct mlx4_priv *priv = mlx4_priv(dev);
  564. int i;
  565. if (dev->flags & MLX4_FLAG_MSI_X)
  566. mlx4_MAP_EQ(dev, MLX4_CATAS_EVENT_MASK, 1,
  567. priv->eq_table.eq[MLX4_EQ_CATAS].eqn);
  568. mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 1,
  569. priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);
  570. mlx4_free_irqs(dev);
  571. for (i = 0; i < MLX4_EQ_CATAS; ++i)
  572. mlx4_free_eq(dev, &priv->eq_table.eq[i]);
  573. if (dev->flags & MLX4_FLAG_MSI_X)
  574. mlx4_free_eq(dev, &priv->eq_table.eq[MLX4_EQ_CATAS]);
  575. mlx4_unmap_clr_int(dev);
  576. for (i = 0; i < ARRAY_SIZE(priv->eq_table.uar_map); ++i)
  577. if (priv->eq_table.uar_map[i])
  578. iounmap(priv->eq_table.uar_map[i]);
  579. mlx4_bitmap_cleanup(&priv->eq_table.bitmap);
  580. }