mthca_eq.c 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975
  1. /*
  2. * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
  3. * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
  4. *
  5. * This software is available to you under a choice of one of two
  6. * licenses. You may choose to be licensed under the terms of the GNU
  7. * General Public License (GPL) Version 2, available from the file
  8. * COPYING in the main directory of this source tree, or the
  9. * OpenIB.org BSD license below:
  10. *
  11. * Redistribution and use in source and binary forms, with or
  12. * without modification, are permitted provided that the following
  13. * conditions are met:
  14. *
  15. * - Redistributions of source code must retain the above
  16. * copyright notice, this list of conditions and the following
  17. * disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials
  22. * provided with the distribution.
  23. *
  24. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31. * SOFTWARE.
  32. *
  33. * $Id: mthca_eq.c 1382 2004-12-24 02:21:02Z roland $
  34. */
  35. #include <linux/init.h>
  36. #include <linux/errno.h>
  37. #include <linux/interrupt.h>
  38. #include <linux/pci.h>
  39. #include "mthca_dev.h"
  40. #include "mthca_cmd.h"
  41. #include "mthca_config_reg.h"
  42. enum {
  43. MTHCA_NUM_ASYNC_EQE = 0x80,
  44. MTHCA_NUM_CMD_EQE = 0x80,
  45. MTHCA_NUM_SPARE_EQE = 0x80,
  46. MTHCA_EQ_ENTRY_SIZE = 0x20
  47. };
  48. /*
  49. * Must be packed because start is 64 bits but only aligned to 32 bits.
  50. */
  51. struct mthca_eq_context {
  52. __be32 flags;
  53. __be64 start;
  54. __be32 logsize_usrpage;
  55. __be32 tavor_pd; /* reserved for Arbel */
  56. u8 reserved1[3];
  57. u8 intr;
  58. __be32 arbel_pd; /* lost_count for Tavor */
  59. __be32 lkey;
  60. u32 reserved2[2];
  61. __be32 consumer_index;
  62. __be32 producer_index;
  63. u32 reserved3[4];
  64. } __attribute__((packed));
  65. #define MTHCA_EQ_STATUS_OK ( 0 << 28)
  66. #define MTHCA_EQ_STATUS_OVERFLOW ( 9 << 28)
  67. #define MTHCA_EQ_STATUS_WRITE_FAIL (10 << 28)
  68. #define MTHCA_EQ_OWNER_SW ( 0 << 24)
  69. #define MTHCA_EQ_OWNER_HW ( 1 << 24)
  70. #define MTHCA_EQ_FLAG_TR ( 1 << 18)
  71. #define MTHCA_EQ_FLAG_OI ( 1 << 17)
  72. #define MTHCA_EQ_STATE_ARMED ( 1 << 8)
  73. #define MTHCA_EQ_STATE_FIRED ( 2 << 8)
  74. #define MTHCA_EQ_STATE_ALWAYS_ARMED ( 3 << 8)
  75. #define MTHCA_EQ_STATE_ARBEL ( 8 << 8)
  76. enum {
  77. MTHCA_EVENT_TYPE_COMP = 0x00,
  78. MTHCA_EVENT_TYPE_PATH_MIG = 0x01,
  79. MTHCA_EVENT_TYPE_COMM_EST = 0x02,
  80. MTHCA_EVENT_TYPE_SQ_DRAINED = 0x03,
  81. MTHCA_EVENT_TYPE_SRQ_QP_LAST_WQE = 0x13,
  82. MTHCA_EVENT_TYPE_SRQ_LIMIT = 0x14,
  83. MTHCA_EVENT_TYPE_CQ_ERROR = 0x04,
  84. MTHCA_EVENT_TYPE_WQ_CATAS_ERROR = 0x05,
  85. MTHCA_EVENT_TYPE_EEC_CATAS_ERROR = 0x06,
  86. MTHCA_EVENT_TYPE_PATH_MIG_FAILED = 0x07,
  87. MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10,
  88. MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11,
  89. MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12,
  90. MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR = 0x08,
  91. MTHCA_EVENT_TYPE_PORT_CHANGE = 0x09,
  92. MTHCA_EVENT_TYPE_EQ_OVERFLOW = 0x0f,
  93. MTHCA_EVENT_TYPE_ECC_DETECT = 0x0e,
  94. MTHCA_EVENT_TYPE_CMD = 0x0a
  95. };
  96. #define MTHCA_ASYNC_EVENT_MASK ((1ULL << MTHCA_EVENT_TYPE_PATH_MIG) | \
  97. (1ULL << MTHCA_EVENT_TYPE_COMM_EST) | \
  98. (1ULL << MTHCA_EVENT_TYPE_SQ_DRAINED) | \
  99. (1ULL << MTHCA_EVENT_TYPE_CQ_ERROR) | \
  100. (1ULL << MTHCA_EVENT_TYPE_WQ_CATAS_ERROR) | \
  101. (1ULL << MTHCA_EVENT_TYPE_EEC_CATAS_ERROR) | \
  102. (1ULL << MTHCA_EVENT_TYPE_PATH_MIG_FAILED) | \
  103. (1ULL << MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
  104. (1ULL << MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR) | \
  105. (1ULL << MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR) | \
  106. (1ULL << MTHCA_EVENT_TYPE_PORT_CHANGE) | \
  107. (1ULL << MTHCA_EVENT_TYPE_ECC_DETECT))
  108. #define MTHCA_SRQ_EVENT_MASK ((1ULL << MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR) | \
  109. (1ULL << MTHCA_EVENT_TYPE_SRQ_QP_LAST_WQE) | \
  110. (1ULL << MTHCA_EVENT_TYPE_SRQ_LIMIT))
  111. #define MTHCA_CMD_EVENT_MASK (1ULL << MTHCA_EVENT_TYPE_CMD)
  112. #define MTHCA_EQ_DB_INC_CI (1 << 24)
  113. #define MTHCA_EQ_DB_REQ_NOT (2 << 24)
  114. #define MTHCA_EQ_DB_DISARM_CQ (3 << 24)
  115. #define MTHCA_EQ_DB_SET_CI (4 << 24)
  116. #define MTHCA_EQ_DB_ALWAYS_ARM (5 << 24)
  117. struct mthca_eqe {
  118. u8 reserved1;
  119. u8 type;
  120. u8 reserved2;
  121. u8 subtype;
  122. union {
  123. u32 raw[6];
  124. struct {
  125. __be32 cqn;
  126. } __attribute__((packed)) comp;
  127. struct {
  128. u16 reserved1;
  129. __be16 token;
  130. u32 reserved2;
  131. u8 reserved3[3];
  132. u8 status;
  133. __be64 out_param;
  134. } __attribute__((packed)) cmd;
  135. struct {
  136. __be32 qpn;
  137. } __attribute__((packed)) qp;
  138. struct {
  139. __be32 srqn;
  140. } __attribute__((packed)) srq;
  141. struct {
  142. __be32 cqn;
  143. u32 reserved1;
  144. u8 reserved2[3];
  145. u8 syndrome;
  146. } __attribute__((packed)) cq_err;
  147. struct {
  148. u32 reserved1[2];
  149. __be32 port;
  150. } __attribute__((packed)) port_change;
  151. } event;
  152. u8 reserved3[3];
  153. u8 owner;
  154. } __attribute__((packed));
  155. #define MTHCA_EQ_ENTRY_OWNER_SW (0 << 7)
  156. #define MTHCA_EQ_ENTRY_OWNER_HW (1 << 7)
  157. static inline u64 async_mask(struct mthca_dev *dev)
  158. {
  159. return dev->mthca_flags & MTHCA_FLAG_SRQ ?
  160. MTHCA_ASYNC_EVENT_MASK | MTHCA_SRQ_EVENT_MASK :
  161. MTHCA_ASYNC_EVENT_MASK;
  162. }
  163. static inline void tavor_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)
  164. {
  165. __be32 doorbell[2];
  166. doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_SET_CI | eq->eqn);
  167. doorbell[1] = cpu_to_be32(ci & (eq->nent - 1));
  168. /*
  169. * This barrier makes sure that all updates to ownership bits
  170. * done by set_eqe_hw() hit memory before the consumer index
  171. * is updated. set_eq_ci() allows the HCA to possibly write
  172. * more EQ entries, and we want to avoid the exceedingly
  173. * unlikely possibility of the HCA writing an entry and then
  174. * having set_eqe_hw() overwrite the owner field.
  175. */
  176. wmb();
  177. mthca_write64(doorbell,
  178. dev->kar + MTHCA_EQ_DOORBELL,
  179. MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
  180. }
  181. static inline void arbel_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)
  182. {
  183. /* See comment in tavor_set_eq_ci() above. */
  184. wmb();
  185. __raw_writel((__force u32) cpu_to_be32(ci),
  186. dev->eq_regs.arbel.eq_set_ci_base + eq->eqn * 8);
  187. /* We still want ordering, just not swabbing, so add a barrier */
  188. mb();
  189. }
  190. static inline void set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)
  191. {
  192. if (mthca_is_memfree(dev))
  193. arbel_set_eq_ci(dev, eq, ci);
  194. else
  195. tavor_set_eq_ci(dev, eq, ci);
  196. }
  197. static inline void tavor_eq_req_not(struct mthca_dev *dev, int eqn)
  198. {
  199. __be32 doorbell[2];
  200. doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_REQ_NOT | eqn);
  201. doorbell[1] = 0;
  202. mthca_write64(doorbell,
  203. dev->kar + MTHCA_EQ_DOORBELL,
  204. MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
  205. }
  206. static inline void arbel_eq_req_not(struct mthca_dev *dev, u32 eqn_mask)
  207. {
  208. writel(eqn_mask, dev->eq_regs.arbel.eq_arm);
  209. }
  210. static inline void disarm_cq(struct mthca_dev *dev, int eqn, int cqn)
  211. {
  212. if (!mthca_is_memfree(dev)) {
  213. __be32 doorbell[2];
  214. doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_DISARM_CQ | eqn);
  215. doorbell[1] = cpu_to_be32(cqn);
  216. mthca_write64(doorbell,
  217. dev->kar + MTHCA_EQ_DOORBELL,
  218. MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
  219. }
  220. }
  221. static inline struct mthca_eqe *get_eqe(struct mthca_eq *eq, u32 entry)
  222. {
  223. unsigned long off = (entry & (eq->nent - 1)) * MTHCA_EQ_ENTRY_SIZE;
  224. return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE;
  225. }
  226. static inline struct mthca_eqe* next_eqe_sw(struct mthca_eq *eq)
  227. {
  228. struct mthca_eqe* eqe;
  229. eqe = get_eqe(eq, eq->cons_index);
  230. return (MTHCA_EQ_ENTRY_OWNER_HW & eqe->owner) ? NULL : eqe;
  231. }
  232. static inline void set_eqe_hw(struct mthca_eqe *eqe)
  233. {
  234. eqe->owner = MTHCA_EQ_ENTRY_OWNER_HW;
  235. }
  236. static void port_change(struct mthca_dev *dev, int port, int active)
  237. {
  238. struct ib_event record;
  239. mthca_dbg(dev, "Port change to %s for port %d\n",
  240. active ? "active" : "down", port);
  241. record.device = &dev->ib_dev;
  242. record.event = active ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
  243. record.element.port_num = port;
  244. ib_dispatch_event(&record);
  245. }
  246. static int mthca_eq_int(struct mthca_dev *dev, struct mthca_eq *eq)
  247. {
  248. struct mthca_eqe *eqe;
  249. int disarm_cqn;
  250. int eqes_found = 0;
  251. int set_ci = 0;
  252. while ((eqe = next_eqe_sw(eq))) {
  253. /*
  254. * Make sure we read EQ entry contents after we've
  255. * checked the ownership bit.
  256. */
  257. rmb();
  258. switch (eqe->type) {
  259. case MTHCA_EVENT_TYPE_COMP:
  260. disarm_cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff;
  261. disarm_cq(dev, eq->eqn, disarm_cqn);
  262. mthca_cq_completion(dev, disarm_cqn);
  263. break;
  264. case MTHCA_EVENT_TYPE_PATH_MIG:
  265. mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
  266. IB_EVENT_PATH_MIG);
  267. break;
  268. case MTHCA_EVENT_TYPE_COMM_EST:
  269. mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
  270. IB_EVENT_COMM_EST);
  271. break;
  272. case MTHCA_EVENT_TYPE_SQ_DRAINED:
  273. mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
  274. IB_EVENT_SQ_DRAINED);
  275. break;
  276. case MTHCA_EVENT_TYPE_SRQ_QP_LAST_WQE:
  277. mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
  278. IB_EVENT_QP_LAST_WQE_REACHED);
  279. break;
  280. case MTHCA_EVENT_TYPE_SRQ_LIMIT:
  281. mthca_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & 0xffffff,
  282. IB_EVENT_SRQ_LIMIT_REACHED);
  283. break;
  284. case MTHCA_EVENT_TYPE_WQ_CATAS_ERROR:
  285. mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
  286. IB_EVENT_QP_FATAL);
  287. break;
  288. case MTHCA_EVENT_TYPE_PATH_MIG_FAILED:
  289. mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
  290. IB_EVENT_PATH_MIG_ERR);
  291. break;
  292. case MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
  293. mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
  294. IB_EVENT_QP_REQ_ERR);
  295. break;
  296. case MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR:
  297. mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,
  298. IB_EVENT_QP_ACCESS_ERR);
  299. break;
  300. case MTHCA_EVENT_TYPE_CMD:
  301. mthca_cmd_event(dev,
  302. be16_to_cpu(eqe->event.cmd.token),
  303. eqe->event.cmd.status,
  304. be64_to_cpu(eqe->event.cmd.out_param));
  305. break;
  306. case MTHCA_EVENT_TYPE_PORT_CHANGE:
  307. port_change(dev,
  308. (be32_to_cpu(eqe->event.port_change.port) >> 28) & 3,
  309. eqe->subtype == 0x4);
  310. break;
  311. case MTHCA_EVENT_TYPE_CQ_ERROR:
  312. mthca_warn(dev, "CQ %s on CQN %06x\n",
  313. eqe->event.cq_err.syndrome == 1 ?
  314. "overrun" : "access violation",
  315. be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff);
  316. mthca_cq_event(dev, be32_to_cpu(eqe->event.cq_err.cqn),
  317. IB_EVENT_CQ_ERR);
  318. break;
  319. case MTHCA_EVENT_TYPE_EQ_OVERFLOW:
  320. mthca_warn(dev, "EQ overrun on EQN %d\n", eq->eqn);
  321. break;
  322. case MTHCA_EVENT_TYPE_EEC_CATAS_ERROR:
  323. case MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR:
  324. case MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR:
  325. case MTHCA_EVENT_TYPE_ECC_DETECT:
  326. default:
  327. mthca_warn(dev, "Unhandled event %02x(%02x) on EQ %d\n",
  328. eqe->type, eqe->subtype, eq->eqn);
  329. break;
  330. };
  331. set_eqe_hw(eqe);
  332. ++eq->cons_index;
  333. eqes_found = 1;
  334. ++set_ci;
  335. /*
  336. * The HCA will think the queue has overflowed if we
  337. * don't tell it we've been processing events. We
  338. * create our EQs with MTHCA_NUM_SPARE_EQE extra
  339. * entries, so we must update our consumer index at
  340. * least that often.
  341. */
  342. if (unlikely(set_ci >= MTHCA_NUM_SPARE_EQE)) {
  343. /*
  344. * Conditional on hca_type is OK here because
  345. * this is a rare case, not the fast path.
  346. */
  347. set_eq_ci(dev, eq, eq->cons_index);
  348. set_ci = 0;
  349. }
  350. }
  351. /*
  352. * Rely on caller to set consumer index so that we don't have
  353. * to test hca_type in our interrupt handling fast path.
  354. */
  355. return eqes_found;
  356. }
  357. static irqreturn_t mthca_tavor_interrupt(int irq, void *dev_ptr, struct pt_regs *regs)
  358. {
  359. struct mthca_dev *dev = dev_ptr;
  360. u32 ecr;
  361. int i;
  362. if (dev->eq_table.clr_mask)
  363. writel(dev->eq_table.clr_mask, dev->eq_table.clr_int);
  364. ecr = readl(dev->eq_regs.tavor.ecr_base + 4);
  365. if (!ecr)
  366. return IRQ_NONE;
  367. writel(ecr, dev->eq_regs.tavor.ecr_base +
  368. MTHCA_ECR_CLR_BASE - MTHCA_ECR_BASE + 4);
  369. for (i = 0; i < MTHCA_NUM_EQ; ++i)
  370. if (ecr & dev->eq_table.eq[i].eqn_mask) {
  371. if (mthca_eq_int(dev, &dev->eq_table.eq[i]))
  372. tavor_set_eq_ci(dev, &dev->eq_table.eq[i],
  373. dev->eq_table.eq[i].cons_index);
  374. tavor_eq_req_not(dev, dev->eq_table.eq[i].eqn);
  375. }
  376. return IRQ_HANDLED;
  377. }
  378. static irqreturn_t mthca_tavor_msi_x_interrupt(int irq, void *eq_ptr,
  379. struct pt_regs *regs)
  380. {
  381. struct mthca_eq *eq = eq_ptr;
  382. struct mthca_dev *dev = eq->dev;
  383. mthca_eq_int(dev, eq);
  384. tavor_set_eq_ci(dev, eq, eq->cons_index);
  385. tavor_eq_req_not(dev, eq->eqn);
  386. /* MSI-X vectors always belong to us */
  387. return IRQ_HANDLED;
  388. }
  389. static irqreturn_t mthca_arbel_interrupt(int irq, void *dev_ptr, struct pt_regs *regs)
  390. {
  391. struct mthca_dev *dev = dev_ptr;
  392. int work = 0;
  393. int i;
  394. if (dev->eq_table.clr_mask)
  395. writel(dev->eq_table.clr_mask, dev->eq_table.clr_int);
  396. for (i = 0; i < MTHCA_NUM_EQ; ++i)
  397. if (mthca_eq_int(dev, &dev->eq_table.eq[i])) {
  398. work = 1;
  399. arbel_set_eq_ci(dev, &dev->eq_table.eq[i],
  400. dev->eq_table.eq[i].cons_index);
  401. }
  402. arbel_eq_req_not(dev, dev->eq_table.arm_mask);
  403. return IRQ_RETVAL(work);
  404. }
  405. static irqreturn_t mthca_arbel_msi_x_interrupt(int irq, void *eq_ptr,
  406. struct pt_regs *regs)
  407. {
  408. struct mthca_eq *eq = eq_ptr;
  409. struct mthca_dev *dev = eq->dev;
  410. mthca_eq_int(dev, eq);
  411. arbel_set_eq_ci(dev, eq, eq->cons_index);
  412. arbel_eq_req_not(dev, eq->eqn_mask);
  413. /* MSI-X vectors always belong to us */
  414. return IRQ_HANDLED;
  415. }
  416. static int __devinit mthca_create_eq(struct mthca_dev *dev,
  417. int nent,
  418. u8 intr,
  419. struct mthca_eq *eq)
  420. {
  421. int npages;
  422. u64 *dma_list = NULL;
  423. dma_addr_t t;
  424. struct mthca_mailbox *mailbox;
  425. struct mthca_eq_context *eq_context;
  426. int err = -ENOMEM;
  427. int i;
  428. u8 status;
  429. eq->dev = dev;
  430. eq->nent = roundup_pow_of_two(max(nent, 2));
  431. npages = ALIGN(eq->nent * MTHCA_EQ_ENTRY_SIZE, PAGE_SIZE) / PAGE_SIZE;
  432. eq->page_list = kmalloc(npages * sizeof *eq->page_list,
  433. GFP_KERNEL);
  434. if (!eq->page_list)
  435. goto err_out;
  436. for (i = 0; i < npages; ++i)
  437. eq->page_list[i].buf = NULL;
  438. dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
  439. if (!dma_list)
  440. goto err_out_free;
  441. mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
  442. if (IS_ERR(mailbox))
  443. goto err_out_free;
  444. eq_context = mailbox->buf;
  445. for (i = 0; i < npages; ++i) {
  446. eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev,
  447. PAGE_SIZE, &t, GFP_KERNEL);
  448. if (!eq->page_list[i].buf)
  449. goto err_out_free_pages;
  450. dma_list[i] = t;
  451. pci_unmap_addr_set(&eq->page_list[i], mapping, t);
  452. memset(eq->page_list[i].buf, 0, PAGE_SIZE);
  453. }
  454. for (i = 0; i < eq->nent; ++i)
  455. set_eqe_hw(get_eqe(eq, i));
  456. eq->eqn = mthca_alloc(&dev->eq_table.alloc);
  457. if (eq->eqn == -1)
  458. goto err_out_free_pages;
  459. err = mthca_mr_alloc_phys(dev, dev->driver_pd.pd_num,
  460. dma_list, PAGE_SHIFT, npages,
  461. 0, npages * PAGE_SIZE,
  462. MTHCA_MPT_FLAG_LOCAL_WRITE |
  463. MTHCA_MPT_FLAG_LOCAL_READ,
  464. &eq->mr);
  465. if (err)
  466. goto err_out_free_eq;
  467. memset(eq_context, 0, sizeof *eq_context);
  468. eq_context->flags = cpu_to_be32(MTHCA_EQ_STATUS_OK |
  469. MTHCA_EQ_OWNER_HW |
  470. MTHCA_EQ_STATE_ARMED |
  471. MTHCA_EQ_FLAG_TR);
  472. if (mthca_is_memfree(dev))
  473. eq_context->flags |= cpu_to_be32(MTHCA_EQ_STATE_ARBEL);
  474. eq_context->logsize_usrpage = cpu_to_be32((ffs(eq->nent) - 1) << 24);
  475. if (mthca_is_memfree(dev)) {
  476. eq_context->arbel_pd = cpu_to_be32(dev->driver_pd.pd_num);
  477. } else {
  478. eq_context->logsize_usrpage |= cpu_to_be32(dev->driver_uar.index);
  479. eq_context->tavor_pd = cpu_to_be32(dev->driver_pd.pd_num);
  480. }
  481. eq_context->intr = intr;
  482. eq_context->lkey = cpu_to_be32(eq->mr.ibmr.lkey);
  483. err = mthca_SW2HW_EQ(dev, mailbox, eq->eqn, &status);
  484. if (err) {
  485. mthca_warn(dev, "SW2HW_EQ failed (%d)\n", err);
  486. goto err_out_free_mr;
  487. }
  488. if (status) {
  489. mthca_warn(dev, "SW2HW_EQ returned status 0x%02x\n",
  490. status);
  491. err = -EINVAL;
  492. goto err_out_free_mr;
  493. }
  494. kfree(dma_list);
  495. mthca_free_mailbox(dev, mailbox);
  496. eq->eqn_mask = swab32(1 << eq->eqn);
  497. eq->cons_index = 0;
  498. dev->eq_table.arm_mask |= eq->eqn_mask;
  499. mthca_dbg(dev, "Allocated EQ %d with %d entries\n",
  500. eq->eqn, eq->nent);
  501. return err;
  502. err_out_free_mr:
  503. mthca_free_mr(dev, &eq->mr);
  504. err_out_free_eq:
  505. mthca_free(&dev->eq_table.alloc, eq->eqn);
  506. err_out_free_pages:
  507. for (i = 0; i < npages; ++i)
  508. if (eq->page_list[i].buf)
  509. dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
  510. eq->page_list[i].buf,
  511. pci_unmap_addr(&eq->page_list[i],
  512. mapping));
  513. mthca_free_mailbox(dev, mailbox);
  514. err_out_free:
  515. kfree(eq->page_list);
  516. kfree(dma_list);
  517. err_out:
  518. return err;
  519. }
  520. static void mthca_free_eq(struct mthca_dev *dev,
  521. struct mthca_eq *eq)
  522. {
  523. struct mthca_mailbox *mailbox;
  524. int err;
  525. u8 status;
  526. int npages = (eq->nent * MTHCA_EQ_ENTRY_SIZE + PAGE_SIZE - 1) /
  527. PAGE_SIZE;
  528. int i;
  529. mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
  530. if (IS_ERR(mailbox))
  531. return;
  532. err = mthca_HW2SW_EQ(dev, mailbox, eq->eqn, &status);
  533. if (err)
  534. mthca_warn(dev, "HW2SW_EQ failed (%d)\n", err);
  535. if (status)
  536. mthca_warn(dev, "HW2SW_EQ returned status 0x%02x\n", status);
  537. dev->eq_table.arm_mask &= ~eq->eqn_mask;
  538. if (0) {
  539. mthca_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn);
  540. for (i = 0; i < sizeof (struct mthca_eq_context) / 4; ++i) {
  541. if (i % 4 == 0)
  542. printk("[%02x] ", i * 4);
  543. printk(" %08x", be32_to_cpup(mailbox->buf + i * 4));
  544. if ((i + 1) % 4 == 0)
  545. printk("\n");
  546. }
  547. }
  548. mthca_free_mr(dev, &eq->mr);
  549. for (i = 0; i < npages; ++i)
  550. pci_free_consistent(dev->pdev, PAGE_SIZE,
  551. eq->page_list[i].buf,
  552. pci_unmap_addr(&eq->page_list[i], mapping));
  553. kfree(eq->page_list);
  554. mthca_free_mailbox(dev, mailbox);
  555. }
  556. static void mthca_free_irqs(struct mthca_dev *dev)
  557. {
  558. int i;
  559. if (dev->eq_table.have_irq)
  560. free_irq(dev->pdev->irq, dev);
  561. for (i = 0; i < MTHCA_NUM_EQ; ++i)
  562. if (dev->eq_table.eq[i].have_irq)
  563. free_irq(dev->eq_table.eq[i].msi_x_vector,
  564. dev->eq_table.eq + i);
  565. }
  566. static int __devinit mthca_map_reg(struct mthca_dev *dev,
  567. unsigned long offset, unsigned long size,
  568. void __iomem **map)
  569. {
  570. unsigned long base = pci_resource_start(dev->pdev, 0);
  571. if (!request_mem_region(base + offset, size, DRV_NAME))
  572. return -EBUSY;
  573. *map = ioremap(base + offset, size);
  574. if (!*map) {
  575. release_mem_region(base + offset, size);
  576. return -ENOMEM;
  577. }
  578. return 0;
  579. }
  580. static void mthca_unmap_reg(struct mthca_dev *dev, unsigned long offset,
  581. unsigned long size, void __iomem *map)
  582. {
  583. unsigned long base = pci_resource_start(dev->pdev, 0);
  584. release_mem_region(base + offset, size);
  585. iounmap(map);
  586. }
  587. static int __devinit mthca_map_eq_regs(struct mthca_dev *dev)
  588. {
  589. unsigned long mthca_base;
  590. mthca_base = pci_resource_start(dev->pdev, 0);
  591. if (mthca_is_memfree(dev)) {
  592. /*
  593. * We assume that the EQ arm and EQ set CI registers
  594. * fall within the first BAR. We can't trust the
  595. * values firmware gives us, since those addresses are
  596. * valid on the HCA's side of the PCI bus but not
  597. * necessarily the host side.
  598. */
  599. if (mthca_map_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
  600. dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
  601. &dev->clr_base)) {
  602. mthca_err(dev, "Couldn't map interrupt clear register, "
  603. "aborting.\n");
  604. return -ENOMEM;
  605. }
  606. /*
  607. * Add 4 because we limit ourselves to EQs 0 ... 31,
  608. * so we only need the low word of the register.
  609. */
  610. if (mthca_map_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) &
  611. dev->fw.arbel.eq_arm_base) + 4, 4,
  612. &dev->eq_regs.arbel.eq_arm)) {
  613. mthca_err(dev, "Couldn't map EQ arm register, aborting.\n");
  614. mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
  615. dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
  616. dev->clr_base);
  617. return -ENOMEM;
  618. }
  619. if (mthca_map_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
  620. dev->fw.arbel.eq_set_ci_base,
  621. MTHCA_EQ_SET_CI_SIZE,
  622. &dev->eq_regs.arbel.eq_set_ci_base)) {
  623. mthca_err(dev, "Couldn't map EQ CI register, aborting.\n");
  624. mthca_unmap_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) &
  625. dev->fw.arbel.eq_arm_base) + 4, 4,
  626. dev->eq_regs.arbel.eq_arm);
  627. mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
  628. dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
  629. dev->clr_base);
  630. return -ENOMEM;
  631. }
  632. } else {
  633. if (mthca_map_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE,
  634. &dev->clr_base)) {
  635. mthca_err(dev, "Couldn't map interrupt clear register, "
  636. "aborting.\n");
  637. return -ENOMEM;
  638. }
  639. if (mthca_map_reg(dev, MTHCA_ECR_BASE,
  640. MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE,
  641. &dev->eq_regs.tavor.ecr_base)) {
  642. mthca_err(dev, "Couldn't map ecr register, "
  643. "aborting.\n");
  644. mthca_unmap_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE,
  645. dev->clr_base);
  646. return -ENOMEM;
  647. }
  648. }
  649. return 0;
  650. }
  651. static void mthca_unmap_eq_regs(struct mthca_dev *dev)
  652. {
  653. if (mthca_is_memfree(dev)) {
  654. mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
  655. dev->fw.arbel.eq_set_ci_base,
  656. MTHCA_EQ_SET_CI_SIZE,
  657. dev->eq_regs.arbel.eq_set_ci_base);
  658. mthca_unmap_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) &
  659. dev->fw.arbel.eq_arm_base) + 4, 4,
  660. dev->eq_regs.arbel.eq_arm);
  661. mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &
  662. dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,
  663. dev->clr_base);
  664. } else {
  665. mthca_unmap_reg(dev, MTHCA_ECR_BASE,
  666. MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE,
  667. dev->eq_regs.tavor.ecr_base);
  668. mthca_unmap_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE,
  669. dev->clr_base);
  670. }
  671. }
  672. int __devinit mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt)
  673. {
  674. int ret;
  675. u8 status;
  676. /*
  677. * We assume that mapping one page is enough for the whole EQ
  678. * context table. This is fine with all current HCAs, because
  679. * we only use 32 EQs and each EQ uses 32 bytes of context
  680. * memory, or 1 KB total.
  681. */
  682. dev->eq_table.icm_virt = icm_virt;
  683. dev->eq_table.icm_page = alloc_page(GFP_HIGHUSER);
  684. if (!dev->eq_table.icm_page)
  685. return -ENOMEM;
  686. dev->eq_table.icm_dma = pci_map_page(dev->pdev, dev->eq_table.icm_page, 0,
  687. PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
  688. if (pci_dma_mapping_error(dev->eq_table.icm_dma)) {
  689. __free_page(dev->eq_table.icm_page);
  690. return -ENOMEM;
  691. }
  692. ret = mthca_MAP_ICM_page(dev, dev->eq_table.icm_dma, icm_virt, &status);
  693. if (!ret && status)
  694. ret = -EINVAL;
  695. if (ret) {
  696. pci_unmap_page(dev->pdev, dev->eq_table.icm_dma, PAGE_SIZE,
  697. PCI_DMA_BIDIRECTIONAL);
  698. __free_page(dev->eq_table.icm_page);
  699. }
  700. return ret;
  701. }
  702. void mthca_unmap_eq_icm(struct mthca_dev *dev)
  703. {
  704. u8 status;
  705. mthca_UNMAP_ICM(dev, dev->eq_table.icm_virt, 1, &status);
  706. pci_unmap_page(dev->pdev, dev->eq_table.icm_dma, PAGE_SIZE,
  707. PCI_DMA_BIDIRECTIONAL);
  708. __free_page(dev->eq_table.icm_page);
  709. }
  710. int __devinit mthca_init_eq_table(struct mthca_dev *dev)
  711. {
  712. int err;
  713. u8 status;
  714. u8 intr;
  715. int i;
  716. err = mthca_alloc_init(&dev->eq_table.alloc,
  717. dev->limits.num_eqs,
  718. dev->limits.num_eqs - 1,
  719. dev->limits.reserved_eqs);
  720. if (err)
  721. return err;
  722. err = mthca_map_eq_regs(dev);
  723. if (err)
  724. goto err_out_free;
  725. if (dev->mthca_flags & MTHCA_FLAG_MSI ||
  726. dev->mthca_flags & MTHCA_FLAG_MSI_X) {
  727. dev->eq_table.clr_mask = 0;
  728. } else {
  729. dev->eq_table.clr_mask =
  730. swab32(1 << (dev->eq_table.inta_pin & 31));
  731. dev->eq_table.clr_int = dev->clr_base +
  732. (dev->eq_table.inta_pin < 32 ? 4 : 0);
  733. }
  734. dev->eq_table.arm_mask = 0;
  735. intr = (dev->mthca_flags & MTHCA_FLAG_MSI) ?
  736. 128 : dev->eq_table.inta_pin;
  737. err = mthca_create_eq(dev, dev->limits.num_cqs + MTHCA_NUM_SPARE_EQE,
  738. (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 128 : intr,
  739. &dev->eq_table.eq[MTHCA_EQ_COMP]);
  740. if (err)
  741. goto err_out_unmap;
  742. err = mthca_create_eq(dev, MTHCA_NUM_ASYNC_EQE + MTHCA_NUM_SPARE_EQE,
  743. (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 129 : intr,
  744. &dev->eq_table.eq[MTHCA_EQ_ASYNC]);
  745. if (err)
  746. goto err_out_comp;
  747. err = mthca_create_eq(dev, MTHCA_NUM_CMD_EQE + MTHCA_NUM_SPARE_EQE,
  748. (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 130 : intr,
  749. &dev->eq_table.eq[MTHCA_EQ_CMD]);
  750. if (err)
  751. goto err_out_async;
  752. if (dev->mthca_flags & MTHCA_FLAG_MSI_X) {
  753. static const char *eq_name[] = {
  754. [MTHCA_EQ_COMP] = DRV_NAME " (comp)",
  755. [MTHCA_EQ_ASYNC] = DRV_NAME " (async)",
  756. [MTHCA_EQ_CMD] = DRV_NAME " (cmd)"
  757. };
  758. for (i = 0; i < MTHCA_NUM_EQ; ++i) {
  759. err = request_irq(dev->eq_table.eq[i].msi_x_vector,
  760. mthca_is_memfree(dev) ?
  761. mthca_arbel_msi_x_interrupt :
  762. mthca_tavor_msi_x_interrupt,
  763. 0, eq_name[i], dev->eq_table.eq + i);
  764. if (err)
  765. goto err_out_cmd;
  766. dev->eq_table.eq[i].have_irq = 1;
  767. }
  768. } else {
  769. err = request_irq(dev->pdev->irq,
  770. mthca_is_memfree(dev) ?
  771. mthca_arbel_interrupt :
  772. mthca_tavor_interrupt,
  773. SA_SHIRQ, DRV_NAME, dev);
  774. if (err)
  775. goto err_out_cmd;
  776. dev->eq_table.have_irq = 1;
  777. }
  778. err = mthca_MAP_EQ(dev, async_mask(dev),
  779. 0, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, &status);
  780. if (err)
  781. mthca_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n",
  782. dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, err);
  783. if (status)
  784. mthca_warn(dev, "MAP_EQ for async EQ %d returned status 0x%02x\n",
  785. dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, status);
  786. err = mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK,
  787. 0, dev->eq_table.eq[MTHCA_EQ_CMD].eqn, &status);
  788. if (err)
  789. mthca_warn(dev, "MAP_EQ for cmd EQ %d failed (%d)\n",
  790. dev->eq_table.eq[MTHCA_EQ_CMD].eqn, err);
  791. if (status)
  792. mthca_warn(dev, "MAP_EQ for cmd EQ %d returned status 0x%02x\n",
  793. dev->eq_table.eq[MTHCA_EQ_CMD].eqn, status);
  794. for (i = 0; i < MTHCA_NUM_EQ; ++i)
  795. if (mthca_is_memfree(dev))
  796. arbel_eq_req_not(dev, dev->eq_table.eq[i].eqn_mask);
  797. else
  798. tavor_eq_req_not(dev, dev->eq_table.eq[i].eqn);
  799. return 0;
  800. err_out_cmd:
  801. mthca_free_irqs(dev);
  802. mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_CMD]);
  803. err_out_async:
  804. mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_ASYNC]);
  805. err_out_comp:
  806. mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_COMP]);
  807. err_out_unmap:
  808. mthca_unmap_eq_regs(dev);
  809. err_out_free:
  810. mthca_alloc_cleanup(&dev->eq_table.alloc);
  811. return err;
  812. }
  813. void mthca_cleanup_eq_table(struct mthca_dev *dev)
  814. {
  815. u8 status;
  816. int i;
  817. mthca_free_irqs(dev);
  818. mthca_MAP_EQ(dev, async_mask(dev),
  819. 1, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, &status);
  820. mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK,
  821. 1, dev->eq_table.eq[MTHCA_EQ_CMD].eqn, &status);
  822. for (i = 0; i < MTHCA_NUM_EQ; ++i)
  823. mthca_free_eq(dev, &dev->eq_table.eq[i]);
  824. mthca_unmap_eq_regs(dev);
  825. mthca_alloc_cleanup(&dev->eq_table.alloc);
  826. }