qib_init.c 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731
  1. /*
  2. * Copyright (c) 2012 Intel Corporation. All rights reserved.
  3. * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
  4. * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
  5. *
  6. * This software is available to you under a choice of one of two
  7. * licenses. You may choose to be licensed under the terms of the GNU
  8. * General Public License (GPL) Version 2, available from the file
  9. * COPYING in the main directory of this source tree, or the
  10. * OpenIB.org BSD license below:
  11. *
  12. * Redistribution and use in source and binary forms, with or
  13. * without modification, are permitted provided that the following
  14. * conditions are met:
  15. *
  16. * - Redistributions of source code must retain the above
  17. * copyright notice, this list of conditions and the following
  18. * disclaimer.
  19. *
  20. * - Redistributions in binary form must reproduce the above
  21. * copyright notice, this list of conditions and the following
  22. * disclaimer in the documentation and/or other materials
  23. * provided with the distribution.
  24. *
  25. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32. * SOFTWARE.
  33. */
  34. #include <linux/pci.h>
  35. #include <linux/netdevice.h>
  36. #include <linux/vmalloc.h>
  37. #include <linux/delay.h>
  38. #include <linux/idr.h>
  39. #include <linux/module.h>
  40. #include <linux/printk.h>
  41. #include "qib.h"
  42. #include "qib_common.h"
  43. #include "qib_mad.h"
  44. #undef pr_fmt
  45. #define pr_fmt(fmt) QIB_DRV_NAME ": " fmt
  46. /*
  47. * min buffers we want to have per context, after driver
  48. */
  49. #define QIB_MIN_USER_CTXT_BUFCNT 7
  50. #define QLOGIC_IB_R_SOFTWARE_MASK 0xFF
  51. #define QLOGIC_IB_R_SOFTWARE_SHIFT 24
  52. #define QLOGIC_IB_R_EMULATOR_MASK (1ULL<<62)
  53. /*
  54. * Number of ctxts we are configured to use (to allow for more pio
  55. * buffers per ctxt, etc.) Zero means use chip value.
  56. */
  57. ushort qib_cfgctxts;
  58. module_param_named(cfgctxts, qib_cfgctxts, ushort, S_IRUGO);
  59. MODULE_PARM_DESC(cfgctxts, "Set max number of contexts to use");
  60. /*
  61. * If set, do not write to any regs if avoidable, hack to allow
  62. * check for deranged default register values.
  63. */
  64. ushort qib_mini_init;
  65. module_param_named(mini_init, qib_mini_init, ushort, S_IRUGO);
  66. MODULE_PARM_DESC(mini_init, "If set, do minimal diag init");
  67. unsigned qib_n_krcv_queues;
  68. module_param_named(krcvqs, qib_n_krcv_queues, uint, S_IRUGO);
  69. MODULE_PARM_DESC(krcvqs, "number of kernel receive queues per IB port");
  70. unsigned qib_cc_table_size;
  71. module_param_named(cc_table_size, qib_cc_table_size, uint, S_IRUGO);
  72. MODULE_PARM_DESC(cc_table_size, "Congestion control table entries 0 (CCA disabled - default), min = 128, max = 1984");
  73. /*
  74. * qib_wc_pat parameter:
  75. * 0 is WC via MTRR
  76. * 1 is WC via PAT
  77. * If PAT initialization fails, code reverts back to MTRR
  78. */
  79. unsigned qib_wc_pat = 1; /* default (1) is to use PAT, not MTRR */
  80. module_param_named(wc_pat, qib_wc_pat, uint, S_IRUGO);
  81. MODULE_PARM_DESC(wc_pat, "enable write-combining via PAT mechanism");
  82. struct workqueue_struct *qib_cq_wq;
  83. static void verify_interrupt(unsigned long);
  84. static struct idr qib_unit_table;
  85. u32 qib_cpulist_count;
  86. unsigned long *qib_cpulist;
  87. /* set number of contexts we'll actually use */
  88. void qib_set_ctxtcnt(struct qib_devdata *dd)
  89. {
  90. if (!qib_cfgctxts) {
  91. dd->cfgctxts = dd->first_user_ctxt + num_online_cpus();
  92. if (dd->cfgctxts > dd->ctxtcnt)
  93. dd->cfgctxts = dd->ctxtcnt;
  94. } else if (qib_cfgctxts < dd->num_pports)
  95. dd->cfgctxts = dd->ctxtcnt;
  96. else if (qib_cfgctxts <= dd->ctxtcnt)
  97. dd->cfgctxts = qib_cfgctxts;
  98. else
  99. dd->cfgctxts = dd->ctxtcnt;
  100. dd->freectxts = (dd->first_user_ctxt > dd->cfgctxts) ? 0 :
  101. dd->cfgctxts - dd->first_user_ctxt;
  102. }
  103. /*
  104. * Common code for creating the receive context array.
  105. */
  106. int qib_create_ctxts(struct qib_devdata *dd)
  107. {
  108. unsigned i;
  109. int ret;
  110. /*
  111. * Allocate full ctxtcnt array, rather than just cfgctxts, because
  112. * cleanup iterates across all possible ctxts.
  113. */
  114. dd->rcd = kzalloc(sizeof(*dd->rcd) * dd->ctxtcnt, GFP_KERNEL);
  115. if (!dd->rcd) {
  116. qib_dev_err(dd,
  117. "Unable to allocate ctxtdata array, failing\n");
  118. ret = -ENOMEM;
  119. goto done;
  120. }
  121. /* create (one or more) kctxt */
  122. for (i = 0; i < dd->first_user_ctxt; ++i) {
  123. struct qib_pportdata *ppd;
  124. struct qib_ctxtdata *rcd;
  125. if (dd->skip_kctxt_mask & (1 << i))
  126. continue;
  127. ppd = dd->pport + (i % dd->num_pports);
  128. rcd = qib_create_ctxtdata(ppd, i);
  129. if (!rcd) {
  130. qib_dev_err(dd,
  131. "Unable to allocate ctxtdata for Kernel ctxt, failing\n");
  132. ret = -ENOMEM;
  133. goto done;
  134. }
  135. rcd->pkeys[0] = QIB_DEFAULT_P_KEY;
  136. rcd->seq_cnt = 1;
  137. }
  138. ret = 0;
  139. done:
  140. return ret;
  141. }
  142. /*
  143. * Common code for user and kernel context setup.
  144. */
  145. struct qib_ctxtdata *qib_create_ctxtdata(struct qib_pportdata *ppd, u32 ctxt)
  146. {
  147. struct qib_devdata *dd = ppd->dd;
  148. struct qib_ctxtdata *rcd;
  149. rcd = kzalloc(sizeof(*rcd), GFP_KERNEL);
  150. if (rcd) {
  151. INIT_LIST_HEAD(&rcd->qp_wait_list);
  152. rcd->ppd = ppd;
  153. rcd->dd = dd;
  154. rcd->cnt = 1;
  155. rcd->ctxt = ctxt;
  156. dd->rcd[ctxt] = rcd;
  157. dd->f_init_ctxt(rcd);
  158. /*
  159. * To avoid wasting a lot of memory, we allocate 32KB chunks
  160. * of physically contiguous memory, advance through it until
  161. * used up and then allocate more. Of course, we need
  162. * memory to store those extra pointers, now. 32KB seems to
  163. * be the most that is "safe" under memory pressure
  164. * (creating large files and then copying them over
  165. * NFS while doing lots of MPI jobs). The OOM killer can
  166. * get invoked, even though we say we can sleep and this can
  167. * cause significant system problems....
  168. */
  169. rcd->rcvegrbuf_size = 0x8000;
  170. rcd->rcvegrbufs_perchunk =
  171. rcd->rcvegrbuf_size / dd->rcvegrbufsize;
  172. rcd->rcvegrbuf_chunks = (rcd->rcvegrcnt +
  173. rcd->rcvegrbufs_perchunk - 1) /
  174. rcd->rcvegrbufs_perchunk;
  175. BUG_ON(!is_power_of_2(rcd->rcvegrbufs_perchunk));
  176. rcd->rcvegrbufs_perchunk_shift =
  177. ilog2(rcd->rcvegrbufs_perchunk);
  178. }
  179. return rcd;
  180. }
  181. /*
  182. * Common code for initializing the physical port structure.
  183. */
  184. void qib_init_pportdata(struct qib_pportdata *ppd, struct qib_devdata *dd,
  185. u8 hw_pidx, u8 port)
  186. {
  187. int size;
  188. ppd->dd = dd;
  189. ppd->hw_pidx = hw_pidx;
  190. ppd->port = port; /* IB port number, not index */
  191. spin_lock_init(&ppd->sdma_lock);
  192. spin_lock_init(&ppd->lflags_lock);
  193. init_waitqueue_head(&ppd->state_wait);
  194. init_timer(&ppd->symerr_clear_timer);
  195. ppd->symerr_clear_timer.function = qib_clear_symerror_on_linkup;
  196. ppd->symerr_clear_timer.data = (unsigned long)ppd;
  197. ppd->qib_wq = NULL;
  198. spin_lock_init(&ppd->cc_shadow_lock);
  199. if (qib_cc_table_size < IB_CCT_MIN_ENTRIES)
  200. goto bail;
  201. ppd->cc_supported_table_entries = min(max_t(int, qib_cc_table_size,
  202. IB_CCT_MIN_ENTRIES), IB_CCT_ENTRIES*IB_CC_TABLE_CAP_DEFAULT);
  203. ppd->cc_max_table_entries =
  204. ppd->cc_supported_table_entries/IB_CCT_ENTRIES;
  205. size = IB_CC_TABLE_CAP_DEFAULT * sizeof(struct ib_cc_table_entry)
  206. * IB_CCT_ENTRIES;
  207. ppd->ccti_entries = kzalloc(size, GFP_KERNEL);
  208. if (!ppd->ccti_entries) {
  209. qib_dev_err(dd,
  210. "failed to allocate congestion control table for port %d!\n",
  211. port);
  212. goto bail;
  213. }
  214. size = IB_CC_CCS_ENTRIES * sizeof(struct ib_cc_congestion_entry);
  215. ppd->congestion_entries = kzalloc(size, GFP_KERNEL);
  216. if (!ppd->congestion_entries) {
  217. qib_dev_err(dd,
  218. "failed to allocate congestion setting list for port %d!\n",
  219. port);
  220. goto bail_1;
  221. }
  222. size = sizeof(struct cc_table_shadow);
  223. ppd->ccti_entries_shadow = kzalloc(size, GFP_KERNEL);
  224. if (!ppd->ccti_entries_shadow) {
  225. qib_dev_err(dd,
  226. "failed to allocate shadow ccti list for port %d!\n",
  227. port);
  228. goto bail_2;
  229. }
  230. size = sizeof(struct ib_cc_congestion_setting_attr);
  231. ppd->congestion_entries_shadow = kzalloc(size, GFP_KERNEL);
  232. if (!ppd->congestion_entries_shadow) {
  233. qib_dev_err(dd,
  234. "failed to allocate shadow congestion setting list for port %d!\n",
  235. port);
  236. goto bail_3;
  237. }
  238. return;
  239. bail_3:
  240. kfree(ppd->ccti_entries_shadow);
  241. ppd->ccti_entries_shadow = NULL;
  242. bail_2:
  243. kfree(ppd->congestion_entries);
  244. ppd->congestion_entries = NULL;
  245. bail_1:
  246. kfree(ppd->ccti_entries);
  247. ppd->ccti_entries = NULL;
  248. bail:
  249. /* User is intentionally disabling the congestion control agent */
  250. if (!qib_cc_table_size)
  251. return;
  252. if (qib_cc_table_size < IB_CCT_MIN_ENTRIES) {
  253. qib_cc_table_size = 0;
  254. qib_dev_err(dd,
  255. "Congestion Control table size %d less than minimum %d for port %d\n",
  256. qib_cc_table_size, IB_CCT_MIN_ENTRIES, port);
  257. }
  258. qib_dev_err(dd, "Congestion Control Agent disabled for port %d\n",
  259. port);
  260. return;
  261. }
  262. static int init_pioavailregs(struct qib_devdata *dd)
  263. {
  264. int ret, pidx;
  265. u64 *status_page;
  266. dd->pioavailregs_dma = dma_alloc_coherent(
  267. &dd->pcidev->dev, PAGE_SIZE, &dd->pioavailregs_phys,
  268. GFP_KERNEL);
  269. if (!dd->pioavailregs_dma) {
  270. qib_dev_err(dd,
  271. "failed to allocate PIOavail reg area in memory\n");
  272. ret = -ENOMEM;
  273. goto done;
  274. }
  275. /*
  276. * We really want L2 cache aligned, but for current CPUs of
  277. * interest, they are the same.
  278. */
  279. status_page = (u64 *)
  280. ((char *) dd->pioavailregs_dma +
  281. ((2 * L1_CACHE_BYTES +
  282. dd->pioavregs * sizeof(u64)) & ~L1_CACHE_BYTES));
  283. /* device status comes first, for backwards compatibility */
  284. dd->devstatusp = status_page;
  285. *status_page++ = 0;
  286. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  287. dd->pport[pidx].statusp = status_page;
  288. *status_page++ = 0;
  289. }
  290. /*
  291. * Setup buffer to hold freeze and other messages, accessible to
  292. * apps, following statusp. This is per-unit, not per port.
  293. */
  294. dd->freezemsg = (char *) status_page;
  295. *dd->freezemsg = 0;
  296. /* length of msg buffer is "whatever is left" */
  297. ret = (char *) status_page - (char *) dd->pioavailregs_dma;
  298. dd->freezelen = PAGE_SIZE - ret;
  299. ret = 0;
  300. done:
  301. return ret;
  302. }
  303. /**
  304. * init_shadow_tids - allocate the shadow TID array
  305. * @dd: the qlogic_ib device
  306. *
  307. * allocate the shadow TID array, so we can qib_munlock previous
  308. * entries. It may make more sense to move the pageshadow to the
  309. * ctxt data structure, so we only allocate memory for ctxts actually
  310. * in use, since we at 8k per ctxt, now.
  311. * We don't want failures here to prevent use of the driver/chip,
  312. * so no return value.
  313. */
  314. static void init_shadow_tids(struct qib_devdata *dd)
  315. {
  316. struct page **pages;
  317. dma_addr_t *addrs;
  318. pages = vzalloc(dd->cfgctxts * dd->rcvtidcnt * sizeof(struct page *));
  319. if (!pages) {
  320. qib_dev_err(dd,
  321. "failed to allocate shadow page * array, no expected sends!\n");
  322. goto bail;
  323. }
  324. addrs = vzalloc(dd->cfgctxts * dd->rcvtidcnt * sizeof(dma_addr_t));
  325. if (!addrs) {
  326. qib_dev_err(dd,
  327. "failed to allocate shadow dma handle array, no expected sends!\n");
  328. goto bail_free;
  329. }
  330. dd->pageshadow = pages;
  331. dd->physshadow = addrs;
  332. return;
  333. bail_free:
  334. vfree(pages);
  335. bail:
  336. dd->pageshadow = NULL;
  337. }
  338. /*
  339. * Do initialization for device that is only needed on
  340. * first detect, not on resets.
  341. */
  342. static int loadtime_init(struct qib_devdata *dd)
  343. {
  344. int ret = 0;
  345. if (((dd->revision >> QLOGIC_IB_R_SOFTWARE_SHIFT) &
  346. QLOGIC_IB_R_SOFTWARE_MASK) != QIB_CHIP_SWVERSION) {
  347. qib_dev_err(dd,
  348. "Driver only handles version %d, chip swversion is %d (%llx), failng\n",
  349. QIB_CHIP_SWVERSION,
  350. (int)(dd->revision >>
  351. QLOGIC_IB_R_SOFTWARE_SHIFT) &
  352. QLOGIC_IB_R_SOFTWARE_MASK,
  353. (unsigned long long) dd->revision);
  354. ret = -ENOSYS;
  355. goto done;
  356. }
  357. if (dd->revision & QLOGIC_IB_R_EMULATOR_MASK)
  358. qib_devinfo(dd->pcidev, "%s", dd->boardversion);
  359. spin_lock_init(&dd->pioavail_lock);
  360. spin_lock_init(&dd->sendctrl_lock);
  361. spin_lock_init(&dd->uctxt_lock);
  362. spin_lock_init(&dd->qib_diag_trans_lock);
  363. spin_lock_init(&dd->eep_st_lock);
  364. mutex_init(&dd->eep_lock);
  365. if (qib_mini_init)
  366. goto done;
  367. ret = init_pioavailregs(dd);
  368. init_shadow_tids(dd);
  369. qib_get_eeprom_info(dd);
  370. /* setup time (don't start yet) to verify we got interrupt */
  371. init_timer(&dd->intrchk_timer);
  372. dd->intrchk_timer.function = verify_interrupt;
  373. dd->intrchk_timer.data = (unsigned long) dd;
  374. done:
  375. return ret;
  376. }
  377. /**
  378. * init_after_reset - re-initialize after a reset
  379. * @dd: the qlogic_ib device
  380. *
  381. * sanity check at least some of the values after reset, and
  382. * ensure no receive or transmit (explicitly, in case reset
  383. * failed
  384. */
  385. static int init_after_reset(struct qib_devdata *dd)
  386. {
  387. int i;
  388. /*
  389. * Ensure chip does no sends or receives, tail updates, or
  390. * pioavail updates while we re-initialize. This is mostly
  391. * for the driver data structures, not chip registers.
  392. */
  393. for (i = 0; i < dd->num_pports; ++i) {
  394. /*
  395. * ctxt == -1 means "all contexts". Only really safe for
  396. * _dis_abling things, as here.
  397. */
  398. dd->f_rcvctrl(dd->pport + i, QIB_RCVCTRL_CTXT_DIS |
  399. QIB_RCVCTRL_INTRAVAIL_DIS |
  400. QIB_RCVCTRL_TAILUPD_DIS, -1);
  401. /* Redundant across ports for some, but no big deal. */
  402. dd->f_sendctrl(dd->pport + i, QIB_SENDCTRL_SEND_DIS |
  403. QIB_SENDCTRL_AVAIL_DIS);
  404. }
  405. return 0;
  406. }
  407. static void enable_chip(struct qib_devdata *dd)
  408. {
  409. u64 rcvmask;
  410. int i;
  411. /*
  412. * Enable PIO send, and update of PIOavail regs to memory.
  413. */
  414. for (i = 0; i < dd->num_pports; ++i)
  415. dd->f_sendctrl(dd->pport + i, QIB_SENDCTRL_SEND_ENB |
  416. QIB_SENDCTRL_AVAIL_ENB);
  417. /*
  418. * Enable kernel ctxts' receive and receive interrupt.
  419. * Other ctxts done as user opens and inits them.
  420. */
  421. rcvmask = QIB_RCVCTRL_CTXT_ENB | QIB_RCVCTRL_INTRAVAIL_ENB;
  422. rcvmask |= (dd->flags & QIB_NODMA_RTAIL) ?
  423. QIB_RCVCTRL_TAILUPD_DIS : QIB_RCVCTRL_TAILUPD_ENB;
  424. for (i = 0; dd->rcd && i < dd->first_user_ctxt; ++i) {
  425. struct qib_ctxtdata *rcd = dd->rcd[i];
  426. if (rcd)
  427. dd->f_rcvctrl(rcd->ppd, rcvmask, i);
  428. }
  429. }
  430. static void verify_interrupt(unsigned long opaque)
  431. {
  432. struct qib_devdata *dd = (struct qib_devdata *) opaque;
  433. if (!dd)
  434. return; /* being torn down */
  435. /*
  436. * If we don't have a lid or any interrupts, let the user know and
  437. * don't bother checking again.
  438. */
  439. if (dd->int_counter == 0) {
  440. if (!dd->f_intr_fallback(dd))
  441. dev_err(&dd->pcidev->dev,
  442. "No interrupts detected, not usable.\n");
  443. else /* re-arm the timer to see if fallback works */
  444. mod_timer(&dd->intrchk_timer, jiffies + HZ/2);
  445. }
  446. }
  447. static void init_piobuf_state(struct qib_devdata *dd)
  448. {
  449. int i, pidx;
  450. u32 uctxts;
  451. /*
  452. * Ensure all buffers are free, and fifos empty. Buffers
  453. * are common, so only do once for port 0.
  454. *
  455. * After enable and qib_chg_pioavailkernel so we can safely
  456. * enable pioavail updates and PIOENABLE. After this, packets
  457. * are ready and able to go out.
  458. */
  459. dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_ALL);
  460. for (pidx = 0; pidx < dd->num_pports; ++pidx)
  461. dd->f_sendctrl(dd->pport + pidx, QIB_SENDCTRL_FLUSH);
  462. /*
  463. * If not all sendbufs are used, add the one to each of the lower
  464. * numbered contexts. pbufsctxt and lastctxt_piobuf are
  465. * calculated in chip-specific code because it may cause some
  466. * chip-specific adjustments to be made.
  467. */
  468. uctxts = dd->cfgctxts - dd->first_user_ctxt;
  469. dd->ctxts_extrabuf = dd->pbufsctxt ?
  470. dd->lastctxt_piobuf - (dd->pbufsctxt * uctxts) : 0;
  471. /*
  472. * Set up the shadow copies of the piobufavail registers,
  473. * which we compare against the chip registers for now, and
  474. * the in memory DMA'ed copies of the registers.
  475. * By now pioavail updates to memory should have occurred, so
  476. * copy them into our working/shadow registers; this is in
  477. * case something went wrong with abort, but mostly to get the
  478. * initial values of the generation bit correct.
  479. */
  480. for (i = 0; i < dd->pioavregs; i++) {
  481. __le64 tmp;
  482. tmp = dd->pioavailregs_dma[i];
  483. /*
  484. * Don't need to worry about pioavailkernel here
  485. * because we will call qib_chg_pioavailkernel() later
  486. * in initialization, to busy out buffers as needed.
  487. */
  488. dd->pioavailshadow[i] = le64_to_cpu(tmp);
  489. }
  490. while (i < ARRAY_SIZE(dd->pioavailshadow))
  491. dd->pioavailshadow[i++] = 0; /* for debugging sanity */
  492. /* after pioavailshadow is setup */
  493. qib_chg_pioavailkernel(dd, 0, dd->piobcnt2k + dd->piobcnt4k,
  494. TXCHK_CHG_TYPE_KERN, NULL);
  495. dd->f_initvl15_bufs(dd);
  496. }
  497. /**
  498. * qib_create_workqueues - create per port workqueues
  499. * @dd: the qlogic_ib device
  500. */
  501. static int qib_create_workqueues(struct qib_devdata *dd)
  502. {
  503. int pidx;
  504. struct qib_pportdata *ppd;
  505. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  506. ppd = dd->pport + pidx;
  507. if (!ppd->qib_wq) {
  508. char wq_name[8]; /* 3 + 2 + 1 + 1 + 1 */
  509. snprintf(wq_name, sizeof(wq_name), "qib%d_%d",
  510. dd->unit, pidx);
  511. ppd->qib_wq =
  512. create_singlethread_workqueue(wq_name);
  513. if (!ppd->qib_wq)
  514. goto wq_error;
  515. }
  516. }
  517. return 0;
  518. wq_error:
  519. pr_err("create_singlethread_workqueue failed for port %d\n",
  520. pidx + 1);
  521. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  522. ppd = dd->pport + pidx;
  523. if (ppd->qib_wq) {
  524. destroy_workqueue(ppd->qib_wq);
  525. ppd->qib_wq = NULL;
  526. }
  527. }
  528. return -ENOMEM;
  529. }
  530. /**
  531. * qib_init - do the actual initialization sequence on the chip
  532. * @dd: the qlogic_ib device
  533. * @reinit: reinitializing, so don't allocate new memory
  534. *
  535. * Do the actual initialization sequence on the chip. This is done
  536. * both from the init routine called from the PCI infrastructure, and
  537. * when we reset the chip, or detect that it was reset internally,
  538. * or it's administratively re-enabled.
  539. *
  540. * Memory allocation here and in called routines is only done in
  541. * the first case (reinit == 0). We have to be careful, because even
  542. * without memory allocation, we need to re-write all the chip registers
  543. * TIDs, etc. after the reset or enable has completed.
  544. */
  545. int qib_init(struct qib_devdata *dd, int reinit)
  546. {
  547. int ret = 0, pidx, lastfail = 0;
  548. u32 portok = 0;
  549. unsigned i;
  550. struct qib_ctxtdata *rcd;
  551. struct qib_pportdata *ppd;
  552. unsigned long flags;
  553. /* Set linkstate to unknown, so we can watch for a transition. */
  554. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  555. ppd = dd->pport + pidx;
  556. spin_lock_irqsave(&ppd->lflags_lock, flags);
  557. ppd->lflags &= ~(QIBL_LINKACTIVE | QIBL_LINKARMED |
  558. QIBL_LINKDOWN | QIBL_LINKINIT |
  559. QIBL_LINKV);
  560. spin_unlock_irqrestore(&ppd->lflags_lock, flags);
  561. }
  562. if (reinit)
  563. ret = init_after_reset(dd);
  564. else
  565. ret = loadtime_init(dd);
  566. if (ret)
  567. goto done;
  568. /* Bypass most chip-init, to get to device creation */
  569. if (qib_mini_init)
  570. return 0;
  571. ret = dd->f_late_initreg(dd);
  572. if (ret)
  573. goto done;
  574. /* dd->rcd can be NULL if early init failed */
  575. for (i = 0; dd->rcd && i < dd->first_user_ctxt; ++i) {
  576. /*
  577. * Set up the (kernel) rcvhdr queue and egr TIDs. If doing
  578. * re-init, the simplest way to handle this is to free
  579. * existing, and re-allocate.
  580. * Need to re-create rest of ctxt 0 ctxtdata as well.
  581. */
  582. rcd = dd->rcd[i];
  583. if (!rcd)
  584. continue;
  585. lastfail = qib_create_rcvhdrq(dd, rcd);
  586. if (!lastfail)
  587. lastfail = qib_setup_eagerbufs(rcd);
  588. if (lastfail) {
  589. qib_dev_err(dd,
  590. "failed to allocate kernel ctxt's rcvhdrq and/or egr bufs\n");
  591. continue;
  592. }
  593. }
  594. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  595. int mtu;
  596. if (lastfail)
  597. ret = lastfail;
  598. ppd = dd->pport + pidx;
  599. mtu = ib_mtu_enum_to_int(qib_ibmtu);
  600. if (mtu == -1) {
  601. mtu = QIB_DEFAULT_MTU;
  602. qib_ibmtu = 0; /* don't leave invalid value */
  603. }
  604. /* set max we can ever have for this driver load */
  605. ppd->init_ibmaxlen = min(mtu > 2048 ?
  606. dd->piosize4k : dd->piosize2k,
  607. dd->rcvegrbufsize +
  608. (dd->rcvhdrentsize << 2));
  609. /*
  610. * Have to initialize ibmaxlen, but this will normally
  611. * change immediately in qib_set_mtu().
  612. */
  613. ppd->ibmaxlen = ppd->init_ibmaxlen;
  614. qib_set_mtu(ppd, mtu);
  615. spin_lock_irqsave(&ppd->lflags_lock, flags);
  616. ppd->lflags |= QIBL_IB_LINK_DISABLED;
  617. spin_unlock_irqrestore(&ppd->lflags_lock, flags);
  618. lastfail = dd->f_bringup_serdes(ppd);
  619. if (lastfail) {
  620. qib_devinfo(dd->pcidev,
  621. "Failed to bringup IB port %u\n", ppd->port);
  622. lastfail = -ENETDOWN;
  623. continue;
  624. }
  625. portok++;
  626. }
  627. if (!portok) {
  628. /* none of the ports initialized */
  629. if (!ret && lastfail)
  630. ret = lastfail;
  631. else if (!ret)
  632. ret = -ENETDOWN;
  633. /* but continue on, so we can debug cause */
  634. }
  635. enable_chip(dd);
  636. init_piobuf_state(dd);
  637. done:
  638. if (!ret) {
  639. /* chip is OK for user apps; mark it as initialized */
  640. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  641. ppd = dd->pport + pidx;
  642. /*
  643. * Set status even if port serdes is not initialized
  644. * so that diags will work.
  645. */
  646. *ppd->statusp |= QIB_STATUS_CHIP_PRESENT |
  647. QIB_STATUS_INITTED;
  648. if (!ppd->link_speed_enabled)
  649. continue;
  650. if (dd->flags & QIB_HAS_SEND_DMA)
  651. ret = qib_setup_sdma(ppd);
  652. init_timer(&ppd->hol_timer);
  653. ppd->hol_timer.function = qib_hol_event;
  654. ppd->hol_timer.data = (unsigned long)ppd;
  655. ppd->hol_state = QIB_HOL_UP;
  656. }
  657. /* now we can enable all interrupts from the chip */
  658. dd->f_set_intr_state(dd, 1);
  659. /*
  660. * Setup to verify we get an interrupt, and fallback
  661. * to an alternate if necessary and possible.
  662. */
  663. mod_timer(&dd->intrchk_timer, jiffies + HZ/2);
  664. /* start stats retrieval timer */
  665. mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER);
  666. }
  667. /* if ret is non-zero, we probably should do some cleanup here... */
  668. return ret;
  669. }
  670. /*
  671. * These next two routines are placeholders in case we don't have per-arch
  672. * code for controlling write combining. If explicit control of write
  673. * combining is not available, performance will probably be awful.
  674. */
  675. int __attribute__((weak)) qib_enable_wc(struct qib_devdata *dd)
  676. {
  677. return -EOPNOTSUPP;
  678. }
  679. void __attribute__((weak)) qib_disable_wc(struct qib_devdata *dd)
  680. {
  681. }
  682. static inline struct qib_devdata *__qib_lookup(int unit)
  683. {
  684. return idr_find(&qib_unit_table, unit);
  685. }
  686. struct qib_devdata *qib_lookup(int unit)
  687. {
  688. struct qib_devdata *dd;
  689. unsigned long flags;
  690. spin_lock_irqsave(&qib_devs_lock, flags);
  691. dd = __qib_lookup(unit);
  692. spin_unlock_irqrestore(&qib_devs_lock, flags);
  693. return dd;
  694. }
  695. /*
  696. * Stop the timers during unit shutdown, or after an error late
  697. * in initialization.
  698. */
  699. static void qib_stop_timers(struct qib_devdata *dd)
  700. {
  701. struct qib_pportdata *ppd;
  702. int pidx;
  703. if (dd->stats_timer.data) {
  704. del_timer_sync(&dd->stats_timer);
  705. dd->stats_timer.data = 0;
  706. }
  707. if (dd->intrchk_timer.data) {
  708. del_timer_sync(&dd->intrchk_timer);
  709. dd->intrchk_timer.data = 0;
  710. }
  711. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  712. ppd = dd->pport + pidx;
  713. if (ppd->hol_timer.data)
  714. del_timer_sync(&ppd->hol_timer);
  715. if (ppd->led_override_timer.data) {
  716. del_timer_sync(&ppd->led_override_timer);
  717. atomic_set(&ppd->led_override_timer_active, 0);
  718. }
  719. if (ppd->symerr_clear_timer.data)
  720. del_timer_sync(&ppd->symerr_clear_timer);
  721. }
  722. }
  723. /**
  724. * qib_shutdown_device - shut down a device
  725. * @dd: the qlogic_ib device
  726. *
  727. * This is called to make the device quiet when we are about to
  728. * unload the driver, and also when the device is administratively
  729. * disabled. It does not free any data structures.
  730. * Everything it does has to be setup again by qib_init(dd, 1)
  731. */
  732. static void qib_shutdown_device(struct qib_devdata *dd)
  733. {
  734. struct qib_pportdata *ppd;
  735. unsigned pidx;
  736. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  737. ppd = dd->pport + pidx;
  738. spin_lock_irq(&ppd->lflags_lock);
  739. ppd->lflags &= ~(QIBL_LINKDOWN | QIBL_LINKINIT |
  740. QIBL_LINKARMED | QIBL_LINKACTIVE |
  741. QIBL_LINKV);
  742. spin_unlock_irq(&ppd->lflags_lock);
  743. *ppd->statusp &= ~(QIB_STATUS_IB_CONF | QIB_STATUS_IB_READY);
  744. }
  745. dd->flags &= ~QIB_INITTED;
  746. /* mask interrupts, but not errors */
  747. dd->f_set_intr_state(dd, 0);
  748. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  749. ppd = dd->pport + pidx;
  750. dd->f_rcvctrl(ppd, QIB_RCVCTRL_TAILUPD_DIS |
  751. QIB_RCVCTRL_CTXT_DIS |
  752. QIB_RCVCTRL_INTRAVAIL_DIS |
  753. QIB_RCVCTRL_PKEY_ENB, -1);
  754. /*
  755. * Gracefully stop all sends allowing any in progress to
  756. * trickle out first.
  757. */
  758. dd->f_sendctrl(ppd, QIB_SENDCTRL_CLEAR);
  759. }
  760. /*
  761. * Enough for anything that's going to trickle out to have actually
  762. * done so.
  763. */
  764. udelay(20);
  765. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  766. ppd = dd->pport + pidx;
  767. dd->f_setextled(ppd, 0); /* make sure LEDs are off */
  768. if (dd->flags & QIB_HAS_SEND_DMA)
  769. qib_teardown_sdma(ppd);
  770. dd->f_sendctrl(ppd, QIB_SENDCTRL_AVAIL_DIS |
  771. QIB_SENDCTRL_SEND_DIS);
  772. /*
  773. * Clear SerdesEnable.
  774. * We can't count on interrupts since we are stopping.
  775. */
  776. dd->f_quiet_serdes(ppd);
  777. if (ppd->qib_wq) {
  778. destroy_workqueue(ppd->qib_wq);
  779. ppd->qib_wq = NULL;
  780. }
  781. }
  782. qib_update_eeprom_log(dd);
  783. }
  784. /**
  785. * qib_free_ctxtdata - free a context's allocated data
  786. * @dd: the qlogic_ib device
  787. * @rcd: the ctxtdata structure
  788. *
  789. * free up any allocated data for a context
  790. * This should not touch anything that would affect a simultaneous
  791. * re-allocation of context data, because it is called after qib_mutex
  792. * is released (and can be called from reinit as well).
  793. * It should never change any chip state, or global driver state.
  794. */
  795. void qib_free_ctxtdata(struct qib_devdata *dd, struct qib_ctxtdata *rcd)
  796. {
  797. if (!rcd)
  798. return;
  799. if (rcd->rcvhdrq) {
  800. dma_free_coherent(&dd->pcidev->dev, rcd->rcvhdrq_size,
  801. rcd->rcvhdrq, rcd->rcvhdrq_phys);
  802. rcd->rcvhdrq = NULL;
  803. if (rcd->rcvhdrtail_kvaddr) {
  804. dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
  805. rcd->rcvhdrtail_kvaddr,
  806. rcd->rcvhdrqtailaddr_phys);
  807. rcd->rcvhdrtail_kvaddr = NULL;
  808. }
  809. }
  810. if (rcd->rcvegrbuf) {
  811. unsigned e;
  812. for (e = 0; e < rcd->rcvegrbuf_chunks; e++) {
  813. void *base = rcd->rcvegrbuf[e];
  814. size_t size = rcd->rcvegrbuf_size;
  815. dma_free_coherent(&dd->pcidev->dev, size,
  816. base, rcd->rcvegrbuf_phys[e]);
  817. }
  818. kfree(rcd->rcvegrbuf);
  819. rcd->rcvegrbuf = NULL;
  820. kfree(rcd->rcvegrbuf_phys);
  821. rcd->rcvegrbuf_phys = NULL;
  822. rcd->rcvegrbuf_chunks = 0;
  823. }
  824. kfree(rcd->tid_pg_list);
  825. vfree(rcd->user_event_mask);
  826. vfree(rcd->subctxt_uregbase);
  827. vfree(rcd->subctxt_rcvegrbuf);
  828. vfree(rcd->subctxt_rcvhdr_base);
  829. kfree(rcd);
  830. }
  831. /*
  832. * Perform a PIO buffer bandwidth write test, to verify proper system
  833. * configuration. Even when all the setup calls work, occasionally
  834. * BIOS or other issues can prevent write combining from working, or
  835. * can cause other bandwidth problems to the chip.
  836. *
  837. * This test simply writes the same buffer over and over again, and
  838. * measures close to the peak bandwidth to the chip (not testing
  839. * data bandwidth to the wire). On chips that use an address-based
  840. * trigger to send packets to the wire, this is easy. On chips that
  841. * use a count to trigger, we want to make sure that the packet doesn't
  842. * go out on the wire, or trigger flow control checks.
  843. */
  844. static void qib_verify_pioperf(struct qib_devdata *dd)
  845. {
  846. u32 pbnum, cnt, lcnt;
  847. u32 __iomem *piobuf;
  848. u32 *addr;
  849. u64 msecs, emsecs;
  850. piobuf = dd->f_getsendbuf(dd->pport, 0ULL, &pbnum);
  851. if (!piobuf) {
  852. qib_devinfo(dd->pcidev,
  853. "No PIObufs for checking perf, skipping\n");
  854. return;
  855. }
  856. /*
  857. * Enough to give us a reasonable test, less than piobuf size, and
  858. * likely multiple of store buffer length.
  859. */
  860. cnt = 1024;
  861. addr = vmalloc(cnt);
  862. if (!addr) {
  863. qib_devinfo(dd->pcidev,
  864. "Couldn't get memory for checking PIO perf,"
  865. " skipping\n");
  866. goto done;
  867. }
  868. preempt_disable(); /* we want reasonably accurate elapsed time */
  869. msecs = 1 + jiffies_to_msecs(jiffies);
  870. for (lcnt = 0; lcnt < 10000U; lcnt++) {
  871. /* wait until we cross msec boundary */
  872. if (jiffies_to_msecs(jiffies) >= msecs)
  873. break;
  874. udelay(1);
  875. }
  876. dd->f_set_armlaunch(dd, 0);
  877. /*
  878. * length 0, no dwords actually sent
  879. */
  880. writeq(0, piobuf);
  881. qib_flush_wc();
  882. /*
  883. * This is only roughly accurate, since even with preempt we
  884. * still take interrupts that could take a while. Running for
  885. * >= 5 msec seems to get us "close enough" to accurate values.
  886. */
  887. msecs = jiffies_to_msecs(jiffies);
  888. for (emsecs = lcnt = 0; emsecs <= 5UL; lcnt++) {
  889. qib_pio_copy(piobuf + 64, addr, cnt >> 2);
  890. emsecs = jiffies_to_msecs(jiffies) - msecs;
  891. }
  892. /* 1 GiB/sec, slightly over IB SDR line rate */
  893. if (lcnt < (emsecs * 1024U))
  894. qib_dev_err(dd,
  895. "Performance problem: bandwidth to PIO buffers is only %u MiB/sec\n",
  896. lcnt / (u32) emsecs);
  897. preempt_enable();
  898. vfree(addr);
  899. done:
  900. /* disarm piobuf, so it's available again */
  901. dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_BUF(pbnum));
  902. qib_sendbuf_done(dd, pbnum);
  903. dd->f_set_armlaunch(dd, 1);
  904. }
  905. void qib_free_devdata(struct qib_devdata *dd)
  906. {
  907. unsigned long flags;
  908. spin_lock_irqsave(&qib_devs_lock, flags);
  909. idr_remove(&qib_unit_table, dd->unit);
  910. list_del(&dd->list);
  911. spin_unlock_irqrestore(&qib_devs_lock, flags);
  912. ib_dealloc_device(&dd->verbs_dev.ibdev);
  913. }
  914. /*
  915. * Allocate our primary per-unit data structure. Must be done via verbs
  916. * allocator, because the verbs cleanup process both does cleanup and
  917. * free of the data structure.
  918. * "extra" is for chip-specific data.
  919. *
  920. * Use the idr mechanism to get a unit number for this unit.
  921. */
  922. struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra)
  923. {
  924. unsigned long flags;
  925. struct qib_devdata *dd;
  926. int ret;
  927. if (!idr_pre_get(&qib_unit_table, GFP_KERNEL)) {
  928. dd = ERR_PTR(-ENOMEM);
  929. goto bail;
  930. }
  931. dd = (struct qib_devdata *) ib_alloc_device(sizeof(*dd) + extra);
  932. if (!dd) {
  933. dd = ERR_PTR(-ENOMEM);
  934. goto bail;
  935. }
  936. spin_lock_irqsave(&qib_devs_lock, flags);
  937. ret = idr_get_new(&qib_unit_table, dd, &dd->unit);
  938. if (ret >= 0)
  939. list_add(&dd->list, &qib_dev_list);
  940. spin_unlock_irqrestore(&qib_devs_lock, flags);
  941. if (ret < 0) {
  942. qib_early_err(&pdev->dev,
  943. "Could not allocate unit ID: error %d\n", -ret);
  944. ib_dealloc_device(&dd->verbs_dev.ibdev);
  945. dd = ERR_PTR(ret);
  946. goto bail;
  947. }
  948. if (!qib_cpulist_count) {
  949. u32 count = num_online_cpus();
  950. qib_cpulist = kzalloc(BITS_TO_LONGS(count) *
  951. sizeof(long), GFP_KERNEL);
  952. if (qib_cpulist)
  953. qib_cpulist_count = count;
  954. else
  955. qib_early_err(&pdev->dev,
  956. "Could not alloc cpulist info, cpu affinity might be wrong\n");
  957. }
  958. bail:
  959. return dd;
  960. }
  961. /*
  962. * Called from freeze mode handlers, and from PCI error
  963. * reporting code. Should be paranoid about state of
  964. * system and data structures.
  965. */
  966. void qib_disable_after_error(struct qib_devdata *dd)
  967. {
  968. if (dd->flags & QIB_INITTED) {
  969. u32 pidx;
  970. dd->flags &= ~QIB_INITTED;
  971. if (dd->pport)
  972. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  973. struct qib_pportdata *ppd;
  974. ppd = dd->pport + pidx;
  975. if (dd->flags & QIB_PRESENT) {
  976. qib_set_linkstate(ppd,
  977. QIB_IB_LINKDOWN_DISABLE);
  978. dd->f_setextled(ppd, 0);
  979. }
  980. *ppd->statusp &= ~QIB_STATUS_IB_READY;
  981. }
  982. }
  983. /*
  984. * Mark as having had an error for driver, and also
  985. * for /sys and status word mapped to user programs.
  986. * This marks unit as not usable, until reset.
  987. */
  988. if (dd->devstatusp)
  989. *dd->devstatusp |= QIB_STATUS_HWERROR;
  990. }
  991. static void qib_remove_one(struct pci_dev *);
  992. static int qib_init_one(struct pci_dev *, const struct pci_device_id *);
  993. #define DRIVER_LOAD_MSG "QLogic " QIB_DRV_NAME " loaded: "
  994. #define PFX QIB_DRV_NAME ": "
  995. static DEFINE_PCI_DEVICE_TABLE(qib_pci_tbl) = {
  996. { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_QLOGIC_IB_6120) },
  997. { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_IB_7220) },
  998. { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_IB_7322) },
  999. { 0, }
  1000. };
  1001. MODULE_DEVICE_TABLE(pci, qib_pci_tbl);
  1002. struct pci_driver qib_driver = {
  1003. .name = QIB_DRV_NAME,
  1004. .probe = qib_init_one,
  1005. .remove = qib_remove_one,
  1006. .id_table = qib_pci_tbl,
  1007. .err_handler = &qib_pci_err_handler,
  1008. };
  1009. /*
  1010. * Do all the generic driver unit- and chip-independent memory
  1011. * allocation and initialization.
  1012. */
  1013. static int __init qlogic_ib_init(void)
  1014. {
  1015. int ret;
  1016. ret = qib_dev_init();
  1017. if (ret)
  1018. goto bail;
  1019. qib_cq_wq = create_singlethread_workqueue("qib_cq");
  1020. if (!qib_cq_wq) {
  1021. ret = -ENOMEM;
  1022. goto bail_dev;
  1023. }
  1024. /*
  1025. * These must be called before the driver is registered with
  1026. * the PCI subsystem.
  1027. */
  1028. idr_init(&qib_unit_table);
  1029. if (!idr_pre_get(&qib_unit_table, GFP_KERNEL)) {
  1030. pr_err("idr_pre_get() failed\n");
  1031. ret = -ENOMEM;
  1032. goto bail_cq_wq;
  1033. }
  1034. ret = pci_register_driver(&qib_driver);
  1035. if (ret < 0) {
  1036. pr_err("Unable to register driver: error %d\n", -ret);
  1037. goto bail_unit;
  1038. }
  1039. /* not fatal if it doesn't work */
  1040. if (qib_init_qibfs())
  1041. pr_err("Unable to register ipathfs\n");
  1042. goto bail; /* all OK */
  1043. bail_unit:
  1044. idr_destroy(&qib_unit_table);
  1045. bail_cq_wq:
  1046. destroy_workqueue(qib_cq_wq);
  1047. bail_dev:
  1048. qib_dev_cleanup();
  1049. bail:
  1050. return ret;
  1051. }
  1052. module_init(qlogic_ib_init);
  1053. /*
  1054. * Do the non-unit driver cleanup, memory free, etc. at unload.
  1055. */
  1056. static void __exit qlogic_ib_cleanup(void)
  1057. {
  1058. int ret;
  1059. ret = qib_exit_qibfs();
  1060. if (ret)
  1061. pr_err(
  1062. "Unable to cleanup counter filesystem: error %d\n",
  1063. -ret);
  1064. pci_unregister_driver(&qib_driver);
  1065. destroy_workqueue(qib_cq_wq);
  1066. qib_cpulist_count = 0;
  1067. kfree(qib_cpulist);
  1068. idr_destroy(&qib_unit_table);
  1069. qib_dev_cleanup();
  1070. }
  1071. module_exit(qlogic_ib_cleanup);
  1072. /* this can only be called after a successful initialization */
  1073. static void cleanup_device_data(struct qib_devdata *dd)
  1074. {
  1075. int ctxt;
  1076. int pidx;
  1077. struct qib_ctxtdata **tmp;
  1078. unsigned long flags;
  1079. /* users can't do anything more with chip */
  1080. for (pidx = 0; pidx < dd->num_pports; ++pidx) {
  1081. if (dd->pport[pidx].statusp)
  1082. *dd->pport[pidx].statusp &= ~QIB_STATUS_CHIP_PRESENT;
  1083. spin_lock(&dd->pport[pidx].cc_shadow_lock);
  1084. kfree(dd->pport[pidx].congestion_entries);
  1085. dd->pport[pidx].congestion_entries = NULL;
  1086. kfree(dd->pport[pidx].ccti_entries);
  1087. dd->pport[pidx].ccti_entries = NULL;
  1088. kfree(dd->pport[pidx].ccti_entries_shadow);
  1089. dd->pport[pidx].ccti_entries_shadow = NULL;
  1090. kfree(dd->pport[pidx].congestion_entries_shadow);
  1091. dd->pport[pidx].congestion_entries_shadow = NULL;
  1092. spin_unlock(&dd->pport[pidx].cc_shadow_lock);
  1093. }
  1094. if (!qib_wc_pat)
  1095. qib_disable_wc(dd);
  1096. if (dd->pioavailregs_dma) {
  1097. dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
  1098. (void *) dd->pioavailregs_dma,
  1099. dd->pioavailregs_phys);
  1100. dd->pioavailregs_dma = NULL;
  1101. }
  1102. if (dd->pageshadow) {
  1103. struct page **tmpp = dd->pageshadow;
  1104. dma_addr_t *tmpd = dd->physshadow;
  1105. int i, cnt = 0;
  1106. for (ctxt = 0; ctxt < dd->cfgctxts; ctxt++) {
  1107. int ctxt_tidbase = ctxt * dd->rcvtidcnt;
  1108. int maxtid = ctxt_tidbase + dd->rcvtidcnt;
  1109. for (i = ctxt_tidbase; i < maxtid; i++) {
  1110. if (!tmpp[i])
  1111. continue;
  1112. pci_unmap_page(dd->pcidev, tmpd[i],
  1113. PAGE_SIZE, PCI_DMA_FROMDEVICE);
  1114. qib_release_user_pages(&tmpp[i], 1);
  1115. tmpp[i] = NULL;
  1116. cnt++;
  1117. }
  1118. }
  1119. tmpp = dd->pageshadow;
  1120. dd->pageshadow = NULL;
  1121. vfree(tmpp);
  1122. }
  1123. /*
  1124. * Free any resources still in use (usually just kernel contexts)
  1125. * at unload; we do for ctxtcnt, because that's what we allocate.
  1126. * We acquire lock to be really paranoid that rcd isn't being
  1127. * accessed from some interrupt-related code (that should not happen,
  1128. * but best to be sure).
  1129. */
  1130. spin_lock_irqsave(&dd->uctxt_lock, flags);
  1131. tmp = dd->rcd;
  1132. dd->rcd = NULL;
  1133. spin_unlock_irqrestore(&dd->uctxt_lock, flags);
  1134. for (ctxt = 0; tmp && ctxt < dd->ctxtcnt; ctxt++) {
  1135. struct qib_ctxtdata *rcd = tmp[ctxt];
  1136. tmp[ctxt] = NULL; /* debugging paranoia */
  1137. qib_free_ctxtdata(dd, rcd);
  1138. }
  1139. kfree(tmp);
  1140. kfree(dd->boardname);
  1141. }
  1142. /*
  1143. * Clean up on unit shutdown, or error during unit load after
  1144. * successful initialization.
  1145. */
  1146. static void qib_postinit_cleanup(struct qib_devdata *dd)
  1147. {
  1148. /*
  1149. * Clean up chip-specific stuff.
  1150. * We check for NULL here, because it's outside
  1151. * the kregbase check, and we need to call it
  1152. * after the free_irq. Thus it's possible that
  1153. * the function pointers were never initialized.
  1154. */
  1155. if (dd->f_cleanup)
  1156. dd->f_cleanup(dd);
  1157. qib_pcie_ddcleanup(dd);
  1158. cleanup_device_data(dd);
  1159. qib_free_devdata(dd);
  1160. }
  1161. static int qib_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
  1162. {
  1163. int ret, j, pidx, initfail;
  1164. struct qib_devdata *dd = NULL;
  1165. ret = qib_pcie_init(pdev, ent);
  1166. if (ret)
  1167. goto bail;
  1168. /*
  1169. * Do device-specific initialiation, function table setup, dd
  1170. * allocation, etc.
  1171. */
  1172. switch (ent->device) {
  1173. case PCI_DEVICE_ID_QLOGIC_IB_6120:
  1174. #ifdef CONFIG_PCI_MSI
  1175. dd = qib_init_iba6120_funcs(pdev, ent);
  1176. #else
  1177. qib_early_err(&pdev->dev,
  1178. "QLogic PCIE device 0x%x cannot work if CONFIG_PCI_MSI is not enabled\n",
  1179. ent->device);
  1180. dd = ERR_PTR(-ENODEV);
  1181. #endif
  1182. break;
  1183. case PCI_DEVICE_ID_QLOGIC_IB_7220:
  1184. dd = qib_init_iba7220_funcs(pdev, ent);
  1185. break;
  1186. case PCI_DEVICE_ID_QLOGIC_IB_7322:
  1187. dd = qib_init_iba7322_funcs(pdev, ent);
  1188. break;
  1189. default:
  1190. qib_early_err(&pdev->dev,
  1191. "Failing on unknown QLogic deviceid 0x%x\n",
  1192. ent->device);
  1193. ret = -ENODEV;
  1194. }
  1195. if (IS_ERR(dd))
  1196. ret = PTR_ERR(dd);
  1197. if (ret)
  1198. goto bail; /* error already printed */
  1199. ret = qib_create_workqueues(dd);
  1200. if (ret)
  1201. goto bail;
  1202. /* do the generic initialization */
  1203. initfail = qib_init(dd, 0);
  1204. ret = qib_register_ib_device(dd);
  1205. /*
  1206. * Now ready for use. this should be cleared whenever we
  1207. * detect a reset, or initiate one. If earlier failure,
  1208. * we still create devices, so diags, etc. can be used
  1209. * to determine cause of problem.
  1210. */
  1211. if (!qib_mini_init && !initfail && !ret)
  1212. dd->flags |= QIB_INITTED;
  1213. j = qib_device_create(dd);
  1214. if (j)
  1215. qib_dev_err(dd, "Failed to create /dev devices: %d\n", -j);
  1216. j = qibfs_add(dd);
  1217. if (j)
  1218. qib_dev_err(dd, "Failed filesystem setup for counters: %d\n",
  1219. -j);
  1220. if (qib_mini_init || initfail || ret) {
  1221. qib_stop_timers(dd);
  1222. flush_workqueue(ib_wq);
  1223. for (pidx = 0; pidx < dd->num_pports; ++pidx)
  1224. dd->f_quiet_serdes(dd->pport + pidx);
  1225. if (qib_mini_init)
  1226. goto bail;
  1227. if (!j) {
  1228. (void) qibfs_remove(dd);
  1229. qib_device_remove(dd);
  1230. }
  1231. if (!ret)
  1232. qib_unregister_ib_device(dd);
  1233. qib_postinit_cleanup(dd);
  1234. if (initfail)
  1235. ret = initfail;
  1236. goto bail;
  1237. }
  1238. if (!qib_wc_pat) {
  1239. ret = qib_enable_wc(dd);
  1240. if (ret) {
  1241. qib_dev_err(dd,
  1242. "Write combining not enabled (err %d): performance may be poor\n",
  1243. -ret);
  1244. ret = 0;
  1245. }
  1246. }
  1247. qib_verify_pioperf(dd);
  1248. bail:
  1249. return ret;
  1250. }
  1251. static void qib_remove_one(struct pci_dev *pdev)
  1252. {
  1253. struct qib_devdata *dd = pci_get_drvdata(pdev);
  1254. int ret;
  1255. /* unregister from IB core */
  1256. qib_unregister_ib_device(dd);
  1257. /*
  1258. * Disable the IB link, disable interrupts on the device,
  1259. * clear dma engines, etc.
  1260. */
  1261. if (!qib_mini_init)
  1262. qib_shutdown_device(dd);
  1263. qib_stop_timers(dd);
  1264. /* wait until all of our (qsfp) queue_work() calls complete */
  1265. flush_workqueue(ib_wq);
  1266. ret = qibfs_remove(dd);
  1267. if (ret)
  1268. qib_dev_err(dd, "Failed counters filesystem cleanup: %d\n",
  1269. -ret);
  1270. qib_device_remove(dd);
  1271. qib_postinit_cleanup(dd);
  1272. }
  1273. /**
  1274. * qib_create_rcvhdrq - create a receive header queue
  1275. * @dd: the qlogic_ib device
  1276. * @rcd: the context data
  1277. *
  1278. * This must be contiguous memory (from an i/o perspective), and must be
  1279. * DMA'able (which means for some systems, it will go through an IOMMU,
  1280. * or be forced into a low address range).
  1281. */
  1282. int qib_create_rcvhdrq(struct qib_devdata *dd, struct qib_ctxtdata *rcd)
  1283. {
  1284. unsigned amt;
  1285. if (!rcd->rcvhdrq) {
  1286. dma_addr_t phys_hdrqtail;
  1287. gfp_t gfp_flags;
  1288. amt = ALIGN(dd->rcvhdrcnt * dd->rcvhdrentsize *
  1289. sizeof(u32), PAGE_SIZE);
  1290. gfp_flags = (rcd->ctxt >= dd->first_user_ctxt) ?
  1291. GFP_USER : GFP_KERNEL;
  1292. rcd->rcvhdrq = dma_alloc_coherent(
  1293. &dd->pcidev->dev, amt, &rcd->rcvhdrq_phys,
  1294. gfp_flags | __GFP_COMP);
  1295. if (!rcd->rcvhdrq) {
  1296. qib_dev_err(dd,
  1297. "attempt to allocate %d bytes for ctxt %u rcvhdrq failed\n",
  1298. amt, rcd->ctxt);
  1299. goto bail;
  1300. }
  1301. if (rcd->ctxt >= dd->first_user_ctxt) {
  1302. rcd->user_event_mask = vmalloc_user(PAGE_SIZE);
  1303. if (!rcd->user_event_mask)
  1304. goto bail_free_hdrq;
  1305. }
  1306. if (!(dd->flags & QIB_NODMA_RTAIL)) {
  1307. rcd->rcvhdrtail_kvaddr = dma_alloc_coherent(
  1308. &dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail,
  1309. gfp_flags);
  1310. if (!rcd->rcvhdrtail_kvaddr)
  1311. goto bail_free;
  1312. rcd->rcvhdrqtailaddr_phys = phys_hdrqtail;
  1313. }
  1314. rcd->rcvhdrq_size = amt;
  1315. }
  1316. /* clear for security and sanity on each use */
  1317. memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size);
  1318. if (rcd->rcvhdrtail_kvaddr)
  1319. memset(rcd->rcvhdrtail_kvaddr, 0, PAGE_SIZE);
  1320. return 0;
  1321. bail_free:
  1322. qib_dev_err(dd,
  1323. "attempt to allocate 1 page for ctxt %u rcvhdrqtailaddr failed\n",
  1324. rcd->ctxt);
  1325. vfree(rcd->user_event_mask);
  1326. rcd->user_event_mask = NULL;
  1327. bail_free_hdrq:
  1328. dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq,
  1329. rcd->rcvhdrq_phys);
  1330. rcd->rcvhdrq = NULL;
  1331. bail:
  1332. return -ENOMEM;
  1333. }
  1334. /**
  1335. * allocate eager buffers, both kernel and user contexts.
  1336. * @rcd: the context we are setting up.
  1337. *
  1338. * Allocate the eager TID buffers and program them into hip.
  1339. * They are no longer completely contiguous, we do multiple allocation
  1340. * calls. Otherwise we get the OOM code involved, by asking for too
  1341. * much per call, with disastrous results on some kernels.
  1342. */
  1343. int qib_setup_eagerbufs(struct qib_ctxtdata *rcd)
  1344. {
  1345. struct qib_devdata *dd = rcd->dd;
  1346. unsigned e, egrcnt, egrperchunk, chunk, egrsize, egroff;
  1347. size_t size;
  1348. gfp_t gfp_flags;
  1349. /*
  1350. * GFP_USER, but without GFP_FS, so buffer cache can be
  1351. * coalesced (we hope); otherwise, even at order 4,
  1352. * heavy filesystem activity makes these fail, and we can
  1353. * use compound pages.
  1354. */
  1355. gfp_flags = __GFP_WAIT | __GFP_IO | __GFP_COMP;
  1356. egrcnt = rcd->rcvegrcnt;
  1357. egroff = rcd->rcvegr_tid_base;
  1358. egrsize = dd->rcvegrbufsize;
  1359. chunk = rcd->rcvegrbuf_chunks;
  1360. egrperchunk = rcd->rcvegrbufs_perchunk;
  1361. size = rcd->rcvegrbuf_size;
  1362. if (!rcd->rcvegrbuf) {
  1363. rcd->rcvegrbuf =
  1364. kzalloc(chunk * sizeof(rcd->rcvegrbuf[0]),
  1365. GFP_KERNEL);
  1366. if (!rcd->rcvegrbuf)
  1367. goto bail;
  1368. }
  1369. if (!rcd->rcvegrbuf_phys) {
  1370. rcd->rcvegrbuf_phys =
  1371. kmalloc(chunk * sizeof(rcd->rcvegrbuf_phys[0]),
  1372. GFP_KERNEL);
  1373. if (!rcd->rcvegrbuf_phys)
  1374. goto bail_rcvegrbuf;
  1375. }
  1376. for (e = 0; e < rcd->rcvegrbuf_chunks; e++) {
  1377. if (rcd->rcvegrbuf[e])
  1378. continue;
  1379. rcd->rcvegrbuf[e] =
  1380. dma_alloc_coherent(&dd->pcidev->dev, size,
  1381. &rcd->rcvegrbuf_phys[e],
  1382. gfp_flags);
  1383. if (!rcd->rcvegrbuf[e])
  1384. goto bail_rcvegrbuf_phys;
  1385. }
  1386. rcd->rcvegr_phys = rcd->rcvegrbuf_phys[0];
  1387. for (e = chunk = 0; chunk < rcd->rcvegrbuf_chunks; chunk++) {
  1388. dma_addr_t pa = rcd->rcvegrbuf_phys[chunk];
  1389. unsigned i;
  1390. /* clear for security and sanity on each use */
  1391. memset(rcd->rcvegrbuf[chunk], 0, size);
  1392. for (i = 0; e < egrcnt && i < egrperchunk; e++, i++) {
  1393. dd->f_put_tid(dd, e + egroff +
  1394. (u64 __iomem *)
  1395. ((char __iomem *)
  1396. dd->kregbase +
  1397. dd->rcvegrbase),
  1398. RCVHQ_RCV_TYPE_EAGER, pa);
  1399. pa += egrsize;
  1400. }
  1401. cond_resched(); /* don't hog the cpu */
  1402. }
  1403. return 0;
  1404. bail_rcvegrbuf_phys:
  1405. for (e = 0; e < rcd->rcvegrbuf_chunks && rcd->rcvegrbuf[e]; e++)
  1406. dma_free_coherent(&dd->pcidev->dev, size,
  1407. rcd->rcvegrbuf[e], rcd->rcvegrbuf_phys[e]);
  1408. kfree(rcd->rcvegrbuf_phys);
  1409. rcd->rcvegrbuf_phys = NULL;
  1410. bail_rcvegrbuf:
  1411. kfree(rcd->rcvegrbuf);
  1412. rcd->rcvegrbuf = NULL;
  1413. bail:
  1414. return -ENOMEM;
  1415. }
  1416. /*
  1417. * Note: Changes to this routine should be mirrored
  1418. * for the diagnostics routine qib_remap_ioaddr32().
  1419. * There is also related code for VL15 buffers in qib_init_7322_variables().
  1420. * The teardown code that unmaps is in qib_pcie_ddcleanup()
  1421. */
  1422. int init_chip_wc_pat(struct qib_devdata *dd, u32 vl15buflen)
  1423. {
  1424. u64 __iomem *qib_kregbase = NULL;
  1425. void __iomem *qib_piobase = NULL;
  1426. u64 __iomem *qib_userbase = NULL;
  1427. u64 qib_kreglen;
  1428. u64 qib_pio2koffset = dd->piobufbase & 0xffffffff;
  1429. u64 qib_pio4koffset = dd->piobufbase >> 32;
  1430. u64 qib_pio2klen = dd->piobcnt2k * dd->palign;
  1431. u64 qib_pio4klen = dd->piobcnt4k * dd->align4k;
  1432. u64 qib_physaddr = dd->physaddr;
  1433. u64 qib_piolen;
  1434. u64 qib_userlen = 0;
  1435. /*
  1436. * Free the old mapping because the kernel will try to reuse the
  1437. * old mapping and not create a new mapping with the
  1438. * write combining attribute.
  1439. */
  1440. iounmap(dd->kregbase);
  1441. dd->kregbase = NULL;
  1442. /*
  1443. * Assumes chip address space looks like:
  1444. * - kregs + sregs + cregs + uregs (in any order)
  1445. * - piobufs (2K and 4K bufs in either order)
  1446. * or:
  1447. * - kregs + sregs + cregs (in any order)
  1448. * - piobufs (2K and 4K bufs in either order)
  1449. * - uregs
  1450. */
  1451. if (dd->piobcnt4k == 0) {
  1452. qib_kreglen = qib_pio2koffset;
  1453. qib_piolen = qib_pio2klen;
  1454. } else if (qib_pio2koffset < qib_pio4koffset) {
  1455. qib_kreglen = qib_pio2koffset;
  1456. qib_piolen = qib_pio4koffset + qib_pio4klen - qib_kreglen;
  1457. } else {
  1458. qib_kreglen = qib_pio4koffset;
  1459. qib_piolen = qib_pio2koffset + qib_pio2klen - qib_kreglen;
  1460. }
  1461. qib_piolen += vl15buflen;
  1462. /* Map just the configured ports (not all hw ports) */
  1463. if (dd->uregbase > qib_kreglen)
  1464. qib_userlen = dd->ureg_align * dd->cfgctxts;
  1465. /* Sanity checks passed, now create the new mappings */
  1466. qib_kregbase = ioremap_nocache(qib_physaddr, qib_kreglen);
  1467. if (!qib_kregbase)
  1468. goto bail;
  1469. qib_piobase = ioremap_wc(qib_physaddr + qib_kreglen, qib_piolen);
  1470. if (!qib_piobase)
  1471. goto bail_kregbase;
  1472. if (qib_userlen) {
  1473. qib_userbase = ioremap_nocache(qib_physaddr + dd->uregbase,
  1474. qib_userlen);
  1475. if (!qib_userbase)
  1476. goto bail_piobase;
  1477. }
  1478. dd->kregbase = qib_kregbase;
  1479. dd->kregend = (u64 __iomem *)
  1480. ((char __iomem *) qib_kregbase + qib_kreglen);
  1481. dd->piobase = qib_piobase;
  1482. dd->pio2kbase = (void __iomem *)
  1483. (((char __iomem *) dd->piobase) +
  1484. qib_pio2koffset - qib_kreglen);
  1485. if (dd->piobcnt4k)
  1486. dd->pio4kbase = (void __iomem *)
  1487. (((char __iomem *) dd->piobase) +
  1488. qib_pio4koffset - qib_kreglen);
  1489. if (qib_userlen)
  1490. /* ureg will now be accessed relative to dd->userbase */
  1491. dd->userbase = qib_userbase;
  1492. return 0;
  1493. bail_piobase:
  1494. iounmap(qib_piobase);
  1495. bail_kregbase:
  1496. iounmap(qib_kregbase);
  1497. bail:
  1498. return -ENOMEM;
  1499. }