ntb_hw.c 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439
  1. /*
  2. * This file is provided under a dual BSD/GPLv2 license. When using or
  3. * redistributing this file, you may do so under either license.
  4. *
  5. * GPL LICENSE SUMMARY
  6. *
  7. * Copyright(c) 2012 Intel Corporation. All rights reserved.
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of version 2 of the GNU General Public License as
  11. * published by the Free Software Foundation.
  12. *
  13. * BSD LICENSE
  14. *
  15. * Copyright(c) 2012 Intel Corporation. All rights reserved.
  16. *
  17. * Redistribution and use in source and binary forms, with or without
  18. * modification, are permitted provided that the following conditions
  19. * are met:
  20. *
  21. * * Redistributions of source code must retain the above copyright
  22. * notice, this list of conditions and the following disclaimer.
  23. * * Redistributions in binary form must reproduce the above copy
  24. * notice, this list of conditions and the following disclaimer in
  25. * the documentation and/or other materials provided with the
  26. * distribution.
  27. * * Neither the name of Intel Corporation nor the names of its
  28. * contributors may be used to endorse or promote products derived
  29. * from this software without specific prior written permission.
  30. *
  31. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  32. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  33. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  34. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  35. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  36. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  37. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  38. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  39. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  40. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  41. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42. *
  43. * Intel PCIe NTB Linux driver
  44. *
  45. * Contact Information:
  46. * Jon Mason <jon.mason@intel.com>
  47. */
  48. #include <linux/debugfs.h>
  49. #include <linux/delay.h>
  50. #include <linux/init.h>
  51. #include <linux/interrupt.h>
  52. #include <linux/module.h>
  53. #include <linux/pci.h>
  54. #include <linux/random.h>
  55. #include <linux/slab.h>
  56. #include "ntb_hw.h"
  57. #include "ntb_regs.h"
  58. #define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver"
  59. #define NTB_VER "1.0"
  60. MODULE_DESCRIPTION(NTB_NAME);
  61. MODULE_VERSION(NTB_VER);
  62. MODULE_LICENSE("Dual BSD/GPL");
  63. MODULE_AUTHOR("Intel Corporation");
  64. static bool xeon_errata_workaround = true;
  65. module_param(xeon_errata_workaround, bool, 0644);
  66. MODULE_PARM_DESC(xeon_errata_workaround, "Workaround for the Xeon Errata");
  67. enum {
  68. NTB_CONN_TRANSPARENT = 0,
  69. NTB_CONN_B2B,
  70. NTB_CONN_RP,
  71. };
  72. enum {
  73. NTB_DEV_USD = 0,
  74. NTB_DEV_DSD,
  75. };
  76. enum {
  77. SNB_HW = 0,
  78. BWD_HW,
  79. };
  80. static struct dentry *debugfs_dir;
  81. #define BWD_LINK_RECOVERY_TIME 500
  82. /* Translate memory window 0,1 to BAR 2,4 */
  83. #define MW_TO_BAR(mw) (mw * NTB_MAX_NUM_MW + 2)
  84. static DEFINE_PCI_DEVICE_TABLE(ntb_pci_tbl) = {
  85. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
  86. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
  87. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
  88. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
  89. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
  90. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
  91. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
  92. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
  93. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
  94. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
  95. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
  96. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
  97. {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
  98. {0}
  99. };
  100. MODULE_DEVICE_TABLE(pci, ntb_pci_tbl);
  101. /**
  102. * ntb_register_event_callback() - register event callback
  103. * @ndev: pointer to ntb_device instance
  104. * @func: callback function to register
  105. *
  106. * This function registers a callback for any HW driver events such as link
  107. * up/down, power management notices and etc.
  108. *
  109. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  110. */
  111. int ntb_register_event_callback(struct ntb_device *ndev,
  112. void (*func)(void *handle, enum ntb_hw_event event))
  113. {
  114. if (ndev->event_cb)
  115. return -EINVAL;
  116. ndev->event_cb = func;
  117. return 0;
  118. }
  119. /**
  120. * ntb_unregister_event_callback() - unregisters the event callback
  121. * @ndev: pointer to ntb_device instance
  122. *
  123. * This function unregisters the existing callback from transport
  124. */
  125. void ntb_unregister_event_callback(struct ntb_device *ndev)
  126. {
  127. ndev->event_cb = NULL;
  128. }
  129. /**
  130. * ntb_register_db_callback() - register a callback for doorbell interrupt
  131. * @ndev: pointer to ntb_device instance
  132. * @idx: doorbell index to register callback, zero based
  133. * @data: pointer to be returned to caller with every callback
  134. * @func: callback function to register
  135. *
  136. * This function registers a callback function for the doorbell interrupt
  137. * on the primary side. The function will unmask the doorbell as well to
  138. * allow interrupt.
  139. *
  140. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  141. */
  142. int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
  143. void *data, void (*func)(void *data, int db_num))
  144. {
  145. unsigned long mask;
  146. if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) {
  147. dev_warn(&ndev->pdev->dev, "Invalid Index.\n");
  148. return -EINVAL;
  149. }
  150. ndev->db_cb[idx].callback = func;
  151. ndev->db_cb[idx].data = data;
  152. /* unmask interrupt */
  153. mask = readw(ndev->reg_ofs.ldb_mask);
  154. clear_bit(idx * ndev->bits_per_vector, &mask);
  155. writew(mask, ndev->reg_ofs.ldb_mask);
  156. return 0;
  157. }
  158. /**
  159. * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
  160. * @ndev: pointer to ntb_device instance
  161. * @idx: doorbell index to register callback, zero based
  162. *
  163. * This function unregisters a callback function for the doorbell interrupt
  164. * on the primary side. The function will also mask the said doorbell.
  165. */
  166. void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
  167. {
  168. unsigned long mask;
  169. if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback)
  170. return;
  171. mask = readw(ndev->reg_ofs.ldb_mask);
  172. set_bit(idx * ndev->bits_per_vector, &mask);
  173. writew(mask, ndev->reg_ofs.ldb_mask);
  174. ndev->db_cb[idx].callback = NULL;
  175. }
  176. /**
  177. * ntb_find_transport() - find the transport pointer
  178. * @transport: pointer to pci device
  179. *
  180. * Given the pci device pointer, return the transport pointer passed in when
  181. * the transport attached when it was inited.
  182. *
  183. * RETURNS: pointer to transport.
  184. */
  185. void *ntb_find_transport(struct pci_dev *pdev)
  186. {
  187. struct ntb_device *ndev = pci_get_drvdata(pdev);
  188. return ndev->ntb_transport;
  189. }
  190. /**
  191. * ntb_register_transport() - Register NTB transport with NTB HW driver
  192. * @transport: transport identifier
  193. *
  194. * This function allows a transport to reserve the hardware driver for
  195. * NTB usage.
  196. *
  197. * RETURNS: pointer to ntb_device, NULL on error.
  198. */
  199. struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport)
  200. {
  201. struct ntb_device *ndev = pci_get_drvdata(pdev);
  202. if (ndev->ntb_transport)
  203. return NULL;
  204. ndev->ntb_transport = transport;
  205. return ndev;
  206. }
  207. /**
  208. * ntb_unregister_transport() - Unregister the transport with the NTB HW driver
  209. * @ndev - ntb_device of the transport to be freed
  210. *
  211. * This function unregisters the transport from the HW driver and performs any
  212. * necessary cleanups.
  213. */
  214. void ntb_unregister_transport(struct ntb_device *ndev)
  215. {
  216. int i;
  217. if (!ndev->ntb_transport)
  218. return;
  219. for (i = 0; i < ndev->max_cbs; i++)
  220. ntb_unregister_db_callback(ndev, i);
  221. ntb_unregister_event_callback(ndev);
  222. ndev->ntb_transport = NULL;
  223. }
  224. /**
  225. * ntb_write_local_spad() - write to the secondary scratchpad register
  226. * @ndev: pointer to ntb_device instance
  227. * @idx: index to the scratchpad register, 0 based
  228. * @val: the data value to put into the register
  229. *
  230. * This function allows writing of a 32bit value to the indexed scratchpad
  231. * register. This writes over the data mirrored to the local scratchpad register
  232. * by the remote system.
  233. *
  234. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  235. */
  236. int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
  237. {
  238. if (idx >= ndev->limits.max_spads)
  239. return -EINVAL;
  240. dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n",
  241. val, idx);
  242. writel(val, ndev->reg_ofs.spad_read + idx * 4);
  243. return 0;
  244. }
  245. /**
  246. * ntb_read_local_spad() - read from the primary scratchpad register
  247. * @ndev: pointer to ntb_device instance
  248. * @idx: index to scratchpad register, 0 based
  249. * @val: pointer to 32bit integer for storing the register value
  250. *
  251. * This function allows reading of the 32bit scratchpad register on
  252. * the primary (internal) side. This allows the local system to read data
  253. * written and mirrored to the scratchpad register by the remote system.
  254. *
  255. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  256. */
  257. int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
  258. {
  259. if (idx >= ndev->limits.max_spads)
  260. return -EINVAL;
  261. *val = readl(ndev->reg_ofs.spad_write + idx * 4);
  262. dev_dbg(&ndev->pdev->dev,
  263. "Reading %x from local scratch pad index %d\n", *val, idx);
  264. return 0;
  265. }
  266. /**
  267. * ntb_write_remote_spad() - write to the secondary scratchpad register
  268. * @ndev: pointer to ntb_device instance
  269. * @idx: index to the scratchpad register, 0 based
  270. * @val: the data value to put into the register
  271. *
  272. * This function allows writing of a 32bit value to the indexed scratchpad
  273. * register. The register resides on the secondary (external) side. This allows
  274. * the local system to write data to be mirrored to the remote systems
  275. * scratchpad register.
  276. *
  277. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  278. */
  279. int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
  280. {
  281. if (idx >= ndev->limits.max_spads)
  282. return -EINVAL;
  283. dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n",
  284. val, idx);
  285. writel(val, ndev->reg_ofs.spad_write + idx * 4);
  286. return 0;
  287. }
  288. /**
  289. * ntb_read_remote_spad() - read from the primary scratchpad register
  290. * @ndev: pointer to ntb_device instance
  291. * @idx: index to scratchpad register, 0 based
  292. * @val: pointer to 32bit integer for storing the register value
  293. *
  294. * This function allows reading of the 32bit scratchpad register on
  295. * the primary (internal) side. This alloows the local system to read the data
  296. * it wrote to be mirrored on the remote system.
  297. *
  298. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  299. */
  300. int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
  301. {
  302. if (idx >= ndev->limits.max_spads)
  303. return -EINVAL;
  304. *val = readl(ndev->reg_ofs.spad_read + idx * 4);
  305. dev_dbg(&ndev->pdev->dev,
  306. "Reading %x from remote scratch pad index %d\n", *val, idx);
  307. return 0;
  308. }
  309. /**
  310. * ntb_get_mw_base() - get addr for the NTB memory window
  311. * @ndev: pointer to ntb_device instance
  312. * @mw: memory window number
  313. *
  314. * This function provides the base address of the memory window specified.
  315. *
  316. * RETURNS: address, or NULL on error.
  317. */
  318. resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw)
  319. {
  320. if (mw >= ntb_max_mw(ndev))
  321. return 0;
  322. return pci_resource_start(ndev->pdev, MW_TO_BAR(mw));
  323. }
  324. /**
  325. * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
  326. * @ndev: pointer to ntb_device instance
  327. * @mw: memory window number
  328. *
  329. * This function provides the base virtual address of the memory window
  330. * specified.
  331. *
  332. * RETURNS: pointer to virtual address, or NULL on error.
  333. */
  334. void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)
  335. {
  336. if (mw >= ntb_max_mw(ndev))
  337. return NULL;
  338. return ndev->mw[mw].vbase;
  339. }
  340. /**
  341. * ntb_get_mw_size() - return size of NTB memory window
  342. * @ndev: pointer to ntb_device instance
  343. * @mw: memory window number
  344. *
  345. * This function provides the physical size of the memory window specified
  346. *
  347. * RETURNS: the size of the memory window or zero on error
  348. */
  349. u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)
  350. {
  351. if (mw >= ntb_max_mw(ndev))
  352. return 0;
  353. return ndev->mw[mw].bar_sz;
  354. }
  355. /**
  356. * ntb_set_mw_addr - set the memory window address
  357. * @ndev: pointer to ntb_device instance
  358. * @mw: memory window number
  359. * @addr: base address for data
  360. *
  361. * This function sets the base physical address of the memory window. This
  362. * memory address is where data from the remote system will be transfered into
  363. * or out of depending on how the transport is configured.
  364. */
  365. void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr)
  366. {
  367. if (mw >= ntb_max_mw(ndev))
  368. return;
  369. dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr,
  370. MW_TO_BAR(mw));
  371. ndev->mw[mw].phys_addr = addr;
  372. switch (MW_TO_BAR(mw)) {
  373. case NTB_BAR_23:
  374. writeq(addr, ndev->reg_ofs.bar2_xlat);
  375. break;
  376. case NTB_BAR_45:
  377. writeq(addr, ndev->reg_ofs.bar4_xlat);
  378. break;
  379. }
  380. }
  381. /**
  382. * ntb_ring_doorbell() - Set the doorbell on the secondary/external side
  383. * @ndev: pointer to ntb_device instance
  384. * @db: doorbell to ring
  385. *
  386. * This function allows triggering of a doorbell on the secondary/external
  387. * side that will initiate an interrupt on the remote host
  388. *
  389. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  390. */
  391. void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db)
  392. {
  393. dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db);
  394. if (ndev->hw_type == BWD_HW)
  395. writeq((u64) 1 << db, ndev->reg_ofs.rdb);
  396. else
  397. writew(((1 << ndev->bits_per_vector) - 1) <<
  398. (db * ndev->bits_per_vector), ndev->reg_ofs.rdb);
  399. }
  400. static void bwd_recover_link(struct ntb_device *ndev)
  401. {
  402. u32 status;
  403. /* Driver resets the NTB ModPhy lanes - magic! */
  404. writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6);
  405. writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4);
  406. writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4);
  407. writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6);
  408. /* Driver waits 100ms to allow the NTB ModPhy to settle */
  409. msleep(100);
  410. /* Clear AER Errors, write to clear */
  411. status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET);
  412. dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status);
  413. status &= PCI_ERR_COR_REP_ROLL;
  414. writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET);
  415. /* Clear unexpected electrical idle event in LTSSM, write to clear */
  416. status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
  417. dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status);
  418. status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
  419. writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
  420. /* Clear DeSkew Buffer error, write to clear */
  421. status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET);
  422. dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status);
  423. status |= BWD_DESKEWSTS_DBERR;
  424. writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET);
  425. status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
  426. dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status);
  427. status &= BWD_IBIST_ERR_OFLOW;
  428. writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
  429. /* Releases the NTB state machine to allow the link to retrain */
  430. status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
  431. dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status);
  432. status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT;
  433. writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
  434. }
  435. static void ntb_link_event(struct ntb_device *ndev, int link_state)
  436. {
  437. unsigned int event;
  438. if (ndev->link_status == link_state)
  439. return;
  440. if (link_state == NTB_LINK_UP) {
  441. u16 status;
  442. dev_info(&ndev->pdev->dev, "Link Up\n");
  443. ndev->link_status = NTB_LINK_UP;
  444. event = NTB_EVENT_HW_LINK_UP;
  445. if (ndev->hw_type == BWD_HW ||
  446. ndev->conn_type == NTB_CONN_TRANSPARENT)
  447. status = readw(ndev->reg_ofs.lnk_stat);
  448. else {
  449. int rc = pci_read_config_word(ndev->pdev,
  450. SNB_LINK_STATUS_OFFSET,
  451. &status);
  452. if (rc)
  453. return;
  454. }
  455. ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
  456. ndev->link_speed = (status & NTB_LINK_SPEED_MASK);
  457. dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n",
  458. ndev->link_width, ndev->link_speed);
  459. } else {
  460. dev_info(&ndev->pdev->dev, "Link Down\n");
  461. ndev->link_status = NTB_LINK_DOWN;
  462. event = NTB_EVENT_HW_LINK_DOWN;
  463. /* Don't modify link width/speed, we need it in link recovery */
  464. }
  465. /* notify the upper layer if we have an event change */
  466. if (ndev->event_cb)
  467. ndev->event_cb(ndev->ntb_transport, event);
  468. }
  469. static int ntb_link_status(struct ntb_device *ndev)
  470. {
  471. int link_state;
  472. if (ndev->hw_type == BWD_HW) {
  473. u32 ntb_cntl;
  474. ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
  475. if (ntb_cntl & BWD_CNTL_LINK_DOWN)
  476. link_state = NTB_LINK_DOWN;
  477. else
  478. link_state = NTB_LINK_UP;
  479. } else {
  480. u16 status;
  481. int rc;
  482. rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
  483. &status);
  484. if (rc)
  485. return rc;
  486. if (status & NTB_LINK_STATUS_ACTIVE)
  487. link_state = NTB_LINK_UP;
  488. else
  489. link_state = NTB_LINK_DOWN;
  490. }
  491. ntb_link_event(ndev, link_state);
  492. return 0;
  493. }
  494. static void bwd_link_recovery(struct work_struct *work)
  495. {
  496. struct ntb_device *ndev = container_of(work, struct ntb_device,
  497. lr_timer.work);
  498. u32 status32;
  499. bwd_recover_link(ndev);
  500. /* There is a potential race between the 2 NTB devices recovering at the
  501. * same time. If the times are the same, the link will not recover and
  502. * the driver will be stuck in this loop forever. Add a random interval
  503. * to the recovery time to prevent this race.
  504. */
  505. msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME);
  506. status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
  507. if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT)
  508. goto retry;
  509. status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
  510. if (status32 & BWD_IBIST_ERR_OFLOW)
  511. goto retry;
  512. status32 = readl(ndev->reg_ofs.lnk_cntl);
  513. if (!(status32 & BWD_CNTL_LINK_DOWN)) {
  514. unsigned char speed, width;
  515. u16 status16;
  516. status16 = readw(ndev->reg_ofs.lnk_stat);
  517. width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
  518. speed = (status16 & NTB_LINK_SPEED_MASK);
  519. if (ndev->link_width != width || ndev->link_speed != speed)
  520. goto retry;
  521. }
  522. schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
  523. return;
  524. retry:
  525. schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT);
  526. }
  527. /* BWD doesn't have link status interrupt, poll on that platform */
  528. static void bwd_link_poll(struct work_struct *work)
  529. {
  530. struct ntb_device *ndev = container_of(work, struct ntb_device,
  531. hb_timer.work);
  532. unsigned long ts = jiffies;
  533. /* If we haven't gotten an interrupt in a while, check the BWD link
  534. * status bit
  535. */
  536. if (ts > ndev->last_ts + NTB_HB_TIMEOUT) {
  537. int rc = ntb_link_status(ndev);
  538. if (rc)
  539. dev_err(&ndev->pdev->dev,
  540. "Error determining link status\n");
  541. /* Check to see if a link error is the cause of the link down */
  542. if (ndev->link_status == NTB_LINK_DOWN) {
  543. u32 status32 = readl(ndev->reg_base +
  544. BWD_LTSSMSTATEJMP_OFFSET);
  545. if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) {
  546. schedule_delayed_work(&ndev->lr_timer, 0);
  547. return;
  548. }
  549. }
  550. }
  551. schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
  552. }
  553. static int ntb_xeon_setup(struct ntb_device *ndev)
  554. {
  555. int rc;
  556. u8 val;
  557. ndev->hw_type = SNB_HW;
  558. rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &val);
  559. if (rc)
  560. return rc;
  561. if (val & SNB_PPD_DEV_TYPE)
  562. ndev->dev_type = NTB_DEV_USD;
  563. else
  564. ndev->dev_type = NTB_DEV_DSD;
  565. switch (val & SNB_PPD_CONN_TYPE) {
  566. case NTB_CONN_B2B:
  567. dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
  568. ndev->conn_type = NTB_CONN_B2B;
  569. ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
  570. ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
  571. ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
  572. ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
  573. ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
  574. ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
  575. /* There is a Xeon hardware errata related to writes to
  576. * SDOORBELL or B2BDOORBELL in conjunction with inbound access
  577. * to NTB MMIO Space, which may hang the system. To workaround
  578. * this use the second memory window to access the interrupt and
  579. * scratch pad registers on the remote system.
  580. */
  581. if (xeon_errata_workaround) {
  582. if (!ndev->mw[1].bar_sz)
  583. return -EINVAL;
  584. ndev->limits.max_mw = SNB_ERRATA_MAX_MW;
  585. ndev->reg_ofs.spad_write = ndev->mw[1].vbase +
  586. SNB_SPAD_OFFSET;
  587. ndev->reg_ofs.rdb = ndev->mw[1].vbase +
  588. SNB_PDOORBELL_OFFSET;
  589. /* Set the Limit register to 4k, the minimum size, to
  590. * prevent an illegal access
  591. */
  592. writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +
  593. SNB_PBAR4LMT_OFFSET);
  594. } else {
  595. ndev->limits.max_mw = SNB_MAX_MW;
  596. ndev->reg_ofs.spad_write = ndev->reg_base +
  597. SNB_B2B_SPAD_OFFSET;
  598. ndev->reg_ofs.rdb = ndev->reg_base +
  599. SNB_B2B_DOORBELL_OFFSET;
  600. /* Disable the Limit register, just incase it is set to
  601. * something silly
  602. */
  603. writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);
  604. }
  605. /* The Xeon errata workaround requires setting SBAR Base
  606. * addresses to known values, so that the PBAR XLAT can be
  607. * pointed at SBAR0 of the remote system.
  608. */
  609. if (ndev->dev_type == NTB_DEV_USD) {
  610. writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
  611. SNB_PBAR2XLAT_OFFSET);
  612. if (xeon_errata_workaround)
  613. writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
  614. SNB_PBAR4XLAT_OFFSET);
  615. else {
  616. writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
  617. SNB_PBAR4XLAT_OFFSET);
  618. /* B2B_XLAT_OFFSET is a 64bit register, but can
  619. * only take 32bit writes
  620. */
  621. writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
  622. ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
  623. writel(SNB_MBAR01_DSD_ADDR >> 32,
  624. ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
  625. }
  626. writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
  627. SNB_SBAR0BASE_OFFSET);
  628. writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
  629. SNB_SBAR2BASE_OFFSET);
  630. writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base +
  631. SNB_SBAR4BASE_OFFSET);
  632. } else {
  633. writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
  634. SNB_PBAR2XLAT_OFFSET);
  635. if (xeon_errata_workaround)
  636. writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
  637. SNB_PBAR4XLAT_OFFSET);
  638. else {
  639. writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base +
  640. SNB_PBAR4XLAT_OFFSET);
  641. /* B2B_XLAT_OFFSET is a 64bit register, but can
  642. * only take 32bit writes
  643. */
  644. writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
  645. ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
  646. writel(SNB_MBAR01_USD_ADDR >> 32,
  647. ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
  648. }
  649. writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
  650. SNB_SBAR0BASE_OFFSET);
  651. writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
  652. SNB_SBAR2BASE_OFFSET);
  653. writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
  654. SNB_SBAR4BASE_OFFSET);
  655. }
  656. break;
  657. case NTB_CONN_RP:
  658. dev_info(&ndev->pdev->dev, "Conn Type = RP\n");
  659. ndev->conn_type = NTB_CONN_RP;
  660. if (xeon_errata_workaround) {
  661. dev_err(&ndev->pdev->dev,
  662. "NTB-RP disabled due to hardware errata. To disregard this warning and potentially lock-up the system, add the parameter 'xeon_errata_workaround=0'.\n");
  663. return -EINVAL;
  664. }
  665. /* Scratch pads need to have exclusive access from the primary
  666. * or secondary side. Halve the num spads so that each side can
  667. * have an equal amount.
  668. */
  669. ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
  670. /* Note: The SDOORBELL is the cause of the errata. You REALLY
  671. * don't want to touch it.
  672. */
  673. ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
  674. ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
  675. ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
  676. /* Offset the start of the spads to correspond to whether it is
  677. * primary or secondary
  678. */
  679. ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +
  680. ndev->limits.max_spads * 4;
  681. ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
  682. ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
  683. ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
  684. ndev->limits.max_mw = SNB_MAX_MW;
  685. break;
  686. case NTB_CONN_TRANSPARENT:
  687. dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");
  688. ndev->conn_type = NTB_CONN_TRANSPARENT;
  689. /* Scratch pads need to have exclusive access from the primary
  690. * or secondary side. Halve the num spads so that each side can
  691. * have an equal amount.
  692. */
  693. ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
  694. ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
  695. ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
  696. ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
  697. ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
  698. /* Offset the start of the spads to correspond to whether it is
  699. * primary or secondary
  700. */
  701. ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +
  702. ndev->limits.max_spads * 4;
  703. ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;
  704. ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;
  705. ndev->limits.max_mw = SNB_MAX_MW;
  706. break;
  707. default:
  708. /* Most likely caused by the remote NTB-RP device not being
  709. * configured
  710. */
  711. dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", val);
  712. return -EINVAL;
  713. }
  714. ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;
  715. ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;
  716. ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
  717. ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
  718. ndev->limits.msix_cnt = SNB_MSIX_CNT;
  719. ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
  720. return 0;
  721. }
  722. static int ntb_bwd_setup(struct ntb_device *ndev)
  723. {
  724. int rc;
  725. u32 val;
  726. ndev->hw_type = BWD_HW;
  727. rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val);
  728. if (rc)
  729. return rc;
  730. switch ((val & BWD_PPD_CONN_TYPE) >> 8) {
  731. case NTB_CONN_B2B:
  732. ndev->conn_type = NTB_CONN_B2B;
  733. break;
  734. case NTB_CONN_RP:
  735. default:
  736. dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
  737. return -EINVAL;
  738. }
  739. if (val & BWD_PPD_DEV_TYPE)
  740. ndev->dev_type = NTB_DEV_DSD;
  741. else
  742. ndev->dev_type = NTB_DEV_USD;
  743. /* Initiate PCI-E link training */
  744. rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET,
  745. val | BWD_PPD_INIT_LINK);
  746. if (rc)
  747. return rc;
  748. ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET;
  749. ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET;
  750. ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET;
  751. ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET;
  752. ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET;
  753. ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET;
  754. ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET;
  755. ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET;
  756. ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET;
  757. ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET;
  758. ndev->limits.max_mw = BWD_MAX_MW;
  759. ndev->limits.max_spads = BWD_MAX_SPADS;
  760. ndev->limits.max_db_bits = BWD_MAX_DB_BITS;
  761. ndev->limits.msix_cnt = BWD_MSIX_CNT;
  762. ndev->bits_per_vector = BWD_DB_BITS_PER_VEC;
  763. /* Since bwd doesn't have a link interrupt, setup a poll timer */
  764. INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll);
  765. INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery);
  766. schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
  767. return 0;
  768. }
  769. static int ntb_device_setup(struct ntb_device *ndev)
  770. {
  771. int rc;
  772. switch (ndev->pdev->device) {
  773. case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
  774. case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
  775. case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
  776. case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
  777. case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
  778. case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
  779. case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
  780. case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
  781. case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
  782. case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
  783. case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
  784. case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
  785. rc = ntb_xeon_setup(ndev);
  786. break;
  787. case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
  788. rc = ntb_bwd_setup(ndev);
  789. break;
  790. default:
  791. rc = -ENODEV;
  792. }
  793. if (rc)
  794. return rc;
  795. dev_info(&ndev->pdev->dev, "Device Type = %s\n",
  796. ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");
  797. if (ndev->conn_type == NTB_CONN_B2B)
  798. /* Enable Bus Master and Memory Space on the secondary side */
  799. writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
  800. ndev->reg_ofs.spci_cmd);
  801. return 0;
  802. }
  803. static void ntb_device_free(struct ntb_device *ndev)
  804. {
  805. if (ndev->hw_type == BWD_HW) {
  806. cancel_delayed_work_sync(&ndev->hb_timer);
  807. cancel_delayed_work_sync(&ndev->lr_timer);
  808. }
  809. }
  810. static irqreturn_t bwd_callback_msix_irq(int irq, void *data)
  811. {
  812. struct ntb_db_cb *db_cb = data;
  813. struct ntb_device *ndev = db_cb->ndev;
  814. dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
  815. db_cb->db_num);
  816. if (db_cb->callback)
  817. db_cb->callback(db_cb->data, db_cb->db_num);
  818. /* No need to check for the specific HB irq, any interrupt means
  819. * we're connected.
  820. */
  821. ndev->last_ts = jiffies;
  822. writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb);
  823. return IRQ_HANDLED;
  824. }
  825. static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
  826. {
  827. struct ntb_db_cb *db_cb = data;
  828. struct ntb_device *ndev = db_cb->ndev;
  829. dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
  830. db_cb->db_num);
  831. if (db_cb->callback)
  832. db_cb->callback(db_cb->data, db_cb->db_num);
  833. /* On Sandybridge, there are 16 bits in the interrupt register
  834. * but only 4 vectors. So, 5 bits are assigned to the first 3
  835. * vectors, with the 4th having a single bit for link
  836. * interrupts.
  837. */
  838. writew(((1 << ndev->bits_per_vector) - 1) <<
  839. (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb);
  840. return IRQ_HANDLED;
  841. }
  842. /* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */
  843. static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
  844. {
  845. struct ntb_device *ndev = dev;
  846. int rc;
  847. dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);
  848. rc = ntb_link_status(ndev);
  849. if (rc)
  850. dev_err(&ndev->pdev->dev, "Error determining link status\n");
  851. /* bit 15 is always the link bit */
  852. writew(1 << ndev->limits.max_db_bits, ndev->reg_ofs.ldb);
  853. return IRQ_HANDLED;
  854. }
  855. static irqreturn_t ntb_interrupt(int irq, void *dev)
  856. {
  857. struct ntb_device *ndev = dev;
  858. unsigned int i = 0;
  859. if (ndev->hw_type == BWD_HW) {
  860. u64 ldb = readq(ndev->reg_ofs.ldb);
  861. dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb);
  862. while (ldb) {
  863. i = __ffs(ldb);
  864. ldb &= ldb - 1;
  865. bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
  866. }
  867. } else {
  868. u16 ldb = readw(ndev->reg_ofs.ldb);
  869. dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb);
  870. if (ldb & SNB_DB_HW_LINK) {
  871. xeon_event_msix_irq(irq, dev);
  872. ldb &= ~SNB_DB_HW_LINK;
  873. }
  874. while (ldb) {
  875. i = __ffs(ldb);
  876. ldb &= ldb - 1;
  877. xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
  878. }
  879. }
  880. return IRQ_HANDLED;
  881. }
  882. static int ntb_setup_msix(struct ntb_device *ndev)
  883. {
  884. struct pci_dev *pdev = ndev->pdev;
  885. struct msix_entry *msix;
  886. int msix_entries;
  887. int rc, i, pos;
  888. u16 val;
  889. pos = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
  890. if (!pos) {
  891. rc = -EIO;
  892. goto err;
  893. }
  894. rc = pci_read_config_word(pdev, pos + PCI_MSIX_FLAGS, &val);
  895. if (rc)
  896. goto err;
  897. msix_entries = msix_table_size(val);
  898. if (msix_entries > ndev->limits.msix_cnt) {
  899. rc = -EINVAL;
  900. goto err;
  901. }
  902. ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries,
  903. GFP_KERNEL);
  904. if (!ndev->msix_entries) {
  905. rc = -ENOMEM;
  906. goto err;
  907. }
  908. for (i = 0; i < msix_entries; i++)
  909. ndev->msix_entries[i].entry = i;
  910. rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries);
  911. if (rc < 0)
  912. goto err1;
  913. if (rc > 0) {
  914. /* On SNB, the link interrupt is always tied to 4th vector. If
  915. * we can't get all 4, then we can't use MSI-X.
  916. */
  917. if (ndev->hw_type != BWD_HW) {
  918. rc = -EIO;
  919. goto err1;
  920. }
  921. dev_warn(&pdev->dev,
  922. "Only %d MSI-X vectors. Limiting the number of queues to that number.\n",
  923. rc);
  924. msix_entries = rc;
  925. }
  926. for (i = 0; i < msix_entries; i++) {
  927. msix = &ndev->msix_entries[i];
  928. WARN_ON(!msix->vector);
  929. /* Use the last MSI-X vector for Link status */
  930. if (ndev->hw_type == BWD_HW) {
  931. rc = request_irq(msix->vector, bwd_callback_msix_irq, 0,
  932. "ntb-callback-msix", &ndev->db_cb[i]);
  933. if (rc)
  934. goto err2;
  935. } else {
  936. if (i == msix_entries - 1) {
  937. rc = request_irq(msix->vector,
  938. xeon_event_msix_irq, 0,
  939. "ntb-event-msix", ndev);
  940. if (rc)
  941. goto err2;
  942. } else {
  943. rc = request_irq(msix->vector,
  944. xeon_callback_msix_irq, 0,
  945. "ntb-callback-msix",
  946. &ndev->db_cb[i]);
  947. if (rc)
  948. goto err2;
  949. }
  950. }
  951. }
  952. ndev->num_msix = msix_entries;
  953. if (ndev->hw_type == BWD_HW)
  954. ndev->max_cbs = msix_entries;
  955. else
  956. ndev->max_cbs = msix_entries - 1;
  957. return 0;
  958. err2:
  959. while (--i >= 0) {
  960. msix = &ndev->msix_entries[i];
  961. if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
  962. free_irq(msix->vector, ndev);
  963. else
  964. free_irq(msix->vector, &ndev->db_cb[i]);
  965. }
  966. pci_disable_msix(pdev);
  967. err1:
  968. kfree(ndev->msix_entries);
  969. dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n");
  970. err:
  971. ndev->num_msix = 0;
  972. return rc;
  973. }
  974. static int ntb_setup_msi(struct ntb_device *ndev)
  975. {
  976. struct pci_dev *pdev = ndev->pdev;
  977. int rc;
  978. rc = pci_enable_msi(pdev);
  979. if (rc)
  980. return rc;
  981. rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev);
  982. if (rc) {
  983. pci_disable_msi(pdev);
  984. dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
  985. return rc;
  986. }
  987. return 0;
  988. }
  989. static int ntb_setup_intx(struct ntb_device *ndev)
  990. {
  991. struct pci_dev *pdev = ndev->pdev;
  992. int rc;
  993. pci_msi_off(pdev);
  994. /* Verify intx is enabled */
  995. pci_intx(pdev, 1);
  996. rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx",
  997. ndev);
  998. if (rc)
  999. return rc;
  1000. return 0;
  1001. }
  1002. static int ntb_setup_interrupts(struct ntb_device *ndev)
  1003. {
  1004. int rc;
  1005. /* On BWD, disable all interrupts. On SNB, disable all but Link
  1006. * Interrupt. The rest will be unmasked as callbacks are registered.
  1007. */
  1008. if (ndev->hw_type == BWD_HW)
  1009. writeq(~0, ndev->reg_ofs.ldb_mask);
  1010. else
  1011. writew(~(1 << ndev->limits.max_db_bits),
  1012. ndev->reg_ofs.ldb_mask);
  1013. rc = ntb_setup_msix(ndev);
  1014. if (!rc)
  1015. goto done;
  1016. ndev->bits_per_vector = 1;
  1017. ndev->max_cbs = ndev->limits.max_db_bits;
  1018. rc = ntb_setup_msi(ndev);
  1019. if (!rc)
  1020. goto done;
  1021. rc = ntb_setup_intx(ndev);
  1022. if (rc) {
  1023. dev_err(&ndev->pdev->dev, "no usable interrupts\n");
  1024. return rc;
  1025. }
  1026. done:
  1027. return 0;
  1028. }
  1029. static void ntb_free_interrupts(struct ntb_device *ndev)
  1030. {
  1031. struct pci_dev *pdev = ndev->pdev;
  1032. /* mask interrupts */
  1033. if (ndev->hw_type == BWD_HW)
  1034. writeq(~0, ndev->reg_ofs.ldb_mask);
  1035. else
  1036. writew(~0, ndev->reg_ofs.ldb_mask);
  1037. if (ndev->num_msix) {
  1038. struct msix_entry *msix;
  1039. u32 i;
  1040. for (i = 0; i < ndev->num_msix; i++) {
  1041. msix = &ndev->msix_entries[i];
  1042. if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
  1043. free_irq(msix->vector, ndev);
  1044. else
  1045. free_irq(msix->vector, &ndev->db_cb[i]);
  1046. }
  1047. pci_disable_msix(pdev);
  1048. } else {
  1049. free_irq(pdev->irq, ndev);
  1050. if (pci_dev_msi_enabled(pdev))
  1051. pci_disable_msi(pdev);
  1052. }
  1053. }
  1054. static int ntb_create_callbacks(struct ntb_device *ndev)
  1055. {
  1056. int i;
  1057. /* Chicken-egg issue. We won't know how many callbacks are necessary
  1058. * until we see how many MSI-X vectors we get, but these pointers need
  1059. * to be passed into the MSI-X register function. So, we allocate the
  1060. * max, knowing that they might not all be used, to work around this.
  1061. */
  1062. ndev->db_cb = kcalloc(ndev->limits.max_db_bits,
  1063. sizeof(struct ntb_db_cb),
  1064. GFP_KERNEL);
  1065. if (!ndev->db_cb)
  1066. return -ENOMEM;
  1067. for (i = 0; i < ndev->limits.max_db_bits; i++) {
  1068. ndev->db_cb[i].db_num = i;
  1069. ndev->db_cb[i].ndev = ndev;
  1070. }
  1071. return 0;
  1072. }
  1073. static void ntb_free_callbacks(struct ntb_device *ndev)
  1074. {
  1075. int i;
  1076. for (i = 0; i < ndev->limits.max_db_bits; i++)
  1077. ntb_unregister_db_callback(ndev, i);
  1078. kfree(ndev->db_cb);
  1079. }
  1080. static void ntb_setup_debugfs(struct ntb_device *ndev)
  1081. {
  1082. if (!debugfs_initialized())
  1083. return;
  1084. if (!debugfs_dir)
  1085. debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
  1086. ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev),
  1087. debugfs_dir);
  1088. }
  1089. static void ntb_free_debugfs(struct ntb_device *ndev)
  1090. {
  1091. debugfs_remove_recursive(ndev->debugfs_dir);
  1092. if (debugfs_dir && simple_empty(debugfs_dir)) {
  1093. debugfs_remove_recursive(debugfs_dir);
  1094. debugfs_dir = NULL;
  1095. }
  1096. }
  1097. static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
  1098. {
  1099. struct ntb_device *ndev;
  1100. int rc, i;
  1101. ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL);
  1102. if (!ndev)
  1103. return -ENOMEM;
  1104. ndev->pdev = pdev;
  1105. ndev->link_status = NTB_LINK_DOWN;
  1106. pci_set_drvdata(pdev, ndev);
  1107. ntb_setup_debugfs(ndev);
  1108. rc = pci_enable_device(pdev);
  1109. if (rc)
  1110. goto err;
  1111. pci_set_master(ndev->pdev);
  1112. rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, KBUILD_MODNAME);
  1113. if (rc)
  1114. goto err1;
  1115. ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO);
  1116. if (!ndev->reg_base) {
  1117. dev_warn(&pdev->dev, "Cannot remap BAR 0\n");
  1118. rc = -EIO;
  1119. goto err2;
  1120. }
  1121. for (i = 0; i < NTB_MAX_NUM_MW; i++) {
  1122. ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
  1123. ndev->mw[i].vbase =
  1124. ioremap_wc(pci_resource_start(pdev, MW_TO_BAR(i)),
  1125. ndev->mw[i].bar_sz);
  1126. dev_info(&pdev->dev, "MW %d size %llu\n", i,
  1127. (unsigned long long) ndev->mw[i].bar_sz);
  1128. if (!ndev->mw[i].vbase) {
  1129. dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
  1130. MW_TO_BAR(i));
  1131. rc = -EIO;
  1132. goto err3;
  1133. }
  1134. }
  1135. rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
  1136. if (rc) {
  1137. rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
  1138. if (rc)
  1139. goto err3;
  1140. dev_warn(&pdev->dev, "Cannot DMA highmem\n");
  1141. }
  1142. rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
  1143. if (rc) {
  1144. rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
  1145. if (rc)
  1146. goto err3;
  1147. dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
  1148. }
  1149. rc = ntb_device_setup(ndev);
  1150. if (rc)
  1151. goto err3;
  1152. rc = ntb_create_callbacks(ndev);
  1153. if (rc)
  1154. goto err4;
  1155. rc = ntb_setup_interrupts(ndev);
  1156. if (rc)
  1157. goto err5;
  1158. /* The scratchpad registers keep the values between rmmod/insmod,
  1159. * blast them now
  1160. */
  1161. for (i = 0; i < ndev->limits.max_spads; i++) {
  1162. ntb_write_local_spad(ndev, i, 0);
  1163. ntb_write_remote_spad(ndev, i, 0);
  1164. }
  1165. rc = ntb_transport_init(pdev);
  1166. if (rc)
  1167. goto err6;
  1168. /* Let's bring the NTB link up */
  1169. writel(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP,
  1170. ndev->reg_ofs.lnk_cntl);
  1171. return 0;
  1172. err6:
  1173. ntb_free_interrupts(ndev);
  1174. err5:
  1175. ntb_free_callbacks(ndev);
  1176. err4:
  1177. ntb_device_free(ndev);
  1178. err3:
  1179. for (i--; i >= 0; i--)
  1180. iounmap(ndev->mw[i].vbase);
  1181. iounmap(ndev->reg_base);
  1182. err2:
  1183. pci_release_selected_regions(pdev, NTB_BAR_MASK);
  1184. err1:
  1185. pci_disable_device(pdev);
  1186. err:
  1187. ntb_free_debugfs(ndev);
  1188. kfree(ndev);
  1189. dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
  1190. return rc;
  1191. }
  1192. static void ntb_pci_remove(struct pci_dev *pdev)
  1193. {
  1194. struct ntb_device *ndev = pci_get_drvdata(pdev);
  1195. int i;
  1196. u32 ntb_cntl;
  1197. /* Bring NTB link down */
  1198. ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
  1199. ntb_cntl |= NTB_CNTL_LINK_DISABLE;
  1200. writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
  1201. ntb_transport_free(ndev->ntb_transport);
  1202. ntb_free_interrupts(ndev);
  1203. ntb_free_callbacks(ndev);
  1204. ntb_device_free(ndev);
  1205. for (i = 0; i < NTB_MAX_NUM_MW; i++)
  1206. iounmap(ndev->mw[i].vbase);
  1207. iounmap(ndev->reg_base);
  1208. pci_release_selected_regions(pdev, NTB_BAR_MASK);
  1209. pci_disable_device(pdev);
  1210. ntb_free_debugfs(ndev);
  1211. kfree(ndev);
  1212. }
  1213. static struct pci_driver ntb_pci_driver = {
  1214. .name = KBUILD_MODNAME,
  1215. .id_table = ntb_pci_tbl,
  1216. .probe = ntb_pci_probe,
  1217. .remove = ntb_pci_remove,
  1218. };
  1219. module_pci_driver(ntb_pci_driver);