eeh_driver.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732
  1. /*
  2. * PCI Error Recovery Driver for RPA-compliant PPC64 platform.
  3. * Copyright IBM Corp. 2004 2005
  4. * Copyright Linas Vepstas <linas@linas.org> 2004, 2005
  5. *
  6. * All rights reserved.
  7. *
  8. * This program is free software; you can redistribute it and/or modify
  9. * it under the terms of the GNU General Public License as published by
  10. * the Free Software Foundation; either version 2 of the License, or (at
  11. * your option) any later version.
  12. *
  13. * This program is distributed in the hope that it will be useful, but
  14. * WITHOUT ANY WARRANTY; without even the implied warranty of
  15. * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  16. * NON INFRINGEMENT. See the GNU General Public License for more
  17. * details.
  18. *
  19. * You should have received a copy of the GNU General Public License
  20. * along with this program; if not, write to the Free Software
  21. * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22. *
  23. * Send comments and feedback to Linas Vepstas <linas@austin.ibm.com>
  24. */
  25. #include <linux/delay.h>
  26. #include <linux/interrupt.h>
  27. #include <linux/irq.h>
  28. #include <linux/module.h>
  29. #include <linux/pci.h>
  30. #include <asm/eeh.h>
  31. #include <asm/eeh_event.h>
  32. #include <asm/ppc-pci.h>
  33. #include <asm/pci-bridge.h>
  34. #include <asm/prom.h>
  35. #include <asm/rtas.h>
  36. /**
  37. * eeh_pcid_name - Retrieve name of PCI device driver
  38. * @pdev: PCI device
  39. *
  40. * This routine is used to retrieve the name of PCI device driver
  41. * if that's valid.
  42. */
  43. static inline const char *eeh_pcid_name(struct pci_dev *pdev)
  44. {
  45. if (pdev && pdev->dev.driver)
  46. return pdev->dev.driver->name;
  47. return "";
  48. }
  49. /**
  50. * eeh_pcid_get - Get the PCI device driver
  51. * @pdev: PCI device
  52. *
  53. * The function is used to retrieve the PCI device driver for
  54. * the indicated PCI device. Besides, we will increase the reference
  55. * of the PCI device driver to prevent that being unloaded on
  56. * the fly. Otherwise, kernel crash would be seen.
  57. */
  58. static inline struct pci_driver *eeh_pcid_get(struct pci_dev *pdev)
  59. {
  60. if (!pdev || !pdev->driver)
  61. return NULL;
  62. if (!try_module_get(pdev->driver->driver.owner))
  63. return NULL;
  64. return pdev->driver;
  65. }
  66. /**
  67. * eeh_pcid_put - Dereference on the PCI device driver
  68. * @pdev: PCI device
  69. *
  70. * The function is called to do dereference on the PCI device
  71. * driver of the indicated PCI device.
  72. */
  73. static inline void eeh_pcid_put(struct pci_dev *pdev)
  74. {
  75. if (!pdev || !pdev->driver)
  76. return;
  77. module_put(pdev->driver->driver.owner);
  78. }
  79. #if 0
  80. static void print_device_node_tree(struct pci_dn *pdn, int dent)
  81. {
  82. int i;
  83. struct device_node *pc;
  84. if (!pdn)
  85. return;
  86. for (i = 0; i < dent; i++)
  87. printk(" ");
  88. printk("dn=%s mode=%x \tcfg_addr=%x pe_addr=%x \tfull=%s\n",
  89. pdn->node->name, pdn->eeh_mode, pdn->eeh_config_addr,
  90. pdn->eeh_pe_config_addr, pdn->node->full_name);
  91. dent += 3;
  92. pc = pdn->node->child;
  93. while (pc) {
  94. print_device_node_tree(PCI_DN(pc), dent);
  95. pc = pc->sibling;
  96. }
  97. }
  98. #endif
  99. /**
  100. * eeh_disable_irq - Disable interrupt for the recovering device
  101. * @dev: PCI device
  102. *
  103. * This routine must be called when reporting temporary or permanent
  104. * error to the particular PCI device to disable interrupt of that
  105. * device. If the device has enabled MSI or MSI-X interrupt, we needn't
  106. * do real work because EEH should freeze DMA transfers for those PCI
  107. * devices encountering EEH errors, which includes MSI or MSI-X.
  108. */
  109. static void eeh_disable_irq(struct pci_dev *dev)
  110. {
  111. struct eeh_dev *edev = pci_dev_to_eeh_dev(dev);
  112. /* Don't disable MSI and MSI-X interrupts. They are
  113. * effectively disabled by the DMA Stopped state
  114. * when an EEH error occurs.
  115. */
  116. if (dev->msi_enabled || dev->msix_enabled)
  117. return;
  118. if (!irq_has_action(dev->irq))
  119. return;
  120. edev->mode |= EEH_DEV_IRQ_DISABLED;
  121. disable_irq_nosync(dev->irq);
  122. }
  123. /**
  124. * eeh_enable_irq - Enable interrupt for the recovering device
  125. * @dev: PCI device
  126. *
  127. * This routine must be called to enable interrupt while failed
  128. * device could be resumed.
  129. */
  130. static void eeh_enable_irq(struct pci_dev *dev)
  131. {
  132. struct eeh_dev *edev = pci_dev_to_eeh_dev(dev);
  133. struct irq_desc *desc;
  134. if ((edev->mode) & EEH_DEV_IRQ_DISABLED) {
  135. edev->mode &= ~EEH_DEV_IRQ_DISABLED;
  136. desc = irq_to_desc(dev->irq);
  137. if (desc && desc->depth > 0)
  138. enable_irq(dev->irq);
  139. }
  140. }
  141. /**
  142. * eeh_report_error - Report pci error to each device driver
  143. * @data: eeh device
  144. * @userdata: return value
  145. *
  146. * Report an EEH error to each device driver, collect up and
  147. * merge the device driver responses. Cumulative response
  148. * passed back in "userdata".
  149. */
  150. static void *eeh_report_error(void *data, void *userdata)
  151. {
  152. struct eeh_dev *edev = (struct eeh_dev *)data;
  153. struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
  154. enum pci_ers_result rc, *res = userdata;
  155. struct pci_driver *driver;
  156. /* We might not have the associated PCI device,
  157. * then we should continue for next one.
  158. */
  159. if (!dev) return NULL;
  160. dev->error_state = pci_channel_io_frozen;
  161. driver = eeh_pcid_get(dev);
  162. if (!driver) return NULL;
  163. eeh_disable_irq(dev);
  164. if (!driver->err_handler ||
  165. !driver->err_handler->error_detected) {
  166. eeh_pcid_put(dev);
  167. return NULL;
  168. }
  169. rc = driver->err_handler->error_detected(dev, pci_channel_io_frozen);
  170. /* A driver that needs a reset trumps all others */
  171. if (rc == PCI_ERS_RESULT_NEED_RESET) *res = rc;
  172. if (*res == PCI_ERS_RESULT_NONE) *res = rc;
  173. eeh_pcid_put(dev);
  174. return NULL;
  175. }
  176. /**
  177. * eeh_report_mmio_enabled - Tell drivers that MMIO has been enabled
  178. * @data: eeh device
  179. * @userdata: return value
  180. *
  181. * Tells each device driver that IO ports, MMIO and config space I/O
  182. * are now enabled. Collects up and merges the device driver responses.
  183. * Cumulative response passed back in "userdata".
  184. */
  185. static void *eeh_report_mmio_enabled(void *data, void *userdata)
  186. {
  187. struct eeh_dev *edev = (struct eeh_dev *)data;
  188. struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
  189. enum pci_ers_result rc, *res = userdata;
  190. struct pci_driver *driver;
  191. driver = eeh_pcid_get(dev);
  192. if (!driver) return NULL;
  193. if (!driver->err_handler ||
  194. !driver->err_handler->mmio_enabled) {
  195. eeh_pcid_put(dev);
  196. return NULL;
  197. }
  198. rc = driver->err_handler->mmio_enabled(dev);
  199. /* A driver that needs a reset trumps all others */
  200. if (rc == PCI_ERS_RESULT_NEED_RESET) *res = rc;
  201. if (*res == PCI_ERS_RESULT_NONE) *res = rc;
  202. eeh_pcid_put(dev);
  203. return NULL;
  204. }
  205. /**
  206. * eeh_report_reset - Tell device that slot has been reset
  207. * @data: eeh device
  208. * @userdata: return value
  209. *
  210. * This routine must be called while EEH tries to reset particular
  211. * PCI device so that the associated PCI device driver could take
  212. * some actions, usually to save data the driver needs so that the
  213. * driver can work again while the device is recovered.
  214. */
  215. static void *eeh_report_reset(void *data, void *userdata)
  216. {
  217. struct eeh_dev *edev = (struct eeh_dev *)data;
  218. struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
  219. enum pci_ers_result rc, *res = userdata;
  220. struct pci_driver *driver;
  221. if (!dev) return NULL;
  222. dev->error_state = pci_channel_io_normal;
  223. driver = eeh_pcid_get(dev);
  224. if (!driver) return NULL;
  225. eeh_enable_irq(dev);
  226. if (!driver->err_handler ||
  227. !driver->err_handler->slot_reset) {
  228. eeh_pcid_put(dev);
  229. return NULL;
  230. }
  231. rc = driver->err_handler->slot_reset(dev);
  232. if ((*res == PCI_ERS_RESULT_NONE) ||
  233. (*res == PCI_ERS_RESULT_RECOVERED)) *res = rc;
  234. if (*res == PCI_ERS_RESULT_DISCONNECT &&
  235. rc == PCI_ERS_RESULT_NEED_RESET) *res = rc;
  236. eeh_pcid_put(dev);
  237. return NULL;
  238. }
  239. /**
  240. * eeh_report_resume - Tell device to resume normal operations
  241. * @data: eeh device
  242. * @userdata: return value
  243. *
  244. * This routine must be called to notify the device driver that it
  245. * could resume so that the device driver can do some initialization
  246. * to make the recovered device work again.
  247. */
  248. static void *eeh_report_resume(void *data, void *userdata)
  249. {
  250. struct eeh_dev *edev = (struct eeh_dev *)data;
  251. struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
  252. struct pci_driver *driver;
  253. if (!dev) return NULL;
  254. dev->error_state = pci_channel_io_normal;
  255. driver = eeh_pcid_get(dev);
  256. if (!driver) return NULL;
  257. eeh_enable_irq(dev);
  258. if (!driver->err_handler ||
  259. !driver->err_handler->resume) {
  260. eeh_pcid_put(dev);
  261. return NULL;
  262. }
  263. driver->err_handler->resume(dev);
  264. eeh_pcid_put(dev);
  265. return NULL;
  266. }
  267. /**
  268. * eeh_report_failure - Tell device driver that device is dead.
  269. * @data: eeh device
  270. * @userdata: return value
  271. *
  272. * This informs the device driver that the device is permanently
  273. * dead, and that no further recovery attempts will be made on it.
  274. */
  275. static void *eeh_report_failure(void *data, void *userdata)
  276. {
  277. struct eeh_dev *edev = (struct eeh_dev *)data;
  278. struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
  279. struct pci_driver *driver;
  280. if (!dev) return NULL;
  281. dev->error_state = pci_channel_io_perm_failure;
  282. driver = eeh_pcid_get(dev);
  283. if (!driver) return NULL;
  284. eeh_disable_irq(dev);
  285. if (!driver->err_handler ||
  286. !driver->err_handler->error_detected) {
  287. eeh_pcid_put(dev);
  288. return NULL;
  289. }
  290. driver->err_handler->error_detected(dev, pci_channel_io_perm_failure);
  291. eeh_pcid_put(dev);
  292. return NULL;
  293. }
  294. static void *eeh_rmv_device(void *data, void *userdata)
  295. {
  296. struct pci_driver *driver;
  297. struct eeh_dev *edev = (struct eeh_dev *)data;
  298. struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
  299. int *removed = (int *)userdata;
  300. /*
  301. * Actually, we should remove the PCI bridges as well.
  302. * However, that's lots of complexity to do that,
  303. * particularly some of devices under the bridge might
  304. * support EEH. So we just care about PCI devices for
  305. * simplicity here.
  306. */
  307. if (!dev || (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE))
  308. return NULL;
  309. driver = eeh_pcid_get(dev);
  310. if (driver && driver->err_handler)
  311. return NULL;
  312. /* Remove it from PCI subsystem */
  313. pr_debug("EEH: Removing %s without EEH sensitive driver\n",
  314. pci_name(dev));
  315. edev->bus = dev->bus;
  316. edev->mode |= EEH_DEV_DISCONNECTED;
  317. (*removed)++;
  318. pci_stop_and_remove_bus_device(dev);
  319. return NULL;
  320. }
  321. static void *eeh_pe_detach_dev(void *data, void *userdata)
  322. {
  323. struct eeh_pe *pe = (struct eeh_pe *)data;
  324. struct eeh_dev *edev, *tmp;
  325. eeh_pe_for_each_dev(pe, edev, tmp) {
  326. if (!(edev->mode & EEH_DEV_DISCONNECTED))
  327. continue;
  328. edev->mode &= ~(EEH_DEV_DISCONNECTED | EEH_DEV_IRQ_DISABLED);
  329. eeh_rmv_from_parent_pe(edev);
  330. }
  331. return NULL;
  332. }
  333. /**
  334. * eeh_reset_device - Perform actual reset of a pci slot
  335. * @pe: EEH PE
  336. * @bus: PCI bus corresponding to the isolcated slot
  337. *
  338. * This routine must be called to do reset on the indicated PE.
  339. * During the reset, udev might be invoked because those affected
  340. * PCI devices will be removed and then added.
  341. */
  342. static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus)
  343. {
  344. struct pci_bus *frozen_bus = eeh_pe_bus_get(pe);
  345. struct timeval tstamp;
  346. int cnt, rc, removed = 0;
  347. /* pcibios will clear the counter; save the value */
  348. cnt = pe->freeze_count;
  349. tstamp = pe->tstamp;
  350. /*
  351. * We don't remove the corresponding PE instances because
  352. * we need the information afterwords. The attached EEH
  353. * devices are expected to be attached soon when calling
  354. * into pcibios_add_pci_devices().
  355. */
  356. eeh_pe_state_mark(pe, EEH_PE_KEEP);
  357. if (bus)
  358. pcibios_remove_pci_devices(bus);
  359. else if (frozen_bus)
  360. eeh_pe_dev_traverse(pe, eeh_rmv_device, &removed);
  361. /* Reset the pci controller. (Asserts RST#; resets config space).
  362. * Reconfigure bridges and devices. Don't try to bring the system
  363. * up if the reset failed for some reason.
  364. */
  365. rc = eeh_reset_pe(pe);
  366. if (rc)
  367. return rc;
  368. /* Restore PE */
  369. eeh_ops->configure_bridge(pe);
  370. eeh_pe_restore_bars(pe);
  371. /* Give the system 5 seconds to finish running the user-space
  372. * hotplug shutdown scripts, e.g. ifdown for ethernet. Yes,
  373. * this is a hack, but if we don't do this, and try to bring
  374. * the device up before the scripts have taken it down,
  375. * potentially weird things happen.
  376. */
  377. if (bus) {
  378. pr_info("EEH: Sleep 5s ahead of complete hotplug\n");
  379. ssleep(5);
  380. /*
  381. * The EEH device is still connected with its parent
  382. * PE. We should disconnect it so the binding can be
  383. * rebuilt when adding PCI devices.
  384. */
  385. eeh_pe_traverse(pe, eeh_pe_detach_dev, NULL);
  386. pcibios_add_pci_devices(bus);
  387. } else if (frozen_bus && removed) {
  388. pr_info("EEH: Sleep 5s ahead of partial hotplug\n");
  389. ssleep(5);
  390. eeh_pe_traverse(pe, eeh_pe_detach_dev, NULL);
  391. pcibios_add_pci_devices(frozen_bus);
  392. }
  393. eeh_pe_state_clear(pe, EEH_PE_KEEP);
  394. pe->tstamp = tstamp;
  395. pe->freeze_count = cnt;
  396. return 0;
  397. }
  398. /* The longest amount of time to wait for a pci device
  399. * to come back on line, in seconds.
  400. */
  401. #define MAX_WAIT_FOR_RECOVERY 150
  402. static void eeh_handle_normal_event(struct eeh_pe *pe)
  403. {
  404. struct pci_bus *frozen_bus;
  405. int rc = 0;
  406. enum pci_ers_result result = PCI_ERS_RESULT_NONE;
  407. frozen_bus = eeh_pe_bus_get(pe);
  408. if (!frozen_bus) {
  409. pr_err("%s: Cannot find PCI bus for PHB#%d-PE#%x\n",
  410. __func__, pe->phb->global_number, pe->addr);
  411. return;
  412. }
  413. eeh_pe_update_time_stamp(pe);
  414. pe->freeze_count++;
  415. if (pe->freeze_count > EEH_MAX_ALLOWED_FREEZES)
  416. goto excess_failures;
  417. pr_warning("EEH: This PCI device has failed %d times in the last hour\n",
  418. pe->freeze_count);
  419. /* Walk the various device drivers attached to this slot through
  420. * a reset sequence, giving each an opportunity to do what it needs
  421. * to accomplish the reset. Each child gets a report of the
  422. * status ... if any child can't handle the reset, then the entire
  423. * slot is dlpar removed and added.
  424. */
  425. pr_info("EEH: Notify device drivers to shutdown\n");
  426. eeh_pe_dev_traverse(pe, eeh_report_error, &result);
  427. /* Get the current PCI slot state. This can take a long time,
  428. * sometimes over 3 seconds for certain systems.
  429. */
  430. rc = eeh_ops->wait_state(pe, MAX_WAIT_FOR_RECOVERY*1000);
  431. if (rc < 0 || rc == EEH_STATE_NOT_SUPPORT) {
  432. pr_warning("EEH: Permanent failure\n");
  433. goto hard_fail;
  434. }
  435. /* Since rtas may enable MMIO when posting the error log,
  436. * don't post the error log until after all dev drivers
  437. * have been informed.
  438. */
  439. pr_info("EEH: Collect temporary log\n");
  440. eeh_slot_error_detail(pe, EEH_LOG_TEMP);
  441. /* If all device drivers were EEH-unaware, then shut
  442. * down all of the device drivers, and hope they
  443. * go down willingly, without panicing the system.
  444. */
  445. if (result == PCI_ERS_RESULT_NONE) {
  446. pr_info("EEH: Reset with hotplug activity\n");
  447. rc = eeh_reset_device(pe, frozen_bus);
  448. if (rc) {
  449. pr_warning("%s: Unable to reset, err=%d\n",
  450. __func__, rc);
  451. goto hard_fail;
  452. }
  453. }
  454. /* If all devices reported they can proceed, then re-enable MMIO */
  455. if (result == PCI_ERS_RESULT_CAN_RECOVER) {
  456. pr_info("EEH: Enable I/O for affected devices\n");
  457. rc = eeh_pci_enable(pe, EEH_OPT_THAW_MMIO);
  458. if (rc < 0)
  459. goto hard_fail;
  460. if (rc) {
  461. result = PCI_ERS_RESULT_NEED_RESET;
  462. } else {
  463. pr_info("EEH: Notify device drivers to resume I/O\n");
  464. result = PCI_ERS_RESULT_NONE;
  465. eeh_pe_dev_traverse(pe, eeh_report_mmio_enabled, &result);
  466. }
  467. }
  468. /* If all devices reported they can proceed, then re-enable DMA */
  469. if (result == PCI_ERS_RESULT_CAN_RECOVER) {
  470. pr_info("EEH: Enabled DMA for affected devices\n");
  471. rc = eeh_pci_enable(pe, EEH_OPT_THAW_DMA);
  472. if (rc < 0)
  473. goto hard_fail;
  474. if (rc)
  475. result = PCI_ERS_RESULT_NEED_RESET;
  476. else
  477. result = PCI_ERS_RESULT_RECOVERED;
  478. }
  479. /* If any device has a hard failure, then shut off everything. */
  480. if (result == PCI_ERS_RESULT_DISCONNECT) {
  481. pr_warning("EEH: Device driver gave up\n");
  482. goto hard_fail;
  483. }
  484. /* If any device called out for a reset, then reset the slot */
  485. if (result == PCI_ERS_RESULT_NEED_RESET) {
  486. pr_info("EEH: Reset without hotplug activity\n");
  487. rc = eeh_reset_device(pe, NULL);
  488. if (rc) {
  489. pr_warning("%s: Cannot reset, err=%d\n",
  490. __func__, rc);
  491. goto hard_fail;
  492. }
  493. pr_info("EEH: Notify device drivers "
  494. "the completion of reset\n");
  495. result = PCI_ERS_RESULT_NONE;
  496. eeh_pe_dev_traverse(pe, eeh_report_reset, &result);
  497. }
  498. /* All devices should claim they have recovered by now. */
  499. if ((result != PCI_ERS_RESULT_RECOVERED) &&
  500. (result != PCI_ERS_RESULT_NONE)) {
  501. pr_warning("EEH: Not recovered\n");
  502. goto hard_fail;
  503. }
  504. /* Tell all device drivers that they can resume operations */
  505. pr_info("EEH: Notify device driver to resume\n");
  506. eeh_pe_dev_traverse(pe, eeh_report_resume, NULL);
  507. return;
  508. excess_failures:
  509. /*
  510. * About 90% of all real-life EEH failures in the field
  511. * are due to poorly seated PCI cards. Only 10% or so are
  512. * due to actual, failed cards.
  513. */
  514. pr_err("EEH: PHB#%d-PE#%x has failed %d times in the\n"
  515. "last hour and has been permanently disabled.\n"
  516. "Please try reseating or replacing it.\n",
  517. pe->phb->global_number, pe->addr,
  518. pe->freeze_count);
  519. goto perm_error;
  520. hard_fail:
  521. pr_err("EEH: Unable to recover from failure from PHB#%d-PE#%x.\n"
  522. "Please try reseating or replacing it\n",
  523. pe->phb->global_number, pe->addr);
  524. perm_error:
  525. eeh_slot_error_detail(pe, EEH_LOG_PERM);
  526. /* Notify all devices that they're about to go down. */
  527. eeh_pe_dev_traverse(pe, eeh_report_failure, NULL);
  528. /* Shut down the device drivers for good. */
  529. if (frozen_bus)
  530. pcibios_remove_pci_devices(frozen_bus);
  531. }
  532. static void eeh_handle_special_event(void)
  533. {
  534. struct eeh_pe *pe, *phb_pe;
  535. struct pci_bus *bus;
  536. struct pci_controller *hose, *tmp;
  537. unsigned long flags;
  538. int rc = 0;
  539. /*
  540. * The return value from next_error() has been classified as follows.
  541. * It might be good to enumerate them. However, next_error() is only
  542. * supported by PowerNV platform for now. So it would be fine to use
  543. * integer directly:
  544. *
  545. * 4 - Dead IOC 3 - Dead PHB
  546. * 2 - Fenced PHB 1 - Frozen PE
  547. * 0 - No error found
  548. *
  549. */
  550. rc = eeh_ops->next_error(&pe);
  551. if (rc <= 0)
  552. return;
  553. switch (rc) {
  554. case 4:
  555. /* Mark all PHBs in dead state */
  556. eeh_serialize_lock(&flags);
  557. list_for_each_entry_safe(hose, tmp,
  558. &hose_list, list_node) {
  559. phb_pe = eeh_phb_pe_get(hose);
  560. if (!phb_pe) continue;
  561. eeh_pe_state_mark(phb_pe,
  562. EEH_PE_ISOLATED | EEH_PE_PHB_DEAD);
  563. }
  564. eeh_serialize_unlock(flags);
  565. /* Purge all events */
  566. eeh_remove_event(NULL);
  567. break;
  568. case 3:
  569. case 2:
  570. case 1:
  571. /* Mark the PE in fenced state */
  572. eeh_serialize_lock(&flags);
  573. if (rc == 3)
  574. eeh_pe_state_mark(pe,
  575. EEH_PE_ISOLATED | EEH_PE_PHB_DEAD);
  576. else
  577. eeh_pe_state_mark(pe,
  578. EEH_PE_ISOLATED | EEH_PE_RECOVERING);
  579. eeh_serialize_unlock(flags);
  580. /* Purge all events of the PHB */
  581. eeh_remove_event(pe);
  582. break;
  583. default:
  584. pr_err("%s: Invalid value %d from next_error()\n",
  585. __func__, rc);
  586. return;
  587. }
  588. /*
  589. * For fenced PHB and frozen PE, it's handled as normal
  590. * event. We have to remove the affected PHBs for dead
  591. * PHB and IOC
  592. */
  593. if (rc == 2 || rc == 1)
  594. eeh_handle_normal_event(pe);
  595. else {
  596. list_for_each_entry_safe(hose, tmp,
  597. &hose_list, list_node) {
  598. phb_pe = eeh_phb_pe_get(hose);
  599. if (!phb_pe || !(phb_pe->state & EEH_PE_PHB_DEAD))
  600. continue;
  601. bus = eeh_pe_bus_get(phb_pe);
  602. /* Notify all devices that they're about to go down. */
  603. eeh_pe_dev_traverse(pe, eeh_report_failure, NULL);
  604. pcibios_remove_pci_devices(bus);
  605. }
  606. }
  607. }
  608. /**
  609. * eeh_handle_event - Reset a PCI device after hard lockup.
  610. * @pe: EEH PE
  611. *
  612. * While PHB detects address or data parity errors on particular PCI
  613. * slot, the associated PE will be frozen. Besides, DMA's occurring
  614. * to wild addresses (which usually happen due to bugs in device
  615. * drivers or in PCI adapter firmware) can cause EEH error. #SERR,
  616. * #PERR or other misc PCI-related errors also can trigger EEH errors.
  617. *
  618. * Recovery process consists of unplugging the device driver (which
  619. * generated hotplug events to userspace), then issuing a PCI #RST to
  620. * the device, then reconfiguring the PCI config space for all bridges
  621. * & devices under this slot, and then finally restarting the device
  622. * drivers (which cause a second set of hotplug events to go out to
  623. * userspace).
  624. */
  625. void eeh_handle_event(struct eeh_pe *pe)
  626. {
  627. if (pe)
  628. eeh_handle_normal_event(pe);
  629. else
  630. eeh_handle_special_event();
  631. }