dma.c 26 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102
  1. /*
  2. * Filename: dma.c
  3. *
  4. *
  5. * Authors: Joshua Morris <josh.h.morris@us.ibm.com>
  6. * Philip Kelleher <pjk1939@linux.vnet.ibm.com>
  7. *
  8. * (C) Copyright 2013 IBM Corporation
  9. *
  10. * This program is free software; you can redistribute it and/or
  11. * modify it under the terms of the GNU General Public License as
  12. * published by the Free Software Foundation; either version 2 of the
  13. * License, or (at your option) any later version.
  14. *
  15. * This program is distributed in the hope that it will be useful, but
  16. * WITHOUT ANY WARRANTY; without even the implied warranty of
  17. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18. * General Public License for more details.
  19. *
  20. * You should have received a copy of the GNU General Public License
  21. * along with this program; if not, write to the Free Software Foundation,
  22. * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  23. */
  24. #include <linux/slab.h>
  25. #include "rsxx_priv.h"
  26. struct rsxx_dma {
  27. struct list_head list;
  28. u8 cmd;
  29. unsigned int laddr; /* Logical address */
  30. struct {
  31. u32 off;
  32. u32 cnt;
  33. } sub_page;
  34. dma_addr_t dma_addr;
  35. struct page *page;
  36. unsigned int pg_off; /* Page Offset */
  37. rsxx_dma_cb cb;
  38. void *cb_data;
  39. };
  40. /* This timeout is used to detect a stalled DMA channel */
  41. #define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000)
  42. struct hw_status {
  43. u8 status;
  44. u8 tag;
  45. __le16 count;
  46. __le32 _rsvd2;
  47. __le64 _rsvd3;
  48. } __packed;
  49. enum rsxx_dma_status {
  50. DMA_SW_ERR = 0x1,
  51. DMA_HW_FAULT = 0x2,
  52. DMA_CANCELLED = 0x4,
  53. };
  54. struct hw_cmd {
  55. u8 command;
  56. u8 tag;
  57. u8 _rsvd;
  58. u8 sub_page; /* Bit[0:2]: 512byte offset */
  59. /* Bit[4:6]: 512byte count */
  60. __le32 device_addr;
  61. __le64 host_addr;
  62. } __packed;
  63. enum rsxx_hw_cmd {
  64. HW_CMD_BLK_DISCARD = 0x70,
  65. HW_CMD_BLK_WRITE = 0x80,
  66. HW_CMD_BLK_READ = 0xC0,
  67. HW_CMD_BLK_RECON_READ = 0xE0,
  68. };
  69. enum rsxx_hw_status {
  70. HW_STATUS_CRC = 0x01,
  71. HW_STATUS_HARD_ERR = 0x02,
  72. HW_STATUS_SOFT_ERR = 0x04,
  73. HW_STATUS_FAULT = 0x08,
  74. };
  75. static struct kmem_cache *rsxx_dma_pool;
  76. struct dma_tracker {
  77. int next_tag;
  78. struct rsxx_dma *dma;
  79. };
  80. #define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \
  81. (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS))
  82. struct dma_tracker_list {
  83. spinlock_t lock;
  84. int head;
  85. struct dma_tracker list[0];
  86. };
  87. /*----------------- Misc Utility Functions -------------------*/
  88. static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
  89. {
  90. unsigned long long tgt_addr8;
  91. tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
  92. card->_stripe.upper_mask) |
  93. ((addr8) & card->_stripe.lower_mask);
  94. do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
  95. return tgt_addr8;
  96. }
  97. static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
  98. {
  99. unsigned int tgt;
  100. tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
  101. return tgt;
  102. }
  103. void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
  104. {
  105. /* Reset all DMA Command/Status Queues */
  106. iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
  107. }
  108. static unsigned int get_dma_size(struct rsxx_dma *dma)
  109. {
  110. if (dma->sub_page.cnt)
  111. return dma->sub_page.cnt << 9;
  112. else
  113. return RSXX_HW_BLK_SIZE;
  114. }
  115. /*----------------- DMA Tracker -------------------*/
  116. static void set_tracker_dma(struct dma_tracker_list *trackers,
  117. int tag,
  118. struct rsxx_dma *dma)
  119. {
  120. trackers->list[tag].dma = dma;
  121. }
  122. static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
  123. int tag)
  124. {
  125. return trackers->list[tag].dma;
  126. }
  127. static int pop_tracker(struct dma_tracker_list *trackers)
  128. {
  129. int tag;
  130. spin_lock(&trackers->lock);
  131. tag = trackers->head;
  132. if (tag != -1) {
  133. trackers->head = trackers->list[tag].next_tag;
  134. trackers->list[tag].next_tag = -1;
  135. }
  136. spin_unlock(&trackers->lock);
  137. return tag;
  138. }
  139. static void push_tracker(struct dma_tracker_list *trackers, int tag)
  140. {
  141. spin_lock(&trackers->lock);
  142. trackers->list[tag].next_tag = trackers->head;
  143. trackers->head = tag;
  144. trackers->list[tag].dma = NULL;
  145. spin_unlock(&trackers->lock);
  146. }
  147. /*----------------- Interrupt Coalescing -------------*/
  148. /*
  149. * Interrupt Coalescing Register Format:
  150. * Interrupt Timer (64ns units) [15:0]
  151. * Interrupt Count [24:16]
  152. * Reserved [31:25]
  153. */
  154. #define INTR_COAL_LATENCY_MASK (0x0000ffff)
  155. #define INTR_COAL_COUNT_SHIFT 16
  156. #define INTR_COAL_COUNT_BITS 9
  157. #define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \
  158. INTR_COAL_COUNT_SHIFT)
  159. #define INTR_COAL_LATENCY_UNITS_NS 64
  160. static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
  161. {
  162. u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
  163. if (mode == RSXX_INTR_COAL_DISABLED)
  164. return 0;
  165. return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
  166. (latency_units & INTR_COAL_LATENCY_MASK);
  167. }
  168. static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
  169. {
  170. int i;
  171. u32 q_depth = 0;
  172. u32 intr_coal;
  173. if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE ||
  174. unlikely(card->eeh_state))
  175. return;
  176. for (i = 0; i < card->n_targets; i++)
  177. q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
  178. intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
  179. q_depth / 2,
  180. card->config.data.intr_coal.latency);
  181. iowrite32(intr_coal, card->regmap + INTR_COAL);
  182. }
  183. /*----------------- RSXX DMA Handling -------------------*/
  184. static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
  185. struct rsxx_dma *dma,
  186. unsigned int status)
  187. {
  188. if (status & DMA_SW_ERR)
  189. ctrl->stats.dma_sw_err++;
  190. if (status & DMA_HW_FAULT)
  191. ctrl->stats.dma_hw_fault++;
  192. if (status & DMA_CANCELLED)
  193. ctrl->stats.dma_cancelled++;
  194. if (dma->dma_addr)
  195. pci_unmap_page(ctrl->card->dev, dma->dma_addr,
  196. get_dma_size(dma),
  197. dma->cmd == HW_CMD_BLK_WRITE ?
  198. PCI_DMA_TODEVICE :
  199. PCI_DMA_FROMDEVICE);
  200. if (dma->cb)
  201. dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
  202. kmem_cache_free(rsxx_dma_pool, dma);
  203. }
  204. int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
  205. struct list_head *q)
  206. {
  207. struct rsxx_dma *dma;
  208. struct rsxx_dma *tmp;
  209. int cnt = 0;
  210. list_for_each_entry_safe(dma, tmp, q, list) {
  211. list_del(&dma->list);
  212. rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
  213. cnt++;
  214. }
  215. return cnt;
  216. }
  217. static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
  218. struct rsxx_dma *dma)
  219. {
  220. /*
  221. * Requeued DMAs go to the front of the queue so they are issued
  222. * first.
  223. */
  224. spin_lock_bh(&ctrl->queue_lock);
  225. ctrl->stats.sw_q_depth++;
  226. list_add(&dma->list, &ctrl->queue);
  227. spin_unlock_bh(&ctrl->queue_lock);
  228. }
  229. static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
  230. struct rsxx_dma *dma,
  231. u8 hw_st)
  232. {
  233. unsigned int status = 0;
  234. int requeue_cmd = 0;
  235. dev_dbg(CARD_TO_DEV(ctrl->card),
  236. "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
  237. dma->cmd, dma->laddr, hw_st);
  238. if (hw_st & HW_STATUS_CRC)
  239. ctrl->stats.crc_errors++;
  240. if (hw_st & HW_STATUS_HARD_ERR)
  241. ctrl->stats.hard_errors++;
  242. if (hw_st & HW_STATUS_SOFT_ERR)
  243. ctrl->stats.soft_errors++;
  244. switch (dma->cmd) {
  245. case HW_CMD_BLK_READ:
  246. if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
  247. if (ctrl->card->scrub_hard) {
  248. dma->cmd = HW_CMD_BLK_RECON_READ;
  249. requeue_cmd = 1;
  250. ctrl->stats.reads_retried++;
  251. } else {
  252. status |= DMA_HW_FAULT;
  253. ctrl->stats.reads_failed++;
  254. }
  255. } else if (hw_st & HW_STATUS_FAULT) {
  256. status |= DMA_HW_FAULT;
  257. ctrl->stats.reads_failed++;
  258. }
  259. break;
  260. case HW_CMD_BLK_RECON_READ:
  261. if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
  262. /* Data could not be reconstructed. */
  263. status |= DMA_HW_FAULT;
  264. ctrl->stats.reads_failed++;
  265. }
  266. break;
  267. case HW_CMD_BLK_WRITE:
  268. status |= DMA_HW_FAULT;
  269. ctrl->stats.writes_failed++;
  270. break;
  271. case HW_CMD_BLK_DISCARD:
  272. status |= DMA_HW_FAULT;
  273. ctrl->stats.discards_failed++;
  274. break;
  275. default:
  276. dev_err(CARD_TO_DEV(ctrl->card),
  277. "Unknown command in DMA!(cmd: x%02x "
  278. "laddr x%08x st: x%02x\n",
  279. dma->cmd, dma->laddr, hw_st);
  280. status |= DMA_SW_ERR;
  281. break;
  282. }
  283. if (requeue_cmd)
  284. rsxx_requeue_dma(ctrl, dma);
  285. else
  286. rsxx_complete_dma(ctrl, dma, status);
  287. }
  288. static void dma_engine_stalled(unsigned long data)
  289. {
  290. struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data;
  291. int cnt;
  292. if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
  293. unlikely(ctrl->card->eeh_state))
  294. return;
  295. if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
  296. /*
  297. * The dma engine was stalled because the SW_CMD_IDX write
  298. * was lost. Issue it again to recover.
  299. */
  300. dev_warn(CARD_TO_DEV(ctrl->card),
  301. "SW_CMD_IDX write was lost, re-writing...\n");
  302. iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
  303. mod_timer(&ctrl->activity_timer,
  304. jiffies + DMA_ACTIVITY_TIMEOUT);
  305. } else {
  306. dev_warn(CARD_TO_DEV(ctrl->card),
  307. "DMA channel %d has stalled, faulting interface.\n",
  308. ctrl->id);
  309. ctrl->card->dma_fault = 1;
  310. /* Clean up the DMA queue */
  311. spin_lock(&ctrl->queue_lock);
  312. cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
  313. spin_unlock(&ctrl->queue_lock);
  314. cnt += rsxx_dma_cancel(ctrl);
  315. if (cnt)
  316. dev_info(CARD_TO_DEV(ctrl->card),
  317. "Freed %d queued DMAs on channel %d\n",
  318. cnt, ctrl->id);
  319. }
  320. }
  321. static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl)
  322. {
  323. struct rsxx_dma *dma;
  324. int tag;
  325. int cmds_pending = 0;
  326. struct hw_cmd *hw_cmd_buf;
  327. hw_cmd_buf = ctrl->cmd.buf;
  328. if (unlikely(ctrl->card->halt) ||
  329. unlikely(ctrl->card->eeh_state))
  330. return;
  331. while (1) {
  332. spin_lock_bh(&ctrl->queue_lock);
  333. if (list_empty(&ctrl->queue)) {
  334. spin_unlock_bh(&ctrl->queue_lock);
  335. break;
  336. }
  337. spin_unlock_bh(&ctrl->queue_lock);
  338. tag = pop_tracker(ctrl->trackers);
  339. if (tag == -1)
  340. break;
  341. spin_lock_bh(&ctrl->queue_lock);
  342. dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
  343. list_del(&dma->list);
  344. ctrl->stats.sw_q_depth--;
  345. spin_unlock_bh(&ctrl->queue_lock);
  346. /*
  347. * This will catch any DMAs that slipped in right before the
  348. * fault, but was queued after all the other DMAs were
  349. * cancelled.
  350. */
  351. if (unlikely(ctrl->card->dma_fault)) {
  352. push_tracker(ctrl->trackers, tag);
  353. rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
  354. continue;
  355. }
  356. set_tracker_dma(ctrl->trackers, tag, dma);
  357. hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd;
  358. hw_cmd_buf[ctrl->cmd.idx].tag = tag;
  359. hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0;
  360. hw_cmd_buf[ctrl->cmd.idx].sub_page =
  361. ((dma->sub_page.cnt & 0x7) << 4) |
  362. (dma->sub_page.off & 0x7);
  363. hw_cmd_buf[ctrl->cmd.idx].device_addr =
  364. cpu_to_le32(dma->laddr);
  365. hw_cmd_buf[ctrl->cmd.idx].host_addr =
  366. cpu_to_le64(dma->dma_addr);
  367. dev_dbg(CARD_TO_DEV(ctrl->card),
  368. "Issue DMA%d(laddr %d tag %d) to idx %d\n",
  369. ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
  370. ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
  371. cmds_pending++;
  372. if (dma->cmd == HW_CMD_BLK_WRITE)
  373. ctrl->stats.writes_issued++;
  374. else if (dma->cmd == HW_CMD_BLK_DISCARD)
  375. ctrl->stats.discards_issued++;
  376. else
  377. ctrl->stats.reads_issued++;
  378. }
  379. /* Let HW know we've queued commands. */
  380. if (cmds_pending) {
  381. atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
  382. mod_timer(&ctrl->activity_timer,
  383. jiffies + DMA_ACTIVITY_TIMEOUT);
  384. if (unlikely(ctrl->card->eeh_state)) {
  385. del_timer_sync(&ctrl->activity_timer);
  386. return;
  387. }
  388. iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
  389. }
  390. }
  391. static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl)
  392. {
  393. struct rsxx_dma *dma;
  394. unsigned long flags;
  395. u16 count;
  396. u8 status;
  397. u8 tag;
  398. struct hw_status *hw_st_buf;
  399. hw_st_buf = ctrl->status.buf;
  400. if (unlikely(ctrl->card->halt) ||
  401. unlikely(ctrl->card->dma_fault) ||
  402. unlikely(ctrl->card->eeh_state))
  403. return;
  404. count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
  405. while (count == ctrl->e_cnt) {
  406. /*
  407. * The read memory-barrier is necessary to keep aggressive
  408. * processors/optimizers (such as the PPC Apple G5) from
  409. * reordering the following status-buffer tag & status read
  410. * *before* the count read on subsequent iterations of the
  411. * loop!
  412. */
  413. rmb();
  414. status = hw_st_buf[ctrl->status.idx].status;
  415. tag = hw_st_buf[ctrl->status.idx].tag;
  416. dma = get_tracker_dma(ctrl->trackers, tag);
  417. if (dma == NULL) {
  418. spin_lock_irqsave(&ctrl->card->irq_lock, flags);
  419. rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
  420. spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
  421. dev_err(CARD_TO_DEV(ctrl->card),
  422. "No tracker for tag %d "
  423. "(idx %d id %d)\n",
  424. tag, ctrl->status.idx, ctrl->id);
  425. return;
  426. }
  427. dev_dbg(CARD_TO_DEV(ctrl->card),
  428. "Completing DMA%d"
  429. "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
  430. ctrl->id, dma->laddr, tag, status, count,
  431. ctrl->status.idx);
  432. atomic_dec(&ctrl->stats.hw_q_depth);
  433. mod_timer(&ctrl->activity_timer,
  434. jiffies + DMA_ACTIVITY_TIMEOUT);
  435. if (status)
  436. rsxx_handle_dma_error(ctrl, dma, status);
  437. else
  438. rsxx_complete_dma(ctrl, dma, 0);
  439. push_tracker(ctrl->trackers, tag);
  440. ctrl->status.idx = (ctrl->status.idx + 1) &
  441. RSXX_CS_IDX_MASK;
  442. ctrl->e_cnt++;
  443. count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
  444. }
  445. dma_intr_coal_auto_tune(ctrl->card);
  446. if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
  447. del_timer_sync(&ctrl->activity_timer);
  448. spin_lock_irqsave(&ctrl->card->irq_lock, flags);
  449. rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
  450. spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
  451. spin_lock_bh(&ctrl->queue_lock);
  452. if (ctrl->stats.sw_q_depth)
  453. queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
  454. spin_unlock_bh(&ctrl->queue_lock);
  455. }
  456. static void rsxx_schedule_issue(struct work_struct *work)
  457. {
  458. struct rsxx_dma_ctrl *ctrl;
  459. ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
  460. mutex_lock(&ctrl->work_lock);
  461. rsxx_issue_dmas(ctrl);
  462. mutex_unlock(&ctrl->work_lock);
  463. }
  464. static void rsxx_schedule_done(struct work_struct *work)
  465. {
  466. struct rsxx_dma_ctrl *ctrl;
  467. ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
  468. mutex_lock(&ctrl->work_lock);
  469. rsxx_dma_done(ctrl);
  470. mutex_unlock(&ctrl->work_lock);
  471. }
  472. static int rsxx_queue_discard(struct rsxx_cardinfo *card,
  473. struct list_head *q,
  474. unsigned int laddr,
  475. rsxx_dma_cb cb,
  476. void *cb_data)
  477. {
  478. struct rsxx_dma *dma;
  479. dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
  480. if (!dma)
  481. return -ENOMEM;
  482. dma->cmd = HW_CMD_BLK_DISCARD;
  483. dma->laddr = laddr;
  484. dma->dma_addr = 0;
  485. dma->sub_page.off = 0;
  486. dma->sub_page.cnt = 0;
  487. dma->page = NULL;
  488. dma->pg_off = 0;
  489. dma->cb = cb;
  490. dma->cb_data = cb_data;
  491. dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
  492. list_add_tail(&dma->list, q);
  493. return 0;
  494. }
  495. static int rsxx_queue_dma(struct rsxx_cardinfo *card,
  496. struct list_head *q,
  497. int dir,
  498. unsigned int dma_off,
  499. unsigned int dma_len,
  500. unsigned int laddr,
  501. struct page *page,
  502. unsigned int pg_off,
  503. rsxx_dma_cb cb,
  504. void *cb_data)
  505. {
  506. struct rsxx_dma *dma;
  507. dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
  508. if (!dma)
  509. return -ENOMEM;
  510. dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len,
  511. dir ? PCI_DMA_TODEVICE :
  512. PCI_DMA_FROMDEVICE);
  513. if (!dma->dma_addr) {
  514. kmem_cache_free(rsxx_dma_pool, dma);
  515. return -ENOMEM;
  516. }
  517. dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
  518. dma->laddr = laddr;
  519. dma->sub_page.off = (dma_off >> 9);
  520. dma->sub_page.cnt = (dma_len >> 9);
  521. dma->page = page;
  522. dma->pg_off = pg_off;
  523. dma->cb = cb;
  524. dma->cb_data = cb_data;
  525. dev_dbg(CARD_TO_DEV(card),
  526. "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
  527. dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
  528. dma->sub_page.cnt, dma->page, dma->pg_off);
  529. /* Queue the DMA */
  530. list_add_tail(&dma->list, q);
  531. return 0;
  532. }
  533. int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
  534. struct bio *bio,
  535. atomic_t *n_dmas,
  536. rsxx_dma_cb cb,
  537. void *cb_data)
  538. {
  539. struct list_head dma_list[RSXX_MAX_TARGETS];
  540. struct bio_vec *bvec;
  541. unsigned long long addr8;
  542. unsigned int laddr;
  543. unsigned int bv_len;
  544. unsigned int bv_off;
  545. unsigned int dma_off;
  546. unsigned int dma_len;
  547. int dma_cnt[RSXX_MAX_TARGETS];
  548. int tgt;
  549. int st;
  550. int i;
  551. addr8 = bio->bi_sector << 9; /* sectors are 512 bytes */
  552. atomic_set(n_dmas, 0);
  553. for (i = 0; i < card->n_targets; i++) {
  554. INIT_LIST_HEAD(&dma_list[i]);
  555. dma_cnt[i] = 0;
  556. }
  557. if (bio->bi_rw & REQ_DISCARD) {
  558. bv_len = bio->bi_size;
  559. while (bv_len > 0) {
  560. tgt = rsxx_get_dma_tgt(card, addr8);
  561. laddr = rsxx_addr8_to_laddr(addr8, card);
  562. st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
  563. cb, cb_data);
  564. if (st)
  565. goto bvec_err;
  566. dma_cnt[tgt]++;
  567. atomic_inc(n_dmas);
  568. addr8 += RSXX_HW_BLK_SIZE;
  569. bv_len -= RSXX_HW_BLK_SIZE;
  570. }
  571. } else {
  572. bio_for_each_segment(bvec, bio, i) {
  573. bv_len = bvec->bv_len;
  574. bv_off = bvec->bv_offset;
  575. while (bv_len > 0) {
  576. tgt = rsxx_get_dma_tgt(card, addr8);
  577. laddr = rsxx_addr8_to_laddr(addr8, card);
  578. dma_off = addr8 & RSXX_HW_BLK_MASK;
  579. dma_len = min(bv_len,
  580. RSXX_HW_BLK_SIZE - dma_off);
  581. st = rsxx_queue_dma(card, &dma_list[tgt],
  582. bio_data_dir(bio),
  583. dma_off, dma_len,
  584. laddr, bvec->bv_page,
  585. bv_off, cb, cb_data);
  586. if (st)
  587. goto bvec_err;
  588. dma_cnt[tgt]++;
  589. atomic_inc(n_dmas);
  590. addr8 += dma_len;
  591. bv_off += dma_len;
  592. bv_len -= dma_len;
  593. }
  594. }
  595. }
  596. for (i = 0; i < card->n_targets; i++) {
  597. if (!list_empty(&dma_list[i])) {
  598. spin_lock_bh(&card->ctrl[i].queue_lock);
  599. card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
  600. list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
  601. spin_unlock_bh(&card->ctrl[i].queue_lock);
  602. queue_work(card->ctrl[i].issue_wq,
  603. &card->ctrl[i].issue_dma_work);
  604. }
  605. }
  606. return 0;
  607. bvec_err:
  608. for (i = 0; i < card->n_targets; i++) {
  609. spin_lock_bh(&card->ctrl[i].queue_lock);
  610. rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i]);
  611. spin_unlock_bh(&card->ctrl[i].queue_lock);
  612. }
  613. return st;
  614. }
  615. /*----------------- DMA Engine Initialization & Setup -------------------*/
  616. int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl)
  617. {
  618. ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8,
  619. &ctrl->status.dma_addr);
  620. ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8,
  621. &ctrl->cmd.dma_addr);
  622. if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
  623. return -ENOMEM;
  624. memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
  625. iowrite32(lower_32_bits(ctrl->status.dma_addr),
  626. ctrl->regmap + SB_ADD_LO);
  627. iowrite32(upper_32_bits(ctrl->status.dma_addr),
  628. ctrl->regmap + SB_ADD_HI);
  629. memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
  630. iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
  631. iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
  632. ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
  633. if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
  634. dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
  635. ctrl->status.idx);
  636. return -EINVAL;
  637. }
  638. iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
  639. iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
  640. ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
  641. if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
  642. dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
  643. ctrl->status.idx);
  644. return -EINVAL;
  645. }
  646. iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
  647. iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
  648. return 0;
  649. }
  650. static int rsxx_dma_ctrl_init(struct pci_dev *dev,
  651. struct rsxx_dma_ctrl *ctrl)
  652. {
  653. int i;
  654. int st;
  655. memset(&ctrl->stats, 0, sizeof(ctrl->stats));
  656. ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8);
  657. if (!ctrl->trackers)
  658. return -ENOMEM;
  659. ctrl->trackers->head = 0;
  660. for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
  661. ctrl->trackers->list[i].next_tag = i + 1;
  662. ctrl->trackers->list[i].dma = NULL;
  663. }
  664. ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
  665. spin_lock_init(&ctrl->trackers->lock);
  666. spin_lock_init(&ctrl->queue_lock);
  667. mutex_init(&ctrl->work_lock);
  668. INIT_LIST_HEAD(&ctrl->queue);
  669. setup_timer(&ctrl->activity_timer, dma_engine_stalled,
  670. (unsigned long)ctrl);
  671. ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
  672. if (!ctrl->issue_wq)
  673. return -ENOMEM;
  674. ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
  675. if (!ctrl->done_wq)
  676. return -ENOMEM;
  677. INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue);
  678. INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done);
  679. st = rsxx_hw_buffers_init(dev, ctrl);
  680. if (st)
  681. return st;
  682. return 0;
  683. }
  684. static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
  685. unsigned int stripe_size8)
  686. {
  687. if (!is_power_of_2(stripe_size8)) {
  688. dev_err(CARD_TO_DEV(card),
  689. "stripe_size is NOT a power of 2!\n");
  690. return -EINVAL;
  691. }
  692. card->_stripe.lower_mask = stripe_size8 - 1;
  693. card->_stripe.upper_mask = ~(card->_stripe.lower_mask);
  694. card->_stripe.upper_shift = ffs(card->n_targets) - 1;
  695. card->_stripe.target_mask = card->n_targets - 1;
  696. card->_stripe.target_shift = ffs(stripe_size8) - 1;
  697. dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n",
  698. card->_stripe.lower_mask);
  699. dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n",
  700. card->_stripe.upper_shift);
  701. dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n",
  702. card->_stripe.upper_mask);
  703. dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n",
  704. card->_stripe.target_mask);
  705. dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
  706. card->_stripe.target_shift);
  707. return 0;
  708. }
  709. int rsxx_dma_configure(struct rsxx_cardinfo *card)
  710. {
  711. u32 intr_coal;
  712. intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
  713. card->config.data.intr_coal.count,
  714. card->config.data.intr_coal.latency);
  715. iowrite32(intr_coal, card->regmap + INTR_COAL);
  716. return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
  717. }
  718. int rsxx_dma_setup(struct rsxx_cardinfo *card)
  719. {
  720. unsigned long flags;
  721. int st;
  722. int i;
  723. dev_info(CARD_TO_DEV(card),
  724. "Initializing %d DMA targets\n",
  725. card->n_targets);
  726. /* Regmap is divided up into 4K chunks. One for each DMA channel */
  727. for (i = 0; i < card->n_targets; i++)
  728. card->ctrl[i].regmap = card->regmap + (i * 4096);
  729. card->dma_fault = 0;
  730. /* Reset the DMA queues */
  731. rsxx_dma_queue_reset(card);
  732. /************* Setup DMA Control *************/
  733. for (i = 0; i < card->n_targets; i++) {
  734. st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
  735. if (st)
  736. goto failed_dma_setup;
  737. card->ctrl[i].card = card;
  738. card->ctrl[i].id = i;
  739. }
  740. card->scrub_hard = 1;
  741. if (card->config_valid)
  742. rsxx_dma_configure(card);
  743. /* Enable the interrupts after all setup has completed. */
  744. for (i = 0; i < card->n_targets; i++) {
  745. spin_lock_irqsave(&card->irq_lock, flags);
  746. rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
  747. spin_unlock_irqrestore(&card->irq_lock, flags);
  748. }
  749. return 0;
  750. failed_dma_setup:
  751. for (i = 0; i < card->n_targets; i++) {
  752. struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
  753. if (ctrl->issue_wq) {
  754. destroy_workqueue(ctrl->issue_wq);
  755. ctrl->issue_wq = NULL;
  756. }
  757. if (ctrl->done_wq) {
  758. destroy_workqueue(ctrl->done_wq);
  759. ctrl->done_wq = NULL;
  760. }
  761. if (ctrl->trackers)
  762. vfree(ctrl->trackers);
  763. if (ctrl->status.buf)
  764. pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
  765. ctrl->status.buf,
  766. ctrl->status.dma_addr);
  767. if (ctrl->cmd.buf)
  768. pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
  769. ctrl->cmd.buf, ctrl->cmd.dma_addr);
  770. }
  771. return st;
  772. }
  773. int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
  774. {
  775. struct rsxx_dma *dma;
  776. int i;
  777. int cnt = 0;
  778. /* Clean up issued DMAs */
  779. for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
  780. dma = get_tracker_dma(ctrl->trackers, i);
  781. if (dma) {
  782. atomic_dec(&ctrl->stats.hw_q_depth);
  783. rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
  784. push_tracker(ctrl->trackers, i);
  785. cnt++;
  786. }
  787. }
  788. return cnt;
  789. }
  790. void rsxx_dma_destroy(struct rsxx_cardinfo *card)
  791. {
  792. struct rsxx_dma_ctrl *ctrl;
  793. int i;
  794. for (i = 0; i < card->n_targets; i++) {
  795. ctrl = &card->ctrl[i];
  796. if (ctrl->issue_wq) {
  797. destroy_workqueue(ctrl->issue_wq);
  798. ctrl->issue_wq = NULL;
  799. }
  800. if (ctrl->done_wq) {
  801. destroy_workqueue(ctrl->done_wq);
  802. ctrl->done_wq = NULL;
  803. }
  804. if (timer_pending(&ctrl->activity_timer))
  805. del_timer_sync(&ctrl->activity_timer);
  806. /* Clean up the DMA queue */
  807. spin_lock_bh(&ctrl->queue_lock);
  808. rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
  809. spin_unlock_bh(&ctrl->queue_lock);
  810. rsxx_dma_cancel(ctrl);
  811. vfree(ctrl->trackers);
  812. pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
  813. ctrl->status.buf, ctrl->status.dma_addr);
  814. pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
  815. ctrl->cmd.buf, ctrl->cmd.dma_addr);
  816. }
  817. }
  818. int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
  819. {
  820. int i;
  821. int j;
  822. int cnt;
  823. struct rsxx_dma *dma;
  824. struct list_head *issued_dmas;
  825. issued_dmas = kzalloc(sizeof(*issued_dmas) * card->n_targets,
  826. GFP_KERNEL);
  827. if (!issued_dmas)
  828. return -ENOMEM;
  829. for (i = 0; i < card->n_targets; i++) {
  830. INIT_LIST_HEAD(&issued_dmas[i]);
  831. cnt = 0;
  832. for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
  833. dma = get_tracker_dma(card->ctrl[i].trackers, j);
  834. if (dma == NULL)
  835. continue;
  836. if (dma->cmd == HW_CMD_BLK_WRITE)
  837. card->ctrl[i].stats.writes_issued--;
  838. else if (dma->cmd == HW_CMD_BLK_DISCARD)
  839. card->ctrl[i].stats.discards_issued--;
  840. else
  841. card->ctrl[i].stats.reads_issued--;
  842. list_add_tail(&dma->list, &issued_dmas[i]);
  843. push_tracker(card->ctrl[i].trackers, j);
  844. cnt++;
  845. }
  846. spin_lock_bh(&card->ctrl[i].queue_lock);
  847. list_splice(&issued_dmas[i], &card->ctrl[i].queue);
  848. atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
  849. card->ctrl[i].stats.sw_q_depth += cnt;
  850. card->ctrl[i].e_cnt = 0;
  851. list_for_each_entry(dma, &card->ctrl[i].queue, list) {
  852. if (dma->dma_addr)
  853. pci_unmap_page(card->dev, dma->dma_addr,
  854. get_dma_size(dma),
  855. dma->cmd == HW_CMD_BLK_WRITE ?
  856. PCI_DMA_TODEVICE :
  857. PCI_DMA_FROMDEVICE);
  858. }
  859. spin_unlock_bh(&card->ctrl[i].queue_lock);
  860. }
  861. kfree(issued_dmas);
  862. return 0;
  863. }
  864. int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card)
  865. {
  866. struct rsxx_dma *dma;
  867. int i;
  868. for (i = 0; i < card->n_targets; i++) {
  869. spin_lock_bh(&card->ctrl[i].queue_lock);
  870. list_for_each_entry(dma, &card->ctrl[i].queue, list) {
  871. dma->dma_addr = pci_map_page(card->dev, dma->page,
  872. dma->pg_off, get_dma_size(dma),
  873. dma->cmd == HW_CMD_BLK_WRITE ?
  874. PCI_DMA_TODEVICE :
  875. PCI_DMA_FROMDEVICE);
  876. if (!dma->dma_addr) {
  877. spin_unlock_bh(&card->ctrl[i].queue_lock);
  878. kmem_cache_free(rsxx_dma_pool, dma);
  879. return -ENOMEM;
  880. }
  881. }
  882. spin_unlock_bh(&card->ctrl[i].queue_lock);
  883. }
  884. return 0;
  885. }
  886. int rsxx_dma_init(void)
  887. {
  888. rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
  889. if (!rsxx_dma_pool)
  890. return -ENOMEM;
  891. return 0;
  892. }
  893. void rsxx_dma_cleanup(void)
  894. {
  895. kmem_cache_destroy(rsxx_dma_pool);
  896. }