mthca_mr.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975
  1. /*
  2. * Copyright (c) 2004 Topspin Communications. All rights reserved.
  3. * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
  4. *
  5. * This software is available to you under a choice of one of two
  6. * licenses. You may choose to be licensed under the terms of the GNU
  7. * General Public License (GPL) Version 2, available from the file
  8. * COPYING in the main directory of this source tree, or the
  9. * OpenIB.org BSD license below:
  10. *
  11. * Redistribution and use in source and binary forms, with or
  12. * without modification, are permitted provided that the following
  13. * conditions are met:
  14. *
  15. * - Redistributions of source code must retain the above
  16. * copyright notice, this list of conditions and the following
  17. * disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials
  22. * provided with the distribution.
  23. *
  24. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31. * SOFTWARE.
  32. */
  33. #include <linux/slab.h>
  34. #include <linux/errno.h>
  35. #include "mthca_dev.h"
  36. #include "mthca_cmd.h"
  37. #include "mthca_memfree.h"
  38. struct mthca_mtt {
  39. struct mthca_buddy *buddy;
  40. int order;
  41. u32 first_seg;
  42. };
  43. /*
  44. * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
  45. */
  46. struct mthca_mpt_entry {
  47. __be32 flags;
  48. __be32 page_size;
  49. __be32 key;
  50. __be32 pd;
  51. __be64 start;
  52. __be64 length;
  53. __be32 lkey;
  54. __be32 window_count;
  55. __be32 window_count_limit;
  56. __be64 mtt_seg;
  57. __be32 mtt_sz; /* Arbel only */
  58. u32 reserved[2];
  59. } __attribute__((packed));
  60. #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28)
  61. #define MTHCA_MPT_FLAG_MIO (1 << 17)
  62. #define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15)
  63. #define MTHCA_MPT_FLAG_PHYSICAL (1 << 9)
  64. #define MTHCA_MPT_FLAG_REGION (1 << 8)
  65. #define MTHCA_MTT_FLAG_PRESENT 1
  66. #define MTHCA_MPT_STATUS_SW 0xF0
  67. #define MTHCA_MPT_STATUS_HW 0x00
  68. #define SINAI_FMR_KEY_INC 0x1000000
  69. /*
  70. * Buddy allocator for MTT segments (currently not very efficient
  71. * since it doesn't keep a free list and just searches linearly
  72. * through the bitmaps)
  73. */
  74. static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order)
  75. {
  76. int o;
  77. int m;
  78. u32 seg;
  79. spin_lock(&buddy->lock);
  80. for (o = order; o <= buddy->max_order; ++o) {
  81. m = 1 << (buddy->max_order - o);
  82. seg = find_first_bit(buddy->bits[o], m);
  83. if (seg < m)
  84. goto found;
  85. }
  86. spin_unlock(&buddy->lock);
  87. return -1;
  88. found:
  89. clear_bit(seg, buddy->bits[o]);
  90. while (o > order) {
  91. --o;
  92. seg <<= 1;
  93. set_bit(seg ^ 1, buddy->bits[o]);
  94. }
  95. spin_unlock(&buddy->lock);
  96. seg <<= order;
  97. return seg;
  98. }
  99. static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order)
  100. {
  101. seg >>= order;
  102. spin_lock(&buddy->lock);
  103. while (test_bit(seg ^ 1, buddy->bits[order])) {
  104. clear_bit(seg ^ 1, buddy->bits[order]);
  105. seg >>= 1;
  106. ++order;
  107. }
  108. set_bit(seg, buddy->bits[order]);
  109. spin_unlock(&buddy->lock);
  110. }
  111. static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order)
  112. {
  113. int i, s;
  114. buddy->max_order = max_order;
  115. spin_lock_init(&buddy->lock);
  116. buddy->bits = kzalloc((buddy->max_order + 1) * sizeof (long *),
  117. GFP_KERNEL);
  118. if (!buddy->bits)
  119. goto err_out;
  120. for (i = 0; i <= buddy->max_order; ++i) {
  121. s = BITS_TO_LONGS(1 << (buddy->max_order - i));
  122. buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL);
  123. if (!buddy->bits[i])
  124. goto err_out_free;
  125. bitmap_zero(buddy->bits[i],
  126. 1 << (buddy->max_order - i));
  127. }
  128. set_bit(0, buddy->bits[buddy->max_order]);
  129. return 0;
  130. err_out_free:
  131. for (i = 0; i <= buddy->max_order; ++i)
  132. kfree(buddy->bits[i]);
  133. kfree(buddy->bits);
  134. err_out:
  135. return -ENOMEM;
  136. }
  137. static void mthca_buddy_cleanup(struct mthca_buddy *buddy)
  138. {
  139. int i;
  140. for (i = 0; i <= buddy->max_order; ++i)
  141. kfree(buddy->bits[i]);
  142. kfree(buddy->bits);
  143. }
  144. static u32 mthca_alloc_mtt_range(struct mthca_dev *dev, int order,
  145. struct mthca_buddy *buddy)
  146. {
  147. u32 seg = mthca_buddy_alloc(buddy, order);
  148. if (seg == -1)
  149. return -1;
  150. if (mthca_is_memfree(dev))
  151. if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg,
  152. seg + (1 << order) - 1)) {
  153. mthca_buddy_free(buddy, seg, order);
  154. seg = -1;
  155. }
  156. return seg;
  157. }
  158. static struct mthca_mtt *__mthca_alloc_mtt(struct mthca_dev *dev, int size,
  159. struct mthca_buddy *buddy)
  160. {
  161. struct mthca_mtt *mtt;
  162. int i;
  163. if (size <= 0)
  164. return ERR_PTR(-EINVAL);
  165. mtt = kmalloc(sizeof *mtt, GFP_KERNEL);
  166. if (!mtt)
  167. return ERR_PTR(-ENOMEM);
  168. mtt->buddy = buddy;
  169. mtt->order = 0;
  170. for (i = MTHCA_MTT_SEG_SIZE / 8; i < size; i <<= 1)
  171. ++mtt->order;
  172. mtt->first_seg = mthca_alloc_mtt_range(dev, mtt->order, buddy);
  173. if (mtt->first_seg == -1) {
  174. kfree(mtt);
  175. return ERR_PTR(-ENOMEM);
  176. }
  177. return mtt;
  178. }
  179. struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size)
  180. {
  181. return __mthca_alloc_mtt(dev, size, &dev->mr_table.mtt_buddy);
  182. }
  183. void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt)
  184. {
  185. if (!mtt)
  186. return;
  187. mthca_buddy_free(mtt->buddy, mtt->first_seg, mtt->order);
  188. mthca_table_put_range(dev, dev->mr_table.mtt_table,
  189. mtt->first_seg,
  190. mtt->first_seg + (1 << mtt->order) - 1);
  191. kfree(mtt);
  192. }
  193. static int __mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt,
  194. int start_index, u64 *buffer_list, int list_len)
  195. {
  196. struct mthca_mailbox *mailbox;
  197. __be64 *mtt_entry;
  198. int err = 0;
  199. u8 status;
  200. int i;
  201. mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
  202. if (IS_ERR(mailbox))
  203. return PTR_ERR(mailbox);
  204. mtt_entry = mailbox->buf;
  205. while (list_len > 0) {
  206. mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base +
  207. mtt->first_seg * MTHCA_MTT_SEG_SIZE +
  208. start_index * 8);
  209. mtt_entry[1] = 0;
  210. for (i = 0; i < list_len && i < MTHCA_MAILBOX_SIZE / 8 - 2; ++i)
  211. mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] |
  212. MTHCA_MTT_FLAG_PRESENT);
  213. /*
  214. * If we have an odd number of entries to write, add
  215. * one more dummy entry for firmware efficiency.
  216. */
  217. if (i & 1)
  218. mtt_entry[i + 2] = 0;
  219. err = mthca_WRITE_MTT(dev, mailbox, (i + 1) & ~1, &status);
  220. if (err) {
  221. mthca_warn(dev, "WRITE_MTT failed (%d)\n", err);
  222. goto out;
  223. }
  224. if (status) {
  225. mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n",
  226. status);
  227. err = -EINVAL;
  228. goto out;
  229. }
  230. list_len -= i;
  231. start_index += i;
  232. buffer_list += i;
  233. }
  234. out:
  235. mthca_free_mailbox(dev, mailbox);
  236. return err;
  237. }
  238. int mthca_write_mtt_size(struct mthca_dev *dev)
  239. {
  240. if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy ||
  241. !(dev->mthca_flags & MTHCA_FLAG_FMR))
  242. /*
  243. * Be friendly to WRITE_MTT command
  244. * and leave two empty slots for the
  245. * index and reserved fields of the
  246. * mailbox.
  247. */
  248. return PAGE_SIZE / sizeof (u64) - 2;
  249. /* For Arbel, all MTTs must fit in the same page. */
  250. return mthca_is_memfree(dev) ? (PAGE_SIZE / sizeof (u64)) : 0x7ffffff;
  251. }
  252. static void mthca_tavor_write_mtt_seg(struct mthca_dev *dev,
  253. struct mthca_mtt *mtt, int start_index,
  254. u64 *buffer_list, int list_len)
  255. {
  256. u64 __iomem *mtts;
  257. int i;
  258. mtts = dev->mr_table.tavor_fmr.mtt_base + mtt->first_seg * MTHCA_MTT_SEG_SIZE +
  259. start_index * sizeof (u64);
  260. for (i = 0; i < list_len; ++i)
  261. mthca_write64_raw(cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT),
  262. mtts + i);
  263. }
  264. static void mthca_arbel_write_mtt_seg(struct mthca_dev *dev,
  265. struct mthca_mtt *mtt, int start_index,
  266. u64 *buffer_list, int list_len)
  267. {
  268. __be64 *mtts;
  269. dma_addr_t dma_handle;
  270. int i;
  271. int s = start_index * sizeof (u64);
  272. /* For Arbel, all MTTs must fit in the same page. */
  273. BUG_ON(s / PAGE_SIZE != (s + list_len * sizeof(u64) - 1) / PAGE_SIZE);
  274. /* Require full segments */
  275. BUG_ON(s % MTHCA_MTT_SEG_SIZE);
  276. mtts = mthca_table_find(dev->mr_table.mtt_table, mtt->first_seg +
  277. s / MTHCA_MTT_SEG_SIZE, &dma_handle);
  278. BUG_ON(!mtts);
  279. for (i = 0; i < list_len; ++i)
  280. mtts[i] = cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT);
  281. dma_sync_single(&dev->pdev->dev, dma_handle, list_len * sizeof (u64), DMA_TO_DEVICE);
  282. }
  283. int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt,
  284. int start_index, u64 *buffer_list, int list_len)
  285. {
  286. int size = mthca_write_mtt_size(dev);
  287. int chunk;
  288. if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy ||
  289. !(dev->mthca_flags & MTHCA_FLAG_FMR))
  290. return __mthca_write_mtt(dev, mtt, start_index, buffer_list, list_len);
  291. while (list_len > 0) {
  292. chunk = min(size, list_len);
  293. if (mthca_is_memfree(dev))
  294. mthca_arbel_write_mtt_seg(dev, mtt, start_index,
  295. buffer_list, chunk);
  296. else
  297. mthca_tavor_write_mtt_seg(dev, mtt, start_index,
  298. buffer_list, chunk);
  299. list_len -= chunk;
  300. start_index += chunk;
  301. buffer_list += chunk;
  302. }
  303. return 0;
  304. }
  305. static inline u32 tavor_hw_index_to_key(u32 ind)
  306. {
  307. return ind;
  308. }
  309. static inline u32 tavor_key_to_hw_index(u32 key)
  310. {
  311. return key;
  312. }
  313. static inline u32 arbel_hw_index_to_key(u32 ind)
  314. {
  315. return (ind >> 24) | (ind << 8);
  316. }
  317. static inline u32 arbel_key_to_hw_index(u32 key)
  318. {
  319. return (key << 24) | (key >> 8);
  320. }
  321. static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind)
  322. {
  323. if (mthca_is_memfree(dev))
  324. return arbel_hw_index_to_key(ind);
  325. else
  326. return tavor_hw_index_to_key(ind);
  327. }
  328. static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key)
  329. {
  330. if (mthca_is_memfree(dev))
  331. return arbel_key_to_hw_index(key);
  332. else
  333. return tavor_key_to_hw_index(key);
  334. }
  335. static inline u32 adjust_key(struct mthca_dev *dev, u32 key)
  336. {
  337. if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
  338. return ((key << 20) & 0x800000) | (key & 0x7fffff);
  339. else
  340. return key;
  341. }
  342. int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift,
  343. u64 iova, u64 total_size, u32 access, struct mthca_mr *mr)
  344. {
  345. struct mthca_mailbox *mailbox;
  346. struct mthca_mpt_entry *mpt_entry;
  347. u32 key;
  348. int i;
  349. int err;
  350. u8 status;
  351. WARN_ON(buffer_size_shift >= 32);
  352. key = mthca_alloc(&dev->mr_table.mpt_alloc);
  353. if (key == -1)
  354. return -ENOMEM;
  355. key = adjust_key(dev, key);
  356. mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
  357. if (mthca_is_memfree(dev)) {
  358. err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
  359. if (err)
  360. goto err_out_mpt_free;
  361. }
  362. mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
  363. if (IS_ERR(mailbox)) {
  364. err = PTR_ERR(mailbox);
  365. goto err_out_table;
  366. }
  367. mpt_entry = mailbox->buf;
  368. mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS |
  369. MTHCA_MPT_FLAG_MIO |
  370. MTHCA_MPT_FLAG_REGION |
  371. access);
  372. if (!mr->mtt)
  373. mpt_entry->flags |= cpu_to_be32(MTHCA_MPT_FLAG_PHYSICAL);
  374. mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12);
  375. mpt_entry->key = cpu_to_be32(key);
  376. mpt_entry->pd = cpu_to_be32(pd);
  377. mpt_entry->start = cpu_to_be64(iova);
  378. mpt_entry->length = cpu_to_be64(total_size);
  379. memset(&mpt_entry->lkey, 0,
  380. sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
  381. if (mr->mtt)
  382. mpt_entry->mtt_seg =
  383. cpu_to_be64(dev->mr_table.mtt_base +
  384. mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE);
  385. if (0) {
  386. mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
  387. for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
  388. if (i % 4 == 0)
  389. printk("[%02x] ", i * 4);
  390. printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i]));
  391. if ((i + 1) % 4 == 0)
  392. printk("\n");
  393. }
  394. }
  395. err = mthca_SW2HW_MPT(dev, mailbox,
  396. key & (dev->limits.num_mpts - 1),
  397. &status);
  398. if (err) {
  399. mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
  400. goto err_out_mailbox;
  401. } else if (status) {
  402. mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
  403. status);
  404. err = -EINVAL;
  405. goto err_out_mailbox;
  406. }
  407. mthca_free_mailbox(dev, mailbox);
  408. return err;
  409. err_out_mailbox:
  410. mthca_free_mailbox(dev, mailbox);
  411. err_out_table:
  412. mthca_table_put(dev, dev->mr_table.mpt_table, key);
  413. err_out_mpt_free:
  414. mthca_free(&dev->mr_table.mpt_alloc, key);
  415. return err;
  416. }
  417. int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd,
  418. u32 access, struct mthca_mr *mr)
  419. {
  420. mr->mtt = NULL;
  421. return mthca_mr_alloc(dev, pd, 12, 0, ~0ULL, access, mr);
  422. }
  423. int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
  424. u64 *buffer_list, int buffer_size_shift,
  425. int list_len, u64 iova, u64 total_size,
  426. u32 access, struct mthca_mr *mr)
  427. {
  428. int err;
  429. mr->mtt = mthca_alloc_mtt(dev, list_len);
  430. if (IS_ERR(mr->mtt))
  431. return PTR_ERR(mr->mtt);
  432. err = mthca_write_mtt(dev, mr->mtt, 0, buffer_list, list_len);
  433. if (err) {
  434. mthca_free_mtt(dev, mr->mtt);
  435. return err;
  436. }
  437. err = mthca_mr_alloc(dev, pd, buffer_size_shift, iova,
  438. total_size, access, mr);
  439. if (err)
  440. mthca_free_mtt(dev, mr->mtt);
  441. return err;
  442. }
  443. /* Free mr or fmr */
  444. static void mthca_free_region(struct mthca_dev *dev, u32 lkey)
  445. {
  446. mthca_table_put(dev, dev->mr_table.mpt_table,
  447. key_to_hw_index(dev, lkey));
  448. mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, lkey));
  449. }
  450. void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr)
  451. {
  452. int err;
  453. u8 status;
  454. err = mthca_HW2SW_MPT(dev, NULL,
  455. key_to_hw_index(dev, mr->ibmr.lkey) &
  456. (dev->limits.num_mpts - 1),
  457. &status);
  458. if (err)
  459. mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err);
  460. else if (status)
  461. mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n",
  462. status);
  463. mthca_free_region(dev, mr->ibmr.lkey);
  464. mthca_free_mtt(dev, mr->mtt);
  465. }
  466. int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd,
  467. u32 access, struct mthca_fmr *mr)
  468. {
  469. struct mthca_mpt_entry *mpt_entry;
  470. struct mthca_mailbox *mailbox;
  471. u64 mtt_seg;
  472. u32 key, idx;
  473. u8 status;
  474. int list_len = mr->attr.max_pages;
  475. int err = -ENOMEM;
  476. int i;
  477. if (mr->attr.page_shift < 12 || mr->attr.page_shift >= 32)
  478. return -EINVAL;
  479. /* For Arbel, all MTTs must fit in the same page. */
  480. if (mthca_is_memfree(dev) &&
  481. mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE)
  482. return -EINVAL;
  483. mr->maps = 0;
  484. key = mthca_alloc(&dev->mr_table.mpt_alloc);
  485. if (key == -1)
  486. return -ENOMEM;
  487. key = adjust_key(dev, key);
  488. idx = key & (dev->limits.num_mpts - 1);
  489. mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
  490. if (mthca_is_memfree(dev)) {
  491. err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
  492. if (err)
  493. goto err_out_mpt_free;
  494. mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key, NULL);
  495. BUG_ON(!mr->mem.arbel.mpt);
  496. } else
  497. mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base +
  498. sizeof *(mr->mem.tavor.mpt) * idx;
  499. mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy);
  500. if (IS_ERR(mr->mtt)) {
  501. err = PTR_ERR(mr->mtt);
  502. goto err_out_table;
  503. }
  504. mtt_seg = mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE;
  505. if (mthca_is_memfree(dev)) {
  506. mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table,
  507. mr->mtt->first_seg,
  508. &mr->mem.arbel.dma_handle);
  509. BUG_ON(!mr->mem.arbel.mtts);
  510. } else
  511. mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg;
  512. mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
  513. if (IS_ERR(mailbox)) {
  514. err = PTR_ERR(mailbox);
  515. goto err_out_free_mtt;
  516. }
  517. mpt_entry = mailbox->buf;
  518. mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS |
  519. MTHCA_MPT_FLAG_MIO |
  520. MTHCA_MPT_FLAG_REGION |
  521. access);
  522. mpt_entry->page_size = cpu_to_be32(mr->attr.page_shift - 12);
  523. mpt_entry->key = cpu_to_be32(key);
  524. mpt_entry->pd = cpu_to_be32(pd);
  525. memset(&mpt_entry->start, 0,
  526. sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start));
  527. mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg);
  528. if (0) {
  529. mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
  530. for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
  531. if (i % 4 == 0)
  532. printk("[%02x] ", i * 4);
  533. printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i]));
  534. if ((i + 1) % 4 == 0)
  535. printk("\n");
  536. }
  537. }
  538. err = mthca_SW2HW_MPT(dev, mailbox,
  539. key & (dev->limits.num_mpts - 1),
  540. &status);
  541. if (err) {
  542. mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
  543. goto err_out_mailbox_free;
  544. }
  545. if (status) {
  546. mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
  547. status);
  548. err = -EINVAL;
  549. goto err_out_mailbox_free;
  550. }
  551. mthca_free_mailbox(dev, mailbox);
  552. return 0;
  553. err_out_mailbox_free:
  554. mthca_free_mailbox(dev, mailbox);
  555. err_out_free_mtt:
  556. mthca_free_mtt(dev, mr->mtt);
  557. err_out_table:
  558. mthca_table_put(dev, dev->mr_table.mpt_table, key);
  559. err_out_mpt_free:
  560. mthca_free(&dev->mr_table.mpt_alloc, key);
  561. return err;
  562. }
  563. int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr)
  564. {
  565. if (fmr->maps)
  566. return -EBUSY;
  567. mthca_free_region(dev, fmr->ibmr.lkey);
  568. mthca_free_mtt(dev, fmr->mtt);
  569. return 0;
  570. }
  571. static inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list,
  572. int list_len, u64 iova)
  573. {
  574. int i, page_mask;
  575. if (list_len > fmr->attr.max_pages)
  576. return -EINVAL;
  577. page_mask = (1 << fmr->attr.page_shift) - 1;
  578. /* We are getting page lists, so va must be page aligned. */
  579. if (iova & page_mask)
  580. return -EINVAL;
  581. /* Trust the user not to pass misaligned data in page_list */
  582. if (0)
  583. for (i = 0; i < list_len; ++i) {
  584. if (page_list[i] & ~page_mask)
  585. return -EINVAL;
  586. }
  587. if (fmr->maps >= fmr->attr.max_maps)
  588. return -EINVAL;
  589. return 0;
  590. }
  591. int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
  592. int list_len, u64 iova)
  593. {
  594. struct mthca_fmr *fmr = to_mfmr(ibfmr);
  595. struct mthca_dev *dev = to_mdev(ibfmr->device);
  596. struct mthca_mpt_entry mpt_entry;
  597. u32 key;
  598. int i, err;
  599. err = mthca_check_fmr(fmr, page_list, list_len, iova);
  600. if (err)
  601. return err;
  602. ++fmr->maps;
  603. key = tavor_key_to_hw_index(fmr->ibmr.lkey);
  604. key += dev->limits.num_mpts;
  605. fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key);
  606. writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
  607. for (i = 0; i < list_len; ++i) {
  608. __be64 mtt_entry = cpu_to_be64(page_list[i] |
  609. MTHCA_MTT_FLAG_PRESENT);
  610. mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i);
  611. }
  612. mpt_entry.lkey = cpu_to_be32(key);
  613. mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift));
  614. mpt_entry.start = cpu_to_be64(iova);
  615. __raw_writel((__force u32) mpt_entry.lkey, &fmr->mem.tavor.mpt->key);
  616. memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start,
  617. offsetof(struct mthca_mpt_entry, window_count) -
  618. offsetof(struct mthca_mpt_entry, start));
  619. writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt);
  620. return 0;
  621. }
  622. int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
  623. int list_len, u64 iova)
  624. {
  625. struct mthca_fmr *fmr = to_mfmr(ibfmr);
  626. struct mthca_dev *dev = to_mdev(ibfmr->device);
  627. u32 key;
  628. int i, err;
  629. err = mthca_check_fmr(fmr, page_list, list_len, iova);
  630. if (err)
  631. return err;
  632. ++fmr->maps;
  633. key = arbel_key_to_hw_index(fmr->ibmr.lkey);
  634. if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
  635. key += SINAI_FMR_KEY_INC;
  636. else
  637. key += dev->limits.num_mpts;
  638. fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key);
  639. *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
  640. wmb();
  641. for (i = 0; i < list_len; ++i)
  642. fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] |
  643. MTHCA_MTT_FLAG_PRESENT);
  644. dma_sync_single(&dev->pdev->dev, fmr->mem.arbel.dma_handle,
  645. list_len * sizeof(u64), DMA_TO_DEVICE);
  646. fmr->mem.arbel.mpt->key = cpu_to_be32(key);
  647. fmr->mem.arbel.mpt->lkey = cpu_to_be32(key);
  648. fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift));
  649. fmr->mem.arbel.mpt->start = cpu_to_be64(iova);
  650. wmb();
  651. *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW;
  652. wmb();
  653. return 0;
  654. }
  655. void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
  656. {
  657. if (!fmr->maps)
  658. return;
  659. fmr->maps = 0;
  660. writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
  661. }
  662. void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
  663. {
  664. if (!fmr->maps)
  665. return;
  666. fmr->maps = 0;
  667. *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
  668. }
  669. int mthca_init_mr_table(struct mthca_dev *dev)
  670. {
  671. unsigned long addr;
  672. int mpts, mtts, err, i;
  673. err = mthca_alloc_init(&dev->mr_table.mpt_alloc,
  674. dev->limits.num_mpts,
  675. ~0, dev->limits.reserved_mrws);
  676. if (err)
  677. return err;
  678. if (!mthca_is_memfree(dev) &&
  679. (dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN))
  680. dev->limits.fmr_reserved_mtts = 0;
  681. else
  682. dev->mthca_flags |= MTHCA_FLAG_FMR;
  683. if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
  684. mthca_dbg(dev, "Memory key throughput optimization activated.\n");
  685. err = mthca_buddy_init(&dev->mr_table.mtt_buddy,
  686. fls(dev->limits.num_mtt_segs - 1));
  687. if (err)
  688. goto err_mtt_buddy;
  689. dev->mr_table.tavor_fmr.mpt_base = NULL;
  690. dev->mr_table.tavor_fmr.mtt_base = NULL;
  691. if (dev->limits.fmr_reserved_mtts) {
  692. i = fls(dev->limits.fmr_reserved_mtts - 1);
  693. if (i >= 31) {
  694. mthca_warn(dev, "Unable to reserve 2^31 FMR MTTs.\n");
  695. err = -EINVAL;
  696. goto err_fmr_mpt;
  697. }
  698. mpts = mtts = 1 << i;
  699. } else {
  700. mtts = dev->limits.num_mtt_segs;
  701. mpts = dev->limits.num_mpts;
  702. }
  703. if (!mthca_is_memfree(dev) &&
  704. (dev->mthca_flags & MTHCA_FLAG_FMR)) {
  705. addr = pci_resource_start(dev->pdev, 4) +
  706. ((pci_resource_len(dev->pdev, 4) - 1) &
  707. dev->mr_table.mpt_base);
  708. dev->mr_table.tavor_fmr.mpt_base =
  709. ioremap(addr, mpts * sizeof(struct mthca_mpt_entry));
  710. if (!dev->mr_table.tavor_fmr.mpt_base) {
  711. mthca_warn(dev, "MPT ioremap for FMR failed.\n");
  712. err = -ENOMEM;
  713. goto err_fmr_mpt;
  714. }
  715. addr = pci_resource_start(dev->pdev, 4) +
  716. ((pci_resource_len(dev->pdev, 4) - 1) &
  717. dev->mr_table.mtt_base);
  718. dev->mr_table.tavor_fmr.mtt_base =
  719. ioremap(addr, mtts * MTHCA_MTT_SEG_SIZE);
  720. if (!dev->mr_table.tavor_fmr.mtt_base) {
  721. mthca_warn(dev, "MTT ioremap for FMR failed.\n");
  722. err = -ENOMEM;
  723. goto err_fmr_mtt;
  724. }
  725. }
  726. if (dev->limits.fmr_reserved_mtts) {
  727. err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, fls(mtts - 1));
  728. if (err)
  729. goto err_fmr_mtt_buddy;
  730. /* Prevent regular MRs from using FMR keys */
  731. err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, fls(mtts - 1));
  732. if (err)
  733. goto err_reserve_fmr;
  734. dev->mr_table.fmr_mtt_buddy =
  735. &dev->mr_table.tavor_fmr.mtt_buddy;
  736. } else
  737. dev->mr_table.fmr_mtt_buddy = &dev->mr_table.mtt_buddy;
  738. /* FMR table is always the first, take reserved MTTs out of there */
  739. if (dev->limits.reserved_mtts) {
  740. i = fls(dev->limits.reserved_mtts - 1);
  741. if (mthca_alloc_mtt_range(dev, i,
  742. dev->mr_table.fmr_mtt_buddy) == -1) {
  743. mthca_warn(dev, "MTT table of order %d is too small.\n",
  744. dev->mr_table.fmr_mtt_buddy->max_order);
  745. err = -ENOMEM;
  746. goto err_reserve_mtts;
  747. }
  748. }
  749. return 0;
  750. err_reserve_mtts:
  751. err_reserve_fmr:
  752. if (dev->limits.fmr_reserved_mtts)
  753. mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy);
  754. err_fmr_mtt_buddy:
  755. if (dev->mr_table.tavor_fmr.mtt_base)
  756. iounmap(dev->mr_table.tavor_fmr.mtt_base);
  757. err_fmr_mtt:
  758. if (dev->mr_table.tavor_fmr.mpt_base)
  759. iounmap(dev->mr_table.tavor_fmr.mpt_base);
  760. err_fmr_mpt:
  761. mthca_buddy_cleanup(&dev->mr_table.mtt_buddy);
  762. err_mtt_buddy:
  763. mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
  764. return err;
  765. }
  766. void mthca_cleanup_mr_table(struct mthca_dev *dev)
  767. {
  768. /* XXX check if any MRs are still allocated? */
  769. if (dev->limits.fmr_reserved_mtts)
  770. mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy);
  771. mthca_buddy_cleanup(&dev->mr_table.mtt_buddy);
  772. if (dev->mr_table.tavor_fmr.mtt_base)
  773. iounmap(dev->mr_table.tavor_fmr.mtt_base);
  774. if (dev->mr_table.tavor_fmr.mpt_base)
  775. iounmap(dev->mr_table.tavor_fmr.mpt_base);
  776. mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
  777. }