mthca_mr.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473
  1. /*
  2. * Copyright (c) 2004 Topspin Communications. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. *
  32. * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $
  33. */
  34. #include <linux/slab.h>
  35. #include <linux/init.h>
  36. #include <linux/errno.h>
  37. #include "mthca_dev.h"
  38. #include "mthca_cmd.h"
  39. #include "mthca_memfree.h"
  40. /*
  41. * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
  42. */
  43. struct mthca_mpt_entry {
  44. u32 flags;
  45. u32 page_size;
  46. u32 key;
  47. u32 pd;
  48. u64 start;
  49. u64 length;
  50. u32 lkey;
  51. u32 window_count;
  52. u32 window_count_limit;
  53. u64 mtt_seg;
  54. u32 mtt_sz; /* Arbel only */
  55. u32 reserved[2];
  56. } __attribute__((packed));
  57. #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28)
  58. #define MTHCA_MPT_FLAG_MIO (1 << 17)
  59. #define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15)
  60. #define MTHCA_MPT_FLAG_PHYSICAL (1 << 9)
  61. #define MTHCA_MPT_FLAG_REGION (1 << 8)
  62. #define MTHCA_MTT_FLAG_PRESENT 1
  63. /*
  64. * Buddy allocator for MTT segments (currently not very efficient
  65. * since it doesn't keep a free list and just searches linearly
  66. * through the bitmaps)
  67. */
  68. static u32 __mthca_alloc_mtt(struct mthca_dev *dev, int order)
  69. {
  70. int o;
  71. int m;
  72. u32 seg;
  73. spin_lock(&dev->mr_table.mpt_alloc.lock);
  74. for (o = order; o <= dev->mr_table.max_mtt_order; ++o) {
  75. m = 1 << (dev->mr_table.max_mtt_order - o);
  76. seg = find_first_bit(dev->mr_table.mtt_buddy[o], m);
  77. if (seg < m)
  78. goto found;
  79. }
  80. spin_unlock(&dev->mr_table.mpt_alloc.lock);
  81. return -1;
  82. found:
  83. clear_bit(seg, dev->mr_table.mtt_buddy[o]);
  84. while (o > order) {
  85. --o;
  86. seg <<= 1;
  87. set_bit(seg ^ 1, dev->mr_table.mtt_buddy[o]);
  88. }
  89. spin_unlock(&dev->mr_table.mpt_alloc.lock);
  90. seg <<= order;
  91. return seg;
  92. }
  93. static void __mthca_free_mtt(struct mthca_dev *dev, u32 seg, int order)
  94. {
  95. seg >>= order;
  96. spin_lock(&dev->mr_table.mpt_alloc.lock);
  97. while (test_bit(seg ^ 1, dev->mr_table.mtt_buddy[order])) {
  98. clear_bit(seg ^ 1, dev->mr_table.mtt_buddy[order]);
  99. seg >>= 1;
  100. ++order;
  101. }
  102. set_bit(seg, dev->mr_table.mtt_buddy[order]);
  103. spin_unlock(&dev->mr_table.mpt_alloc.lock);
  104. }
  105. static u32 mthca_alloc_mtt(struct mthca_dev *dev, int order)
  106. {
  107. u32 seg = __mthca_alloc_mtt(dev, order);
  108. if (seg == -1)
  109. return -1;
  110. if (dev->hca_type == ARBEL_NATIVE)
  111. if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg,
  112. seg + (1 << order) - 1)) {
  113. __mthca_free_mtt(dev, seg, order);
  114. seg = -1;
  115. }
  116. return seg;
  117. }
  118. static void mthca_free_mtt(struct mthca_dev *dev, u32 seg, int order)
  119. {
  120. __mthca_free_mtt(dev, seg, order);
  121. if (dev->hca_type == ARBEL_NATIVE)
  122. mthca_table_put_range(dev, dev->mr_table.mtt_table, seg,
  123. seg + (1 << order) - 1);
  124. }
  125. static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind)
  126. {
  127. if (dev->hca_type == ARBEL_NATIVE)
  128. return (ind >> 24) | (ind << 8);
  129. else
  130. return ind;
  131. }
  132. static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key)
  133. {
  134. if (dev->hca_type == ARBEL_NATIVE)
  135. return (key << 24) | (key >> 8);
  136. else
  137. return key;
  138. }
  139. int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd,
  140. u32 access, struct mthca_mr *mr)
  141. {
  142. void *mailbox = NULL;
  143. struct mthca_mpt_entry *mpt_entry;
  144. u32 key;
  145. int err;
  146. u8 status;
  147. might_sleep();
  148. mr->order = -1;
  149. key = mthca_alloc(&dev->mr_table.mpt_alloc);
  150. if (key == -1)
  151. return -ENOMEM;
  152. mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
  153. if (dev->hca_type == ARBEL_NATIVE) {
  154. err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
  155. if (err)
  156. goto err_out_mpt_free;
  157. }
  158. mailbox = kmalloc(sizeof *mpt_entry + MTHCA_CMD_MAILBOX_EXTRA,
  159. GFP_KERNEL);
  160. if (!mailbox) {
  161. err = -ENOMEM;
  162. goto err_out_table;
  163. }
  164. mpt_entry = MAILBOX_ALIGN(mailbox);
  165. mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS |
  166. MTHCA_MPT_FLAG_MIO |
  167. MTHCA_MPT_FLAG_PHYSICAL |
  168. MTHCA_MPT_FLAG_REGION |
  169. access);
  170. mpt_entry->page_size = 0;
  171. mpt_entry->key = cpu_to_be32(key);
  172. mpt_entry->pd = cpu_to_be32(pd);
  173. mpt_entry->start = 0;
  174. mpt_entry->length = ~0ULL;
  175. memset(&mpt_entry->lkey, 0,
  176. sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
  177. err = mthca_SW2HW_MPT(dev, mpt_entry,
  178. key & (dev->limits.num_mpts - 1),
  179. &status);
  180. if (err) {
  181. mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
  182. goto err_out_table;
  183. } else if (status) {
  184. mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
  185. status);
  186. err = -EINVAL;
  187. goto err_out_table;
  188. }
  189. kfree(mailbox);
  190. return err;
  191. err_out_table:
  192. if (dev->hca_type == ARBEL_NATIVE)
  193. mthca_table_put(dev, dev->mr_table.mpt_table, key);
  194. err_out_mpt_free:
  195. mthca_free(&dev->mr_table.mpt_alloc, key);
  196. kfree(mailbox);
  197. return err;
  198. }
  199. int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
  200. u64 *buffer_list, int buffer_size_shift,
  201. int list_len, u64 iova, u64 total_size,
  202. u32 access, struct mthca_mr *mr)
  203. {
  204. void *mailbox;
  205. u64 *mtt_entry;
  206. struct mthca_mpt_entry *mpt_entry;
  207. u32 key;
  208. int err = -ENOMEM;
  209. u8 status;
  210. int i;
  211. might_sleep();
  212. WARN_ON(buffer_size_shift >= 32);
  213. key = mthca_alloc(&dev->mr_table.mpt_alloc);
  214. if (key == -1)
  215. return -ENOMEM;
  216. mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
  217. if (dev->hca_type == ARBEL_NATIVE) {
  218. err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
  219. if (err)
  220. goto err_out_mpt_free;
  221. }
  222. for (i = dev->limits.mtt_seg_size / 8, mr->order = 0;
  223. i < list_len;
  224. i <<= 1, ++mr->order)
  225. ; /* nothing */
  226. mr->first_seg = mthca_alloc_mtt(dev, mr->order);
  227. if (mr->first_seg == -1)
  228. goto err_out_table;
  229. /*
  230. * If list_len is odd, we add one more dummy entry for
  231. * firmware efficiency.
  232. */
  233. mailbox = kmalloc(max(sizeof *mpt_entry,
  234. (size_t) 8 * (list_len + (list_len & 1) + 2)) +
  235. MTHCA_CMD_MAILBOX_EXTRA,
  236. GFP_KERNEL);
  237. if (!mailbox)
  238. goto err_out_free_mtt;
  239. mtt_entry = MAILBOX_ALIGN(mailbox);
  240. mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base +
  241. mr->first_seg * dev->limits.mtt_seg_size);
  242. mtt_entry[1] = 0;
  243. for (i = 0; i < list_len; ++i)
  244. mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] |
  245. MTHCA_MTT_FLAG_PRESENT);
  246. if (list_len & 1) {
  247. mtt_entry[i + 2] = 0;
  248. ++list_len;
  249. }
  250. if (0) {
  251. mthca_dbg(dev, "Dumping MPT entry\n");
  252. for (i = 0; i < list_len + 2; ++i)
  253. printk(KERN_ERR "[%2d] %016llx\n",
  254. i, (unsigned long long) be64_to_cpu(mtt_entry[i]));
  255. }
  256. err = mthca_WRITE_MTT(dev, mtt_entry, list_len, &status);
  257. if (err) {
  258. mthca_warn(dev, "WRITE_MTT failed (%d)\n", err);
  259. goto err_out_mailbox_free;
  260. }
  261. if (status) {
  262. mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n",
  263. status);
  264. err = -EINVAL;
  265. goto err_out_mailbox_free;
  266. }
  267. mpt_entry = MAILBOX_ALIGN(mailbox);
  268. mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS |
  269. MTHCA_MPT_FLAG_MIO |
  270. MTHCA_MPT_FLAG_REGION |
  271. access);
  272. mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12);
  273. mpt_entry->key = cpu_to_be32(key);
  274. mpt_entry->pd = cpu_to_be32(pd);
  275. mpt_entry->start = cpu_to_be64(iova);
  276. mpt_entry->length = cpu_to_be64(total_size);
  277. memset(&mpt_entry->lkey, 0,
  278. sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
  279. mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base +
  280. mr->first_seg * dev->limits.mtt_seg_size);
  281. if (0) {
  282. mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
  283. for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
  284. if (i % 4 == 0)
  285. printk("[%02x] ", i * 4);
  286. printk(" %08x", be32_to_cpu(((u32 *) mpt_entry)[i]));
  287. if ((i + 1) % 4 == 0)
  288. printk("\n");
  289. }
  290. }
  291. err = mthca_SW2HW_MPT(dev, mpt_entry,
  292. key & (dev->limits.num_mpts - 1),
  293. &status);
  294. if (err)
  295. mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
  296. else if (status) {
  297. mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
  298. status);
  299. err = -EINVAL;
  300. }
  301. kfree(mailbox);
  302. return err;
  303. err_out_mailbox_free:
  304. kfree(mailbox);
  305. err_out_free_mtt:
  306. mthca_free_mtt(dev, mr->first_seg, mr->order);
  307. err_out_table:
  308. if (dev->hca_type == ARBEL_NATIVE)
  309. mthca_table_put(dev, dev->mr_table.mpt_table, key);
  310. err_out_mpt_free:
  311. mthca_free(&dev->mr_table.mpt_alloc, key);
  312. return err;
  313. }
  314. void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr)
  315. {
  316. int err;
  317. u8 status;
  318. might_sleep();
  319. err = mthca_HW2SW_MPT(dev, NULL,
  320. key_to_hw_index(dev, mr->ibmr.lkey) &
  321. (dev->limits.num_mpts - 1),
  322. &status);
  323. if (err)
  324. mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err);
  325. else if (status)
  326. mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n",
  327. status);
  328. if (mr->order >= 0)
  329. mthca_free_mtt(dev, mr->first_seg, mr->order);
  330. if (dev->hca_type == ARBEL_NATIVE)
  331. mthca_table_put(dev, dev->mr_table.mpt_table,
  332. key_to_hw_index(dev, mr->ibmr.lkey));
  333. mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, mr->ibmr.lkey));
  334. }
  335. int __devinit mthca_init_mr_table(struct mthca_dev *dev)
  336. {
  337. int err;
  338. int i, s;
  339. err = mthca_alloc_init(&dev->mr_table.mpt_alloc,
  340. dev->limits.num_mpts,
  341. ~0, dev->limits.reserved_mrws);
  342. if (err)
  343. return err;
  344. err = -ENOMEM;
  345. for (i = 1, dev->mr_table.max_mtt_order = 0;
  346. i < dev->limits.num_mtt_segs;
  347. i <<= 1, ++dev->mr_table.max_mtt_order)
  348. ; /* nothing */
  349. dev->mr_table.mtt_buddy = kmalloc((dev->mr_table.max_mtt_order + 1) *
  350. sizeof (long *),
  351. GFP_KERNEL);
  352. if (!dev->mr_table.mtt_buddy)
  353. goto err_out;
  354. for (i = 0; i <= dev->mr_table.max_mtt_order; ++i)
  355. dev->mr_table.mtt_buddy[i] = NULL;
  356. for (i = 0; i <= dev->mr_table.max_mtt_order; ++i) {
  357. s = BITS_TO_LONGS(1 << (dev->mr_table.max_mtt_order - i));
  358. dev->mr_table.mtt_buddy[i] = kmalloc(s * sizeof (long),
  359. GFP_KERNEL);
  360. if (!dev->mr_table.mtt_buddy[i])
  361. goto err_out_free;
  362. bitmap_zero(dev->mr_table.mtt_buddy[i],
  363. 1 << (dev->mr_table.max_mtt_order - i));
  364. }
  365. set_bit(0, dev->mr_table.mtt_buddy[dev->mr_table.max_mtt_order]);
  366. for (i = 0; i < dev->mr_table.max_mtt_order; ++i)
  367. if (1 << i >= dev->limits.reserved_mtts)
  368. break;
  369. if (i == dev->mr_table.max_mtt_order) {
  370. mthca_err(dev, "MTT table of order %d is "
  371. "too small.\n", i);
  372. goto err_out_free;
  373. }
  374. (void) mthca_alloc_mtt(dev, i);
  375. return 0;
  376. err_out_free:
  377. for (i = 0; i <= dev->mr_table.max_mtt_order; ++i)
  378. kfree(dev->mr_table.mtt_buddy[i]);
  379. err_out:
  380. mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
  381. return err;
  382. }
  383. void __devexit mthca_cleanup_mr_table(struct mthca_dev *dev)
  384. {
  385. int i;
  386. /* XXX check if any MRs are still allocated? */
  387. for (i = 0; i <= dev->mr_table.max_mtt_order; ++i)
  388. kfree(dev->mr_table.mtt_buddy[i]);
  389. kfree(dev->mr_table.mtt_buddy);
  390. mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
  391. }