xprt_rdma.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337
  1. /*
  2. * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the BSD-type
  8. * license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or without
  11. * modification, are permitted provided that the following conditions
  12. * are met:
  13. *
  14. * Redistributions of source code must retain the above copyright
  15. * notice, this list of conditions and the following disclaimer.
  16. *
  17. * Redistributions in binary form must reproduce the above
  18. * copyright notice, this list of conditions and the following
  19. * disclaimer in the documentation and/or other materials provided
  20. * with the distribution.
  21. *
  22. * Neither the name of the Network Appliance, Inc. nor the names of
  23. * its contributors may be used to endorse or promote products
  24. * derived from this software without specific prior written
  25. * permission.
  26. *
  27. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  28. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  29. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  30. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  31. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  32. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  33. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  34. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  35. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  36. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  37. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  38. */
  39. #ifndef _LINUX_SUNRPC_XPRT_RDMA_H
  40. #define _LINUX_SUNRPC_XPRT_RDMA_H
  41. #include <linux/wait.h> /* wait_queue_head_t, etc */
  42. #include <linux/spinlock.h> /* spinlock_t, etc */
  43. #include <asm/atomic.h> /* atomic_t, etc */
  44. #include <rdma/rdma_cm.h> /* RDMA connection api */
  45. #include <rdma/ib_verbs.h> /* RDMA verbs api */
  46. #include <linux/sunrpc/clnt.h> /* rpc_xprt */
  47. #include <linux/sunrpc/rpc_rdma.h> /* RPC/RDMA protocol */
  48. #include <linux/sunrpc/xprtrdma.h> /* xprt parameters */
  49. /*
  50. * Interface Adapter -- one per transport instance
  51. */
  52. struct rpcrdma_ia {
  53. struct rdma_cm_id *ri_id;
  54. struct ib_pd *ri_pd;
  55. struct ib_mr *ri_bind_mem;
  56. u32 ri_dma_lkey;
  57. int ri_have_dma_lkey;
  58. struct completion ri_done;
  59. int ri_async_rc;
  60. enum rpcrdma_memreg ri_memreg_strategy;
  61. };
  62. /*
  63. * RDMA Endpoint -- one per transport instance
  64. */
  65. struct rpcrdma_ep {
  66. atomic_t rep_cqcount;
  67. int rep_cqinit;
  68. int rep_connected;
  69. struct rpcrdma_ia *rep_ia;
  70. struct ib_cq *rep_cq;
  71. struct ib_qp_init_attr rep_attr;
  72. wait_queue_head_t rep_connect_wait;
  73. struct ib_sge rep_pad; /* holds zeroed pad */
  74. struct ib_mr *rep_pad_mr; /* holds zeroed pad */
  75. void (*rep_func)(struct rpcrdma_ep *);
  76. struct rpc_xprt *rep_xprt; /* for rep_func */
  77. struct rdma_conn_param rep_remote_cma;
  78. struct sockaddr_storage rep_remote_addr;
  79. };
  80. #define INIT_CQCOUNT(ep) atomic_set(&(ep)->rep_cqcount, (ep)->rep_cqinit)
  81. #define DECR_CQCOUNT(ep) atomic_sub_return(1, &(ep)->rep_cqcount)
  82. /*
  83. * struct rpcrdma_rep -- this structure encapsulates state required to recv
  84. * and complete a reply, asychronously. It needs several pieces of
  85. * state:
  86. * o recv buffer (posted to provider)
  87. * o ib_sge (also donated to provider)
  88. * o status of reply (length, success or not)
  89. * o bookkeeping state to get run by tasklet (list, etc)
  90. *
  91. * These are allocated during initialization, per-transport instance;
  92. * however, the tasklet execution list itself is global, as it should
  93. * always be pretty short.
  94. *
  95. * N of these are associated with a transport instance, and stored in
  96. * struct rpcrdma_buffer. N is the max number of outstanding requests.
  97. */
  98. /* temporary static scatter/gather max */
  99. #define RPCRDMA_MAX_DATA_SEGS (8) /* max scatter/gather */
  100. #define RPCRDMA_MAX_SEGS (RPCRDMA_MAX_DATA_SEGS + 2) /* head+tail = 2 */
  101. #define MAX_RPCRDMAHDR (\
  102. /* max supported RPC/RDMA header */ \
  103. sizeof(struct rpcrdma_msg) + (2 * sizeof(u32)) + \
  104. (sizeof(struct rpcrdma_read_chunk) * RPCRDMA_MAX_SEGS) + sizeof(u32))
  105. struct rpcrdma_buffer;
  106. struct rpcrdma_rep {
  107. unsigned int rr_len; /* actual received reply length */
  108. struct rpcrdma_buffer *rr_buffer; /* home base for this structure */
  109. struct rpc_xprt *rr_xprt; /* needed for request/reply matching */
  110. void (*rr_func)(struct rpcrdma_rep *);/* called by tasklet in softint */
  111. struct list_head rr_list; /* tasklet list */
  112. wait_queue_head_t rr_unbind; /* optional unbind wait */
  113. struct ib_sge rr_iov; /* for posting */
  114. struct ib_mr *rr_handle; /* handle for mem in rr_iov */
  115. char rr_base[MAX_RPCRDMAHDR]; /* minimal inline receive buffer */
  116. };
  117. /*
  118. * struct rpcrdma_req -- structure central to the request/reply sequence.
  119. *
  120. * N of these are associated with a transport instance, and stored in
  121. * struct rpcrdma_buffer. N is the max number of outstanding requests.
  122. *
  123. * It includes pre-registered buffer memory for send AND recv.
  124. * The recv buffer, however, is not owned by this structure, and
  125. * is "donated" to the hardware when a recv is posted. When a
  126. * reply is handled, the recv buffer used is given back to the
  127. * struct rpcrdma_req associated with the request.
  128. *
  129. * In addition to the basic memory, this structure includes an array
  130. * of iovs for send operations. The reason is that the iovs passed to
  131. * ib_post_{send,recv} must not be modified until the work request
  132. * completes.
  133. *
  134. * NOTES:
  135. * o RPCRDMA_MAX_SEGS is the max number of addressible chunk elements we
  136. * marshal. The number needed varies depending on the iov lists that
  137. * are passed to us, the memory registration mode we are in, and if
  138. * physical addressing is used, the layout.
  139. */
  140. struct rpcrdma_mr_seg { /* chunk descriptors */
  141. union { /* chunk memory handles */
  142. struct ib_mr *rl_mr; /* if registered directly */
  143. struct rpcrdma_mw { /* if registered from region */
  144. union {
  145. struct ib_mw *mw;
  146. struct ib_fmr *fmr;
  147. struct {
  148. struct ib_fast_reg_page_list *fr_pgl;
  149. struct ib_mr *fr_mr;
  150. } frmr;
  151. } r;
  152. struct list_head mw_list;
  153. } *rl_mw;
  154. } mr_chunk;
  155. u64 mr_base; /* registration result */
  156. u32 mr_rkey; /* registration result */
  157. u32 mr_len; /* length of chunk or segment */
  158. int mr_nsegs; /* number of segments in chunk or 0 */
  159. enum dma_data_direction mr_dir; /* segment mapping direction */
  160. dma_addr_t mr_dma; /* segment mapping address */
  161. size_t mr_dmalen; /* segment mapping length */
  162. struct page *mr_page; /* owning page, if any */
  163. char *mr_offset; /* kva if no page, else offset */
  164. };
  165. struct rpcrdma_req {
  166. size_t rl_size; /* actual length of buffer */
  167. unsigned int rl_niovs; /* 0, 2 or 4 */
  168. unsigned int rl_nchunks; /* non-zero if chunks */
  169. unsigned int rl_connect_cookie; /* retry detection */
  170. struct rpcrdma_buffer *rl_buffer; /* home base for this structure */
  171. struct rpcrdma_rep *rl_reply;/* holder for reply buffer */
  172. struct rpcrdma_mr_seg rl_segments[RPCRDMA_MAX_SEGS];/* chunk segments */
  173. struct ib_sge rl_send_iov[4]; /* for active requests */
  174. struct ib_sge rl_iov; /* for posting */
  175. struct ib_mr *rl_handle; /* handle for mem in rl_iov */
  176. char rl_base[MAX_RPCRDMAHDR]; /* start of actual buffer */
  177. __u32 rl_xdr_buf[0]; /* start of returned rpc rq_buffer */
  178. };
  179. #define rpcr_to_rdmar(r) \
  180. container_of((r)->rq_buffer, struct rpcrdma_req, rl_xdr_buf[0])
  181. /*
  182. * struct rpcrdma_buffer -- holds list/queue of pre-registered memory for
  183. * inline requests/replies, and client/server credits.
  184. *
  185. * One of these is associated with a transport instance
  186. */
  187. struct rpcrdma_buffer {
  188. spinlock_t rb_lock; /* protects indexes */
  189. atomic_t rb_credits; /* most recent server credits */
  190. unsigned long rb_cwndscale; /* cached framework rpc_cwndscale */
  191. int rb_max_requests;/* client max requests */
  192. struct list_head rb_mws; /* optional memory windows/fmrs/frmrs */
  193. int rb_send_index;
  194. struct rpcrdma_req **rb_send_bufs;
  195. int rb_recv_index;
  196. struct rpcrdma_rep **rb_recv_bufs;
  197. char *rb_pool;
  198. };
  199. #define rdmab_to_ia(b) (&container_of((b), struct rpcrdma_xprt, rx_buf)->rx_ia)
  200. /*
  201. * Internal structure for transport instance creation. This
  202. * exists primarily for modularity.
  203. *
  204. * This data should be set with mount options
  205. */
  206. struct rpcrdma_create_data_internal {
  207. struct sockaddr_storage addr; /* RDMA server address */
  208. unsigned int max_requests; /* max requests (slots) in flight */
  209. unsigned int rsize; /* mount rsize - max read hdr+data */
  210. unsigned int wsize; /* mount wsize - max write hdr+data */
  211. unsigned int inline_rsize; /* max non-rdma read data payload */
  212. unsigned int inline_wsize; /* max non-rdma write data payload */
  213. unsigned int padding; /* non-rdma write header padding */
  214. };
  215. #define RPCRDMA_INLINE_READ_THRESHOLD(rq) \
  216. (rpcx_to_rdmad(rq->rq_task->tk_xprt).inline_rsize)
  217. #define RPCRDMA_INLINE_WRITE_THRESHOLD(rq)\
  218. (rpcx_to_rdmad(rq->rq_task->tk_xprt).inline_wsize)
  219. #define RPCRDMA_INLINE_PAD_VALUE(rq)\
  220. rpcx_to_rdmad(rq->rq_task->tk_xprt).padding
  221. /*
  222. * Statistics for RPCRDMA
  223. */
  224. struct rpcrdma_stats {
  225. unsigned long read_chunk_count;
  226. unsigned long write_chunk_count;
  227. unsigned long reply_chunk_count;
  228. unsigned long long total_rdma_request;
  229. unsigned long long total_rdma_reply;
  230. unsigned long long pullup_copy_count;
  231. unsigned long long fixup_copy_count;
  232. unsigned long hardway_register_count;
  233. unsigned long failed_marshal_count;
  234. unsigned long bad_reply_count;
  235. };
  236. /*
  237. * RPCRDMA transport -- encapsulates the structures above for
  238. * integration with RPC.
  239. *
  240. * The contained structures are embedded, not pointers,
  241. * for convenience. This structure need not be visible externally.
  242. *
  243. * It is allocated and initialized during mount, and released
  244. * during unmount.
  245. */
  246. struct rpcrdma_xprt {
  247. struct rpc_xprt xprt;
  248. struct rpcrdma_ia rx_ia;
  249. struct rpcrdma_ep rx_ep;
  250. struct rpcrdma_buffer rx_buf;
  251. struct rpcrdma_create_data_internal rx_data;
  252. struct delayed_work rdma_connect;
  253. struct rpcrdma_stats rx_stats;
  254. };
  255. #define rpcx_to_rdmax(x) container_of(x, struct rpcrdma_xprt, xprt)
  256. #define rpcx_to_rdmad(x) (rpcx_to_rdmax(x)->rx_data)
  257. /*
  258. * Interface Adapter calls - xprtrdma/verbs.c
  259. */
  260. int rpcrdma_ia_open(struct rpcrdma_xprt *, struct sockaddr *, int);
  261. void rpcrdma_ia_close(struct rpcrdma_ia *);
  262. /*
  263. * Endpoint calls - xprtrdma/verbs.c
  264. */
  265. int rpcrdma_ep_create(struct rpcrdma_ep *, struct rpcrdma_ia *,
  266. struct rpcrdma_create_data_internal *);
  267. int rpcrdma_ep_destroy(struct rpcrdma_ep *, struct rpcrdma_ia *);
  268. int rpcrdma_ep_connect(struct rpcrdma_ep *, struct rpcrdma_ia *);
  269. int rpcrdma_ep_disconnect(struct rpcrdma_ep *, struct rpcrdma_ia *);
  270. int rpcrdma_ep_post(struct rpcrdma_ia *, struct rpcrdma_ep *,
  271. struct rpcrdma_req *);
  272. int rpcrdma_ep_post_recv(struct rpcrdma_ia *, struct rpcrdma_ep *,
  273. struct rpcrdma_rep *);
  274. /*
  275. * Buffer calls - xprtrdma/verbs.c
  276. */
  277. int rpcrdma_buffer_create(struct rpcrdma_buffer *, struct rpcrdma_ep *,
  278. struct rpcrdma_ia *,
  279. struct rpcrdma_create_data_internal *);
  280. void rpcrdma_buffer_destroy(struct rpcrdma_buffer *);
  281. struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *);
  282. void rpcrdma_buffer_put(struct rpcrdma_req *);
  283. void rpcrdma_recv_buffer_get(struct rpcrdma_req *);
  284. void rpcrdma_recv_buffer_put(struct rpcrdma_rep *);
  285. int rpcrdma_register_internal(struct rpcrdma_ia *, void *, int,
  286. struct ib_mr **, struct ib_sge *);
  287. int rpcrdma_deregister_internal(struct rpcrdma_ia *,
  288. struct ib_mr *, struct ib_sge *);
  289. int rpcrdma_register_external(struct rpcrdma_mr_seg *,
  290. int, int, struct rpcrdma_xprt *);
  291. int rpcrdma_deregister_external(struct rpcrdma_mr_seg *,
  292. struct rpcrdma_xprt *, void *);
  293. /*
  294. * RPC/RDMA connection management calls - xprtrdma/rpc_rdma.c
  295. */
  296. void rpcrdma_conn_func(struct rpcrdma_ep *);
  297. void rpcrdma_reply_handler(struct rpcrdma_rep *);
  298. /*
  299. * RPC/RDMA protocol calls - xprtrdma/rpc_rdma.c
  300. */
  301. int rpcrdma_marshal_req(struct rpc_rqst *);
  302. #endif /* _LINUX_SUNRPC_XPRT_RDMA_H */