cma.c 89 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549
  1. /*
  2. * Copyright (c) 2005 Voltaire Inc. All rights reserved.
  3. * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
  4. * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
  5. * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
  6. *
  7. * This software is available to you under a choice of one of two
  8. * licenses. You may choose to be licensed under the terms of the GNU
  9. * General Public License (GPL) Version 2, available from the file
  10. * COPYING in the main directory of this source tree, or the
  11. * OpenIB.org BSD license below:
  12. *
  13. * Redistribution and use in source and binary forms, with or
  14. * without modification, are permitted provided that the following
  15. * conditions are met:
  16. *
  17. * - Redistributions of source code must retain the above
  18. * copyright notice, this list of conditions and the following
  19. * disclaimer.
  20. *
  21. * - Redistributions in binary form must reproduce the above
  22. * copyright notice, this list of conditions and the following
  23. * disclaimer in the documentation and/or other materials
  24. * provided with the distribution.
  25. *
  26. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33. * SOFTWARE.
  34. */
  35. #include <linux/completion.h>
  36. #include <linux/in.h>
  37. #include <linux/in6.h>
  38. #include <linux/mutex.h>
  39. #include <linux/random.h>
  40. #include <linux/idr.h>
  41. #include <linux/inetdevice.h>
  42. #include <linux/slab.h>
  43. #include <linux/module.h>
  44. #include <net/route.h>
  45. #include <net/tcp.h>
  46. #include <net/ipv6.h>
  47. #include <rdma/rdma_cm.h>
  48. #include <rdma/rdma_cm_ib.h>
  49. #include <rdma/rdma_netlink.h>
  50. #include <rdma/ib_cache.h>
  51. #include <rdma/ib_cm.h>
  52. #include <rdma/ib_sa.h>
  53. #include <rdma/iw_cm.h>
  54. MODULE_AUTHOR("Sean Hefty");
  55. MODULE_DESCRIPTION("Generic RDMA CM Agent");
  56. MODULE_LICENSE("Dual BSD/GPL");
  57. #define CMA_CM_RESPONSE_TIMEOUT 20
  58. #define CMA_MAX_CM_RETRIES 15
  59. #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  60. #define CMA_IBOE_PACKET_LIFETIME 18
  61. static void cma_add_one(struct ib_device *device);
  62. static void cma_remove_one(struct ib_device *device);
  63. static struct ib_client cma_client = {
  64. .name = "cma",
  65. .add = cma_add_one,
  66. .remove = cma_remove_one
  67. };
  68. static struct ib_sa_client sa_client;
  69. static struct rdma_addr_client addr_client;
  70. static LIST_HEAD(dev_list);
  71. static LIST_HEAD(listen_any_list);
  72. static DEFINE_MUTEX(lock);
  73. static struct workqueue_struct *cma_wq;
  74. static DEFINE_IDR(sdp_ps);
  75. static DEFINE_IDR(tcp_ps);
  76. static DEFINE_IDR(udp_ps);
  77. static DEFINE_IDR(ipoib_ps);
  78. static DEFINE_IDR(ib_ps);
  79. struct cma_device {
  80. struct list_head list;
  81. struct ib_device *device;
  82. struct completion comp;
  83. atomic_t refcount;
  84. struct list_head id_list;
  85. };
  86. struct rdma_bind_list {
  87. struct idr *ps;
  88. struct hlist_head owners;
  89. unsigned short port;
  90. };
  91. enum {
  92. CMA_OPTION_AFONLY,
  93. };
  94. /*
  95. * Device removal can occur at anytime, so we need extra handling to
  96. * serialize notifying the user of device removal with other callbacks.
  97. * We do this by disabling removal notification while a callback is in process,
  98. * and reporting it after the callback completes.
  99. */
  100. struct rdma_id_private {
  101. struct rdma_cm_id id;
  102. struct rdma_bind_list *bind_list;
  103. struct hlist_node node;
  104. struct list_head list; /* listen_any_list or cma_device.list */
  105. struct list_head listen_list; /* per device listens */
  106. struct cma_device *cma_dev;
  107. struct list_head mc_list;
  108. int internal_id;
  109. enum rdma_cm_state state;
  110. spinlock_t lock;
  111. struct mutex qp_mutex;
  112. struct completion comp;
  113. atomic_t refcount;
  114. struct mutex handler_mutex;
  115. int backlog;
  116. int timeout_ms;
  117. struct ib_sa_query *query;
  118. int query_id;
  119. union {
  120. struct ib_cm_id *ib;
  121. struct iw_cm_id *iw;
  122. } cm_id;
  123. u32 seq_num;
  124. u32 qkey;
  125. u32 qp_num;
  126. pid_t owner;
  127. u32 options;
  128. u8 srq;
  129. u8 tos;
  130. u8 reuseaddr;
  131. u8 afonly;
  132. };
  133. struct cma_multicast {
  134. struct rdma_id_private *id_priv;
  135. union {
  136. struct ib_sa_multicast *ib;
  137. } multicast;
  138. struct list_head list;
  139. void *context;
  140. struct sockaddr_storage addr;
  141. struct kref mcref;
  142. };
  143. struct cma_work {
  144. struct work_struct work;
  145. struct rdma_id_private *id;
  146. enum rdma_cm_state old_state;
  147. enum rdma_cm_state new_state;
  148. struct rdma_cm_event event;
  149. };
  150. struct cma_ndev_work {
  151. struct work_struct work;
  152. struct rdma_id_private *id;
  153. struct rdma_cm_event event;
  154. };
  155. struct iboe_mcast_work {
  156. struct work_struct work;
  157. struct rdma_id_private *id;
  158. struct cma_multicast *mc;
  159. };
  160. union cma_ip_addr {
  161. struct in6_addr ip6;
  162. struct {
  163. __be32 pad[3];
  164. __be32 addr;
  165. } ip4;
  166. };
  167. struct cma_hdr {
  168. u8 cma_version;
  169. u8 ip_version; /* IP version: 7:4 */
  170. __be16 port;
  171. union cma_ip_addr src_addr;
  172. union cma_ip_addr dst_addr;
  173. };
  174. struct sdp_hh {
  175. u8 bsdh[16];
  176. u8 sdp_version; /* Major version: 7:4 */
  177. u8 ip_version; /* IP version: 7:4 */
  178. u8 sdp_specific1[10];
  179. __be16 port;
  180. __be16 sdp_specific2;
  181. union cma_ip_addr src_addr;
  182. union cma_ip_addr dst_addr;
  183. };
  184. struct sdp_hah {
  185. u8 bsdh[16];
  186. u8 sdp_version;
  187. };
  188. #define CMA_VERSION 0x00
  189. #define SDP_MAJ_VERSION 0x2
  190. static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
  191. {
  192. unsigned long flags;
  193. int ret;
  194. spin_lock_irqsave(&id_priv->lock, flags);
  195. ret = (id_priv->state == comp);
  196. spin_unlock_irqrestore(&id_priv->lock, flags);
  197. return ret;
  198. }
  199. static int cma_comp_exch(struct rdma_id_private *id_priv,
  200. enum rdma_cm_state comp, enum rdma_cm_state exch)
  201. {
  202. unsigned long flags;
  203. int ret;
  204. spin_lock_irqsave(&id_priv->lock, flags);
  205. if ((ret = (id_priv->state == comp)))
  206. id_priv->state = exch;
  207. spin_unlock_irqrestore(&id_priv->lock, flags);
  208. return ret;
  209. }
  210. static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
  211. enum rdma_cm_state exch)
  212. {
  213. unsigned long flags;
  214. enum rdma_cm_state old;
  215. spin_lock_irqsave(&id_priv->lock, flags);
  216. old = id_priv->state;
  217. id_priv->state = exch;
  218. spin_unlock_irqrestore(&id_priv->lock, flags);
  219. return old;
  220. }
  221. static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
  222. {
  223. return hdr->ip_version >> 4;
  224. }
  225. static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
  226. {
  227. hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
  228. }
  229. static inline u8 sdp_get_majv(u8 sdp_version)
  230. {
  231. return sdp_version >> 4;
  232. }
  233. static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
  234. {
  235. return hh->ip_version >> 4;
  236. }
  237. static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
  238. {
  239. hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
  240. }
  241. static void cma_attach_to_dev(struct rdma_id_private *id_priv,
  242. struct cma_device *cma_dev)
  243. {
  244. atomic_inc(&cma_dev->refcount);
  245. id_priv->cma_dev = cma_dev;
  246. id_priv->id.device = cma_dev->device;
  247. id_priv->id.route.addr.dev_addr.transport =
  248. rdma_node_get_transport(cma_dev->device->node_type);
  249. list_add_tail(&id_priv->list, &cma_dev->id_list);
  250. }
  251. static inline void cma_deref_dev(struct cma_device *cma_dev)
  252. {
  253. if (atomic_dec_and_test(&cma_dev->refcount))
  254. complete(&cma_dev->comp);
  255. }
  256. static inline void release_mc(struct kref *kref)
  257. {
  258. struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
  259. kfree(mc->multicast.ib);
  260. kfree(mc);
  261. }
  262. static void cma_release_dev(struct rdma_id_private *id_priv)
  263. {
  264. mutex_lock(&lock);
  265. list_del(&id_priv->list);
  266. cma_deref_dev(id_priv->cma_dev);
  267. id_priv->cma_dev = NULL;
  268. mutex_unlock(&lock);
  269. }
  270. static int cma_set_qkey(struct rdma_id_private *id_priv)
  271. {
  272. struct ib_sa_mcmember_rec rec;
  273. int ret = 0;
  274. if (id_priv->qkey)
  275. return 0;
  276. switch (id_priv->id.ps) {
  277. case RDMA_PS_UDP:
  278. id_priv->qkey = RDMA_UDP_QKEY;
  279. break;
  280. case RDMA_PS_IPOIB:
  281. ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
  282. ret = ib_sa_get_mcmember_rec(id_priv->id.device,
  283. id_priv->id.port_num, &rec.mgid,
  284. &rec);
  285. if (!ret)
  286. id_priv->qkey = be32_to_cpu(rec.qkey);
  287. break;
  288. default:
  289. break;
  290. }
  291. return ret;
  292. }
  293. static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num)
  294. {
  295. int i;
  296. int err;
  297. struct ib_port_attr props;
  298. union ib_gid tmp;
  299. err = ib_query_port(device, port_num, &props);
  300. if (err)
  301. return err;
  302. for (i = 0; i < props.gid_tbl_len; ++i) {
  303. err = ib_query_gid(device, port_num, i, &tmp);
  304. if (err)
  305. return err;
  306. if (!memcmp(&tmp, gid, sizeof tmp))
  307. return 0;
  308. }
  309. return -EADDRNOTAVAIL;
  310. }
  311. static int cma_acquire_dev(struct rdma_id_private *id_priv)
  312. {
  313. struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
  314. struct cma_device *cma_dev;
  315. union ib_gid gid, iboe_gid;
  316. int ret = -ENODEV;
  317. u8 port;
  318. enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ?
  319. IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
  320. if (dev_ll != IB_LINK_LAYER_INFINIBAND &&
  321. id_priv->id.ps == RDMA_PS_IPOIB)
  322. return -EINVAL;
  323. mutex_lock(&lock);
  324. iboe_addr_get_sgid(dev_addr, &iboe_gid);
  325. memcpy(&gid, dev_addr->src_dev_addr +
  326. rdma_addr_gid_offset(dev_addr), sizeof gid);
  327. list_for_each_entry(cma_dev, &dev_list, list) {
  328. for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
  329. if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) {
  330. if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB &&
  331. rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET)
  332. ret = find_gid_port(cma_dev->device, &iboe_gid, port);
  333. else
  334. ret = find_gid_port(cma_dev->device, &gid, port);
  335. if (!ret) {
  336. id_priv->id.port_num = port;
  337. goto out;
  338. }
  339. }
  340. }
  341. }
  342. out:
  343. if (!ret)
  344. cma_attach_to_dev(id_priv, cma_dev);
  345. mutex_unlock(&lock);
  346. return ret;
  347. }
  348. static void cma_deref_id(struct rdma_id_private *id_priv)
  349. {
  350. if (atomic_dec_and_test(&id_priv->refcount))
  351. complete(&id_priv->comp);
  352. }
  353. static int cma_disable_callback(struct rdma_id_private *id_priv,
  354. enum rdma_cm_state state)
  355. {
  356. mutex_lock(&id_priv->handler_mutex);
  357. if (id_priv->state != state) {
  358. mutex_unlock(&id_priv->handler_mutex);
  359. return -EINVAL;
  360. }
  361. return 0;
  362. }
  363. struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
  364. void *context, enum rdma_port_space ps,
  365. enum ib_qp_type qp_type)
  366. {
  367. struct rdma_id_private *id_priv;
  368. id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
  369. if (!id_priv)
  370. return ERR_PTR(-ENOMEM);
  371. id_priv->owner = task_pid_nr(current);
  372. id_priv->state = RDMA_CM_IDLE;
  373. id_priv->id.context = context;
  374. id_priv->id.event_handler = event_handler;
  375. id_priv->id.ps = ps;
  376. id_priv->id.qp_type = qp_type;
  377. spin_lock_init(&id_priv->lock);
  378. mutex_init(&id_priv->qp_mutex);
  379. init_completion(&id_priv->comp);
  380. atomic_set(&id_priv->refcount, 1);
  381. mutex_init(&id_priv->handler_mutex);
  382. INIT_LIST_HEAD(&id_priv->listen_list);
  383. INIT_LIST_HEAD(&id_priv->mc_list);
  384. get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
  385. return &id_priv->id;
  386. }
  387. EXPORT_SYMBOL(rdma_create_id);
  388. static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
  389. {
  390. struct ib_qp_attr qp_attr;
  391. int qp_attr_mask, ret;
  392. qp_attr.qp_state = IB_QPS_INIT;
  393. ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
  394. if (ret)
  395. return ret;
  396. ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
  397. if (ret)
  398. return ret;
  399. qp_attr.qp_state = IB_QPS_RTR;
  400. ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
  401. if (ret)
  402. return ret;
  403. qp_attr.qp_state = IB_QPS_RTS;
  404. qp_attr.sq_psn = 0;
  405. ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
  406. return ret;
  407. }
  408. static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
  409. {
  410. struct ib_qp_attr qp_attr;
  411. int qp_attr_mask, ret;
  412. qp_attr.qp_state = IB_QPS_INIT;
  413. ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
  414. if (ret)
  415. return ret;
  416. return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
  417. }
  418. int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
  419. struct ib_qp_init_attr *qp_init_attr)
  420. {
  421. struct rdma_id_private *id_priv;
  422. struct ib_qp *qp;
  423. int ret;
  424. id_priv = container_of(id, struct rdma_id_private, id);
  425. if (id->device != pd->device)
  426. return -EINVAL;
  427. qp = ib_create_qp(pd, qp_init_attr);
  428. if (IS_ERR(qp))
  429. return PTR_ERR(qp);
  430. if (id->qp_type == IB_QPT_UD)
  431. ret = cma_init_ud_qp(id_priv, qp);
  432. else
  433. ret = cma_init_conn_qp(id_priv, qp);
  434. if (ret)
  435. goto err;
  436. id->qp = qp;
  437. id_priv->qp_num = qp->qp_num;
  438. id_priv->srq = (qp->srq != NULL);
  439. return 0;
  440. err:
  441. ib_destroy_qp(qp);
  442. return ret;
  443. }
  444. EXPORT_SYMBOL(rdma_create_qp);
  445. void rdma_destroy_qp(struct rdma_cm_id *id)
  446. {
  447. struct rdma_id_private *id_priv;
  448. id_priv = container_of(id, struct rdma_id_private, id);
  449. mutex_lock(&id_priv->qp_mutex);
  450. ib_destroy_qp(id_priv->id.qp);
  451. id_priv->id.qp = NULL;
  452. mutex_unlock(&id_priv->qp_mutex);
  453. }
  454. EXPORT_SYMBOL(rdma_destroy_qp);
  455. static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
  456. struct rdma_conn_param *conn_param)
  457. {
  458. struct ib_qp_attr qp_attr;
  459. int qp_attr_mask, ret;
  460. mutex_lock(&id_priv->qp_mutex);
  461. if (!id_priv->id.qp) {
  462. ret = 0;
  463. goto out;
  464. }
  465. /* Need to update QP attributes from default values. */
  466. qp_attr.qp_state = IB_QPS_INIT;
  467. ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
  468. if (ret)
  469. goto out;
  470. ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
  471. if (ret)
  472. goto out;
  473. qp_attr.qp_state = IB_QPS_RTR;
  474. ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
  475. if (ret)
  476. goto out;
  477. if (conn_param)
  478. qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
  479. ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
  480. out:
  481. mutex_unlock(&id_priv->qp_mutex);
  482. return ret;
  483. }
  484. static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
  485. struct rdma_conn_param *conn_param)
  486. {
  487. struct ib_qp_attr qp_attr;
  488. int qp_attr_mask, ret;
  489. mutex_lock(&id_priv->qp_mutex);
  490. if (!id_priv->id.qp) {
  491. ret = 0;
  492. goto out;
  493. }
  494. qp_attr.qp_state = IB_QPS_RTS;
  495. ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
  496. if (ret)
  497. goto out;
  498. if (conn_param)
  499. qp_attr.max_rd_atomic = conn_param->initiator_depth;
  500. ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
  501. out:
  502. mutex_unlock(&id_priv->qp_mutex);
  503. return ret;
  504. }
  505. static int cma_modify_qp_err(struct rdma_id_private *id_priv)
  506. {
  507. struct ib_qp_attr qp_attr;
  508. int ret;
  509. mutex_lock(&id_priv->qp_mutex);
  510. if (!id_priv->id.qp) {
  511. ret = 0;
  512. goto out;
  513. }
  514. qp_attr.qp_state = IB_QPS_ERR;
  515. ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
  516. out:
  517. mutex_unlock(&id_priv->qp_mutex);
  518. return ret;
  519. }
  520. static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
  521. struct ib_qp_attr *qp_attr, int *qp_attr_mask)
  522. {
  523. struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
  524. int ret;
  525. u16 pkey;
  526. if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) ==
  527. IB_LINK_LAYER_INFINIBAND)
  528. pkey = ib_addr_get_pkey(dev_addr);
  529. else
  530. pkey = 0xffff;
  531. ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
  532. pkey, &qp_attr->pkey_index);
  533. if (ret)
  534. return ret;
  535. qp_attr->port_num = id_priv->id.port_num;
  536. *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
  537. if (id_priv->id.qp_type == IB_QPT_UD) {
  538. ret = cma_set_qkey(id_priv);
  539. if (ret)
  540. return ret;
  541. qp_attr->qkey = id_priv->qkey;
  542. *qp_attr_mask |= IB_QP_QKEY;
  543. } else {
  544. qp_attr->qp_access_flags = 0;
  545. *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
  546. }
  547. return 0;
  548. }
  549. int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
  550. int *qp_attr_mask)
  551. {
  552. struct rdma_id_private *id_priv;
  553. int ret = 0;
  554. id_priv = container_of(id, struct rdma_id_private, id);
  555. switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
  556. case RDMA_TRANSPORT_IB:
  557. if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
  558. ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
  559. else
  560. ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
  561. qp_attr_mask);
  562. if (qp_attr->qp_state == IB_QPS_RTR)
  563. qp_attr->rq_psn = id_priv->seq_num;
  564. break;
  565. case RDMA_TRANSPORT_IWARP:
  566. if (!id_priv->cm_id.iw) {
  567. qp_attr->qp_access_flags = 0;
  568. *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
  569. } else
  570. ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
  571. qp_attr_mask);
  572. break;
  573. default:
  574. ret = -ENOSYS;
  575. break;
  576. }
  577. return ret;
  578. }
  579. EXPORT_SYMBOL(rdma_init_qp_attr);
  580. static inline int cma_zero_addr(struct sockaddr *addr)
  581. {
  582. struct in6_addr *ip6;
  583. if (addr->sa_family == AF_INET)
  584. return ipv4_is_zeronet(
  585. ((struct sockaddr_in *)addr)->sin_addr.s_addr);
  586. else {
  587. ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
  588. return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
  589. ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
  590. }
  591. }
  592. static inline int cma_loopback_addr(struct sockaddr *addr)
  593. {
  594. if (addr->sa_family == AF_INET)
  595. return ipv4_is_loopback(
  596. ((struct sockaddr_in *) addr)->sin_addr.s_addr);
  597. else
  598. return ipv6_addr_loopback(
  599. &((struct sockaddr_in6 *) addr)->sin6_addr);
  600. }
  601. static inline int cma_any_addr(struct sockaddr *addr)
  602. {
  603. return cma_zero_addr(addr) || cma_loopback_addr(addr);
  604. }
  605. static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
  606. {
  607. if (src->sa_family != dst->sa_family)
  608. return -1;
  609. switch (src->sa_family) {
  610. case AF_INET:
  611. return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
  612. ((struct sockaddr_in *) dst)->sin_addr.s_addr;
  613. default:
  614. return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
  615. &((struct sockaddr_in6 *) dst)->sin6_addr);
  616. }
  617. }
  618. static inline __be16 cma_port(struct sockaddr *addr)
  619. {
  620. if (addr->sa_family == AF_INET)
  621. return ((struct sockaddr_in *) addr)->sin_port;
  622. else
  623. return ((struct sockaddr_in6 *) addr)->sin6_port;
  624. }
  625. static inline int cma_any_port(struct sockaddr *addr)
  626. {
  627. return !cma_port(addr);
  628. }
  629. static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
  630. u8 *ip_ver, __be16 *port,
  631. union cma_ip_addr **src, union cma_ip_addr **dst)
  632. {
  633. switch (ps) {
  634. case RDMA_PS_SDP:
  635. if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
  636. SDP_MAJ_VERSION)
  637. return -EINVAL;
  638. *ip_ver = sdp_get_ip_ver(hdr);
  639. *port = ((struct sdp_hh *) hdr)->port;
  640. *src = &((struct sdp_hh *) hdr)->src_addr;
  641. *dst = &((struct sdp_hh *) hdr)->dst_addr;
  642. break;
  643. default:
  644. if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
  645. return -EINVAL;
  646. *ip_ver = cma_get_ip_ver(hdr);
  647. *port = ((struct cma_hdr *) hdr)->port;
  648. *src = &((struct cma_hdr *) hdr)->src_addr;
  649. *dst = &((struct cma_hdr *) hdr)->dst_addr;
  650. break;
  651. }
  652. if (*ip_ver != 4 && *ip_ver != 6)
  653. return -EINVAL;
  654. return 0;
  655. }
  656. static void cma_save_net_info(struct rdma_addr *addr,
  657. struct rdma_addr *listen_addr,
  658. u8 ip_ver, __be16 port,
  659. union cma_ip_addr *src, union cma_ip_addr *dst)
  660. {
  661. struct sockaddr_in *listen4, *ip4;
  662. struct sockaddr_in6 *listen6, *ip6;
  663. switch (ip_ver) {
  664. case 4:
  665. listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
  666. ip4 = (struct sockaddr_in *) &addr->src_addr;
  667. ip4->sin_family = listen4->sin_family;
  668. ip4->sin_addr.s_addr = dst->ip4.addr;
  669. ip4->sin_port = listen4->sin_port;
  670. ip4 = (struct sockaddr_in *) &addr->dst_addr;
  671. ip4->sin_family = listen4->sin_family;
  672. ip4->sin_addr.s_addr = src->ip4.addr;
  673. ip4->sin_port = port;
  674. break;
  675. case 6:
  676. listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
  677. ip6 = (struct sockaddr_in6 *) &addr->src_addr;
  678. ip6->sin6_family = listen6->sin6_family;
  679. ip6->sin6_addr = dst->ip6;
  680. ip6->sin6_port = listen6->sin6_port;
  681. ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
  682. ip6->sin6_family = listen6->sin6_family;
  683. ip6->sin6_addr = src->ip6;
  684. ip6->sin6_port = port;
  685. break;
  686. default:
  687. break;
  688. }
  689. }
  690. static inline int cma_user_data_offset(enum rdma_port_space ps)
  691. {
  692. switch (ps) {
  693. case RDMA_PS_SDP:
  694. return 0;
  695. default:
  696. return sizeof(struct cma_hdr);
  697. }
  698. }
  699. static void cma_cancel_route(struct rdma_id_private *id_priv)
  700. {
  701. switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) {
  702. case IB_LINK_LAYER_INFINIBAND:
  703. if (id_priv->query)
  704. ib_sa_cancel_query(id_priv->query_id, id_priv->query);
  705. break;
  706. default:
  707. break;
  708. }
  709. }
  710. static void cma_cancel_listens(struct rdma_id_private *id_priv)
  711. {
  712. struct rdma_id_private *dev_id_priv;
  713. /*
  714. * Remove from listen_any_list to prevent added devices from spawning
  715. * additional listen requests.
  716. */
  717. mutex_lock(&lock);
  718. list_del(&id_priv->list);
  719. while (!list_empty(&id_priv->listen_list)) {
  720. dev_id_priv = list_entry(id_priv->listen_list.next,
  721. struct rdma_id_private, listen_list);
  722. /* sync with device removal to avoid duplicate destruction */
  723. list_del_init(&dev_id_priv->list);
  724. list_del(&dev_id_priv->listen_list);
  725. mutex_unlock(&lock);
  726. rdma_destroy_id(&dev_id_priv->id);
  727. mutex_lock(&lock);
  728. }
  729. mutex_unlock(&lock);
  730. }
  731. static void cma_cancel_operation(struct rdma_id_private *id_priv,
  732. enum rdma_cm_state state)
  733. {
  734. switch (state) {
  735. case RDMA_CM_ADDR_QUERY:
  736. rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
  737. break;
  738. case RDMA_CM_ROUTE_QUERY:
  739. cma_cancel_route(id_priv);
  740. break;
  741. case RDMA_CM_LISTEN:
  742. if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)
  743. && !id_priv->cma_dev)
  744. cma_cancel_listens(id_priv);
  745. break;
  746. default:
  747. break;
  748. }
  749. }
  750. static void cma_release_port(struct rdma_id_private *id_priv)
  751. {
  752. struct rdma_bind_list *bind_list = id_priv->bind_list;
  753. if (!bind_list)
  754. return;
  755. mutex_lock(&lock);
  756. hlist_del(&id_priv->node);
  757. if (hlist_empty(&bind_list->owners)) {
  758. idr_remove(bind_list->ps, bind_list->port);
  759. kfree(bind_list);
  760. }
  761. mutex_unlock(&lock);
  762. }
  763. static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
  764. {
  765. struct cma_multicast *mc;
  766. while (!list_empty(&id_priv->mc_list)) {
  767. mc = container_of(id_priv->mc_list.next,
  768. struct cma_multicast, list);
  769. list_del(&mc->list);
  770. switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) {
  771. case IB_LINK_LAYER_INFINIBAND:
  772. ib_sa_free_multicast(mc->multicast.ib);
  773. kfree(mc);
  774. break;
  775. case IB_LINK_LAYER_ETHERNET:
  776. kref_put(&mc->mcref, release_mc);
  777. break;
  778. default:
  779. break;
  780. }
  781. }
  782. }
  783. void rdma_destroy_id(struct rdma_cm_id *id)
  784. {
  785. struct rdma_id_private *id_priv;
  786. enum rdma_cm_state state;
  787. id_priv = container_of(id, struct rdma_id_private, id);
  788. state = cma_exch(id_priv, RDMA_CM_DESTROYING);
  789. cma_cancel_operation(id_priv, state);
  790. /*
  791. * Wait for any active callback to finish. New callbacks will find
  792. * the id_priv state set to destroying and abort.
  793. */
  794. mutex_lock(&id_priv->handler_mutex);
  795. mutex_unlock(&id_priv->handler_mutex);
  796. if (id_priv->cma_dev) {
  797. switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
  798. case RDMA_TRANSPORT_IB:
  799. if (id_priv->cm_id.ib)
  800. ib_destroy_cm_id(id_priv->cm_id.ib);
  801. break;
  802. case RDMA_TRANSPORT_IWARP:
  803. if (id_priv->cm_id.iw)
  804. iw_destroy_cm_id(id_priv->cm_id.iw);
  805. break;
  806. default:
  807. break;
  808. }
  809. cma_leave_mc_groups(id_priv);
  810. cma_release_dev(id_priv);
  811. }
  812. cma_release_port(id_priv);
  813. cma_deref_id(id_priv);
  814. wait_for_completion(&id_priv->comp);
  815. if (id_priv->internal_id)
  816. cma_deref_id(id_priv->id.context);
  817. kfree(id_priv->id.route.path_rec);
  818. kfree(id_priv);
  819. }
  820. EXPORT_SYMBOL(rdma_destroy_id);
  821. static int cma_rep_recv(struct rdma_id_private *id_priv)
  822. {
  823. int ret;
  824. ret = cma_modify_qp_rtr(id_priv, NULL);
  825. if (ret)
  826. goto reject;
  827. ret = cma_modify_qp_rts(id_priv, NULL);
  828. if (ret)
  829. goto reject;
  830. ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
  831. if (ret)
  832. goto reject;
  833. return 0;
  834. reject:
  835. cma_modify_qp_err(id_priv);
  836. ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
  837. NULL, 0, NULL, 0);
  838. return ret;
  839. }
  840. static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
  841. {
  842. if (id_priv->id.ps == RDMA_PS_SDP &&
  843. sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
  844. SDP_MAJ_VERSION)
  845. return -EINVAL;
  846. return 0;
  847. }
  848. static void cma_set_rep_event_data(struct rdma_cm_event *event,
  849. struct ib_cm_rep_event_param *rep_data,
  850. void *private_data)
  851. {
  852. event->param.conn.private_data = private_data;
  853. event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
  854. event->param.conn.responder_resources = rep_data->responder_resources;
  855. event->param.conn.initiator_depth = rep_data->initiator_depth;
  856. event->param.conn.flow_control = rep_data->flow_control;
  857. event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
  858. event->param.conn.srq = rep_data->srq;
  859. event->param.conn.qp_num = rep_data->remote_qpn;
  860. }
  861. static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
  862. {
  863. struct rdma_id_private *id_priv = cm_id->context;
  864. struct rdma_cm_event event;
  865. int ret = 0;
  866. if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
  867. cma_disable_callback(id_priv, RDMA_CM_CONNECT)) ||
  868. (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
  869. cma_disable_callback(id_priv, RDMA_CM_DISCONNECT)))
  870. return 0;
  871. memset(&event, 0, sizeof event);
  872. switch (ib_event->event) {
  873. case IB_CM_REQ_ERROR:
  874. case IB_CM_REP_ERROR:
  875. event.event = RDMA_CM_EVENT_UNREACHABLE;
  876. event.status = -ETIMEDOUT;
  877. break;
  878. case IB_CM_REP_RECEIVED:
  879. event.status = cma_verify_rep(id_priv, ib_event->private_data);
  880. if (event.status)
  881. event.event = RDMA_CM_EVENT_CONNECT_ERROR;
  882. else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
  883. event.status = cma_rep_recv(id_priv);
  884. event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
  885. RDMA_CM_EVENT_ESTABLISHED;
  886. } else
  887. event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
  888. cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
  889. ib_event->private_data);
  890. break;
  891. case IB_CM_RTU_RECEIVED:
  892. case IB_CM_USER_ESTABLISHED:
  893. event.event = RDMA_CM_EVENT_ESTABLISHED;
  894. break;
  895. case IB_CM_DREQ_ERROR:
  896. event.status = -ETIMEDOUT; /* fall through */
  897. case IB_CM_DREQ_RECEIVED:
  898. case IB_CM_DREP_RECEIVED:
  899. if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
  900. RDMA_CM_DISCONNECT))
  901. goto out;
  902. event.event = RDMA_CM_EVENT_DISCONNECTED;
  903. break;
  904. case IB_CM_TIMEWAIT_EXIT:
  905. event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
  906. break;
  907. case IB_CM_MRA_RECEIVED:
  908. /* ignore event */
  909. goto out;
  910. case IB_CM_REJ_RECEIVED:
  911. cma_modify_qp_err(id_priv);
  912. event.status = ib_event->param.rej_rcvd.reason;
  913. event.event = RDMA_CM_EVENT_REJECTED;
  914. event.param.conn.private_data = ib_event->private_data;
  915. event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
  916. break;
  917. default:
  918. printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
  919. ib_event->event);
  920. goto out;
  921. }
  922. ret = id_priv->id.event_handler(&id_priv->id, &event);
  923. if (ret) {
  924. /* Destroy the CM ID by returning a non-zero value. */
  925. id_priv->cm_id.ib = NULL;
  926. cma_exch(id_priv, RDMA_CM_DESTROYING);
  927. mutex_unlock(&id_priv->handler_mutex);
  928. rdma_destroy_id(&id_priv->id);
  929. return ret;
  930. }
  931. out:
  932. mutex_unlock(&id_priv->handler_mutex);
  933. return ret;
  934. }
  935. static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
  936. struct ib_cm_event *ib_event)
  937. {
  938. struct rdma_id_private *id_priv;
  939. struct rdma_cm_id *id;
  940. struct rdma_route *rt;
  941. union cma_ip_addr *src, *dst;
  942. __be16 port;
  943. u8 ip_ver;
  944. int ret;
  945. if (cma_get_net_info(ib_event->private_data, listen_id->ps,
  946. &ip_ver, &port, &src, &dst))
  947. return NULL;
  948. id = rdma_create_id(listen_id->event_handler, listen_id->context,
  949. listen_id->ps, ib_event->param.req_rcvd.qp_type);
  950. if (IS_ERR(id))
  951. return NULL;
  952. cma_save_net_info(&id->route.addr, &listen_id->route.addr,
  953. ip_ver, port, src, dst);
  954. rt = &id->route;
  955. rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
  956. rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
  957. GFP_KERNEL);
  958. if (!rt->path_rec)
  959. goto err;
  960. rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
  961. if (rt->num_paths == 2)
  962. rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
  963. if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) {
  964. rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
  965. rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
  966. ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
  967. } else {
  968. ret = rdma_translate_ip((struct sockaddr *) &rt->addr.src_addr,
  969. &rt->addr.dev_addr);
  970. if (ret)
  971. goto err;
  972. }
  973. rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
  974. id_priv = container_of(id, struct rdma_id_private, id);
  975. id_priv->state = RDMA_CM_CONNECT;
  976. return id_priv;
  977. err:
  978. rdma_destroy_id(id);
  979. return NULL;
  980. }
  981. static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
  982. struct ib_cm_event *ib_event)
  983. {
  984. struct rdma_id_private *id_priv;
  985. struct rdma_cm_id *id;
  986. union cma_ip_addr *src, *dst;
  987. __be16 port;
  988. u8 ip_ver;
  989. int ret;
  990. id = rdma_create_id(listen_id->event_handler, listen_id->context,
  991. listen_id->ps, IB_QPT_UD);
  992. if (IS_ERR(id))
  993. return NULL;
  994. if (cma_get_net_info(ib_event->private_data, listen_id->ps,
  995. &ip_ver, &port, &src, &dst))
  996. goto err;
  997. cma_save_net_info(&id->route.addr, &listen_id->route.addr,
  998. ip_ver, port, src, dst);
  999. if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
  1000. ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
  1001. &id->route.addr.dev_addr);
  1002. if (ret)
  1003. goto err;
  1004. }
  1005. id_priv = container_of(id, struct rdma_id_private, id);
  1006. id_priv->state = RDMA_CM_CONNECT;
  1007. return id_priv;
  1008. err:
  1009. rdma_destroy_id(id);
  1010. return NULL;
  1011. }
  1012. static void cma_set_req_event_data(struct rdma_cm_event *event,
  1013. struct ib_cm_req_event_param *req_data,
  1014. void *private_data, int offset)
  1015. {
  1016. event->param.conn.private_data = private_data + offset;
  1017. event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
  1018. event->param.conn.responder_resources = req_data->responder_resources;
  1019. event->param.conn.initiator_depth = req_data->initiator_depth;
  1020. event->param.conn.flow_control = req_data->flow_control;
  1021. event->param.conn.retry_count = req_data->retry_count;
  1022. event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
  1023. event->param.conn.srq = req_data->srq;
  1024. event->param.conn.qp_num = req_data->remote_qpn;
  1025. }
  1026. static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event)
  1027. {
  1028. return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
  1029. (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
  1030. ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
  1031. (id->qp_type == IB_QPT_UD)) ||
  1032. (!id->qp_type));
  1033. }
  1034. static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
  1035. {
  1036. struct rdma_id_private *listen_id, *conn_id;
  1037. struct rdma_cm_event event;
  1038. int offset, ret;
  1039. listen_id = cm_id->context;
  1040. if (!cma_check_req_qp_type(&listen_id->id, ib_event))
  1041. return -EINVAL;
  1042. if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
  1043. return -ECONNABORTED;
  1044. memset(&event, 0, sizeof event);
  1045. offset = cma_user_data_offset(listen_id->id.ps);
  1046. event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
  1047. if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
  1048. conn_id = cma_new_udp_id(&listen_id->id, ib_event);
  1049. event.param.ud.private_data = ib_event->private_data + offset;
  1050. event.param.ud.private_data_len =
  1051. IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
  1052. } else {
  1053. conn_id = cma_new_conn_id(&listen_id->id, ib_event);
  1054. cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
  1055. ib_event->private_data, offset);
  1056. }
  1057. if (!conn_id) {
  1058. ret = -ENOMEM;
  1059. goto err1;
  1060. }
  1061. mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
  1062. ret = cma_acquire_dev(conn_id);
  1063. if (ret)
  1064. goto err2;
  1065. conn_id->cm_id.ib = cm_id;
  1066. cm_id->context = conn_id;
  1067. cm_id->cm_handler = cma_ib_handler;
  1068. /*
  1069. * Protect against the user destroying conn_id from another thread
  1070. * until we're done accessing it.
  1071. */
  1072. atomic_inc(&conn_id->refcount);
  1073. ret = conn_id->id.event_handler(&conn_id->id, &event);
  1074. if (ret)
  1075. goto err3;
  1076. /*
  1077. * Acquire mutex to prevent user executing rdma_destroy_id()
  1078. * while we're accessing the cm_id.
  1079. */
  1080. mutex_lock(&lock);
  1081. if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD))
  1082. ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
  1083. mutex_unlock(&lock);
  1084. mutex_unlock(&conn_id->handler_mutex);
  1085. mutex_unlock(&listen_id->handler_mutex);
  1086. cma_deref_id(conn_id);
  1087. return 0;
  1088. err3:
  1089. cma_deref_id(conn_id);
  1090. /* Destroy the CM ID by returning a non-zero value. */
  1091. conn_id->cm_id.ib = NULL;
  1092. err2:
  1093. cma_exch(conn_id, RDMA_CM_DESTROYING);
  1094. mutex_unlock(&conn_id->handler_mutex);
  1095. err1:
  1096. mutex_unlock(&listen_id->handler_mutex);
  1097. if (conn_id)
  1098. rdma_destroy_id(&conn_id->id);
  1099. return ret;
  1100. }
  1101. static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
  1102. {
  1103. return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
  1104. }
  1105. static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
  1106. struct ib_cm_compare_data *compare)
  1107. {
  1108. struct cma_hdr *cma_data, *cma_mask;
  1109. struct sdp_hh *sdp_data, *sdp_mask;
  1110. __be32 ip4_addr;
  1111. struct in6_addr ip6_addr;
  1112. memset(compare, 0, sizeof *compare);
  1113. cma_data = (void *) compare->data;
  1114. cma_mask = (void *) compare->mask;
  1115. sdp_data = (void *) compare->data;
  1116. sdp_mask = (void *) compare->mask;
  1117. switch (addr->sa_family) {
  1118. case AF_INET:
  1119. ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
  1120. if (ps == RDMA_PS_SDP) {
  1121. sdp_set_ip_ver(sdp_data, 4);
  1122. sdp_set_ip_ver(sdp_mask, 0xF);
  1123. sdp_data->dst_addr.ip4.addr = ip4_addr;
  1124. sdp_mask->dst_addr.ip4.addr = htonl(~0);
  1125. } else {
  1126. cma_set_ip_ver(cma_data, 4);
  1127. cma_set_ip_ver(cma_mask, 0xF);
  1128. if (!cma_any_addr(addr)) {
  1129. cma_data->dst_addr.ip4.addr = ip4_addr;
  1130. cma_mask->dst_addr.ip4.addr = htonl(~0);
  1131. }
  1132. }
  1133. break;
  1134. case AF_INET6:
  1135. ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
  1136. if (ps == RDMA_PS_SDP) {
  1137. sdp_set_ip_ver(sdp_data, 6);
  1138. sdp_set_ip_ver(sdp_mask, 0xF);
  1139. sdp_data->dst_addr.ip6 = ip6_addr;
  1140. memset(&sdp_mask->dst_addr.ip6, 0xFF,
  1141. sizeof sdp_mask->dst_addr.ip6);
  1142. } else {
  1143. cma_set_ip_ver(cma_data, 6);
  1144. cma_set_ip_ver(cma_mask, 0xF);
  1145. if (!cma_any_addr(addr)) {
  1146. cma_data->dst_addr.ip6 = ip6_addr;
  1147. memset(&cma_mask->dst_addr.ip6, 0xFF,
  1148. sizeof cma_mask->dst_addr.ip6);
  1149. }
  1150. }
  1151. break;
  1152. default:
  1153. break;
  1154. }
  1155. }
  1156. static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
  1157. {
  1158. struct rdma_id_private *id_priv = iw_id->context;
  1159. struct rdma_cm_event event;
  1160. struct sockaddr_in *sin;
  1161. int ret = 0;
  1162. if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
  1163. return 0;
  1164. memset(&event, 0, sizeof event);
  1165. switch (iw_event->event) {
  1166. case IW_CM_EVENT_CLOSE:
  1167. event.event = RDMA_CM_EVENT_DISCONNECTED;
  1168. break;
  1169. case IW_CM_EVENT_CONNECT_REPLY:
  1170. sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
  1171. *sin = iw_event->local_addr;
  1172. sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
  1173. *sin = iw_event->remote_addr;
  1174. switch (iw_event->status) {
  1175. case 0:
  1176. event.event = RDMA_CM_EVENT_ESTABLISHED;
  1177. event.param.conn.initiator_depth = iw_event->ird;
  1178. event.param.conn.responder_resources = iw_event->ord;
  1179. break;
  1180. case -ECONNRESET:
  1181. case -ECONNREFUSED:
  1182. event.event = RDMA_CM_EVENT_REJECTED;
  1183. break;
  1184. case -ETIMEDOUT:
  1185. event.event = RDMA_CM_EVENT_UNREACHABLE;
  1186. break;
  1187. default:
  1188. event.event = RDMA_CM_EVENT_CONNECT_ERROR;
  1189. break;
  1190. }
  1191. break;
  1192. case IW_CM_EVENT_ESTABLISHED:
  1193. event.event = RDMA_CM_EVENT_ESTABLISHED;
  1194. event.param.conn.initiator_depth = iw_event->ird;
  1195. event.param.conn.responder_resources = iw_event->ord;
  1196. break;
  1197. default:
  1198. BUG_ON(1);
  1199. }
  1200. event.status = iw_event->status;
  1201. event.param.conn.private_data = iw_event->private_data;
  1202. event.param.conn.private_data_len = iw_event->private_data_len;
  1203. ret = id_priv->id.event_handler(&id_priv->id, &event);
  1204. if (ret) {
  1205. /* Destroy the CM ID by returning a non-zero value. */
  1206. id_priv->cm_id.iw = NULL;
  1207. cma_exch(id_priv, RDMA_CM_DESTROYING);
  1208. mutex_unlock(&id_priv->handler_mutex);
  1209. rdma_destroy_id(&id_priv->id);
  1210. return ret;
  1211. }
  1212. mutex_unlock(&id_priv->handler_mutex);
  1213. return ret;
  1214. }
  1215. static int iw_conn_req_handler(struct iw_cm_id *cm_id,
  1216. struct iw_cm_event *iw_event)
  1217. {
  1218. struct rdma_cm_id *new_cm_id;
  1219. struct rdma_id_private *listen_id, *conn_id;
  1220. struct sockaddr_in *sin;
  1221. struct net_device *dev = NULL;
  1222. struct rdma_cm_event event;
  1223. int ret;
  1224. struct ib_device_attr attr;
  1225. listen_id = cm_id->context;
  1226. if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
  1227. return -ECONNABORTED;
  1228. /* Create a new RDMA id for the new IW CM ID */
  1229. new_cm_id = rdma_create_id(listen_id->id.event_handler,
  1230. listen_id->id.context,
  1231. RDMA_PS_TCP, IB_QPT_RC);
  1232. if (IS_ERR(new_cm_id)) {
  1233. ret = -ENOMEM;
  1234. goto out;
  1235. }
  1236. conn_id = container_of(new_cm_id, struct rdma_id_private, id);
  1237. mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
  1238. conn_id->state = RDMA_CM_CONNECT;
  1239. dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
  1240. if (!dev) {
  1241. ret = -EADDRNOTAVAIL;
  1242. mutex_unlock(&conn_id->handler_mutex);
  1243. rdma_destroy_id(new_cm_id);
  1244. goto out;
  1245. }
  1246. ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
  1247. if (ret) {
  1248. mutex_unlock(&conn_id->handler_mutex);
  1249. rdma_destroy_id(new_cm_id);
  1250. goto out;
  1251. }
  1252. ret = cma_acquire_dev(conn_id);
  1253. if (ret) {
  1254. mutex_unlock(&conn_id->handler_mutex);
  1255. rdma_destroy_id(new_cm_id);
  1256. goto out;
  1257. }
  1258. conn_id->cm_id.iw = cm_id;
  1259. cm_id->context = conn_id;
  1260. cm_id->cm_handler = cma_iw_handler;
  1261. sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
  1262. *sin = iw_event->local_addr;
  1263. sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
  1264. *sin = iw_event->remote_addr;
  1265. ret = ib_query_device(conn_id->id.device, &attr);
  1266. if (ret) {
  1267. mutex_unlock(&conn_id->handler_mutex);
  1268. rdma_destroy_id(new_cm_id);
  1269. goto out;
  1270. }
  1271. memset(&event, 0, sizeof event);
  1272. event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
  1273. event.param.conn.private_data = iw_event->private_data;
  1274. event.param.conn.private_data_len = iw_event->private_data_len;
  1275. event.param.conn.initiator_depth = iw_event->ird;
  1276. event.param.conn.responder_resources = iw_event->ord;
  1277. /*
  1278. * Protect against the user destroying conn_id from another thread
  1279. * until we're done accessing it.
  1280. */
  1281. atomic_inc(&conn_id->refcount);
  1282. ret = conn_id->id.event_handler(&conn_id->id, &event);
  1283. if (ret) {
  1284. /* User wants to destroy the CM ID */
  1285. conn_id->cm_id.iw = NULL;
  1286. cma_exch(conn_id, RDMA_CM_DESTROYING);
  1287. mutex_unlock(&conn_id->handler_mutex);
  1288. cma_deref_id(conn_id);
  1289. rdma_destroy_id(&conn_id->id);
  1290. goto out;
  1291. }
  1292. mutex_unlock(&conn_id->handler_mutex);
  1293. cma_deref_id(conn_id);
  1294. out:
  1295. if (dev)
  1296. dev_put(dev);
  1297. mutex_unlock(&listen_id->handler_mutex);
  1298. return ret;
  1299. }
  1300. static int cma_ib_listen(struct rdma_id_private *id_priv)
  1301. {
  1302. struct ib_cm_compare_data compare_data;
  1303. struct sockaddr *addr;
  1304. struct ib_cm_id *id;
  1305. __be64 svc_id;
  1306. int ret;
  1307. id = ib_create_cm_id(id_priv->id.device, cma_req_handler, id_priv);
  1308. if (IS_ERR(id))
  1309. return PTR_ERR(id);
  1310. id_priv->cm_id.ib = id;
  1311. addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
  1312. svc_id = cma_get_service_id(id_priv->id.ps, addr);
  1313. if (cma_any_addr(addr) && !id_priv->afonly)
  1314. ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
  1315. else {
  1316. cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
  1317. ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
  1318. }
  1319. if (ret) {
  1320. ib_destroy_cm_id(id_priv->cm_id.ib);
  1321. id_priv->cm_id.ib = NULL;
  1322. }
  1323. return ret;
  1324. }
  1325. static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
  1326. {
  1327. int ret;
  1328. struct sockaddr_in *sin;
  1329. struct iw_cm_id *id;
  1330. id = iw_create_cm_id(id_priv->id.device,
  1331. iw_conn_req_handler,
  1332. id_priv);
  1333. if (IS_ERR(id))
  1334. return PTR_ERR(id);
  1335. id_priv->cm_id.iw = id;
  1336. sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
  1337. id_priv->cm_id.iw->local_addr = *sin;
  1338. ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
  1339. if (ret) {
  1340. iw_destroy_cm_id(id_priv->cm_id.iw);
  1341. id_priv->cm_id.iw = NULL;
  1342. }
  1343. return ret;
  1344. }
  1345. static int cma_listen_handler(struct rdma_cm_id *id,
  1346. struct rdma_cm_event *event)
  1347. {
  1348. struct rdma_id_private *id_priv = id->context;
  1349. id->context = id_priv->id.context;
  1350. id->event_handler = id_priv->id.event_handler;
  1351. return id_priv->id.event_handler(id, event);
  1352. }
  1353. static void cma_listen_on_dev(struct rdma_id_private *id_priv,
  1354. struct cma_device *cma_dev)
  1355. {
  1356. struct rdma_id_private *dev_id_priv;
  1357. struct rdma_cm_id *id;
  1358. int ret;
  1359. id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps,
  1360. id_priv->id.qp_type);
  1361. if (IS_ERR(id))
  1362. return;
  1363. dev_id_priv = container_of(id, struct rdma_id_private, id);
  1364. dev_id_priv->state = RDMA_CM_ADDR_BOUND;
  1365. memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
  1366. ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr));
  1367. cma_attach_to_dev(dev_id_priv, cma_dev);
  1368. list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
  1369. atomic_inc(&id_priv->refcount);
  1370. dev_id_priv->internal_id = 1;
  1371. dev_id_priv->afonly = id_priv->afonly;
  1372. ret = rdma_listen(id, id_priv->backlog);
  1373. if (ret)
  1374. printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
  1375. "listening on device %s\n", ret, cma_dev->device->name);
  1376. }
  1377. static void cma_listen_on_all(struct rdma_id_private *id_priv)
  1378. {
  1379. struct cma_device *cma_dev;
  1380. mutex_lock(&lock);
  1381. list_add_tail(&id_priv->list, &listen_any_list);
  1382. list_for_each_entry(cma_dev, &dev_list, list)
  1383. cma_listen_on_dev(id_priv, cma_dev);
  1384. mutex_unlock(&lock);
  1385. }
  1386. void rdma_set_service_type(struct rdma_cm_id *id, int tos)
  1387. {
  1388. struct rdma_id_private *id_priv;
  1389. id_priv = container_of(id, struct rdma_id_private, id);
  1390. id_priv->tos = (u8) tos;
  1391. }
  1392. EXPORT_SYMBOL(rdma_set_service_type);
  1393. static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
  1394. void *context)
  1395. {
  1396. struct cma_work *work = context;
  1397. struct rdma_route *route;
  1398. route = &work->id->id.route;
  1399. if (!status) {
  1400. route->num_paths = 1;
  1401. *route->path_rec = *path_rec;
  1402. } else {
  1403. work->old_state = RDMA_CM_ROUTE_QUERY;
  1404. work->new_state = RDMA_CM_ADDR_RESOLVED;
  1405. work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
  1406. work->event.status = status;
  1407. }
  1408. queue_work(cma_wq, &work->work);
  1409. }
  1410. static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
  1411. struct cma_work *work)
  1412. {
  1413. struct rdma_addr *addr = &id_priv->id.route.addr;
  1414. struct ib_sa_path_rec path_rec;
  1415. ib_sa_comp_mask comp_mask;
  1416. struct sockaddr_in6 *sin6;
  1417. memset(&path_rec, 0, sizeof path_rec);
  1418. rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
  1419. rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
  1420. path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
  1421. path_rec.numb_path = 1;
  1422. path_rec.reversible = 1;
  1423. path_rec.service_id = cma_get_service_id(id_priv->id.ps,
  1424. (struct sockaddr *) &addr->dst_addr);
  1425. comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
  1426. IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
  1427. IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
  1428. if (addr->src_addr.ss_family == AF_INET) {
  1429. path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
  1430. comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
  1431. } else {
  1432. sin6 = (struct sockaddr_in6 *) &addr->src_addr;
  1433. path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
  1434. comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
  1435. }
  1436. id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
  1437. id_priv->id.port_num, &path_rec,
  1438. comp_mask, timeout_ms,
  1439. GFP_KERNEL, cma_query_handler,
  1440. work, &id_priv->query);
  1441. return (id_priv->query_id < 0) ? id_priv->query_id : 0;
  1442. }
  1443. static void cma_work_handler(struct work_struct *_work)
  1444. {
  1445. struct cma_work *work = container_of(_work, struct cma_work, work);
  1446. struct rdma_id_private *id_priv = work->id;
  1447. int destroy = 0;
  1448. mutex_lock(&id_priv->handler_mutex);
  1449. if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
  1450. goto out;
  1451. if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
  1452. cma_exch(id_priv, RDMA_CM_DESTROYING);
  1453. destroy = 1;
  1454. }
  1455. out:
  1456. mutex_unlock(&id_priv->handler_mutex);
  1457. cma_deref_id(id_priv);
  1458. if (destroy)
  1459. rdma_destroy_id(&id_priv->id);
  1460. kfree(work);
  1461. }
  1462. static void cma_ndev_work_handler(struct work_struct *_work)
  1463. {
  1464. struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
  1465. struct rdma_id_private *id_priv = work->id;
  1466. int destroy = 0;
  1467. mutex_lock(&id_priv->handler_mutex);
  1468. if (id_priv->state == RDMA_CM_DESTROYING ||
  1469. id_priv->state == RDMA_CM_DEVICE_REMOVAL)
  1470. goto out;
  1471. if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
  1472. cma_exch(id_priv, RDMA_CM_DESTROYING);
  1473. destroy = 1;
  1474. }
  1475. out:
  1476. mutex_unlock(&id_priv->handler_mutex);
  1477. cma_deref_id(id_priv);
  1478. if (destroy)
  1479. rdma_destroy_id(&id_priv->id);
  1480. kfree(work);
  1481. }
  1482. static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
  1483. {
  1484. struct rdma_route *route = &id_priv->id.route;
  1485. struct cma_work *work;
  1486. int ret;
  1487. work = kzalloc(sizeof *work, GFP_KERNEL);
  1488. if (!work)
  1489. return -ENOMEM;
  1490. work->id = id_priv;
  1491. INIT_WORK(&work->work, cma_work_handler);
  1492. work->old_state = RDMA_CM_ROUTE_QUERY;
  1493. work->new_state = RDMA_CM_ROUTE_RESOLVED;
  1494. work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
  1495. route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
  1496. if (!route->path_rec) {
  1497. ret = -ENOMEM;
  1498. goto err1;
  1499. }
  1500. ret = cma_query_ib_route(id_priv, timeout_ms, work);
  1501. if (ret)
  1502. goto err2;
  1503. return 0;
  1504. err2:
  1505. kfree(route->path_rec);
  1506. route->path_rec = NULL;
  1507. err1:
  1508. kfree(work);
  1509. return ret;
  1510. }
  1511. int rdma_set_ib_paths(struct rdma_cm_id *id,
  1512. struct ib_sa_path_rec *path_rec, int num_paths)
  1513. {
  1514. struct rdma_id_private *id_priv;
  1515. int ret;
  1516. id_priv = container_of(id, struct rdma_id_private, id);
  1517. if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
  1518. RDMA_CM_ROUTE_RESOLVED))
  1519. return -EINVAL;
  1520. id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
  1521. GFP_KERNEL);
  1522. if (!id->route.path_rec) {
  1523. ret = -ENOMEM;
  1524. goto err;
  1525. }
  1526. id->route.num_paths = num_paths;
  1527. return 0;
  1528. err:
  1529. cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
  1530. return ret;
  1531. }
  1532. EXPORT_SYMBOL(rdma_set_ib_paths);
  1533. static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
  1534. {
  1535. struct cma_work *work;
  1536. work = kzalloc(sizeof *work, GFP_KERNEL);
  1537. if (!work)
  1538. return -ENOMEM;
  1539. work->id = id_priv;
  1540. INIT_WORK(&work->work, cma_work_handler);
  1541. work->old_state = RDMA_CM_ROUTE_QUERY;
  1542. work->new_state = RDMA_CM_ROUTE_RESOLVED;
  1543. work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
  1544. queue_work(cma_wq, &work->work);
  1545. return 0;
  1546. }
  1547. static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
  1548. {
  1549. struct rdma_route *route = &id_priv->id.route;
  1550. struct rdma_addr *addr = &route->addr;
  1551. struct cma_work *work;
  1552. int ret;
  1553. struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr;
  1554. struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr;
  1555. struct net_device *ndev = NULL;
  1556. u16 vid;
  1557. if (src_addr->sin_family != dst_addr->sin_family)
  1558. return -EINVAL;
  1559. work = kzalloc(sizeof *work, GFP_KERNEL);
  1560. if (!work)
  1561. return -ENOMEM;
  1562. work->id = id_priv;
  1563. INIT_WORK(&work->work, cma_work_handler);
  1564. route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
  1565. if (!route->path_rec) {
  1566. ret = -ENOMEM;
  1567. goto err1;
  1568. }
  1569. route->num_paths = 1;
  1570. if (addr->dev_addr.bound_dev_if)
  1571. ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
  1572. if (!ndev) {
  1573. ret = -ENODEV;
  1574. goto err2;
  1575. }
  1576. vid = rdma_vlan_dev_vlan_id(ndev);
  1577. iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid);
  1578. iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid);
  1579. route->path_rec->hop_limit = 1;
  1580. route->path_rec->reversible = 1;
  1581. route->path_rec->pkey = cpu_to_be16(0xffff);
  1582. route->path_rec->mtu_selector = IB_SA_EQ;
  1583. route->path_rec->sl = netdev_get_prio_tc_map(
  1584. ndev->priv_flags & IFF_802_1Q_VLAN ?
  1585. vlan_dev_real_dev(ndev) : ndev,
  1586. rt_tos2priority(id_priv->tos));
  1587. route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
  1588. route->path_rec->rate_selector = IB_SA_EQ;
  1589. route->path_rec->rate = iboe_get_rate(ndev);
  1590. dev_put(ndev);
  1591. route->path_rec->packet_life_time_selector = IB_SA_EQ;
  1592. route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
  1593. if (!route->path_rec->mtu) {
  1594. ret = -EINVAL;
  1595. goto err2;
  1596. }
  1597. work->old_state = RDMA_CM_ROUTE_QUERY;
  1598. work->new_state = RDMA_CM_ROUTE_RESOLVED;
  1599. work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
  1600. work->event.status = 0;
  1601. queue_work(cma_wq, &work->work);
  1602. return 0;
  1603. err2:
  1604. kfree(route->path_rec);
  1605. route->path_rec = NULL;
  1606. err1:
  1607. kfree(work);
  1608. return ret;
  1609. }
  1610. int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
  1611. {
  1612. struct rdma_id_private *id_priv;
  1613. int ret;
  1614. id_priv = container_of(id, struct rdma_id_private, id);
  1615. if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
  1616. return -EINVAL;
  1617. atomic_inc(&id_priv->refcount);
  1618. switch (rdma_node_get_transport(id->device->node_type)) {
  1619. case RDMA_TRANSPORT_IB:
  1620. switch (rdma_port_get_link_layer(id->device, id->port_num)) {
  1621. case IB_LINK_LAYER_INFINIBAND:
  1622. ret = cma_resolve_ib_route(id_priv, timeout_ms);
  1623. break;
  1624. case IB_LINK_LAYER_ETHERNET:
  1625. ret = cma_resolve_iboe_route(id_priv);
  1626. break;
  1627. default:
  1628. ret = -ENOSYS;
  1629. }
  1630. break;
  1631. case RDMA_TRANSPORT_IWARP:
  1632. ret = cma_resolve_iw_route(id_priv, timeout_ms);
  1633. break;
  1634. default:
  1635. ret = -ENOSYS;
  1636. break;
  1637. }
  1638. if (ret)
  1639. goto err;
  1640. return 0;
  1641. err:
  1642. cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
  1643. cma_deref_id(id_priv);
  1644. return ret;
  1645. }
  1646. EXPORT_SYMBOL(rdma_resolve_route);
  1647. static int cma_bind_loopback(struct rdma_id_private *id_priv)
  1648. {
  1649. struct cma_device *cma_dev;
  1650. struct ib_port_attr port_attr;
  1651. union ib_gid gid;
  1652. u16 pkey;
  1653. int ret;
  1654. u8 p;
  1655. mutex_lock(&lock);
  1656. if (list_empty(&dev_list)) {
  1657. ret = -ENODEV;
  1658. goto out;
  1659. }
  1660. list_for_each_entry(cma_dev, &dev_list, list)
  1661. for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
  1662. if (!ib_query_port(cma_dev->device, p, &port_attr) &&
  1663. port_attr.state == IB_PORT_ACTIVE)
  1664. goto port_found;
  1665. p = 1;
  1666. cma_dev = list_entry(dev_list.next, struct cma_device, list);
  1667. port_found:
  1668. ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
  1669. if (ret)
  1670. goto out;
  1671. ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
  1672. if (ret)
  1673. goto out;
  1674. id_priv->id.route.addr.dev_addr.dev_type =
  1675. (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ?
  1676. ARPHRD_INFINIBAND : ARPHRD_ETHER;
  1677. rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
  1678. ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
  1679. id_priv->id.port_num = p;
  1680. cma_attach_to_dev(id_priv, cma_dev);
  1681. out:
  1682. mutex_unlock(&lock);
  1683. return ret;
  1684. }
  1685. static void addr_handler(int status, struct sockaddr *src_addr,
  1686. struct rdma_dev_addr *dev_addr, void *context)
  1687. {
  1688. struct rdma_id_private *id_priv = context;
  1689. struct rdma_cm_event event;
  1690. memset(&event, 0, sizeof event);
  1691. mutex_lock(&id_priv->handler_mutex);
  1692. if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
  1693. RDMA_CM_ADDR_RESOLVED))
  1694. goto out;
  1695. if (!status && !id_priv->cma_dev)
  1696. status = cma_acquire_dev(id_priv);
  1697. if (status) {
  1698. if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
  1699. RDMA_CM_ADDR_BOUND))
  1700. goto out;
  1701. event.event = RDMA_CM_EVENT_ADDR_ERROR;
  1702. event.status = status;
  1703. } else {
  1704. memcpy(&id_priv->id.route.addr.src_addr, src_addr,
  1705. ip_addr_size(src_addr));
  1706. event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
  1707. }
  1708. if (id_priv->id.event_handler(&id_priv->id, &event)) {
  1709. cma_exch(id_priv, RDMA_CM_DESTROYING);
  1710. mutex_unlock(&id_priv->handler_mutex);
  1711. cma_deref_id(id_priv);
  1712. rdma_destroy_id(&id_priv->id);
  1713. return;
  1714. }
  1715. out:
  1716. mutex_unlock(&id_priv->handler_mutex);
  1717. cma_deref_id(id_priv);
  1718. }
  1719. static int cma_resolve_loopback(struct rdma_id_private *id_priv)
  1720. {
  1721. struct cma_work *work;
  1722. struct sockaddr *src, *dst;
  1723. union ib_gid gid;
  1724. int ret;
  1725. work = kzalloc(sizeof *work, GFP_KERNEL);
  1726. if (!work)
  1727. return -ENOMEM;
  1728. if (!id_priv->cma_dev) {
  1729. ret = cma_bind_loopback(id_priv);
  1730. if (ret)
  1731. goto err;
  1732. }
  1733. rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
  1734. rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
  1735. src = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
  1736. if (cma_zero_addr(src)) {
  1737. dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
  1738. if ((src->sa_family = dst->sa_family) == AF_INET) {
  1739. ((struct sockaddr_in *)src)->sin_addr =
  1740. ((struct sockaddr_in *)dst)->sin_addr;
  1741. } else {
  1742. ((struct sockaddr_in6 *)src)->sin6_addr =
  1743. ((struct sockaddr_in6 *)dst)->sin6_addr;
  1744. }
  1745. }
  1746. work->id = id_priv;
  1747. INIT_WORK(&work->work, cma_work_handler);
  1748. work->old_state = RDMA_CM_ADDR_QUERY;
  1749. work->new_state = RDMA_CM_ADDR_RESOLVED;
  1750. work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
  1751. queue_work(cma_wq, &work->work);
  1752. return 0;
  1753. err:
  1754. kfree(work);
  1755. return ret;
  1756. }
  1757. static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
  1758. struct sockaddr *dst_addr)
  1759. {
  1760. if (!src_addr || !src_addr->sa_family) {
  1761. src_addr = (struct sockaddr *) &id->route.addr.src_addr;
  1762. if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) {
  1763. ((struct sockaddr_in6 *) src_addr)->sin6_scope_id =
  1764. ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
  1765. }
  1766. }
  1767. return rdma_bind_addr(id, src_addr);
  1768. }
  1769. int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
  1770. struct sockaddr *dst_addr, int timeout_ms)
  1771. {
  1772. struct rdma_id_private *id_priv;
  1773. int ret;
  1774. id_priv = container_of(id, struct rdma_id_private, id);
  1775. if (id_priv->state == RDMA_CM_IDLE) {
  1776. ret = cma_bind_addr(id, src_addr, dst_addr);
  1777. if (ret)
  1778. return ret;
  1779. }
  1780. if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY))
  1781. return -EINVAL;
  1782. atomic_inc(&id_priv->refcount);
  1783. memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
  1784. if (cma_any_addr(dst_addr))
  1785. ret = cma_resolve_loopback(id_priv);
  1786. else
  1787. ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr,
  1788. dst_addr, &id->route.addr.dev_addr,
  1789. timeout_ms, addr_handler, id_priv);
  1790. if (ret)
  1791. goto err;
  1792. return 0;
  1793. err:
  1794. cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
  1795. cma_deref_id(id_priv);
  1796. return ret;
  1797. }
  1798. EXPORT_SYMBOL(rdma_resolve_addr);
  1799. int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
  1800. {
  1801. struct rdma_id_private *id_priv;
  1802. unsigned long flags;
  1803. int ret;
  1804. id_priv = container_of(id, struct rdma_id_private, id);
  1805. spin_lock_irqsave(&id_priv->lock, flags);
  1806. if (id_priv->state == RDMA_CM_IDLE) {
  1807. id_priv->reuseaddr = reuse;
  1808. ret = 0;
  1809. } else {
  1810. ret = -EINVAL;
  1811. }
  1812. spin_unlock_irqrestore(&id_priv->lock, flags);
  1813. return ret;
  1814. }
  1815. EXPORT_SYMBOL(rdma_set_reuseaddr);
  1816. int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
  1817. {
  1818. struct rdma_id_private *id_priv;
  1819. unsigned long flags;
  1820. int ret;
  1821. id_priv = container_of(id, struct rdma_id_private, id);
  1822. spin_lock_irqsave(&id_priv->lock, flags);
  1823. if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
  1824. id_priv->options |= (1 << CMA_OPTION_AFONLY);
  1825. id_priv->afonly = afonly;
  1826. ret = 0;
  1827. } else {
  1828. ret = -EINVAL;
  1829. }
  1830. spin_unlock_irqrestore(&id_priv->lock, flags);
  1831. return ret;
  1832. }
  1833. EXPORT_SYMBOL(rdma_set_afonly);
  1834. static void cma_bind_port(struct rdma_bind_list *bind_list,
  1835. struct rdma_id_private *id_priv)
  1836. {
  1837. struct sockaddr_in *sin;
  1838. sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
  1839. sin->sin_port = htons(bind_list->port);
  1840. id_priv->bind_list = bind_list;
  1841. hlist_add_head(&id_priv->node, &bind_list->owners);
  1842. }
  1843. static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
  1844. unsigned short snum)
  1845. {
  1846. struct rdma_bind_list *bind_list;
  1847. int port, ret;
  1848. bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
  1849. if (!bind_list)
  1850. return -ENOMEM;
  1851. do {
  1852. ret = idr_get_new_above(ps, bind_list, snum, &port);
  1853. } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
  1854. if (ret)
  1855. goto err1;
  1856. if (port != snum) {
  1857. ret = -EADDRNOTAVAIL;
  1858. goto err2;
  1859. }
  1860. bind_list->ps = ps;
  1861. bind_list->port = (unsigned short) port;
  1862. cma_bind_port(bind_list, id_priv);
  1863. return 0;
  1864. err2:
  1865. idr_remove(ps, port);
  1866. err1:
  1867. kfree(bind_list);
  1868. return ret;
  1869. }
  1870. static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
  1871. {
  1872. static unsigned int last_used_port;
  1873. int low, high, remaining;
  1874. unsigned int rover;
  1875. inet_get_local_port_range(&low, &high);
  1876. remaining = (high - low) + 1;
  1877. rover = net_random() % remaining + low;
  1878. retry:
  1879. if (last_used_port != rover &&
  1880. !idr_find(ps, (unsigned short) rover)) {
  1881. int ret = cma_alloc_port(ps, id_priv, rover);
  1882. /*
  1883. * Remember previously used port number in order to avoid
  1884. * re-using same port immediately after it is closed.
  1885. */
  1886. if (!ret)
  1887. last_used_port = rover;
  1888. if (ret != -EADDRNOTAVAIL)
  1889. return ret;
  1890. }
  1891. if (--remaining) {
  1892. rover++;
  1893. if ((rover < low) || (rover > high))
  1894. rover = low;
  1895. goto retry;
  1896. }
  1897. return -EADDRNOTAVAIL;
  1898. }
  1899. /*
  1900. * Check that the requested port is available. This is called when trying to
  1901. * bind to a specific port, or when trying to listen on a bound port. In
  1902. * the latter case, the provided id_priv may already be on the bind_list, but
  1903. * we still need to check that it's okay to start listening.
  1904. */
  1905. static int cma_check_port(struct rdma_bind_list *bind_list,
  1906. struct rdma_id_private *id_priv, uint8_t reuseaddr)
  1907. {
  1908. struct rdma_id_private *cur_id;
  1909. struct sockaddr *addr, *cur_addr;
  1910. struct hlist_node *node;
  1911. addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
  1912. hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
  1913. if (id_priv == cur_id)
  1914. continue;
  1915. if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr &&
  1916. cur_id->reuseaddr)
  1917. continue;
  1918. cur_addr = (struct sockaddr *) &cur_id->id.route.addr.src_addr;
  1919. if (id_priv->afonly && cur_id->afonly &&
  1920. (addr->sa_family != cur_addr->sa_family))
  1921. continue;
  1922. if (cma_any_addr(addr) || cma_any_addr(cur_addr))
  1923. return -EADDRNOTAVAIL;
  1924. if (!cma_addr_cmp(addr, cur_addr))
  1925. return -EADDRINUSE;
  1926. }
  1927. return 0;
  1928. }
  1929. static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
  1930. {
  1931. struct rdma_bind_list *bind_list;
  1932. unsigned short snum;
  1933. int ret;
  1934. snum = ntohs(cma_port((struct sockaddr *) &id_priv->id.route.addr.src_addr));
  1935. if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
  1936. return -EACCES;
  1937. bind_list = idr_find(ps, snum);
  1938. if (!bind_list) {
  1939. ret = cma_alloc_port(ps, id_priv, snum);
  1940. } else {
  1941. ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
  1942. if (!ret)
  1943. cma_bind_port(bind_list, id_priv);
  1944. }
  1945. return ret;
  1946. }
  1947. static int cma_bind_listen(struct rdma_id_private *id_priv)
  1948. {
  1949. struct rdma_bind_list *bind_list = id_priv->bind_list;
  1950. int ret = 0;
  1951. mutex_lock(&lock);
  1952. if (bind_list->owners.first->next)
  1953. ret = cma_check_port(bind_list, id_priv, 0);
  1954. mutex_unlock(&lock);
  1955. return ret;
  1956. }
  1957. static int cma_get_port(struct rdma_id_private *id_priv)
  1958. {
  1959. struct idr *ps;
  1960. int ret;
  1961. switch (id_priv->id.ps) {
  1962. case RDMA_PS_SDP:
  1963. ps = &sdp_ps;
  1964. break;
  1965. case RDMA_PS_TCP:
  1966. ps = &tcp_ps;
  1967. break;
  1968. case RDMA_PS_UDP:
  1969. ps = &udp_ps;
  1970. break;
  1971. case RDMA_PS_IPOIB:
  1972. ps = &ipoib_ps;
  1973. break;
  1974. case RDMA_PS_IB:
  1975. ps = &ib_ps;
  1976. break;
  1977. default:
  1978. return -EPROTONOSUPPORT;
  1979. }
  1980. mutex_lock(&lock);
  1981. if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr))
  1982. ret = cma_alloc_any_port(ps, id_priv);
  1983. else
  1984. ret = cma_use_port(ps, id_priv);
  1985. mutex_unlock(&lock);
  1986. return ret;
  1987. }
  1988. static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
  1989. struct sockaddr *addr)
  1990. {
  1991. #if IS_ENABLED(CONFIG_IPV6)
  1992. struct sockaddr_in6 *sin6;
  1993. if (addr->sa_family != AF_INET6)
  1994. return 0;
  1995. sin6 = (struct sockaddr_in6 *) addr;
  1996. if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) &&
  1997. !sin6->sin6_scope_id)
  1998. return -EINVAL;
  1999. dev_addr->bound_dev_if = sin6->sin6_scope_id;
  2000. #endif
  2001. return 0;
  2002. }
  2003. int rdma_listen(struct rdma_cm_id *id, int backlog)
  2004. {
  2005. struct rdma_id_private *id_priv;
  2006. int ret;
  2007. id_priv = container_of(id, struct rdma_id_private, id);
  2008. if (id_priv->state == RDMA_CM_IDLE) {
  2009. ((struct sockaddr *) &id->route.addr.src_addr)->sa_family = AF_INET;
  2010. ret = rdma_bind_addr(id, (struct sockaddr *) &id->route.addr.src_addr);
  2011. if (ret)
  2012. return ret;
  2013. }
  2014. if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
  2015. return -EINVAL;
  2016. if (id_priv->reuseaddr) {
  2017. ret = cma_bind_listen(id_priv);
  2018. if (ret)
  2019. goto err;
  2020. }
  2021. id_priv->backlog = backlog;
  2022. if (id->device) {
  2023. switch (rdma_node_get_transport(id->device->node_type)) {
  2024. case RDMA_TRANSPORT_IB:
  2025. ret = cma_ib_listen(id_priv);
  2026. if (ret)
  2027. goto err;
  2028. break;
  2029. case RDMA_TRANSPORT_IWARP:
  2030. ret = cma_iw_listen(id_priv, backlog);
  2031. if (ret)
  2032. goto err;
  2033. break;
  2034. default:
  2035. ret = -ENOSYS;
  2036. goto err;
  2037. }
  2038. } else
  2039. cma_listen_on_all(id_priv);
  2040. return 0;
  2041. err:
  2042. id_priv->backlog = 0;
  2043. cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
  2044. return ret;
  2045. }
  2046. EXPORT_SYMBOL(rdma_listen);
  2047. int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
  2048. {
  2049. struct rdma_id_private *id_priv;
  2050. int ret;
  2051. if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
  2052. return -EAFNOSUPPORT;
  2053. id_priv = container_of(id, struct rdma_id_private, id);
  2054. if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
  2055. return -EINVAL;
  2056. ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
  2057. if (ret)
  2058. goto err1;
  2059. if (!cma_any_addr(addr)) {
  2060. ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
  2061. if (ret)
  2062. goto err1;
  2063. ret = cma_acquire_dev(id_priv);
  2064. if (ret)
  2065. goto err1;
  2066. }
  2067. memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
  2068. if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
  2069. if (addr->sa_family == AF_INET)
  2070. id_priv->afonly = 1;
  2071. #if IS_ENABLED(CONFIG_IPV6)
  2072. else if (addr->sa_family == AF_INET6)
  2073. id_priv->afonly = init_net.ipv6.sysctl.bindv6only;
  2074. #endif
  2075. }
  2076. ret = cma_get_port(id_priv);
  2077. if (ret)
  2078. goto err2;
  2079. return 0;
  2080. err2:
  2081. if (id_priv->cma_dev)
  2082. cma_release_dev(id_priv);
  2083. err1:
  2084. cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
  2085. return ret;
  2086. }
  2087. EXPORT_SYMBOL(rdma_bind_addr);
  2088. static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
  2089. struct rdma_route *route)
  2090. {
  2091. struct cma_hdr *cma_hdr;
  2092. struct sdp_hh *sdp_hdr;
  2093. if (route->addr.src_addr.ss_family == AF_INET) {
  2094. struct sockaddr_in *src4, *dst4;
  2095. src4 = (struct sockaddr_in *) &route->addr.src_addr;
  2096. dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
  2097. switch (ps) {
  2098. case RDMA_PS_SDP:
  2099. sdp_hdr = hdr;
  2100. if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
  2101. return -EINVAL;
  2102. sdp_set_ip_ver(sdp_hdr, 4);
  2103. sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
  2104. sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
  2105. sdp_hdr->port = src4->sin_port;
  2106. break;
  2107. default:
  2108. cma_hdr = hdr;
  2109. cma_hdr->cma_version = CMA_VERSION;
  2110. cma_set_ip_ver(cma_hdr, 4);
  2111. cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
  2112. cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
  2113. cma_hdr->port = src4->sin_port;
  2114. break;
  2115. }
  2116. } else {
  2117. struct sockaddr_in6 *src6, *dst6;
  2118. src6 = (struct sockaddr_in6 *) &route->addr.src_addr;
  2119. dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr;
  2120. switch (ps) {
  2121. case RDMA_PS_SDP:
  2122. sdp_hdr = hdr;
  2123. if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
  2124. return -EINVAL;
  2125. sdp_set_ip_ver(sdp_hdr, 6);
  2126. sdp_hdr->src_addr.ip6 = src6->sin6_addr;
  2127. sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
  2128. sdp_hdr->port = src6->sin6_port;
  2129. break;
  2130. default:
  2131. cma_hdr = hdr;
  2132. cma_hdr->cma_version = CMA_VERSION;
  2133. cma_set_ip_ver(cma_hdr, 6);
  2134. cma_hdr->src_addr.ip6 = src6->sin6_addr;
  2135. cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
  2136. cma_hdr->port = src6->sin6_port;
  2137. break;
  2138. }
  2139. }
  2140. return 0;
  2141. }
  2142. static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
  2143. struct ib_cm_event *ib_event)
  2144. {
  2145. struct rdma_id_private *id_priv = cm_id->context;
  2146. struct rdma_cm_event event;
  2147. struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
  2148. int ret = 0;
  2149. if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
  2150. return 0;
  2151. memset(&event, 0, sizeof event);
  2152. switch (ib_event->event) {
  2153. case IB_CM_SIDR_REQ_ERROR:
  2154. event.event = RDMA_CM_EVENT_UNREACHABLE;
  2155. event.status = -ETIMEDOUT;
  2156. break;
  2157. case IB_CM_SIDR_REP_RECEIVED:
  2158. event.param.ud.private_data = ib_event->private_data;
  2159. event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
  2160. if (rep->status != IB_SIDR_SUCCESS) {
  2161. event.event = RDMA_CM_EVENT_UNREACHABLE;
  2162. event.status = ib_event->param.sidr_rep_rcvd.status;
  2163. break;
  2164. }
  2165. ret = cma_set_qkey(id_priv);
  2166. if (ret) {
  2167. event.event = RDMA_CM_EVENT_ADDR_ERROR;
  2168. event.status = -EINVAL;
  2169. break;
  2170. }
  2171. if (id_priv->qkey != rep->qkey) {
  2172. event.event = RDMA_CM_EVENT_UNREACHABLE;
  2173. event.status = -EINVAL;
  2174. break;
  2175. }
  2176. ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
  2177. id_priv->id.route.path_rec,
  2178. &event.param.ud.ah_attr);
  2179. event.param.ud.qp_num = rep->qpn;
  2180. event.param.ud.qkey = rep->qkey;
  2181. event.event = RDMA_CM_EVENT_ESTABLISHED;
  2182. event.status = 0;
  2183. break;
  2184. default:
  2185. printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
  2186. ib_event->event);
  2187. goto out;
  2188. }
  2189. ret = id_priv->id.event_handler(&id_priv->id, &event);
  2190. if (ret) {
  2191. /* Destroy the CM ID by returning a non-zero value. */
  2192. id_priv->cm_id.ib = NULL;
  2193. cma_exch(id_priv, RDMA_CM_DESTROYING);
  2194. mutex_unlock(&id_priv->handler_mutex);
  2195. rdma_destroy_id(&id_priv->id);
  2196. return ret;
  2197. }
  2198. out:
  2199. mutex_unlock(&id_priv->handler_mutex);
  2200. return ret;
  2201. }
  2202. static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
  2203. struct rdma_conn_param *conn_param)
  2204. {
  2205. struct ib_cm_sidr_req_param req;
  2206. struct rdma_route *route;
  2207. struct ib_cm_id *id;
  2208. int ret;
  2209. req.private_data_len = sizeof(struct cma_hdr) +
  2210. conn_param->private_data_len;
  2211. if (req.private_data_len < conn_param->private_data_len)
  2212. return -EINVAL;
  2213. req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
  2214. if (!req.private_data)
  2215. return -ENOMEM;
  2216. if (conn_param->private_data && conn_param->private_data_len)
  2217. memcpy((void *) req.private_data + sizeof(struct cma_hdr),
  2218. conn_param->private_data, conn_param->private_data_len);
  2219. route = &id_priv->id.route;
  2220. ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
  2221. if (ret)
  2222. goto out;
  2223. id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
  2224. id_priv);
  2225. if (IS_ERR(id)) {
  2226. ret = PTR_ERR(id);
  2227. goto out;
  2228. }
  2229. id_priv->cm_id.ib = id;
  2230. req.path = route->path_rec;
  2231. req.service_id = cma_get_service_id(id_priv->id.ps,
  2232. (struct sockaddr *) &route->addr.dst_addr);
  2233. req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
  2234. req.max_cm_retries = CMA_MAX_CM_RETRIES;
  2235. ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
  2236. if (ret) {
  2237. ib_destroy_cm_id(id_priv->cm_id.ib);
  2238. id_priv->cm_id.ib = NULL;
  2239. }
  2240. out:
  2241. kfree(req.private_data);
  2242. return ret;
  2243. }
  2244. static int cma_connect_ib(struct rdma_id_private *id_priv,
  2245. struct rdma_conn_param *conn_param)
  2246. {
  2247. struct ib_cm_req_param req;
  2248. struct rdma_route *route;
  2249. void *private_data;
  2250. struct ib_cm_id *id;
  2251. int offset, ret;
  2252. memset(&req, 0, sizeof req);
  2253. offset = cma_user_data_offset(id_priv->id.ps);
  2254. req.private_data_len = offset + conn_param->private_data_len;
  2255. if (req.private_data_len < conn_param->private_data_len)
  2256. return -EINVAL;
  2257. private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
  2258. if (!private_data)
  2259. return -ENOMEM;
  2260. if (conn_param->private_data && conn_param->private_data_len)
  2261. memcpy(private_data + offset, conn_param->private_data,
  2262. conn_param->private_data_len);
  2263. id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
  2264. if (IS_ERR(id)) {
  2265. ret = PTR_ERR(id);
  2266. goto out;
  2267. }
  2268. id_priv->cm_id.ib = id;
  2269. route = &id_priv->id.route;
  2270. ret = cma_format_hdr(private_data, id_priv->id.ps, route);
  2271. if (ret)
  2272. goto out;
  2273. req.private_data = private_data;
  2274. req.primary_path = &route->path_rec[0];
  2275. if (route->num_paths == 2)
  2276. req.alternate_path = &route->path_rec[1];
  2277. req.service_id = cma_get_service_id(id_priv->id.ps,
  2278. (struct sockaddr *) &route->addr.dst_addr);
  2279. req.qp_num = id_priv->qp_num;
  2280. req.qp_type = id_priv->id.qp_type;
  2281. req.starting_psn = id_priv->seq_num;
  2282. req.responder_resources = conn_param->responder_resources;
  2283. req.initiator_depth = conn_param->initiator_depth;
  2284. req.flow_control = conn_param->flow_control;
  2285. req.retry_count = min_t(u8, 7, conn_param->retry_count);
  2286. req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
  2287. req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
  2288. req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
  2289. req.max_cm_retries = CMA_MAX_CM_RETRIES;
  2290. req.srq = id_priv->srq ? 1 : 0;
  2291. ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
  2292. out:
  2293. if (ret && !IS_ERR(id)) {
  2294. ib_destroy_cm_id(id);
  2295. id_priv->cm_id.ib = NULL;
  2296. }
  2297. kfree(private_data);
  2298. return ret;
  2299. }
  2300. static int cma_connect_iw(struct rdma_id_private *id_priv,
  2301. struct rdma_conn_param *conn_param)
  2302. {
  2303. struct iw_cm_id *cm_id;
  2304. struct sockaddr_in* sin;
  2305. int ret;
  2306. struct iw_cm_conn_param iw_param;
  2307. cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
  2308. if (IS_ERR(cm_id))
  2309. return PTR_ERR(cm_id);
  2310. id_priv->cm_id.iw = cm_id;
  2311. sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
  2312. cm_id->local_addr = *sin;
  2313. sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
  2314. cm_id->remote_addr = *sin;
  2315. ret = cma_modify_qp_rtr(id_priv, conn_param);
  2316. if (ret)
  2317. goto out;
  2318. if (conn_param) {
  2319. iw_param.ord = conn_param->initiator_depth;
  2320. iw_param.ird = conn_param->responder_resources;
  2321. iw_param.private_data = conn_param->private_data;
  2322. iw_param.private_data_len = conn_param->private_data_len;
  2323. iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
  2324. } else {
  2325. memset(&iw_param, 0, sizeof iw_param);
  2326. iw_param.qpn = id_priv->qp_num;
  2327. }
  2328. ret = iw_cm_connect(cm_id, &iw_param);
  2329. out:
  2330. if (ret) {
  2331. iw_destroy_cm_id(cm_id);
  2332. id_priv->cm_id.iw = NULL;
  2333. }
  2334. return ret;
  2335. }
  2336. int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
  2337. {
  2338. struct rdma_id_private *id_priv;
  2339. int ret;
  2340. id_priv = container_of(id, struct rdma_id_private, id);
  2341. if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
  2342. return -EINVAL;
  2343. if (!id->qp) {
  2344. id_priv->qp_num = conn_param->qp_num;
  2345. id_priv->srq = conn_param->srq;
  2346. }
  2347. switch (rdma_node_get_transport(id->device->node_type)) {
  2348. case RDMA_TRANSPORT_IB:
  2349. if (id->qp_type == IB_QPT_UD)
  2350. ret = cma_resolve_ib_udp(id_priv, conn_param);
  2351. else
  2352. ret = cma_connect_ib(id_priv, conn_param);
  2353. break;
  2354. case RDMA_TRANSPORT_IWARP:
  2355. ret = cma_connect_iw(id_priv, conn_param);
  2356. break;
  2357. default:
  2358. ret = -ENOSYS;
  2359. break;
  2360. }
  2361. if (ret)
  2362. goto err;
  2363. return 0;
  2364. err:
  2365. cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
  2366. return ret;
  2367. }
  2368. EXPORT_SYMBOL(rdma_connect);
  2369. static int cma_accept_ib(struct rdma_id_private *id_priv,
  2370. struct rdma_conn_param *conn_param)
  2371. {
  2372. struct ib_cm_rep_param rep;
  2373. int ret;
  2374. ret = cma_modify_qp_rtr(id_priv, conn_param);
  2375. if (ret)
  2376. goto out;
  2377. ret = cma_modify_qp_rts(id_priv, conn_param);
  2378. if (ret)
  2379. goto out;
  2380. memset(&rep, 0, sizeof rep);
  2381. rep.qp_num = id_priv->qp_num;
  2382. rep.starting_psn = id_priv->seq_num;
  2383. rep.private_data = conn_param->private_data;
  2384. rep.private_data_len = conn_param->private_data_len;
  2385. rep.responder_resources = conn_param->responder_resources;
  2386. rep.initiator_depth = conn_param->initiator_depth;
  2387. rep.failover_accepted = 0;
  2388. rep.flow_control = conn_param->flow_control;
  2389. rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
  2390. rep.srq = id_priv->srq ? 1 : 0;
  2391. ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
  2392. out:
  2393. return ret;
  2394. }
  2395. static int cma_accept_iw(struct rdma_id_private *id_priv,
  2396. struct rdma_conn_param *conn_param)
  2397. {
  2398. struct iw_cm_conn_param iw_param;
  2399. int ret;
  2400. ret = cma_modify_qp_rtr(id_priv, conn_param);
  2401. if (ret)
  2402. return ret;
  2403. iw_param.ord = conn_param->initiator_depth;
  2404. iw_param.ird = conn_param->responder_resources;
  2405. iw_param.private_data = conn_param->private_data;
  2406. iw_param.private_data_len = conn_param->private_data_len;
  2407. if (id_priv->id.qp) {
  2408. iw_param.qpn = id_priv->qp_num;
  2409. } else
  2410. iw_param.qpn = conn_param->qp_num;
  2411. return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
  2412. }
  2413. static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
  2414. enum ib_cm_sidr_status status,
  2415. const void *private_data, int private_data_len)
  2416. {
  2417. struct ib_cm_sidr_rep_param rep;
  2418. int ret;
  2419. memset(&rep, 0, sizeof rep);
  2420. rep.status = status;
  2421. if (status == IB_SIDR_SUCCESS) {
  2422. ret = cma_set_qkey(id_priv);
  2423. if (ret)
  2424. return ret;
  2425. rep.qp_num = id_priv->qp_num;
  2426. rep.qkey = id_priv->qkey;
  2427. }
  2428. rep.private_data = private_data;
  2429. rep.private_data_len = private_data_len;
  2430. return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
  2431. }
  2432. int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
  2433. {
  2434. struct rdma_id_private *id_priv;
  2435. int ret;
  2436. id_priv = container_of(id, struct rdma_id_private, id);
  2437. id_priv->owner = task_pid_nr(current);
  2438. if (!cma_comp(id_priv, RDMA_CM_CONNECT))
  2439. return -EINVAL;
  2440. if (!id->qp && conn_param) {
  2441. id_priv->qp_num = conn_param->qp_num;
  2442. id_priv->srq = conn_param->srq;
  2443. }
  2444. switch (rdma_node_get_transport(id->device->node_type)) {
  2445. case RDMA_TRANSPORT_IB:
  2446. if (id->qp_type == IB_QPT_UD) {
  2447. if (conn_param)
  2448. ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
  2449. conn_param->private_data,
  2450. conn_param->private_data_len);
  2451. else
  2452. ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
  2453. NULL, 0);
  2454. } else {
  2455. if (conn_param)
  2456. ret = cma_accept_ib(id_priv, conn_param);
  2457. else
  2458. ret = cma_rep_recv(id_priv);
  2459. }
  2460. break;
  2461. case RDMA_TRANSPORT_IWARP:
  2462. ret = cma_accept_iw(id_priv, conn_param);
  2463. break;
  2464. default:
  2465. ret = -ENOSYS;
  2466. break;
  2467. }
  2468. if (ret)
  2469. goto reject;
  2470. return 0;
  2471. reject:
  2472. cma_modify_qp_err(id_priv);
  2473. rdma_reject(id, NULL, 0);
  2474. return ret;
  2475. }
  2476. EXPORT_SYMBOL(rdma_accept);
  2477. int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
  2478. {
  2479. struct rdma_id_private *id_priv;
  2480. int ret;
  2481. id_priv = container_of(id, struct rdma_id_private, id);
  2482. if (!id_priv->cm_id.ib)
  2483. return -EINVAL;
  2484. switch (id->device->node_type) {
  2485. case RDMA_NODE_IB_CA:
  2486. ret = ib_cm_notify(id_priv->cm_id.ib, event);
  2487. break;
  2488. default:
  2489. ret = 0;
  2490. break;
  2491. }
  2492. return ret;
  2493. }
  2494. EXPORT_SYMBOL(rdma_notify);
  2495. int rdma_reject(struct rdma_cm_id *id, const void *private_data,
  2496. u8 private_data_len)
  2497. {
  2498. struct rdma_id_private *id_priv;
  2499. int ret;
  2500. id_priv = container_of(id, struct rdma_id_private, id);
  2501. if (!id_priv->cm_id.ib)
  2502. return -EINVAL;
  2503. switch (rdma_node_get_transport(id->device->node_type)) {
  2504. case RDMA_TRANSPORT_IB:
  2505. if (id->qp_type == IB_QPT_UD)
  2506. ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
  2507. private_data, private_data_len);
  2508. else
  2509. ret = ib_send_cm_rej(id_priv->cm_id.ib,
  2510. IB_CM_REJ_CONSUMER_DEFINED, NULL,
  2511. 0, private_data, private_data_len);
  2512. break;
  2513. case RDMA_TRANSPORT_IWARP:
  2514. ret = iw_cm_reject(id_priv->cm_id.iw,
  2515. private_data, private_data_len);
  2516. break;
  2517. default:
  2518. ret = -ENOSYS;
  2519. break;
  2520. }
  2521. return ret;
  2522. }
  2523. EXPORT_SYMBOL(rdma_reject);
  2524. int rdma_disconnect(struct rdma_cm_id *id)
  2525. {
  2526. struct rdma_id_private *id_priv;
  2527. int ret;
  2528. id_priv = container_of(id, struct rdma_id_private, id);
  2529. if (!id_priv->cm_id.ib)
  2530. return -EINVAL;
  2531. switch (rdma_node_get_transport(id->device->node_type)) {
  2532. case RDMA_TRANSPORT_IB:
  2533. ret = cma_modify_qp_err(id_priv);
  2534. if (ret)
  2535. goto out;
  2536. /* Initiate or respond to a disconnect. */
  2537. if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
  2538. ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
  2539. break;
  2540. case RDMA_TRANSPORT_IWARP:
  2541. ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
  2542. break;
  2543. default:
  2544. ret = -EINVAL;
  2545. break;
  2546. }
  2547. out:
  2548. return ret;
  2549. }
  2550. EXPORT_SYMBOL(rdma_disconnect);
  2551. static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
  2552. {
  2553. struct rdma_id_private *id_priv;
  2554. struct cma_multicast *mc = multicast->context;
  2555. struct rdma_cm_event event;
  2556. int ret;
  2557. id_priv = mc->id_priv;
  2558. if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) &&
  2559. cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED))
  2560. return 0;
  2561. mutex_lock(&id_priv->qp_mutex);
  2562. if (!status && id_priv->id.qp)
  2563. status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
  2564. be16_to_cpu(multicast->rec.mlid));
  2565. mutex_unlock(&id_priv->qp_mutex);
  2566. memset(&event, 0, sizeof event);
  2567. event.status = status;
  2568. event.param.ud.private_data = mc->context;
  2569. if (!status) {
  2570. event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
  2571. ib_init_ah_from_mcmember(id_priv->id.device,
  2572. id_priv->id.port_num, &multicast->rec,
  2573. &event.param.ud.ah_attr);
  2574. event.param.ud.qp_num = 0xFFFFFF;
  2575. event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
  2576. } else
  2577. event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
  2578. ret = id_priv->id.event_handler(&id_priv->id, &event);
  2579. if (ret) {
  2580. cma_exch(id_priv, RDMA_CM_DESTROYING);
  2581. mutex_unlock(&id_priv->handler_mutex);
  2582. rdma_destroy_id(&id_priv->id);
  2583. return 0;
  2584. }
  2585. mutex_unlock(&id_priv->handler_mutex);
  2586. return 0;
  2587. }
  2588. static void cma_set_mgid(struct rdma_id_private *id_priv,
  2589. struct sockaddr *addr, union ib_gid *mgid)
  2590. {
  2591. unsigned char mc_map[MAX_ADDR_LEN];
  2592. struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
  2593. struct sockaddr_in *sin = (struct sockaddr_in *) addr;
  2594. struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
  2595. if (cma_any_addr(addr)) {
  2596. memset(mgid, 0, sizeof *mgid);
  2597. } else if ((addr->sa_family == AF_INET6) &&
  2598. ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
  2599. 0xFF10A01B)) {
  2600. /* IPv6 address is an SA assigned MGID. */
  2601. memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
  2602. } else if ((addr->sa_family == AF_INET6)) {
  2603. ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
  2604. if (id_priv->id.ps == RDMA_PS_UDP)
  2605. mc_map[7] = 0x01; /* Use RDMA CM signature */
  2606. *mgid = *(union ib_gid *) (mc_map + 4);
  2607. } else {
  2608. ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
  2609. if (id_priv->id.ps == RDMA_PS_UDP)
  2610. mc_map[7] = 0x01; /* Use RDMA CM signature */
  2611. *mgid = *(union ib_gid *) (mc_map + 4);
  2612. }
  2613. }
  2614. static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
  2615. struct cma_multicast *mc)
  2616. {
  2617. struct ib_sa_mcmember_rec rec;
  2618. struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
  2619. ib_sa_comp_mask comp_mask;
  2620. int ret;
  2621. ib_addr_get_mgid(dev_addr, &rec.mgid);
  2622. ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
  2623. &rec.mgid, &rec);
  2624. if (ret)
  2625. return ret;
  2626. cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
  2627. if (id_priv->id.ps == RDMA_PS_UDP)
  2628. rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
  2629. rdma_addr_get_sgid(dev_addr, &rec.port_gid);
  2630. rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
  2631. rec.join_state = 1;
  2632. comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
  2633. IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
  2634. IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
  2635. IB_SA_MCMEMBER_REC_FLOW_LABEL |
  2636. IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
  2637. if (id_priv->id.ps == RDMA_PS_IPOIB)
  2638. comp_mask |= IB_SA_MCMEMBER_REC_RATE |
  2639. IB_SA_MCMEMBER_REC_RATE_SELECTOR |
  2640. IB_SA_MCMEMBER_REC_MTU_SELECTOR |
  2641. IB_SA_MCMEMBER_REC_MTU |
  2642. IB_SA_MCMEMBER_REC_HOP_LIMIT;
  2643. mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
  2644. id_priv->id.port_num, &rec,
  2645. comp_mask, GFP_KERNEL,
  2646. cma_ib_mc_handler, mc);
  2647. return PTR_RET(mc->multicast.ib);
  2648. }
  2649. static void iboe_mcast_work_handler(struct work_struct *work)
  2650. {
  2651. struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
  2652. struct cma_multicast *mc = mw->mc;
  2653. struct ib_sa_multicast *m = mc->multicast.ib;
  2654. mc->multicast.ib->context = mc;
  2655. cma_ib_mc_handler(0, m);
  2656. kref_put(&mc->mcref, release_mc);
  2657. kfree(mw);
  2658. }
  2659. static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
  2660. {
  2661. struct sockaddr_in *sin = (struct sockaddr_in *)addr;
  2662. struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
  2663. if (cma_any_addr(addr)) {
  2664. memset(mgid, 0, sizeof *mgid);
  2665. } else if (addr->sa_family == AF_INET6) {
  2666. memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
  2667. } else {
  2668. mgid->raw[0] = 0xff;
  2669. mgid->raw[1] = 0x0e;
  2670. mgid->raw[2] = 0;
  2671. mgid->raw[3] = 0;
  2672. mgid->raw[4] = 0;
  2673. mgid->raw[5] = 0;
  2674. mgid->raw[6] = 0;
  2675. mgid->raw[7] = 0;
  2676. mgid->raw[8] = 0;
  2677. mgid->raw[9] = 0;
  2678. mgid->raw[10] = 0xff;
  2679. mgid->raw[11] = 0xff;
  2680. *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
  2681. }
  2682. }
  2683. static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
  2684. struct cma_multicast *mc)
  2685. {
  2686. struct iboe_mcast_work *work;
  2687. struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
  2688. int err;
  2689. struct sockaddr *addr = (struct sockaddr *)&mc->addr;
  2690. struct net_device *ndev = NULL;
  2691. if (cma_zero_addr((struct sockaddr *)&mc->addr))
  2692. return -EINVAL;
  2693. work = kzalloc(sizeof *work, GFP_KERNEL);
  2694. if (!work)
  2695. return -ENOMEM;
  2696. mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
  2697. if (!mc->multicast.ib) {
  2698. err = -ENOMEM;
  2699. goto out1;
  2700. }
  2701. cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);
  2702. mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
  2703. if (id_priv->id.ps == RDMA_PS_UDP)
  2704. mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
  2705. if (dev_addr->bound_dev_if)
  2706. ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
  2707. if (!ndev) {
  2708. err = -ENODEV;
  2709. goto out2;
  2710. }
  2711. mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
  2712. mc->multicast.ib->rec.hop_limit = 1;
  2713. mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
  2714. dev_put(ndev);
  2715. if (!mc->multicast.ib->rec.mtu) {
  2716. err = -EINVAL;
  2717. goto out2;
  2718. }
  2719. iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid);
  2720. work->id = id_priv;
  2721. work->mc = mc;
  2722. INIT_WORK(&work->work, iboe_mcast_work_handler);
  2723. kref_get(&mc->mcref);
  2724. queue_work(cma_wq, &work->work);
  2725. return 0;
  2726. out2:
  2727. kfree(mc->multicast.ib);
  2728. out1:
  2729. kfree(work);
  2730. return err;
  2731. }
  2732. int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
  2733. void *context)
  2734. {
  2735. struct rdma_id_private *id_priv;
  2736. struct cma_multicast *mc;
  2737. int ret;
  2738. id_priv = container_of(id, struct rdma_id_private, id);
  2739. if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
  2740. !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
  2741. return -EINVAL;
  2742. mc = kmalloc(sizeof *mc, GFP_KERNEL);
  2743. if (!mc)
  2744. return -ENOMEM;
  2745. memcpy(&mc->addr, addr, ip_addr_size(addr));
  2746. mc->context = context;
  2747. mc->id_priv = id_priv;
  2748. spin_lock(&id_priv->lock);
  2749. list_add(&mc->list, &id_priv->mc_list);
  2750. spin_unlock(&id_priv->lock);
  2751. switch (rdma_node_get_transport(id->device->node_type)) {
  2752. case RDMA_TRANSPORT_IB:
  2753. switch (rdma_port_get_link_layer(id->device, id->port_num)) {
  2754. case IB_LINK_LAYER_INFINIBAND:
  2755. ret = cma_join_ib_multicast(id_priv, mc);
  2756. break;
  2757. case IB_LINK_LAYER_ETHERNET:
  2758. kref_init(&mc->mcref);
  2759. ret = cma_iboe_join_multicast(id_priv, mc);
  2760. break;
  2761. default:
  2762. ret = -EINVAL;
  2763. }
  2764. break;
  2765. default:
  2766. ret = -ENOSYS;
  2767. break;
  2768. }
  2769. if (ret) {
  2770. spin_lock_irq(&id_priv->lock);
  2771. list_del(&mc->list);
  2772. spin_unlock_irq(&id_priv->lock);
  2773. kfree(mc);
  2774. }
  2775. return ret;
  2776. }
  2777. EXPORT_SYMBOL(rdma_join_multicast);
  2778. void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
  2779. {
  2780. struct rdma_id_private *id_priv;
  2781. struct cma_multicast *mc;
  2782. id_priv = container_of(id, struct rdma_id_private, id);
  2783. spin_lock_irq(&id_priv->lock);
  2784. list_for_each_entry(mc, &id_priv->mc_list, list) {
  2785. if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
  2786. list_del(&mc->list);
  2787. spin_unlock_irq(&id_priv->lock);
  2788. if (id->qp)
  2789. ib_detach_mcast(id->qp,
  2790. &mc->multicast.ib->rec.mgid,
  2791. be16_to_cpu(mc->multicast.ib->rec.mlid));
  2792. if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) {
  2793. switch (rdma_port_get_link_layer(id->device, id->port_num)) {
  2794. case IB_LINK_LAYER_INFINIBAND:
  2795. ib_sa_free_multicast(mc->multicast.ib);
  2796. kfree(mc);
  2797. break;
  2798. case IB_LINK_LAYER_ETHERNET:
  2799. kref_put(&mc->mcref, release_mc);
  2800. break;
  2801. default:
  2802. break;
  2803. }
  2804. }
  2805. return;
  2806. }
  2807. }
  2808. spin_unlock_irq(&id_priv->lock);
  2809. }
  2810. EXPORT_SYMBOL(rdma_leave_multicast);
  2811. static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
  2812. {
  2813. struct rdma_dev_addr *dev_addr;
  2814. struct cma_ndev_work *work;
  2815. dev_addr = &id_priv->id.route.addr.dev_addr;
  2816. if ((dev_addr->bound_dev_if == ndev->ifindex) &&
  2817. memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
  2818. printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
  2819. ndev->name, &id_priv->id);
  2820. work = kzalloc(sizeof *work, GFP_KERNEL);
  2821. if (!work)
  2822. return -ENOMEM;
  2823. INIT_WORK(&work->work, cma_ndev_work_handler);
  2824. work->id = id_priv;
  2825. work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
  2826. atomic_inc(&id_priv->refcount);
  2827. queue_work(cma_wq, &work->work);
  2828. }
  2829. return 0;
  2830. }
  2831. static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
  2832. void *ctx)
  2833. {
  2834. struct net_device *ndev = (struct net_device *)ctx;
  2835. struct cma_device *cma_dev;
  2836. struct rdma_id_private *id_priv;
  2837. int ret = NOTIFY_DONE;
  2838. if (dev_net(ndev) != &init_net)
  2839. return NOTIFY_DONE;
  2840. if (event != NETDEV_BONDING_FAILOVER)
  2841. return NOTIFY_DONE;
  2842. if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
  2843. return NOTIFY_DONE;
  2844. mutex_lock(&lock);
  2845. list_for_each_entry(cma_dev, &dev_list, list)
  2846. list_for_each_entry(id_priv, &cma_dev->id_list, list) {
  2847. ret = cma_netdev_change(ndev, id_priv);
  2848. if (ret)
  2849. goto out;
  2850. }
  2851. out:
  2852. mutex_unlock(&lock);
  2853. return ret;
  2854. }
  2855. static struct notifier_block cma_nb = {
  2856. .notifier_call = cma_netdev_callback
  2857. };
  2858. static void cma_add_one(struct ib_device *device)
  2859. {
  2860. struct cma_device *cma_dev;
  2861. struct rdma_id_private *id_priv;
  2862. cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
  2863. if (!cma_dev)
  2864. return;
  2865. cma_dev->device = device;
  2866. init_completion(&cma_dev->comp);
  2867. atomic_set(&cma_dev->refcount, 1);
  2868. INIT_LIST_HEAD(&cma_dev->id_list);
  2869. ib_set_client_data(device, &cma_client, cma_dev);
  2870. mutex_lock(&lock);
  2871. list_add_tail(&cma_dev->list, &dev_list);
  2872. list_for_each_entry(id_priv, &listen_any_list, list)
  2873. cma_listen_on_dev(id_priv, cma_dev);
  2874. mutex_unlock(&lock);
  2875. }
  2876. static int cma_remove_id_dev(struct rdma_id_private *id_priv)
  2877. {
  2878. struct rdma_cm_event event;
  2879. enum rdma_cm_state state;
  2880. int ret = 0;
  2881. /* Record that we want to remove the device */
  2882. state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
  2883. if (state == RDMA_CM_DESTROYING)
  2884. return 0;
  2885. cma_cancel_operation(id_priv, state);
  2886. mutex_lock(&id_priv->handler_mutex);
  2887. /* Check for destruction from another callback. */
  2888. if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
  2889. goto out;
  2890. memset(&event, 0, sizeof event);
  2891. event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
  2892. ret = id_priv->id.event_handler(&id_priv->id, &event);
  2893. out:
  2894. mutex_unlock(&id_priv->handler_mutex);
  2895. return ret;
  2896. }
  2897. static void cma_process_remove(struct cma_device *cma_dev)
  2898. {
  2899. struct rdma_id_private *id_priv;
  2900. int ret;
  2901. mutex_lock(&lock);
  2902. while (!list_empty(&cma_dev->id_list)) {
  2903. id_priv = list_entry(cma_dev->id_list.next,
  2904. struct rdma_id_private, list);
  2905. list_del(&id_priv->listen_list);
  2906. list_del_init(&id_priv->list);
  2907. atomic_inc(&id_priv->refcount);
  2908. mutex_unlock(&lock);
  2909. ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
  2910. cma_deref_id(id_priv);
  2911. if (ret)
  2912. rdma_destroy_id(&id_priv->id);
  2913. mutex_lock(&lock);
  2914. }
  2915. mutex_unlock(&lock);
  2916. cma_deref_dev(cma_dev);
  2917. wait_for_completion(&cma_dev->comp);
  2918. }
  2919. static void cma_remove_one(struct ib_device *device)
  2920. {
  2921. struct cma_device *cma_dev;
  2922. cma_dev = ib_get_client_data(device, &cma_client);
  2923. if (!cma_dev)
  2924. return;
  2925. mutex_lock(&lock);
  2926. list_del(&cma_dev->list);
  2927. mutex_unlock(&lock);
  2928. cma_process_remove(cma_dev);
  2929. kfree(cma_dev);
  2930. }
  2931. static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
  2932. {
  2933. struct nlmsghdr *nlh;
  2934. struct rdma_cm_id_stats *id_stats;
  2935. struct rdma_id_private *id_priv;
  2936. struct rdma_cm_id *id = NULL;
  2937. struct cma_device *cma_dev;
  2938. int i_dev = 0, i_id = 0;
  2939. /*
  2940. * We export all of the IDs as a sequence of messages. Each
  2941. * ID gets its own netlink message.
  2942. */
  2943. mutex_lock(&lock);
  2944. list_for_each_entry(cma_dev, &dev_list, list) {
  2945. if (i_dev < cb->args[0]) {
  2946. i_dev++;
  2947. continue;
  2948. }
  2949. i_id = 0;
  2950. list_for_each_entry(id_priv, &cma_dev->id_list, list) {
  2951. if (i_id < cb->args[1]) {
  2952. i_id++;
  2953. continue;
  2954. }
  2955. id_stats = ibnl_put_msg(skb, &nlh, cb->nlh->nlmsg_seq,
  2956. sizeof *id_stats, RDMA_NL_RDMA_CM,
  2957. RDMA_NL_RDMA_CM_ID_STATS);
  2958. if (!id_stats)
  2959. goto out;
  2960. memset(id_stats, 0, sizeof *id_stats);
  2961. id = &id_priv->id;
  2962. id_stats->node_type = id->route.addr.dev_addr.dev_type;
  2963. id_stats->port_num = id->port_num;
  2964. id_stats->bound_dev_if =
  2965. id->route.addr.dev_addr.bound_dev_if;
  2966. if (id->route.addr.src_addr.ss_family == AF_INET) {
  2967. if (ibnl_put_attr(skb, nlh,
  2968. sizeof(struct sockaddr_in),
  2969. &id->route.addr.src_addr,
  2970. RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
  2971. goto out;
  2972. }
  2973. if (ibnl_put_attr(skb, nlh,
  2974. sizeof(struct sockaddr_in),
  2975. &id->route.addr.dst_addr,
  2976. RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
  2977. goto out;
  2978. }
  2979. } else if (id->route.addr.src_addr.ss_family == AF_INET6) {
  2980. if (ibnl_put_attr(skb, nlh,
  2981. sizeof(struct sockaddr_in6),
  2982. &id->route.addr.src_addr,
  2983. RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
  2984. goto out;
  2985. }
  2986. if (ibnl_put_attr(skb, nlh,
  2987. sizeof(struct sockaddr_in6),
  2988. &id->route.addr.dst_addr,
  2989. RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
  2990. goto out;
  2991. }
  2992. }
  2993. id_stats->pid = id_priv->owner;
  2994. id_stats->port_space = id->ps;
  2995. id_stats->cm_state = id_priv->state;
  2996. id_stats->qp_num = id_priv->qp_num;
  2997. id_stats->qp_type = id->qp_type;
  2998. i_id++;
  2999. }
  3000. cb->args[1] = 0;
  3001. i_dev++;
  3002. }
  3003. out:
  3004. mutex_unlock(&lock);
  3005. cb->args[0] = i_dev;
  3006. cb->args[1] = i_id;
  3007. return skb->len;
  3008. }
  3009. static const struct ibnl_client_cbs cma_cb_table[] = {
  3010. [RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats,
  3011. .module = THIS_MODULE },
  3012. };
  3013. static int __init cma_init(void)
  3014. {
  3015. int ret;
  3016. cma_wq = create_singlethread_workqueue("rdma_cm");
  3017. if (!cma_wq)
  3018. return -ENOMEM;
  3019. ib_sa_register_client(&sa_client);
  3020. rdma_addr_register_client(&addr_client);
  3021. register_netdevice_notifier(&cma_nb);
  3022. ret = ib_register_client(&cma_client);
  3023. if (ret)
  3024. goto err;
  3025. if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))
  3026. printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n");
  3027. return 0;
  3028. err:
  3029. unregister_netdevice_notifier(&cma_nb);
  3030. rdma_addr_unregister_client(&addr_client);
  3031. ib_sa_unregister_client(&sa_client);
  3032. destroy_workqueue(cma_wq);
  3033. return ret;
  3034. }
  3035. static void __exit cma_cleanup(void)
  3036. {
  3037. ibnl_remove_client(RDMA_NL_RDMA_CM);
  3038. ib_unregister_client(&cma_client);
  3039. unregister_netdevice_notifier(&cma_nb);
  3040. rdma_addr_unregister_client(&addr_client);
  3041. ib_sa_unregister_client(&sa_client);
  3042. destroy_workqueue(cma_wq);
  3043. idr_destroy(&sdp_ps);
  3044. idr_destroy(&tcp_ps);
  3045. idr_destroy(&udp_ps);
  3046. idr_destroy(&ipoib_ps);
  3047. idr_destroy(&ib_ps);
  3048. }
  3049. module_init(cma_init);
  3050. module_exit(cma_cleanup);