iscsi_tcp.c 61 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319
  1. /*
  2. * iSCSI Initiator over TCP/IP Data-Path
  3. *
  4. * Copyright (C) 2004 Dmitry Yusupov
  5. * Copyright (C) 2004 Alex Aizman
  6. * Copyright (C) 2005 - 2006 Mike Christie
  7. * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
  8. * maintained by open-iscsi@googlegroups.com
  9. *
  10. * This program is free software; you can redistribute it and/or modify
  11. * it under the terms of the GNU General Public License as published
  12. * by the Free Software Foundation; either version 2 of the License, or
  13. * (at your option) any later version.
  14. *
  15. * This program is distributed in the hope that it will be useful, but
  16. * WITHOUT ANY WARRANTY; without even the implied warranty of
  17. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18. * General Public License for more details.
  19. *
  20. * See the file COPYING included with this distribution for more details.
  21. *
  22. * Credits:
  23. * Christoph Hellwig
  24. * FUJITA Tomonori
  25. * Arne Redlich
  26. * Zhenyu Wang
  27. */
  28. #include <linux/types.h>
  29. #include <linux/list.h>
  30. #include <linux/inet.h>
  31. #include <linux/file.h>
  32. #include <linux/blkdev.h>
  33. #include <linux/crypto.h>
  34. #include <linux/delay.h>
  35. #include <linux/kfifo.h>
  36. #include <linux/scatterlist.h>
  37. #include <net/tcp.h>
  38. #include <scsi/scsi_cmnd.h>
  39. #include <scsi/scsi_device.h>
  40. #include <scsi/scsi_host.h>
  41. #include <scsi/scsi.h>
  42. #include <scsi/scsi_transport_iscsi.h>
  43. #include "iscsi_tcp.h"
  44. MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
  45. "Alex Aizman <itn780@yahoo.com>");
  46. MODULE_DESCRIPTION("iSCSI/TCP data-path");
  47. MODULE_LICENSE("GPL");
  48. /* #define DEBUG_TCP */
  49. #define DEBUG_ASSERT
  50. #ifdef DEBUG_TCP
  51. #define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
  52. #else
  53. #define debug_tcp(fmt...)
  54. #endif
  55. #ifndef DEBUG_ASSERT
  56. #ifdef BUG_ON
  57. #undef BUG_ON
  58. #endif
  59. #define BUG_ON(expr)
  60. #endif
  61. static unsigned int iscsi_max_lun = 512;
  62. module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
  63. static inline void
  64. iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
  65. {
  66. sg_init_one(&ibuf->sg, vbuf, size);
  67. ibuf->sent = 0;
  68. ibuf->use_sendmsg = 1;
  69. }
  70. static inline void
  71. iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
  72. {
  73. sg_init_table(&ibuf->sg, 1);
  74. sg_set_page(&ibuf->sg, sg_page(sg), sg->length, sg->offset);
  75. /*
  76. * Fastpath: sg element fits into single page
  77. */
  78. if (sg->length + sg->offset <= PAGE_SIZE && !PageSlab(sg_page(sg)))
  79. ibuf->use_sendmsg = 0;
  80. else
  81. ibuf->use_sendmsg = 1;
  82. ibuf->sent = 0;
  83. }
  84. static inline int
  85. iscsi_buf_left(struct iscsi_buf *ibuf)
  86. {
  87. int rc;
  88. rc = ibuf->sg.length - ibuf->sent;
  89. BUG_ON(rc < 0);
  90. return rc;
  91. }
  92. static inline void
  93. iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
  94. u8* crc)
  95. {
  96. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  97. crypto_hash_digest(&tcp_conn->tx_hash, &buf->sg, buf->sg.length, crc);
  98. buf->sg.length += sizeof(u32);
  99. }
  100. static inline int
  101. iscsi_hdr_extract(struct iscsi_tcp_conn *tcp_conn)
  102. {
  103. struct sk_buff *skb = tcp_conn->in.skb;
  104. tcp_conn->in.zero_copy_hdr = 0;
  105. if (tcp_conn->in.copy >= tcp_conn->hdr_size &&
  106. tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
  107. /*
  108. * Zero-copy PDU Header: using connection context
  109. * to store header pointer.
  110. */
  111. if (skb_shinfo(skb)->frag_list == NULL &&
  112. !skb_shinfo(skb)->nr_frags) {
  113. tcp_conn->in.hdr = (struct iscsi_hdr *)
  114. ((char*)skb->data + tcp_conn->in.offset);
  115. tcp_conn->in.zero_copy_hdr = 1;
  116. } else {
  117. /* ignoring return code since we checked
  118. * in.copy before */
  119. skb_copy_bits(skb, tcp_conn->in.offset,
  120. &tcp_conn->hdr, tcp_conn->hdr_size);
  121. tcp_conn->in.hdr = &tcp_conn->hdr;
  122. }
  123. tcp_conn->in.offset += tcp_conn->hdr_size;
  124. tcp_conn->in.copy -= tcp_conn->hdr_size;
  125. } else {
  126. int hdr_remains;
  127. int copylen;
  128. /*
  129. * PDU header scattered across SKB's,
  130. * copying it... This'll happen quite rarely.
  131. */
  132. if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER)
  133. tcp_conn->in.hdr_offset = 0;
  134. hdr_remains = tcp_conn->hdr_size - tcp_conn->in.hdr_offset;
  135. BUG_ON(hdr_remains <= 0);
  136. copylen = min(tcp_conn->in.copy, hdr_remains);
  137. skb_copy_bits(skb, tcp_conn->in.offset,
  138. (char*)&tcp_conn->hdr + tcp_conn->in.hdr_offset,
  139. copylen);
  140. debug_tcp("PDU gather offset %d bytes %d in.offset %d "
  141. "in.copy %d\n", tcp_conn->in.hdr_offset, copylen,
  142. tcp_conn->in.offset, tcp_conn->in.copy);
  143. tcp_conn->in.offset += copylen;
  144. tcp_conn->in.copy -= copylen;
  145. if (copylen < hdr_remains) {
  146. tcp_conn->in_progress = IN_PROGRESS_HEADER_GATHER;
  147. tcp_conn->in.hdr_offset += copylen;
  148. return -EAGAIN;
  149. }
  150. tcp_conn->in.hdr = &tcp_conn->hdr;
  151. tcp_conn->discontiguous_hdr_cnt++;
  152. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  153. }
  154. return 0;
  155. }
  156. /*
  157. * must be called with session lock
  158. */
  159. static void
  160. iscsi_tcp_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  161. {
  162. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  163. struct iscsi_r2t_info *r2t;
  164. struct scsi_cmnd *sc;
  165. /* flush ctask's r2t queues */
  166. while (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
  167. __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
  168. sizeof(void*));
  169. debug_scsi("iscsi_tcp_cleanup_ctask pending r2t dropped\n");
  170. }
  171. sc = ctask->sc;
  172. if (unlikely(!sc))
  173. return;
  174. tcp_ctask->xmstate = XMSTATE_IDLE;
  175. tcp_ctask->r2t = NULL;
  176. }
  177. /**
  178. * iscsi_data_rsp - SCSI Data-In Response processing
  179. * @conn: iscsi connection
  180. * @ctask: scsi command task
  181. **/
  182. static int
  183. iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  184. {
  185. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  186. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  187. struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
  188. struct iscsi_session *session = conn->session;
  189. struct scsi_cmnd *sc = ctask->sc;
  190. int datasn = be32_to_cpu(rhdr->datasn);
  191. iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr);
  192. /*
  193. * setup Data-In byte counter (gets decremented..)
  194. */
  195. ctask->data_count = tcp_conn->in.datalen;
  196. if (tcp_conn->in.datalen == 0)
  197. return 0;
  198. if (tcp_ctask->exp_datasn != datasn) {
  199. debug_tcp("%s: ctask->exp_datasn(%d) != rhdr->datasn(%d)\n",
  200. __FUNCTION__, tcp_ctask->exp_datasn, datasn);
  201. return ISCSI_ERR_DATASN;
  202. }
  203. tcp_ctask->exp_datasn++;
  204. tcp_ctask->data_offset = be32_to_cpu(rhdr->offset);
  205. if (tcp_ctask->data_offset + tcp_conn->in.datalen > scsi_bufflen(sc)) {
  206. debug_tcp("%s: data_offset(%d) + data_len(%d) > total_length_in(%d)\n",
  207. __FUNCTION__, tcp_ctask->data_offset,
  208. tcp_conn->in.datalen, scsi_bufflen(sc));
  209. return ISCSI_ERR_DATA_OFFSET;
  210. }
  211. if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
  212. conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
  213. if (rhdr->flags & ISCSI_FLAG_DATA_UNDERFLOW) {
  214. int res_count = be32_to_cpu(rhdr->residual_count);
  215. if (res_count > 0 &&
  216. res_count <= scsi_bufflen(sc)) {
  217. scsi_set_resid(sc, res_count);
  218. sc->result = (DID_OK << 16) | rhdr->cmd_status;
  219. } else
  220. sc->result = (DID_BAD_TARGET << 16) |
  221. rhdr->cmd_status;
  222. } else if (rhdr->flags & ISCSI_FLAG_DATA_OVERFLOW) {
  223. scsi_set_resid(sc, be32_to_cpu(rhdr->residual_count));
  224. sc->result = (DID_OK << 16) | rhdr->cmd_status;
  225. } else
  226. sc->result = (DID_OK << 16) | rhdr->cmd_status;
  227. }
  228. conn->datain_pdus_cnt++;
  229. return 0;
  230. }
  231. /**
  232. * iscsi_solicit_data_init - initialize first Data-Out
  233. * @conn: iscsi connection
  234. * @ctask: scsi command task
  235. * @r2t: R2T info
  236. *
  237. * Notes:
  238. * Initialize first Data-Out within this R2T sequence and finds
  239. * proper data_offset within this SCSI command.
  240. *
  241. * This function is called with connection lock taken.
  242. **/
  243. static void
  244. iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
  245. struct iscsi_r2t_info *r2t)
  246. {
  247. struct iscsi_data *hdr;
  248. struct scsi_cmnd *sc = ctask->sc;
  249. int i, sg_count = 0;
  250. struct scatterlist *sg;
  251. hdr = &r2t->dtask.hdr;
  252. memset(hdr, 0, sizeof(struct iscsi_data));
  253. hdr->ttt = r2t->ttt;
  254. hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
  255. r2t->solicit_datasn++;
  256. hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
  257. memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
  258. hdr->itt = ctask->hdr->itt;
  259. hdr->exp_statsn = r2t->exp_statsn;
  260. hdr->offset = cpu_to_be32(r2t->data_offset);
  261. if (r2t->data_length > conn->max_xmit_dlength) {
  262. hton24(hdr->dlength, conn->max_xmit_dlength);
  263. r2t->data_count = conn->max_xmit_dlength;
  264. hdr->flags = 0;
  265. } else {
  266. hton24(hdr->dlength, r2t->data_length);
  267. r2t->data_count = r2t->data_length;
  268. hdr->flags = ISCSI_FLAG_CMD_FINAL;
  269. }
  270. conn->dataout_pdus_cnt++;
  271. r2t->sent = 0;
  272. iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
  273. sizeof(struct iscsi_hdr));
  274. sg = scsi_sglist(sc);
  275. r2t->sg = NULL;
  276. for (i = 0; i < scsi_sg_count(sc); i++, sg += 1) {
  277. /* FIXME: prefetch ? */
  278. if (sg_count + sg->length > r2t->data_offset) {
  279. int page_offset;
  280. /* sg page found! */
  281. /* offset within this page */
  282. page_offset = r2t->data_offset - sg_count;
  283. /* fill in this buffer */
  284. iscsi_buf_init_sg(&r2t->sendbuf, sg);
  285. r2t->sendbuf.sg.offset += page_offset;
  286. r2t->sendbuf.sg.length -= page_offset;
  287. /* xmit logic will continue with next one */
  288. r2t->sg = sg + 1;
  289. break;
  290. }
  291. sg_count += sg->length;
  292. }
  293. BUG_ON(r2t->sg == NULL);
  294. }
  295. /**
  296. * iscsi_r2t_rsp - iSCSI R2T Response processing
  297. * @conn: iscsi connection
  298. * @ctask: scsi command task
  299. **/
  300. static int
  301. iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  302. {
  303. struct iscsi_r2t_info *r2t;
  304. struct iscsi_session *session = conn->session;
  305. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  306. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  307. struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
  308. int r2tsn = be32_to_cpu(rhdr->r2tsn);
  309. int rc;
  310. if (tcp_conn->in.datalen) {
  311. printk(KERN_ERR "iscsi_tcp: invalid R2t with datalen %d\n",
  312. tcp_conn->in.datalen);
  313. return ISCSI_ERR_DATALEN;
  314. }
  315. if (tcp_ctask->exp_datasn != r2tsn){
  316. debug_tcp("%s: ctask->exp_datasn(%d) != rhdr->r2tsn(%d)\n",
  317. __FUNCTION__, tcp_ctask->exp_datasn, r2tsn);
  318. return ISCSI_ERR_R2TSN;
  319. }
  320. /* fill-in new R2T associated with the task */
  321. spin_lock(&session->lock);
  322. iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr);
  323. if (!ctask->sc || ctask->mtask ||
  324. session->state != ISCSI_STATE_LOGGED_IN) {
  325. printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
  326. "recovery...\n", ctask->itt);
  327. spin_unlock(&session->lock);
  328. return 0;
  329. }
  330. rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
  331. BUG_ON(!rc);
  332. r2t->exp_statsn = rhdr->statsn;
  333. r2t->data_length = be32_to_cpu(rhdr->data_length);
  334. if (r2t->data_length == 0) {
  335. printk(KERN_ERR "iscsi_tcp: invalid R2T with zero data len\n");
  336. spin_unlock(&session->lock);
  337. return ISCSI_ERR_DATALEN;
  338. }
  339. if (r2t->data_length > session->max_burst)
  340. debug_scsi("invalid R2T with data len %u and max burst %u."
  341. "Attempting to execute request.\n",
  342. r2t->data_length, session->max_burst);
  343. r2t->data_offset = be32_to_cpu(rhdr->data_offset);
  344. if (r2t->data_offset + r2t->data_length > scsi_bufflen(ctask->sc)) {
  345. spin_unlock(&session->lock);
  346. printk(KERN_ERR "iscsi_tcp: invalid R2T with data len %u at "
  347. "offset %u and total length %d\n", r2t->data_length,
  348. r2t->data_offset, scsi_bufflen(ctask->sc));
  349. return ISCSI_ERR_DATALEN;
  350. }
  351. r2t->ttt = rhdr->ttt; /* no flip */
  352. r2t->solicit_datasn = 0;
  353. iscsi_solicit_data_init(conn, ctask, r2t);
  354. tcp_ctask->exp_datasn = r2tsn + 1;
  355. __kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*));
  356. tcp_ctask->xmstate |= XMSTATE_SOL_HDR_INIT;
  357. list_move_tail(&ctask->running, &conn->xmitqueue);
  358. scsi_queue_work(session->host, &conn->xmitwork);
  359. conn->r2t_pdus_cnt++;
  360. spin_unlock(&session->lock);
  361. return 0;
  362. }
  363. static int
  364. iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
  365. {
  366. int rc = 0, opcode, ahslen;
  367. struct iscsi_hdr *hdr;
  368. struct iscsi_session *session = conn->session;
  369. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  370. uint32_t cdgst, rdgst = 0, itt;
  371. hdr = tcp_conn->in.hdr;
  372. /* verify PDU length */
  373. tcp_conn->in.datalen = ntoh24(hdr->dlength);
  374. if (tcp_conn->in.datalen > conn->max_recv_dlength) {
  375. printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
  376. tcp_conn->in.datalen, conn->max_recv_dlength);
  377. return ISCSI_ERR_DATALEN;
  378. }
  379. tcp_conn->data_copied = 0;
  380. /* read AHS */
  381. ahslen = hdr->hlength << 2;
  382. tcp_conn->in.offset += ahslen;
  383. tcp_conn->in.copy -= ahslen;
  384. if (tcp_conn->in.copy < 0) {
  385. printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
  386. "%d bytes\n", ahslen);
  387. return ISCSI_ERR_AHSLEN;
  388. }
  389. /* calculate read padding */
  390. tcp_conn->in.padding = tcp_conn->in.datalen & (ISCSI_PAD_LEN-1);
  391. if (tcp_conn->in.padding) {
  392. tcp_conn->in.padding = ISCSI_PAD_LEN - tcp_conn->in.padding;
  393. debug_scsi("read padding %d bytes\n", tcp_conn->in.padding);
  394. }
  395. if (conn->hdrdgst_en) {
  396. struct scatterlist sg;
  397. sg_init_one(&sg, (u8 *)hdr,
  398. sizeof(struct iscsi_hdr) + ahslen);
  399. crypto_hash_digest(&tcp_conn->rx_hash, &sg, sg.length,
  400. (u8 *)&cdgst);
  401. rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
  402. ahslen);
  403. if (cdgst != rdgst) {
  404. printk(KERN_ERR "iscsi_tcp: hdrdgst error "
  405. "recv 0x%x calc 0x%x\n", rdgst, cdgst);
  406. return ISCSI_ERR_HDR_DGST;
  407. }
  408. }
  409. opcode = hdr->opcode & ISCSI_OPCODE_MASK;
  410. /* verify itt (itt encoding: age+cid+itt) */
  411. rc = iscsi_verify_itt(conn, hdr, &itt);
  412. if (rc == ISCSI_ERR_NO_SCSI_CMD) {
  413. tcp_conn->in.datalen = 0; /* force drop */
  414. return 0;
  415. } else if (rc)
  416. return rc;
  417. debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
  418. opcode, tcp_conn->in.offset, tcp_conn->in.copy,
  419. ahslen, tcp_conn->in.datalen);
  420. switch(opcode) {
  421. case ISCSI_OP_SCSI_DATA_IN:
  422. tcp_conn->in.ctask = session->cmds[itt];
  423. rc = iscsi_data_rsp(conn, tcp_conn->in.ctask);
  424. if (rc)
  425. return rc;
  426. /* fall through */
  427. case ISCSI_OP_SCSI_CMD_RSP:
  428. tcp_conn->in.ctask = session->cmds[itt];
  429. if (tcp_conn->in.datalen)
  430. goto copy_hdr;
  431. spin_lock(&session->lock);
  432. rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
  433. spin_unlock(&session->lock);
  434. break;
  435. case ISCSI_OP_R2T:
  436. tcp_conn->in.ctask = session->cmds[itt];
  437. if (ahslen)
  438. rc = ISCSI_ERR_AHSLEN;
  439. else if (tcp_conn->in.ctask->sc->sc_data_direction ==
  440. DMA_TO_DEVICE)
  441. rc = iscsi_r2t_rsp(conn, tcp_conn->in.ctask);
  442. else
  443. rc = ISCSI_ERR_PROTO;
  444. break;
  445. case ISCSI_OP_LOGIN_RSP:
  446. case ISCSI_OP_TEXT_RSP:
  447. case ISCSI_OP_REJECT:
  448. case ISCSI_OP_ASYNC_EVENT:
  449. /*
  450. * It is possible that we could get a PDU with a buffer larger
  451. * than 8K, but there are no targets that currently do this.
  452. * For now we fail until we find a vendor that needs it
  453. */
  454. if (ISCSI_DEF_MAX_RECV_SEG_LEN <
  455. tcp_conn->in.datalen) {
  456. printk(KERN_ERR "iscsi_tcp: received buffer of len %u "
  457. "but conn buffer is only %u (opcode %0x)\n",
  458. tcp_conn->in.datalen,
  459. ISCSI_DEF_MAX_RECV_SEG_LEN, opcode);
  460. rc = ISCSI_ERR_PROTO;
  461. break;
  462. }
  463. if (tcp_conn->in.datalen)
  464. goto copy_hdr;
  465. /* fall through */
  466. case ISCSI_OP_LOGOUT_RSP:
  467. case ISCSI_OP_NOOP_IN:
  468. case ISCSI_OP_SCSI_TMFUNC_RSP:
  469. rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
  470. break;
  471. default:
  472. rc = ISCSI_ERR_BAD_OPCODE;
  473. break;
  474. }
  475. return rc;
  476. copy_hdr:
  477. /*
  478. * if we did zero copy for the header but we will need multiple
  479. * skbs to complete the command then we have to copy the header
  480. * for later use
  481. */
  482. if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy <=
  483. (tcp_conn->in.datalen + tcp_conn->in.padding +
  484. (conn->datadgst_en ? 4 : 0))) {
  485. debug_tcp("Copying header for later use. in.copy %d in.datalen"
  486. " %d\n", tcp_conn->in.copy, tcp_conn->in.datalen);
  487. memcpy(&tcp_conn->hdr, tcp_conn->in.hdr,
  488. sizeof(struct iscsi_hdr));
  489. tcp_conn->in.hdr = &tcp_conn->hdr;
  490. tcp_conn->in.zero_copy_hdr = 0;
  491. }
  492. return 0;
  493. }
  494. /**
  495. * iscsi_ctask_copy - copy skb bits to the destanation cmd task
  496. * @conn: iscsi tcp connection
  497. * @ctask: scsi command task
  498. * @buf: buffer to copy to
  499. * @buf_size: size of buffer
  500. * @offset: offset within the buffer
  501. *
  502. * Notes:
  503. * The function calls skb_copy_bits() and updates per-connection and
  504. * per-cmd byte counters.
  505. *
  506. * Read counters (in bytes):
  507. *
  508. * conn->in.offset offset within in progress SKB
  509. * conn->in.copy left to copy from in progress SKB
  510. * including padding
  511. * conn->in.copied copied already from in progress SKB
  512. * conn->data_copied copied already from in progress buffer
  513. * ctask->sent total bytes sent up to the MidLayer
  514. * ctask->data_count left to copy from in progress Data-In
  515. * buf_left left to copy from in progress buffer
  516. **/
  517. static inline int
  518. iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
  519. void *buf, int buf_size, int offset)
  520. {
  521. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  522. int buf_left = buf_size - (tcp_conn->data_copied + offset);
  523. unsigned size = min(tcp_conn->in.copy, buf_left);
  524. int rc;
  525. size = min(size, ctask->data_count);
  526. debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
  527. size, tcp_conn->in.offset, tcp_conn->in.copied);
  528. BUG_ON(size <= 0);
  529. BUG_ON(tcp_ctask->sent + size > scsi_bufflen(ctask->sc));
  530. rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
  531. (char*)buf + (offset + tcp_conn->data_copied), size);
  532. /* must fit into skb->len */
  533. BUG_ON(rc);
  534. tcp_conn->in.offset += size;
  535. tcp_conn->in.copy -= size;
  536. tcp_conn->in.copied += size;
  537. tcp_conn->data_copied += size;
  538. tcp_ctask->sent += size;
  539. ctask->data_count -= size;
  540. BUG_ON(tcp_conn->in.copy < 0);
  541. BUG_ON(ctask->data_count < 0);
  542. if (buf_size != (tcp_conn->data_copied + offset)) {
  543. if (!ctask->data_count) {
  544. BUG_ON(buf_size - tcp_conn->data_copied < 0);
  545. /* done with this PDU */
  546. return buf_size - tcp_conn->data_copied;
  547. }
  548. return -EAGAIN;
  549. }
  550. /* done with this buffer or with both - PDU and buffer */
  551. tcp_conn->data_copied = 0;
  552. return 0;
  553. }
  554. /**
  555. * iscsi_tcp_copy - copy skb bits to the destanation buffer
  556. * @conn: iscsi tcp connection
  557. *
  558. * Notes:
  559. * The function calls skb_copy_bits() and updates per-connection
  560. * byte counters.
  561. **/
  562. static inline int
  563. iscsi_tcp_copy(struct iscsi_conn *conn, int buf_size)
  564. {
  565. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  566. int buf_left = buf_size - tcp_conn->data_copied;
  567. int size = min(tcp_conn->in.copy, buf_left);
  568. int rc;
  569. debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
  570. size, tcp_conn->in.offset, tcp_conn->data_copied);
  571. BUG_ON(size <= 0);
  572. rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
  573. (char*)conn->data + tcp_conn->data_copied, size);
  574. BUG_ON(rc);
  575. tcp_conn->in.offset += size;
  576. tcp_conn->in.copy -= size;
  577. tcp_conn->in.copied += size;
  578. tcp_conn->data_copied += size;
  579. if (buf_size != tcp_conn->data_copied)
  580. return -EAGAIN;
  581. return 0;
  582. }
  583. static inline void
  584. partial_sg_digest_update(struct hash_desc *desc, struct scatterlist *sg,
  585. int offset, int length)
  586. {
  587. struct scatterlist temp;
  588. sg_init_table(&temp, 1);
  589. sg_set_page(&temp, sg_page(sg), length, offset);
  590. crypto_hash_update(desc, &temp, length);
  591. }
  592. static void
  593. iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
  594. {
  595. struct scatterlist tmp;
  596. sg_init_one(&tmp, buf, len);
  597. crypto_hash_update(&tcp_conn->rx_hash, &tmp, len);
  598. }
  599. static int iscsi_scsi_data_in(struct iscsi_conn *conn)
  600. {
  601. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  602. struct iscsi_cmd_task *ctask = tcp_conn->in.ctask;
  603. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  604. struct scsi_cmnd *sc = ctask->sc;
  605. struct scatterlist *sg;
  606. int i, offset, rc = 0;
  607. BUG_ON((void*)ctask != sc->SCp.ptr);
  608. offset = tcp_ctask->data_offset;
  609. sg = scsi_sglist(sc);
  610. if (tcp_ctask->data_offset)
  611. for (i = 0; i < tcp_ctask->sg_count; i++)
  612. offset -= sg[i].length;
  613. /* we've passed through partial sg*/
  614. if (offset < 0)
  615. offset = 0;
  616. for (i = tcp_ctask->sg_count; i < scsi_sg_count(sc); i++) {
  617. char *dest;
  618. dest = kmap_atomic(sg_page(&sg[i]), KM_SOFTIRQ0);
  619. rc = iscsi_ctask_copy(tcp_conn, ctask, dest + sg[i].offset,
  620. sg[i].length, offset);
  621. kunmap_atomic(dest, KM_SOFTIRQ0);
  622. if (rc == -EAGAIN)
  623. /* continue with the next SKB/PDU */
  624. return rc;
  625. if (!rc) {
  626. if (conn->datadgst_en) {
  627. if (!offset)
  628. crypto_hash_update(
  629. &tcp_conn->rx_hash,
  630. &sg[i], sg[i].length);
  631. else
  632. partial_sg_digest_update(
  633. &tcp_conn->rx_hash,
  634. &sg[i],
  635. sg[i].offset + offset,
  636. sg[i].length - offset);
  637. }
  638. offset = 0;
  639. tcp_ctask->sg_count++;
  640. }
  641. if (!ctask->data_count) {
  642. if (rc && conn->datadgst_en)
  643. /*
  644. * data-in is complete, but buffer not...
  645. */
  646. partial_sg_digest_update(&tcp_conn->rx_hash,
  647. &sg[i],
  648. sg[i].offset,
  649. sg[i].length-rc);
  650. rc = 0;
  651. break;
  652. }
  653. if (!tcp_conn->in.copy)
  654. return -EAGAIN;
  655. }
  656. BUG_ON(ctask->data_count);
  657. /* check for non-exceptional status */
  658. if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) {
  659. debug_scsi("done [sc %lx res %d itt 0x%x flags 0x%x]\n",
  660. (long)sc, sc->result, ctask->itt,
  661. tcp_conn->in.hdr->flags);
  662. spin_lock(&conn->session->lock);
  663. __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
  664. spin_unlock(&conn->session->lock);
  665. }
  666. return rc;
  667. }
  668. static int
  669. iscsi_data_recv(struct iscsi_conn *conn)
  670. {
  671. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  672. int rc = 0, opcode;
  673. opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
  674. switch (opcode) {
  675. case ISCSI_OP_SCSI_DATA_IN:
  676. rc = iscsi_scsi_data_in(conn);
  677. break;
  678. case ISCSI_OP_SCSI_CMD_RSP:
  679. case ISCSI_OP_TEXT_RSP:
  680. case ISCSI_OP_LOGIN_RSP:
  681. case ISCSI_OP_ASYNC_EVENT:
  682. case ISCSI_OP_REJECT:
  683. /*
  684. * Collect data segment to the connection's data
  685. * placeholder
  686. */
  687. if (iscsi_tcp_copy(conn, tcp_conn->in.datalen)) {
  688. rc = -EAGAIN;
  689. goto exit;
  690. }
  691. rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, conn->data,
  692. tcp_conn->in.datalen);
  693. if (!rc && conn->datadgst_en && opcode != ISCSI_OP_LOGIN_RSP)
  694. iscsi_recv_digest_update(tcp_conn, conn->data,
  695. tcp_conn->in.datalen);
  696. break;
  697. default:
  698. BUG_ON(1);
  699. }
  700. exit:
  701. return rc;
  702. }
  703. /**
  704. * iscsi_tcp_data_recv - TCP receive in sendfile fashion
  705. * @rd_desc: read descriptor
  706. * @skb: socket buffer
  707. * @offset: offset in skb
  708. * @len: skb->len - offset
  709. **/
  710. static int
  711. iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
  712. unsigned int offset, size_t len)
  713. {
  714. int rc;
  715. struct iscsi_conn *conn = rd_desc->arg.data;
  716. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  717. int processed;
  718. char pad[ISCSI_PAD_LEN];
  719. struct scatterlist sg;
  720. /*
  721. * Save current SKB and its offset in the corresponding
  722. * connection context.
  723. */
  724. tcp_conn->in.copy = skb->len - offset;
  725. tcp_conn->in.offset = offset;
  726. tcp_conn->in.skb = skb;
  727. tcp_conn->in.len = tcp_conn->in.copy;
  728. BUG_ON(tcp_conn->in.copy <= 0);
  729. debug_tcp("in %d bytes\n", tcp_conn->in.copy);
  730. more:
  731. tcp_conn->in.copied = 0;
  732. rc = 0;
  733. if (unlikely(conn->suspend_rx)) {
  734. debug_tcp("conn %d Rx suspended!\n", conn->id);
  735. return 0;
  736. }
  737. if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
  738. tcp_conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
  739. rc = iscsi_hdr_extract(tcp_conn);
  740. if (rc) {
  741. if (rc == -EAGAIN)
  742. goto nomore;
  743. else {
  744. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  745. return 0;
  746. }
  747. }
  748. /*
  749. * Verify and process incoming PDU header.
  750. */
  751. rc = iscsi_tcp_hdr_recv(conn);
  752. if (!rc && tcp_conn->in.datalen) {
  753. if (conn->datadgst_en)
  754. crypto_hash_init(&tcp_conn->rx_hash);
  755. tcp_conn->in_progress = IN_PROGRESS_DATA_RECV;
  756. } else if (rc) {
  757. iscsi_conn_failure(conn, rc);
  758. return 0;
  759. }
  760. }
  761. if (tcp_conn->in_progress == IN_PROGRESS_DDIGEST_RECV &&
  762. tcp_conn->in.copy) {
  763. uint32_t recv_digest;
  764. debug_tcp("extra data_recv offset %d copy %d\n",
  765. tcp_conn->in.offset, tcp_conn->in.copy);
  766. if (!tcp_conn->data_copied) {
  767. if (tcp_conn->in.padding) {
  768. debug_tcp("padding -> %d\n",
  769. tcp_conn->in.padding);
  770. memset(pad, 0, tcp_conn->in.padding);
  771. sg_init_one(&sg, pad, tcp_conn->in.padding);
  772. crypto_hash_update(&tcp_conn->rx_hash,
  773. &sg, sg.length);
  774. }
  775. crypto_hash_final(&tcp_conn->rx_hash,
  776. (u8 *) &tcp_conn->in.datadgst);
  777. debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst);
  778. }
  779. rc = iscsi_tcp_copy(conn, sizeof(uint32_t));
  780. if (rc) {
  781. if (rc == -EAGAIN)
  782. goto again;
  783. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  784. return 0;
  785. }
  786. memcpy(&recv_digest, conn->data, sizeof(uint32_t));
  787. if (recv_digest != tcp_conn->in.datadgst) {
  788. debug_tcp("iscsi_tcp: data digest error!"
  789. "0x%x != 0x%x\n", recv_digest,
  790. tcp_conn->in.datadgst);
  791. iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
  792. return 0;
  793. } else {
  794. debug_tcp("iscsi_tcp: data digest match!"
  795. "0x%x == 0x%x\n", recv_digest,
  796. tcp_conn->in.datadgst);
  797. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  798. }
  799. }
  800. if (tcp_conn->in_progress == IN_PROGRESS_DATA_RECV &&
  801. tcp_conn->in.copy) {
  802. debug_tcp("data_recv offset %d copy %d\n",
  803. tcp_conn->in.offset, tcp_conn->in.copy);
  804. rc = iscsi_data_recv(conn);
  805. if (rc) {
  806. if (rc == -EAGAIN)
  807. goto again;
  808. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  809. return 0;
  810. }
  811. if (tcp_conn->in.padding)
  812. tcp_conn->in_progress = IN_PROGRESS_PAD_RECV;
  813. else if (conn->datadgst_en)
  814. tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
  815. else
  816. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  817. tcp_conn->data_copied = 0;
  818. }
  819. if (tcp_conn->in_progress == IN_PROGRESS_PAD_RECV &&
  820. tcp_conn->in.copy) {
  821. int copylen = min(tcp_conn->in.padding - tcp_conn->data_copied,
  822. tcp_conn->in.copy);
  823. tcp_conn->in.copy -= copylen;
  824. tcp_conn->in.offset += copylen;
  825. tcp_conn->data_copied += copylen;
  826. if (tcp_conn->data_copied != tcp_conn->in.padding)
  827. tcp_conn->in_progress = IN_PROGRESS_PAD_RECV;
  828. else if (conn->datadgst_en)
  829. tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
  830. else
  831. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  832. tcp_conn->data_copied = 0;
  833. }
  834. debug_tcp("f, processed %d from out of %d padding %d\n",
  835. tcp_conn->in.offset - offset, (int)len, tcp_conn->in.padding);
  836. BUG_ON(tcp_conn->in.offset - offset > len);
  837. if (tcp_conn->in.offset - offset != len) {
  838. debug_tcp("continue to process %d bytes\n",
  839. (int)len - (tcp_conn->in.offset - offset));
  840. goto more;
  841. }
  842. nomore:
  843. processed = tcp_conn->in.offset - offset;
  844. BUG_ON(processed == 0);
  845. return processed;
  846. again:
  847. processed = tcp_conn->in.offset - offset;
  848. debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
  849. processed, (int)len, (int)rd_desc->count);
  850. BUG_ON(processed == 0);
  851. BUG_ON(processed > len);
  852. conn->rxdata_octets += processed;
  853. return processed;
  854. }
  855. static void
  856. iscsi_tcp_data_ready(struct sock *sk, int flag)
  857. {
  858. struct iscsi_conn *conn = sk->sk_user_data;
  859. read_descriptor_t rd_desc;
  860. read_lock(&sk->sk_callback_lock);
  861. /*
  862. * Use rd_desc to pass 'conn' to iscsi_tcp_data_recv.
  863. * We set count to 1 because we want the network layer to
  864. * hand us all the skbs that are available. iscsi_tcp_data_recv
  865. * handled pdus that cross buffers or pdus that still need data.
  866. */
  867. rd_desc.arg.data = conn;
  868. rd_desc.count = 1;
  869. tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
  870. read_unlock(&sk->sk_callback_lock);
  871. }
  872. static void
  873. iscsi_tcp_state_change(struct sock *sk)
  874. {
  875. struct iscsi_tcp_conn *tcp_conn;
  876. struct iscsi_conn *conn;
  877. struct iscsi_session *session;
  878. void (*old_state_change)(struct sock *);
  879. read_lock(&sk->sk_callback_lock);
  880. conn = (struct iscsi_conn*)sk->sk_user_data;
  881. session = conn->session;
  882. if ((sk->sk_state == TCP_CLOSE_WAIT ||
  883. sk->sk_state == TCP_CLOSE) &&
  884. !atomic_read(&sk->sk_rmem_alloc)) {
  885. debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
  886. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  887. }
  888. tcp_conn = conn->dd_data;
  889. old_state_change = tcp_conn->old_state_change;
  890. read_unlock(&sk->sk_callback_lock);
  891. old_state_change(sk);
  892. }
  893. /**
  894. * iscsi_write_space - Called when more output buffer space is available
  895. * @sk: socket space is available for
  896. **/
  897. static void
  898. iscsi_write_space(struct sock *sk)
  899. {
  900. struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
  901. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  902. tcp_conn->old_write_space(sk);
  903. debug_tcp("iscsi_write_space: cid %d\n", conn->id);
  904. scsi_queue_work(conn->session->host, &conn->xmitwork);
  905. }
  906. static void
  907. iscsi_conn_set_callbacks(struct iscsi_conn *conn)
  908. {
  909. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  910. struct sock *sk = tcp_conn->sock->sk;
  911. /* assign new callbacks */
  912. write_lock_bh(&sk->sk_callback_lock);
  913. sk->sk_user_data = conn;
  914. tcp_conn->old_data_ready = sk->sk_data_ready;
  915. tcp_conn->old_state_change = sk->sk_state_change;
  916. tcp_conn->old_write_space = sk->sk_write_space;
  917. sk->sk_data_ready = iscsi_tcp_data_ready;
  918. sk->sk_state_change = iscsi_tcp_state_change;
  919. sk->sk_write_space = iscsi_write_space;
  920. write_unlock_bh(&sk->sk_callback_lock);
  921. }
  922. static void
  923. iscsi_conn_restore_callbacks(struct iscsi_tcp_conn *tcp_conn)
  924. {
  925. struct sock *sk = tcp_conn->sock->sk;
  926. /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
  927. write_lock_bh(&sk->sk_callback_lock);
  928. sk->sk_user_data = NULL;
  929. sk->sk_data_ready = tcp_conn->old_data_ready;
  930. sk->sk_state_change = tcp_conn->old_state_change;
  931. sk->sk_write_space = tcp_conn->old_write_space;
  932. sk->sk_no_check = 0;
  933. write_unlock_bh(&sk->sk_callback_lock);
  934. }
  935. /**
  936. * iscsi_send - generic send routine
  937. * @sk: kernel's socket
  938. * @buf: buffer to write from
  939. * @size: actual size to write
  940. * @flags: socket's flags
  941. */
  942. static inline int
  943. iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
  944. {
  945. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  946. struct socket *sk = tcp_conn->sock;
  947. int offset = buf->sg.offset + buf->sent, res;
  948. /*
  949. * if we got use_sg=0 or are sending something we kmallocd
  950. * then we did not have to do kmap (kmap returns page_address)
  951. *
  952. * if we got use_sg > 0, but had to drop down, we do not
  953. * set clustering so this should only happen for that
  954. * slab case.
  955. */
  956. if (buf->use_sendmsg)
  957. res = sock_no_sendpage(sk, sg_page(&buf->sg), offset, size, flags);
  958. else
  959. res = tcp_conn->sendpage(sk, sg_page(&buf->sg), offset, size, flags);
  960. if (res >= 0) {
  961. conn->txdata_octets += res;
  962. buf->sent += res;
  963. return res;
  964. }
  965. tcp_conn->sendpage_failures_cnt++;
  966. if (res == -EAGAIN)
  967. res = -ENOBUFS;
  968. else
  969. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  970. return res;
  971. }
  972. /**
  973. * iscsi_sendhdr - send PDU Header via tcp_sendpage()
  974. * @conn: iscsi connection
  975. * @buf: buffer to write from
  976. * @datalen: lenght of data to be sent after the header
  977. *
  978. * Notes:
  979. * (Tx, Fast Path)
  980. **/
  981. static inline int
  982. iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
  983. {
  984. int flags = 0; /* MSG_DONTWAIT; */
  985. int res, size;
  986. size = buf->sg.length - buf->sent;
  987. BUG_ON(buf->sent + size > buf->sg.length);
  988. if (buf->sent + size != buf->sg.length || datalen)
  989. flags |= MSG_MORE;
  990. res = iscsi_send(conn, buf, size, flags);
  991. debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
  992. if (res >= 0) {
  993. if (size != res)
  994. return -EAGAIN;
  995. return 0;
  996. }
  997. return res;
  998. }
  999. /**
  1000. * iscsi_sendpage - send one page of iSCSI Data-Out.
  1001. * @conn: iscsi connection
  1002. * @buf: buffer to write from
  1003. * @count: remaining data
  1004. * @sent: number of bytes sent
  1005. *
  1006. * Notes:
  1007. * (Tx, Fast Path)
  1008. **/
  1009. static inline int
  1010. iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
  1011. int *count, int *sent)
  1012. {
  1013. int flags = 0; /* MSG_DONTWAIT; */
  1014. int res, size;
  1015. size = buf->sg.length - buf->sent;
  1016. BUG_ON(buf->sent + size > buf->sg.length);
  1017. if (size > *count)
  1018. size = *count;
  1019. if (buf->sent + size != buf->sg.length || *count != size)
  1020. flags |= MSG_MORE;
  1021. res = iscsi_send(conn, buf, size, flags);
  1022. debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
  1023. size, buf->sent, *count, *sent, res);
  1024. if (res >= 0) {
  1025. *count -= res;
  1026. *sent += res;
  1027. if (size != res)
  1028. return -EAGAIN;
  1029. return 0;
  1030. }
  1031. return res;
  1032. }
  1033. static inline void
  1034. iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
  1035. struct iscsi_tcp_cmd_task *tcp_ctask)
  1036. {
  1037. crypto_hash_init(&tcp_conn->tx_hash);
  1038. tcp_ctask->digest_count = 4;
  1039. }
  1040. /**
  1041. * iscsi_solicit_data_cont - initialize next Data-Out
  1042. * @conn: iscsi connection
  1043. * @ctask: scsi command task
  1044. * @r2t: R2T info
  1045. * @left: bytes left to transfer
  1046. *
  1047. * Notes:
  1048. * Initialize next Data-Out within this R2T sequence and continue
  1049. * to process next Scatter-Gather element(if any) of this SCSI command.
  1050. *
  1051. * Called under connection lock.
  1052. **/
  1053. static void
  1054. iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
  1055. struct iscsi_r2t_info *r2t, int left)
  1056. {
  1057. struct iscsi_data *hdr;
  1058. int new_offset;
  1059. hdr = &r2t->dtask.hdr;
  1060. memset(hdr, 0, sizeof(struct iscsi_data));
  1061. hdr->ttt = r2t->ttt;
  1062. hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
  1063. r2t->solicit_datasn++;
  1064. hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
  1065. memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
  1066. hdr->itt = ctask->hdr->itt;
  1067. hdr->exp_statsn = r2t->exp_statsn;
  1068. new_offset = r2t->data_offset + r2t->sent;
  1069. hdr->offset = cpu_to_be32(new_offset);
  1070. if (left > conn->max_xmit_dlength) {
  1071. hton24(hdr->dlength, conn->max_xmit_dlength);
  1072. r2t->data_count = conn->max_xmit_dlength;
  1073. } else {
  1074. hton24(hdr->dlength, left);
  1075. r2t->data_count = left;
  1076. hdr->flags = ISCSI_FLAG_CMD_FINAL;
  1077. }
  1078. conn->dataout_pdus_cnt++;
  1079. iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
  1080. sizeof(struct iscsi_hdr));
  1081. if (iscsi_buf_left(&r2t->sendbuf))
  1082. return;
  1083. iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
  1084. r2t->sg += 1;
  1085. }
  1086. static void iscsi_set_padding(struct iscsi_tcp_cmd_task *tcp_ctask,
  1087. unsigned long len)
  1088. {
  1089. tcp_ctask->pad_count = len & (ISCSI_PAD_LEN - 1);
  1090. if (!tcp_ctask->pad_count)
  1091. return;
  1092. tcp_ctask->pad_count = ISCSI_PAD_LEN - tcp_ctask->pad_count;
  1093. debug_scsi("write padding %d bytes\n", tcp_ctask->pad_count);
  1094. tcp_ctask->xmstate |= XMSTATE_W_PAD;
  1095. }
  1096. /**
  1097. * iscsi_tcp_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
  1098. * @conn: iscsi connection
  1099. * @ctask: scsi command task
  1100. * @sc: scsi command
  1101. **/
  1102. static void
  1103. iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
  1104. {
  1105. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1106. BUG_ON(__kfifo_len(tcp_ctask->r2tqueue));
  1107. tcp_ctask->xmstate = XMSTATE_CMD_HDR_INIT;
  1108. }
  1109. /**
  1110. * iscsi_tcp_mtask_xmit - xmit management(immediate) task
  1111. * @conn: iscsi connection
  1112. * @mtask: task management task
  1113. *
  1114. * Notes:
  1115. * The function can return -EAGAIN in which case caller must
  1116. * call it again later, or recover. '0' return code means successful
  1117. * xmit.
  1118. *
  1119. * Management xmit state machine consists of these states:
  1120. * XMSTATE_IMM_HDR_INIT - calculate digest of PDU Header
  1121. * XMSTATE_IMM_HDR - PDU Header xmit in progress
  1122. * XMSTATE_IMM_DATA - PDU Data xmit in progress
  1123. * XMSTATE_IDLE - management PDU is done
  1124. **/
  1125. static int
  1126. iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
  1127. {
  1128. struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
  1129. int rc;
  1130. debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
  1131. conn->id, tcp_mtask->xmstate, mtask->itt);
  1132. if (tcp_mtask->xmstate & XMSTATE_IMM_HDR_INIT) {
  1133. iscsi_buf_init_iov(&tcp_mtask->headbuf, (char*)mtask->hdr,
  1134. sizeof(struct iscsi_hdr));
  1135. if (mtask->data_count) {
  1136. tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
  1137. iscsi_buf_init_iov(&tcp_mtask->sendbuf,
  1138. (char*)mtask->data,
  1139. mtask->data_count);
  1140. }
  1141. if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
  1142. conn->stop_stage != STOP_CONN_RECOVER &&
  1143. conn->hdrdgst_en)
  1144. iscsi_hdr_digest(conn, &tcp_mtask->headbuf,
  1145. (u8*)tcp_mtask->hdrext);
  1146. tcp_mtask->sent = 0;
  1147. tcp_mtask->xmstate &= ~XMSTATE_IMM_HDR_INIT;
  1148. tcp_mtask->xmstate |= XMSTATE_IMM_HDR;
  1149. }
  1150. if (tcp_mtask->xmstate & XMSTATE_IMM_HDR) {
  1151. rc = iscsi_sendhdr(conn, &tcp_mtask->headbuf,
  1152. mtask->data_count);
  1153. if (rc)
  1154. return rc;
  1155. tcp_mtask->xmstate &= ~XMSTATE_IMM_HDR;
  1156. }
  1157. if (tcp_mtask->xmstate & XMSTATE_IMM_DATA) {
  1158. BUG_ON(!mtask->data_count);
  1159. tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
  1160. /* FIXME: implement.
  1161. * Virtual buffer could be spreaded across multiple pages...
  1162. */
  1163. do {
  1164. int rc;
  1165. rc = iscsi_sendpage(conn, &tcp_mtask->sendbuf,
  1166. &mtask->data_count, &tcp_mtask->sent);
  1167. if (rc) {
  1168. tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
  1169. return rc;
  1170. }
  1171. } while (mtask->data_count);
  1172. }
  1173. BUG_ON(tcp_mtask->xmstate != XMSTATE_IDLE);
  1174. if (mtask->hdr->itt == RESERVED_ITT) {
  1175. struct iscsi_session *session = conn->session;
  1176. spin_lock_bh(&session->lock);
  1177. list_del(&conn->mtask->running);
  1178. __kfifo_put(session->mgmtpool.queue, (void*)&conn->mtask,
  1179. sizeof(void*));
  1180. spin_unlock_bh(&session->lock);
  1181. }
  1182. return 0;
  1183. }
  1184. static int
  1185. iscsi_send_cmd_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  1186. {
  1187. struct scsi_cmnd *sc = ctask->sc;
  1188. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1189. int rc = 0;
  1190. if (tcp_ctask->xmstate & XMSTATE_CMD_HDR_INIT) {
  1191. tcp_ctask->sent = 0;
  1192. tcp_ctask->sg_count = 0;
  1193. tcp_ctask->exp_datasn = 0;
  1194. if (sc->sc_data_direction == DMA_TO_DEVICE) {
  1195. struct scatterlist *sg = scsi_sglist(sc);
  1196. iscsi_buf_init_sg(&tcp_ctask->sendbuf, sg);
  1197. tcp_ctask->sg = sg + 1;
  1198. tcp_ctask->bad_sg = sg + scsi_sg_count(sc);
  1199. debug_scsi("cmd [itt 0x%x total %d imm_data %d "
  1200. "unsol count %d, unsol offset %d]\n",
  1201. ctask->itt, scsi_bufflen(sc),
  1202. ctask->imm_count, ctask->unsol_count,
  1203. ctask->unsol_offset);
  1204. }
  1205. iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)ctask->hdr,
  1206. sizeof(struct iscsi_hdr));
  1207. if (conn->hdrdgst_en)
  1208. iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
  1209. (u8*)tcp_ctask->hdrext);
  1210. tcp_ctask->xmstate &= ~XMSTATE_CMD_HDR_INIT;
  1211. tcp_ctask->xmstate |= XMSTATE_CMD_HDR_XMIT;
  1212. }
  1213. if (tcp_ctask->xmstate & XMSTATE_CMD_HDR_XMIT) {
  1214. rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count);
  1215. if (rc)
  1216. return rc;
  1217. tcp_ctask->xmstate &= ~XMSTATE_CMD_HDR_XMIT;
  1218. if (sc->sc_data_direction != DMA_TO_DEVICE)
  1219. return 0;
  1220. if (ctask->imm_count) {
  1221. tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
  1222. iscsi_set_padding(tcp_ctask, ctask->imm_count);
  1223. if (ctask->conn->datadgst_en) {
  1224. iscsi_data_digest_init(ctask->conn->dd_data,
  1225. tcp_ctask);
  1226. tcp_ctask->immdigest = 0;
  1227. }
  1228. }
  1229. if (ctask->unsol_count)
  1230. tcp_ctask->xmstate |=
  1231. XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
  1232. }
  1233. return rc;
  1234. }
  1235. static int
  1236. iscsi_send_padding(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  1237. {
  1238. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1239. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1240. int sent = 0, rc;
  1241. if (tcp_ctask->xmstate & XMSTATE_W_PAD) {
  1242. iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
  1243. tcp_ctask->pad_count);
  1244. if (conn->datadgst_en)
  1245. crypto_hash_update(&tcp_conn->tx_hash,
  1246. &tcp_ctask->sendbuf.sg,
  1247. tcp_ctask->sendbuf.sg.length);
  1248. } else if (!(tcp_ctask->xmstate & XMSTATE_W_RESEND_PAD))
  1249. return 0;
  1250. tcp_ctask->xmstate &= ~XMSTATE_W_PAD;
  1251. tcp_ctask->xmstate &= ~XMSTATE_W_RESEND_PAD;
  1252. debug_scsi("sending %d pad bytes for itt 0x%x\n",
  1253. tcp_ctask->pad_count, ctask->itt);
  1254. rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
  1255. &sent);
  1256. if (rc) {
  1257. debug_scsi("padding send failed %d\n", rc);
  1258. tcp_ctask->xmstate |= XMSTATE_W_RESEND_PAD;
  1259. }
  1260. return rc;
  1261. }
  1262. static int
  1263. iscsi_send_digest(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
  1264. struct iscsi_buf *buf, uint32_t *digest)
  1265. {
  1266. struct iscsi_tcp_cmd_task *tcp_ctask;
  1267. struct iscsi_tcp_conn *tcp_conn;
  1268. int rc, sent = 0;
  1269. if (!conn->datadgst_en)
  1270. return 0;
  1271. tcp_ctask = ctask->dd_data;
  1272. tcp_conn = conn->dd_data;
  1273. if (!(tcp_ctask->xmstate & XMSTATE_W_RESEND_DATA_DIGEST)) {
  1274. crypto_hash_final(&tcp_conn->tx_hash, (u8*)digest);
  1275. iscsi_buf_init_iov(buf, (char*)digest, 4);
  1276. }
  1277. tcp_ctask->xmstate &= ~XMSTATE_W_RESEND_DATA_DIGEST;
  1278. rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
  1279. if (!rc)
  1280. debug_scsi("sent digest 0x%x for itt 0x%x\n", *digest,
  1281. ctask->itt);
  1282. else {
  1283. debug_scsi("sending digest 0x%x failed for itt 0x%x!\n",
  1284. *digest, ctask->itt);
  1285. tcp_ctask->xmstate |= XMSTATE_W_RESEND_DATA_DIGEST;
  1286. }
  1287. return rc;
  1288. }
  1289. static int
  1290. iscsi_send_data(struct iscsi_cmd_task *ctask, struct iscsi_buf *sendbuf,
  1291. struct scatterlist **sg, int *sent, int *count,
  1292. struct iscsi_buf *digestbuf, uint32_t *digest)
  1293. {
  1294. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1295. struct iscsi_conn *conn = ctask->conn;
  1296. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1297. int rc, buf_sent, offset;
  1298. while (*count) {
  1299. buf_sent = 0;
  1300. offset = sendbuf->sent;
  1301. rc = iscsi_sendpage(conn, sendbuf, count, &buf_sent);
  1302. *sent = *sent + buf_sent;
  1303. if (buf_sent && conn->datadgst_en)
  1304. partial_sg_digest_update(&tcp_conn->tx_hash,
  1305. &sendbuf->sg, sendbuf->sg.offset + offset,
  1306. buf_sent);
  1307. if (!iscsi_buf_left(sendbuf) && *sg != tcp_ctask->bad_sg) {
  1308. iscsi_buf_init_sg(sendbuf, *sg);
  1309. *sg = *sg + 1;
  1310. }
  1311. if (rc)
  1312. return rc;
  1313. }
  1314. rc = iscsi_send_padding(conn, ctask);
  1315. if (rc)
  1316. return rc;
  1317. return iscsi_send_digest(conn, ctask, digestbuf, digest);
  1318. }
  1319. static int
  1320. iscsi_send_unsol_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  1321. {
  1322. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1323. struct iscsi_data_task *dtask;
  1324. int rc;
  1325. tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
  1326. if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) {
  1327. dtask = &tcp_ctask->unsol_dtask;
  1328. iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr);
  1329. iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
  1330. sizeof(struct iscsi_hdr));
  1331. if (conn->hdrdgst_en)
  1332. iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
  1333. (u8*)dtask->hdrext);
  1334. tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT;
  1335. iscsi_set_padding(tcp_ctask, ctask->data_count);
  1336. }
  1337. rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count);
  1338. if (rc) {
  1339. tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
  1340. tcp_ctask->xmstate |= XMSTATE_UNS_HDR;
  1341. return rc;
  1342. }
  1343. if (conn->datadgst_en) {
  1344. dtask = &tcp_ctask->unsol_dtask;
  1345. iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);
  1346. dtask->digest = 0;
  1347. }
  1348. debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
  1349. ctask->itt, ctask->unsol_count, tcp_ctask->sent);
  1350. return 0;
  1351. }
  1352. static int
  1353. iscsi_send_unsol_pdu(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  1354. {
  1355. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1356. int rc;
  1357. if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) {
  1358. BUG_ON(!ctask->unsol_count);
  1359. tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR;
  1360. send_hdr:
  1361. rc = iscsi_send_unsol_hdr(conn, ctask);
  1362. if (rc)
  1363. return rc;
  1364. }
  1365. if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
  1366. struct iscsi_data_task *dtask = &tcp_ctask->unsol_dtask;
  1367. int start = tcp_ctask->sent;
  1368. rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
  1369. &tcp_ctask->sent, &ctask->data_count,
  1370. &dtask->digestbuf, &dtask->digest);
  1371. ctask->unsol_count -= tcp_ctask->sent - start;
  1372. if (rc)
  1373. return rc;
  1374. tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
  1375. /*
  1376. * Done with the Data-Out. Next, check if we need
  1377. * to send another unsolicited Data-Out.
  1378. */
  1379. if (ctask->unsol_count) {
  1380. debug_scsi("sending more uns\n");
  1381. tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
  1382. goto send_hdr;
  1383. }
  1384. }
  1385. return 0;
  1386. }
  1387. static int iscsi_send_sol_pdu(struct iscsi_conn *conn,
  1388. struct iscsi_cmd_task *ctask)
  1389. {
  1390. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1391. struct iscsi_session *session = conn->session;
  1392. struct iscsi_r2t_info *r2t;
  1393. struct iscsi_data_task *dtask;
  1394. int left, rc;
  1395. if (tcp_ctask->xmstate & XMSTATE_SOL_HDR_INIT) {
  1396. if (!tcp_ctask->r2t) {
  1397. spin_lock_bh(&session->lock);
  1398. __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
  1399. sizeof(void*));
  1400. spin_unlock_bh(&session->lock);
  1401. }
  1402. send_hdr:
  1403. r2t = tcp_ctask->r2t;
  1404. dtask = &r2t->dtask;
  1405. if (conn->hdrdgst_en)
  1406. iscsi_hdr_digest(conn, &r2t->headbuf,
  1407. (u8*)dtask->hdrext);
  1408. tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR_INIT;
  1409. tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
  1410. }
  1411. if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
  1412. r2t = tcp_ctask->r2t;
  1413. dtask = &r2t->dtask;
  1414. rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);
  1415. if (rc)
  1416. return rc;
  1417. tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
  1418. tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
  1419. if (conn->datadgst_en) {
  1420. iscsi_data_digest_init(conn->dd_data, tcp_ctask);
  1421. dtask->digest = 0;
  1422. }
  1423. iscsi_set_padding(tcp_ctask, r2t->data_count);
  1424. debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
  1425. r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
  1426. r2t->sent);
  1427. }
  1428. if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
  1429. r2t = tcp_ctask->r2t;
  1430. dtask = &r2t->dtask;
  1431. rc = iscsi_send_data(ctask, &r2t->sendbuf, &r2t->sg,
  1432. &r2t->sent, &r2t->data_count,
  1433. &dtask->digestbuf, &dtask->digest);
  1434. if (rc)
  1435. return rc;
  1436. tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
  1437. /*
  1438. * Done with this Data-Out. Next, check if we have
  1439. * to send another Data-Out for this R2T.
  1440. */
  1441. BUG_ON(r2t->data_length - r2t->sent < 0);
  1442. left = r2t->data_length - r2t->sent;
  1443. if (left) {
  1444. iscsi_solicit_data_cont(conn, ctask, r2t, left);
  1445. goto send_hdr;
  1446. }
  1447. /*
  1448. * Done with this R2T. Check if there are more
  1449. * outstanding R2Ts ready to be processed.
  1450. */
  1451. spin_lock_bh(&session->lock);
  1452. tcp_ctask->r2t = NULL;
  1453. __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
  1454. sizeof(void*));
  1455. if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t,
  1456. sizeof(void*))) {
  1457. tcp_ctask->r2t = r2t;
  1458. spin_unlock_bh(&session->lock);
  1459. goto send_hdr;
  1460. }
  1461. spin_unlock_bh(&session->lock);
  1462. }
  1463. return 0;
  1464. }
  1465. /**
  1466. * iscsi_tcp_ctask_xmit - xmit normal PDU task
  1467. * @conn: iscsi connection
  1468. * @ctask: iscsi command task
  1469. *
  1470. * Notes:
  1471. * The function can return -EAGAIN in which case caller must
  1472. * call it again later, or recover. '0' return code means successful
  1473. * xmit.
  1474. * The function is devided to logical helpers (above) for the different
  1475. * xmit stages.
  1476. *
  1477. *iscsi_send_cmd_hdr()
  1478. * XMSTATE_CMD_HDR_INIT - prepare Header and Data buffers Calculate
  1479. * Header Digest
  1480. * XMSTATE_CMD_HDR_XMIT - Transmit header in progress
  1481. *
  1482. *iscsi_send_padding
  1483. * XMSTATE_W_PAD - Prepare and send pading
  1484. * XMSTATE_W_RESEND_PAD - retry send pading
  1485. *
  1486. *iscsi_send_digest
  1487. * XMSTATE_W_RESEND_DATA_DIGEST - Finalize and send Data Digest
  1488. * XMSTATE_W_RESEND_DATA_DIGEST - retry sending digest
  1489. *
  1490. *iscsi_send_unsol_hdr
  1491. * XMSTATE_UNS_INIT - prepare un-solicit data header and digest
  1492. * XMSTATE_UNS_HDR - send un-solicit header
  1493. *
  1494. *iscsi_send_unsol_pdu
  1495. * XMSTATE_UNS_DATA - send un-solicit data in progress
  1496. *
  1497. *iscsi_send_sol_pdu
  1498. * XMSTATE_SOL_HDR_INIT - solicit data header and digest initialize
  1499. * XMSTATE_SOL_HDR - send solicit header
  1500. * XMSTATE_SOL_DATA - send solicit data
  1501. *
  1502. *iscsi_tcp_ctask_xmit
  1503. * XMSTATE_IMM_DATA - xmit managment data (??)
  1504. **/
  1505. static int
  1506. iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  1507. {
  1508. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1509. int rc = 0;
  1510. debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
  1511. conn->id, tcp_ctask->xmstate, ctask->itt);
  1512. rc = iscsi_send_cmd_hdr(conn, ctask);
  1513. if (rc)
  1514. return rc;
  1515. if (ctask->sc->sc_data_direction != DMA_TO_DEVICE)
  1516. return 0;
  1517. if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) {
  1518. rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
  1519. &tcp_ctask->sent, &ctask->imm_count,
  1520. &tcp_ctask->immbuf, &tcp_ctask->immdigest);
  1521. if (rc)
  1522. return rc;
  1523. tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA;
  1524. }
  1525. rc = iscsi_send_unsol_pdu(conn, ctask);
  1526. if (rc)
  1527. return rc;
  1528. rc = iscsi_send_sol_pdu(conn, ctask);
  1529. if (rc)
  1530. return rc;
  1531. return rc;
  1532. }
  1533. static struct iscsi_cls_conn *
  1534. iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
  1535. {
  1536. struct iscsi_conn *conn;
  1537. struct iscsi_cls_conn *cls_conn;
  1538. struct iscsi_tcp_conn *tcp_conn;
  1539. cls_conn = iscsi_conn_setup(cls_session, conn_idx);
  1540. if (!cls_conn)
  1541. return NULL;
  1542. conn = cls_conn->dd_data;
  1543. /*
  1544. * due to strange issues with iser these are not set
  1545. * in iscsi_conn_setup
  1546. */
  1547. conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN;
  1548. tcp_conn = kzalloc(sizeof(*tcp_conn), GFP_KERNEL);
  1549. if (!tcp_conn)
  1550. goto tcp_conn_alloc_fail;
  1551. conn->dd_data = tcp_conn;
  1552. tcp_conn->iscsi_conn = conn;
  1553. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  1554. /* initial operational parameters */
  1555. tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
  1556. tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
  1557. CRYPTO_ALG_ASYNC);
  1558. tcp_conn->tx_hash.flags = 0;
  1559. if (IS_ERR(tcp_conn->tx_hash.tfm)) {
  1560. printk(KERN_ERR "Could not create connection due to crc32c "
  1561. "loading error %ld. Make sure the crc32c module is "
  1562. "built as a module or into the kernel\n",
  1563. PTR_ERR(tcp_conn->tx_hash.tfm));
  1564. goto free_tcp_conn;
  1565. }
  1566. tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
  1567. CRYPTO_ALG_ASYNC);
  1568. tcp_conn->rx_hash.flags = 0;
  1569. if (IS_ERR(tcp_conn->rx_hash.tfm)) {
  1570. printk(KERN_ERR "Could not create connection due to crc32c "
  1571. "loading error %ld. Make sure the crc32c module is "
  1572. "built as a module or into the kernel\n",
  1573. PTR_ERR(tcp_conn->rx_hash.tfm));
  1574. goto free_tx_tfm;
  1575. }
  1576. return cls_conn;
  1577. free_tx_tfm:
  1578. crypto_free_hash(tcp_conn->tx_hash.tfm);
  1579. free_tcp_conn:
  1580. kfree(tcp_conn);
  1581. tcp_conn_alloc_fail:
  1582. iscsi_conn_teardown(cls_conn);
  1583. return NULL;
  1584. }
  1585. static void
  1586. iscsi_tcp_release_conn(struct iscsi_conn *conn)
  1587. {
  1588. struct iscsi_session *session = conn->session;
  1589. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1590. struct socket *sock = tcp_conn->sock;
  1591. if (!sock)
  1592. return;
  1593. sock_hold(sock->sk);
  1594. iscsi_conn_restore_callbacks(tcp_conn);
  1595. sock_put(sock->sk);
  1596. spin_lock_bh(&session->lock);
  1597. tcp_conn->sock = NULL;
  1598. conn->recv_lock = NULL;
  1599. spin_unlock_bh(&session->lock);
  1600. sockfd_put(sock);
  1601. }
  1602. static void
  1603. iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
  1604. {
  1605. struct iscsi_conn *conn = cls_conn->dd_data;
  1606. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1607. iscsi_tcp_release_conn(conn);
  1608. iscsi_conn_teardown(cls_conn);
  1609. if (tcp_conn->tx_hash.tfm)
  1610. crypto_free_hash(tcp_conn->tx_hash.tfm);
  1611. if (tcp_conn->rx_hash.tfm)
  1612. crypto_free_hash(tcp_conn->rx_hash.tfm);
  1613. kfree(tcp_conn);
  1614. }
  1615. static void
  1616. iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
  1617. {
  1618. struct iscsi_conn *conn = cls_conn->dd_data;
  1619. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1620. iscsi_conn_stop(cls_conn, flag);
  1621. iscsi_tcp_release_conn(conn);
  1622. tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
  1623. }
  1624. static int iscsi_tcp_get_addr(struct iscsi_conn *conn, struct socket *sock,
  1625. char *buf, int *port,
  1626. int (*getname)(struct socket *, struct sockaddr *,
  1627. int *addrlen))
  1628. {
  1629. struct sockaddr_storage *addr;
  1630. struct sockaddr_in6 *sin6;
  1631. struct sockaddr_in *sin;
  1632. int rc = 0, len;
  1633. addr = kmalloc(sizeof(*addr), GFP_KERNEL);
  1634. if (!addr)
  1635. return -ENOMEM;
  1636. if (getname(sock, (struct sockaddr *) addr, &len)) {
  1637. rc = -ENODEV;
  1638. goto free_addr;
  1639. }
  1640. switch (addr->ss_family) {
  1641. case AF_INET:
  1642. sin = (struct sockaddr_in *)addr;
  1643. spin_lock_bh(&conn->session->lock);
  1644. sprintf(buf, NIPQUAD_FMT, NIPQUAD(sin->sin_addr.s_addr));
  1645. *port = be16_to_cpu(sin->sin_port);
  1646. spin_unlock_bh(&conn->session->lock);
  1647. break;
  1648. case AF_INET6:
  1649. sin6 = (struct sockaddr_in6 *)addr;
  1650. spin_lock_bh(&conn->session->lock);
  1651. sprintf(buf, NIP6_FMT, NIP6(sin6->sin6_addr));
  1652. *port = be16_to_cpu(sin6->sin6_port);
  1653. spin_unlock_bh(&conn->session->lock);
  1654. break;
  1655. }
  1656. free_addr:
  1657. kfree(addr);
  1658. return rc;
  1659. }
  1660. static int
  1661. iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
  1662. struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
  1663. int is_leading)
  1664. {
  1665. struct iscsi_conn *conn = cls_conn->dd_data;
  1666. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1667. struct sock *sk;
  1668. struct socket *sock;
  1669. int err;
  1670. /* lookup for existing socket */
  1671. sock = sockfd_lookup((int)transport_eph, &err);
  1672. if (!sock) {
  1673. printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
  1674. return -EEXIST;
  1675. }
  1676. /*
  1677. * copy these values now because if we drop the session
  1678. * userspace may still want to query the values since we will
  1679. * be using them for the reconnect
  1680. */
  1681. err = iscsi_tcp_get_addr(conn, sock, conn->portal_address,
  1682. &conn->portal_port, kernel_getpeername);
  1683. if (err)
  1684. goto free_socket;
  1685. err = iscsi_tcp_get_addr(conn, sock, conn->local_address,
  1686. &conn->local_port, kernel_getsockname);
  1687. if (err)
  1688. goto free_socket;
  1689. err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
  1690. if (err)
  1691. goto free_socket;
  1692. /* bind iSCSI connection and socket */
  1693. tcp_conn->sock = sock;
  1694. /* setup Socket parameters */
  1695. sk = sock->sk;
  1696. sk->sk_reuse = 1;
  1697. sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
  1698. sk->sk_allocation = GFP_ATOMIC;
  1699. /* FIXME: disable Nagle's algorithm */
  1700. /*
  1701. * Intercept TCP callbacks for sendfile like receive
  1702. * processing.
  1703. */
  1704. conn->recv_lock = &sk->sk_callback_lock;
  1705. iscsi_conn_set_callbacks(conn);
  1706. tcp_conn->sendpage = tcp_conn->sock->ops->sendpage;
  1707. /*
  1708. * set receive state machine into initial state
  1709. */
  1710. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  1711. return 0;
  1712. free_socket:
  1713. sockfd_put(sock);
  1714. return err;
  1715. }
  1716. /* called with host lock */
  1717. static void
  1718. iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
  1719. {
  1720. struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
  1721. tcp_mtask->xmstate = XMSTATE_IMM_HDR_INIT;
  1722. }
  1723. static int
  1724. iscsi_r2tpool_alloc(struct iscsi_session *session)
  1725. {
  1726. int i;
  1727. int cmd_i;
  1728. /*
  1729. * initialize per-task: R2T pool and xmit queue
  1730. */
  1731. for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
  1732. struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
  1733. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1734. /*
  1735. * pre-allocated x4 as much r2ts to handle race when
  1736. * target acks DataOut faster than we data_xmit() queues
  1737. * could replenish r2tqueue.
  1738. */
  1739. /* R2T pool */
  1740. if (iscsi_pool_init(&tcp_ctask->r2tpool, session->max_r2t * 4,
  1741. (void***)&tcp_ctask->r2ts,
  1742. sizeof(struct iscsi_r2t_info))) {
  1743. goto r2t_alloc_fail;
  1744. }
  1745. /* R2T xmit queue */
  1746. tcp_ctask->r2tqueue = kfifo_alloc(
  1747. session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
  1748. if (tcp_ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
  1749. iscsi_pool_free(&tcp_ctask->r2tpool,
  1750. (void**)tcp_ctask->r2ts);
  1751. goto r2t_alloc_fail;
  1752. }
  1753. }
  1754. return 0;
  1755. r2t_alloc_fail:
  1756. for (i = 0; i < cmd_i; i++) {
  1757. struct iscsi_cmd_task *ctask = session->cmds[i];
  1758. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1759. kfifo_free(tcp_ctask->r2tqueue);
  1760. iscsi_pool_free(&tcp_ctask->r2tpool,
  1761. (void**)tcp_ctask->r2ts);
  1762. }
  1763. return -ENOMEM;
  1764. }
  1765. static void
  1766. iscsi_r2tpool_free(struct iscsi_session *session)
  1767. {
  1768. int i;
  1769. for (i = 0; i < session->cmds_max; i++) {
  1770. struct iscsi_cmd_task *ctask = session->cmds[i];
  1771. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1772. kfifo_free(tcp_ctask->r2tqueue);
  1773. iscsi_pool_free(&tcp_ctask->r2tpool,
  1774. (void**)tcp_ctask->r2ts);
  1775. }
  1776. }
  1777. static int
  1778. iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
  1779. char *buf, int buflen)
  1780. {
  1781. struct iscsi_conn *conn = cls_conn->dd_data;
  1782. struct iscsi_session *session = conn->session;
  1783. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1784. int value;
  1785. switch(param) {
  1786. case ISCSI_PARAM_HDRDGST_EN:
  1787. iscsi_set_param(cls_conn, param, buf, buflen);
  1788. tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
  1789. if (conn->hdrdgst_en)
  1790. tcp_conn->hdr_size += sizeof(__u32);
  1791. break;
  1792. case ISCSI_PARAM_DATADGST_EN:
  1793. iscsi_set_param(cls_conn, param, buf, buflen);
  1794. tcp_conn->sendpage = conn->datadgst_en ?
  1795. sock_no_sendpage : tcp_conn->sock->ops->sendpage;
  1796. break;
  1797. case ISCSI_PARAM_MAX_R2T:
  1798. sscanf(buf, "%d", &value);
  1799. if (session->max_r2t == roundup_pow_of_two(value))
  1800. break;
  1801. iscsi_r2tpool_free(session);
  1802. iscsi_set_param(cls_conn, param, buf, buflen);
  1803. if (session->max_r2t & (session->max_r2t - 1))
  1804. session->max_r2t = roundup_pow_of_two(session->max_r2t);
  1805. if (iscsi_r2tpool_alloc(session))
  1806. return -ENOMEM;
  1807. break;
  1808. default:
  1809. return iscsi_set_param(cls_conn, param, buf, buflen);
  1810. }
  1811. return 0;
  1812. }
  1813. static int
  1814. iscsi_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
  1815. enum iscsi_param param, char *buf)
  1816. {
  1817. struct iscsi_conn *conn = cls_conn->dd_data;
  1818. int len;
  1819. switch(param) {
  1820. case ISCSI_PARAM_CONN_PORT:
  1821. spin_lock_bh(&conn->session->lock);
  1822. len = sprintf(buf, "%hu\n", conn->portal_port);
  1823. spin_unlock_bh(&conn->session->lock);
  1824. break;
  1825. case ISCSI_PARAM_CONN_ADDRESS:
  1826. spin_lock_bh(&conn->session->lock);
  1827. len = sprintf(buf, "%s\n", conn->portal_address);
  1828. spin_unlock_bh(&conn->session->lock);
  1829. break;
  1830. default:
  1831. return iscsi_conn_get_param(cls_conn, param, buf);
  1832. }
  1833. return len;
  1834. }
  1835. static int
  1836. iscsi_tcp_host_get_param(struct Scsi_Host *shost, enum iscsi_host_param param,
  1837. char *buf)
  1838. {
  1839. struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
  1840. int len;
  1841. switch (param) {
  1842. case ISCSI_HOST_PARAM_IPADDRESS:
  1843. spin_lock_bh(&session->lock);
  1844. if (!session->leadconn)
  1845. len = -ENODEV;
  1846. else
  1847. len = sprintf(buf, "%s\n",
  1848. session->leadconn->local_address);
  1849. spin_unlock_bh(&session->lock);
  1850. break;
  1851. default:
  1852. return iscsi_host_get_param(shost, param, buf);
  1853. }
  1854. return len;
  1855. }
  1856. static void
  1857. iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
  1858. {
  1859. struct iscsi_conn *conn = cls_conn->dd_data;
  1860. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1861. stats->txdata_octets = conn->txdata_octets;
  1862. stats->rxdata_octets = conn->rxdata_octets;
  1863. stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
  1864. stats->dataout_pdus = conn->dataout_pdus_cnt;
  1865. stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
  1866. stats->datain_pdus = conn->datain_pdus_cnt;
  1867. stats->r2t_pdus = conn->r2t_pdus_cnt;
  1868. stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
  1869. stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
  1870. stats->custom_length = 3;
  1871. strcpy(stats->custom[0].desc, "tx_sendpage_failures");
  1872. stats->custom[0].value = tcp_conn->sendpage_failures_cnt;
  1873. strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
  1874. stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt;
  1875. strcpy(stats->custom[2].desc, "eh_abort_cnt");
  1876. stats->custom[2].value = conn->eh_abort_cnt;
  1877. }
  1878. static struct iscsi_cls_session *
  1879. iscsi_tcp_session_create(struct iscsi_transport *iscsit,
  1880. struct scsi_transport_template *scsit,
  1881. uint16_t cmds_max, uint16_t qdepth,
  1882. uint32_t initial_cmdsn, uint32_t *hostno)
  1883. {
  1884. struct iscsi_cls_session *cls_session;
  1885. struct iscsi_session *session;
  1886. uint32_t hn;
  1887. int cmd_i;
  1888. cls_session = iscsi_session_setup(iscsit, scsit, cmds_max, qdepth,
  1889. sizeof(struct iscsi_tcp_cmd_task),
  1890. sizeof(struct iscsi_tcp_mgmt_task),
  1891. initial_cmdsn, &hn);
  1892. if (!cls_session)
  1893. return NULL;
  1894. *hostno = hn;
  1895. session = class_to_transport_session(cls_session);
  1896. for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
  1897. struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
  1898. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1899. ctask->hdr = &tcp_ctask->hdr;
  1900. }
  1901. for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
  1902. struct iscsi_mgmt_task *mtask = session->mgmt_cmds[cmd_i];
  1903. struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
  1904. mtask->hdr = &tcp_mtask->hdr;
  1905. }
  1906. if (iscsi_r2tpool_alloc(class_to_transport_session(cls_session)))
  1907. goto r2tpool_alloc_fail;
  1908. return cls_session;
  1909. r2tpool_alloc_fail:
  1910. iscsi_session_teardown(cls_session);
  1911. return NULL;
  1912. }
  1913. static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session)
  1914. {
  1915. iscsi_r2tpool_free(class_to_transport_session(cls_session));
  1916. iscsi_session_teardown(cls_session);
  1917. }
  1918. static int iscsi_tcp_slave_configure(struct scsi_device *sdev)
  1919. {
  1920. blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
  1921. blk_queue_dma_alignment(sdev->request_queue, 0);
  1922. return 0;
  1923. }
  1924. static struct scsi_host_template iscsi_sht = {
  1925. .module = THIS_MODULE,
  1926. .name = "iSCSI Initiator over TCP/IP",
  1927. .queuecommand = iscsi_queuecommand,
  1928. .change_queue_depth = iscsi_change_queue_depth,
  1929. .can_queue = ISCSI_DEF_XMIT_CMDS_MAX - 1,
  1930. .sg_tablesize = ISCSI_SG_TABLESIZE,
  1931. .max_sectors = 0xFFFF,
  1932. .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
  1933. .eh_abort_handler = iscsi_eh_abort,
  1934. .eh_host_reset_handler = iscsi_eh_host_reset,
  1935. .use_clustering = DISABLE_CLUSTERING,
  1936. .slave_configure = iscsi_tcp_slave_configure,
  1937. .proc_name = "iscsi_tcp",
  1938. .this_id = -1,
  1939. };
  1940. static struct iscsi_transport iscsi_tcp_transport = {
  1941. .owner = THIS_MODULE,
  1942. .name = "tcp",
  1943. .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
  1944. | CAP_DATADGST,
  1945. .param_mask = ISCSI_MAX_RECV_DLENGTH |
  1946. ISCSI_MAX_XMIT_DLENGTH |
  1947. ISCSI_HDRDGST_EN |
  1948. ISCSI_DATADGST_EN |
  1949. ISCSI_INITIAL_R2T_EN |
  1950. ISCSI_MAX_R2T |
  1951. ISCSI_IMM_DATA_EN |
  1952. ISCSI_FIRST_BURST |
  1953. ISCSI_MAX_BURST |
  1954. ISCSI_PDU_INORDER_EN |
  1955. ISCSI_DATASEQ_INORDER_EN |
  1956. ISCSI_ERL |
  1957. ISCSI_CONN_PORT |
  1958. ISCSI_CONN_ADDRESS |
  1959. ISCSI_EXP_STATSN |
  1960. ISCSI_PERSISTENT_PORT |
  1961. ISCSI_PERSISTENT_ADDRESS |
  1962. ISCSI_TARGET_NAME | ISCSI_TPGT |
  1963. ISCSI_USERNAME | ISCSI_PASSWORD |
  1964. ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN,
  1965. .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
  1966. ISCSI_HOST_INITIATOR_NAME |
  1967. ISCSI_HOST_NETDEV_NAME,
  1968. .host_template = &iscsi_sht,
  1969. .conndata_size = sizeof(struct iscsi_conn),
  1970. .max_conn = 1,
  1971. .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN,
  1972. /* session management */
  1973. .create_session = iscsi_tcp_session_create,
  1974. .destroy_session = iscsi_tcp_session_destroy,
  1975. /* connection management */
  1976. .create_conn = iscsi_tcp_conn_create,
  1977. .bind_conn = iscsi_tcp_conn_bind,
  1978. .destroy_conn = iscsi_tcp_conn_destroy,
  1979. .set_param = iscsi_conn_set_param,
  1980. .get_conn_param = iscsi_tcp_conn_get_param,
  1981. .get_session_param = iscsi_session_get_param,
  1982. .start_conn = iscsi_conn_start,
  1983. .stop_conn = iscsi_tcp_conn_stop,
  1984. /* iscsi host params */
  1985. .get_host_param = iscsi_tcp_host_get_param,
  1986. .set_host_param = iscsi_host_set_param,
  1987. /* IO */
  1988. .send_pdu = iscsi_conn_send_pdu,
  1989. .get_stats = iscsi_conn_get_stats,
  1990. .init_cmd_task = iscsi_tcp_cmd_init,
  1991. .init_mgmt_task = iscsi_tcp_mgmt_init,
  1992. .xmit_cmd_task = iscsi_tcp_ctask_xmit,
  1993. .xmit_mgmt_task = iscsi_tcp_mtask_xmit,
  1994. .cleanup_cmd_task = iscsi_tcp_cleanup_ctask,
  1995. /* recovery */
  1996. .session_recovery_timedout = iscsi_session_recovery_timedout,
  1997. };
  1998. static int __init
  1999. iscsi_tcp_init(void)
  2000. {
  2001. if (iscsi_max_lun < 1) {
  2002. printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
  2003. iscsi_max_lun);
  2004. return -EINVAL;
  2005. }
  2006. iscsi_tcp_transport.max_lun = iscsi_max_lun;
  2007. if (!iscsi_register_transport(&iscsi_tcp_transport))
  2008. return -ENODEV;
  2009. return 0;
  2010. }
  2011. static void __exit
  2012. iscsi_tcp_exit(void)
  2013. {
  2014. iscsi_unregister_transport(&iscsi_tcp_transport);
  2015. }
  2016. module_init(iscsi_tcp_init);
  2017. module_exit(iscsi_tcp_exit);