iscsi_tcp.c 62 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318
  1. /*
  2. * iSCSI Initiator over TCP/IP Data-Path
  3. *
  4. * Copyright (C) 2004 Dmitry Yusupov
  5. * Copyright (C) 2004 Alex Aizman
  6. * Copyright (C) 2005 - 2006 Mike Christie
  7. * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
  8. * maintained by open-iscsi@googlegroups.com
  9. *
  10. * This program is free software; you can redistribute it and/or modify
  11. * it under the terms of the GNU General Public License as published
  12. * by the Free Software Foundation; either version 2 of the License, or
  13. * (at your option) any later version.
  14. *
  15. * This program is distributed in the hope that it will be useful, but
  16. * WITHOUT ANY WARRANTY; without even the implied warranty of
  17. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18. * General Public License for more details.
  19. *
  20. * See the file COPYING included with this distribution for more details.
  21. *
  22. * Credits:
  23. * Christoph Hellwig
  24. * FUJITA Tomonori
  25. * Arne Redlich
  26. * Zhenyu Wang
  27. */
  28. #include <linux/types.h>
  29. #include <linux/list.h>
  30. #include <linux/inet.h>
  31. #include <linux/file.h>
  32. #include <linux/blkdev.h>
  33. #include <linux/crypto.h>
  34. #include <linux/delay.h>
  35. #include <linux/kfifo.h>
  36. #include <linux/scatterlist.h>
  37. #include <net/tcp.h>
  38. #include <scsi/scsi_cmnd.h>
  39. #include <scsi/scsi_device.h>
  40. #include <scsi/scsi_host.h>
  41. #include <scsi/scsi.h>
  42. #include <scsi/scsi_transport_iscsi.h>
  43. #include "iscsi_tcp.h"
  44. MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
  45. "Alex Aizman <itn780@yahoo.com>");
  46. MODULE_DESCRIPTION("iSCSI/TCP data-path");
  47. MODULE_LICENSE("GPL");
  48. /* #define DEBUG_TCP */
  49. #define DEBUG_ASSERT
  50. #ifdef DEBUG_TCP
  51. #define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
  52. #else
  53. #define debug_tcp(fmt...)
  54. #endif
  55. #ifndef DEBUG_ASSERT
  56. #ifdef BUG_ON
  57. #undef BUG_ON
  58. #endif
  59. #define BUG_ON(expr)
  60. #endif
  61. static unsigned int iscsi_max_lun = 512;
  62. module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
  63. static inline void
  64. iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
  65. {
  66. sg_init_one(&ibuf->sg, vbuf, size);
  67. ibuf->sent = 0;
  68. ibuf->use_sendmsg = 1;
  69. }
  70. static inline void
  71. iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
  72. {
  73. sg_init_table(&ibuf->sg, 1);
  74. sg_set_page(&ibuf->sg, sg_page(sg), sg->length, sg->offset);
  75. /*
  76. * Fastpath: sg element fits into single page
  77. */
  78. if (sg->length + sg->offset <= PAGE_SIZE && !PageSlab(sg_page(sg)))
  79. ibuf->use_sendmsg = 0;
  80. else
  81. ibuf->use_sendmsg = 1;
  82. ibuf->sent = 0;
  83. }
  84. static inline int
  85. iscsi_buf_left(struct iscsi_buf *ibuf)
  86. {
  87. int rc;
  88. rc = ibuf->sg.length - ibuf->sent;
  89. BUG_ON(rc < 0);
  90. return rc;
  91. }
  92. static inline void
  93. iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
  94. u8* crc)
  95. {
  96. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  97. crypto_hash_digest(&tcp_conn->tx_hash, &buf->sg, buf->sg.length, crc);
  98. buf->sg.length += sizeof(u32);
  99. }
  100. static inline int
  101. iscsi_hdr_extract(struct iscsi_tcp_conn *tcp_conn)
  102. {
  103. struct sk_buff *skb = tcp_conn->in.skb;
  104. tcp_conn->in.zero_copy_hdr = 0;
  105. if (tcp_conn->in.copy >= tcp_conn->hdr_size &&
  106. tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
  107. /*
  108. * Zero-copy PDU Header: using connection context
  109. * to store header pointer.
  110. */
  111. if (skb_shinfo(skb)->frag_list == NULL &&
  112. !skb_shinfo(skb)->nr_frags) {
  113. tcp_conn->in.hdr = (struct iscsi_hdr *)
  114. ((char*)skb->data + tcp_conn->in.offset);
  115. tcp_conn->in.zero_copy_hdr = 1;
  116. } else {
  117. /* ignoring return code since we checked
  118. * in.copy before */
  119. skb_copy_bits(skb, tcp_conn->in.offset,
  120. &tcp_conn->hdr, tcp_conn->hdr_size);
  121. tcp_conn->in.hdr = &tcp_conn->hdr;
  122. }
  123. tcp_conn->in.offset += tcp_conn->hdr_size;
  124. tcp_conn->in.copy -= tcp_conn->hdr_size;
  125. } else {
  126. int hdr_remains;
  127. int copylen;
  128. /*
  129. * PDU header scattered across SKB's,
  130. * copying it... This'll happen quite rarely.
  131. */
  132. if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER)
  133. tcp_conn->in.hdr_offset = 0;
  134. hdr_remains = tcp_conn->hdr_size - tcp_conn->in.hdr_offset;
  135. BUG_ON(hdr_remains <= 0);
  136. copylen = min(tcp_conn->in.copy, hdr_remains);
  137. skb_copy_bits(skb, tcp_conn->in.offset,
  138. (char*)&tcp_conn->hdr + tcp_conn->in.hdr_offset,
  139. copylen);
  140. debug_tcp("PDU gather offset %d bytes %d in.offset %d "
  141. "in.copy %d\n", tcp_conn->in.hdr_offset, copylen,
  142. tcp_conn->in.offset, tcp_conn->in.copy);
  143. tcp_conn->in.offset += copylen;
  144. tcp_conn->in.copy -= copylen;
  145. if (copylen < hdr_remains) {
  146. tcp_conn->in_progress = IN_PROGRESS_HEADER_GATHER;
  147. tcp_conn->in.hdr_offset += copylen;
  148. return -EAGAIN;
  149. }
  150. tcp_conn->in.hdr = &tcp_conn->hdr;
  151. tcp_conn->discontiguous_hdr_cnt++;
  152. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  153. }
  154. return 0;
  155. }
  156. /*
  157. * must be called with session lock
  158. */
  159. static void
  160. iscsi_tcp_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  161. {
  162. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  163. struct iscsi_r2t_info *r2t;
  164. struct scsi_cmnd *sc;
  165. /* flush ctask's r2t queues */
  166. while (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
  167. __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
  168. sizeof(void*));
  169. debug_scsi("iscsi_tcp_cleanup_ctask pending r2t dropped\n");
  170. }
  171. sc = ctask->sc;
  172. if (unlikely(!sc))
  173. return;
  174. tcp_ctask->xmstate = XMSTATE_VALUE_IDLE;
  175. tcp_ctask->r2t = NULL;
  176. }
  177. /**
  178. * iscsi_data_rsp - SCSI Data-In Response processing
  179. * @conn: iscsi connection
  180. * @ctask: scsi command task
  181. **/
  182. static int
  183. iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  184. {
  185. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  186. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  187. struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
  188. struct iscsi_session *session = conn->session;
  189. struct scsi_cmnd *sc = ctask->sc;
  190. int datasn = be32_to_cpu(rhdr->datasn);
  191. iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr);
  192. /*
  193. * setup Data-In byte counter (gets decremented..)
  194. */
  195. ctask->data_count = tcp_conn->in.datalen;
  196. if (tcp_conn->in.datalen == 0)
  197. return 0;
  198. if (tcp_ctask->exp_datasn != datasn) {
  199. debug_tcp("%s: ctask->exp_datasn(%d) != rhdr->datasn(%d)\n",
  200. __FUNCTION__, tcp_ctask->exp_datasn, datasn);
  201. return ISCSI_ERR_DATASN;
  202. }
  203. tcp_ctask->exp_datasn++;
  204. tcp_ctask->data_offset = be32_to_cpu(rhdr->offset);
  205. if (tcp_ctask->data_offset + tcp_conn->in.datalen > scsi_bufflen(sc)) {
  206. debug_tcp("%s: data_offset(%d) + data_len(%d) > total_length_in(%d)\n",
  207. __FUNCTION__, tcp_ctask->data_offset,
  208. tcp_conn->in.datalen, scsi_bufflen(sc));
  209. return ISCSI_ERR_DATA_OFFSET;
  210. }
  211. if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
  212. conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
  213. if (rhdr->flags & ISCSI_FLAG_DATA_UNDERFLOW) {
  214. int res_count = be32_to_cpu(rhdr->residual_count);
  215. if (res_count > 0 &&
  216. res_count <= scsi_bufflen(sc)) {
  217. scsi_set_resid(sc, res_count);
  218. sc->result = (DID_OK << 16) | rhdr->cmd_status;
  219. } else
  220. sc->result = (DID_BAD_TARGET << 16) |
  221. rhdr->cmd_status;
  222. } else if (rhdr->flags & ISCSI_FLAG_DATA_OVERFLOW) {
  223. scsi_set_resid(sc, be32_to_cpu(rhdr->residual_count));
  224. sc->result = (DID_OK << 16) | rhdr->cmd_status;
  225. } else
  226. sc->result = (DID_OK << 16) | rhdr->cmd_status;
  227. }
  228. conn->datain_pdus_cnt++;
  229. return 0;
  230. }
  231. /**
  232. * iscsi_solicit_data_init - initialize first Data-Out
  233. * @conn: iscsi connection
  234. * @ctask: scsi command task
  235. * @r2t: R2T info
  236. *
  237. * Notes:
  238. * Initialize first Data-Out within this R2T sequence and finds
  239. * proper data_offset within this SCSI command.
  240. *
  241. * This function is called with connection lock taken.
  242. **/
  243. static void
  244. iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
  245. struct iscsi_r2t_info *r2t)
  246. {
  247. struct iscsi_data *hdr;
  248. struct scsi_cmnd *sc = ctask->sc;
  249. int i, sg_count = 0;
  250. struct scatterlist *sg;
  251. hdr = &r2t->dtask.hdr;
  252. memset(hdr, 0, sizeof(struct iscsi_data));
  253. hdr->ttt = r2t->ttt;
  254. hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
  255. r2t->solicit_datasn++;
  256. hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
  257. memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
  258. hdr->itt = ctask->hdr->itt;
  259. hdr->exp_statsn = r2t->exp_statsn;
  260. hdr->offset = cpu_to_be32(r2t->data_offset);
  261. if (r2t->data_length > conn->max_xmit_dlength) {
  262. hton24(hdr->dlength, conn->max_xmit_dlength);
  263. r2t->data_count = conn->max_xmit_dlength;
  264. hdr->flags = 0;
  265. } else {
  266. hton24(hdr->dlength, r2t->data_length);
  267. r2t->data_count = r2t->data_length;
  268. hdr->flags = ISCSI_FLAG_CMD_FINAL;
  269. }
  270. conn->dataout_pdus_cnt++;
  271. r2t->sent = 0;
  272. iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
  273. sizeof(struct iscsi_hdr));
  274. sg = scsi_sglist(sc);
  275. r2t->sg = NULL;
  276. for (i = 0; i < scsi_sg_count(sc); i++, sg += 1) {
  277. /* FIXME: prefetch ? */
  278. if (sg_count + sg->length > r2t->data_offset) {
  279. int page_offset;
  280. /* sg page found! */
  281. /* offset within this page */
  282. page_offset = r2t->data_offset - sg_count;
  283. /* fill in this buffer */
  284. iscsi_buf_init_sg(&r2t->sendbuf, sg);
  285. r2t->sendbuf.sg.offset += page_offset;
  286. r2t->sendbuf.sg.length -= page_offset;
  287. /* xmit logic will continue with next one */
  288. r2t->sg = sg + 1;
  289. break;
  290. }
  291. sg_count += sg->length;
  292. }
  293. BUG_ON(r2t->sg == NULL);
  294. }
  295. /**
  296. * iscsi_r2t_rsp - iSCSI R2T Response processing
  297. * @conn: iscsi connection
  298. * @ctask: scsi command task
  299. **/
  300. static int
  301. iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  302. {
  303. struct iscsi_r2t_info *r2t;
  304. struct iscsi_session *session = conn->session;
  305. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  306. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  307. struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
  308. int r2tsn = be32_to_cpu(rhdr->r2tsn);
  309. int rc;
  310. if (tcp_conn->in.datalen) {
  311. printk(KERN_ERR "iscsi_tcp: invalid R2t with datalen %d\n",
  312. tcp_conn->in.datalen);
  313. return ISCSI_ERR_DATALEN;
  314. }
  315. if (tcp_ctask->exp_datasn != r2tsn){
  316. debug_tcp("%s: ctask->exp_datasn(%d) != rhdr->r2tsn(%d)\n",
  317. __FUNCTION__, tcp_ctask->exp_datasn, r2tsn);
  318. return ISCSI_ERR_R2TSN;
  319. }
  320. /* fill-in new R2T associated with the task */
  321. spin_lock(&session->lock);
  322. iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr);
  323. if (!ctask->sc || ctask->mtask ||
  324. session->state != ISCSI_STATE_LOGGED_IN) {
  325. printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
  326. "recovery...\n", ctask->itt);
  327. spin_unlock(&session->lock);
  328. return 0;
  329. }
  330. rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
  331. BUG_ON(!rc);
  332. r2t->exp_statsn = rhdr->statsn;
  333. r2t->data_length = be32_to_cpu(rhdr->data_length);
  334. if (r2t->data_length == 0) {
  335. printk(KERN_ERR "iscsi_tcp: invalid R2T with zero data len\n");
  336. spin_unlock(&session->lock);
  337. return ISCSI_ERR_DATALEN;
  338. }
  339. if (r2t->data_length > session->max_burst)
  340. debug_scsi("invalid R2T with data len %u and max burst %u."
  341. "Attempting to execute request.\n",
  342. r2t->data_length, session->max_burst);
  343. r2t->data_offset = be32_to_cpu(rhdr->data_offset);
  344. if (r2t->data_offset + r2t->data_length > scsi_bufflen(ctask->sc)) {
  345. spin_unlock(&session->lock);
  346. printk(KERN_ERR "iscsi_tcp: invalid R2T with data len %u at "
  347. "offset %u and total length %d\n", r2t->data_length,
  348. r2t->data_offset, scsi_bufflen(ctask->sc));
  349. return ISCSI_ERR_DATALEN;
  350. }
  351. r2t->ttt = rhdr->ttt; /* no flip */
  352. r2t->solicit_datasn = 0;
  353. iscsi_solicit_data_init(conn, ctask, r2t);
  354. tcp_ctask->exp_datasn = r2tsn + 1;
  355. __kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*));
  356. set_bit(XMSTATE_BIT_SOL_HDR_INIT, &tcp_ctask->xmstate);
  357. list_move_tail(&ctask->running, &conn->xmitqueue);
  358. scsi_queue_work(session->host, &conn->xmitwork);
  359. conn->r2t_pdus_cnt++;
  360. spin_unlock(&session->lock);
  361. return 0;
  362. }
  363. static int
  364. iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
  365. {
  366. int rc = 0, opcode, ahslen;
  367. struct iscsi_hdr *hdr;
  368. struct iscsi_session *session = conn->session;
  369. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  370. uint32_t cdgst, rdgst = 0, itt;
  371. hdr = tcp_conn->in.hdr;
  372. /* verify PDU length */
  373. tcp_conn->in.datalen = ntoh24(hdr->dlength);
  374. if (tcp_conn->in.datalen > conn->max_recv_dlength) {
  375. printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
  376. tcp_conn->in.datalen, conn->max_recv_dlength);
  377. return ISCSI_ERR_DATALEN;
  378. }
  379. tcp_conn->data_copied = 0;
  380. /* read AHS */
  381. ahslen = hdr->hlength << 2;
  382. tcp_conn->in.offset += ahslen;
  383. tcp_conn->in.copy -= ahslen;
  384. if (tcp_conn->in.copy < 0) {
  385. printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
  386. "%d bytes\n", ahslen);
  387. return ISCSI_ERR_AHSLEN;
  388. }
  389. /* calculate read padding */
  390. tcp_conn->in.padding = tcp_conn->in.datalen & (ISCSI_PAD_LEN-1);
  391. if (tcp_conn->in.padding) {
  392. tcp_conn->in.padding = ISCSI_PAD_LEN - tcp_conn->in.padding;
  393. debug_scsi("read padding %d bytes\n", tcp_conn->in.padding);
  394. }
  395. if (conn->hdrdgst_en) {
  396. struct scatterlist sg;
  397. sg_init_one(&sg, (u8 *)hdr,
  398. sizeof(struct iscsi_hdr) + ahslen);
  399. crypto_hash_digest(&tcp_conn->rx_hash, &sg, sg.length,
  400. (u8 *)&cdgst);
  401. rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
  402. ahslen);
  403. if (cdgst != rdgst) {
  404. printk(KERN_ERR "iscsi_tcp: hdrdgst error "
  405. "recv 0x%x calc 0x%x\n", rdgst, cdgst);
  406. return ISCSI_ERR_HDR_DGST;
  407. }
  408. }
  409. opcode = hdr->opcode & ISCSI_OPCODE_MASK;
  410. /* verify itt (itt encoding: age+cid+itt) */
  411. rc = iscsi_verify_itt(conn, hdr, &itt);
  412. if (rc == ISCSI_ERR_NO_SCSI_CMD) {
  413. tcp_conn->in.datalen = 0; /* force drop */
  414. return 0;
  415. } else if (rc)
  416. return rc;
  417. debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
  418. opcode, tcp_conn->in.offset, tcp_conn->in.copy,
  419. ahslen, tcp_conn->in.datalen);
  420. switch(opcode) {
  421. case ISCSI_OP_SCSI_DATA_IN:
  422. tcp_conn->in.ctask = session->cmds[itt];
  423. rc = iscsi_data_rsp(conn, tcp_conn->in.ctask);
  424. if (rc)
  425. return rc;
  426. /* fall through */
  427. case ISCSI_OP_SCSI_CMD_RSP:
  428. tcp_conn->in.ctask = session->cmds[itt];
  429. if (tcp_conn->in.datalen)
  430. goto copy_hdr;
  431. spin_lock(&session->lock);
  432. rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
  433. spin_unlock(&session->lock);
  434. break;
  435. case ISCSI_OP_R2T:
  436. tcp_conn->in.ctask = session->cmds[itt];
  437. if (ahslen)
  438. rc = ISCSI_ERR_AHSLEN;
  439. else if (tcp_conn->in.ctask->sc->sc_data_direction ==
  440. DMA_TO_DEVICE)
  441. rc = iscsi_r2t_rsp(conn, tcp_conn->in.ctask);
  442. else
  443. rc = ISCSI_ERR_PROTO;
  444. break;
  445. case ISCSI_OP_LOGIN_RSP:
  446. case ISCSI_OP_TEXT_RSP:
  447. case ISCSI_OP_REJECT:
  448. case ISCSI_OP_ASYNC_EVENT:
  449. /*
  450. * It is possible that we could get a PDU with a buffer larger
  451. * than 8K, but there are no targets that currently do this.
  452. * For now we fail until we find a vendor that needs it
  453. */
  454. if (ISCSI_DEF_MAX_RECV_SEG_LEN <
  455. tcp_conn->in.datalen) {
  456. printk(KERN_ERR "iscsi_tcp: received buffer of len %u "
  457. "but conn buffer is only %u (opcode %0x)\n",
  458. tcp_conn->in.datalen,
  459. ISCSI_DEF_MAX_RECV_SEG_LEN, opcode);
  460. rc = ISCSI_ERR_PROTO;
  461. break;
  462. }
  463. if (tcp_conn->in.datalen)
  464. goto copy_hdr;
  465. /* fall through */
  466. case ISCSI_OP_LOGOUT_RSP:
  467. case ISCSI_OP_NOOP_IN:
  468. case ISCSI_OP_SCSI_TMFUNC_RSP:
  469. rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
  470. break;
  471. default:
  472. rc = ISCSI_ERR_BAD_OPCODE;
  473. break;
  474. }
  475. return rc;
  476. copy_hdr:
  477. /*
  478. * if we did zero copy for the header but we will need multiple
  479. * skbs to complete the command then we have to copy the header
  480. * for later use
  481. */
  482. if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy <=
  483. (tcp_conn->in.datalen + tcp_conn->in.padding +
  484. (conn->datadgst_en ? 4 : 0))) {
  485. debug_tcp("Copying header for later use. in.copy %d in.datalen"
  486. " %d\n", tcp_conn->in.copy, tcp_conn->in.datalen);
  487. memcpy(&tcp_conn->hdr, tcp_conn->in.hdr,
  488. sizeof(struct iscsi_hdr));
  489. tcp_conn->in.hdr = &tcp_conn->hdr;
  490. tcp_conn->in.zero_copy_hdr = 0;
  491. }
  492. return 0;
  493. }
  494. /**
  495. * iscsi_ctask_copy - copy skb bits to the destanation cmd task
  496. * @conn: iscsi tcp connection
  497. * @ctask: scsi command task
  498. * @buf: buffer to copy to
  499. * @buf_size: size of buffer
  500. * @offset: offset within the buffer
  501. *
  502. * Notes:
  503. * The function calls skb_copy_bits() and updates per-connection and
  504. * per-cmd byte counters.
  505. *
  506. * Read counters (in bytes):
  507. *
  508. * conn->in.offset offset within in progress SKB
  509. * conn->in.copy left to copy from in progress SKB
  510. * including padding
  511. * conn->in.copied copied already from in progress SKB
  512. * conn->data_copied copied already from in progress buffer
  513. * ctask->sent total bytes sent up to the MidLayer
  514. * ctask->data_count left to copy from in progress Data-In
  515. * buf_left left to copy from in progress buffer
  516. **/
  517. static inline int
  518. iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
  519. void *buf, int buf_size, int offset)
  520. {
  521. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  522. int buf_left = buf_size - (tcp_conn->data_copied + offset);
  523. unsigned size = min(tcp_conn->in.copy, buf_left);
  524. int rc;
  525. size = min(size, ctask->data_count);
  526. debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
  527. size, tcp_conn->in.offset, tcp_conn->in.copied);
  528. BUG_ON(size <= 0);
  529. BUG_ON(tcp_ctask->sent + size > scsi_bufflen(ctask->sc));
  530. rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
  531. (char*)buf + (offset + tcp_conn->data_copied), size);
  532. /* must fit into skb->len */
  533. BUG_ON(rc);
  534. tcp_conn->in.offset += size;
  535. tcp_conn->in.copy -= size;
  536. tcp_conn->in.copied += size;
  537. tcp_conn->data_copied += size;
  538. tcp_ctask->sent += size;
  539. ctask->data_count -= size;
  540. BUG_ON(tcp_conn->in.copy < 0);
  541. BUG_ON(ctask->data_count < 0);
  542. if (buf_size != (tcp_conn->data_copied + offset)) {
  543. if (!ctask->data_count) {
  544. BUG_ON(buf_size - tcp_conn->data_copied < 0);
  545. /* done with this PDU */
  546. return buf_size - tcp_conn->data_copied;
  547. }
  548. return -EAGAIN;
  549. }
  550. /* done with this buffer or with both - PDU and buffer */
  551. tcp_conn->data_copied = 0;
  552. return 0;
  553. }
  554. /**
  555. * iscsi_tcp_copy - copy skb bits to the destanation buffer
  556. * @conn: iscsi tcp connection
  557. *
  558. * Notes:
  559. * The function calls skb_copy_bits() and updates per-connection
  560. * byte counters.
  561. **/
  562. static inline int
  563. iscsi_tcp_copy(struct iscsi_conn *conn, int buf_size)
  564. {
  565. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  566. int buf_left = buf_size - tcp_conn->data_copied;
  567. int size = min(tcp_conn->in.copy, buf_left);
  568. int rc;
  569. debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
  570. size, tcp_conn->in.offset, tcp_conn->data_copied);
  571. BUG_ON(size <= 0);
  572. rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
  573. (char*)conn->data + tcp_conn->data_copied, size);
  574. BUG_ON(rc);
  575. tcp_conn->in.offset += size;
  576. tcp_conn->in.copy -= size;
  577. tcp_conn->in.copied += size;
  578. tcp_conn->data_copied += size;
  579. if (buf_size != tcp_conn->data_copied)
  580. return -EAGAIN;
  581. return 0;
  582. }
  583. static inline void
  584. partial_sg_digest_update(struct hash_desc *desc, struct scatterlist *sg,
  585. int offset, int length)
  586. {
  587. struct scatterlist temp;
  588. sg_init_table(&temp, 1);
  589. sg_set_page(&temp, sg_page(sg), length, offset);
  590. crypto_hash_update(desc, &temp, length);
  591. }
  592. static void
  593. iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
  594. {
  595. struct scatterlist tmp;
  596. sg_init_one(&tmp, buf, len);
  597. crypto_hash_update(&tcp_conn->rx_hash, &tmp, len);
  598. }
  599. static int iscsi_scsi_data_in(struct iscsi_conn *conn)
  600. {
  601. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  602. struct iscsi_cmd_task *ctask = tcp_conn->in.ctask;
  603. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  604. struct scsi_cmnd *sc = ctask->sc;
  605. struct scatterlist *sg;
  606. int i, offset, rc = 0;
  607. BUG_ON((void*)ctask != sc->SCp.ptr);
  608. offset = tcp_ctask->data_offset;
  609. sg = scsi_sglist(sc);
  610. if (tcp_ctask->data_offset)
  611. for (i = 0; i < tcp_ctask->sg_count; i++)
  612. offset -= sg[i].length;
  613. /* we've passed through partial sg*/
  614. if (offset < 0)
  615. offset = 0;
  616. for (i = tcp_ctask->sg_count; i < scsi_sg_count(sc); i++) {
  617. char *dest;
  618. dest = kmap_atomic(sg_page(&sg[i]), KM_SOFTIRQ0);
  619. rc = iscsi_ctask_copy(tcp_conn, ctask, dest + sg[i].offset,
  620. sg[i].length, offset);
  621. kunmap_atomic(dest, KM_SOFTIRQ0);
  622. if (rc == -EAGAIN)
  623. /* continue with the next SKB/PDU */
  624. return rc;
  625. if (!rc) {
  626. if (conn->datadgst_en) {
  627. if (!offset)
  628. crypto_hash_update(
  629. &tcp_conn->rx_hash,
  630. &sg[i], sg[i].length);
  631. else
  632. partial_sg_digest_update(
  633. &tcp_conn->rx_hash,
  634. &sg[i],
  635. sg[i].offset + offset,
  636. sg[i].length - offset);
  637. }
  638. offset = 0;
  639. tcp_ctask->sg_count++;
  640. }
  641. if (!ctask->data_count) {
  642. if (rc && conn->datadgst_en)
  643. /*
  644. * data-in is complete, but buffer not...
  645. */
  646. partial_sg_digest_update(&tcp_conn->rx_hash,
  647. &sg[i],
  648. sg[i].offset,
  649. sg[i].length-rc);
  650. rc = 0;
  651. break;
  652. }
  653. if (!tcp_conn->in.copy)
  654. return -EAGAIN;
  655. }
  656. BUG_ON(ctask->data_count);
  657. /* check for non-exceptional status */
  658. if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) {
  659. debug_scsi("done [sc %lx res %d itt 0x%x flags 0x%x]\n",
  660. (long)sc, sc->result, ctask->itt,
  661. tcp_conn->in.hdr->flags);
  662. spin_lock(&conn->session->lock);
  663. __iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
  664. spin_unlock(&conn->session->lock);
  665. }
  666. return rc;
  667. }
  668. static int
  669. iscsi_data_recv(struct iscsi_conn *conn)
  670. {
  671. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  672. int rc = 0, opcode;
  673. opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
  674. switch (opcode) {
  675. case ISCSI_OP_SCSI_DATA_IN:
  676. rc = iscsi_scsi_data_in(conn);
  677. break;
  678. case ISCSI_OP_SCSI_CMD_RSP:
  679. case ISCSI_OP_TEXT_RSP:
  680. case ISCSI_OP_LOGIN_RSP:
  681. case ISCSI_OP_ASYNC_EVENT:
  682. case ISCSI_OP_REJECT:
  683. /*
  684. * Collect data segment to the connection's data
  685. * placeholder
  686. */
  687. if (iscsi_tcp_copy(conn, tcp_conn->in.datalen)) {
  688. rc = -EAGAIN;
  689. goto exit;
  690. }
  691. rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, conn->data,
  692. tcp_conn->in.datalen);
  693. if (!rc && conn->datadgst_en && opcode != ISCSI_OP_LOGIN_RSP)
  694. iscsi_recv_digest_update(tcp_conn, conn->data,
  695. tcp_conn->in.datalen);
  696. break;
  697. default:
  698. BUG_ON(1);
  699. }
  700. exit:
  701. return rc;
  702. }
  703. /**
  704. * iscsi_tcp_data_recv - TCP receive in sendfile fashion
  705. * @rd_desc: read descriptor
  706. * @skb: socket buffer
  707. * @offset: offset in skb
  708. * @len: skb->len - offset
  709. **/
  710. static int
  711. iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
  712. unsigned int offset, size_t len)
  713. {
  714. int rc;
  715. struct iscsi_conn *conn = rd_desc->arg.data;
  716. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  717. int processed;
  718. char pad[ISCSI_PAD_LEN];
  719. struct scatterlist sg;
  720. /*
  721. * Save current SKB and its offset in the corresponding
  722. * connection context.
  723. */
  724. tcp_conn->in.copy = skb->len - offset;
  725. tcp_conn->in.offset = offset;
  726. tcp_conn->in.skb = skb;
  727. tcp_conn->in.len = tcp_conn->in.copy;
  728. BUG_ON(tcp_conn->in.copy <= 0);
  729. debug_tcp("in %d bytes\n", tcp_conn->in.copy);
  730. more:
  731. tcp_conn->in.copied = 0;
  732. rc = 0;
  733. if (unlikely(conn->suspend_rx)) {
  734. debug_tcp("conn %d Rx suspended!\n", conn->id);
  735. return 0;
  736. }
  737. if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
  738. tcp_conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
  739. rc = iscsi_hdr_extract(tcp_conn);
  740. if (rc) {
  741. if (rc == -EAGAIN)
  742. goto nomore;
  743. else {
  744. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  745. return 0;
  746. }
  747. }
  748. /*
  749. * Verify and process incoming PDU header.
  750. */
  751. rc = iscsi_tcp_hdr_recv(conn);
  752. if (!rc && tcp_conn->in.datalen) {
  753. if (conn->datadgst_en)
  754. crypto_hash_init(&tcp_conn->rx_hash);
  755. tcp_conn->in_progress = IN_PROGRESS_DATA_RECV;
  756. } else if (rc) {
  757. iscsi_conn_failure(conn, rc);
  758. return 0;
  759. }
  760. }
  761. if (tcp_conn->in_progress == IN_PROGRESS_DDIGEST_RECV &&
  762. tcp_conn->in.copy) {
  763. uint32_t recv_digest;
  764. debug_tcp("extra data_recv offset %d copy %d\n",
  765. tcp_conn->in.offset, tcp_conn->in.copy);
  766. if (!tcp_conn->data_copied) {
  767. if (tcp_conn->in.padding) {
  768. debug_tcp("padding -> %d\n",
  769. tcp_conn->in.padding);
  770. memset(pad, 0, tcp_conn->in.padding);
  771. sg_init_one(&sg, pad, tcp_conn->in.padding);
  772. crypto_hash_update(&tcp_conn->rx_hash,
  773. &sg, sg.length);
  774. }
  775. crypto_hash_final(&tcp_conn->rx_hash,
  776. (u8 *) &tcp_conn->in.datadgst);
  777. debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst);
  778. }
  779. rc = iscsi_tcp_copy(conn, sizeof(uint32_t));
  780. if (rc) {
  781. if (rc == -EAGAIN)
  782. goto again;
  783. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  784. return 0;
  785. }
  786. memcpy(&recv_digest, conn->data, sizeof(uint32_t));
  787. if (recv_digest != tcp_conn->in.datadgst) {
  788. debug_tcp("iscsi_tcp: data digest error!"
  789. "0x%x != 0x%x\n", recv_digest,
  790. tcp_conn->in.datadgst);
  791. iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
  792. return 0;
  793. } else {
  794. debug_tcp("iscsi_tcp: data digest match!"
  795. "0x%x == 0x%x\n", recv_digest,
  796. tcp_conn->in.datadgst);
  797. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  798. }
  799. }
  800. if (tcp_conn->in_progress == IN_PROGRESS_DATA_RECV &&
  801. tcp_conn->in.copy) {
  802. debug_tcp("data_recv offset %d copy %d\n",
  803. tcp_conn->in.offset, tcp_conn->in.copy);
  804. rc = iscsi_data_recv(conn);
  805. if (rc) {
  806. if (rc == -EAGAIN)
  807. goto again;
  808. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  809. return 0;
  810. }
  811. if (tcp_conn->in.padding)
  812. tcp_conn->in_progress = IN_PROGRESS_PAD_RECV;
  813. else if (conn->datadgst_en)
  814. tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
  815. else
  816. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  817. tcp_conn->data_copied = 0;
  818. }
  819. if (tcp_conn->in_progress == IN_PROGRESS_PAD_RECV &&
  820. tcp_conn->in.copy) {
  821. int copylen = min(tcp_conn->in.padding - tcp_conn->data_copied,
  822. tcp_conn->in.copy);
  823. tcp_conn->in.copy -= copylen;
  824. tcp_conn->in.offset += copylen;
  825. tcp_conn->data_copied += copylen;
  826. if (tcp_conn->data_copied != tcp_conn->in.padding)
  827. tcp_conn->in_progress = IN_PROGRESS_PAD_RECV;
  828. else if (conn->datadgst_en)
  829. tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
  830. else
  831. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  832. tcp_conn->data_copied = 0;
  833. }
  834. debug_tcp("f, processed %d from out of %d padding %d\n",
  835. tcp_conn->in.offset - offset, (int)len, tcp_conn->in.padding);
  836. BUG_ON(tcp_conn->in.offset - offset > len);
  837. if (tcp_conn->in.offset - offset != len) {
  838. debug_tcp("continue to process %d bytes\n",
  839. (int)len - (tcp_conn->in.offset - offset));
  840. goto more;
  841. }
  842. nomore:
  843. processed = tcp_conn->in.offset - offset;
  844. BUG_ON(processed == 0);
  845. return processed;
  846. again:
  847. processed = tcp_conn->in.offset - offset;
  848. debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
  849. processed, (int)len, (int)rd_desc->count);
  850. BUG_ON(processed == 0);
  851. BUG_ON(processed > len);
  852. conn->rxdata_octets += processed;
  853. return processed;
  854. }
  855. static void
  856. iscsi_tcp_data_ready(struct sock *sk, int flag)
  857. {
  858. struct iscsi_conn *conn = sk->sk_user_data;
  859. read_descriptor_t rd_desc;
  860. read_lock(&sk->sk_callback_lock);
  861. /*
  862. * Use rd_desc to pass 'conn' to iscsi_tcp_data_recv.
  863. * We set count to 1 because we want the network layer to
  864. * hand us all the skbs that are available. iscsi_tcp_data_recv
  865. * handled pdus that cross buffers or pdus that still need data.
  866. */
  867. rd_desc.arg.data = conn;
  868. rd_desc.count = 1;
  869. tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
  870. read_unlock(&sk->sk_callback_lock);
  871. }
  872. static void
  873. iscsi_tcp_state_change(struct sock *sk)
  874. {
  875. struct iscsi_tcp_conn *tcp_conn;
  876. struct iscsi_conn *conn;
  877. struct iscsi_session *session;
  878. void (*old_state_change)(struct sock *);
  879. read_lock(&sk->sk_callback_lock);
  880. conn = (struct iscsi_conn*)sk->sk_user_data;
  881. session = conn->session;
  882. if ((sk->sk_state == TCP_CLOSE_WAIT ||
  883. sk->sk_state == TCP_CLOSE) &&
  884. !atomic_read(&sk->sk_rmem_alloc)) {
  885. debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
  886. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  887. }
  888. tcp_conn = conn->dd_data;
  889. old_state_change = tcp_conn->old_state_change;
  890. read_unlock(&sk->sk_callback_lock);
  891. old_state_change(sk);
  892. }
  893. /**
  894. * iscsi_write_space - Called when more output buffer space is available
  895. * @sk: socket space is available for
  896. **/
  897. static void
  898. iscsi_write_space(struct sock *sk)
  899. {
  900. struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
  901. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  902. tcp_conn->old_write_space(sk);
  903. debug_tcp("iscsi_write_space: cid %d\n", conn->id);
  904. scsi_queue_work(conn->session->host, &conn->xmitwork);
  905. }
  906. static void
  907. iscsi_conn_set_callbacks(struct iscsi_conn *conn)
  908. {
  909. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  910. struct sock *sk = tcp_conn->sock->sk;
  911. /* assign new callbacks */
  912. write_lock_bh(&sk->sk_callback_lock);
  913. sk->sk_user_data = conn;
  914. tcp_conn->old_data_ready = sk->sk_data_ready;
  915. tcp_conn->old_state_change = sk->sk_state_change;
  916. tcp_conn->old_write_space = sk->sk_write_space;
  917. sk->sk_data_ready = iscsi_tcp_data_ready;
  918. sk->sk_state_change = iscsi_tcp_state_change;
  919. sk->sk_write_space = iscsi_write_space;
  920. write_unlock_bh(&sk->sk_callback_lock);
  921. }
  922. static void
  923. iscsi_conn_restore_callbacks(struct iscsi_tcp_conn *tcp_conn)
  924. {
  925. struct sock *sk = tcp_conn->sock->sk;
  926. /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
  927. write_lock_bh(&sk->sk_callback_lock);
  928. sk->sk_user_data = NULL;
  929. sk->sk_data_ready = tcp_conn->old_data_ready;
  930. sk->sk_state_change = tcp_conn->old_state_change;
  931. sk->sk_write_space = tcp_conn->old_write_space;
  932. sk->sk_no_check = 0;
  933. write_unlock_bh(&sk->sk_callback_lock);
  934. }
  935. /**
  936. * iscsi_send - generic send routine
  937. * @sk: kernel's socket
  938. * @buf: buffer to write from
  939. * @size: actual size to write
  940. * @flags: socket's flags
  941. */
  942. static inline int
  943. iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
  944. {
  945. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  946. struct socket *sk = tcp_conn->sock;
  947. int offset = buf->sg.offset + buf->sent, res;
  948. /*
  949. * if we got use_sg=0 or are sending something we kmallocd
  950. * then we did not have to do kmap (kmap returns page_address)
  951. *
  952. * if we got use_sg > 0, but had to drop down, we do not
  953. * set clustering so this should only happen for that
  954. * slab case.
  955. */
  956. if (buf->use_sendmsg)
  957. res = sock_no_sendpage(sk, sg_page(&buf->sg), offset, size, flags);
  958. else
  959. res = tcp_conn->sendpage(sk, sg_page(&buf->sg), offset, size, flags);
  960. if (res >= 0) {
  961. conn->txdata_octets += res;
  962. buf->sent += res;
  963. return res;
  964. }
  965. tcp_conn->sendpage_failures_cnt++;
  966. if (res == -EAGAIN)
  967. res = -ENOBUFS;
  968. else
  969. iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
  970. return res;
  971. }
  972. /**
  973. * iscsi_sendhdr - send PDU Header via tcp_sendpage()
  974. * @conn: iscsi connection
  975. * @buf: buffer to write from
  976. * @datalen: lenght of data to be sent after the header
  977. *
  978. * Notes:
  979. * (Tx, Fast Path)
  980. **/
  981. static inline int
  982. iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
  983. {
  984. int flags = 0; /* MSG_DONTWAIT; */
  985. int res, size;
  986. size = buf->sg.length - buf->sent;
  987. BUG_ON(buf->sent + size > buf->sg.length);
  988. if (buf->sent + size != buf->sg.length || datalen)
  989. flags |= MSG_MORE;
  990. res = iscsi_send(conn, buf, size, flags);
  991. debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
  992. if (res >= 0) {
  993. if (size != res)
  994. return -EAGAIN;
  995. return 0;
  996. }
  997. return res;
  998. }
  999. /**
  1000. * iscsi_sendpage - send one page of iSCSI Data-Out.
  1001. * @conn: iscsi connection
  1002. * @buf: buffer to write from
  1003. * @count: remaining data
  1004. * @sent: number of bytes sent
  1005. *
  1006. * Notes:
  1007. * (Tx, Fast Path)
  1008. **/
  1009. static inline int
  1010. iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
  1011. int *count, int *sent)
  1012. {
  1013. int flags = 0; /* MSG_DONTWAIT; */
  1014. int res, size;
  1015. size = buf->sg.length - buf->sent;
  1016. BUG_ON(buf->sent + size > buf->sg.length);
  1017. if (size > *count)
  1018. size = *count;
  1019. if (buf->sent + size != buf->sg.length || *count != size)
  1020. flags |= MSG_MORE;
  1021. res = iscsi_send(conn, buf, size, flags);
  1022. debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
  1023. size, buf->sent, *count, *sent, res);
  1024. if (res >= 0) {
  1025. *count -= res;
  1026. *sent += res;
  1027. if (size != res)
  1028. return -EAGAIN;
  1029. return 0;
  1030. }
  1031. return res;
  1032. }
  1033. static inline void
  1034. iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
  1035. struct iscsi_tcp_cmd_task *tcp_ctask)
  1036. {
  1037. crypto_hash_init(&tcp_conn->tx_hash);
  1038. tcp_ctask->digest_count = 4;
  1039. }
  1040. /**
  1041. * iscsi_solicit_data_cont - initialize next Data-Out
  1042. * @conn: iscsi connection
  1043. * @ctask: scsi command task
  1044. * @r2t: R2T info
  1045. * @left: bytes left to transfer
  1046. *
  1047. * Notes:
  1048. * Initialize next Data-Out within this R2T sequence and continue
  1049. * to process next Scatter-Gather element(if any) of this SCSI command.
  1050. *
  1051. * Called under connection lock.
  1052. **/
  1053. static void
  1054. iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
  1055. struct iscsi_r2t_info *r2t, int left)
  1056. {
  1057. struct iscsi_data *hdr;
  1058. int new_offset;
  1059. hdr = &r2t->dtask.hdr;
  1060. memset(hdr, 0, sizeof(struct iscsi_data));
  1061. hdr->ttt = r2t->ttt;
  1062. hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
  1063. r2t->solicit_datasn++;
  1064. hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
  1065. memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
  1066. hdr->itt = ctask->hdr->itt;
  1067. hdr->exp_statsn = r2t->exp_statsn;
  1068. new_offset = r2t->data_offset + r2t->sent;
  1069. hdr->offset = cpu_to_be32(new_offset);
  1070. if (left > conn->max_xmit_dlength) {
  1071. hton24(hdr->dlength, conn->max_xmit_dlength);
  1072. r2t->data_count = conn->max_xmit_dlength;
  1073. } else {
  1074. hton24(hdr->dlength, left);
  1075. r2t->data_count = left;
  1076. hdr->flags = ISCSI_FLAG_CMD_FINAL;
  1077. }
  1078. conn->dataout_pdus_cnt++;
  1079. iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
  1080. sizeof(struct iscsi_hdr));
  1081. if (iscsi_buf_left(&r2t->sendbuf))
  1082. return;
  1083. iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
  1084. r2t->sg += 1;
  1085. }
  1086. static void iscsi_set_padding(struct iscsi_tcp_cmd_task *tcp_ctask,
  1087. unsigned long len)
  1088. {
  1089. tcp_ctask->pad_count = len & (ISCSI_PAD_LEN - 1);
  1090. if (!tcp_ctask->pad_count)
  1091. return;
  1092. tcp_ctask->pad_count = ISCSI_PAD_LEN - tcp_ctask->pad_count;
  1093. debug_scsi("write padding %d bytes\n", tcp_ctask->pad_count);
  1094. set_bit(XMSTATE_BIT_W_PAD, &tcp_ctask->xmstate);
  1095. }
  1096. /**
  1097. * iscsi_tcp_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
  1098. * @conn: iscsi connection
  1099. * @ctask: scsi command task
  1100. * @sc: scsi command
  1101. **/
  1102. static void
  1103. iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
  1104. {
  1105. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1106. BUG_ON(__kfifo_len(tcp_ctask->r2tqueue));
  1107. tcp_ctask->xmstate = 1 << XMSTATE_BIT_CMD_HDR_INIT;
  1108. }
  1109. /**
  1110. * iscsi_tcp_mtask_xmit - xmit management(immediate) task
  1111. * @conn: iscsi connection
  1112. * @mtask: task management task
  1113. *
  1114. * Notes:
  1115. * The function can return -EAGAIN in which case caller must
  1116. * call it again later, or recover. '0' return code means successful
  1117. * xmit.
  1118. *
  1119. * Management xmit state machine consists of these states:
  1120. * XMSTATE_BIT_IMM_HDR_INIT - calculate digest of PDU Header
  1121. * XMSTATE_BIT_IMM_HDR - PDU Header xmit in progress
  1122. * XMSTATE_BIT_IMM_DATA - PDU Data xmit in progress
  1123. * XMSTATE_VALUE_IDLE - management PDU is done
  1124. **/
  1125. static int
  1126. iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
  1127. {
  1128. struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
  1129. int rc;
  1130. debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
  1131. conn->id, tcp_mtask->xmstate, mtask->itt);
  1132. if (test_bit(XMSTATE_BIT_IMM_HDR_INIT, &tcp_mtask->xmstate)) {
  1133. iscsi_buf_init_iov(&tcp_mtask->headbuf, (char*)mtask->hdr,
  1134. sizeof(struct iscsi_hdr));
  1135. if (mtask->data_count) {
  1136. set_bit(XMSTATE_BIT_IMM_DATA, &tcp_mtask->xmstate);
  1137. iscsi_buf_init_iov(&tcp_mtask->sendbuf,
  1138. (char*)mtask->data,
  1139. mtask->data_count);
  1140. }
  1141. if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
  1142. conn->stop_stage != STOP_CONN_RECOVER &&
  1143. conn->hdrdgst_en)
  1144. iscsi_hdr_digest(conn, &tcp_mtask->headbuf,
  1145. (u8*)tcp_mtask->hdrext);
  1146. tcp_mtask->sent = 0;
  1147. clear_bit(XMSTATE_BIT_IMM_HDR_INIT, &tcp_mtask->xmstate);
  1148. set_bit(XMSTATE_BIT_IMM_HDR, &tcp_mtask->xmstate);
  1149. }
  1150. if (test_bit(XMSTATE_BIT_IMM_HDR, &tcp_mtask->xmstate)) {
  1151. rc = iscsi_sendhdr(conn, &tcp_mtask->headbuf,
  1152. mtask->data_count);
  1153. if (rc)
  1154. return rc;
  1155. clear_bit(XMSTATE_BIT_IMM_HDR, &tcp_mtask->xmstate);
  1156. }
  1157. if (test_and_clear_bit(XMSTATE_BIT_IMM_DATA, &tcp_mtask->xmstate)) {
  1158. BUG_ON(!mtask->data_count);
  1159. /* FIXME: implement.
  1160. * Virtual buffer could be spreaded across multiple pages...
  1161. */
  1162. do {
  1163. int rc;
  1164. rc = iscsi_sendpage(conn, &tcp_mtask->sendbuf,
  1165. &mtask->data_count, &tcp_mtask->sent);
  1166. if (rc) {
  1167. set_bit(XMSTATE_BIT_IMM_DATA, &tcp_mtask->xmstate);
  1168. return rc;
  1169. }
  1170. } while (mtask->data_count);
  1171. }
  1172. BUG_ON(tcp_mtask->xmstate != XMSTATE_VALUE_IDLE);
  1173. if (mtask->hdr->itt == RESERVED_ITT) {
  1174. struct iscsi_session *session = conn->session;
  1175. spin_lock_bh(&session->lock);
  1176. list_del(&conn->mtask->running);
  1177. __kfifo_put(session->mgmtpool.queue, (void*)&conn->mtask,
  1178. sizeof(void*));
  1179. spin_unlock_bh(&session->lock);
  1180. }
  1181. return 0;
  1182. }
  1183. static int
  1184. iscsi_send_cmd_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  1185. {
  1186. struct scsi_cmnd *sc = ctask->sc;
  1187. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1188. int rc = 0;
  1189. if (test_bit(XMSTATE_BIT_CMD_HDR_INIT, &tcp_ctask->xmstate)) {
  1190. tcp_ctask->sent = 0;
  1191. tcp_ctask->sg_count = 0;
  1192. tcp_ctask->exp_datasn = 0;
  1193. if (sc->sc_data_direction == DMA_TO_DEVICE) {
  1194. struct scatterlist *sg = scsi_sglist(sc);
  1195. iscsi_buf_init_sg(&tcp_ctask->sendbuf, sg);
  1196. tcp_ctask->sg = sg + 1;
  1197. tcp_ctask->bad_sg = sg + scsi_sg_count(sc);
  1198. debug_scsi("cmd [itt 0x%x total %d imm_data %d "
  1199. "unsol count %d, unsol offset %d]\n",
  1200. ctask->itt, scsi_bufflen(sc),
  1201. ctask->imm_count, ctask->unsol_count,
  1202. ctask->unsol_offset);
  1203. }
  1204. iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)ctask->hdr,
  1205. sizeof(struct iscsi_hdr));
  1206. if (conn->hdrdgst_en)
  1207. iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
  1208. (u8*)tcp_ctask->hdrext);
  1209. clear_bit(XMSTATE_BIT_CMD_HDR_INIT, &tcp_ctask->xmstate);
  1210. set_bit(XMSTATE_BIT_CMD_HDR_XMIT, &tcp_ctask->xmstate);
  1211. }
  1212. if (test_bit(XMSTATE_BIT_CMD_HDR_XMIT, &tcp_ctask->xmstate)) {
  1213. rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count);
  1214. if (rc)
  1215. return rc;
  1216. clear_bit(XMSTATE_BIT_CMD_HDR_XMIT, &tcp_ctask->xmstate);
  1217. if (sc->sc_data_direction != DMA_TO_DEVICE)
  1218. return 0;
  1219. if (ctask->imm_count) {
  1220. set_bit(XMSTATE_BIT_IMM_DATA, &tcp_ctask->xmstate);
  1221. iscsi_set_padding(tcp_ctask, ctask->imm_count);
  1222. if (ctask->conn->datadgst_en) {
  1223. iscsi_data_digest_init(ctask->conn->dd_data,
  1224. tcp_ctask);
  1225. tcp_ctask->immdigest = 0;
  1226. }
  1227. }
  1228. if (ctask->unsol_count) {
  1229. set_bit(XMSTATE_BIT_UNS_HDR, &tcp_ctask->xmstate);
  1230. set_bit(XMSTATE_BIT_UNS_INIT, &tcp_ctask->xmstate);
  1231. }
  1232. }
  1233. return rc;
  1234. }
  1235. static int
  1236. iscsi_send_padding(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  1237. {
  1238. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1239. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1240. int sent = 0, rc;
  1241. if (test_bit(XMSTATE_BIT_W_PAD, &tcp_ctask->xmstate)) {
  1242. iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
  1243. tcp_ctask->pad_count);
  1244. if (conn->datadgst_en)
  1245. crypto_hash_update(&tcp_conn->tx_hash,
  1246. &tcp_ctask->sendbuf.sg,
  1247. tcp_ctask->sendbuf.sg.length);
  1248. } else if (!test_bit(XMSTATE_BIT_W_RESEND_PAD, &tcp_ctask->xmstate))
  1249. return 0;
  1250. clear_bit(XMSTATE_BIT_W_PAD, &tcp_ctask->xmstate);
  1251. clear_bit(XMSTATE_BIT_W_RESEND_PAD, &tcp_ctask->xmstate);
  1252. debug_scsi("sending %d pad bytes for itt 0x%x\n",
  1253. tcp_ctask->pad_count, ctask->itt);
  1254. rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
  1255. &sent);
  1256. if (rc) {
  1257. debug_scsi("padding send failed %d\n", rc);
  1258. set_bit(XMSTATE_BIT_W_RESEND_PAD, &tcp_ctask->xmstate);
  1259. }
  1260. return rc;
  1261. }
  1262. static int
  1263. iscsi_send_digest(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
  1264. struct iscsi_buf *buf, uint32_t *digest)
  1265. {
  1266. struct iscsi_tcp_cmd_task *tcp_ctask;
  1267. struct iscsi_tcp_conn *tcp_conn;
  1268. int rc, sent = 0;
  1269. if (!conn->datadgst_en)
  1270. return 0;
  1271. tcp_ctask = ctask->dd_data;
  1272. tcp_conn = conn->dd_data;
  1273. if (!test_bit(XMSTATE_BIT_W_RESEND_DATA_DIGEST, &tcp_ctask->xmstate)) {
  1274. crypto_hash_final(&tcp_conn->tx_hash, (u8*)digest);
  1275. iscsi_buf_init_iov(buf, (char*)digest, 4);
  1276. }
  1277. clear_bit(XMSTATE_BIT_W_RESEND_DATA_DIGEST, &tcp_ctask->xmstate);
  1278. rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
  1279. if (!rc)
  1280. debug_scsi("sent digest 0x%x for itt 0x%x\n", *digest,
  1281. ctask->itt);
  1282. else {
  1283. debug_scsi("sending digest 0x%x failed for itt 0x%x!\n",
  1284. *digest, ctask->itt);
  1285. set_bit(XMSTATE_BIT_W_RESEND_DATA_DIGEST, &tcp_ctask->xmstate);
  1286. }
  1287. return rc;
  1288. }
  1289. static int
  1290. iscsi_send_data(struct iscsi_cmd_task *ctask, struct iscsi_buf *sendbuf,
  1291. struct scatterlist **sg, int *sent, int *count,
  1292. struct iscsi_buf *digestbuf, uint32_t *digest)
  1293. {
  1294. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1295. struct iscsi_conn *conn = ctask->conn;
  1296. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1297. int rc, buf_sent, offset;
  1298. while (*count) {
  1299. buf_sent = 0;
  1300. offset = sendbuf->sent;
  1301. rc = iscsi_sendpage(conn, sendbuf, count, &buf_sent);
  1302. *sent = *sent + buf_sent;
  1303. if (buf_sent && conn->datadgst_en)
  1304. partial_sg_digest_update(&tcp_conn->tx_hash,
  1305. &sendbuf->sg, sendbuf->sg.offset + offset,
  1306. buf_sent);
  1307. if (!iscsi_buf_left(sendbuf) && *sg != tcp_ctask->bad_sg) {
  1308. iscsi_buf_init_sg(sendbuf, *sg);
  1309. *sg = *sg + 1;
  1310. }
  1311. if (rc)
  1312. return rc;
  1313. }
  1314. rc = iscsi_send_padding(conn, ctask);
  1315. if (rc)
  1316. return rc;
  1317. return iscsi_send_digest(conn, ctask, digestbuf, digest);
  1318. }
  1319. static int
  1320. iscsi_send_unsol_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  1321. {
  1322. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1323. struct iscsi_data_task *dtask;
  1324. int rc;
  1325. set_bit(XMSTATE_BIT_UNS_DATA, &tcp_ctask->xmstate);
  1326. if (test_bit(XMSTATE_BIT_UNS_INIT, &tcp_ctask->xmstate)) {
  1327. dtask = &tcp_ctask->unsol_dtask;
  1328. iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr);
  1329. iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
  1330. sizeof(struct iscsi_hdr));
  1331. if (conn->hdrdgst_en)
  1332. iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
  1333. (u8*)dtask->hdrext);
  1334. clear_bit(XMSTATE_BIT_UNS_INIT, &tcp_ctask->xmstate);
  1335. iscsi_set_padding(tcp_ctask, ctask->data_count);
  1336. }
  1337. rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count);
  1338. if (rc) {
  1339. clear_bit(XMSTATE_BIT_UNS_DATA, &tcp_ctask->xmstate);
  1340. set_bit(XMSTATE_BIT_UNS_HDR, &tcp_ctask->xmstate);
  1341. return rc;
  1342. }
  1343. if (conn->datadgst_en) {
  1344. dtask = &tcp_ctask->unsol_dtask;
  1345. iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);
  1346. dtask->digest = 0;
  1347. }
  1348. debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
  1349. ctask->itt, ctask->unsol_count, tcp_ctask->sent);
  1350. return 0;
  1351. }
  1352. static int
  1353. iscsi_send_unsol_pdu(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  1354. {
  1355. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1356. int rc;
  1357. if (test_and_clear_bit(XMSTATE_BIT_UNS_HDR, &tcp_ctask->xmstate)) {
  1358. BUG_ON(!ctask->unsol_count);
  1359. send_hdr:
  1360. rc = iscsi_send_unsol_hdr(conn, ctask);
  1361. if (rc)
  1362. return rc;
  1363. }
  1364. if (test_bit(XMSTATE_BIT_UNS_DATA, &tcp_ctask->xmstate)) {
  1365. struct iscsi_data_task *dtask = &tcp_ctask->unsol_dtask;
  1366. int start = tcp_ctask->sent;
  1367. rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
  1368. &tcp_ctask->sent, &ctask->data_count,
  1369. &dtask->digestbuf, &dtask->digest);
  1370. ctask->unsol_count -= tcp_ctask->sent - start;
  1371. if (rc)
  1372. return rc;
  1373. clear_bit(XMSTATE_BIT_UNS_DATA, &tcp_ctask->xmstate);
  1374. /*
  1375. * Done with the Data-Out. Next, check if we need
  1376. * to send another unsolicited Data-Out.
  1377. */
  1378. if (ctask->unsol_count) {
  1379. debug_scsi("sending more uns\n");
  1380. set_bit(XMSTATE_BIT_UNS_INIT, &tcp_ctask->xmstate);
  1381. goto send_hdr;
  1382. }
  1383. }
  1384. return 0;
  1385. }
  1386. static int iscsi_send_sol_pdu(struct iscsi_conn *conn,
  1387. struct iscsi_cmd_task *ctask)
  1388. {
  1389. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1390. struct iscsi_session *session = conn->session;
  1391. struct iscsi_r2t_info *r2t;
  1392. struct iscsi_data_task *dtask;
  1393. int left, rc;
  1394. if (test_bit(XMSTATE_BIT_SOL_HDR_INIT, &tcp_ctask->xmstate)) {
  1395. if (!tcp_ctask->r2t) {
  1396. spin_lock_bh(&session->lock);
  1397. __kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
  1398. sizeof(void*));
  1399. spin_unlock_bh(&session->lock);
  1400. }
  1401. send_hdr:
  1402. r2t = tcp_ctask->r2t;
  1403. dtask = &r2t->dtask;
  1404. if (conn->hdrdgst_en)
  1405. iscsi_hdr_digest(conn, &r2t->headbuf,
  1406. (u8*)dtask->hdrext);
  1407. clear_bit(XMSTATE_BIT_SOL_HDR_INIT, &tcp_ctask->xmstate);
  1408. set_bit(XMSTATE_BIT_SOL_HDR, &tcp_ctask->xmstate);
  1409. }
  1410. if (test_bit(XMSTATE_BIT_SOL_HDR, &tcp_ctask->xmstate)) {
  1411. r2t = tcp_ctask->r2t;
  1412. dtask = &r2t->dtask;
  1413. rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);
  1414. if (rc)
  1415. return rc;
  1416. clear_bit(XMSTATE_BIT_SOL_HDR, &tcp_ctask->xmstate);
  1417. set_bit(XMSTATE_BIT_SOL_DATA, &tcp_ctask->xmstate);
  1418. if (conn->datadgst_en) {
  1419. iscsi_data_digest_init(conn->dd_data, tcp_ctask);
  1420. dtask->digest = 0;
  1421. }
  1422. iscsi_set_padding(tcp_ctask, r2t->data_count);
  1423. debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
  1424. r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
  1425. r2t->sent);
  1426. }
  1427. if (test_bit(XMSTATE_BIT_SOL_DATA, &tcp_ctask->xmstate)) {
  1428. r2t = tcp_ctask->r2t;
  1429. dtask = &r2t->dtask;
  1430. rc = iscsi_send_data(ctask, &r2t->sendbuf, &r2t->sg,
  1431. &r2t->sent, &r2t->data_count,
  1432. &dtask->digestbuf, &dtask->digest);
  1433. if (rc)
  1434. return rc;
  1435. clear_bit(XMSTATE_BIT_SOL_DATA, &tcp_ctask->xmstate);
  1436. /*
  1437. * Done with this Data-Out. Next, check if we have
  1438. * to send another Data-Out for this R2T.
  1439. */
  1440. BUG_ON(r2t->data_length - r2t->sent < 0);
  1441. left = r2t->data_length - r2t->sent;
  1442. if (left) {
  1443. iscsi_solicit_data_cont(conn, ctask, r2t, left);
  1444. goto send_hdr;
  1445. }
  1446. /*
  1447. * Done with this R2T. Check if there are more
  1448. * outstanding R2Ts ready to be processed.
  1449. */
  1450. spin_lock_bh(&session->lock);
  1451. tcp_ctask->r2t = NULL;
  1452. __kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
  1453. sizeof(void*));
  1454. if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t,
  1455. sizeof(void*))) {
  1456. tcp_ctask->r2t = r2t;
  1457. spin_unlock_bh(&session->lock);
  1458. goto send_hdr;
  1459. }
  1460. spin_unlock_bh(&session->lock);
  1461. }
  1462. return 0;
  1463. }
  1464. /**
  1465. * iscsi_tcp_ctask_xmit - xmit normal PDU task
  1466. * @conn: iscsi connection
  1467. * @ctask: iscsi command task
  1468. *
  1469. * Notes:
  1470. * The function can return -EAGAIN in which case caller must
  1471. * call it again later, or recover. '0' return code means successful
  1472. * xmit.
  1473. * The function is devided to logical helpers (above) for the different
  1474. * xmit stages.
  1475. *
  1476. *iscsi_send_cmd_hdr()
  1477. * XMSTATE_BIT_CMD_HDR_INIT - prepare Header and Data buffers Calculate
  1478. * Header Digest
  1479. * XMSTATE_BIT_CMD_HDR_XMIT - Transmit header in progress
  1480. *
  1481. *iscsi_send_padding
  1482. * XMSTATE_BIT_W_PAD - Prepare and send pading
  1483. * XMSTATE_BIT_W_RESEND_PAD - retry send pading
  1484. *
  1485. *iscsi_send_digest
  1486. * XMSTATE_BIT_W_RESEND_DATA_DIGEST - Finalize and send Data Digest
  1487. * XMSTATE_BIT_W_RESEND_DATA_DIGEST - retry sending digest
  1488. *
  1489. *iscsi_send_unsol_hdr
  1490. * XMSTATE_BIT_UNS_INIT - prepare un-solicit data header and digest
  1491. * XMSTATE_BIT_UNS_HDR - send un-solicit header
  1492. *
  1493. *iscsi_send_unsol_pdu
  1494. * XMSTATE_BIT_UNS_DATA - send un-solicit data in progress
  1495. *
  1496. *iscsi_send_sol_pdu
  1497. * XMSTATE_BIT_SOL_HDR_INIT - solicit data header and digest initialize
  1498. * XMSTATE_BIT_SOL_HDR - send solicit header
  1499. * XMSTATE_BIT_SOL_DATA - send solicit data
  1500. *
  1501. *iscsi_tcp_ctask_xmit
  1502. * XMSTATE_BIT_IMM_DATA - xmit managment data (??)
  1503. **/
  1504. static int
  1505. iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
  1506. {
  1507. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1508. int rc = 0;
  1509. debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
  1510. conn->id, tcp_ctask->xmstate, ctask->itt);
  1511. rc = iscsi_send_cmd_hdr(conn, ctask);
  1512. if (rc)
  1513. return rc;
  1514. if (ctask->sc->sc_data_direction != DMA_TO_DEVICE)
  1515. return 0;
  1516. if (test_bit(XMSTATE_BIT_IMM_DATA, &tcp_ctask->xmstate)) {
  1517. rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
  1518. &tcp_ctask->sent, &ctask->imm_count,
  1519. &tcp_ctask->immbuf, &tcp_ctask->immdigest);
  1520. if (rc)
  1521. return rc;
  1522. clear_bit(XMSTATE_BIT_IMM_DATA, &tcp_ctask->xmstate);
  1523. }
  1524. rc = iscsi_send_unsol_pdu(conn, ctask);
  1525. if (rc)
  1526. return rc;
  1527. rc = iscsi_send_sol_pdu(conn, ctask);
  1528. if (rc)
  1529. return rc;
  1530. return rc;
  1531. }
  1532. static struct iscsi_cls_conn *
  1533. iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
  1534. {
  1535. struct iscsi_conn *conn;
  1536. struct iscsi_cls_conn *cls_conn;
  1537. struct iscsi_tcp_conn *tcp_conn;
  1538. cls_conn = iscsi_conn_setup(cls_session, conn_idx);
  1539. if (!cls_conn)
  1540. return NULL;
  1541. conn = cls_conn->dd_data;
  1542. /*
  1543. * due to strange issues with iser these are not set
  1544. * in iscsi_conn_setup
  1545. */
  1546. conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN;
  1547. tcp_conn = kzalloc(sizeof(*tcp_conn), GFP_KERNEL);
  1548. if (!tcp_conn)
  1549. goto tcp_conn_alloc_fail;
  1550. conn->dd_data = tcp_conn;
  1551. tcp_conn->iscsi_conn = conn;
  1552. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  1553. /* initial operational parameters */
  1554. tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
  1555. tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
  1556. CRYPTO_ALG_ASYNC);
  1557. tcp_conn->tx_hash.flags = 0;
  1558. if (IS_ERR(tcp_conn->tx_hash.tfm)) {
  1559. printk(KERN_ERR "Could not create connection due to crc32c "
  1560. "loading error %ld. Make sure the crc32c module is "
  1561. "built as a module or into the kernel\n",
  1562. PTR_ERR(tcp_conn->tx_hash.tfm));
  1563. goto free_tcp_conn;
  1564. }
  1565. tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
  1566. CRYPTO_ALG_ASYNC);
  1567. tcp_conn->rx_hash.flags = 0;
  1568. if (IS_ERR(tcp_conn->rx_hash.tfm)) {
  1569. printk(KERN_ERR "Could not create connection due to crc32c "
  1570. "loading error %ld. Make sure the crc32c module is "
  1571. "built as a module or into the kernel\n",
  1572. PTR_ERR(tcp_conn->rx_hash.tfm));
  1573. goto free_tx_tfm;
  1574. }
  1575. return cls_conn;
  1576. free_tx_tfm:
  1577. crypto_free_hash(tcp_conn->tx_hash.tfm);
  1578. free_tcp_conn:
  1579. kfree(tcp_conn);
  1580. tcp_conn_alloc_fail:
  1581. iscsi_conn_teardown(cls_conn);
  1582. return NULL;
  1583. }
  1584. static void
  1585. iscsi_tcp_release_conn(struct iscsi_conn *conn)
  1586. {
  1587. struct iscsi_session *session = conn->session;
  1588. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1589. struct socket *sock = tcp_conn->sock;
  1590. if (!sock)
  1591. return;
  1592. sock_hold(sock->sk);
  1593. iscsi_conn_restore_callbacks(tcp_conn);
  1594. sock_put(sock->sk);
  1595. spin_lock_bh(&session->lock);
  1596. tcp_conn->sock = NULL;
  1597. conn->recv_lock = NULL;
  1598. spin_unlock_bh(&session->lock);
  1599. sockfd_put(sock);
  1600. }
  1601. static void
  1602. iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
  1603. {
  1604. struct iscsi_conn *conn = cls_conn->dd_data;
  1605. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1606. iscsi_tcp_release_conn(conn);
  1607. iscsi_conn_teardown(cls_conn);
  1608. if (tcp_conn->tx_hash.tfm)
  1609. crypto_free_hash(tcp_conn->tx_hash.tfm);
  1610. if (tcp_conn->rx_hash.tfm)
  1611. crypto_free_hash(tcp_conn->rx_hash.tfm);
  1612. kfree(tcp_conn);
  1613. }
  1614. static void
  1615. iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
  1616. {
  1617. struct iscsi_conn *conn = cls_conn->dd_data;
  1618. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1619. iscsi_conn_stop(cls_conn, flag);
  1620. iscsi_tcp_release_conn(conn);
  1621. tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
  1622. }
  1623. static int iscsi_tcp_get_addr(struct iscsi_conn *conn, struct socket *sock,
  1624. char *buf, int *port,
  1625. int (*getname)(struct socket *, struct sockaddr *,
  1626. int *addrlen))
  1627. {
  1628. struct sockaddr_storage *addr;
  1629. struct sockaddr_in6 *sin6;
  1630. struct sockaddr_in *sin;
  1631. int rc = 0, len;
  1632. addr = kmalloc(sizeof(*addr), GFP_KERNEL);
  1633. if (!addr)
  1634. return -ENOMEM;
  1635. if (getname(sock, (struct sockaddr *) addr, &len)) {
  1636. rc = -ENODEV;
  1637. goto free_addr;
  1638. }
  1639. switch (addr->ss_family) {
  1640. case AF_INET:
  1641. sin = (struct sockaddr_in *)addr;
  1642. spin_lock_bh(&conn->session->lock);
  1643. sprintf(buf, NIPQUAD_FMT, NIPQUAD(sin->sin_addr.s_addr));
  1644. *port = be16_to_cpu(sin->sin_port);
  1645. spin_unlock_bh(&conn->session->lock);
  1646. break;
  1647. case AF_INET6:
  1648. sin6 = (struct sockaddr_in6 *)addr;
  1649. spin_lock_bh(&conn->session->lock);
  1650. sprintf(buf, NIP6_FMT, NIP6(sin6->sin6_addr));
  1651. *port = be16_to_cpu(sin6->sin6_port);
  1652. spin_unlock_bh(&conn->session->lock);
  1653. break;
  1654. }
  1655. free_addr:
  1656. kfree(addr);
  1657. return rc;
  1658. }
  1659. static int
  1660. iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
  1661. struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
  1662. int is_leading)
  1663. {
  1664. struct iscsi_conn *conn = cls_conn->dd_data;
  1665. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1666. struct sock *sk;
  1667. struct socket *sock;
  1668. int err;
  1669. /* lookup for existing socket */
  1670. sock = sockfd_lookup((int)transport_eph, &err);
  1671. if (!sock) {
  1672. printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
  1673. return -EEXIST;
  1674. }
  1675. /*
  1676. * copy these values now because if we drop the session
  1677. * userspace may still want to query the values since we will
  1678. * be using them for the reconnect
  1679. */
  1680. err = iscsi_tcp_get_addr(conn, sock, conn->portal_address,
  1681. &conn->portal_port, kernel_getpeername);
  1682. if (err)
  1683. goto free_socket;
  1684. err = iscsi_tcp_get_addr(conn, sock, conn->local_address,
  1685. &conn->local_port, kernel_getsockname);
  1686. if (err)
  1687. goto free_socket;
  1688. err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
  1689. if (err)
  1690. goto free_socket;
  1691. /* bind iSCSI connection and socket */
  1692. tcp_conn->sock = sock;
  1693. /* setup Socket parameters */
  1694. sk = sock->sk;
  1695. sk->sk_reuse = 1;
  1696. sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
  1697. sk->sk_allocation = GFP_ATOMIC;
  1698. /* FIXME: disable Nagle's algorithm */
  1699. /*
  1700. * Intercept TCP callbacks for sendfile like receive
  1701. * processing.
  1702. */
  1703. conn->recv_lock = &sk->sk_callback_lock;
  1704. iscsi_conn_set_callbacks(conn);
  1705. tcp_conn->sendpage = tcp_conn->sock->ops->sendpage;
  1706. /*
  1707. * set receive state machine into initial state
  1708. */
  1709. tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
  1710. return 0;
  1711. free_socket:
  1712. sockfd_put(sock);
  1713. return err;
  1714. }
  1715. /* called with host lock */
  1716. static void
  1717. iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
  1718. {
  1719. struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
  1720. tcp_mtask->xmstate = 1 << XMSTATE_BIT_IMM_HDR_INIT;
  1721. }
  1722. static int
  1723. iscsi_r2tpool_alloc(struct iscsi_session *session)
  1724. {
  1725. int i;
  1726. int cmd_i;
  1727. /*
  1728. * initialize per-task: R2T pool and xmit queue
  1729. */
  1730. for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
  1731. struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
  1732. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1733. /*
  1734. * pre-allocated x4 as much r2ts to handle race when
  1735. * target acks DataOut faster than we data_xmit() queues
  1736. * could replenish r2tqueue.
  1737. */
  1738. /* R2T pool */
  1739. if (iscsi_pool_init(&tcp_ctask->r2tpool, session->max_r2t * 4,
  1740. (void***)&tcp_ctask->r2ts,
  1741. sizeof(struct iscsi_r2t_info))) {
  1742. goto r2t_alloc_fail;
  1743. }
  1744. /* R2T xmit queue */
  1745. tcp_ctask->r2tqueue = kfifo_alloc(
  1746. session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
  1747. if (tcp_ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
  1748. iscsi_pool_free(&tcp_ctask->r2tpool,
  1749. (void**)tcp_ctask->r2ts);
  1750. goto r2t_alloc_fail;
  1751. }
  1752. }
  1753. return 0;
  1754. r2t_alloc_fail:
  1755. for (i = 0; i < cmd_i; i++) {
  1756. struct iscsi_cmd_task *ctask = session->cmds[i];
  1757. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1758. kfifo_free(tcp_ctask->r2tqueue);
  1759. iscsi_pool_free(&tcp_ctask->r2tpool,
  1760. (void**)tcp_ctask->r2ts);
  1761. }
  1762. return -ENOMEM;
  1763. }
  1764. static void
  1765. iscsi_r2tpool_free(struct iscsi_session *session)
  1766. {
  1767. int i;
  1768. for (i = 0; i < session->cmds_max; i++) {
  1769. struct iscsi_cmd_task *ctask = session->cmds[i];
  1770. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1771. kfifo_free(tcp_ctask->r2tqueue);
  1772. iscsi_pool_free(&tcp_ctask->r2tpool,
  1773. (void**)tcp_ctask->r2ts);
  1774. }
  1775. }
  1776. static int
  1777. iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
  1778. char *buf, int buflen)
  1779. {
  1780. struct iscsi_conn *conn = cls_conn->dd_data;
  1781. struct iscsi_session *session = conn->session;
  1782. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1783. int value;
  1784. switch(param) {
  1785. case ISCSI_PARAM_HDRDGST_EN:
  1786. iscsi_set_param(cls_conn, param, buf, buflen);
  1787. tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
  1788. if (conn->hdrdgst_en)
  1789. tcp_conn->hdr_size += sizeof(__u32);
  1790. break;
  1791. case ISCSI_PARAM_DATADGST_EN:
  1792. iscsi_set_param(cls_conn, param, buf, buflen);
  1793. tcp_conn->sendpage = conn->datadgst_en ?
  1794. sock_no_sendpage : tcp_conn->sock->ops->sendpage;
  1795. break;
  1796. case ISCSI_PARAM_MAX_R2T:
  1797. sscanf(buf, "%d", &value);
  1798. if (session->max_r2t == roundup_pow_of_two(value))
  1799. break;
  1800. iscsi_r2tpool_free(session);
  1801. iscsi_set_param(cls_conn, param, buf, buflen);
  1802. if (session->max_r2t & (session->max_r2t - 1))
  1803. session->max_r2t = roundup_pow_of_two(session->max_r2t);
  1804. if (iscsi_r2tpool_alloc(session))
  1805. return -ENOMEM;
  1806. break;
  1807. default:
  1808. return iscsi_set_param(cls_conn, param, buf, buflen);
  1809. }
  1810. return 0;
  1811. }
  1812. static int
  1813. iscsi_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
  1814. enum iscsi_param param, char *buf)
  1815. {
  1816. struct iscsi_conn *conn = cls_conn->dd_data;
  1817. int len;
  1818. switch(param) {
  1819. case ISCSI_PARAM_CONN_PORT:
  1820. spin_lock_bh(&conn->session->lock);
  1821. len = sprintf(buf, "%hu\n", conn->portal_port);
  1822. spin_unlock_bh(&conn->session->lock);
  1823. break;
  1824. case ISCSI_PARAM_CONN_ADDRESS:
  1825. spin_lock_bh(&conn->session->lock);
  1826. len = sprintf(buf, "%s\n", conn->portal_address);
  1827. spin_unlock_bh(&conn->session->lock);
  1828. break;
  1829. default:
  1830. return iscsi_conn_get_param(cls_conn, param, buf);
  1831. }
  1832. return len;
  1833. }
  1834. static int
  1835. iscsi_tcp_host_get_param(struct Scsi_Host *shost, enum iscsi_host_param param,
  1836. char *buf)
  1837. {
  1838. struct iscsi_session *session = iscsi_hostdata(shost->hostdata);
  1839. int len;
  1840. switch (param) {
  1841. case ISCSI_HOST_PARAM_IPADDRESS:
  1842. spin_lock_bh(&session->lock);
  1843. if (!session->leadconn)
  1844. len = -ENODEV;
  1845. else
  1846. len = sprintf(buf, "%s\n",
  1847. session->leadconn->local_address);
  1848. spin_unlock_bh(&session->lock);
  1849. break;
  1850. default:
  1851. return iscsi_host_get_param(shost, param, buf);
  1852. }
  1853. return len;
  1854. }
  1855. static void
  1856. iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
  1857. {
  1858. struct iscsi_conn *conn = cls_conn->dd_data;
  1859. struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
  1860. stats->txdata_octets = conn->txdata_octets;
  1861. stats->rxdata_octets = conn->rxdata_octets;
  1862. stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
  1863. stats->dataout_pdus = conn->dataout_pdus_cnt;
  1864. stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
  1865. stats->datain_pdus = conn->datain_pdus_cnt;
  1866. stats->r2t_pdus = conn->r2t_pdus_cnt;
  1867. stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
  1868. stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
  1869. stats->custom_length = 3;
  1870. strcpy(stats->custom[0].desc, "tx_sendpage_failures");
  1871. stats->custom[0].value = tcp_conn->sendpage_failures_cnt;
  1872. strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
  1873. stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt;
  1874. strcpy(stats->custom[2].desc, "eh_abort_cnt");
  1875. stats->custom[2].value = conn->eh_abort_cnt;
  1876. }
  1877. static struct iscsi_cls_session *
  1878. iscsi_tcp_session_create(struct iscsi_transport *iscsit,
  1879. struct scsi_transport_template *scsit,
  1880. uint16_t cmds_max, uint16_t qdepth,
  1881. uint32_t initial_cmdsn, uint32_t *hostno)
  1882. {
  1883. struct iscsi_cls_session *cls_session;
  1884. struct iscsi_session *session;
  1885. uint32_t hn;
  1886. int cmd_i;
  1887. cls_session = iscsi_session_setup(iscsit, scsit, cmds_max, qdepth,
  1888. sizeof(struct iscsi_tcp_cmd_task),
  1889. sizeof(struct iscsi_tcp_mgmt_task),
  1890. initial_cmdsn, &hn);
  1891. if (!cls_session)
  1892. return NULL;
  1893. *hostno = hn;
  1894. session = class_to_transport_session(cls_session);
  1895. for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
  1896. struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
  1897. struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
  1898. ctask->hdr = &tcp_ctask->hdr;
  1899. }
  1900. for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
  1901. struct iscsi_mgmt_task *mtask = session->mgmt_cmds[cmd_i];
  1902. struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
  1903. mtask->hdr = &tcp_mtask->hdr;
  1904. }
  1905. if (iscsi_r2tpool_alloc(class_to_transport_session(cls_session)))
  1906. goto r2tpool_alloc_fail;
  1907. return cls_session;
  1908. r2tpool_alloc_fail:
  1909. iscsi_session_teardown(cls_session);
  1910. return NULL;
  1911. }
  1912. static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session)
  1913. {
  1914. iscsi_r2tpool_free(class_to_transport_session(cls_session));
  1915. iscsi_session_teardown(cls_session);
  1916. }
  1917. static int iscsi_tcp_slave_configure(struct scsi_device *sdev)
  1918. {
  1919. blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
  1920. blk_queue_dma_alignment(sdev->request_queue, 0);
  1921. return 0;
  1922. }
  1923. static struct scsi_host_template iscsi_sht = {
  1924. .module = THIS_MODULE,
  1925. .name = "iSCSI Initiator over TCP/IP",
  1926. .queuecommand = iscsi_queuecommand,
  1927. .change_queue_depth = iscsi_change_queue_depth,
  1928. .can_queue = ISCSI_DEF_XMIT_CMDS_MAX - 1,
  1929. .sg_tablesize = ISCSI_SG_TABLESIZE,
  1930. .max_sectors = 0xFFFF,
  1931. .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
  1932. .eh_abort_handler = iscsi_eh_abort,
  1933. .eh_host_reset_handler = iscsi_eh_host_reset,
  1934. .use_clustering = DISABLE_CLUSTERING,
  1935. .slave_configure = iscsi_tcp_slave_configure,
  1936. .proc_name = "iscsi_tcp",
  1937. .this_id = -1,
  1938. };
  1939. static struct iscsi_transport iscsi_tcp_transport = {
  1940. .owner = THIS_MODULE,
  1941. .name = "tcp",
  1942. .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
  1943. | CAP_DATADGST,
  1944. .param_mask = ISCSI_MAX_RECV_DLENGTH |
  1945. ISCSI_MAX_XMIT_DLENGTH |
  1946. ISCSI_HDRDGST_EN |
  1947. ISCSI_DATADGST_EN |
  1948. ISCSI_INITIAL_R2T_EN |
  1949. ISCSI_MAX_R2T |
  1950. ISCSI_IMM_DATA_EN |
  1951. ISCSI_FIRST_BURST |
  1952. ISCSI_MAX_BURST |
  1953. ISCSI_PDU_INORDER_EN |
  1954. ISCSI_DATASEQ_INORDER_EN |
  1955. ISCSI_ERL |
  1956. ISCSI_CONN_PORT |
  1957. ISCSI_CONN_ADDRESS |
  1958. ISCSI_EXP_STATSN |
  1959. ISCSI_PERSISTENT_PORT |
  1960. ISCSI_PERSISTENT_ADDRESS |
  1961. ISCSI_TARGET_NAME | ISCSI_TPGT |
  1962. ISCSI_USERNAME | ISCSI_PASSWORD |
  1963. ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN,
  1964. .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
  1965. ISCSI_HOST_INITIATOR_NAME |
  1966. ISCSI_HOST_NETDEV_NAME,
  1967. .host_template = &iscsi_sht,
  1968. .conndata_size = sizeof(struct iscsi_conn),
  1969. .max_conn = 1,
  1970. .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN,
  1971. /* session management */
  1972. .create_session = iscsi_tcp_session_create,
  1973. .destroy_session = iscsi_tcp_session_destroy,
  1974. /* connection management */
  1975. .create_conn = iscsi_tcp_conn_create,
  1976. .bind_conn = iscsi_tcp_conn_bind,
  1977. .destroy_conn = iscsi_tcp_conn_destroy,
  1978. .set_param = iscsi_conn_set_param,
  1979. .get_conn_param = iscsi_tcp_conn_get_param,
  1980. .get_session_param = iscsi_session_get_param,
  1981. .start_conn = iscsi_conn_start,
  1982. .stop_conn = iscsi_tcp_conn_stop,
  1983. /* iscsi host params */
  1984. .get_host_param = iscsi_tcp_host_get_param,
  1985. .set_host_param = iscsi_host_set_param,
  1986. /* IO */
  1987. .send_pdu = iscsi_conn_send_pdu,
  1988. .get_stats = iscsi_conn_get_stats,
  1989. .init_cmd_task = iscsi_tcp_cmd_init,
  1990. .init_mgmt_task = iscsi_tcp_mgmt_init,
  1991. .xmit_cmd_task = iscsi_tcp_ctask_xmit,
  1992. .xmit_mgmt_task = iscsi_tcp_mtask_xmit,
  1993. .cleanup_cmd_task = iscsi_tcp_cleanup_ctask,
  1994. /* recovery */
  1995. .session_recovery_timedout = iscsi_session_recovery_timedout,
  1996. };
  1997. static int __init
  1998. iscsi_tcp_init(void)
  1999. {
  2000. if (iscsi_max_lun < 1) {
  2001. printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
  2002. iscsi_max_lun);
  2003. return -EINVAL;
  2004. }
  2005. iscsi_tcp_transport.max_lun = iscsi_max_lun;
  2006. if (!iscsi_register_transport(&iscsi_tcp_transport))
  2007. return -ENODEV;
  2008. return 0;
  2009. }
  2010. static void __exit
  2011. iscsi_tcp_exit(void)
  2012. {
  2013. iscsi_unregister_transport(&iscsi_tcp_transport);
  2014. }
  2015. module_init(iscsi_tcp_init);
  2016. module_exit(iscsi_tcp_exit);