qdio_main.c 40 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700
  1. /*
  2. * linux/drivers/s390/cio/qdio_main.c
  3. *
  4. * Linux for s390 qdio support, buffer handling, qdio API and module support.
  5. *
  6. * Copyright 2000,2008 IBM Corp.
  7. * Author(s): Utz Bacher <utz.bacher@de.ibm.com>
  8. * Jan Glauber <jang@linux.vnet.ibm.com>
  9. * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com>
  10. */
  11. #include <linux/module.h>
  12. #include <linux/init.h>
  13. #include <linux/kernel.h>
  14. #include <linux/timer.h>
  15. #include <linux/delay.h>
  16. #include <linux/gfp.h>
  17. #include <linux/kernel_stat.h>
  18. #include <asm/atomic.h>
  19. #include <asm/debug.h>
  20. #include <asm/qdio.h>
  21. #include "cio.h"
  22. #include "css.h"
  23. #include "device.h"
  24. #include "qdio.h"
  25. #include "qdio_debug.h"
  26. MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\
  27. "Jan Glauber <jang@linux.vnet.ibm.com>");
  28. MODULE_DESCRIPTION("QDIO base support");
  29. MODULE_LICENSE("GPL");
  30. static inline int do_siga_sync(struct subchannel_id schid,
  31. unsigned int out_mask, unsigned int in_mask)
  32. {
  33. register unsigned long __fc asm ("0") = 2;
  34. register struct subchannel_id __schid asm ("1") = schid;
  35. register unsigned long out asm ("2") = out_mask;
  36. register unsigned long in asm ("3") = in_mask;
  37. int cc;
  38. asm volatile(
  39. " siga 0\n"
  40. " ipm %0\n"
  41. " srl %0,28\n"
  42. : "=d" (cc)
  43. : "d" (__fc), "d" (__schid), "d" (out), "d" (in) : "cc");
  44. return cc;
  45. }
  46. static inline int do_siga_input(struct subchannel_id schid, unsigned int mask)
  47. {
  48. register unsigned long __fc asm ("0") = 1;
  49. register struct subchannel_id __schid asm ("1") = schid;
  50. register unsigned long __mask asm ("2") = mask;
  51. int cc;
  52. asm volatile(
  53. " siga 0\n"
  54. " ipm %0\n"
  55. " srl %0,28\n"
  56. : "=d" (cc)
  57. : "d" (__fc), "d" (__schid), "d" (__mask) : "cc", "memory");
  58. return cc;
  59. }
  60. /**
  61. * do_siga_output - perform SIGA-w/wt function
  62. * @schid: subchannel id or in case of QEBSM the subchannel token
  63. * @mask: which output queues to process
  64. * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer
  65. * @fc: function code to perform
  66. *
  67. * Returns cc or QDIO_ERROR_SIGA_ACCESS_EXCEPTION.
  68. * Note: For IQDC unicast queues only the highest priority queue is processed.
  69. */
  70. static inline int do_siga_output(unsigned long schid, unsigned long mask,
  71. unsigned int *bb, unsigned int fc)
  72. {
  73. register unsigned long __fc asm("0") = fc;
  74. register unsigned long __schid asm("1") = schid;
  75. register unsigned long __mask asm("2") = mask;
  76. int cc = QDIO_ERROR_SIGA_ACCESS_EXCEPTION;
  77. asm volatile(
  78. " siga 0\n"
  79. "0: ipm %0\n"
  80. " srl %0,28\n"
  81. "1:\n"
  82. EX_TABLE(0b, 1b)
  83. : "+d" (cc), "+d" (__fc), "+d" (__schid), "+d" (__mask)
  84. : : "cc", "memory");
  85. *bb = ((unsigned int) __fc) >> 31;
  86. return cc;
  87. }
  88. static inline int qdio_check_ccq(struct qdio_q *q, unsigned int ccq)
  89. {
  90. /* all done or next buffer state different */
  91. if (ccq == 0 || ccq == 32)
  92. return 0;
  93. /* not all buffers processed */
  94. if (ccq == 96 || ccq == 97)
  95. return 1;
  96. /* notify devices immediately */
  97. DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
  98. return -EIO;
  99. }
  100. /**
  101. * qdio_do_eqbs - extract buffer states for QEBSM
  102. * @q: queue to manipulate
  103. * @state: state of the extracted buffers
  104. * @start: buffer number to start at
  105. * @count: count of buffers to examine
  106. * @auto_ack: automatically acknowledge buffers
  107. *
  108. * Returns the number of successfully extracted equal buffer states.
  109. * Stops processing if a state is different from the last buffers state.
  110. */
  111. static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,
  112. int start, int count, int auto_ack)
  113. {
  114. unsigned int ccq = 0;
  115. int tmp_count = count, tmp_start = start;
  116. int nr = q->nr;
  117. int rc;
  118. BUG_ON(!q->irq_ptr->sch_token);
  119. qperf_inc(q, eqbs);
  120. if (!q->is_input_q)
  121. nr += q->irq_ptr->nr_input_qs;
  122. again:
  123. ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,
  124. auto_ack);
  125. rc = qdio_check_ccq(q, ccq);
  126. /* At least one buffer was processed, return and extract the remaining
  127. * buffers later.
  128. */
  129. if ((ccq == 96) && (count != tmp_count)) {
  130. qperf_inc(q, eqbs_partial);
  131. return (count - tmp_count);
  132. }
  133. if (rc == 1) {
  134. DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);
  135. goto again;
  136. }
  137. if (rc < 0) {
  138. DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
  139. DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
  140. q->handler(q->irq_ptr->cdev,
  141. QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
  142. 0, -1, -1, q->irq_ptr->int_parm);
  143. return 0;
  144. }
  145. return count - tmp_count;
  146. }
  147. /**
  148. * qdio_do_sqbs - set buffer states for QEBSM
  149. * @q: queue to manipulate
  150. * @state: new state of the buffers
  151. * @start: first buffer number to change
  152. * @count: how many buffers to change
  153. *
  154. * Returns the number of successfully changed buffers.
  155. * Does retrying until the specified count of buffer states is set or an
  156. * error occurs.
  157. */
  158. static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,
  159. int count)
  160. {
  161. unsigned int ccq = 0;
  162. int tmp_count = count, tmp_start = start;
  163. int nr = q->nr;
  164. int rc;
  165. if (!count)
  166. return 0;
  167. BUG_ON(!q->irq_ptr->sch_token);
  168. qperf_inc(q, sqbs);
  169. if (!q->is_input_q)
  170. nr += q->irq_ptr->nr_input_qs;
  171. again:
  172. ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);
  173. rc = qdio_check_ccq(q, ccq);
  174. if (rc == 1) {
  175. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);
  176. qperf_inc(q, sqbs_partial);
  177. goto again;
  178. }
  179. if (rc < 0) {
  180. DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
  181. DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
  182. q->handler(q->irq_ptr->cdev,
  183. QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
  184. 0, -1, -1, q->irq_ptr->int_parm);
  185. return 0;
  186. }
  187. WARN_ON(tmp_count);
  188. return count - tmp_count;
  189. }
  190. /* returns number of examined buffers and their common state in *state */
  191. static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,
  192. unsigned char *state, unsigned int count,
  193. int auto_ack)
  194. {
  195. unsigned char __state = 0;
  196. int i;
  197. BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK);
  198. BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q);
  199. if (is_qebsm(q))
  200. return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
  201. for (i = 0; i < count; i++) {
  202. if (!__state)
  203. __state = q->slsb.val[bufnr];
  204. else if (q->slsb.val[bufnr] != __state)
  205. break;
  206. bufnr = next_buf(bufnr);
  207. }
  208. *state = __state;
  209. return i;
  210. }
  211. static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,
  212. unsigned char *state, int auto_ack)
  213. {
  214. return get_buf_states(q, bufnr, state, 1, auto_ack);
  215. }
  216. /* wrap-around safe setting of slsb states, returns number of changed buffers */
  217. static inline int set_buf_states(struct qdio_q *q, int bufnr,
  218. unsigned char state, int count)
  219. {
  220. int i;
  221. BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK);
  222. BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q);
  223. if (is_qebsm(q))
  224. return qdio_do_sqbs(q, state, bufnr, count);
  225. for (i = 0; i < count; i++) {
  226. xchg(&q->slsb.val[bufnr], state);
  227. bufnr = next_buf(bufnr);
  228. }
  229. return count;
  230. }
  231. static inline int set_buf_state(struct qdio_q *q, int bufnr,
  232. unsigned char state)
  233. {
  234. return set_buf_states(q, bufnr, state, 1);
  235. }
  236. /* set slsb states to initial state */
  237. void qdio_init_buf_states(struct qdio_irq *irq_ptr)
  238. {
  239. struct qdio_q *q;
  240. int i;
  241. for_each_input_queue(irq_ptr, q, i)
  242. set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,
  243. QDIO_MAX_BUFFERS_PER_Q);
  244. for_each_output_queue(irq_ptr, q, i)
  245. set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,
  246. QDIO_MAX_BUFFERS_PER_Q);
  247. }
  248. static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,
  249. unsigned int input)
  250. {
  251. int cc;
  252. if (!need_siga_sync(q))
  253. return 0;
  254. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);
  255. qperf_inc(q, siga_sync);
  256. cc = do_siga_sync(q->irq_ptr->schid, output, input);
  257. if (cc)
  258. DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);
  259. return cc;
  260. }
  261. static inline int qdio_siga_sync_q(struct qdio_q *q)
  262. {
  263. if (q->is_input_q)
  264. return qdio_siga_sync(q, 0, q->mask);
  265. else
  266. return qdio_siga_sync(q, q->mask, 0);
  267. }
  268. static inline int qdio_siga_sync_out(struct qdio_q *q)
  269. {
  270. return qdio_siga_sync(q, ~0U, 0);
  271. }
  272. static inline int qdio_siga_sync_all(struct qdio_q *q)
  273. {
  274. return qdio_siga_sync(q, ~0U, ~0U);
  275. }
  276. static int qdio_siga_output(struct qdio_q *q, unsigned int *busy_bit)
  277. {
  278. unsigned long schid;
  279. unsigned int fc = 0;
  280. u64 start_time = 0;
  281. int cc;
  282. if (q->u.out.use_enh_siga)
  283. fc = 3;
  284. if (is_qebsm(q)) {
  285. schid = q->irq_ptr->sch_token;
  286. fc |= 0x80;
  287. }
  288. else
  289. schid = *((u32 *)&q->irq_ptr->schid);
  290. again:
  291. cc = do_siga_output(schid, q->mask, busy_bit, fc);
  292. /* hipersocket busy condition */
  293. if (*busy_bit) {
  294. WARN_ON(queue_type(q) != QDIO_IQDIO_QFMT || cc != 2);
  295. if (!start_time) {
  296. start_time = get_clock();
  297. goto again;
  298. }
  299. if ((get_clock() - start_time) < QDIO_BUSY_BIT_PATIENCE)
  300. goto again;
  301. }
  302. return cc;
  303. }
  304. static inline int qdio_siga_input(struct qdio_q *q)
  305. {
  306. int cc;
  307. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);
  308. qperf_inc(q, siga_read);
  309. cc = do_siga_input(q->irq_ptr->schid, q->mask);
  310. if (cc)
  311. DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);
  312. return cc;
  313. }
  314. static inline void qdio_sync_after_thinint(struct qdio_q *q)
  315. {
  316. if (pci_out_supported(q)) {
  317. if (need_siga_sync_thinint(q))
  318. qdio_siga_sync_all(q);
  319. else if (need_siga_sync_out_thinint(q))
  320. qdio_siga_sync_out(q);
  321. } else
  322. qdio_siga_sync_q(q);
  323. }
  324. int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,
  325. unsigned char *state)
  326. {
  327. qdio_siga_sync_q(q);
  328. return get_buf_states(q, bufnr, state, 1, 0);
  329. }
  330. static inline void qdio_stop_polling(struct qdio_q *q)
  331. {
  332. if (!q->u.in.polling)
  333. return;
  334. q->u.in.polling = 0;
  335. qperf_inc(q, stop_polling);
  336. /* show the card that we are not polling anymore */
  337. if (is_qebsm(q)) {
  338. set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
  339. q->u.in.ack_count);
  340. q->u.in.ack_count = 0;
  341. } else
  342. set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
  343. }
  344. static inline void account_sbals(struct qdio_q *q, int count)
  345. {
  346. int pos = 0;
  347. q->q_stats.nr_sbal_total += count;
  348. if (count == QDIO_MAX_BUFFERS_MASK) {
  349. q->q_stats.nr_sbals[7]++;
  350. return;
  351. }
  352. while (count >>= 1)
  353. pos++;
  354. q->q_stats.nr_sbals[pos]++;
  355. }
  356. static void announce_buffer_error(struct qdio_q *q, int count)
  357. {
  358. q->qdio_error |= QDIO_ERROR_SLSB_STATE;
  359. /* special handling for no target buffer empty */
  360. if ((!q->is_input_q &&
  361. (q->sbal[q->first_to_check]->element[15].flags & 0xff) == 0x10)) {
  362. qperf_inc(q, target_full);
  363. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x",
  364. q->first_to_check);
  365. return;
  366. }
  367. DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
  368. DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
  369. DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count);
  370. DBF_ERROR("F14:%2x F15:%2x",
  371. q->sbal[q->first_to_check]->element[14].flags & 0xff,
  372. q->sbal[q->first_to_check]->element[15].flags & 0xff);
  373. }
  374. static inline void inbound_primed(struct qdio_q *q, int count)
  375. {
  376. int new;
  377. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim: %02x", count);
  378. /* for QEBSM the ACK was already set by EQBS */
  379. if (is_qebsm(q)) {
  380. if (!q->u.in.polling) {
  381. q->u.in.polling = 1;
  382. q->u.in.ack_count = count;
  383. q->u.in.ack_start = q->first_to_check;
  384. return;
  385. }
  386. /* delete the previous ACK's */
  387. set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
  388. q->u.in.ack_count);
  389. q->u.in.ack_count = count;
  390. q->u.in.ack_start = q->first_to_check;
  391. return;
  392. }
  393. /*
  394. * ACK the newest buffer. The ACK will be removed in qdio_stop_polling
  395. * or by the next inbound run.
  396. */
  397. new = add_buf(q->first_to_check, count - 1);
  398. if (q->u.in.polling) {
  399. /* reset the previous ACK but first set the new one */
  400. set_buf_state(q, new, SLSB_P_INPUT_ACK);
  401. set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
  402. } else {
  403. q->u.in.polling = 1;
  404. set_buf_state(q, new, SLSB_P_INPUT_ACK);
  405. }
  406. q->u.in.ack_start = new;
  407. count--;
  408. if (!count)
  409. return;
  410. /* need to change ALL buffers to get more interrupts */
  411. set_buf_states(q, q->first_to_check, SLSB_P_INPUT_NOT_INIT, count);
  412. }
  413. static int get_inbound_buffer_frontier(struct qdio_q *q)
  414. {
  415. int count, stop;
  416. unsigned char state;
  417. /*
  418. * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
  419. * would return 0.
  420. */
  421. count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
  422. stop = add_buf(q->first_to_check, count);
  423. if (q->first_to_check == stop)
  424. goto out;
  425. /*
  426. * No siga sync here, as a PCI or we after a thin interrupt
  427. * already sync'ed the queues.
  428. */
  429. count = get_buf_states(q, q->first_to_check, &state, count, 1);
  430. if (!count)
  431. goto out;
  432. switch (state) {
  433. case SLSB_P_INPUT_PRIMED:
  434. inbound_primed(q, count);
  435. q->first_to_check = add_buf(q->first_to_check, count);
  436. if (atomic_sub(count, &q->nr_buf_used) == 0)
  437. qperf_inc(q, inbound_queue_full);
  438. if (q->irq_ptr->perf_stat_enabled)
  439. account_sbals(q, count);
  440. break;
  441. case SLSB_P_INPUT_ERROR:
  442. announce_buffer_error(q, count);
  443. /* process the buffer, the upper layer will take care of it */
  444. q->first_to_check = add_buf(q->first_to_check, count);
  445. atomic_sub(count, &q->nr_buf_used);
  446. if (q->irq_ptr->perf_stat_enabled)
  447. account_sbals_error(q, count);
  448. break;
  449. case SLSB_CU_INPUT_EMPTY:
  450. case SLSB_P_INPUT_NOT_INIT:
  451. case SLSB_P_INPUT_ACK:
  452. if (q->irq_ptr->perf_stat_enabled)
  453. q->q_stats.nr_sbal_nop++;
  454. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop");
  455. break;
  456. default:
  457. BUG();
  458. }
  459. out:
  460. return q->first_to_check;
  461. }
  462. static int qdio_inbound_q_moved(struct qdio_q *q)
  463. {
  464. int bufnr;
  465. bufnr = get_inbound_buffer_frontier(q);
  466. if ((bufnr != q->last_move) || q->qdio_error) {
  467. q->last_move = bufnr;
  468. if (!is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR)
  469. q->u.in.timestamp = get_clock();
  470. return 1;
  471. } else
  472. return 0;
  473. }
  474. static inline int qdio_inbound_q_done(struct qdio_q *q)
  475. {
  476. unsigned char state = 0;
  477. if (!atomic_read(&q->nr_buf_used))
  478. return 1;
  479. qdio_siga_sync_q(q);
  480. get_buf_state(q, q->first_to_check, &state, 0);
  481. if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR)
  482. /* more work coming */
  483. return 0;
  484. if (is_thinint_irq(q->irq_ptr))
  485. return 1;
  486. /* don't poll under z/VM */
  487. if (MACHINE_IS_VM)
  488. return 1;
  489. /*
  490. * At this point we know, that inbound first_to_check
  491. * has (probably) not moved (see qdio_inbound_processing).
  492. */
  493. if (get_clock() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) {
  494. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x",
  495. q->first_to_check);
  496. return 1;
  497. } else
  498. return 0;
  499. }
  500. static void qdio_kick_handler(struct qdio_q *q)
  501. {
  502. int start = q->first_to_kick;
  503. int end = q->first_to_check;
  504. int count;
  505. if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
  506. return;
  507. count = sub_buf(end, start);
  508. if (q->is_input_q) {
  509. qperf_inc(q, inbound_handler);
  510. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "kih s:%02x c:%02x", start, count);
  511. } else {
  512. qperf_inc(q, outbound_handler);
  513. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "koh: s:%02x c:%02x",
  514. start, count);
  515. }
  516. q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count,
  517. q->irq_ptr->int_parm);
  518. /* for the next time */
  519. q->first_to_kick = end;
  520. q->qdio_error = 0;
  521. }
  522. static void __qdio_inbound_processing(struct qdio_q *q)
  523. {
  524. qperf_inc(q, tasklet_inbound);
  525. if (!qdio_inbound_q_moved(q))
  526. return;
  527. qdio_kick_handler(q);
  528. if (!qdio_inbound_q_done(q)) {
  529. /* means poll time is not yet over */
  530. qperf_inc(q, tasklet_inbound_resched);
  531. if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
  532. tasklet_schedule(&q->tasklet);
  533. return;
  534. }
  535. }
  536. qdio_stop_polling(q);
  537. /*
  538. * We need to check again to not lose initiative after
  539. * resetting the ACK state.
  540. */
  541. if (!qdio_inbound_q_done(q)) {
  542. qperf_inc(q, tasklet_inbound_resched2);
  543. if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
  544. tasklet_schedule(&q->tasklet);
  545. }
  546. }
  547. void qdio_inbound_processing(unsigned long data)
  548. {
  549. struct qdio_q *q = (struct qdio_q *)data;
  550. __qdio_inbound_processing(q);
  551. }
  552. static int get_outbound_buffer_frontier(struct qdio_q *q)
  553. {
  554. int count, stop;
  555. unsigned char state;
  556. if (((queue_type(q) != QDIO_IQDIO_QFMT) && !pci_out_supported(q)) ||
  557. (queue_type(q) == QDIO_IQDIO_QFMT && multicast_outbound(q)))
  558. qdio_siga_sync_q(q);
  559. /*
  560. * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
  561. * would return 0.
  562. */
  563. count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
  564. stop = add_buf(q->first_to_check, count);
  565. if (q->first_to_check == stop)
  566. return q->first_to_check;
  567. count = get_buf_states(q, q->first_to_check, &state, count, 0);
  568. if (!count)
  569. return q->first_to_check;
  570. switch (state) {
  571. case SLSB_P_OUTPUT_EMPTY:
  572. /* the adapter got it */
  573. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out empty:%1d %02x", q->nr, count);
  574. atomic_sub(count, &q->nr_buf_used);
  575. q->first_to_check = add_buf(q->first_to_check, count);
  576. if (q->irq_ptr->perf_stat_enabled)
  577. account_sbals(q, count);
  578. break;
  579. case SLSB_P_OUTPUT_ERROR:
  580. announce_buffer_error(q, count);
  581. /* process the buffer, the upper layer will take care of it */
  582. q->first_to_check = add_buf(q->first_to_check, count);
  583. atomic_sub(count, &q->nr_buf_used);
  584. if (q->irq_ptr->perf_stat_enabled)
  585. account_sbals_error(q, count);
  586. break;
  587. case SLSB_CU_OUTPUT_PRIMED:
  588. /* the adapter has not fetched the output yet */
  589. if (q->irq_ptr->perf_stat_enabled)
  590. q->q_stats.nr_sbal_nop++;
  591. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", q->nr);
  592. break;
  593. case SLSB_P_OUTPUT_NOT_INIT:
  594. case SLSB_P_OUTPUT_HALTED:
  595. break;
  596. default:
  597. BUG();
  598. }
  599. return q->first_to_check;
  600. }
  601. /* all buffers processed? */
  602. static inline int qdio_outbound_q_done(struct qdio_q *q)
  603. {
  604. return atomic_read(&q->nr_buf_used) == 0;
  605. }
  606. static inline int qdio_outbound_q_moved(struct qdio_q *q)
  607. {
  608. int bufnr;
  609. bufnr = get_outbound_buffer_frontier(q);
  610. if ((bufnr != q->last_move) || q->qdio_error) {
  611. q->last_move = bufnr;
  612. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr);
  613. return 1;
  614. } else
  615. return 0;
  616. }
  617. static int qdio_kick_outbound_q(struct qdio_q *q)
  618. {
  619. unsigned int busy_bit;
  620. int cc;
  621. if (!need_siga_out(q))
  622. return 0;
  623. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);
  624. qperf_inc(q, siga_write);
  625. cc = qdio_siga_output(q, &busy_bit);
  626. switch (cc) {
  627. case 0:
  628. break;
  629. case 2:
  630. if (busy_bit) {
  631. DBF_ERROR("%4x cc2 REP:%1d", SCH_NO(q), q->nr);
  632. cc |= QDIO_ERROR_SIGA_BUSY;
  633. } else
  634. DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);
  635. break;
  636. case 1:
  637. case 3:
  638. DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);
  639. break;
  640. }
  641. return cc;
  642. }
  643. static void __qdio_outbound_processing(struct qdio_q *q)
  644. {
  645. qperf_inc(q, tasklet_outbound);
  646. BUG_ON(atomic_read(&q->nr_buf_used) < 0);
  647. if (qdio_outbound_q_moved(q))
  648. qdio_kick_handler(q);
  649. if (queue_type(q) == QDIO_ZFCP_QFMT)
  650. if (!pci_out_supported(q) && !qdio_outbound_q_done(q))
  651. goto sched;
  652. /* bail out for HiperSockets unicast queues */
  653. if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q))
  654. return;
  655. if ((queue_type(q) == QDIO_IQDIO_QFMT) &&
  656. (atomic_read(&q->nr_buf_used)) > QDIO_IQDIO_POLL_LVL)
  657. goto sched;
  658. if (q->u.out.pci_out_enabled)
  659. return;
  660. /*
  661. * Now we know that queue type is either qeth without pci enabled
  662. * or HiperSockets multicast. Make sure buffer switch from PRIMED to
  663. * EMPTY is noticed and outbound_handler is called after some time.
  664. */
  665. if (qdio_outbound_q_done(q))
  666. del_timer(&q->u.out.timer);
  667. else
  668. if (!timer_pending(&q->u.out.timer))
  669. mod_timer(&q->u.out.timer, jiffies + 10 * HZ);
  670. return;
  671. sched:
  672. if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
  673. return;
  674. tasklet_schedule(&q->tasklet);
  675. }
  676. /* outbound tasklet */
  677. void qdio_outbound_processing(unsigned long data)
  678. {
  679. struct qdio_q *q = (struct qdio_q *)data;
  680. __qdio_outbound_processing(q);
  681. }
  682. void qdio_outbound_timer(unsigned long data)
  683. {
  684. struct qdio_q *q = (struct qdio_q *)data;
  685. if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
  686. return;
  687. tasklet_schedule(&q->tasklet);
  688. }
  689. static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)
  690. {
  691. struct qdio_q *out;
  692. int i;
  693. if (!pci_out_supported(q))
  694. return;
  695. for_each_output_queue(q->irq_ptr, out, i)
  696. if (!qdio_outbound_q_done(out))
  697. tasklet_schedule(&out->tasklet);
  698. }
  699. static void __tiqdio_inbound_processing(struct qdio_q *q)
  700. {
  701. qperf_inc(q, tasklet_inbound);
  702. qdio_sync_after_thinint(q);
  703. /*
  704. * The interrupt could be caused by a PCI request. Check the
  705. * PCI capable outbound queues.
  706. */
  707. qdio_check_outbound_after_thinint(q);
  708. if (!qdio_inbound_q_moved(q))
  709. return;
  710. qdio_kick_handler(q);
  711. if (!qdio_inbound_q_done(q)) {
  712. qperf_inc(q, tasklet_inbound_resched);
  713. if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
  714. tasklet_schedule(&q->tasklet);
  715. return;
  716. }
  717. }
  718. qdio_stop_polling(q);
  719. /*
  720. * We need to check again to not lose initiative after
  721. * resetting the ACK state.
  722. */
  723. if (!qdio_inbound_q_done(q)) {
  724. qperf_inc(q, tasklet_inbound_resched2);
  725. if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
  726. tasklet_schedule(&q->tasklet);
  727. }
  728. }
  729. void tiqdio_inbound_processing(unsigned long data)
  730. {
  731. struct qdio_q *q = (struct qdio_q *)data;
  732. __tiqdio_inbound_processing(q);
  733. }
  734. static inline void qdio_set_state(struct qdio_irq *irq_ptr,
  735. enum qdio_irq_states state)
  736. {
  737. DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state);
  738. irq_ptr->state = state;
  739. mb();
  740. }
  741. static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb)
  742. {
  743. if (irb->esw.esw0.erw.cons) {
  744. DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no);
  745. DBF_ERROR_HEX(irb, 64);
  746. DBF_ERROR_HEX(irb->ecw, 64);
  747. }
  748. }
  749. /* PCI interrupt handler */
  750. static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
  751. {
  752. int i;
  753. struct qdio_q *q;
  754. if (unlikely(irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
  755. return;
  756. for_each_input_queue(irq_ptr, q, i) {
  757. if (q->u.in.queue_start_poll) {
  758. /* skip if polling is enabled or already in work */
  759. if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
  760. &q->u.in.queue_irq_state)) {
  761. qperf_inc(q, int_discarded);
  762. continue;
  763. }
  764. q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr,
  765. q->irq_ptr->int_parm);
  766. } else
  767. tasklet_schedule(&q->tasklet);
  768. }
  769. if (!(irq_ptr->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED))
  770. return;
  771. for_each_output_queue(irq_ptr, q, i) {
  772. if (qdio_outbound_q_done(q))
  773. continue;
  774. if (!siga_syncs_out_pci(q))
  775. qdio_siga_sync_q(q);
  776. tasklet_schedule(&q->tasklet);
  777. }
  778. }
  779. static void qdio_handle_activate_check(struct ccw_device *cdev,
  780. unsigned long intparm, int cstat, int dstat)
  781. {
  782. struct qdio_irq *irq_ptr = cdev->private->qdio_data;
  783. struct qdio_q *q;
  784. DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
  785. DBF_ERROR("intp :%lx", intparm);
  786. DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
  787. if (irq_ptr->nr_input_qs) {
  788. q = irq_ptr->input_qs[0];
  789. } else if (irq_ptr->nr_output_qs) {
  790. q = irq_ptr->output_qs[0];
  791. } else {
  792. dump_stack();
  793. goto no_handler;
  794. }
  795. q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
  796. 0, -1, -1, irq_ptr->int_parm);
  797. no_handler:
  798. qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
  799. }
  800. static void qdio_establish_handle_irq(struct ccw_device *cdev, int cstat,
  801. int dstat)
  802. {
  803. struct qdio_irq *irq_ptr = cdev->private->qdio_data;
  804. DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");
  805. if (cstat)
  806. goto error;
  807. if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END))
  808. goto error;
  809. if (!(dstat & DEV_STAT_DEV_END))
  810. goto error;
  811. qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED);
  812. return;
  813. error:
  814. DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no);
  815. DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
  816. qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
  817. }
  818. /* qdio interrupt handler */
  819. void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
  820. struct irb *irb)
  821. {
  822. struct qdio_irq *irq_ptr = cdev->private->qdio_data;
  823. int cstat, dstat;
  824. if (!intparm || !irq_ptr) {
  825. DBF_ERROR("qint:%4x", cdev->private->schid.sch_no);
  826. return;
  827. }
  828. kstat_cpu(smp_processor_id()).irqs[IOINT_QDI]++;
  829. if (irq_ptr->perf_stat_enabled)
  830. irq_ptr->perf_stat.qdio_int++;
  831. if (IS_ERR(irb)) {
  832. switch (PTR_ERR(irb)) {
  833. case -EIO:
  834. DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no);
  835. qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
  836. wake_up(&cdev->private->wait_q);
  837. return;
  838. default:
  839. WARN_ON(1);
  840. return;
  841. }
  842. }
  843. qdio_irq_check_sense(irq_ptr, irb);
  844. cstat = irb->scsw.cmd.cstat;
  845. dstat = irb->scsw.cmd.dstat;
  846. switch (irq_ptr->state) {
  847. case QDIO_IRQ_STATE_INACTIVE:
  848. qdio_establish_handle_irq(cdev, cstat, dstat);
  849. break;
  850. case QDIO_IRQ_STATE_CLEANUP:
  851. qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
  852. break;
  853. case QDIO_IRQ_STATE_ESTABLISHED:
  854. case QDIO_IRQ_STATE_ACTIVE:
  855. if (cstat & SCHN_STAT_PCI) {
  856. qdio_int_handler_pci(irq_ptr);
  857. return;
  858. }
  859. if (cstat || dstat)
  860. qdio_handle_activate_check(cdev, intparm, cstat,
  861. dstat);
  862. break;
  863. case QDIO_IRQ_STATE_STOPPED:
  864. break;
  865. default:
  866. WARN_ON(1);
  867. }
  868. wake_up(&cdev->private->wait_q);
  869. }
  870. /**
  871. * qdio_get_ssqd_desc - get qdio subchannel description
  872. * @cdev: ccw device to get description for
  873. * @data: where to store the ssqd
  874. *
  875. * Returns 0 or an error code. The results of the chsc are stored in the
  876. * specified structure.
  877. */
  878. int qdio_get_ssqd_desc(struct ccw_device *cdev,
  879. struct qdio_ssqd_desc *data)
  880. {
  881. if (!cdev || !cdev->private)
  882. return -EINVAL;
  883. DBF_EVENT("get ssqd:%4x", cdev->private->schid.sch_no);
  884. return qdio_setup_get_ssqd(NULL, &cdev->private->schid, data);
  885. }
  886. EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc);
  887. static void qdio_shutdown_queues(struct ccw_device *cdev)
  888. {
  889. struct qdio_irq *irq_ptr = cdev->private->qdio_data;
  890. struct qdio_q *q;
  891. int i;
  892. for_each_input_queue(irq_ptr, q, i)
  893. tasklet_kill(&q->tasklet);
  894. for_each_output_queue(irq_ptr, q, i) {
  895. del_timer(&q->u.out.timer);
  896. tasklet_kill(&q->tasklet);
  897. }
  898. }
  899. /**
  900. * qdio_shutdown - shut down a qdio subchannel
  901. * @cdev: associated ccw device
  902. * @how: use halt or clear to shutdown
  903. */
  904. int qdio_shutdown(struct ccw_device *cdev, int how)
  905. {
  906. struct qdio_irq *irq_ptr = cdev->private->qdio_data;
  907. int rc;
  908. unsigned long flags;
  909. if (!irq_ptr)
  910. return -ENODEV;
  911. BUG_ON(irqs_disabled());
  912. DBF_EVENT("qshutdown:%4x", cdev->private->schid.sch_no);
  913. mutex_lock(&irq_ptr->setup_mutex);
  914. /*
  915. * Subchannel was already shot down. We cannot prevent being called
  916. * twice since cio may trigger a shutdown asynchronously.
  917. */
  918. if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
  919. mutex_unlock(&irq_ptr->setup_mutex);
  920. return 0;
  921. }
  922. /*
  923. * Indicate that the device is going down. Scheduling the queue
  924. * tasklets is forbidden from here on.
  925. */
  926. qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
  927. tiqdio_remove_input_queues(irq_ptr);
  928. qdio_shutdown_queues(cdev);
  929. qdio_shutdown_debug_entries(irq_ptr, cdev);
  930. /* cleanup subchannel */
  931. spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
  932. if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
  933. rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
  934. else
  935. /* default behaviour is halt */
  936. rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
  937. if (rc) {
  938. DBF_ERROR("%4x SHUTD ERR", irq_ptr->schid.sch_no);
  939. DBF_ERROR("rc:%4d", rc);
  940. goto no_cleanup;
  941. }
  942. qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP);
  943. spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
  944. wait_event_interruptible_timeout(cdev->private->wait_q,
  945. irq_ptr->state == QDIO_IRQ_STATE_INACTIVE ||
  946. irq_ptr->state == QDIO_IRQ_STATE_ERR,
  947. 10 * HZ);
  948. spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
  949. no_cleanup:
  950. qdio_shutdown_thinint(irq_ptr);
  951. /* restore interrupt handler */
  952. if ((void *)cdev->handler == (void *)qdio_int_handler)
  953. cdev->handler = irq_ptr->orig_handler;
  954. spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
  955. qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
  956. mutex_unlock(&irq_ptr->setup_mutex);
  957. if (rc)
  958. return rc;
  959. return 0;
  960. }
  961. EXPORT_SYMBOL_GPL(qdio_shutdown);
  962. /**
  963. * qdio_free - free data structures for a qdio subchannel
  964. * @cdev: associated ccw device
  965. */
  966. int qdio_free(struct ccw_device *cdev)
  967. {
  968. struct qdio_irq *irq_ptr = cdev->private->qdio_data;
  969. if (!irq_ptr)
  970. return -ENODEV;
  971. DBF_EVENT("qfree:%4x", cdev->private->schid.sch_no);
  972. mutex_lock(&irq_ptr->setup_mutex);
  973. if (irq_ptr->debug_area != NULL) {
  974. debug_unregister(irq_ptr->debug_area);
  975. irq_ptr->debug_area = NULL;
  976. }
  977. cdev->private->qdio_data = NULL;
  978. mutex_unlock(&irq_ptr->setup_mutex);
  979. qdio_release_memory(irq_ptr);
  980. return 0;
  981. }
  982. EXPORT_SYMBOL_GPL(qdio_free);
  983. /**
  984. * qdio_allocate - allocate qdio queues and associated data
  985. * @init_data: initialization data
  986. */
  987. int qdio_allocate(struct qdio_initialize *init_data)
  988. {
  989. struct qdio_irq *irq_ptr;
  990. DBF_EVENT("qallocate:%4x", init_data->cdev->private->schid.sch_no);
  991. if ((init_data->no_input_qs && !init_data->input_handler) ||
  992. (init_data->no_output_qs && !init_data->output_handler))
  993. return -EINVAL;
  994. if ((init_data->no_input_qs > QDIO_MAX_QUEUES_PER_IRQ) ||
  995. (init_data->no_output_qs > QDIO_MAX_QUEUES_PER_IRQ))
  996. return -EINVAL;
  997. if ((!init_data->input_sbal_addr_array) ||
  998. (!init_data->output_sbal_addr_array))
  999. return -EINVAL;
  1000. /* irq_ptr must be in GFP_DMA since it contains ccw1.cda */
  1001. irq_ptr = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
  1002. if (!irq_ptr)
  1003. goto out_err;
  1004. mutex_init(&irq_ptr->setup_mutex);
  1005. qdio_allocate_dbf(init_data, irq_ptr);
  1006. /*
  1007. * Allocate a page for the chsc calls in qdio_establish.
  1008. * Must be pre-allocated since a zfcp recovery will call
  1009. * qdio_establish. In case of low memory and swap on a zfcp disk
  1010. * we may not be able to allocate memory otherwise.
  1011. */
  1012. irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
  1013. if (!irq_ptr->chsc_page)
  1014. goto out_rel;
  1015. /* qdr is used in ccw1.cda which is u32 */
  1016. irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
  1017. if (!irq_ptr->qdr)
  1018. goto out_rel;
  1019. WARN_ON((unsigned long)irq_ptr->qdr & 0xfff);
  1020. if (qdio_allocate_qs(irq_ptr, init_data->no_input_qs,
  1021. init_data->no_output_qs))
  1022. goto out_rel;
  1023. init_data->cdev->private->qdio_data = irq_ptr;
  1024. qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
  1025. return 0;
  1026. out_rel:
  1027. qdio_release_memory(irq_ptr);
  1028. out_err:
  1029. return -ENOMEM;
  1030. }
  1031. EXPORT_SYMBOL_GPL(qdio_allocate);
  1032. /**
  1033. * qdio_establish - establish queues on a qdio subchannel
  1034. * @init_data: initialization data
  1035. */
  1036. int qdio_establish(struct qdio_initialize *init_data)
  1037. {
  1038. struct qdio_irq *irq_ptr;
  1039. struct ccw_device *cdev = init_data->cdev;
  1040. unsigned long saveflags;
  1041. int rc;
  1042. DBF_EVENT("qestablish:%4x", cdev->private->schid.sch_no);
  1043. irq_ptr = cdev->private->qdio_data;
  1044. if (!irq_ptr)
  1045. return -ENODEV;
  1046. if (cdev->private->state != DEV_STATE_ONLINE)
  1047. return -EINVAL;
  1048. mutex_lock(&irq_ptr->setup_mutex);
  1049. qdio_setup_irq(init_data);
  1050. rc = qdio_establish_thinint(irq_ptr);
  1051. if (rc) {
  1052. mutex_unlock(&irq_ptr->setup_mutex);
  1053. qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
  1054. return rc;
  1055. }
  1056. /* establish q */
  1057. irq_ptr->ccw.cmd_code = irq_ptr->equeue.cmd;
  1058. irq_ptr->ccw.flags = CCW_FLAG_SLI;
  1059. irq_ptr->ccw.count = irq_ptr->equeue.count;
  1060. irq_ptr->ccw.cda = (u32)((addr_t)irq_ptr->qdr);
  1061. spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags);
  1062. ccw_device_set_options_mask(cdev, 0);
  1063. rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0);
  1064. if (rc) {
  1065. DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
  1066. DBF_ERROR("rc:%4x", rc);
  1067. }
  1068. spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags);
  1069. if (rc) {
  1070. mutex_unlock(&irq_ptr->setup_mutex);
  1071. qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
  1072. return rc;
  1073. }
  1074. wait_event_interruptible_timeout(cdev->private->wait_q,
  1075. irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED ||
  1076. irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
  1077. if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) {
  1078. mutex_unlock(&irq_ptr->setup_mutex);
  1079. qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
  1080. return -EIO;
  1081. }
  1082. qdio_setup_ssqd_info(irq_ptr);
  1083. DBF_EVENT("qDmmwc:%2x", irq_ptr->ssqd_desc.mmwc);
  1084. DBF_EVENT("qib ac:%4x", irq_ptr->qib.ac);
  1085. /* qebsm is now setup if available, initialize buffer states */
  1086. qdio_init_buf_states(irq_ptr);
  1087. mutex_unlock(&irq_ptr->setup_mutex);
  1088. qdio_print_subchannel_info(irq_ptr, cdev);
  1089. qdio_setup_debug_entries(irq_ptr, cdev);
  1090. return 0;
  1091. }
  1092. EXPORT_SYMBOL_GPL(qdio_establish);
  1093. /**
  1094. * qdio_activate - activate queues on a qdio subchannel
  1095. * @cdev: associated cdev
  1096. */
  1097. int qdio_activate(struct ccw_device *cdev)
  1098. {
  1099. struct qdio_irq *irq_ptr;
  1100. int rc;
  1101. unsigned long saveflags;
  1102. DBF_EVENT("qactivate:%4x", cdev->private->schid.sch_no);
  1103. irq_ptr = cdev->private->qdio_data;
  1104. if (!irq_ptr)
  1105. return -ENODEV;
  1106. if (cdev->private->state != DEV_STATE_ONLINE)
  1107. return -EINVAL;
  1108. mutex_lock(&irq_ptr->setup_mutex);
  1109. if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
  1110. rc = -EBUSY;
  1111. goto out;
  1112. }
  1113. irq_ptr->ccw.cmd_code = irq_ptr->aqueue.cmd;
  1114. irq_ptr->ccw.flags = CCW_FLAG_SLI;
  1115. irq_ptr->ccw.count = irq_ptr->aqueue.count;
  1116. irq_ptr->ccw.cda = 0;
  1117. spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags);
  1118. ccw_device_set_options(cdev, CCWDEV_REPORT_ALL);
  1119. rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ACTIVATE,
  1120. 0, DOIO_DENY_PREFETCH);
  1121. if (rc) {
  1122. DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no);
  1123. DBF_ERROR("rc:%4x", rc);
  1124. }
  1125. spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags);
  1126. if (rc)
  1127. goto out;
  1128. if (is_thinint_irq(irq_ptr))
  1129. tiqdio_add_input_queues(irq_ptr);
  1130. /* wait for subchannel to become active */
  1131. msleep(5);
  1132. switch (irq_ptr->state) {
  1133. case QDIO_IRQ_STATE_STOPPED:
  1134. case QDIO_IRQ_STATE_ERR:
  1135. rc = -EIO;
  1136. break;
  1137. default:
  1138. qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE);
  1139. rc = 0;
  1140. }
  1141. out:
  1142. mutex_unlock(&irq_ptr->setup_mutex);
  1143. return rc;
  1144. }
  1145. EXPORT_SYMBOL_GPL(qdio_activate);
  1146. static inline int buf_in_between(int bufnr, int start, int count)
  1147. {
  1148. int end = add_buf(start, count);
  1149. if (end > start) {
  1150. if (bufnr >= start && bufnr < end)
  1151. return 1;
  1152. else
  1153. return 0;
  1154. }
  1155. /* wrap-around case */
  1156. if ((bufnr >= start && bufnr <= QDIO_MAX_BUFFERS_PER_Q) ||
  1157. (bufnr < end))
  1158. return 1;
  1159. else
  1160. return 0;
  1161. }
  1162. /**
  1163. * handle_inbound - reset processed input buffers
  1164. * @q: queue containing the buffers
  1165. * @callflags: flags
  1166. * @bufnr: first buffer to process
  1167. * @count: how many buffers are emptied
  1168. */
  1169. static int handle_inbound(struct qdio_q *q, unsigned int callflags,
  1170. int bufnr, int count)
  1171. {
  1172. int used, diff;
  1173. qperf_inc(q, inbound_call);
  1174. if (!q->u.in.polling)
  1175. goto set;
  1176. /* protect against stop polling setting an ACK for an emptied slsb */
  1177. if (count == QDIO_MAX_BUFFERS_PER_Q) {
  1178. /* overwriting everything, just delete polling status */
  1179. q->u.in.polling = 0;
  1180. q->u.in.ack_count = 0;
  1181. goto set;
  1182. } else if (buf_in_between(q->u.in.ack_start, bufnr, count)) {
  1183. if (is_qebsm(q)) {
  1184. /* partial overwrite, just update ack_start */
  1185. diff = add_buf(bufnr, count);
  1186. diff = sub_buf(diff, q->u.in.ack_start);
  1187. q->u.in.ack_count -= diff;
  1188. if (q->u.in.ack_count <= 0) {
  1189. q->u.in.polling = 0;
  1190. q->u.in.ack_count = 0;
  1191. goto set;
  1192. }
  1193. q->u.in.ack_start = add_buf(q->u.in.ack_start, diff);
  1194. }
  1195. else
  1196. /* the only ACK will be deleted, so stop polling */
  1197. q->u.in.polling = 0;
  1198. }
  1199. set:
  1200. count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
  1201. used = atomic_add_return(count, &q->nr_buf_used) - count;
  1202. BUG_ON(used + count > QDIO_MAX_BUFFERS_PER_Q);
  1203. /* no need to signal as long as the adapter had free buffers */
  1204. if (used)
  1205. return 0;
  1206. if (need_siga_in(q))
  1207. return qdio_siga_input(q);
  1208. return 0;
  1209. }
  1210. /**
  1211. * handle_outbound - process filled outbound buffers
  1212. * @q: queue containing the buffers
  1213. * @callflags: flags
  1214. * @bufnr: first buffer to process
  1215. * @count: how many buffers are filled
  1216. */
  1217. static int handle_outbound(struct qdio_q *q, unsigned int callflags,
  1218. int bufnr, int count)
  1219. {
  1220. unsigned char state;
  1221. int used, rc = 0;
  1222. qperf_inc(q, outbound_call);
  1223. count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count);
  1224. used = atomic_add_return(count, &q->nr_buf_used);
  1225. BUG_ON(used > QDIO_MAX_BUFFERS_PER_Q);
  1226. if (callflags & QDIO_FLAG_PCI_OUT) {
  1227. q->u.out.pci_out_enabled = 1;
  1228. qperf_inc(q, pci_request_int);
  1229. }
  1230. else
  1231. q->u.out.pci_out_enabled = 0;
  1232. if (queue_type(q) == QDIO_IQDIO_QFMT) {
  1233. if (multicast_outbound(q))
  1234. rc = qdio_kick_outbound_q(q);
  1235. else
  1236. if ((q->irq_ptr->ssqd_desc.mmwc > 1) &&
  1237. (count > 1) &&
  1238. (count <= q->irq_ptr->ssqd_desc.mmwc)) {
  1239. /* exploit enhanced SIGA */
  1240. q->u.out.use_enh_siga = 1;
  1241. rc = qdio_kick_outbound_q(q);
  1242. } else {
  1243. /*
  1244. * One siga-w per buffer required for unicast
  1245. * HiperSockets.
  1246. */
  1247. q->u.out.use_enh_siga = 0;
  1248. while (count--) {
  1249. rc = qdio_kick_outbound_q(q);
  1250. if (rc)
  1251. goto out;
  1252. }
  1253. }
  1254. goto out;
  1255. }
  1256. if (need_siga_sync(q)) {
  1257. qdio_siga_sync_q(q);
  1258. goto out;
  1259. }
  1260. /* try to fast requeue buffers */
  1261. get_buf_state(q, prev_buf(bufnr), &state, 0);
  1262. if (state != SLSB_CU_OUTPUT_PRIMED)
  1263. rc = qdio_kick_outbound_q(q);
  1264. else
  1265. qperf_inc(q, fast_requeue);
  1266. out:
  1267. /* in case of SIGA errors we must process the error immediately */
  1268. if (used >= q->u.out.scan_threshold || rc)
  1269. tasklet_schedule(&q->tasklet);
  1270. else
  1271. /* free the SBALs in case of no further traffic */
  1272. if (!timer_pending(&q->u.out.timer))
  1273. mod_timer(&q->u.out.timer, jiffies + HZ);
  1274. return rc;
  1275. }
  1276. /**
  1277. * do_QDIO - process input or output buffers
  1278. * @cdev: associated ccw_device for the qdio subchannel
  1279. * @callflags: input or output and special flags from the program
  1280. * @q_nr: queue number
  1281. * @bufnr: buffer number
  1282. * @count: how many buffers to process
  1283. */
  1284. int do_QDIO(struct ccw_device *cdev, unsigned int callflags,
  1285. int q_nr, unsigned int bufnr, unsigned int count)
  1286. {
  1287. struct qdio_irq *irq_ptr;
  1288. if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
  1289. return -EINVAL;
  1290. irq_ptr = cdev->private->qdio_data;
  1291. if (!irq_ptr)
  1292. return -ENODEV;
  1293. DBF_DEV_EVENT(DBF_INFO, irq_ptr,
  1294. "do%02x b:%02x c:%02x", callflags, bufnr, count);
  1295. if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
  1296. return -EBUSY;
  1297. if (callflags & QDIO_FLAG_SYNC_INPUT)
  1298. return handle_inbound(irq_ptr->input_qs[q_nr],
  1299. callflags, bufnr, count);
  1300. else if (callflags & QDIO_FLAG_SYNC_OUTPUT)
  1301. return handle_outbound(irq_ptr->output_qs[q_nr],
  1302. callflags, bufnr, count);
  1303. return -EINVAL;
  1304. }
  1305. EXPORT_SYMBOL_GPL(do_QDIO);
  1306. /**
  1307. * qdio_start_irq - process input buffers
  1308. * @cdev: associated ccw_device for the qdio subchannel
  1309. * @nr: input queue number
  1310. *
  1311. * Return codes
  1312. * 0 - success
  1313. * 1 - irqs not started since new data is available
  1314. */
  1315. int qdio_start_irq(struct ccw_device *cdev, int nr)
  1316. {
  1317. struct qdio_q *q;
  1318. struct qdio_irq *irq_ptr = cdev->private->qdio_data;
  1319. if (!irq_ptr)
  1320. return -ENODEV;
  1321. q = irq_ptr->input_qs[nr];
  1322. WARN_ON(queue_irqs_enabled(q));
  1323. if (!shared_ind(q->irq_ptr->dsci))
  1324. xchg(q->irq_ptr->dsci, 0);
  1325. qdio_stop_polling(q);
  1326. clear_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state);
  1327. /*
  1328. * We need to check again to not lose initiative after
  1329. * resetting the ACK state.
  1330. */
  1331. if (!shared_ind(q->irq_ptr->dsci) && *q->irq_ptr->dsci)
  1332. goto rescan;
  1333. if (!qdio_inbound_q_done(q))
  1334. goto rescan;
  1335. return 0;
  1336. rescan:
  1337. if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
  1338. &q->u.in.queue_irq_state))
  1339. return 0;
  1340. else
  1341. return 1;
  1342. }
  1343. EXPORT_SYMBOL(qdio_start_irq);
  1344. /**
  1345. * qdio_get_next_buffers - process input buffers
  1346. * @cdev: associated ccw_device for the qdio subchannel
  1347. * @nr: input queue number
  1348. * @bufnr: first filled buffer number
  1349. * @error: buffers are in error state
  1350. *
  1351. * Return codes
  1352. * < 0 - error
  1353. * = 0 - no new buffers found
  1354. * > 0 - number of processed buffers
  1355. */
  1356. int qdio_get_next_buffers(struct ccw_device *cdev, int nr, int *bufnr,
  1357. int *error)
  1358. {
  1359. struct qdio_q *q;
  1360. int start, end;
  1361. struct qdio_irq *irq_ptr = cdev->private->qdio_data;
  1362. if (!irq_ptr)
  1363. return -ENODEV;
  1364. q = irq_ptr->input_qs[nr];
  1365. WARN_ON(queue_irqs_enabled(q));
  1366. qdio_sync_after_thinint(q);
  1367. /*
  1368. * The interrupt could be caused by a PCI request. Check the
  1369. * PCI capable outbound queues.
  1370. */
  1371. qdio_check_outbound_after_thinint(q);
  1372. if (!qdio_inbound_q_moved(q))
  1373. return 0;
  1374. /* Note: upper-layer MUST stop processing immediately here ... */
  1375. if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
  1376. return -EIO;
  1377. start = q->first_to_kick;
  1378. end = q->first_to_check;
  1379. *bufnr = start;
  1380. *error = q->qdio_error;
  1381. /* for the next time */
  1382. q->first_to_kick = end;
  1383. q->qdio_error = 0;
  1384. return sub_buf(end, start);
  1385. }
  1386. EXPORT_SYMBOL(qdio_get_next_buffers);
  1387. /**
  1388. * qdio_stop_irq - disable interrupt processing for the device
  1389. * @cdev: associated ccw_device for the qdio subchannel
  1390. * @nr: input queue number
  1391. *
  1392. * Return codes
  1393. * 0 - interrupts were already disabled
  1394. * 1 - interrupts successfully disabled
  1395. */
  1396. int qdio_stop_irq(struct ccw_device *cdev, int nr)
  1397. {
  1398. struct qdio_q *q;
  1399. struct qdio_irq *irq_ptr = cdev->private->qdio_data;
  1400. if (!irq_ptr)
  1401. return -ENODEV;
  1402. q = irq_ptr->input_qs[nr];
  1403. if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
  1404. &q->u.in.queue_irq_state))
  1405. return 0;
  1406. else
  1407. return 1;
  1408. }
  1409. EXPORT_SYMBOL(qdio_stop_irq);
  1410. static int __init init_QDIO(void)
  1411. {
  1412. int rc;
  1413. rc = qdio_setup_init();
  1414. if (rc)
  1415. return rc;
  1416. rc = tiqdio_allocate_memory();
  1417. if (rc)
  1418. goto out_cache;
  1419. rc = qdio_debug_init();
  1420. if (rc)
  1421. goto out_ti;
  1422. rc = tiqdio_register_thinints();
  1423. if (rc)
  1424. goto out_debug;
  1425. return 0;
  1426. out_debug:
  1427. qdio_debug_exit();
  1428. out_ti:
  1429. tiqdio_free_memory();
  1430. out_cache:
  1431. qdio_setup_exit();
  1432. return rc;
  1433. }
  1434. static void __exit exit_QDIO(void)
  1435. {
  1436. tiqdio_unregister_thinints();
  1437. tiqdio_free_memory();
  1438. qdio_debug_exit();
  1439. qdio_setup_exit();
  1440. }
  1441. module_init(init_QDIO);
  1442. module_exit(exit_QDIO);