blk-core.c 68 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548
  1. /*
  2. * Copyright (C) 1991, 1992 Linus Torvalds
  3. * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
  4. * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
  5. * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
  6. * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
  7. * - July2000
  8. * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
  9. */
  10. /*
  11. * This handles all read/write requests to block devices
  12. */
  13. #include <linux/kernel.h>
  14. #include <linux/module.h>
  15. #include <linux/backing-dev.h>
  16. #include <linux/bio.h>
  17. #include <linux/blkdev.h>
  18. #include <linux/highmem.h>
  19. #include <linux/mm.h>
  20. #include <linux/kernel_stat.h>
  21. #include <linux/string.h>
  22. #include <linux/init.h>
  23. #include <linux/completion.h>
  24. #include <linux/slab.h>
  25. #include <linux/swap.h>
  26. #include <linux/writeback.h>
  27. #include <linux/task_io_accounting_ops.h>
  28. #include <linux/fault-inject.h>
  29. #define CREATE_TRACE_POINTS
  30. #include <trace/events/block.h>
  31. #include "blk.h"
  32. EXPORT_TRACEPOINT_SYMBOL_GPL(block_remap);
  33. EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
  34. EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
  35. static int __make_request(struct request_queue *q, struct bio *bio);
  36. /*
  37. * For the allocated request tables
  38. */
  39. static struct kmem_cache *request_cachep;
  40. /*
  41. * For queue allocation
  42. */
  43. struct kmem_cache *blk_requestq_cachep;
  44. /*
  45. * Controlling structure to kblockd
  46. */
  47. static struct workqueue_struct *kblockd_workqueue;
  48. static void drive_stat_acct(struct request *rq, int new_io)
  49. {
  50. struct hd_struct *part;
  51. int rw = rq_data_dir(rq);
  52. int cpu;
  53. if (!blk_do_io_stat(rq))
  54. return;
  55. cpu = part_stat_lock();
  56. part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
  57. if (!new_io)
  58. part_stat_inc(cpu, part, merges[rw]);
  59. else {
  60. part_round_stats(cpu, part);
  61. part_inc_in_flight(part, rw);
  62. }
  63. part_stat_unlock();
  64. }
  65. void blk_queue_congestion_threshold(struct request_queue *q)
  66. {
  67. int nr;
  68. nr = q->nr_requests - (q->nr_requests / 8) + 1;
  69. if (nr > q->nr_requests)
  70. nr = q->nr_requests;
  71. q->nr_congestion_on = nr;
  72. nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
  73. if (nr < 1)
  74. nr = 1;
  75. q->nr_congestion_off = nr;
  76. }
  77. /**
  78. * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
  79. * @bdev: device
  80. *
  81. * Locates the passed device's request queue and returns the address of its
  82. * backing_dev_info
  83. *
  84. * Will return NULL if the request queue cannot be located.
  85. */
  86. struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
  87. {
  88. struct backing_dev_info *ret = NULL;
  89. struct request_queue *q = bdev_get_queue(bdev);
  90. if (q)
  91. ret = &q->backing_dev_info;
  92. return ret;
  93. }
  94. EXPORT_SYMBOL(blk_get_backing_dev_info);
  95. void blk_rq_init(struct request_queue *q, struct request *rq)
  96. {
  97. memset(rq, 0, sizeof(*rq));
  98. INIT_LIST_HEAD(&rq->queuelist);
  99. INIT_LIST_HEAD(&rq->timeout_list);
  100. rq->cpu = -1;
  101. rq->q = q;
  102. rq->__sector = (sector_t) -1;
  103. INIT_HLIST_NODE(&rq->hash);
  104. RB_CLEAR_NODE(&rq->rb_node);
  105. rq->cmd = rq->__cmd;
  106. rq->cmd_len = BLK_MAX_CDB;
  107. rq->tag = -1;
  108. rq->ref_count = 1;
  109. rq->start_time = jiffies;
  110. set_start_time_ns(rq);
  111. }
  112. EXPORT_SYMBOL(blk_rq_init);
  113. static void req_bio_endio(struct request *rq, struct bio *bio,
  114. unsigned int nbytes, int error)
  115. {
  116. struct request_queue *q = rq->q;
  117. if (&q->bar_rq != rq) {
  118. if (error)
  119. clear_bit(BIO_UPTODATE, &bio->bi_flags);
  120. else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
  121. error = -EIO;
  122. if (unlikely(nbytes > bio->bi_size)) {
  123. printk(KERN_ERR "%s: want %u bytes done, %u left\n",
  124. __func__, nbytes, bio->bi_size);
  125. nbytes = bio->bi_size;
  126. }
  127. if (unlikely(rq->cmd_flags & REQ_QUIET))
  128. set_bit(BIO_QUIET, &bio->bi_flags);
  129. bio->bi_size -= nbytes;
  130. bio->bi_sector += (nbytes >> 9);
  131. if (bio_integrity(bio))
  132. bio_integrity_advance(bio, nbytes);
  133. if (bio->bi_size == 0)
  134. bio_endio(bio, error);
  135. } else {
  136. /*
  137. * Okay, this is the barrier request in progress, just
  138. * record the error;
  139. */
  140. if (error && !q->orderr)
  141. q->orderr = error;
  142. }
  143. }
  144. void blk_dump_rq_flags(struct request *rq, char *msg)
  145. {
  146. int bit;
  147. printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
  148. rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
  149. rq->cmd_flags);
  150. printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
  151. (unsigned long long)blk_rq_pos(rq),
  152. blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
  153. printk(KERN_INFO " bio %p, biotail %p, buffer %p, len %u\n",
  154. rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq));
  155. if (blk_pc_request(rq)) {
  156. printk(KERN_INFO " cdb: ");
  157. for (bit = 0; bit < BLK_MAX_CDB; bit++)
  158. printk("%02x ", rq->cmd[bit]);
  159. printk("\n");
  160. }
  161. }
  162. EXPORT_SYMBOL(blk_dump_rq_flags);
  163. /*
  164. * "plug" the device if there are no outstanding requests: this will
  165. * force the transfer to start only after we have put all the requests
  166. * on the list.
  167. *
  168. * This is called with interrupts off and no requests on the queue and
  169. * with the queue lock held.
  170. */
  171. void blk_plug_device(struct request_queue *q)
  172. {
  173. WARN_ON(!irqs_disabled());
  174. /*
  175. * don't plug a stopped queue, it must be paired with blk_start_queue()
  176. * which will restart the queueing
  177. */
  178. if (blk_queue_stopped(q))
  179. return;
  180. if (!queue_flag_test_and_set(QUEUE_FLAG_PLUGGED, q)) {
  181. mod_timer(&q->unplug_timer, jiffies + q->unplug_delay);
  182. trace_block_plug(q);
  183. }
  184. }
  185. EXPORT_SYMBOL(blk_plug_device);
  186. /**
  187. * blk_plug_device_unlocked - plug a device without queue lock held
  188. * @q: The &struct request_queue to plug
  189. *
  190. * Description:
  191. * Like @blk_plug_device(), but grabs the queue lock and disables
  192. * interrupts.
  193. **/
  194. void blk_plug_device_unlocked(struct request_queue *q)
  195. {
  196. unsigned long flags;
  197. spin_lock_irqsave(q->queue_lock, flags);
  198. blk_plug_device(q);
  199. spin_unlock_irqrestore(q->queue_lock, flags);
  200. }
  201. EXPORT_SYMBOL(blk_plug_device_unlocked);
  202. /*
  203. * remove the queue from the plugged list, if present. called with
  204. * queue lock held and interrupts disabled.
  205. */
  206. int blk_remove_plug(struct request_queue *q)
  207. {
  208. WARN_ON(!irqs_disabled());
  209. if (!queue_flag_test_and_clear(QUEUE_FLAG_PLUGGED, q))
  210. return 0;
  211. del_timer(&q->unplug_timer);
  212. return 1;
  213. }
  214. EXPORT_SYMBOL(blk_remove_plug);
  215. /*
  216. * remove the plug and let it rip..
  217. */
  218. void __generic_unplug_device(struct request_queue *q)
  219. {
  220. if (unlikely(blk_queue_stopped(q)))
  221. return;
  222. if (!blk_remove_plug(q) && !blk_queue_nonrot(q))
  223. return;
  224. q->request_fn(q);
  225. }
  226. /**
  227. * generic_unplug_device - fire a request queue
  228. * @q: The &struct request_queue in question
  229. *
  230. * Description:
  231. * Linux uses plugging to build bigger requests queues before letting
  232. * the device have at them. If a queue is plugged, the I/O scheduler
  233. * is still adding and merging requests on the queue. Once the queue
  234. * gets unplugged, the request_fn defined for the queue is invoked and
  235. * transfers started.
  236. **/
  237. void generic_unplug_device(struct request_queue *q)
  238. {
  239. if (blk_queue_plugged(q)) {
  240. spin_lock_irq(q->queue_lock);
  241. __generic_unplug_device(q);
  242. spin_unlock_irq(q->queue_lock);
  243. }
  244. }
  245. EXPORT_SYMBOL(generic_unplug_device);
  246. static void blk_backing_dev_unplug(struct backing_dev_info *bdi,
  247. struct page *page)
  248. {
  249. struct request_queue *q = bdi->unplug_io_data;
  250. blk_unplug(q);
  251. }
  252. void blk_unplug_work(struct work_struct *work)
  253. {
  254. struct request_queue *q =
  255. container_of(work, struct request_queue, unplug_work);
  256. trace_block_unplug_io(q);
  257. q->unplug_fn(q);
  258. }
  259. void blk_unplug_timeout(unsigned long data)
  260. {
  261. struct request_queue *q = (struct request_queue *)data;
  262. trace_block_unplug_timer(q);
  263. kblockd_schedule_work(q, &q->unplug_work);
  264. }
  265. void blk_unplug(struct request_queue *q)
  266. {
  267. /*
  268. * devices don't necessarily have an ->unplug_fn defined
  269. */
  270. if (q->unplug_fn) {
  271. trace_block_unplug_io(q);
  272. q->unplug_fn(q);
  273. }
  274. }
  275. EXPORT_SYMBOL(blk_unplug);
  276. /**
  277. * blk_start_queue - restart a previously stopped queue
  278. * @q: The &struct request_queue in question
  279. *
  280. * Description:
  281. * blk_start_queue() will clear the stop flag on the queue, and call
  282. * the request_fn for the queue if it was in a stopped state when
  283. * entered. Also see blk_stop_queue(). Queue lock must be held.
  284. **/
  285. void blk_start_queue(struct request_queue *q)
  286. {
  287. WARN_ON(!irqs_disabled());
  288. queue_flag_clear(QUEUE_FLAG_STOPPED, q);
  289. __blk_run_queue(q);
  290. }
  291. EXPORT_SYMBOL(blk_start_queue);
  292. /**
  293. * blk_stop_queue - stop a queue
  294. * @q: The &struct request_queue in question
  295. *
  296. * Description:
  297. * The Linux block layer assumes that a block driver will consume all
  298. * entries on the request queue when the request_fn strategy is called.
  299. * Often this will not happen, because of hardware limitations (queue
  300. * depth settings). If a device driver gets a 'queue full' response,
  301. * or if it simply chooses not to queue more I/O at one point, it can
  302. * call this function to prevent the request_fn from being called until
  303. * the driver has signalled it's ready to go again. This happens by calling
  304. * blk_start_queue() to restart queue operations. Queue lock must be held.
  305. **/
  306. void blk_stop_queue(struct request_queue *q)
  307. {
  308. blk_remove_plug(q);
  309. queue_flag_set(QUEUE_FLAG_STOPPED, q);
  310. }
  311. EXPORT_SYMBOL(blk_stop_queue);
  312. /**
  313. * blk_sync_queue - cancel any pending callbacks on a queue
  314. * @q: the queue
  315. *
  316. * Description:
  317. * The block layer may perform asynchronous callback activity
  318. * on a queue, such as calling the unplug function after a timeout.
  319. * A block device may call blk_sync_queue to ensure that any
  320. * such activity is cancelled, thus allowing it to release resources
  321. * that the callbacks might use. The caller must already have made sure
  322. * that its ->make_request_fn will not re-add plugging prior to calling
  323. * this function.
  324. *
  325. */
  326. void blk_sync_queue(struct request_queue *q)
  327. {
  328. del_timer_sync(&q->unplug_timer);
  329. del_timer_sync(&q->timeout);
  330. cancel_work_sync(&q->unplug_work);
  331. }
  332. EXPORT_SYMBOL(blk_sync_queue);
  333. /**
  334. * __blk_run_queue - run a single device queue
  335. * @q: The queue to run
  336. *
  337. * Description:
  338. * See @blk_run_queue. This variant must be called with the queue lock
  339. * held and interrupts disabled.
  340. *
  341. */
  342. void __blk_run_queue(struct request_queue *q)
  343. {
  344. blk_remove_plug(q);
  345. if (unlikely(blk_queue_stopped(q)))
  346. return;
  347. if (elv_queue_empty(q))
  348. return;
  349. /*
  350. * Only recurse once to avoid overrunning the stack, let the unplug
  351. * handling reinvoke the handler shortly if we already got there.
  352. */
  353. if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
  354. q->request_fn(q);
  355. queue_flag_clear(QUEUE_FLAG_REENTER, q);
  356. } else {
  357. queue_flag_set(QUEUE_FLAG_PLUGGED, q);
  358. kblockd_schedule_work(q, &q->unplug_work);
  359. }
  360. }
  361. EXPORT_SYMBOL(__blk_run_queue);
  362. /**
  363. * blk_run_queue - run a single device queue
  364. * @q: The queue to run
  365. *
  366. * Description:
  367. * Invoke request handling on this queue, if it has pending work to do.
  368. * May be used to restart queueing when a request has completed.
  369. */
  370. void blk_run_queue(struct request_queue *q)
  371. {
  372. unsigned long flags;
  373. spin_lock_irqsave(q->queue_lock, flags);
  374. __blk_run_queue(q);
  375. spin_unlock_irqrestore(q->queue_lock, flags);
  376. }
  377. EXPORT_SYMBOL(blk_run_queue);
  378. void blk_put_queue(struct request_queue *q)
  379. {
  380. kobject_put(&q->kobj);
  381. }
  382. void blk_cleanup_queue(struct request_queue *q)
  383. {
  384. /*
  385. * We know we have process context here, so we can be a little
  386. * cautious and ensure that pending block actions on this device
  387. * are done before moving on. Going into this function, we should
  388. * not have processes doing IO to this device.
  389. */
  390. blk_sync_queue(q);
  391. del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
  392. mutex_lock(&q->sysfs_lock);
  393. queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q);
  394. mutex_unlock(&q->sysfs_lock);
  395. if (q->elevator)
  396. elevator_exit(q->elevator);
  397. blk_put_queue(q);
  398. }
  399. EXPORT_SYMBOL(blk_cleanup_queue);
  400. static int blk_init_free_list(struct request_queue *q)
  401. {
  402. struct request_list *rl = &q->rq;
  403. if (unlikely(rl->rq_pool))
  404. return 0;
  405. rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
  406. rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
  407. rl->elvpriv = 0;
  408. init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
  409. init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
  410. rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
  411. mempool_free_slab, request_cachep, q->node);
  412. if (!rl->rq_pool)
  413. return -ENOMEM;
  414. return 0;
  415. }
  416. struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
  417. {
  418. return blk_alloc_queue_node(gfp_mask, -1);
  419. }
  420. EXPORT_SYMBOL(blk_alloc_queue);
  421. struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
  422. {
  423. struct request_queue *q;
  424. int err;
  425. q = kmem_cache_alloc_node(blk_requestq_cachep,
  426. gfp_mask | __GFP_ZERO, node_id);
  427. if (!q)
  428. return NULL;
  429. q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug;
  430. q->backing_dev_info.unplug_io_data = q;
  431. q->backing_dev_info.ra_pages =
  432. (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
  433. q->backing_dev_info.state = 0;
  434. q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY;
  435. q->backing_dev_info.name = "block";
  436. err = bdi_init(&q->backing_dev_info);
  437. if (err) {
  438. kmem_cache_free(blk_requestq_cachep, q);
  439. return NULL;
  440. }
  441. setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
  442. laptop_mode_timer_fn, (unsigned long) q);
  443. init_timer(&q->unplug_timer);
  444. setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
  445. INIT_LIST_HEAD(&q->timeout_list);
  446. INIT_WORK(&q->unplug_work, blk_unplug_work);
  447. kobject_init(&q->kobj, &blk_queue_ktype);
  448. mutex_init(&q->sysfs_lock);
  449. spin_lock_init(&q->__queue_lock);
  450. return q;
  451. }
  452. EXPORT_SYMBOL(blk_alloc_queue_node);
  453. /**
  454. * blk_init_queue - prepare a request queue for use with a block device
  455. * @rfn: The function to be called to process requests that have been
  456. * placed on the queue.
  457. * @lock: Request queue spin lock
  458. *
  459. * Description:
  460. * If a block device wishes to use the standard request handling procedures,
  461. * which sorts requests and coalesces adjacent requests, then it must
  462. * call blk_init_queue(). The function @rfn will be called when there
  463. * are requests on the queue that need to be processed. If the device
  464. * supports plugging, then @rfn may not be called immediately when requests
  465. * are available on the queue, but may be called at some time later instead.
  466. * Plugged queues are generally unplugged when a buffer belonging to one
  467. * of the requests on the queue is needed, or due to memory pressure.
  468. *
  469. * @rfn is not required, or even expected, to remove all requests off the
  470. * queue, but only as many as it can handle at a time. If it does leave
  471. * requests on the queue, it is responsible for arranging that the requests
  472. * get dealt with eventually.
  473. *
  474. * The queue spin lock must be held while manipulating the requests on the
  475. * request queue; this lock will be taken also from interrupt context, so irq
  476. * disabling is needed for it.
  477. *
  478. * Function returns a pointer to the initialized request queue, or %NULL if
  479. * it didn't succeed.
  480. *
  481. * Note:
  482. * blk_init_queue() must be paired with a blk_cleanup_queue() call
  483. * when the block device is deactivated (such as at module unload).
  484. **/
  485. struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
  486. {
  487. return blk_init_queue_node(rfn, lock, -1);
  488. }
  489. EXPORT_SYMBOL(blk_init_queue);
  490. struct request_queue *
  491. blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
  492. {
  493. struct request_queue *uninit_q, *q;
  494. uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id);
  495. if (!uninit_q)
  496. return NULL;
  497. q = blk_init_allocated_queue_node(uninit_q, rfn, lock, node_id);
  498. if (!q)
  499. blk_cleanup_queue(uninit_q);
  500. return q;
  501. }
  502. EXPORT_SYMBOL(blk_init_queue_node);
  503. struct request_queue *
  504. blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
  505. spinlock_t *lock)
  506. {
  507. return blk_init_allocated_queue_node(q, rfn, lock, -1);
  508. }
  509. EXPORT_SYMBOL(blk_init_allocated_queue);
  510. struct request_queue *
  511. blk_init_allocated_queue_node(struct request_queue *q, request_fn_proc *rfn,
  512. spinlock_t *lock, int node_id)
  513. {
  514. if (!q)
  515. return NULL;
  516. q->node = node_id;
  517. if (blk_init_free_list(q))
  518. return NULL;
  519. q->request_fn = rfn;
  520. q->prep_rq_fn = NULL;
  521. q->unplug_fn = generic_unplug_device;
  522. q->queue_flags = QUEUE_FLAG_DEFAULT;
  523. q->queue_lock = lock;
  524. /*
  525. * This also sets hw/phys segments, boundary and size
  526. */
  527. blk_queue_make_request(q, __make_request);
  528. q->sg_reserved_size = INT_MAX;
  529. /*
  530. * all done
  531. */
  532. if (!elevator_init(q, NULL)) {
  533. blk_queue_congestion_threshold(q);
  534. return q;
  535. }
  536. return NULL;
  537. }
  538. EXPORT_SYMBOL(blk_init_allocated_queue_node);
  539. int blk_get_queue(struct request_queue *q)
  540. {
  541. if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
  542. kobject_get(&q->kobj);
  543. return 0;
  544. }
  545. return 1;
  546. }
  547. static inline void blk_free_request(struct request_queue *q, struct request *rq)
  548. {
  549. if (rq->cmd_flags & REQ_ELVPRIV)
  550. elv_put_request(q, rq);
  551. mempool_free(rq, q->rq.rq_pool);
  552. }
  553. static struct request *
  554. blk_alloc_request(struct request_queue *q, int flags, int priv, gfp_t gfp_mask)
  555. {
  556. struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);
  557. if (!rq)
  558. return NULL;
  559. blk_rq_init(q, rq);
  560. rq->cmd_flags = flags | REQ_ALLOCED;
  561. if (priv) {
  562. if (unlikely(elv_set_request(q, rq, gfp_mask))) {
  563. mempool_free(rq, q->rq.rq_pool);
  564. return NULL;
  565. }
  566. rq->cmd_flags |= REQ_ELVPRIV;
  567. }
  568. return rq;
  569. }
  570. /*
  571. * ioc_batching returns true if the ioc is a valid batching request and
  572. * should be given priority access to a request.
  573. */
  574. static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
  575. {
  576. if (!ioc)
  577. return 0;
  578. /*
  579. * Make sure the process is able to allocate at least 1 request
  580. * even if the batch times out, otherwise we could theoretically
  581. * lose wakeups.
  582. */
  583. return ioc->nr_batch_requests == q->nr_batching ||
  584. (ioc->nr_batch_requests > 0
  585. && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
  586. }
  587. /*
  588. * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
  589. * will cause the process to be a "batcher" on all queues in the system. This
  590. * is the behaviour we want though - once it gets a wakeup it should be given
  591. * a nice run.
  592. */
  593. static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
  594. {
  595. if (!ioc || ioc_batching(q, ioc))
  596. return;
  597. ioc->nr_batch_requests = q->nr_batching;
  598. ioc->last_waited = jiffies;
  599. }
  600. static void __freed_request(struct request_queue *q, int sync)
  601. {
  602. struct request_list *rl = &q->rq;
  603. if (rl->count[sync] < queue_congestion_off_threshold(q))
  604. blk_clear_queue_congested(q, sync);
  605. if (rl->count[sync] + 1 <= q->nr_requests) {
  606. if (waitqueue_active(&rl->wait[sync]))
  607. wake_up(&rl->wait[sync]);
  608. blk_clear_queue_full(q, sync);
  609. }
  610. }
  611. /*
  612. * A request has just been released. Account for it, update the full and
  613. * congestion status, wake up any waiters. Called under q->queue_lock.
  614. */
  615. static void freed_request(struct request_queue *q, int sync, int priv)
  616. {
  617. struct request_list *rl = &q->rq;
  618. rl->count[sync]--;
  619. if (priv)
  620. rl->elvpriv--;
  621. __freed_request(q, sync);
  622. if (unlikely(rl->starved[sync ^ 1]))
  623. __freed_request(q, sync ^ 1);
  624. }
  625. /*
  626. * Get a free request, queue_lock must be held.
  627. * Returns NULL on failure, with queue_lock held.
  628. * Returns !NULL on success, with queue_lock *not held*.
  629. */
  630. static struct request *get_request(struct request_queue *q, int rw_flags,
  631. struct bio *bio, gfp_t gfp_mask)
  632. {
  633. struct request *rq = NULL;
  634. struct request_list *rl = &q->rq;
  635. struct io_context *ioc = NULL;
  636. const bool is_sync = rw_is_sync(rw_flags) != 0;
  637. int may_queue, priv;
  638. may_queue = elv_may_queue(q, rw_flags);
  639. if (may_queue == ELV_MQUEUE_NO)
  640. goto rq_starved;
  641. if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
  642. if (rl->count[is_sync]+1 >= q->nr_requests) {
  643. ioc = current_io_context(GFP_ATOMIC, q->node);
  644. /*
  645. * The queue will fill after this allocation, so set
  646. * it as full, and mark this process as "batching".
  647. * This process will be allowed to complete a batch of
  648. * requests, others will be blocked.
  649. */
  650. if (!blk_queue_full(q, is_sync)) {
  651. ioc_set_batching(q, ioc);
  652. blk_set_queue_full(q, is_sync);
  653. } else {
  654. if (may_queue != ELV_MQUEUE_MUST
  655. && !ioc_batching(q, ioc)) {
  656. /*
  657. * The queue is full and the allocating
  658. * process is not a "batcher", and not
  659. * exempted by the IO scheduler
  660. */
  661. goto out;
  662. }
  663. }
  664. }
  665. blk_set_queue_congested(q, is_sync);
  666. }
  667. /*
  668. * Only allow batching queuers to allocate up to 50% over the defined
  669. * limit of requests, otherwise we could have thousands of requests
  670. * allocated with any setting of ->nr_requests
  671. */
  672. if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
  673. goto out;
  674. rl->count[is_sync]++;
  675. rl->starved[is_sync] = 0;
  676. priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
  677. if (priv)
  678. rl->elvpriv++;
  679. if (blk_queue_io_stat(q))
  680. rw_flags |= REQ_IO_STAT;
  681. spin_unlock_irq(q->queue_lock);
  682. rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
  683. if (unlikely(!rq)) {
  684. /*
  685. * Allocation failed presumably due to memory. Undo anything
  686. * we might have messed up.
  687. *
  688. * Allocating task should really be put onto the front of the
  689. * wait queue, but this is pretty rare.
  690. */
  691. spin_lock_irq(q->queue_lock);
  692. freed_request(q, is_sync, priv);
  693. /*
  694. * in the very unlikely event that allocation failed and no
  695. * requests for this direction was pending, mark us starved
  696. * so that freeing of a request in the other direction will
  697. * notice us. another possible fix would be to split the
  698. * rq mempool into READ and WRITE
  699. */
  700. rq_starved:
  701. if (unlikely(rl->count[is_sync] == 0))
  702. rl->starved[is_sync] = 1;
  703. goto out;
  704. }
  705. /*
  706. * ioc may be NULL here, and ioc_batching will be false. That's
  707. * OK, if the queue is under the request limit then requests need
  708. * not count toward the nr_batch_requests limit. There will always
  709. * be some limit enforced by BLK_BATCH_TIME.
  710. */
  711. if (ioc_batching(q, ioc))
  712. ioc->nr_batch_requests--;
  713. trace_block_getrq(q, bio, rw_flags & 1);
  714. out:
  715. return rq;
  716. }
  717. /*
  718. * No available requests for this queue, unplug the device and wait for some
  719. * requests to become available.
  720. *
  721. * Called with q->queue_lock held, and returns with it unlocked.
  722. */
  723. static struct request *get_request_wait(struct request_queue *q, int rw_flags,
  724. struct bio *bio)
  725. {
  726. const bool is_sync = rw_is_sync(rw_flags) != 0;
  727. struct request *rq;
  728. rq = get_request(q, rw_flags, bio, GFP_NOIO);
  729. while (!rq) {
  730. DEFINE_WAIT(wait);
  731. struct io_context *ioc;
  732. struct request_list *rl = &q->rq;
  733. prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
  734. TASK_UNINTERRUPTIBLE);
  735. trace_block_sleeprq(q, bio, rw_flags & 1);
  736. __generic_unplug_device(q);
  737. spin_unlock_irq(q->queue_lock);
  738. io_schedule();
  739. /*
  740. * After sleeping, we become a "batching" process and
  741. * will be able to allocate at least one request, and
  742. * up to a big batch of them for a small period time.
  743. * See ioc_batching, ioc_set_batching
  744. */
  745. ioc = current_io_context(GFP_NOIO, q->node);
  746. ioc_set_batching(q, ioc);
  747. spin_lock_irq(q->queue_lock);
  748. finish_wait(&rl->wait[is_sync], &wait);
  749. rq = get_request(q, rw_flags, bio, GFP_NOIO);
  750. };
  751. return rq;
  752. }
  753. struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
  754. {
  755. struct request *rq;
  756. BUG_ON(rw != READ && rw != WRITE);
  757. spin_lock_irq(q->queue_lock);
  758. if (gfp_mask & __GFP_WAIT) {
  759. rq = get_request_wait(q, rw, NULL);
  760. } else {
  761. rq = get_request(q, rw, NULL, gfp_mask);
  762. if (!rq)
  763. spin_unlock_irq(q->queue_lock);
  764. }
  765. /* q->queue_lock is unlocked at this point */
  766. return rq;
  767. }
  768. EXPORT_SYMBOL(blk_get_request);
  769. /**
  770. * blk_make_request - given a bio, allocate a corresponding struct request.
  771. * @q: target request queue
  772. * @bio: The bio describing the memory mappings that will be submitted for IO.
  773. * It may be a chained-bio properly constructed by block/bio layer.
  774. * @gfp_mask: gfp flags to be used for memory allocation
  775. *
  776. * blk_make_request is the parallel of generic_make_request for BLOCK_PC
  777. * type commands. Where the struct request needs to be farther initialized by
  778. * the caller. It is passed a &struct bio, which describes the memory info of
  779. * the I/O transfer.
  780. *
  781. * The caller of blk_make_request must make sure that bi_io_vec
  782. * are set to describe the memory buffers. That bio_data_dir() will return
  783. * the needed direction of the request. (And all bio's in the passed bio-chain
  784. * are properly set accordingly)
  785. *
  786. * If called under none-sleepable conditions, mapped bio buffers must not
  787. * need bouncing, by calling the appropriate masked or flagged allocator,
  788. * suitable for the target device. Otherwise the call to blk_queue_bounce will
  789. * BUG.
  790. *
  791. * WARNING: When allocating/cloning a bio-chain, careful consideration should be
  792. * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for
  793. * anything but the first bio in the chain. Otherwise you risk waiting for IO
  794. * completion of a bio that hasn't been submitted yet, thus resulting in a
  795. * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead
  796. * of bio_alloc(), as that avoids the mempool deadlock.
  797. * If possible a big IO should be split into smaller parts when allocation
  798. * fails. Partial allocation should not be an error, or you risk a live-lock.
  799. */
  800. struct request *blk_make_request(struct request_queue *q, struct bio *bio,
  801. gfp_t gfp_mask)
  802. {
  803. struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask);
  804. if (unlikely(!rq))
  805. return ERR_PTR(-ENOMEM);
  806. for_each_bio(bio) {
  807. struct bio *bounce_bio = bio;
  808. int ret;
  809. blk_queue_bounce(q, &bounce_bio);
  810. ret = blk_rq_append_bio(q, rq, bounce_bio);
  811. if (unlikely(ret)) {
  812. blk_put_request(rq);
  813. return ERR_PTR(ret);
  814. }
  815. }
  816. return rq;
  817. }
  818. EXPORT_SYMBOL(blk_make_request);
  819. /**
  820. * blk_requeue_request - put a request back on queue
  821. * @q: request queue where request should be inserted
  822. * @rq: request to be inserted
  823. *
  824. * Description:
  825. * Drivers often keep queueing requests until the hardware cannot accept
  826. * more, when that condition happens we need to put the request back
  827. * on the queue. Must be called with queue lock held.
  828. */
  829. void blk_requeue_request(struct request_queue *q, struct request *rq)
  830. {
  831. blk_delete_timer(rq);
  832. blk_clear_rq_complete(rq);
  833. trace_block_rq_requeue(q, rq);
  834. if (blk_rq_tagged(rq))
  835. blk_queue_end_tag(q, rq);
  836. BUG_ON(blk_queued_rq(rq));
  837. elv_requeue_request(q, rq);
  838. }
  839. EXPORT_SYMBOL(blk_requeue_request);
  840. /**
  841. * blk_insert_request - insert a special request into a request queue
  842. * @q: request queue where request should be inserted
  843. * @rq: request to be inserted
  844. * @at_head: insert request at head or tail of queue
  845. * @data: private data
  846. *
  847. * Description:
  848. * Many block devices need to execute commands asynchronously, so they don't
  849. * block the whole kernel from preemption during request execution. This is
  850. * accomplished normally by inserting aritficial requests tagged as
  851. * REQ_TYPE_SPECIAL in to the corresponding request queue, and letting them
  852. * be scheduled for actual execution by the request queue.
  853. *
  854. * We have the option of inserting the head or the tail of the queue.
  855. * Typically we use the tail for new ioctls and so forth. We use the head
  856. * of the queue for things like a QUEUE_FULL message from a device, or a
  857. * host that is unable to accept a particular command.
  858. */
  859. void blk_insert_request(struct request_queue *q, struct request *rq,
  860. int at_head, void *data)
  861. {
  862. int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
  863. unsigned long flags;
  864. /*
  865. * tell I/O scheduler that this isn't a regular read/write (ie it
  866. * must not attempt merges on this) and that it acts as a soft
  867. * barrier
  868. */
  869. rq->cmd_type = REQ_TYPE_SPECIAL;
  870. rq->special = data;
  871. spin_lock_irqsave(q->queue_lock, flags);
  872. /*
  873. * If command is tagged, release the tag
  874. */
  875. if (blk_rq_tagged(rq))
  876. blk_queue_end_tag(q, rq);
  877. drive_stat_acct(rq, 1);
  878. __elv_add_request(q, rq, where, 0);
  879. __blk_run_queue(q);
  880. spin_unlock_irqrestore(q->queue_lock, flags);
  881. }
  882. EXPORT_SYMBOL(blk_insert_request);
  883. /*
  884. * add-request adds a request to the linked list.
  885. * queue lock is held and interrupts disabled, as we muck with the
  886. * request queue list.
  887. */
  888. static inline void add_request(struct request_queue *q, struct request *req)
  889. {
  890. drive_stat_acct(req, 1);
  891. /*
  892. * elevator indicated where it wants this request to be
  893. * inserted at elevator_merge time
  894. */
  895. __elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0);
  896. }
  897. static void part_round_stats_single(int cpu, struct hd_struct *part,
  898. unsigned long now)
  899. {
  900. if (now == part->stamp)
  901. return;
  902. if (part_in_flight(part)) {
  903. __part_stat_add(cpu, part, time_in_queue,
  904. part_in_flight(part) * (now - part->stamp));
  905. __part_stat_add(cpu, part, io_ticks, (now - part->stamp));
  906. }
  907. part->stamp = now;
  908. }
  909. /**
  910. * part_round_stats() - Round off the performance stats on a struct disk_stats.
  911. * @cpu: cpu number for stats access
  912. * @part: target partition
  913. *
  914. * The average IO queue length and utilisation statistics are maintained
  915. * by observing the current state of the queue length and the amount of
  916. * time it has been in this state for.
  917. *
  918. * Normally, that accounting is done on IO completion, but that can result
  919. * in more than a second's worth of IO being accounted for within any one
  920. * second, leading to >100% utilisation. To deal with that, we call this
  921. * function to do a round-off before returning the results when reading
  922. * /proc/diskstats. This accounts immediately for all queue usage up to
  923. * the current jiffies and restarts the counters again.
  924. */
  925. void part_round_stats(int cpu, struct hd_struct *part)
  926. {
  927. unsigned long now = jiffies;
  928. if (part->partno)
  929. part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
  930. part_round_stats_single(cpu, part, now);
  931. }
  932. EXPORT_SYMBOL_GPL(part_round_stats);
  933. /*
  934. * queue lock must be held
  935. */
  936. void __blk_put_request(struct request_queue *q, struct request *req)
  937. {
  938. if (unlikely(!q))
  939. return;
  940. if (unlikely(--req->ref_count))
  941. return;
  942. elv_completed_request(q, req);
  943. /* this is a bio leak */
  944. WARN_ON(req->bio != NULL);
  945. /*
  946. * Request may not have originated from ll_rw_blk. if not,
  947. * it didn't come out of our reserved rq pools
  948. */
  949. if (req->cmd_flags & REQ_ALLOCED) {
  950. int is_sync = rq_is_sync(req) != 0;
  951. int priv = req->cmd_flags & REQ_ELVPRIV;
  952. BUG_ON(!list_empty(&req->queuelist));
  953. BUG_ON(!hlist_unhashed(&req->hash));
  954. blk_free_request(q, req);
  955. freed_request(q, is_sync, priv);
  956. }
  957. }
  958. EXPORT_SYMBOL_GPL(__blk_put_request);
  959. void blk_put_request(struct request *req)
  960. {
  961. unsigned long flags;
  962. struct request_queue *q = req->q;
  963. spin_lock_irqsave(q->queue_lock, flags);
  964. __blk_put_request(q, req);
  965. spin_unlock_irqrestore(q->queue_lock, flags);
  966. }
  967. EXPORT_SYMBOL(blk_put_request);
  968. void init_request_from_bio(struct request *req, struct bio *bio)
  969. {
  970. req->cpu = bio->bi_comp_cpu;
  971. req->cmd_type = REQ_TYPE_FS;
  972. /*
  973. * Inherit FAILFAST from bio (for read-ahead, and explicit
  974. * FAILFAST). FAILFAST flags are identical for req and bio.
  975. */
  976. if (bio_rw_flagged(bio, BIO_RW_AHEAD))
  977. req->cmd_flags |= REQ_FAILFAST_MASK;
  978. else
  979. req->cmd_flags |= bio->bi_rw & REQ_FAILFAST_MASK;
  980. if (bio_rw_flagged(bio, BIO_RW_DISCARD))
  981. req->cmd_flags |= REQ_DISCARD;
  982. if (bio_rw_flagged(bio, BIO_RW_BARRIER))
  983. req->cmd_flags |= REQ_HARDBARRIER;
  984. if (bio_rw_flagged(bio, BIO_RW_SYNCIO))
  985. req->cmd_flags |= REQ_RW_SYNC;
  986. if (bio_rw_flagged(bio, BIO_RW_META))
  987. req->cmd_flags |= REQ_RW_META;
  988. if (bio_rw_flagged(bio, BIO_RW_NOIDLE))
  989. req->cmd_flags |= REQ_NOIDLE;
  990. req->errors = 0;
  991. req->__sector = bio->bi_sector;
  992. req->ioprio = bio_prio(bio);
  993. blk_rq_bio_prep(req->q, req, bio);
  994. }
  995. /*
  996. * Only disabling plugging for non-rotational devices if it does tagging
  997. * as well, otherwise we do need the proper merging
  998. */
  999. static inline bool queue_should_plug(struct request_queue *q)
  1000. {
  1001. return !(blk_queue_nonrot(q) && blk_queue_tagged(q));
  1002. }
  1003. static int __make_request(struct request_queue *q, struct bio *bio)
  1004. {
  1005. struct request *req;
  1006. int el_ret;
  1007. unsigned int bytes = bio->bi_size;
  1008. const unsigned short prio = bio_prio(bio);
  1009. const bool sync = bio_rw_flagged(bio, BIO_RW_SYNCIO);
  1010. const bool unplug = bio_rw_flagged(bio, BIO_RW_UNPLUG);
  1011. const unsigned int ff = bio->bi_rw & REQ_FAILFAST_MASK;
  1012. int rw_flags;
  1013. if (bio_rw_flagged(bio, BIO_RW_BARRIER) &&
  1014. (q->next_ordered == QUEUE_ORDERED_NONE)) {
  1015. bio_endio(bio, -EOPNOTSUPP);
  1016. return 0;
  1017. }
  1018. /*
  1019. * low level driver can indicate that it wants pages above a
  1020. * certain limit bounced to low memory (ie for highmem, or even
  1021. * ISA dma in theory)
  1022. */
  1023. blk_queue_bounce(q, &bio);
  1024. spin_lock_irq(q->queue_lock);
  1025. if (unlikely(bio_rw_flagged(bio, BIO_RW_BARRIER)) || elv_queue_empty(q))
  1026. goto get_rq;
  1027. el_ret = elv_merge(q, &req, bio);
  1028. switch (el_ret) {
  1029. case ELEVATOR_BACK_MERGE:
  1030. BUG_ON(!rq_mergeable(req));
  1031. if (!ll_back_merge_fn(q, req, bio))
  1032. break;
  1033. trace_block_bio_backmerge(q, bio);
  1034. if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
  1035. blk_rq_set_mixed_merge(req);
  1036. req->biotail->bi_next = bio;
  1037. req->biotail = bio;
  1038. req->__data_len += bytes;
  1039. req->ioprio = ioprio_best(req->ioprio, prio);
  1040. if (!blk_rq_cpu_valid(req))
  1041. req->cpu = bio->bi_comp_cpu;
  1042. drive_stat_acct(req, 0);
  1043. elv_bio_merged(q, req, bio);
  1044. if (!attempt_back_merge(q, req))
  1045. elv_merged_request(q, req, el_ret);
  1046. goto out;
  1047. case ELEVATOR_FRONT_MERGE:
  1048. BUG_ON(!rq_mergeable(req));
  1049. if (!ll_front_merge_fn(q, req, bio))
  1050. break;
  1051. trace_block_bio_frontmerge(q, bio);
  1052. if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) {
  1053. blk_rq_set_mixed_merge(req);
  1054. req->cmd_flags &= ~REQ_FAILFAST_MASK;
  1055. req->cmd_flags |= ff;
  1056. }
  1057. bio->bi_next = req->bio;
  1058. req->bio = bio;
  1059. /*
  1060. * may not be valid. if the low level driver said
  1061. * it didn't need a bounce buffer then it better
  1062. * not touch req->buffer either...
  1063. */
  1064. req->buffer = bio_data(bio);
  1065. req->__sector = bio->bi_sector;
  1066. req->__data_len += bytes;
  1067. req->ioprio = ioprio_best(req->ioprio, prio);
  1068. if (!blk_rq_cpu_valid(req))
  1069. req->cpu = bio->bi_comp_cpu;
  1070. drive_stat_acct(req, 0);
  1071. elv_bio_merged(q, req, bio);
  1072. if (!attempt_front_merge(q, req))
  1073. elv_merged_request(q, req, el_ret);
  1074. goto out;
  1075. /* ELV_NO_MERGE: elevator says don't/can't merge. */
  1076. default:
  1077. ;
  1078. }
  1079. get_rq:
  1080. /*
  1081. * This sync check and mask will be re-done in init_request_from_bio(),
  1082. * but we need to set it earlier to expose the sync flag to the
  1083. * rq allocator and io schedulers.
  1084. */
  1085. rw_flags = bio_data_dir(bio);
  1086. if (sync)
  1087. rw_flags |= REQ_RW_SYNC;
  1088. /*
  1089. * Grab a free request. This is might sleep but can not fail.
  1090. * Returns with the queue unlocked.
  1091. */
  1092. req = get_request_wait(q, rw_flags, bio);
  1093. /*
  1094. * After dropping the lock and possibly sleeping here, our request
  1095. * may now be mergeable after it had proven unmergeable (above).
  1096. * We don't worry about that case for efficiency. It won't happen
  1097. * often, and the elevators are able to handle it.
  1098. */
  1099. init_request_from_bio(req, bio);
  1100. spin_lock_irq(q->queue_lock);
  1101. if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) ||
  1102. bio_flagged(bio, BIO_CPU_AFFINE))
  1103. req->cpu = blk_cpu_to_group(smp_processor_id());
  1104. if (queue_should_plug(q) && elv_queue_empty(q))
  1105. blk_plug_device(q);
  1106. add_request(q, req);
  1107. out:
  1108. if (unplug || !queue_should_plug(q))
  1109. __generic_unplug_device(q);
  1110. spin_unlock_irq(q->queue_lock);
  1111. return 0;
  1112. }
  1113. /*
  1114. * If bio->bi_dev is a partition, remap the location
  1115. */
  1116. static inline void blk_partition_remap(struct bio *bio)
  1117. {
  1118. struct block_device *bdev = bio->bi_bdev;
  1119. if (bio_sectors(bio) && bdev != bdev->bd_contains) {
  1120. struct hd_struct *p = bdev->bd_part;
  1121. bio->bi_sector += p->start_sect;
  1122. bio->bi_bdev = bdev->bd_contains;
  1123. trace_block_remap(bdev_get_queue(bio->bi_bdev), bio,
  1124. bdev->bd_dev,
  1125. bio->bi_sector - p->start_sect);
  1126. }
  1127. }
  1128. static void handle_bad_sector(struct bio *bio)
  1129. {
  1130. char b[BDEVNAME_SIZE];
  1131. printk(KERN_INFO "attempt to access beyond end of device\n");
  1132. printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n",
  1133. bdevname(bio->bi_bdev, b),
  1134. bio->bi_rw,
  1135. (unsigned long long)bio->bi_sector + bio_sectors(bio),
  1136. (long long)(bio->bi_bdev->bd_inode->i_size >> 9));
  1137. set_bit(BIO_EOF, &bio->bi_flags);
  1138. }
  1139. #ifdef CONFIG_FAIL_MAKE_REQUEST
  1140. static DECLARE_FAULT_ATTR(fail_make_request);
  1141. static int __init setup_fail_make_request(char *str)
  1142. {
  1143. return setup_fault_attr(&fail_make_request, str);
  1144. }
  1145. __setup("fail_make_request=", setup_fail_make_request);
  1146. static int should_fail_request(struct bio *bio)
  1147. {
  1148. struct hd_struct *part = bio->bi_bdev->bd_part;
  1149. if (part_to_disk(part)->part0.make_it_fail || part->make_it_fail)
  1150. return should_fail(&fail_make_request, bio->bi_size);
  1151. return 0;
  1152. }
  1153. static int __init fail_make_request_debugfs(void)
  1154. {
  1155. return init_fault_attr_dentries(&fail_make_request,
  1156. "fail_make_request");
  1157. }
  1158. late_initcall(fail_make_request_debugfs);
  1159. #else /* CONFIG_FAIL_MAKE_REQUEST */
  1160. static inline int should_fail_request(struct bio *bio)
  1161. {
  1162. return 0;
  1163. }
  1164. #endif /* CONFIG_FAIL_MAKE_REQUEST */
  1165. /*
  1166. * Check whether this bio extends beyond the end of the device.
  1167. */
  1168. static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
  1169. {
  1170. sector_t maxsector;
  1171. if (!nr_sectors)
  1172. return 0;
  1173. /* Test device or partition size, when known. */
  1174. maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
  1175. if (maxsector) {
  1176. sector_t sector = bio->bi_sector;
  1177. if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
  1178. /*
  1179. * This may well happen - the kernel calls bread()
  1180. * without checking the size of the device, e.g., when
  1181. * mounting a device.
  1182. */
  1183. handle_bad_sector(bio);
  1184. return 1;
  1185. }
  1186. }
  1187. return 0;
  1188. }
  1189. /**
  1190. * generic_make_request - hand a buffer to its device driver for I/O
  1191. * @bio: The bio describing the location in memory and on the device.
  1192. *
  1193. * generic_make_request() is used to make I/O requests of block
  1194. * devices. It is passed a &struct bio, which describes the I/O that needs
  1195. * to be done.
  1196. *
  1197. * generic_make_request() does not return any status. The
  1198. * success/failure status of the request, along with notification of
  1199. * completion, is delivered asynchronously through the bio->bi_end_io
  1200. * function described (one day) else where.
  1201. *
  1202. * The caller of generic_make_request must make sure that bi_io_vec
  1203. * are set to describe the memory buffer, and that bi_dev and bi_sector are
  1204. * set to describe the device address, and the
  1205. * bi_end_io and optionally bi_private are set to describe how
  1206. * completion notification should be signaled.
  1207. *
  1208. * generic_make_request and the drivers it calls may use bi_next if this
  1209. * bio happens to be merged with someone else, and may change bi_dev and
  1210. * bi_sector for remaps as it sees fit. So the values of these fields
  1211. * should NOT be depended on after the call to generic_make_request.
  1212. */
  1213. static inline void __generic_make_request(struct bio *bio)
  1214. {
  1215. struct request_queue *q;
  1216. sector_t old_sector;
  1217. int ret, nr_sectors = bio_sectors(bio);
  1218. dev_t old_dev;
  1219. int err = -EIO;
  1220. might_sleep();
  1221. if (bio_check_eod(bio, nr_sectors))
  1222. goto end_io;
  1223. /*
  1224. * Resolve the mapping until finished. (drivers are
  1225. * still free to implement/resolve their own stacking
  1226. * by explicitly returning 0)
  1227. *
  1228. * NOTE: we don't repeat the blk_size check for each new device.
  1229. * Stacking drivers are expected to know what they are doing.
  1230. */
  1231. old_sector = -1;
  1232. old_dev = 0;
  1233. do {
  1234. char b[BDEVNAME_SIZE];
  1235. q = bdev_get_queue(bio->bi_bdev);
  1236. if (unlikely(!q)) {
  1237. printk(KERN_ERR
  1238. "generic_make_request: Trying to access "
  1239. "nonexistent block-device %s (%Lu)\n",
  1240. bdevname(bio->bi_bdev, b),
  1241. (long long) bio->bi_sector);
  1242. goto end_io;
  1243. }
  1244. if (unlikely(!bio_rw_flagged(bio, BIO_RW_DISCARD) &&
  1245. nr_sectors > queue_max_hw_sectors(q))) {
  1246. printk(KERN_ERR "bio too big device %s (%u > %u)\n",
  1247. bdevname(bio->bi_bdev, b),
  1248. bio_sectors(bio),
  1249. queue_max_hw_sectors(q));
  1250. goto end_io;
  1251. }
  1252. if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
  1253. goto end_io;
  1254. if (should_fail_request(bio))
  1255. goto end_io;
  1256. /*
  1257. * If this device has partitions, remap block n
  1258. * of partition p to block n+start(p) of the disk.
  1259. */
  1260. blk_partition_remap(bio);
  1261. if (bio_integrity_enabled(bio) && bio_integrity_prep(bio))
  1262. goto end_io;
  1263. if (old_sector != -1)
  1264. trace_block_remap(q, bio, old_dev, old_sector);
  1265. old_sector = bio->bi_sector;
  1266. old_dev = bio->bi_bdev->bd_dev;
  1267. if (bio_check_eod(bio, nr_sectors))
  1268. goto end_io;
  1269. if (bio_rw_flagged(bio, BIO_RW_DISCARD) &&
  1270. !blk_queue_discard(q)) {
  1271. err = -EOPNOTSUPP;
  1272. goto end_io;
  1273. }
  1274. trace_block_bio_queue(q, bio);
  1275. ret = q->make_request_fn(q, bio);
  1276. } while (ret);
  1277. return;
  1278. end_io:
  1279. bio_endio(bio, err);
  1280. }
  1281. /*
  1282. * We only want one ->make_request_fn to be active at a time,
  1283. * else stack usage with stacked devices could be a problem.
  1284. * So use current->bio_list to keep a list of requests
  1285. * submited by a make_request_fn function.
  1286. * current->bio_list is also used as a flag to say if
  1287. * generic_make_request is currently active in this task or not.
  1288. * If it is NULL, then no make_request is active. If it is non-NULL,
  1289. * then a make_request is active, and new requests should be added
  1290. * at the tail
  1291. */
  1292. void generic_make_request(struct bio *bio)
  1293. {
  1294. struct bio_list bio_list_on_stack;
  1295. if (current->bio_list) {
  1296. /* make_request is active */
  1297. bio_list_add(current->bio_list, bio);
  1298. return;
  1299. }
  1300. /* following loop may be a bit non-obvious, and so deserves some
  1301. * explanation.
  1302. * Before entering the loop, bio->bi_next is NULL (as all callers
  1303. * ensure that) so we have a list with a single bio.
  1304. * We pretend that we have just taken it off a longer list, so
  1305. * we assign bio_list to a pointer to the bio_list_on_stack,
  1306. * thus initialising the bio_list of new bios to be
  1307. * added. __generic_make_request may indeed add some more bios
  1308. * through a recursive call to generic_make_request. If it
  1309. * did, we find a non-NULL value in bio_list and re-enter the loop
  1310. * from the top. In this case we really did just take the bio
  1311. * of the top of the list (no pretending) and so remove it from
  1312. * bio_list, and call into __generic_make_request again.
  1313. *
  1314. * The loop was structured like this to make only one call to
  1315. * __generic_make_request (which is important as it is large and
  1316. * inlined) and to keep the structure simple.
  1317. */
  1318. BUG_ON(bio->bi_next);
  1319. bio_list_init(&bio_list_on_stack);
  1320. current->bio_list = &bio_list_on_stack;
  1321. do {
  1322. __generic_make_request(bio);
  1323. bio = bio_list_pop(current->bio_list);
  1324. } while (bio);
  1325. current->bio_list = NULL; /* deactivate */
  1326. }
  1327. EXPORT_SYMBOL(generic_make_request);
  1328. /**
  1329. * submit_bio - submit a bio to the block device layer for I/O
  1330. * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
  1331. * @bio: The &struct bio which describes the I/O
  1332. *
  1333. * submit_bio() is very similar in purpose to generic_make_request(), and
  1334. * uses that function to do most of the work. Both are fairly rough
  1335. * interfaces; @bio must be presetup and ready for I/O.
  1336. *
  1337. */
  1338. void submit_bio(int rw, struct bio *bio)
  1339. {
  1340. int count = bio_sectors(bio);
  1341. bio->bi_rw |= rw;
  1342. /*
  1343. * If it's a regular read/write or a barrier with data attached,
  1344. * go through the normal accounting stuff before submission.
  1345. */
  1346. if (bio_has_data(bio) && !(rw & (1 << BIO_RW_DISCARD))) {
  1347. if (rw & WRITE) {
  1348. count_vm_events(PGPGOUT, count);
  1349. } else {
  1350. task_io_account_read(bio->bi_size);
  1351. count_vm_events(PGPGIN, count);
  1352. }
  1353. if (unlikely(block_dump)) {
  1354. char b[BDEVNAME_SIZE];
  1355. printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n",
  1356. current->comm, task_pid_nr(current),
  1357. (rw & WRITE) ? "WRITE" : "READ",
  1358. (unsigned long long)bio->bi_sector,
  1359. bdevname(bio->bi_bdev, b));
  1360. }
  1361. }
  1362. generic_make_request(bio);
  1363. }
  1364. EXPORT_SYMBOL(submit_bio);
  1365. /**
  1366. * blk_rq_check_limits - Helper function to check a request for the queue limit
  1367. * @q: the queue
  1368. * @rq: the request being checked
  1369. *
  1370. * Description:
  1371. * @rq may have been made based on weaker limitations of upper-level queues
  1372. * in request stacking drivers, and it may violate the limitation of @q.
  1373. * Since the block layer and the underlying device driver trust @rq
  1374. * after it is inserted to @q, it should be checked against @q before
  1375. * the insertion using this generic function.
  1376. *
  1377. * This function should also be useful for request stacking drivers
  1378. * in some cases below, so export this fuction.
  1379. * Request stacking drivers like request-based dm may change the queue
  1380. * limits while requests are in the queue (e.g. dm's table swapping).
  1381. * Such request stacking drivers should check those requests agaist
  1382. * the new queue limits again when they dispatch those requests,
  1383. * although such checkings are also done against the old queue limits
  1384. * when submitting requests.
  1385. */
  1386. int blk_rq_check_limits(struct request_queue *q, struct request *rq)
  1387. {
  1388. if (blk_rq_sectors(rq) > queue_max_sectors(q) ||
  1389. blk_rq_bytes(rq) > queue_max_hw_sectors(q) << 9) {
  1390. printk(KERN_ERR "%s: over max size limit.\n", __func__);
  1391. return -EIO;
  1392. }
  1393. /*
  1394. * queue's settings related to segment counting like q->bounce_pfn
  1395. * may differ from that of other stacking queues.
  1396. * Recalculate it to check the request correctly on this queue's
  1397. * limitation.
  1398. */
  1399. blk_recalc_rq_segments(rq);
  1400. if (rq->nr_phys_segments > queue_max_segments(q)) {
  1401. printk(KERN_ERR "%s: over max segments limit.\n", __func__);
  1402. return -EIO;
  1403. }
  1404. return 0;
  1405. }
  1406. EXPORT_SYMBOL_GPL(blk_rq_check_limits);
  1407. /**
  1408. * blk_insert_cloned_request - Helper for stacking drivers to submit a request
  1409. * @q: the queue to submit the request
  1410. * @rq: the request being queued
  1411. */
  1412. int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
  1413. {
  1414. unsigned long flags;
  1415. if (blk_rq_check_limits(q, rq))
  1416. return -EIO;
  1417. #ifdef CONFIG_FAIL_MAKE_REQUEST
  1418. if (rq->rq_disk && rq->rq_disk->part0.make_it_fail &&
  1419. should_fail(&fail_make_request, blk_rq_bytes(rq)))
  1420. return -EIO;
  1421. #endif
  1422. spin_lock_irqsave(q->queue_lock, flags);
  1423. /*
  1424. * Submitting request must be dequeued before calling this function
  1425. * because it will be linked to another request_queue
  1426. */
  1427. BUG_ON(blk_queued_rq(rq));
  1428. drive_stat_acct(rq, 1);
  1429. __elv_add_request(q, rq, ELEVATOR_INSERT_BACK, 0);
  1430. spin_unlock_irqrestore(q->queue_lock, flags);
  1431. return 0;
  1432. }
  1433. EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
  1434. /**
  1435. * blk_rq_err_bytes - determine number of bytes till the next failure boundary
  1436. * @rq: request to examine
  1437. *
  1438. * Description:
  1439. * A request could be merge of IOs which require different failure
  1440. * handling. This function determines the number of bytes which
  1441. * can be failed from the beginning of the request without
  1442. * crossing into area which need to be retried further.
  1443. *
  1444. * Return:
  1445. * The number of bytes to fail.
  1446. *
  1447. * Context:
  1448. * queue_lock must be held.
  1449. */
  1450. unsigned int blk_rq_err_bytes(const struct request *rq)
  1451. {
  1452. unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
  1453. unsigned int bytes = 0;
  1454. struct bio *bio;
  1455. if (!(rq->cmd_flags & REQ_MIXED_MERGE))
  1456. return blk_rq_bytes(rq);
  1457. /*
  1458. * Currently the only 'mixing' which can happen is between
  1459. * different fastfail types. We can safely fail portions
  1460. * which have all the failfast bits that the first one has -
  1461. * the ones which are at least as eager to fail as the first
  1462. * one.
  1463. */
  1464. for (bio = rq->bio; bio; bio = bio->bi_next) {
  1465. if ((bio->bi_rw & ff) != ff)
  1466. break;
  1467. bytes += bio->bi_size;
  1468. }
  1469. /* this could lead to infinite loop */
  1470. BUG_ON(blk_rq_bytes(rq) && !bytes);
  1471. return bytes;
  1472. }
  1473. EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
  1474. static void blk_account_io_completion(struct request *req, unsigned int bytes)
  1475. {
  1476. if (blk_do_io_stat(req)) {
  1477. const int rw = rq_data_dir(req);
  1478. struct hd_struct *part;
  1479. int cpu;
  1480. cpu = part_stat_lock();
  1481. part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
  1482. part_stat_add(cpu, part, sectors[rw], bytes >> 9);
  1483. part_stat_unlock();
  1484. }
  1485. }
  1486. static void blk_account_io_done(struct request *req)
  1487. {
  1488. /*
  1489. * Account IO completion. bar_rq isn't accounted as a normal
  1490. * IO on queueing nor completion. Accounting the containing
  1491. * request is enough.
  1492. */
  1493. if (blk_do_io_stat(req) && req != &req->q->bar_rq) {
  1494. unsigned long duration = jiffies - req->start_time;
  1495. const int rw = rq_data_dir(req);
  1496. struct hd_struct *part;
  1497. int cpu;
  1498. cpu = part_stat_lock();
  1499. part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
  1500. part_stat_inc(cpu, part, ios[rw]);
  1501. part_stat_add(cpu, part, ticks[rw], duration);
  1502. part_round_stats(cpu, part);
  1503. part_dec_in_flight(part, rw);
  1504. part_stat_unlock();
  1505. }
  1506. }
  1507. /**
  1508. * blk_peek_request - peek at the top of a request queue
  1509. * @q: request queue to peek at
  1510. *
  1511. * Description:
  1512. * Return the request at the top of @q. The returned request
  1513. * should be started using blk_start_request() before LLD starts
  1514. * processing it.
  1515. *
  1516. * Return:
  1517. * Pointer to the request at the top of @q if available. Null
  1518. * otherwise.
  1519. *
  1520. * Context:
  1521. * queue_lock must be held.
  1522. */
  1523. struct request *blk_peek_request(struct request_queue *q)
  1524. {
  1525. struct request *rq;
  1526. int ret;
  1527. while ((rq = __elv_next_request(q)) != NULL) {
  1528. if (!(rq->cmd_flags & REQ_STARTED)) {
  1529. /*
  1530. * This is the first time the device driver
  1531. * sees this request (possibly after
  1532. * requeueing). Notify IO scheduler.
  1533. */
  1534. if (blk_sorted_rq(rq))
  1535. elv_activate_rq(q, rq);
  1536. /*
  1537. * just mark as started even if we don't start
  1538. * it, a request that has been delayed should
  1539. * not be passed by new incoming requests
  1540. */
  1541. rq->cmd_flags |= REQ_STARTED;
  1542. trace_block_rq_issue(q, rq);
  1543. }
  1544. if (!q->boundary_rq || q->boundary_rq == rq) {
  1545. q->end_sector = rq_end_sector(rq);
  1546. q->boundary_rq = NULL;
  1547. }
  1548. if (rq->cmd_flags & REQ_DONTPREP)
  1549. break;
  1550. if (q->dma_drain_size && blk_rq_bytes(rq)) {
  1551. /*
  1552. * make sure space for the drain appears we
  1553. * know we can do this because max_hw_segments
  1554. * has been adjusted to be one fewer than the
  1555. * device can handle
  1556. */
  1557. rq->nr_phys_segments++;
  1558. }
  1559. if (!q->prep_rq_fn)
  1560. break;
  1561. ret = q->prep_rq_fn(q, rq);
  1562. if (ret == BLKPREP_OK) {
  1563. break;
  1564. } else if (ret == BLKPREP_DEFER) {
  1565. /*
  1566. * the request may have been (partially) prepped.
  1567. * we need to keep this request in the front to
  1568. * avoid resource deadlock. REQ_STARTED will
  1569. * prevent other fs requests from passing this one.
  1570. */
  1571. if (q->dma_drain_size && blk_rq_bytes(rq) &&
  1572. !(rq->cmd_flags & REQ_DONTPREP)) {
  1573. /*
  1574. * remove the space for the drain we added
  1575. * so that we don't add it again
  1576. */
  1577. --rq->nr_phys_segments;
  1578. }
  1579. rq = NULL;
  1580. break;
  1581. } else if (ret == BLKPREP_KILL) {
  1582. rq->cmd_flags |= REQ_QUIET;
  1583. /*
  1584. * Mark this request as started so we don't trigger
  1585. * any debug logic in the end I/O path.
  1586. */
  1587. blk_start_request(rq);
  1588. __blk_end_request_all(rq, -EIO);
  1589. } else {
  1590. printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
  1591. break;
  1592. }
  1593. }
  1594. return rq;
  1595. }
  1596. EXPORT_SYMBOL(blk_peek_request);
  1597. void blk_dequeue_request(struct request *rq)
  1598. {
  1599. struct request_queue *q = rq->q;
  1600. BUG_ON(list_empty(&rq->queuelist));
  1601. BUG_ON(ELV_ON_HASH(rq));
  1602. list_del_init(&rq->queuelist);
  1603. /*
  1604. * the time frame between a request being removed from the lists
  1605. * and to it is freed is accounted as io that is in progress at
  1606. * the driver side.
  1607. */
  1608. if (blk_account_rq(rq)) {
  1609. q->in_flight[rq_is_sync(rq)]++;
  1610. set_io_start_time_ns(rq);
  1611. }
  1612. }
  1613. /**
  1614. * blk_start_request - start request processing on the driver
  1615. * @req: request to dequeue
  1616. *
  1617. * Description:
  1618. * Dequeue @req and start timeout timer on it. This hands off the
  1619. * request to the driver.
  1620. *
  1621. * Block internal functions which don't want to start timer should
  1622. * call blk_dequeue_request().
  1623. *
  1624. * Context:
  1625. * queue_lock must be held.
  1626. */
  1627. void blk_start_request(struct request *req)
  1628. {
  1629. blk_dequeue_request(req);
  1630. /*
  1631. * We are now handing the request to the hardware, initialize
  1632. * resid_len to full count and add the timeout handler.
  1633. */
  1634. req->resid_len = blk_rq_bytes(req);
  1635. if (unlikely(blk_bidi_rq(req)))
  1636. req->next_rq->resid_len = blk_rq_bytes(req->next_rq);
  1637. blk_add_timer(req);
  1638. }
  1639. EXPORT_SYMBOL(blk_start_request);
  1640. /**
  1641. * blk_fetch_request - fetch a request from a request queue
  1642. * @q: request queue to fetch a request from
  1643. *
  1644. * Description:
  1645. * Return the request at the top of @q. The request is started on
  1646. * return and LLD can start processing it immediately.
  1647. *
  1648. * Return:
  1649. * Pointer to the request at the top of @q if available. Null
  1650. * otherwise.
  1651. *
  1652. * Context:
  1653. * queue_lock must be held.
  1654. */
  1655. struct request *blk_fetch_request(struct request_queue *q)
  1656. {
  1657. struct request *rq;
  1658. rq = blk_peek_request(q);
  1659. if (rq)
  1660. blk_start_request(rq);
  1661. return rq;
  1662. }
  1663. EXPORT_SYMBOL(blk_fetch_request);
  1664. /**
  1665. * blk_update_request - Special helper function for request stacking drivers
  1666. * @req: the request being processed
  1667. * @error: %0 for success, < %0 for error
  1668. * @nr_bytes: number of bytes to complete @req
  1669. *
  1670. * Description:
  1671. * Ends I/O on a number of bytes attached to @req, but doesn't complete
  1672. * the request structure even if @req doesn't have leftover.
  1673. * If @req has leftover, sets it up for the next range of segments.
  1674. *
  1675. * This special helper function is only for request stacking drivers
  1676. * (e.g. request-based dm) so that they can handle partial completion.
  1677. * Actual device drivers should use blk_end_request instead.
  1678. *
  1679. * Passing the result of blk_rq_bytes() as @nr_bytes guarantees
  1680. * %false return from this function.
  1681. *
  1682. * Return:
  1683. * %false - this request doesn't have any more data
  1684. * %true - this request has more data
  1685. **/
  1686. bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
  1687. {
  1688. int total_bytes, bio_nbytes, next_idx = 0;
  1689. struct bio *bio;
  1690. if (!req->bio)
  1691. return false;
  1692. trace_block_rq_complete(req->q, req);
  1693. /*
  1694. * For fs requests, rq is just carrier of independent bio's
  1695. * and each partial completion should be handled separately.
  1696. * Reset per-request error on each partial completion.
  1697. *
  1698. * TODO: tj: This is too subtle. It would be better to let
  1699. * low level drivers do what they see fit.
  1700. */
  1701. if (blk_fs_request(req))
  1702. req->errors = 0;
  1703. if (error && (blk_fs_request(req) && !(req->cmd_flags & REQ_QUIET))) {
  1704. printk(KERN_ERR "end_request: I/O error, dev %s, sector %llu\n",
  1705. req->rq_disk ? req->rq_disk->disk_name : "?",
  1706. (unsigned long long)blk_rq_pos(req));
  1707. }
  1708. blk_account_io_completion(req, nr_bytes);
  1709. total_bytes = bio_nbytes = 0;
  1710. while ((bio = req->bio) != NULL) {
  1711. int nbytes;
  1712. if (nr_bytes >= bio->bi_size) {
  1713. req->bio = bio->bi_next;
  1714. nbytes = bio->bi_size;
  1715. req_bio_endio(req, bio, nbytes, error);
  1716. next_idx = 0;
  1717. bio_nbytes = 0;
  1718. } else {
  1719. int idx = bio->bi_idx + next_idx;
  1720. if (unlikely(idx >= bio->bi_vcnt)) {
  1721. blk_dump_rq_flags(req, "__end_that");
  1722. printk(KERN_ERR "%s: bio idx %d >= vcnt %d\n",
  1723. __func__, idx, bio->bi_vcnt);
  1724. break;
  1725. }
  1726. nbytes = bio_iovec_idx(bio, idx)->bv_len;
  1727. BIO_BUG_ON(nbytes > bio->bi_size);
  1728. /*
  1729. * not a complete bvec done
  1730. */
  1731. if (unlikely(nbytes > nr_bytes)) {
  1732. bio_nbytes += nr_bytes;
  1733. total_bytes += nr_bytes;
  1734. break;
  1735. }
  1736. /*
  1737. * advance to the next vector
  1738. */
  1739. next_idx++;
  1740. bio_nbytes += nbytes;
  1741. }
  1742. total_bytes += nbytes;
  1743. nr_bytes -= nbytes;
  1744. bio = req->bio;
  1745. if (bio) {
  1746. /*
  1747. * end more in this run, or just return 'not-done'
  1748. */
  1749. if (unlikely(nr_bytes <= 0))
  1750. break;
  1751. }
  1752. }
  1753. /*
  1754. * completely done
  1755. */
  1756. if (!req->bio) {
  1757. /*
  1758. * Reset counters so that the request stacking driver
  1759. * can find how many bytes remain in the request
  1760. * later.
  1761. */
  1762. req->__data_len = 0;
  1763. return false;
  1764. }
  1765. /*
  1766. * if the request wasn't completed, update state
  1767. */
  1768. if (bio_nbytes) {
  1769. req_bio_endio(req, bio, bio_nbytes, error);
  1770. bio->bi_idx += next_idx;
  1771. bio_iovec(bio)->bv_offset += nr_bytes;
  1772. bio_iovec(bio)->bv_len -= nr_bytes;
  1773. }
  1774. req->__data_len -= total_bytes;
  1775. req->buffer = bio_data(req->bio);
  1776. /* update sector only for requests with clear definition of sector */
  1777. if (blk_fs_request(req) || blk_discard_rq(req))
  1778. req->__sector += total_bytes >> 9;
  1779. /* mixed attributes always follow the first bio */
  1780. if (req->cmd_flags & REQ_MIXED_MERGE) {
  1781. req->cmd_flags &= ~REQ_FAILFAST_MASK;
  1782. req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK;
  1783. }
  1784. /*
  1785. * If total number of sectors is less than the first segment
  1786. * size, something has gone terribly wrong.
  1787. */
  1788. if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
  1789. printk(KERN_ERR "blk: request botched\n");
  1790. req->__data_len = blk_rq_cur_bytes(req);
  1791. }
  1792. /* recalculate the number of segments */
  1793. blk_recalc_rq_segments(req);
  1794. return true;
  1795. }
  1796. EXPORT_SYMBOL_GPL(blk_update_request);
  1797. static bool blk_update_bidi_request(struct request *rq, int error,
  1798. unsigned int nr_bytes,
  1799. unsigned int bidi_bytes)
  1800. {
  1801. if (blk_update_request(rq, error, nr_bytes))
  1802. return true;
  1803. /* Bidi request must be completed as a whole */
  1804. if (unlikely(blk_bidi_rq(rq)) &&
  1805. blk_update_request(rq->next_rq, error, bidi_bytes))
  1806. return true;
  1807. add_disk_randomness(rq->rq_disk);
  1808. return false;
  1809. }
  1810. /*
  1811. * queue lock must be held
  1812. */
  1813. static void blk_finish_request(struct request *req, int error)
  1814. {
  1815. if (blk_rq_tagged(req))
  1816. blk_queue_end_tag(req->q, req);
  1817. BUG_ON(blk_queued_rq(req));
  1818. if (unlikely(laptop_mode) && blk_fs_request(req))
  1819. laptop_io_completion(&req->q->backing_dev_info);
  1820. blk_delete_timer(req);
  1821. blk_account_io_done(req);
  1822. if (req->end_io)
  1823. req->end_io(req, error);
  1824. else {
  1825. if (blk_bidi_rq(req))
  1826. __blk_put_request(req->next_rq->q, req->next_rq);
  1827. __blk_put_request(req->q, req);
  1828. }
  1829. }
  1830. /**
  1831. * blk_end_bidi_request - Complete a bidi request
  1832. * @rq: the request to complete
  1833. * @error: %0 for success, < %0 for error
  1834. * @nr_bytes: number of bytes to complete @rq
  1835. * @bidi_bytes: number of bytes to complete @rq->next_rq
  1836. *
  1837. * Description:
  1838. * Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
  1839. * Drivers that supports bidi can safely call this member for any
  1840. * type of request, bidi or uni. In the later case @bidi_bytes is
  1841. * just ignored.
  1842. *
  1843. * Return:
  1844. * %false - we are done with this request
  1845. * %true - still buffers pending for this request
  1846. **/
  1847. static bool blk_end_bidi_request(struct request *rq, int error,
  1848. unsigned int nr_bytes, unsigned int bidi_bytes)
  1849. {
  1850. struct request_queue *q = rq->q;
  1851. unsigned long flags;
  1852. if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
  1853. return true;
  1854. spin_lock_irqsave(q->queue_lock, flags);
  1855. blk_finish_request(rq, error);
  1856. spin_unlock_irqrestore(q->queue_lock, flags);
  1857. return false;
  1858. }
  1859. /**
  1860. * __blk_end_bidi_request - Complete a bidi request with queue lock held
  1861. * @rq: the request to complete
  1862. * @error: %0 for success, < %0 for error
  1863. * @nr_bytes: number of bytes to complete @rq
  1864. * @bidi_bytes: number of bytes to complete @rq->next_rq
  1865. *
  1866. * Description:
  1867. * Identical to blk_end_bidi_request() except that queue lock is
  1868. * assumed to be locked on entry and remains so on return.
  1869. *
  1870. * Return:
  1871. * %false - we are done with this request
  1872. * %true - still buffers pending for this request
  1873. **/
  1874. static bool __blk_end_bidi_request(struct request *rq, int error,
  1875. unsigned int nr_bytes, unsigned int bidi_bytes)
  1876. {
  1877. if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
  1878. return true;
  1879. blk_finish_request(rq, error);
  1880. return false;
  1881. }
  1882. /**
  1883. * blk_end_request - Helper function for drivers to complete the request.
  1884. * @rq: the request being processed
  1885. * @error: %0 for success, < %0 for error
  1886. * @nr_bytes: number of bytes to complete
  1887. *
  1888. * Description:
  1889. * Ends I/O on a number of bytes attached to @rq.
  1890. * If @rq has leftover, sets it up for the next range of segments.
  1891. *
  1892. * Return:
  1893. * %false - we are done with this request
  1894. * %true - still buffers pending for this request
  1895. **/
  1896. bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
  1897. {
  1898. return blk_end_bidi_request(rq, error, nr_bytes, 0);
  1899. }
  1900. EXPORT_SYMBOL(blk_end_request);
  1901. /**
  1902. * blk_end_request_all - Helper function for drives to finish the request.
  1903. * @rq: the request to finish
  1904. * @error: %0 for success, < %0 for error
  1905. *
  1906. * Description:
  1907. * Completely finish @rq.
  1908. */
  1909. void blk_end_request_all(struct request *rq, int error)
  1910. {
  1911. bool pending;
  1912. unsigned int bidi_bytes = 0;
  1913. if (unlikely(blk_bidi_rq(rq)))
  1914. bidi_bytes = blk_rq_bytes(rq->next_rq);
  1915. pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
  1916. BUG_ON(pending);
  1917. }
  1918. EXPORT_SYMBOL(blk_end_request_all);
  1919. /**
  1920. * blk_end_request_cur - Helper function to finish the current request chunk.
  1921. * @rq: the request to finish the current chunk for
  1922. * @error: %0 for success, < %0 for error
  1923. *
  1924. * Description:
  1925. * Complete the current consecutively mapped chunk from @rq.
  1926. *
  1927. * Return:
  1928. * %false - we are done with this request
  1929. * %true - still buffers pending for this request
  1930. */
  1931. bool blk_end_request_cur(struct request *rq, int error)
  1932. {
  1933. return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
  1934. }
  1935. EXPORT_SYMBOL(blk_end_request_cur);
  1936. /**
  1937. * blk_end_request_err - Finish a request till the next failure boundary.
  1938. * @rq: the request to finish till the next failure boundary for
  1939. * @error: must be negative errno
  1940. *
  1941. * Description:
  1942. * Complete @rq till the next failure boundary.
  1943. *
  1944. * Return:
  1945. * %false - we are done with this request
  1946. * %true - still buffers pending for this request
  1947. */
  1948. bool blk_end_request_err(struct request *rq, int error)
  1949. {
  1950. WARN_ON(error >= 0);
  1951. return blk_end_request(rq, error, blk_rq_err_bytes(rq));
  1952. }
  1953. EXPORT_SYMBOL_GPL(blk_end_request_err);
  1954. /**
  1955. * __blk_end_request - Helper function for drivers to complete the request.
  1956. * @rq: the request being processed
  1957. * @error: %0 for success, < %0 for error
  1958. * @nr_bytes: number of bytes to complete
  1959. *
  1960. * Description:
  1961. * Must be called with queue lock held unlike blk_end_request().
  1962. *
  1963. * Return:
  1964. * %false - we are done with this request
  1965. * %true - still buffers pending for this request
  1966. **/
  1967. bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
  1968. {
  1969. return __blk_end_bidi_request(rq, error, nr_bytes, 0);
  1970. }
  1971. EXPORT_SYMBOL(__blk_end_request);
  1972. /**
  1973. * __blk_end_request_all - Helper function for drives to finish the request.
  1974. * @rq: the request to finish
  1975. * @error: %0 for success, < %0 for error
  1976. *
  1977. * Description:
  1978. * Completely finish @rq. Must be called with queue lock held.
  1979. */
  1980. void __blk_end_request_all(struct request *rq, int error)
  1981. {
  1982. bool pending;
  1983. unsigned int bidi_bytes = 0;
  1984. if (unlikely(blk_bidi_rq(rq)))
  1985. bidi_bytes = blk_rq_bytes(rq->next_rq);
  1986. pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
  1987. BUG_ON(pending);
  1988. }
  1989. EXPORT_SYMBOL(__blk_end_request_all);
  1990. /**
  1991. * __blk_end_request_cur - Helper function to finish the current request chunk.
  1992. * @rq: the request to finish the current chunk for
  1993. * @error: %0 for success, < %0 for error
  1994. *
  1995. * Description:
  1996. * Complete the current consecutively mapped chunk from @rq. Must
  1997. * be called with queue lock held.
  1998. *
  1999. * Return:
  2000. * %false - we are done with this request
  2001. * %true - still buffers pending for this request
  2002. */
  2003. bool __blk_end_request_cur(struct request *rq, int error)
  2004. {
  2005. return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
  2006. }
  2007. EXPORT_SYMBOL(__blk_end_request_cur);
  2008. /**
  2009. * __blk_end_request_err - Finish a request till the next failure boundary.
  2010. * @rq: the request to finish till the next failure boundary for
  2011. * @error: must be negative errno
  2012. *
  2013. * Description:
  2014. * Complete @rq till the next failure boundary. Must be called
  2015. * with queue lock held.
  2016. *
  2017. * Return:
  2018. * %false - we are done with this request
  2019. * %true - still buffers pending for this request
  2020. */
  2021. bool __blk_end_request_err(struct request *rq, int error)
  2022. {
  2023. WARN_ON(error >= 0);
  2024. return __blk_end_request(rq, error, blk_rq_err_bytes(rq));
  2025. }
  2026. EXPORT_SYMBOL_GPL(__blk_end_request_err);
  2027. void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
  2028. struct bio *bio)
  2029. {
  2030. /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */
  2031. rq->cmd_flags |= bio->bi_rw & REQ_RW;
  2032. if (bio_has_data(bio)) {
  2033. rq->nr_phys_segments = bio_phys_segments(q, bio);
  2034. rq->buffer = bio_data(bio);
  2035. }
  2036. rq->__data_len = bio->bi_size;
  2037. rq->bio = rq->biotail = bio;
  2038. if (bio->bi_bdev)
  2039. rq->rq_disk = bio->bi_bdev->bd_disk;
  2040. }
  2041. #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
  2042. /**
  2043. * rq_flush_dcache_pages - Helper function to flush all pages in a request
  2044. * @rq: the request to be flushed
  2045. *
  2046. * Description:
  2047. * Flush all pages in @rq.
  2048. */
  2049. void rq_flush_dcache_pages(struct request *rq)
  2050. {
  2051. struct req_iterator iter;
  2052. struct bio_vec *bvec;
  2053. rq_for_each_segment(bvec, rq, iter)
  2054. flush_dcache_page(bvec->bv_page);
  2055. }
  2056. EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
  2057. #endif
  2058. /**
  2059. * blk_lld_busy - Check if underlying low-level drivers of a device are busy
  2060. * @q : the queue of the device being checked
  2061. *
  2062. * Description:
  2063. * Check if underlying low-level drivers of a device are busy.
  2064. * If the drivers want to export their busy state, they must set own
  2065. * exporting function using blk_queue_lld_busy() first.
  2066. *
  2067. * Basically, this function is used only by request stacking drivers
  2068. * to stop dispatching requests to underlying devices when underlying
  2069. * devices are busy. This behavior helps more I/O merging on the queue
  2070. * of the request stacking driver and prevents I/O throughput regression
  2071. * on burst I/O load.
  2072. *
  2073. * Return:
  2074. * 0 - Not busy (The request stacking driver should dispatch request)
  2075. * 1 - Busy (The request stacking driver should stop dispatching request)
  2076. */
  2077. int blk_lld_busy(struct request_queue *q)
  2078. {
  2079. if (q->lld_busy_fn)
  2080. return q->lld_busy_fn(q);
  2081. return 0;
  2082. }
  2083. EXPORT_SYMBOL_GPL(blk_lld_busy);
  2084. /**
  2085. * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
  2086. * @rq: the clone request to be cleaned up
  2087. *
  2088. * Description:
  2089. * Free all bios in @rq for a cloned request.
  2090. */
  2091. void blk_rq_unprep_clone(struct request *rq)
  2092. {
  2093. struct bio *bio;
  2094. while ((bio = rq->bio) != NULL) {
  2095. rq->bio = bio->bi_next;
  2096. bio_put(bio);
  2097. }
  2098. }
  2099. EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
  2100. /*
  2101. * Copy attributes of the original request to the clone request.
  2102. * The actual data parts (e.g. ->cmd, ->buffer, ->sense) are not copied.
  2103. */
  2104. static void __blk_rq_prep_clone(struct request *dst, struct request *src)
  2105. {
  2106. dst->cpu = src->cpu;
  2107. dst->cmd_flags = (rq_data_dir(src) | REQ_NOMERGE);
  2108. dst->cmd_type = src->cmd_type;
  2109. dst->__sector = blk_rq_pos(src);
  2110. dst->__data_len = blk_rq_bytes(src);
  2111. dst->nr_phys_segments = src->nr_phys_segments;
  2112. dst->ioprio = src->ioprio;
  2113. dst->extra_len = src->extra_len;
  2114. }
  2115. /**
  2116. * blk_rq_prep_clone - Helper function to setup clone request
  2117. * @rq: the request to be setup
  2118. * @rq_src: original request to be cloned
  2119. * @bs: bio_set that bios for clone are allocated from
  2120. * @gfp_mask: memory allocation mask for bio
  2121. * @bio_ctr: setup function to be called for each clone bio.
  2122. * Returns %0 for success, non %0 for failure.
  2123. * @data: private data to be passed to @bio_ctr
  2124. *
  2125. * Description:
  2126. * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
  2127. * The actual data parts of @rq_src (e.g. ->cmd, ->buffer, ->sense)
  2128. * are not copied, and copying such parts is the caller's responsibility.
  2129. * Also, pages which the original bios are pointing to are not copied
  2130. * and the cloned bios just point same pages.
  2131. * So cloned bios must be completed before original bios, which means
  2132. * the caller must complete @rq before @rq_src.
  2133. */
  2134. int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
  2135. struct bio_set *bs, gfp_t gfp_mask,
  2136. int (*bio_ctr)(struct bio *, struct bio *, void *),
  2137. void *data)
  2138. {
  2139. struct bio *bio, *bio_src;
  2140. if (!bs)
  2141. bs = fs_bio_set;
  2142. blk_rq_init(NULL, rq);
  2143. __rq_for_each_bio(bio_src, rq_src) {
  2144. bio = bio_alloc_bioset(gfp_mask, bio_src->bi_max_vecs, bs);
  2145. if (!bio)
  2146. goto free_and_out;
  2147. __bio_clone(bio, bio_src);
  2148. if (bio_integrity(bio_src) &&
  2149. bio_integrity_clone(bio, bio_src, gfp_mask, bs))
  2150. goto free_and_out;
  2151. if (bio_ctr && bio_ctr(bio, bio_src, data))
  2152. goto free_and_out;
  2153. if (rq->bio) {
  2154. rq->biotail->bi_next = bio;
  2155. rq->biotail = bio;
  2156. } else
  2157. rq->bio = rq->biotail = bio;
  2158. }
  2159. __blk_rq_prep_clone(rq, rq_src);
  2160. return 0;
  2161. free_and_out:
  2162. if (bio)
  2163. bio_free(bio, bs);
  2164. blk_rq_unprep_clone(rq);
  2165. return -ENOMEM;
  2166. }
  2167. EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
  2168. int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
  2169. {
  2170. return queue_work(kblockd_workqueue, work);
  2171. }
  2172. EXPORT_SYMBOL(kblockd_schedule_work);
  2173. int __init blk_dev_init(void)
  2174. {
  2175. BUILD_BUG_ON(__REQ_NR_BITS > 8 *
  2176. sizeof(((struct request *)0)->cmd_flags));
  2177. kblockd_workqueue = create_workqueue("kblockd");
  2178. if (!kblockd_workqueue)
  2179. panic("Failed to create kblockd\n");
  2180. request_cachep = kmem_cache_create("blkdev_requests",
  2181. sizeof(struct request), 0, SLAB_PANIC, NULL);
  2182. blk_requestq_cachep = kmem_cache_create("blkdev_queue",
  2183. sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
  2184. return 0;
  2185. }