blkdev.h 41 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430
  1. #ifndef _LINUX_BLKDEV_H
  2. #define _LINUX_BLKDEV_H
  3. #ifdef CONFIG_BLOCK
  4. #include <linux/sched.h>
  5. #include <linux/major.h>
  6. #include <linux/genhd.h>
  7. #include <linux/list.h>
  8. #include <linux/timer.h>
  9. #include <linux/workqueue.h>
  10. #include <linux/pagemap.h>
  11. #include <linux/backing-dev.h>
  12. #include <linux/wait.h>
  13. #include <linux/mempool.h>
  14. #include <linux/bio.h>
  15. #include <linux/stringify.h>
  16. #include <linux/gfp.h>
  17. #include <linux/bsg.h>
  18. #include <linux/smp.h>
  19. #include <asm/scatterlist.h>
  20. struct module;
  21. struct scsi_ioctl_command;
  22. struct request_queue;
  23. struct elevator_queue;
  24. struct request_pm_state;
  25. struct blk_trace;
  26. struct request;
  27. struct sg_io_hdr;
  28. struct bsg_job;
  29. #define BLKDEV_MIN_RQ 4
  30. #define BLKDEV_MAX_RQ 128 /* Default maximum */
  31. struct request;
  32. typedef void (rq_end_io_fn)(struct request *, int);
  33. struct request_list {
  34. /*
  35. * count[], starved[], and wait[] are indexed by
  36. * BLK_RW_SYNC/BLK_RW_ASYNC
  37. */
  38. int count[2];
  39. int starved[2];
  40. int elvpriv;
  41. mempool_t *rq_pool;
  42. wait_queue_head_t wait[2];
  43. };
  44. /*
  45. * request command types
  46. */
  47. enum rq_cmd_type_bits {
  48. REQ_TYPE_FS = 1, /* fs request */
  49. REQ_TYPE_BLOCK_PC, /* scsi command */
  50. REQ_TYPE_SENSE, /* sense request */
  51. REQ_TYPE_PM_SUSPEND, /* suspend request */
  52. REQ_TYPE_PM_RESUME, /* resume request */
  53. REQ_TYPE_PM_SHUTDOWN, /* shutdown request */
  54. REQ_TYPE_SPECIAL, /* driver defined type */
  55. /*
  56. * for ATA/ATAPI devices. this really doesn't belong here, ide should
  57. * use REQ_TYPE_SPECIAL and use rq->cmd[0] with the range of driver
  58. * private REQ_LB opcodes to differentiate what type of request this is
  59. */
  60. REQ_TYPE_ATA_TASKFILE,
  61. REQ_TYPE_ATA_PC,
  62. };
  63. #define BLK_MAX_CDB 16
  64. /*
  65. * try to put the fields that are referenced together in the same cacheline.
  66. * if you modify this structure, be sure to check block/blk-core.c:blk_rq_init()
  67. * as well!
  68. */
  69. struct request {
  70. struct list_head queuelist;
  71. struct call_single_data csd;
  72. struct request_queue *q;
  73. unsigned int cmd_flags;
  74. enum rq_cmd_type_bits cmd_type;
  75. unsigned long atomic_flags;
  76. int cpu;
  77. /* the following two fields are internal, NEVER access directly */
  78. unsigned int __data_len; /* total data len */
  79. sector_t __sector; /* sector cursor */
  80. struct bio *bio;
  81. struct bio *biotail;
  82. struct hlist_node hash; /* merge hash */
  83. /*
  84. * The rb_node is only used inside the io scheduler, requests
  85. * are pruned when moved to the dispatch queue. So let the
  86. * completion_data share space with the rb_node.
  87. */
  88. union {
  89. struct rb_node rb_node; /* sort/lookup */
  90. void *completion_data;
  91. };
  92. /*
  93. * Three pointers are available for the IO schedulers, if they need
  94. * more they have to dynamically allocate it. Flush requests are
  95. * never put on the IO scheduler. So let the flush fields share
  96. * space with the elevator data.
  97. */
  98. union {
  99. struct {
  100. struct io_cq *icq;
  101. void *priv[2];
  102. } elv;
  103. struct {
  104. unsigned int seq;
  105. struct list_head list;
  106. rq_end_io_fn *saved_end_io;
  107. } flush;
  108. };
  109. struct gendisk *rq_disk;
  110. struct hd_struct *part;
  111. unsigned long start_time;
  112. #ifdef CONFIG_BLK_CGROUP
  113. unsigned long long start_time_ns;
  114. unsigned long long io_start_time_ns; /* when passed to hardware */
  115. #endif
  116. /* Number of scatter-gather DMA addr+len pairs after
  117. * physical address coalescing is performed.
  118. */
  119. unsigned short nr_phys_segments;
  120. #if defined(CONFIG_BLK_DEV_INTEGRITY)
  121. unsigned short nr_integrity_segments;
  122. #endif
  123. unsigned short ioprio;
  124. int ref_count;
  125. void *special; /* opaque pointer available for LLD use */
  126. char *buffer; /* kaddr of the current segment if available */
  127. int tag;
  128. int errors;
  129. /*
  130. * when request is used as a packet command carrier
  131. */
  132. unsigned char __cmd[BLK_MAX_CDB];
  133. unsigned char *cmd;
  134. unsigned short cmd_len;
  135. unsigned int extra_len; /* length of alignment and padding */
  136. unsigned int sense_len;
  137. unsigned int resid_len; /* residual count */
  138. void *sense;
  139. unsigned long deadline;
  140. struct list_head timeout_list;
  141. unsigned int timeout;
  142. int retries;
  143. /*
  144. * completion callback.
  145. */
  146. rq_end_io_fn *end_io;
  147. void *end_io_data;
  148. /* for bidi */
  149. struct request *next_rq;
  150. };
  151. static inline unsigned short req_get_ioprio(struct request *req)
  152. {
  153. return req->ioprio;
  154. }
  155. /*
  156. * State information carried for REQ_TYPE_PM_SUSPEND and REQ_TYPE_PM_RESUME
  157. * requests. Some step values could eventually be made generic.
  158. */
  159. struct request_pm_state
  160. {
  161. /* PM state machine step value, currently driver specific */
  162. int pm_step;
  163. /* requested PM state value (S1, S2, S3, S4, ...) */
  164. u32 pm_state;
  165. void* data; /* for driver use */
  166. };
  167. #include <linux/elevator.h>
  168. typedef void (request_fn_proc) (struct request_queue *q);
  169. typedef void (make_request_fn) (struct request_queue *q, struct bio *bio);
  170. typedef int (prep_rq_fn) (struct request_queue *, struct request *);
  171. typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
  172. struct bio_vec;
  173. struct bvec_merge_data {
  174. struct block_device *bi_bdev;
  175. sector_t bi_sector;
  176. unsigned bi_size;
  177. unsigned long bi_rw;
  178. };
  179. typedef int (merge_bvec_fn) (struct request_queue *, struct bvec_merge_data *,
  180. struct bio_vec *);
  181. typedef void (softirq_done_fn)(struct request *);
  182. typedef int (dma_drain_needed_fn)(struct request *);
  183. typedef int (lld_busy_fn) (struct request_queue *q);
  184. typedef int (bsg_job_fn) (struct bsg_job *);
  185. enum blk_eh_timer_return {
  186. BLK_EH_NOT_HANDLED,
  187. BLK_EH_HANDLED,
  188. BLK_EH_RESET_TIMER,
  189. };
  190. typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);
  191. enum blk_queue_state {
  192. Queue_down,
  193. Queue_up,
  194. };
  195. struct blk_queue_tag {
  196. struct request **tag_index; /* map of busy tags */
  197. unsigned long *tag_map; /* bit map of free/busy tags */
  198. int busy; /* current depth */
  199. int max_depth; /* what we will send to device */
  200. int real_max_depth; /* what the array can hold */
  201. atomic_t refcnt; /* map can be shared */
  202. };
  203. #define BLK_SCSI_MAX_CMDS (256)
  204. #define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))
  205. struct queue_limits {
  206. unsigned long bounce_pfn;
  207. unsigned long seg_boundary_mask;
  208. unsigned int max_hw_sectors;
  209. unsigned int max_sectors;
  210. unsigned int max_segment_size;
  211. unsigned int physical_block_size;
  212. unsigned int alignment_offset;
  213. unsigned int io_min;
  214. unsigned int io_opt;
  215. unsigned int max_discard_sectors;
  216. unsigned int discard_granularity;
  217. unsigned int discard_alignment;
  218. unsigned short logical_block_size;
  219. unsigned short max_segments;
  220. unsigned short max_integrity_segments;
  221. unsigned char misaligned;
  222. unsigned char discard_misaligned;
  223. unsigned char cluster;
  224. unsigned char discard_zeroes_data;
  225. };
  226. struct request_queue {
  227. /*
  228. * Together with queue_head for cacheline sharing
  229. */
  230. struct list_head queue_head;
  231. struct request *last_merge;
  232. struct elevator_queue *elevator;
  233. /*
  234. * the queue request freelist, one for reads and one for writes
  235. */
  236. struct request_list rq;
  237. request_fn_proc *request_fn;
  238. make_request_fn *make_request_fn;
  239. prep_rq_fn *prep_rq_fn;
  240. unprep_rq_fn *unprep_rq_fn;
  241. merge_bvec_fn *merge_bvec_fn;
  242. softirq_done_fn *softirq_done_fn;
  243. rq_timed_out_fn *rq_timed_out_fn;
  244. dma_drain_needed_fn *dma_drain_needed;
  245. lld_busy_fn *lld_busy_fn;
  246. /*
  247. * Dispatch queue sorting
  248. */
  249. sector_t end_sector;
  250. struct request *boundary_rq;
  251. /*
  252. * Delayed queue handling
  253. */
  254. struct delayed_work delay_work;
  255. struct backing_dev_info backing_dev_info;
  256. /*
  257. * The queue owner gets to use this for whatever they like.
  258. * ll_rw_blk doesn't touch it.
  259. */
  260. void *queuedata;
  261. /*
  262. * various queue flags, see QUEUE_* below
  263. */
  264. unsigned long queue_flags;
  265. /*
  266. * ida allocated id for this queue. Used to index queues from
  267. * ioctx.
  268. */
  269. int id;
  270. /*
  271. * queue needs bounce pages for pages above this limit
  272. */
  273. gfp_t bounce_gfp;
  274. /*
  275. * protects queue structures from reentrancy. ->__queue_lock should
  276. * _never_ be used directly, it is queue private. always use
  277. * ->queue_lock.
  278. */
  279. spinlock_t __queue_lock;
  280. spinlock_t *queue_lock;
  281. /*
  282. * queue kobject
  283. */
  284. struct kobject kobj;
  285. /*
  286. * queue settings
  287. */
  288. unsigned long nr_requests; /* Max # of requests */
  289. unsigned int nr_congestion_on;
  290. unsigned int nr_congestion_off;
  291. unsigned int nr_batching;
  292. unsigned int dma_drain_size;
  293. void *dma_drain_buffer;
  294. unsigned int dma_pad_mask;
  295. unsigned int dma_alignment;
  296. struct blk_queue_tag *queue_tags;
  297. struct list_head tag_busy_list;
  298. unsigned int nr_sorted;
  299. unsigned int in_flight[2];
  300. unsigned int rq_timeout;
  301. struct timer_list timeout;
  302. struct list_head timeout_list;
  303. struct list_head icq_list;
  304. struct queue_limits limits;
  305. /*
  306. * sg stuff
  307. */
  308. unsigned int sg_timeout;
  309. unsigned int sg_reserved_size;
  310. int node;
  311. #ifdef CONFIG_BLK_DEV_IO_TRACE
  312. struct blk_trace *blk_trace;
  313. #endif
  314. /*
  315. * for flush operations
  316. */
  317. unsigned int flush_flags;
  318. unsigned int flush_not_queueable:1;
  319. unsigned int flush_queue_delayed:1;
  320. unsigned int flush_pending_idx:1;
  321. unsigned int flush_running_idx:1;
  322. unsigned long flush_pending_since;
  323. struct list_head flush_queue[2];
  324. struct list_head flush_data_in_flight;
  325. struct request flush_rq;
  326. struct mutex sysfs_lock;
  327. int bypass_depth;
  328. #if defined(CONFIG_BLK_DEV_BSG)
  329. bsg_job_fn *bsg_job_fn;
  330. int bsg_job_size;
  331. struct bsg_class_device bsg_dev;
  332. #endif
  333. #ifdef CONFIG_BLK_CGROUP
  334. struct list_head all_q_node;
  335. #endif
  336. #ifdef CONFIG_BLK_DEV_THROTTLING
  337. /* Throttle data */
  338. struct throtl_data *td;
  339. #endif
  340. };
  341. #define QUEUE_FLAG_QUEUED 1 /* uses generic tag queueing */
  342. #define QUEUE_FLAG_STOPPED 2 /* queue is stopped */
  343. #define QUEUE_FLAG_SYNCFULL 3 /* read queue has been filled */
  344. #define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */
  345. #define QUEUE_FLAG_DEAD 5 /* queue being torn down */
  346. #define QUEUE_FLAG_BYPASS 6 /* act as dumb FIFO queue */
  347. #define QUEUE_FLAG_BIDI 7 /* queue supports bidi requests */
  348. #define QUEUE_FLAG_NOMERGES 8 /* disable merge attempts */
  349. #define QUEUE_FLAG_SAME_COMP 9 /* complete on same CPU-group */
  350. #define QUEUE_FLAG_FAIL_IO 10 /* fake timeout */
  351. #define QUEUE_FLAG_STACKABLE 11 /* supports request stacking */
  352. #define QUEUE_FLAG_NONROT 12 /* non-rotational device (SSD) */
  353. #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */
  354. #define QUEUE_FLAG_IO_STAT 13 /* do IO stats */
  355. #define QUEUE_FLAG_DISCARD 14 /* supports DISCARD */
  356. #define QUEUE_FLAG_NOXMERGES 15 /* No extended merges */
  357. #define QUEUE_FLAG_ADD_RANDOM 16 /* Contributes to random pool */
  358. #define QUEUE_FLAG_SECDISCARD 17 /* supports SECDISCARD */
  359. #define QUEUE_FLAG_SAME_FORCE 18 /* force complete on same CPU */
  360. #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
  361. (1 << QUEUE_FLAG_STACKABLE) | \
  362. (1 << QUEUE_FLAG_SAME_COMP) | \
  363. (1 << QUEUE_FLAG_ADD_RANDOM))
  364. static inline int queue_is_locked(struct request_queue *q)
  365. {
  366. #ifdef CONFIG_SMP
  367. spinlock_t *lock = q->queue_lock;
  368. return lock && spin_is_locked(lock);
  369. #else
  370. return 1;
  371. #endif
  372. }
  373. static inline void queue_flag_set_unlocked(unsigned int flag,
  374. struct request_queue *q)
  375. {
  376. __set_bit(flag, &q->queue_flags);
  377. }
  378. static inline int queue_flag_test_and_clear(unsigned int flag,
  379. struct request_queue *q)
  380. {
  381. WARN_ON_ONCE(!queue_is_locked(q));
  382. if (test_bit(flag, &q->queue_flags)) {
  383. __clear_bit(flag, &q->queue_flags);
  384. return 1;
  385. }
  386. return 0;
  387. }
  388. static inline int queue_flag_test_and_set(unsigned int flag,
  389. struct request_queue *q)
  390. {
  391. WARN_ON_ONCE(!queue_is_locked(q));
  392. if (!test_bit(flag, &q->queue_flags)) {
  393. __set_bit(flag, &q->queue_flags);
  394. return 0;
  395. }
  396. return 1;
  397. }
  398. static inline void queue_flag_set(unsigned int flag, struct request_queue *q)
  399. {
  400. WARN_ON_ONCE(!queue_is_locked(q));
  401. __set_bit(flag, &q->queue_flags);
  402. }
  403. static inline void queue_flag_clear_unlocked(unsigned int flag,
  404. struct request_queue *q)
  405. {
  406. __clear_bit(flag, &q->queue_flags);
  407. }
  408. static inline int queue_in_flight(struct request_queue *q)
  409. {
  410. return q->in_flight[0] + q->in_flight[1];
  411. }
  412. static inline void queue_flag_clear(unsigned int flag, struct request_queue *q)
  413. {
  414. WARN_ON_ONCE(!queue_is_locked(q));
  415. __clear_bit(flag, &q->queue_flags);
  416. }
  417. #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
  418. #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
  419. #define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
  420. #define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
  421. #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
  422. #define blk_queue_noxmerges(q) \
  423. test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
  424. #define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
  425. #define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
  426. #define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
  427. #define blk_queue_stackable(q) \
  428. test_bit(QUEUE_FLAG_STACKABLE, &(q)->queue_flags)
  429. #define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
  430. #define blk_queue_secdiscard(q) (blk_queue_discard(q) && \
  431. test_bit(QUEUE_FLAG_SECDISCARD, &(q)->queue_flags))
  432. #define blk_noretry_request(rq) \
  433. ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
  434. REQ_FAILFAST_DRIVER))
  435. #define blk_account_rq(rq) \
  436. (((rq)->cmd_flags & REQ_STARTED) && \
  437. ((rq)->cmd_type == REQ_TYPE_FS || \
  438. ((rq)->cmd_flags & REQ_DISCARD)))
  439. #define blk_pm_request(rq) \
  440. ((rq)->cmd_type == REQ_TYPE_PM_SUSPEND || \
  441. (rq)->cmd_type == REQ_TYPE_PM_RESUME)
  442. #define blk_rq_cpu_valid(rq) ((rq)->cpu != -1)
  443. #define blk_bidi_rq(rq) ((rq)->next_rq != NULL)
  444. /* rq->queuelist of dequeued request must be list_empty() */
  445. #define blk_queued_rq(rq) (!list_empty(&(rq)->queuelist))
  446. #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
  447. #define rq_data_dir(rq) ((rq)->cmd_flags & 1)
  448. static inline unsigned int blk_queue_cluster(struct request_queue *q)
  449. {
  450. return q->limits.cluster;
  451. }
  452. /*
  453. * We regard a request as sync, if either a read or a sync write
  454. */
  455. static inline bool rw_is_sync(unsigned int rw_flags)
  456. {
  457. return !(rw_flags & REQ_WRITE) || (rw_flags & REQ_SYNC);
  458. }
  459. static inline bool rq_is_sync(struct request *rq)
  460. {
  461. return rw_is_sync(rq->cmd_flags);
  462. }
  463. static inline int blk_queue_full(struct request_queue *q, int sync)
  464. {
  465. if (sync)
  466. return test_bit(QUEUE_FLAG_SYNCFULL, &q->queue_flags);
  467. return test_bit(QUEUE_FLAG_ASYNCFULL, &q->queue_flags);
  468. }
  469. static inline void blk_set_queue_full(struct request_queue *q, int sync)
  470. {
  471. if (sync)
  472. queue_flag_set(QUEUE_FLAG_SYNCFULL, q);
  473. else
  474. queue_flag_set(QUEUE_FLAG_ASYNCFULL, q);
  475. }
  476. static inline void blk_clear_queue_full(struct request_queue *q, int sync)
  477. {
  478. if (sync)
  479. queue_flag_clear(QUEUE_FLAG_SYNCFULL, q);
  480. else
  481. queue_flag_clear(QUEUE_FLAG_ASYNCFULL, q);
  482. }
  483. /*
  484. * mergeable request must not have _NOMERGE or _BARRIER bit set, nor may
  485. * it already be started by driver.
  486. */
  487. #define RQ_NOMERGE_FLAGS \
  488. (REQ_NOMERGE | REQ_STARTED | REQ_SOFTBARRIER | REQ_FLUSH | REQ_FUA)
  489. #define rq_mergeable(rq) \
  490. (!((rq)->cmd_flags & RQ_NOMERGE_FLAGS) && \
  491. (((rq)->cmd_flags & REQ_DISCARD) || \
  492. (rq)->cmd_type == REQ_TYPE_FS))
  493. /*
  494. * q->prep_rq_fn return values
  495. */
  496. #define BLKPREP_OK 0 /* serve it */
  497. #define BLKPREP_KILL 1 /* fatal error, kill */
  498. #define BLKPREP_DEFER 2 /* leave on queue */
  499. extern unsigned long blk_max_low_pfn, blk_max_pfn;
  500. /*
  501. * standard bounce addresses:
  502. *
  503. * BLK_BOUNCE_HIGH : bounce all highmem pages
  504. * BLK_BOUNCE_ANY : don't bounce anything
  505. * BLK_BOUNCE_ISA : bounce pages above ISA DMA boundary
  506. */
  507. #if BITS_PER_LONG == 32
  508. #define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT)
  509. #else
  510. #define BLK_BOUNCE_HIGH -1ULL
  511. #endif
  512. #define BLK_BOUNCE_ANY (-1ULL)
  513. #define BLK_BOUNCE_ISA (DMA_BIT_MASK(24))
  514. /*
  515. * default timeout for SG_IO if none specified
  516. */
  517. #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ)
  518. #define BLK_MIN_SG_TIMEOUT (7 * HZ)
  519. #ifdef CONFIG_BOUNCE
  520. extern int init_emergency_isa_pool(void);
  521. extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);
  522. #else
  523. static inline int init_emergency_isa_pool(void)
  524. {
  525. return 0;
  526. }
  527. static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
  528. {
  529. }
  530. #endif /* CONFIG_MMU */
  531. struct rq_map_data {
  532. struct page **pages;
  533. int page_order;
  534. int nr_entries;
  535. unsigned long offset;
  536. int null_mapped;
  537. int from_user;
  538. };
  539. struct req_iterator {
  540. int i;
  541. struct bio *bio;
  542. };
  543. /* This should not be used directly - use rq_for_each_segment */
  544. #define for_each_bio(_bio) \
  545. for (; _bio; _bio = _bio->bi_next)
  546. #define __rq_for_each_bio(_bio, rq) \
  547. if ((rq->bio)) \
  548. for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
  549. #define rq_for_each_segment(bvl, _rq, _iter) \
  550. __rq_for_each_bio(_iter.bio, _rq) \
  551. bio_for_each_segment(bvl, _iter.bio, _iter.i)
  552. #define rq_iter_last(rq, _iter) \
  553. (_iter.bio->bi_next == NULL && _iter.i == _iter.bio->bi_vcnt-1)
  554. #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
  555. # error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
  556. #endif
  557. #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
  558. extern void rq_flush_dcache_pages(struct request *rq);
  559. #else
  560. static inline void rq_flush_dcache_pages(struct request *rq)
  561. {
  562. }
  563. #endif
  564. extern int blk_register_queue(struct gendisk *disk);
  565. extern void blk_unregister_queue(struct gendisk *disk);
  566. extern void generic_make_request(struct bio *bio);
  567. extern void blk_rq_init(struct request_queue *q, struct request *rq);
  568. extern void blk_put_request(struct request *);
  569. extern void __blk_put_request(struct request_queue *, struct request *);
  570. extern struct request *blk_get_request(struct request_queue *, int, gfp_t);
  571. extern struct request *blk_make_request(struct request_queue *, struct bio *,
  572. gfp_t);
  573. extern void blk_requeue_request(struct request_queue *, struct request *);
  574. extern void blk_add_request_payload(struct request *rq, struct page *page,
  575. unsigned int len);
  576. extern int blk_rq_check_limits(struct request_queue *q, struct request *rq);
  577. extern int blk_lld_busy(struct request_queue *q);
  578. extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
  579. struct bio_set *bs, gfp_t gfp_mask,
  580. int (*bio_ctr)(struct bio *, struct bio *, void *),
  581. void *data);
  582. extern void blk_rq_unprep_clone(struct request *rq);
  583. extern int blk_insert_cloned_request(struct request_queue *q,
  584. struct request *rq);
  585. extern void blk_delay_queue(struct request_queue *, unsigned long);
  586. extern void blk_recount_segments(struct request_queue *, struct bio *);
  587. extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
  588. extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
  589. unsigned int, void __user *);
  590. extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
  591. unsigned int, void __user *);
  592. extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
  593. struct scsi_ioctl_command __user *);
  594. extern void blk_queue_bio(struct request_queue *q, struct bio *bio);
  595. /*
  596. * A queue has just exitted congestion. Note this in the global counter of
  597. * congested queues, and wake up anyone who was waiting for requests to be
  598. * put back.
  599. */
  600. static inline void blk_clear_queue_congested(struct request_queue *q, int sync)
  601. {
  602. clear_bdi_congested(&q->backing_dev_info, sync);
  603. }
  604. /*
  605. * A queue has just entered congestion. Flag that in the queue's VM-visible
  606. * state flags and increment the global gounter of congested queues.
  607. */
  608. static inline void blk_set_queue_congested(struct request_queue *q, int sync)
  609. {
  610. set_bdi_congested(&q->backing_dev_info, sync);
  611. }
  612. extern void blk_start_queue(struct request_queue *q);
  613. extern void blk_stop_queue(struct request_queue *q);
  614. extern void blk_sync_queue(struct request_queue *q);
  615. extern void __blk_stop_queue(struct request_queue *q);
  616. extern void __blk_run_queue(struct request_queue *q);
  617. extern void blk_run_queue(struct request_queue *);
  618. extern void blk_run_queue_async(struct request_queue *q);
  619. extern int blk_rq_map_user(struct request_queue *, struct request *,
  620. struct rq_map_data *, void __user *, unsigned long,
  621. gfp_t);
  622. extern int blk_rq_unmap_user(struct bio *);
  623. extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
  624. extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
  625. struct rq_map_data *, struct sg_iovec *, int,
  626. unsigned int, gfp_t);
  627. extern int blk_execute_rq(struct request_queue *, struct gendisk *,
  628. struct request *, int);
  629. extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
  630. struct request *, int, rq_end_io_fn *);
  631. static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
  632. {
  633. return bdev->bd_disk->queue;
  634. }
  635. /*
  636. * blk_rq_pos() : the current sector
  637. * blk_rq_bytes() : bytes left in the entire request
  638. * blk_rq_cur_bytes() : bytes left in the current segment
  639. * blk_rq_err_bytes() : bytes left till the next error boundary
  640. * blk_rq_sectors() : sectors left in the entire request
  641. * blk_rq_cur_sectors() : sectors left in the current segment
  642. */
  643. static inline sector_t blk_rq_pos(const struct request *rq)
  644. {
  645. return rq->__sector;
  646. }
  647. static inline unsigned int blk_rq_bytes(const struct request *rq)
  648. {
  649. return rq->__data_len;
  650. }
  651. static inline int blk_rq_cur_bytes(const struct request *rq)
  652. {
  653. return rq->bio ? bio_cur_bytes(rq->bio) : 0;
  654. }
  655. extern unsigned int blk_rq_err_bytes(const struct request *rq);
  656. static inline unsigned int blk_rq_sectors(const struct request *rq)
  657. {
  658. return blk_rq_bytes(rq) >> 9;
  659. }
  660. static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
  661. {
  662. return blk_rq_cur_bytes(rq) >> 9;
  663. }
  664. /*
  665. * Request issue related functions.
  666. */
  667. extern struct request *blk_peek_request(struct request_queue *q);
  668. extern void blk_start_request(struct request *rq);
  669. extern struct request *blk_fetch_request(struct request_queue *q);
  670. /*
  671. * Request completion related functions.
  672. *
  673. * blk_update_request() completes given number of bytes and updates
  674. * the request without completing it.
  675. *
  676. * blk_end_request() and friends. __blk_end_request() must be called
  677. * with the request queue spinlock acquired.
  678. *
  679. * Several drivers define their own end_request and call
  680. * blk_end_request() for parts of the original function.
  681. * This prevents code duplication in drivers.
  682. */
  683. extern bool blk_update_request(struct request *rq, int error,
  684. unsigned int nr_bytes);
  685. extern bool blk_end_request(struct request *rq, int error,
  686. unsigned int nr_bytes);
  687. extern void blk_end_request_all(struct request *rq, int error);
  688. extern bool blk_end_request_cur(struct request *rq, int error);
  689. extern bool blk_end_request_err(struct request *rq, int error);
  690. extern bool __blk_end_request(struct request *rq, int error,
  691. unsigned int nr_bytes);
  692. extern void __blk_end_request_all(struct request *rq, int error);
  693. extern bool __blk_end_request_cur(struct request *rq, int error);
  694. extern bool __blk_end_request_err(struct request *rq, int error);
  695. extern void blk_complete_request(struct request *);
  696. extern void __blk_complete_request(struct request *);
  697. extern void blk_abort_request(struct request *);
  698. extern void blk_abort_queue(struct request_queue *);
  699. extern void blk_unprep_request(struct request *);
  700. /*
  701. * Access functions for manipulating queue properties
  702. */
  703. extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
  704. spinlock_t *lock, int node_id);
  705. extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
  706. extern struct request_queue *blk_init_allocated_queue(struct request_queue *,
  707. request_fn_proc *, spinlock_t *);
  708. extern void blk_cleanup_queue(struct request_queue *);
  709. extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
  710. extern void blk_queue_bounce_limit(struct request_queue *, u64);
  711. extern void blk_limits_max_hw_sectors(struct queue_limits *, unsigned int);
  712. extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
  713. extern void blk_queue_max_segments(struct request_queue *, unsigned short);
  714. extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
  715. extern void blk_queue_max_discard_sectors(struct request_queue *q,
  716. unsigned int max_discard_sectors);
  717. extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
  718. extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
  719. extern void blk_queue_alignment_offset(struct request_queue *q,
  720. unsigned int alignment);
  721. extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
  722. extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
  723. extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
  724. extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
  725. extern void blk_set_default_limits(struct queue_limits *lim);
  726. extern void blk_set_stacking_limits(struct queue_limits *lim);
  727. extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
  728. sector_t offset);
  729. extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
  730. sector_t offset);
  731. extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
  732. sector_t offset);
  733. extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
  734. extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
  735. extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
  736. extern int blk_queue_dma_drain(struct request_queue *q,
  737. dma_drain_needed_fn *dma_drain_needed,
  738. void *buf, unsigned int size);
  739. extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
  740. extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
  741. extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
  742. extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
  743. extern void blk_queue_merge_bvec(struct request_queue *, merge_bvec_fn *);
  744. extern void blk_queue_dma_alignment(struct request_queue *, int);
  745. extern void blk_queue_update_dma_alignment(struct request_queue *, int);
  746. extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
  747. extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
  748. extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
  749. extern void blk_queue_flush(struct request_queue *q, unsigned int flush);
  750. extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
  751. extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
  752. extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
  753. extern void blk_dump_rq_flags(struct request *, char *);
  754. extern long nr_blockdev_pages(void);
  755. bool __must_check blk_get_queue(struct request_queue *);
  756. struct request_queue *blk_alloc_queue(gfp_t);
  757. struct request_queue *blk_alloc_queue_node(gfp_t, int);
  758. extern void blk_put_queue(struct request_queue *);
  759. /*
  760. * blk_plug permits building a queue of related requests by holding the I/O
  761. * fragments for a short period. This allows merging of sequential requests
  762. * into single larger request. As the requests are moved from a per-task list to
  763. * the device's request_queue in a batch, this results in improved scalability
  764. * as the lock contention for request_queue lock is reduced.
  765. *
  766. * It is ok not to disable preemption when adding the request to the plug list
  767. * or when attempting a merge, because blk_schedule_flush_list() will only flush
  768. * the plug list when the task sleeps by itself. For details, please see
  769. * schedule() where blk_schedule_flush_plug() is called.
  770. */
  771. struct blk_plug {
  772. unsigned long magic; /* detect uninitialized use-cases */
  773. struct list_head list; /* requests */
  774. struct list_head cb_list; /* md requires an unplug callback */
  775. unsigned int should_sort; /* list to be sorted before flushing? */
  776. };
  777. #define BLK_MAX_REQUEST_COUNT 16
  778. struct blk_plug_cb {
  779. struct list_head list;
  780. void (*callback)(struct blk_plug_cb *);
  781. };
  782. extern void blk_start_plug(struct blk_plug *);
  783. extern void blk_finish_plug(struct blk_plug *);
  784. extern void blk_flush_plug_list(struct blk_plug *, bool);
  785. static inline void blk_flush_plug(struct task_struct *tsk)
  786. {
  787. struct blk_plug *plug = tsk->plug;
  788. if (plug)
  789. blk_flush_plug_list(plug, false);
  790. }
  791. static inline void blk_schedule_flush_plug(struct task_struct *tsk)
  792. {
  793. struct blk_plug *plug = tsk->plug;
  794. if (plug)
  795. blk_flush_plug_list(plug, true);
  796. }
  797. static inline bool blk_needs_flush_plug(struct task_struct *tsk)
  798. {
  799. struct blk_plug *plug = tsk->plug;
  800. return plug && (!list_empty(&plug->list) || !list_empty(&plug->cb_list));
  801. }
  802. /*
  803. * tag stuff
  804. */
  805. #define blk_rq_tagged(rq) ((rq)->cmd_flags & REQ_QUEUED)
  806. extern int blk_queue_start_tag(struct request_queue *, struct request *);
  807. extern struct request *blk_queue_find_tag(struct request_queue *, int);
  808. extern void blk_queue_end_tag(struct request_queue *, struct request *);
  809. extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *);
  810. extern void blk_queue_free_tags(struct request_queue *);
  811. extern int blk_queue_resize_tags(struct request_queue *, int);
  812. extern void blk_queue_invalidate_tags(struct request_queue *);
  813. extern struct blk_queue_tag *blk_init_tags(int);
  814. extern void blk_free_tags(struct blk_queue_tag *);
  815. static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt,
  816. int tag)
  817. {
  818. if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
  819. return NULL;
  820. return bqt->tag_index[tag];
  821. }
  822. #define BLKDEV_DISCARD_SECURE 0x01 /* secure discard */
  823. extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *);
  824. extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
  825. sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
  826. extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
  827. sector_t nr_sects, gfp_t gfp_mask);
  828. static inline int sb_issue_discard(struct super_block *sb, sector_t block,
  829. sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
  830. {
  831. return blkdev_issue_discard(sb->s_bdev, block << (sb->s_blocksize_bits - 9),
  832. nr_blocks << (sb->s_blocksize_bits - 9),
  833. gfp_mask, flags);
  834. }
  835. static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
  836. sector_t nr_blocks, gfp_t gfp_mask)
  837. {
  838. return blkdev_issue_zeroout(sb->s_bdev,
  839. block << (sb->s_blocksize_bits - 9),
  840. nr_blocks << (sb->s_blocksize_bits - 9),
  841. gfp_mask);
  842. }
  843. extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm);
  844. enum blk_default_limits {
  845. BLK_MAX_SEGMENTS = 128,
  846. BLK_SAFE_MAX_SECTORS = 255,
  847. BLK_DEF_MAX_SECTORS = 1024,
  848. BLK_MAX_SEGMENT_SIZE = 65536,
  849. BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
  850. };
  851. #define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist)
  852. static inline unsigned long queue_bounce_pfn(struct request_queue *q)
  853. {
  854. return q->limits.bounce_pfn;
  855. }
  856. static inline unsigned long queue_segment_boundary(struct request_queue *q)
  857. {
  858. return q->limits.seg_boundary_mask;
  859. }
  860. static inline unsigned int queue_max_sectors(struct request_queue *q)
  861. {
  862. return q->limits.max_sectors;
  863. }
  864. static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
  865. {
  866. return q->limits.max_hw_sectors;
  867. }
  868. static inline unsigned short queue_max_segments(struct request_queue *q)
  869. {
  870. return q->limits.max_segments;
  871. }
  872. static inline unsigned int queue_max_segment_size(struct request_queue *q)
  873. {
  874. return q->limits.max_segment_size;
  875. }
  876. static inline unsigned short queue_logical_block_size(struct request_queue *q)
  877. {
  878. int retval = 512;
  879. if (q && q->limits.logical_block_size)
  880. retval = q->limits.logical_block_size;
  881. return retval;
  882. }
  883. static inline unsigned short bdev_logical_block_size(struct block_device *bdev)
  884. {
  885. return queue_logical_block_size(bdev_get_queue(bdev));
  886. }
  887. static inline unsigned int queue_physical_block_size(struct request_queue *q)
  888. {
  889. return q->limits.physical_block_size;
  890. }
  891. static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
  892. {
  893. return queue_physical_block_size(bdev_get_queue(bdev));
  894. }
  895. static inline unsigned int queue_io_min(struct request_queue *q)
  896. {
  897. return q->limits.io_min;
  898. }
  899. static inline int bdev_io_min(struct block_device *bdev)
  900. {
  901. return queue_io_min(bdev_get_queue(bdev));
  902. }
  903. static inline unsigned int queue_io_opt(struct request_queue *q)
  904. {
  905. return q->limits.io_opt;
  906. }
  907. static inline int bdev_io_opt(struct block_device *bdev)
  908. {
  909. return queue_io_opt(bdev_get_queue(bdev));
  910. }
  911. static inline int queue_alignment_offset(struct request_queue *q)
  912. {
  913. if (q->limits.misaligned)
  914. return -1;
  915. return q->limits.alignment_offset;
  916. }
  917. static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
  918. {
  919. unsigned int granularity = max(lim->physical_block_size, lim->io_min);
  920. unsigned int alignment = (sector << 9) & (granularity - 1);
  921. return (granularity + lim->alignment_offset - alignment)
  922. & (granularity - 1);
  923. }
  924. static inline int bdev_alignment_offset(struct block_device *bdev)
  925. {
  926. struct request_queue *q = bdev_get_queue(bdev);
  927. if (q->limits.misaligned)
  928. return -1;
  929. if (bdev != bdev->bd_contains)
  930. return bdev->bd_part->alignment_offset;
  931. return q->limits.alignment_offset;
  932. }
  933. static inline int queue_discard_alignment(struct request_queue *q)
  934. {
  935. if (q->limits.discard_misaligned)
  936. return -1;
  937. return q->limits.discard_alignment;
  938. }
  939. static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
  940. {
  941. unsigned int alignment = (sector << 9) & (lim->discard_granularity - 1);
  942. if (!lim->max_discard_sectors)
  943. return 0;
  944. return (lim->discard_granularity + lim->discard_alignment - alignment)
  945. & (lim->discard_granularity - 1);
  946. }
  947. static inline unsigned int queue_discard_zeroes_data(struct request_queue *q)
  948. {
  949. if (q->limits.max_discard_sectors && q->limits.discard_zeroes_data == 1)
  950. return 1;
  951. return 0;
  952. }
  953. static inline unsigned int bdev_discard_zeroes_data(struct block_device *bdev)
  954. {
  955. return queue_discard_zeroes_data(bdev_get_queue(bdev));
  956. }
  957. static inline int queue_dma_alignment(struct request_queue *q)
  958. {
  959. return q ? q->dma_alignment : 511;
  960. }
  961. static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
  962. unsigned int len)
  963. {
  964. unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
  965. return !(addr & alignment) && !(len & alignment);
  966. }
  967. /* assumes size > 256 */
  968. static inline unsigned int blksize_bits(unsigned int size)
  969. {
  970. unsigned int bits = 8;
  971. do {
  972. bits++;
  973. size >>= 1;
  974. } while (size > 256);
  975. return bits;
  976. }
  977. static inline unsigned int block_size(struct block_device *bdev)
  978. {
  979. return bdev->bd_block_size;
  980. }
  981. static inline bool queue_flush_queueable(struct request_queue *q)
  982. {
  983. return !q->flush_not_queueable;
  984. }
  985. typedef struct {struct page *v;} Sector;
  986. unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);
  987. static inline void put_dev_sector(Sector p)
  988. {
  989. page_cache_release(p.v);
  990. }
  991. struct work_struct;
  992. int kblockd_schedule_work(struct request_queue *q, struct work_struct *work);
  993. #ifdef CONFIG_BLK_CGROUP
  994. /*
  995. * This should not be using sched_clock(). A real patch is in progress
  996. * to fix this up, until that is in place we need to disable preemption
  997. * around sched_clock() in this function and set_io_start_time_ns().
  998. */
  999. static inline void set_start_time_ns(struct request *req)
  1000. {
  1001. preempt_disable();
  1002. req->start_time_ns = sched_clock();
  1003. preempt_enable();
  1004. }
  1005. static inline void set_io_start_time_ns(struct request *req)
  1006. {
  1007. preempt_disable();
  1008. req->io_start_time_ns = sched_clock();
  1009. preempt_enable();
  1010. }
  1011. static inline uint64_t rq_start_time_ns(struct request *req)
  1012. {
  1013. return req->start_time_ns;
  1014. }
  1015. static inline uint64_t rq_io_start_time_ns(struct request *req)
  1016. {
  1017. return req->io_start_time_ns;
  1018. }
  1019. #else
  1020. static inline void set_start_time_ns(struct request *req) {}
  1021. static inline void set_io_start_time_ns(struct request *req) {}
  1022. static inline uint64_t rq_start_time_ns(struct request *req)
  1023. {
  1024. return 0;
  1025. }
  1026. static inline uint64_t rq_io_start_time_ns(struct request *req)
  1027. {
  1028. return 0;
  1029. }
  1030. #endif
  1031. #define MODULE_ALIAS_BLOCKDEV(major,minor) \
  1032. MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
  1033. #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
  1034. MODULE_ALIAS("block-major-" __stringify(major) "-*")
  1035. #if defined(CONFIG_BLK_DEV_INTEGRITY)
  1036. #define INTEGRITY_FLAG_READ 2 /* verify data integrity on read */
  1037. #define INTEGRITY_FLAG_WRITE 4 /* generate data integrity on write */
  1038. struct blk_integrity_exchg {
  1039. void *prot_buf;
  1040. void *data_buf;
  1041. sector_t sector;
  1042. unsigned int data_size;
  1043. unsigned short sector_size;
  1044. const char *disk_name;
  1045. };
  1046. typedef void (integrity_gen_fn) (struct blk_integrity_exchg *);
  1047. typedef int (integrity_vrfy_fn) (struct blk_integrity_exchg *);
  1048. typedef void (integrity_set_tag_fn) (void *, void *, unsigned int);
  1049. typedef void (integrity_get_tag_fn) (void *, void *, unsigned int);
  1050. struct blk_integrity {
  1051. integrity_gen_fn *generate_fn;
  1052. integrity_vrfy_fn *verify_fn;
  1053. integrity_set_tag_fn *set_tag_fn;
  1054. integrity_get_tag_fn *get_tag_fn;
  1055. unsigned short flags;
  1056. unsigned short tuple_size;
  1057. unsigned short sector_size;
  1058. unsigned short tag_size;
  1059. const char *name;
  1060. struct kobject kobj;
  1061. };
  1062. extern bool blk_integrity_is_initialized(struct gendisk *);
  1063. extern int blk_integrity_register(struct gendisk *, struct blk_integrity *);
  1064. extern void blk_integrity_unregister(struct gendisk *);
  1065. extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
  1066. extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
  1067. struct scatterlist *);
  1068. extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
  1069. extern int blk_integrity_merge_rq(struct request_queue *, struct request *,
  1070. struct request *);
  1071. extern int blk_integrity_merge_bio(struct request_queue *, struct request *,
  1072. struct bio *);
  1073. static inline
  1074. struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
  1075. {
  1076. return bdev->bd_disk->integrity;
  1077. }
  1078. static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
  1079. {
  1080. return disk->integrity;
  1081. }
  1082. static inline int blk_integrity_rq(struct request *rq)
  1083. {
  1084. if (rq->bio == NULL)
  1085. return 0;
  1086. return bio_integrity(rq->bio);
  1087. }
  1088. static inline void blk_queue_max_integrity_segments(struct request_queue *q,
  1089. unsigned int segs)
  1090. {
  1091. q->limits.max_integrity_segments = segs;
  1092. }
  1093. static inline unsigned short
  1094. queue_max_integrity_segments(struct request_queue *q)
  1095. {
  1096. return q->limits.max_integrity_segments;
  1097. }
  1098. #else /* CONFIG_BLK_DEV_INTEGRITY */
  1099. struct bio;
  1100. struct block_device;
  1101. struct gendisk;
  1102. struct blk_integrity;
  1103. static inline int blk_integrity_rq(struct request *rq)
  1104. {
  1105. return 0;
  1106. }
  1107. static inline int blk_rq_count_integrity_sg(struct request_queue *q,
  1108. struct bio *b)
  1109. {
  1110. return 0;
  1111. }
  1112. static inline int blk_rq_map_integrity_sg(struct request_queue *q,
  1113. struct bio *b,
  1114. struct scatterlist *s)
  1115. {
  1116. return 0;
  1117. }
  1118. static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
  1119. {
  1120. return 0;
  1121. }
  1122. static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
  1123. {
  1124. return NULL;
  1125. }
  1126. static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
  1127. {
  1128. return 0;
  1129. }
  1130. static inline int blk_integrity_register(struct gendisk *d,
  1131. struct blk_integrity *b)
  1132. {
  1133. return 0;
  1134. }
  1135. static inline void blk_integrity_unregister(struct gendisk *d)
  1136. {
  1137. }
  1138. static inline void blk_queue_max_integrity_segments(struct request_queue *q,
  1139. unsigned int segs)
  1140. {
  1141. }
  1142. static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
  1143. {
  1144. return 0;
  1145. }
  1146. static inline int blk_integrity_merge_rq(struct request_queue *rq,
  1147. struct request *r1,
  1148. struct request *r2)
  1149. {
  1150. return 0;
  1151. }
  1152. static inline int blk_integrity_merge_bio(struct request_queue *rq,
  1153. struct request *r,
  1154. struct bio *b)
  1155. {
  1156. return 0;
  1157. }
  1158. static inline bool blk_integrity_is_initialized(struct gendisk *g)
  1159. {
  1160. return 0;
  1161. }
  1162. #endif /* CONFIG_BLK_DEV_INTEGRITY */
  1163. struct block_device_operations {
  1164. int (*open) (struct block_device *, fmode_t);
  1165. int (*release) (struct gendisk *, fmode_t);
  1166. int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
  1167. int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
  1168. int (*direct_access) (struct block_device *, sector_t,
  1169. void **, unsigned long *);
  1170. unsigned int (*check_events) (struct gendisk *disk,
  1171. unsigned int clearing);
  1172. /* ->media_changed() is DEPRECATED, use ->check_events() instead */
  1173. int (*media_changed) (struct gendisk *);
  1174. void (*unlock_native_capacity) (struct gendisk *);
  1175. int (*revalidate_disk) (struct gendisk *);
  1176. int (*getgeo)(struct block_device *, struct hd_geometry *);
  1177. /* this callback is with swap_lock and sometimes page table lock held */
  1178. void (*swap_slot_free_notify) (struct block_device *, unsigned long);
  1179. struct module *owner;
  1180. };
  1181. extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
  1182. unsigned long);
  1183. #else /* CONFIG_BLOCK */
  1184. /*
  1185. * stubs for when the block layer is configured out
  1186. */
  1187. #define buffer_heads_over_limit 0
  1188. static inline long nr_blockdev_pages(void)
  1189. {
  1190. return 0;
  1191. }
  1192. struct blk_plug {
  1193. };
  1194. static inline void blk_start_plug(struct blk_plug *plug)
  1195. {
  1196. }
  1197. static inline void blk_finish_plug(struct blk_plug *plug)
  1198. {
  1199. }
  1200. static inline void blk_flush_plug(struct task_struct *task)
  1201. {
  1202. }
  1203. static inline void blk_schedule_flush_plug(struct task_struct *task)
  1204. {
  1205. }
  1206. static inline bool blk_needs_flush_plug(struct task_struct *tsk)
  1207. {
  1208. return false;
  1209. }
  1210. #endif /* CONFIG_BLOCK */
  1211. #endif