blktrace.c 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515
  1. /*
  2. * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of the GNU General Public License version 2 as
  6. * published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. * GNU General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public License
  14. * along with this program; if not, write to the Free Software
  15. * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  16. *
  17. */
  18. #include <linux/kernel.h>
  19. #include <linux/blkdev.h>
  20. #include <linux/blktrace_api.h>
  21. #include <linux/percpu.h>
  22. #include <linux/init.h>
  23. #include <linux/mutex.h>
  24. #include <linux/debugfs.h>
  25. #include <linux/time.h>
  26. #include <trace/block.h>
  27. #include <linux/uaccess.h>
  28. #include "trace_output.h"
  29. static unsigned int blktrace_seq __read_mostly = 1;
  30. static struct trace_array *blk_tr;
  31. static bool blk_tracer_enabled __read_mostly;
  32. /* Select an alternative, minimalistic output than the original one */
  33. #define TRACE_BLK_OPT_CLASSIC 0x1
  34. static struct tracer_opt blk_tracer_opts[] = {
  35. /* Default disable the minimalistic output */
  36. { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) },
  37. { }
  38. };
  39. static struct tracer_flags blk_tracer_flags = {
  40. .val = 0,
  41. .opts = blk_tracer_opts,
  42. };
  43. /* Global reference count of probes */
  44. static atomic_t blk_probes_ref = ATOMIC_INIT(0);
  45. static void blk_register_tracepoints(void);
  46. static void blk_unregister_tracepoints(void);
  47. /*
  48. * Send out a notify message.
  49. */
  50. static void trace_note(struct blk_trace *bt, pid_t pid, int action,
  51. const void *data, size_t len)
  52. {
  53. struct blk_io_trace *t;
  54. if (!bt->rchan)
  55. return;
  56. t = relay_reserve(bt->rchan, sizeof(*t) + len);
  57. if (t) {
  58. const int cpu = smp_processor_id();
  59. t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
  60. t->time = ktime_to_ns(ktime_get());
  61. t->device = bt->dev;
  62. t->action = action;
  63. t->pid = pid;
  64. t->cpu = cpu;
  65. t->pdu_len = len;
  66. memcpy((void *) t + sizeof(*t), data, len);
  67. }
  68. }
  69. /*
  70. * Send out a notify for this process, if we haven't done so since a trace
  71. * started
  72. */
  73. static void trace_note_tsk(struct blk_trace *bt, struct task_struct *tsk)
  74. {
  75. tsk->btrace_seq = blktrace_seq;
  76. trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, sizeof(tsk->comm));
  77. }
  78. static void trace_note_time(struct blk_trace *bt)
  79. {
  80. struct timespec now;
  81. unsigned long flags;
  82. u32 words[2];
  83. getnstimeofday(&now);
  84. words[0] = now.tv_sec;
  85. words[1] = now.tv_nsec;
  86. local_irq_save(flags);
  87. trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words));
  88. local_irq_restore(flags);
  89. }
  90. void __trace_note_message(struct blk_trace *bt, const char *fmt, ...)
  91. {
  92. int n;
  93. va_list args;
  94. unsigned long flags;
  95. char *buf;
  96. if (blk_tracer_enabled) {
  97. va_start(args, fmt);
  98. ftrace_vprintk(fmt, args);
  99. va_end(args);
  100. return;
  101. }
  102. if (!bt->msg_data)
  103. return;
  104. local_irq_save(flags);
  105. buf = per_cpu_ptr(bt->msg_data, smp_processor_id());
  106. va_start(args, fmt);
  107. n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args);
  108. va_end(args);
  109. trace_note(bt, 0, BLK_TN_MESSAGE, buf, n);
  110. local_irq_restore(flags);
  111. }
  112. EXPORT_SYMBOL_GPL(__trace_note_message);
  113. static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
  114. pid_t pid)
  115. {
  116. if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
  117. return 1;
  118. if (sector < bt->start_lba || sector > bt->end_lba)
  119. return 1;
  120. if (bt->pid && pid != bt->pid)
  121. return 1;
  122. return 0;
  123. }
  124. /*
  125. * Data direction bit lookup
  126. */
  127. static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ),
  128. BLK_TC_ACT(BLK_TC_WRITE) };
  129. /* The ilog2() calls fall out because they're constant */
  130. #define MASK_TC_BIT(rw, __name) ((rw & (1 << BIO_RW_ ## __name)) << \
  131. (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - BIO_RW_ ## __name))
  132. /*
  133. * The worker for the various blk_add_trace*() types. Fills out a
  134. * blk_io_trace structure and places it in a per-cpu subbuffer.
  135. */
  136. static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
  137. int rw, u32 what, int error, int pdu_len, void *pdu_data)
  138. {
  139. struct task_struct *tsk = current;
  140. struct ring_buffer_event *event = NULL;
  141. struct blk_io_trace *t;
  142. unsigned long flags = 0;
  143. unsigned long *sequence;
  144. pid_t pid;
  145. int cpu, pc = 0;
  146. if (unlikely(bt->trace_state != Blktrace_running &&
  147. !blk_tracer_enabled))
  148. return;
  149. what |= ddir_act[rw & WRITE];
  150. what |= MASK_TC_BIT(rw, BARRIER);
  151. what |= MASK_TC_BIT(rw, SYNCIO);
  152. what |= MASK_TC_BIT(rw, AHEAD);
  153. what |= MASK_TC_BIT(rw, META);
  154. what |= MASK_TC_BIT(rw, DISCARD);
  155. pid = tsk->pid;
  156. if (unlikely(act_log_check(bt, what, sector, pid)))
  157. return;
  158. cpu = raw_smp_processor_id();
  159. if (blk_tracer_enabled) {
  160. tracing_record_cmdline(current);
  161. pc = preempt_count();
  162. event = trace_buffer_lock_reserve(blk_tr, TRACE_BLK,
  163. sizeof(*t) + pdu_len,
  164. 0, pc);
  165. if (!event)
  166. return;
  167. t = ring_buffer_event_data(event);
  168. goto record_it;
  169. }
  170. /*
  171. * A word about the locking here - we disable interrupts to reserve
  172. * some space in the relay per-cpu buffer, to prevent an irq
  173. * from coming in and stepping on our toes.
  174. */
  175. local_irq_save(flags);
  176. if (unlikely(tsk->btrace_seq != blktrace_seq))
  177. trace_note_tsk(bt, tsk);
  178. t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);
  179. if (t) {
  180. sequence = per_cpu_ptr(bt->sequence, cpu);
  181. t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
  182. t->sequence = ++(*sequence);
  183. t->time = ktime_to_ns(ktime_get());
  184. record_it:
  185. /*
  186. * These two are not needed in ftrace as they are in the
  187. * generic trace_entry, filled by tracing_generic_entry_update,
  188. * but for the trace_event->bin() synthesizer benefit we do it
  189. * here too.
  190. */
  191. t->cpu = cpu;
  192. t->pid = pid;
  193. t->sector = sector;
  194. t->bytes = bytes;
  195. t->action = what;
  196. t->device = bt->dev;
  197. t->error = error;
  198. t->pdu_len = pdu_len;
  199. if (pdu_len)
  200. memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);
  201. if (blk_tracer_enabled) {
  202. trace_buffer_unlock_commit(blk_tr, event, 0, pc);
  203. return;
  204. }
  205. }
  206. local_irq_restore(flags);
  207. }
  208. static struct dentry *blk_tree_root;
  209. static DEFINE_MUTEX(blk_tree_mutex);
  210. static void blk_trace_cleanup(struct blk_trace *bt)
  211. {
  212. debugfs_remove(bt->msg_file);
  213. debugfs_remove(bt->dropped_file);
  214. relay_close(bt->rchan);
  215. free_percpu(bt->sequence);
  216. free_percpu(bt->msg_data);
  217. kfree(bt);
  218. if (atomic_dec_and_test(&blk_probes_ref))
  219. blk_unregister_tracepoints();
  220. }
  221. int blk_trace_remove(struct request_queue *q)
  222. {
  223. struct blk_trace *bt;
  224. bt = xchg(&q->blk_trace, NULL);
  225. if (!bt)
  226. return -EINVAL;
  227. if (bt->trace_state != Blktrace_running)
  228. blk_trace_cleanup(bt);
  229. return 0;
  230. }
  231. EXPORT_SYMBOL_GPL(blk_trace_remove);
  232. static int blk_dropped_open(struct inode *inode, struct file *filp)
  233. {
  234. filp->private_data = inode->i_private;
  235. return 0;
  236. }
  237. static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
  238. size_t count, loff_t *ppos)
  239. {
  240. struct blk_trace *bt = filp->private_data;
  241. char buf[16];
  242. snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
  243. return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
  244. }
  245. static const struct file_operations blk_dropped_fops = {
  246. .owner = THIS_MODULE,
  247. .open = blk_dropped_open,
  248. .read = blk_dropped_read,
  249. };
  250. static int blk_msg_open(struct inode *inode, struct file *filp)
  251. {
  252. filp->private_data = inode->i_private;
  253. return 0;
  254. }
  255. static ssize_t blk_msg_write(struct file *filp, const char __user *buffer,
  256. size_t count, loff_t *ppos)
  257. {
  258. char *msg;
  259. struct blk_trace *bt;
  260. if (count > BLK_TN_MAX_MSG)
  261. return -EINVAL;
  262. msg = kmalloc(count, GFP_KERNEL);
  263. if (msg == NULL)
  264. return -ENOMEM;
  265. if (copy_from_user(msg, buffer, count)) {
  266. kfree(msg);
  267. return -EFAULT;
  268. }
  269. bt = filp->private_data;
  270. __trace_note_message(bt, "%s", msg);
  271. kfree(msg);
  272. return count;
  273. }
  274. static const struct file_operations blk_msg_fops = {
  275. .owner = THIS_MODULE,
  276. .open = blk_msg_open,
  277. .write = blk_msg_write,
  278. };
  279. /*
  280. * Keep track of how many times we encountered a full subbuffer, to aid
  281. * the user space app in telling how many lost events there were.
  282. */
  283. static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
  284. void *prev_subbuf, size_t prev_padding)
  285. {
  286. struct blk_trace *bt;
  287. if (!relay_buf_full(buf))
  288. return 1;
  289. bt = buf->chan->private_data;
  290. atomic_inc(&bt->dropped);
  291. return 0;
  292. }
  293. static int blk_remove_buf_file_callback(struct dentry *dentry)
  294. {
  295. struct dentry *parent = dentry->d_parent;
  296. debugfs_remove(dentry);
  297. /*
  298. * this will fail for all but the last file, but that is ok. what we
  299. * care about is the top level buts->name directory going away, when
  300. * the last trace file is gone. Then we don't have to rmdir() that
  301. * manually on trace stop, so it nicely solves the issue with
  302. * force killing of running traces.
  303. */
  304. debugfs_remove(parent);
  305. return 0;
  306. }
  307. static struct dentry *blk_create_buf_file_callback(const char *filename,
  308. struct dentry *parent,
  309. int mode,
  310. struct rchan_buf *buf,
  311. int *is_global)
  312. {
  313. return debugfs_create_file(filename, mode, parent, buf,
  314. &relay_file_operations);
  315. }
  316. static struct rchan_callbacks blk_relay_callbacks = {
  317. .subbuf_start = blk_subbuf_start_callback,
  318. .create_buf_file = blk_create_buf_file_callback,
  319. .remove_buf_file = blk_remove_buf_file_callback,
  320. };
  321. /*
  322. * Setup everything required to start tracing
  323. */
  324. int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
  325. struct blk_user_trace_setup *buts)
  326. {
  327. struct blk_trace *old_bt, *bt = NULL;
  328. struct dentry *dir = NULL;
  329. int ret, i;
  330. if (!buts->buf_size || !buts->buf_nr)
  331. return -EINVAL;
  332. strncpy(buts->name, name, BLKTRACE_BDEV_SIZE);
  333. buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0';
  334. /*
  335. * some device names have larger paths - convert the slashes
  336. * to underscores for this to work as expected
  337. */
  338. for (i = 0; i < strlen(buts->name); i++)
  339. if (buts->name[i] == '/')
  340. buts->name[i] = '_';
  341. ret = -ENOMEM;
  342. bt = kzalloc(sizeof(*bt), GFP_KERNEL);
  343. if (!bt)
  344. goto err;
  345. bt->sequence = alloc_percpu(unsigned long);
  346. if (!bt->sequence)
  347. goto err;
  348. bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));
  349. if (!bt->msg_data)
  350. goto err;
  351. ret = -ENOENT;
  352. mutex_lock(&blk_tree_mutex);
  353. if (!blk_tree_root) {
  354. blk_tree_root = debugfs_create_dir("block", NULL);
  355. if (!blk_tree_root) {
  356. mutex_unlock(&blk_tree_mutex);
  357. goto err;
  358. }
  359. }
  360. mutex_unlock(&blk_tree_mutex);
  361. dir = debugfs_create_dir(buts->name, blk_tree_root);
  362. if (!dir)
  363. goto err;
  364. bt->dir = dir;
  365. bt->dev = dev;
  366. atomic_set(&bt->dropped, 0);
  367. ret = -EIO;
  368. bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt,
  369. &blk_dropped_fops);
  370. if (!bt->dropped_file)
  371. goto err;
  372. bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);
  373. if (!bt->msg_file)
  374. goto err;
  375. bt->rchan = relay_open("trace", dir, buts->buf_size,
  376. buts->buf_nr, &blk_relay_callbacks, bt);
  377. if (!bt->rchan)
  378. goto err;
  379. bt->act_mask = buts->act_mask;
  380. if (!bt->act_mask)
  381. bt->act_mask = (u16) -1;
  382. bt->start_lba = buts->start_lba;
  383. bt->end_lba = buts->end_lba;
  384. if (!bt->end_lba)
  385. bt->end_lba = -1ULL;
  386. bt->pid = buts->pid;
  387. bt->trace_state = Blktrace_setup;
  388. ret = -EBUSY;
  389. old_bt = xchg(&q->blk_trace, bt);
  390. if (old_bt) {
  391. (void) xchg(&q->blk_trace, old_bt);
  392. goto err;
  393. }
  394. if (atomic_add_return(1, &blk_probes_ref) == 1)
  395. blk_register_tracepoints();
  396. return 0;
  397. err:
  398. if (bt) {
  399. if (bt->msg_file)
  400. debugfs_remove(bt->msg_file);
  401. if (bt->dropped_file)
  402. debugfs_remove(bt->dropped_file);
  403. free_percpu(bt->sequence);
  404. free_percpu(bt->msg_data);
  405. if (bt->rchan)
  406. relay_close(bt->rchan);
  407. kfree(bt);
  408. }
  409. return ret;
  410. }
  411. int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
  412. char __user *arg)
  413. {
  414. struct blk_user_trace_setup buts;
  415. int ret;
  416. ret = copy_from_user(&buts, arg, sizeof(buts));
  417. if (ret)
  418. return -EFAULT;
  419. ret = do_blk_trace_setup(q, name, dev, &buts);
  420. if (ret)
  421. return ret;
  422. if (copy_to_user(arg, &buts, sizeof(buts)))
  423. return -EFAULT;
  424. return 0;
  425. }
  426. EXPORT_SYMBOL_GPL(blk_trace_setup);
  427. int blk_trace_startstop(struct request_queue *q, int start)
  428. {
  429. int ret;
  430. struct blk_trace *bt = q->blk_trace;
  431. if (bt == NULL)
  432. return -EINVAL;
  433. /*
  434. * For starting a trace, we can transition from a setup or stopped
  435. * trace. For stopping a trace, the state must be running
  436. */
  437. ret = -EINVAL;
  438. if (start) {
  439. if (bt->trace_state == Blktrace_setup ||
  440. bt->trace_state == Blktrace_stopped) {
  441. blktrace_seq++;
  442. smp_mb();
  443. bt->trace_state = Blktrace_running;
  444. trace_note_time(bt);
  445. ret = 0;
  446. }
  447. } else {
  448. if (bt->trace_state == Blktrace_running) {
  449. bt->trace_state = Blktrace_stopped;
  450. relay_flush(bt->rchan);
  451. ret = 0;
  452. }
  453. }
  454. return ret;
  455. }
  456. EXPORT_SYMBOL_GPL(blk_trace_startstop);
  457. /**
  458. * blk_trace_ioctl: - handle the ioctls associated with tracing
  459. * @bdev: the block device
  460. * @cmd: the ioctl cmd
  461. * @arg: the argument data, if any
  462. *
  463. **/
  464. int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
  465. {
  466. struct request_queue *q;
  467. int ret, start = 0;
  468. char b[BDEVNAME_SIZE];
  469. q = bdev_get_queue(bdev);
  470. if (!q)
  471. return -ENXIO;
  472. mutex_lock(&bdev->bd_mutex);
  473. switch (cmd) {
  474. case BLKTRACESETUP:
  475. bdevname(bdev, b);
  476. ret = blk_trace_setup(q, b, bdev->bd_dev, arg);
  477. break;
  478. case BLKTRACESTART:
  479. start = 1;
  480. case BLKTRACESTOP:
  481. ret = blk_trace_startstop(q, start);
  482. break;
  483. case BLKTRACETEARDOWN:
  484. ret = blk_trace_remove(q);
  485. break;
  486. default:
  487. ret = -ENOTTY;
  488. break;
  489. }
  490. mutex_unlock(&bdev->bd_mutex);
  491. return ret;
  492. }
  493. /**
  494. * blk_trace_shutdown: - stop and cleanup trace structures
  495. * @q: the request queue associated with the device
  496. *
  497. **/
  498. void blk_trace_shutdown(struct request_queue *q)
  499. {
  500. if (q->blk_trace) {
  501. blk_trace_startstop(q, 0);
  502. blk_trace_remove(q);
  503. }
  504. }
  505. /*
  506. * blktrace probes
  507. */
  508. /**
  509. * blk_add_trace_rq - Add a trace for a request oriented action
  510. * @q: queue the io is for
  511. * @rq: the source request
  512. * @what: the action
  513. *
  514. * Description:
  515. * Records an action against a request. Will log the bio offset + size.
  516. *
  517. **/
  518. static void blk_add_trace_rq(struct request_queue *q, struct request *rq,
  519. u32 what)
  520. {
  521. struct blk_trace *bt = q->blk_trace;
  522. int rw = rq->cmd_flags & 0x03;
  523. if (likely(!bt))
  524. return;
  525. if (blk_discard_rq(rq))
  526. rw |= (1 << BIO_RW_DISCARD);
  527. if (blk_pc_request(rq)) {
  528. what |= BLK_TC_ACT(BLK_TC_PC);
  529. __blk_add_trace(bt, 0, rq->data_len, rw, what, rq->errors,
  530. sizeof(rq->cmd), rq->cmd);
  531. } else {
  532. what |= BLK_TC_ACT(BLK_TC_FS);
  533. __blk_add_trace(bt, rq->hard_sector, rq->hard_nr_sectors << 9,
  534. rw, what, rq->errors, 0, NULL);
  535. }
  536. }
  537. static void blk_add_trace_rq_abort(struct request_queue *q, struct request *rq)
  538. {
  539. blk_add_trace_rq(q, rq, BLK_TA_ABORT);
  540. }
  541. static void blk_add_trace_rq_insert(struct request_queue *q, struct request *rq)
  542. {
  543. blk_add_trace_rq(q, rq, BLK_TA_INSERT);
  544. }
  545. static void blk_add_trace_rq_issue(struct request_queue *q, struct request *rq)
  546. {
  547. blk_add_trace_rq(q, rq, BLK_TA_ISSUE);
  548. }
  549. static void blk_add_trace_rq_requeue(struct request_queue *q,
  550. struct request *rq)
  551. {
  552. blk_add_trace_rq(q, rq, BLK_TA_REQUEUE);
  553. }
  554. static void blk_add_trace_rq_complete(struct request_queue *q,
  555. struct request *rq)
  556. {
  557. blk_add_trace_rq(q, rq, BLK_TA_COMPLETE);
  558. }
  559. /**
  560. * blk_add_trace_bio - Add a trace for a bio oriented action
  561. * @q: queue the io is for
  562. * @bio: the source bio
  563. * @what: the action
  564. *
  565. * Description:
  566. * Records an action against a bio. Will log the bio offset + size.
  567. *
  568. **/
  569. static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,
  570. u32 what)
  571. {
  572. struct blk_trace *bt = q->blk_trace;
  573. if (likely(!bt))
  574. return;
  575. __blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw, what,
  576. !bio_flagged(bio, BIO_UPTODATE), 0, NULL);
  577. }
  578. static void blk_add_trace_bio_bounce(struct request_queue *q, struct bio *bio)
  579. {
  580. blk_add_trace_bio(q, bio, BLK_TA_BOUNCE);
  581. }
  582. static void blk_add_trace_bio_complete(struct request_queue *q, struct bio *bio)
  583. {
  584. blk_add_trace_bio(q, bio, BLK_TA_COMPLETE);
  585. }
  586. static void blk_add_trace_bio_backmerge(struct request_queue *q,
  587. struct bio *bio)
  588. {
  589. blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE);
  590. }
  591. static void blk_add_trace_bio_frontmerge(struct request_queue *q,
  592. struct bio *bio)
  593. {
  594. blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE);
  595. }
  596. static void blk_add_trace_bio_queue(struct request_queue *q, struct bio *bio)
  597. {
  598. blk_add_trace_bio(q, bio, BLK_TA_QUEUE);
  599. }
  600. static void blk_add_trace_getrq(struct request_queue *q,
  601. struct bio *bio, int rw)
  602. {
  603. if (bio)
  604. blk_add_trace_bio(q, bio, BLK_TA_GETRQ);
  605. else {
  606. struct blk_trace *bt = q->blk_trace;
  607. if (bt)
  608. __blk_add_trace(bt, 0, 0, rw, BLK_TA_GETRQ, 0, 0, NULL);
  609. }
  610. }
  611. static void blk_add_trace_sleeprq(struct request_queue *q,
  612. struct bio *bio, int rw)
  613. {
  614. if (bio)
  615. blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ);
  616. else {
  617. struct blk_trace *bt = q->blk_trace;
  618. if (bt)
  619. __blk_add_trace(bt, 0, 0, rw, BLK_TA_SLEEPRQ,
  620. 0, 0, NULL);
  621. }
  622. }
  623. static void blk_add_trace_plug(struct request_queue *q)
  624. {
  625. struct blk_trace *bt = q->blk_trace;
  626. if (bt)
  627. __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL);
  628. }
  629. static void blk_add_trace_unplug_io(struct request_queue *q)
  630. {
  631. struct blk_trace *bt = q->blk_trace;
  632. if (bt) {
  633. unsigned int pdu = q->rq.count[READ] + q->rq.count[WRITE];
  634. __be64 rpdu = cpu_to_be64(pdu);
  635. __blk_add_trace(bt, 0, 0, 0, BLK_TA_UNPLUG_IO, 0,
  636. sizeof(rpdu), &rpdu);
  637. }
  638. }
  639. static void blk_add_trace_unplug_timer(struct request_queue *q)
  640. {
  641. struct blk_trace *bt = q->blk_trace;
  642. if (bt) {
  643. unsigned int pdu = q->rq.count[READ] + q->rq.count[WRITE];
  644. __be64 rpdu = cpu_to_be64(pdu);
  645. __blk_add_trace(bt, 0, 0, 0, BLK_TA_UNPLUG_TIMER, 0,
  646. sizeof(rpdu), &rpdu);
  647. }
  648. }
  649. static void blk_add_trace_split(struct request_queue *q, struct bio *bio,
  650. unsigned int pdu)
  651. {
  652. struct blk_trace *bt = q->blk_trace;
  653. if (bt) {
  654. __be64 rpdu = cpu_to_be64(pdu);
  655. __blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw,
  656. BLK_TA_SPLIT, !bio_flagged(bio, BIO_UPTODATE),
  657. sizeof(rpdu), &rpdu);
  658. }
  659. }
  660. /**
  661. * blk_add_trace_remap - Add a trace for a remap operation
  662. * @q: queue the io is for
  663. * @bio: the source bio
  664. * @dev: target device
  665. * @from: source sector
  666. * @to: target sector
  667. *
  668. * Description:
  669. * Device mapper or raid target sometimes need to split a bio because
  670. * it spans a stripe (or similar). Add a trace for that action.
  671. *
  672. **/
  673. static void blk_add_trace_remap(struct request_queue *q, struct bio *bio,
  674. dev_t dev, sector_t from, sector_t to)
  675. {
  676. struct blk_trace *bt = q->blk_trace;
  677. struct blk_io_trace_remap r;
  678. if (likely(!bt))
  679. return;
  680. r.device = cpu_to_be32(dev);
  681. r.device_from = cpu_to_be32(bio->bi_bdev->bd_dev);
  682. r.sector = cpu_to_be64(to);
  683. __blk_add_trace(bt, from, bio->bi_size, bio->bi_rw, BLK_TA_REMAP,
  684. !bio_flagged(bio, BIO_UPTODATE), sizeof(r), &r);
  685. }
  686. /**
  687. * blk_add_driver_data - Add binary message with driver-specific data
  688. * @q: queue the io is for
  689. * @rq: io request
  690. * @data: driver-specific data
  691. * @len: length of driver-specific data
  692. *
  693. * Description:
  694. * Some drivers might want to write driver-specific data per request.
  695. *
  696. **/
  697. void blk_add_driver_data(struct request_queue *q,
  698. struct request *rq,
  699. void *data, size_t len)
  700. {
  701. struct blk_trace *bt = q->blk_trace;
  702. if (likely(!bt))
  703. return;
  704. if (blk_pc_request(rq))
  705. __blk_add_trace(bt, 0, rq->data_len, 0, BLK_TA_DRV_DATA,
  706. rq->errors, len, data);
  707. else
  708. __blk_add_trace(bt, rq->hard_sector, rq->hard_nr_sectors << 9,
  709. 0, BLK_TA_DRV_DATA, rq->errors, len, data);
  710. }
  711. EXPORT_SYMBOL_GPL(blk_add_driver_data);
  712. static void blk_register_tracepoints(void)
  713. {
  714. int ret;
  715. ret = register_trace_block_rq_abort(blk_add_trace_rq_abort);
  716. WARN_ON(ret);
  717. ret = register_trace_block_rq_insert(blk_add_trace_rq_insert);
  718. WARN_ON(ret);
  719. ret = register_trace_block_rq_issue(blk_add_trace_rq_issue);
  720. WARN_ON(ret);
  721. ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue);
  722. WARN_ON(ret);
  723. ret = register_trace_block_rq_complete(blk_add_trace_rq_complete);
  724. WARN_ON(ret);
  725. ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce);
  726. WARN_ON(ret);
  727. ret = register_trace_block_bio_complete(blk_add_trace_bio_complete);
  728. WARN_ON(ret);
  729. ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge);
  730. WARN_ON(ret);
  731. ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge);
  732. WARN_ON(ret);
  733. ret = register_trace_block_bio_queue(blk_add_trace_bio_queue);
  734. WARN_ON(ret);
  735. ret = register_trace_block_getrq(blk_add_trace_getrq);
  736. WARN_ON(ret);
  737. ret = register_trace_block_sleeprq(blk_add_trace_sleeprq);
  738. WARN_ON(ret);
  739. ret = register_trace_block_plug(blk_add_trace_plug);
  740. WARN_ON(ret);
  741. ret = register_trace_block_unplug_timer(blk_add_trace_unplug_timer);
  742. WARN_ON(ret);
  743. ret = register_trace_block_unplug_io(blk_add_trace_unplug_io);
  744. WARN_ON(ret);
  745. ret = register_trace_block_split(blk_add_trace_split);
  746. WARN_ON(ret);
  747. ret = register_trace_block_remap(blk_add_trace_remap);
  748. WARN_ON(ret);
  749. }
  750. static void blk_unregister_tracepoints(void)
  751. {
  752. unregister_trace_block_remap(blk_add_trace_remap);
  753. unregister_trace_block_split(blk_add_trace_split);
  754. unregister_trace_block_unplug_io(blk_add_trace_unplug_io);
  755. unregister_trace_block_unplug_timer(blk_add_trace_unplug_timer);
  756. unregister_trace_block_plug(blk_add_trace_plug);
  757. unregister_trace_block_sleeprq(blk_add_trace_sleeprq);
  758. unregister_trace_block_getrq(blk_add_trace_getrq);
  759. unregister_trace_block_bio_queue(blk_add_trace_bio_queue);
  760. unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge);
  761. unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge);
  762. unregister_trace_block_bio_complete(blk_add_trace_bio_complete);
  763. unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce);
  764. unregister_trace_block_rq_complete(blk_add_trace_rq_complete);
  765. unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue);
  766. unregister_trace_block_rq_issue(blk_add_trace_rq_issue);
  767. unregister_trace_block_rq_insert(blk_add_trace_rq_insert);
  768. unregister_trace_block_rq_abort(blk_add_trace_rq_abort);
  769. tracepoint_synchronize_unregister();
  770. }
  771. /*
  772. * struct blk_io_tracer formatting routines
  773. */
  774. static void fill_rwbs(char *rwbs, const struct blk_io_trace *t)
  775. {
  776. int i = 0;
  777. int tc = t->action >> BLK_TC_SHIFT;
  778. if (tc & BLK_TC_DISCARD)
  779. rwbs[i++] = 'D';
  780. else if (tc & BLK_TC_WRITE)
  781. rwbs[i++] = 'W';
  782. else if (t->bytes)
  783. rwbs[i++] = 'R';
  784. else
  785. rwbs[i++] = 'N';
  786. if (tc & BLK_TC_AHEAD)
  787. rwbs[i++] = 'A';
  788. if (tc & BLK_TC_BARRIER)
  789. rwbs[i++] = 'B';
  790. if (tc & BLK_TC_SYNC)
  791. rwbs[i++] = 'S';
  792. if (tc & BLK_TC_META)
  793. rwbs[i++] = 'M';
  794. rwbs[i] = '\0';
  795. }
  796. static inline
  797. const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent)
  798. {
  799. return (const struct blk_io_trace *)ent;
  800. }
  801. static inline const void *pdu_start(const struct trace_entry *ent)
  802. {
  803. return te_blk_io_trace(ent) + 1;
  804. }
  805. static inline u32 t_sec(const struct trace_entry *ent)
  806. {
  807. return te_blk_io_trace(ent)->bytes >> 9;
  808. }
  809. static inline unsigned long long t_sector(const struct trace_entry *ent)
  810. {
  811. return te_blk_io_trace(ent)->sector;
  812. }
  813. static inline __u16 t_error(const struct trace_entry *ent)
  814. {
  815. return te_blk_io_trace(ent)->error;
  816. }
  817. static __u64 get_pdu_int(const struct trace_entry *ent)
  818. {
  819. const __u64 *val = pdu_start(ent);
  820. return be64_to_cpu(*val);
  821. }
  822. static void get_pdu_remap(const struct trace_entry *ent,
  823. struct blk_io_trace_remap *r)
  824. {
  825. const struct blk_io_trace_remap *__r = pdu_start(ent);
  826. __u64 sector = __r->sector;
  827. r->device = be32_to_cpu(__r->device);
  828. r->device_from = be32_to_cpu(__r->device_from);
  829. r->sector = be64_to_cpu(sector);
  830. }
  831. static int blk_log_action_iter(struct trace_iterator *iter, const char *act)
  832. {
  833. char rwbs[6];
  834. unsigned long long ts = ns2usecs(iter->ts);
  835. unsigned long usec_rem = do_div(ts, USEC_PER_SEC);
  836. unsigned secs = (unsigned long)ts;
  837. const struct trace_entry *ent = iter->ent;
  838. const struct blk_io_trace *t = (const struct blk_io_trace *)ent;
  839. fill_rwbs(rwbs, t);
  840. return trace_seq_printf(&iter->seq,
  841. "%3d,%-3d %2d %5d.%06lu %5u %2s %3s ",
  842. MAJOR(t->device), MINOR(t->device), iter->cpu,
  843. secs, usec_rem, ent->pid, act, rwbs);
  844. }
  845. static int blk_log_action_seq(struct trace_seq *s, const struct blk_io_trace *t,
  846. const char *act)
  847. {
  848. char rwbs[6];
  849. fill_rwbs(rwbs, t);
  850. return trace_seq_printf(s, "%3d,%-3d %2s %3s ",
  851. MAJOR(t->device), MINOR(t->device), act, rwbs);
  852. }
  853. static int blk_log_generic(struct trace_seq *s, const struct trace_entry *ent)
  854. {
  855. char cmd[TASK_COMM_LEN];
  856. trace_find_cmdline(ent->pid, cmd);
  857. if (t_sec(ent))
  858. return trace_seq_printf(s, "%llu + %u [%s]\n",
  859. t_sector(ent), t_sec(ent), cmd);
  860. return trace_seq_printf(s, "[%s]\n", cmd);
  861. }
  862. static int blk_log_with_error(struct trace_seq *s,
  863. const struct trace_entry *ent)
  864. {
  865. if (t_sec(ent))
  866. return trace_seq_printf(s, "%llu + %u [%d]\n", t_sector(ent),
  867. t_sec(ent), t_error(ent));
  868. return trace_seq_printf(s, "%llu [%d]\n", t_sector(ent), t_error(ent));
  869. }
  870. static int blk_log_remap(struct trace_seq *s, const struct trace_entry *ent)
  871. {
  872. struct blk_io_trace_remap r = { .device = 0, };
  873. get_pdu_remap(ent, &r);
  874. return trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n",
  875. t_sector(ent),
  876. t_sec(ent), MAJOR(r.device), MINOR(r.device),
  877. (unsigned long long)r.sector);
  878. }
  879. static int blk_log_plug(struct trace_seq *s, const struct trace_entry *ent)
  880. {
  881. char cmd[TASK_COMM_LEN];
  882. trace_find_cmdline(ent->pid, cmd);
  883. return trace_seq_printf(s, "[%s]\n", cmd);
  884. }
  885. static int blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent)
  886. {
  887. char cmd[TASK_COMM_LEN];
  888. trace_find_cmdline(ent->pid, cmd);
  889. return trace_seq_printf(s, "[%s] %llu\n", cmd, get_pdu_int(ent));
  890. }
  891. static int blk_log_split(struct trace_seq *s, const struct trace_entry *ent)
  892. {
  893. char cmd[TASK_COMM_LEN];
  894. trace_find_cmdline(ent->pid, cmd);
  895. return trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent),
  896. get_pdu_int(ent), cmd);
  897. }
  898. /*
  899. * struct tracer operations
  900. */
  901. static void blk_tracer_print_header(struct seq_file *m)
  902. {
  903. if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
  904. return;
  905. seq_puts(m, "# DEV CPU TIMESTAMP PID ACT FLG\n"
  906. "# | | | | | |\n");
  907. }
  908. static void blk_tracer_start(struct trace_array *tr)
  909. {
  910. if (atomic_add_return(1, &blk_probes_ref) == 1)
  911. blk_register_tracepoints();
  912. trace_flags &= ~TRACE_ITER_CONTEXT_INFO;
  913. }
  914. static int blk_tracer_init(struct trace_array *tr)
  915. {
  916. blk_tr = tr;
  917. blk_tracer_start(tr);
  918. blk_tracer_enabled = true;
  919. return 0;
  920. }
  921. static void blk_tracer_stop(struct trace_array *tr)
  922. {
  923. trace_flags |= TRACE_ITER_CONTEXT_INFO;
  924. if (atomic_dec_and_test(&blk_probes_ref))
  925. blk_unregister_tracepoints();
  926. }
  927. static void blk_tracer_reset(struct trace_array *tr)
  928. {
  929. if (!atomic_read(&blk_probes_ref))
  930. return;
  931. blk_tracer_enabled = false;
  932. blk_tracer_stop(tr);
  933. }
  934. static const struct {
  935. const char *act[2];
  936. int (*print)(struct trace_seq *s, const struct trace_entry *ent);
  937. } what2act[] = {
  938. [__BLK_TA_QUEUE] = {{ "Q", "queue" }, blk_log_generic },
  939. [__BLK_TA_BACKMERGE] = {{ "M", "backmerge" }, blk_log_generic },
  940. [__BLK_TA_FRONTMERGE] = {{ "F", "frontmerge" }, blk_log_generic },
  941. [__BLK_TA_GETRQ] = {{ "G", "getrq" }, blk_log_generic },
  942. [__BLK_TA_SLEEPRQ] = {{ "S", "sleeprq" }, blk_log_generic },
  943. [__BLK_TA_REQUEUE] = {{ "R", "requeue" }, blk_log_with_error },
  944. [__BLK_TA_ISSUE] = {{ "D", "issue" }, blk_log_generic },
  945. [__BLK_TA_COMPLETE] = {{ "C", "complete" }, blk_log_with_error },
  946. [__BLK_TA_PLUG] = {{ "P", "plug" }, blk_log_plug },
  947. [__BLK_TA_UNPLUG_IO] = {{ "U", "unplug_io" }, blk_log_unplug },
  948. [__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug },
  949. [__BLK_TA_INSERT] = {{ "I", "insert" }, blk_log_generic },
  950. [__BLK_TA_SPLIT] = {{ "X", "split" }, blk_log_split },
  951. [__BLK_TA_BOUNCE] = {{ "B", "bounce" }, blk_log_generic },
  952. [__BLK_TA_REMAP] = {{ "A", "remap" }, blk_log_remap },
  953. };
  954. static enum print_line_t blk_trace_event_print(struct trace_iterator *iter,
  955. int flags)
  956. {
  957. struct trace_seq *s = &iter->seq;
  958. const struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
  959. const u16 what = t->action & ((1 << BLK_TC_SHIFT) - 1);
  960. int ret;
  961. if (!trace_print_context(iter))
  962. return TRACE_TYPE_PARTIAL_LINE;
  963. if (unlikely(what == 0 || what > ARRAY_SIZE(what2act)))
  964. ret = trace_seq_printf(s, "Bad pc action %x\n", what);
  965. else {
  966. const bool long_act = !!(trace_flags & TRACE_ITER_VERBOSE);
  967. ret = blk_log_action_seq(s, t, what2act[what].act[long_act]);
  968. if (ret)
  969. ret = what2act[what].print(s, iter->ent);
  970. }
  971. return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
  972. }
  973. static int blk_trace_synthesize_old_trace(struct trace_iterator *iter)
  974. {
  975. struct trace_seq *s = &iter->seq;
  976. struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
  977. const int offset = offsetof(struct blk_io_trace, sector);
  978. struct blk_io_trace old = {
  979. .magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION,
  980. .time = iter->ts,
  981. };
  982. if (!trace_seq_putmem(s, &old, offset))
  983. return 0;
  984. return trace_seq_putmem(s, &t->sector,
  985. sizeof(old) - offset + t->pdu_len);
  986. }
  987. static enum print_line_t
  988. blk_trace_event_print_binary(struct trace_iterator *iter, int flags)
  989. {
  990. return blk_trace_synthesize_old_trace(iter) ?
  991. TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
  992. }
  993. static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter)
  994. {
  995. const struct blk_io_trace *t;
  996. u16 what;
  997. int ret;
  998. if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
  999. return TRACE_TYPE_UNHANDLED;
  1000. t = (const struct blk_io_trace *)iter->ent;
  1001. what = t->action & ((1 << BLK_TC_SHIFT) - 1);
  1002. if (unlikely(what == 0 || what > ARRAY_SIZE(what2act)))
  1003. ret = trace_seq_printf(&iter->seq, "Bad pc action %x\n", what);
  1004. else {
  1005. const bool long_act = !!(trace_flags & TRACE_ITER_VERBOSE);
  1006. ret = blk_log_action_iter(iter, what2act[what].act[long_act]);
  1007. if (ret)
  1008. ret = what2act[what].print(&iter->seq, iter->ent);
  1009. }
  1010. return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
  1011. }
  1012. static struct tracer blk_tracer __read_mostly = {
  1013. .name = "blk",
  1014. .init = blk_tracer_init,
  1015. .reset = blk_tracer_reset,
  1016. .start = blk_tracer_start,
  1017. .stop = blk_tracer_stop,
  1018. .print_header = blk_tracer_print_header,
  1019. .print_line = blk_tracer_print_line,
  1020. .flags = &blk_tracer_flags,
  1021. };
  1022. static struct trace_event trace_blk_event = {
  1023. .type = TRACE_BLK,
  1024. .trace = blk_trace_event_print,
  1025. .binary = blk_trace_event_print_binary,
  1026. };
  1027. static int __init init_blk_tracer(void)
  1028. {
  1029. if (!register_ftrace_event(&trace_blk_event)) {
  1030. pr_warning("Warning: could not register block events\n");
  1031. return 1;
  1032. }
  1033. if (register_tracer(&blk_tracer) != 0) {
  1034. pr_warning("Warning: could not register the block tracer\n");
  1035. unregister_ftrace_event(&trace_blk_event);
  1036. return 1;
  1037. }
  1038. return 0;
  1039. }
  1040. device_initcall(init_blk_tracer);
  1041. static int blk_trace_remove_queue(struct request_queue *q)
  1042. {
  1043. struct blk_trace *bt;
  1044. bt = xchg(&q->blk_trace, NULL);
  1045. if (bt == NULL)
  1046. return -EINVAL;
  1047. kfree(bt);
  1048. return 0;
  1049. }
  1050. /*
  1051. * Setup everything required to start tracing
  1052. */
  1053. static int blk_trace_setup_queue(struct request_queue *q, dev_t dev)
  1054. {
  1055. struct blk_trace *old_bt, *bt = NULL;
  1056. bt = kzalloc(sizeof(*bt), GFP_KERNEL);
  1057. if (!bt)
  1058. return -ENOMEM;
  1059. bt->dev = dev;
  1060. bt->act_mask = (u16)-1;
  1061. bt->end_lba = -1ULL;
  1062. old_bt = xchg(&q->blk_trace, bt);
  1063. if (old_bt != NULL) {
  1064. (void)xchg(&q->blk_trace, old_bt);
  1065. kfree(bt);
  1066. return -EBUSY;
  1067. }
  1068. return 0;
  1069. }
  1070. /*
  1071. * sysfs interface to enable and configure tracing
  1072. */
  1073. static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
  1074. struct device_attribute *attr,
  1075. char *buf);
  1076. static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
  1077. struct device_attribute *attr,
  1078. const char *buf, size_t count);
  1079. #define BLK_TRACE_DEVICE_ATTR(_name) \
  1080. DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \
  1081. sysfs_blk_trace_attr_show, \
  1082. sysfs_blk_trace_attr_store)
  1083. static BLK_TRACE_DEVICE_ATTR(enable);
  1084. static BLK_TRACE_DEVICE_ATTR(act_mask);
  1085. static BLK_TRACE_DEVICE_ATTR(pid);
  1086. static BLK_TRACE_DEVICE_ATTR(start_lba);
  1087. static BLK_TRACE_DEVICE_ATTR(end_lba);
  1088. static struct attribute *blk_trace_attrs[] = {
  1089. &dev_attr_enable.attr,
  1090. &dev_attr_act_mask.attr,
  1091. &dev_attr_pid.attr,
  1092. &dev_attr_start_lba.attr,
  1093. &dev_attr_end_lba.attr,
  1094. NULL
  1095. };
  1096. struct attribute_group blk_trace_attr_group = {
  1097. .name = "trace",
  1098. .attrs = blk_trace_attrs,
  1099. };
  1100. static const struct {
  1101. int mask;
  1102. const char *str;
  1103. } mask_maps[] = {
  1104. { BLK_TC_READ, "read" },
  1105. { BLK_TC_WRITE, "write" },
  1106. { BLK_TC_BARRIER, "barrier" },
  1107. { BLK_TC_SYNC, "sync" },
  1108. { BLK_TC_QUEUE, "queue" },
  1109. { BLK_TC_REQUEUE, "requeue" },
  1110. { BLK_TC_ISSUE, "issue" },
  1111. { BLK_TC_COMPLETE, "complete" },
  1112. { BLK_TC_FS, "fs" },
  1113. { BLK_TC_PC, "pc" },
  1114. { BLK_TC_AHEAD, "ahead" },
  1115. { BLK_TC_META, "meta" },
  1116. { BLK_TC_DISCARD, "discard" },
  1117. { BLK_TC_DRV_DATA, "drv_data" },
  1118. };
  1119. static int blk_trace_str2mask(const char *str)
  1120. {
  1121. int i;
  1122. int mask = 0;
  1123. char *s, *token;
  1124. s = kstrdup(str, GFP_KERNEL);
  1125. if (s == NULL)
  1126. return -ENOMEM;
  1127. s = strstrip(s);
  1128. while (1) {
  1129. token = strsep(&s, ",");
  1130. if (token == NULL)
  1131. break;
  1132. if (*token == '\0')
  1133. continue;
  1134. for (i = 0; i < ARRAY_SIZE(mask_maps); i++) {
  1135. if (strcasecmp(token, mask_maps[i].str) == 0) {
  1136. mask |= mask_maps[i].mask;
  1137. break;
  1138. }
  1139. }
  1140. if (i == ARRAY_SIZE(mask_maps)) {
  1141. mask = -EINVAL;
  1142. break;
  1143. }
  1144. }
  1145. kfree(s);
  1146. return mask;
  1147. }
  1148. static ssize_t blk_trace_mask2str(char *buf, int mask)
  1149. {
  1150. int i;
  1151. char *p = buf;
  1152. for (i = 0; i < ARRAY_SIZE(mask_maps); i++) {
  1153. if (mask & mask_maps[i].mask) {
  1154. p += sprintf(p, "%s%s",
  1155. (p == buf) ? "" : ",", mask_maps[i].str);
  1156. }
  1157. }
  1158. *p++ = '\n';
  1159. return p - buf;
  1160. }
  1161. static struct request_queue *blk_trace_get_queue(struct block_device *bdev)
  1162. {
  1163. if (bdev->bd_disk == NULL)
  1164. return NULL;
  1165. return bdev_get_queue(bdev);
  1166. }
  1167. static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
  1168. struct device_attribute *attr,
  1169. char *buf)
  1170. {
  1171. struct hd_struct *p = dev_to_part(dev);
  1172. struct request_queue *q;
  1173. struct block_device *bdev;
  1174. ssize_t ret = -ENXIO;
  1175. lock_kernel();
  1176. bdev = bdget(part_devt(p));
  1177. if (bdev == NULL)
  1178. goto out_unlock_kernel;
  1179. q = blk_trace_get_queue(bdev);
  1180. if (q == NULL)
  1181. goto out_bdput;
  1182. mutex_lock(&bdev->bd_mutex);
  1183. if (attr == &dev_attr_enable) {
  1184. ret = sprintf(buf, "%u\n", !!q->blk_trace);
  1185. goto out_unlock_bdev;
  1186. }
  1187. if (q->blk_trace == NULL)
  1188. ret = sprintf(buf, "disabled\n");
  1189. else if (attr == &dev_attr_act_mask)
  1190. ret = blk_trace_mask2str(buf, q->blk_trace->act_mask);
  1191. else if (attr == &dev_attr_pid)
  1192. ret = sprintf(buf, "%u\n", q->blk_trace->pid);
  1193. else if (attr == &dev_attr_start_lba)
  1194. ret = sprintf(buf, "%llu\n", q->blk_trace->start_lba);
  1195. else if (attr == &dev_attr_end_lba)
  1196. ret = sprintf(buf, "%llu\n", q->blk_trace->end_lba);
  1197. out_unlock_bdev:
  1198. mutex_unlock(&bdev->bd_mutex);
  1199. out_bdput:
  1200. bdput(bdev);
  1201. out_unlock_kernel:
  1202. unlock_kernel();
  1203. return ret;
  1204. }
  1205. static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
  1206. struct device_attribute *attr,
  1207. const char *buf, size_t count)
  1208. {
  1209. struct block_device *bdev;
  1210. struct request_queue *q;
  1211. struct hd_struct *p;
  1212. u64 value;
  1213. ssize_t ret = -EINVAL;
  1214. if (count == 0)
  1215. goto out;
  1216. if (attr == &dev_attr_act_mask) {
  1217. if (sscanf(buf, "%llx", &value) != 1) {
  1218. /* Assume it is a list of trace category names */
  1219. ret = blk_trace_str2mask(buf);
  1220. if (ret < 0)
  1221. goto out;
  1222. value = ret;
  1223. }
  1224. } else if (sscanf(buf, "%llu", &value) != 1)
  1225. goto out;
  1226. ret = -ENXIO;
  1227. lock_kernel();
  1228. p = dev_to_part(dev);
  1229. bdev = bdget(part_devt(p));
  1230. if (bdev == NULL)
  1231. goto out_unlock_kernel;
  1232. q = blk_trace_get_queue(bdev);
  1233. if (q == NULL)
  1234. goto out_bdput;
  1235. mutex_lock(&bdev->bd_mutex);
  1236. if (attr == &dev_attr_enable) {
  1237. if (value)
  1238. ret = blk_trace_setup_queue(q, bdev->bd_dev);
  1239. else
  1240. ret = blk_trace_remove_queue(q);
  1241. goto out_unlock_bdev;
  1242. }
  1243. ret = 0;
  1244. if (q->blk_trace == NULL)
  1245. ret = blk_trace_setup_queue(q, bdev->bd_dev);
  1246. if (ret == 0) {
  1247. if (attr == &dev_attr_act_mask)
  1248. q->blk_trace->act_mask = value;
  1249. else if (attr == &dev_attr_pid)
  1250. q->blk_trace->pid = value;
  1251. else if (attr == &dev_attr_start_lba)
  1252. q->blk_trace->start_lba = value;
  1253. else if (attr == &dev_attr_end_lba)
  1254. q->blk_trace->end_lba = value;
  1255. }
  1256. out_unlock_bdev:
  1257. mutex_unlock(&bdev->bd_mutex);
  1258. out_bdput:
  1259. bdput(bdev);
  1260. out_unlock_kernel:
  1261. unlock_kernel();
  1262. out:
  1263. return ret ? ret : count;
  1264. }