trace.c 85 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824
  1. /*
  2. * ring buffer based function tracer
  3. *
  4. * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  5. * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
  6. *
  7. * Originally taken from the RT patch by:
  8. * Arnaldo Carvalho de Melo <acme@redhat.com>
  9. *
  10. * Based on code from the latency_tracer, that is:
  11. * Copyright (C) 2004-2006 Ingo Molnar
  12. * Copyright (C) 2004 William Lee Irwin III
  13. */
  14. #include <linux/utsrelease.h>
  15. #include <linux/kallsyms.h>
  16. #include <linux/seq_file.h>
  17. #include <linux/notifier.h>
  18. #include <linux/debugfs.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/hardirq.h>
  21. #include <linux/linkage.h>
  22. #include <linux/uaccess.h>
  23. #include <linux/ftrace.h>
  24. #include <linux/module.h>
  25. #include <linux/percpu.h>
  26. #include <linux/kdebug.h>
  27. #include <linux/ctype.h>
  28. #include <linux/init.h>
  29. #include <linux/poll.h>
  30. #include <linux/gfp.h>
  31. #include <linux/fs.h>
  32. #include <linux/kprobes.h>
  33. #include <linux/seq_file.h>
  34. #include <linux/writeback.h>
  35. #include <linux/stacktrace.h>
  36. #include <linux/ring_buffer.h>
  37. #include <linux/irqflags.h>
  38. #include "trace.h"
  39. #define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
  40. unsigned long __read_mostly tracing_max_latency = (cycle_t)ULONG_MAX;
  41. unsigned long __read_mostly tracing_thresh;
  42. /* For tracers that don't implement custom flags */
  43. static struct tracer_opt dummy_tracer_opt[] = {
  44. { }
  45. };
  46. static struct tracer_flags dummy_tracer_flags = {
  47. .val = 0,
  48. .opts = dummy_tracer_opt
  49. };
  50. static int dummy_set_flag(u32 old_flags, u32 bit, int set)
  51. {
  52. return 0;
  53. }
  54. /*
  55. * Kill all tracing for good (never come back).
  56. * It is initialized to 1 but will turn to zero if the initialization
  57. * of the tracer is successful. But that is the only place that sets
  58. * this back to zero.
  59. */
  60. int tracing_disabled = 1;
  61. static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
  62. static inline void ftrace_disable_cpu(void)
  63. {
  64. preempt_disable();
  65. local_inc(&__get_cpu_var(ftrace_cpu_disabled));
  66. }
  67. static inline void ftrace_enable_cpu(void)
  68. {
  69. local_dec(&__get_cpu_var(ftrace_cpu_disabled));
  70. preempt_enable();
  71. }
  72. static cpumask_t __read_mostly tracing_buffer_mask;
  73. #define for_each_tracing_cpu(cpu) \
  74. for_each_cpu_mask(cpu, tracing_buffer_mask)
  75. /*
  76. * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
  77. *
  78. * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
  79. * is set, then ftrace_dump is called. This will output the contents
  80. * of the ftrace buffers to the console. This is very useful for
  81. * capturing traces that lead to crashes and outputing it to a
  82. * serial console.
  83. *
  84. * It is default off, but you can enable it with either specifying
  85. * "ftrace_dump_on_oops" in the kernel command line, or setting
  86. * /proc/sys/kernel/ftrace_dump_on_oops to true.
  87. */
  88. int ftrace_dump_on_oops;
  89. static int tracing_set_tracer(char *buf);
  90. static int __init set_ftrace(char *str)
  91. {
  92. tracing_set_tracer(str);
  93. return 1;
  94. }
  95. __setup("ftrace", set_ftrace);
  96. static int __init set_ftrace_dump_on_oops(char *str)
  97. {
  98. ftrace_dump_on_oops = 1;
  99. return 1;
  100. }
  101. __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
  102. long
  103. ns2usecs(cycle_t nsec)
  104. {
  105. nsec += 500;
  106. do_div(nsec, 1000);
  107. return nsec;
  108. }
  109. cycle_t ftrace_now(int cpu)
  110. {
  111. u64 ts = ring_buffer_time_stamp(cpu);
  112. ring_buffer_normalize_time_stamp(cpu, &ts);
  113. return ts;
  114. }
  115. /*
  116. * The global_trace is the descriptor that holds the tracing
  117. * buffers for the live tracing. For each CPU, it contains
  118. * a link list of pages that will store trace entries. The
  119. * page descriptor of the pages in the memory is used to hold
  120. * the link list by linking the lru item in the page descriptor
  121. * to each of the pages in the buffer per CPU.
  122. *
  123. * For each active CPU there is a data field that holds the
  124. * pages for the buffer for that CPU. Each CPU has the same number
  125. * of pages allocated for its buffer.
  126. */
  127. static struct trace_array global_trace;
  128. static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
  129. /*
  130. * The max_tr is used to snapshot the global_trace when a maximum
  131. * latency is reached. Some tracers will use this to store a maximum
  132. * trace while it continues examining live traces.
  133. *
  134. * The buffers for the max_tr are set up the same as the global_trace.
  135. * When a snapshot is taken, the link list of the max_tr is swapped
  136. * with the link list of the global_trace and the buffers are reset for
  137. * the global_trace so the tracing can continue.
  138. */
  139. static struct trace_array max_tr;
  140. static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
  141. /* tracer_enabled is used to toggle activation of a tracer */
  142. static int tracer_enabled = 1;
  143. /**
  144. * tracing_is_enabled - return tracer_enabled status
  145. *
  146. * This function is used by other tracers to know the status
  147. * of the tracer_enabled flag. Tracers may use this function
  148. * to know if it should enable their features when starting
  149. * up. See irqsoff tracer for an example (start_irqsoff_tracer).
  150. */
  151. int tracing_is_enabled(void)
  152. {
  153. return tracer_enabled;
  154. }
  155. /* function tracing enabled */
  156. int ftrace_function_enabled;
  157. /*
  158. * trace_buf_size is the size in bytes that is allocated
  159. * for a buffer. Note, the number of bytes is always rounded
  160. * to page size.
  161. *
  162. * This number is purposely set to a low number of 16384.
  163. * If the dump on oops happens, it will be much appreciated
  164. * to not have to wait for all that output. Anyway this can be
  165. * boot time and run time configurable.
  166. */
  167. #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
  168. static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
  169. /* trace_types holds a link list of available tracers. */
  170. static struct tracer *trace_types __read_mostly;
  171. /* current_trace points to the tracer that is currently active */
  172. static struct tracer *current_trace __read_mostly;
  173. /*
  174. * max_tracer_type_len is used to simplify the allocating of
  175. * buffers to read userspace tracer names. We keep track of
  176. * the longest tracer name registered.
  177. */
  178. static int max_tracer_type_len;
  179. /*
  180. * trace_types_lock is used to protect the trace_types list.
  181. * This lock is also used to keep user access serialized.
  182. * Accesses from userspace will grab this lock while userspace
  183. * activities happen inside the kernel.
  184. */
  185. static DEFINE_MUTEX(trace_types_lock);
  186. /* trace_wait is a waitqueue for tasks blocked on trace_poll */
  187. static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
  188. /* trace_flags holds trace_options default values */
  189. unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
  190. TRACE_ITER_ANNOTATE;
  191. /**
  192. * trace_wake_up - wake up tasks waiting for trace input
  193. *
  194. * Simply wakes up any task that is blocked on the trace_wait
  195. * queue. These is used with trace_poll for tasks polling the trace.
  196. */
  197. void trace_wake_up(void)
  198. {
  199. /*
  200. * The runqueue_is_locked() can fail, but this is the best we
  201. * have for now:
  202. */
  203. if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
  204. wake_up(&trace_wait);
  205. }
  206. static int __init set_buf_size(char *str)
  207. {
  208. unsigned long buf_size;
  209. int ret;
  210. if (!str)
  211. return 0;
  212. ret = strict_strtoul(str, 0, &buf_size);
  213. /* nr_entries can not be zero */
  214. if (ret < 0 || buf_size == 0)
  215. return 0;
  216. trace_buf_size = buf_size;
  217. return 1;
  218. }
  219. __setup("trace_buf_size=", set_buf_size);
  220. unsigned long nsecs_to_usecs(unsigned long nsecs)
  221. {
  222. return nsecs / 1000;
  223. }
  224. /* These must match the bit postions in trace_iterator_flags */
  225. static const char *trace_options[] = {
  226. "print-parent",
  227. "sym-offset",
  228. "sym-addr",
  229. "verbose",
  230. "raw",
  231. "hex",
  232. "bin",
  233. "block",
  234. "stacktrace",
  235. "sched-tree",
  236. "ftrace_printk",
  237. "ftrace_preempt",
  238. "branch",
  239. "annotate",
  240. "userstacktrace",
  241. "sym-userobj",
  242. NULL
  243. };
  244. /*
  245. * ftrace_max_lock is used to protect the swapping of buffers
  246. * when taking a max snapshot. The buffers themselves are
  247. * protected by per_cpu spinlocks. But the action of the swap
  248. * needs its own lock.
  249. *
  250. * This is defined as a raw_spinlock_t in order to help
  251. * with performance when lockdep debugging is enabled.
  252. */
  253. static raw_spinlock_t ftrace_max_lock =
  254. (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
  255. /*
  256. * Copy the new maximum trace into the separate maximum-trace
  257. * structure. (this way the maximum trace is permanently saved,
  258. * for later retrieval via /debugfs/tracing/latency_trace)
  259. */
  260. static void
  261. __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
  262. {
  263. struct trace_array_cpu *data = tr->data[cpu];
  264. max_tr.cpu = cpu;
  265. max_tr.time_start = data->preempt_timestamp;
  266. data = max_tr.data[cpu];
  267. data->saved_latency = tracing_max_latency;
  268. memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
  269. data->pid = tsk->pid;
  270. data->uid = tsk->uid;
  271. data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
  272. data->policy = tsk->policy;
  273. data->rt_priority = tsk->rt_priority;
  274. /* record this tasks comm */
  275. tracing_record_cmdline(current);
  276. }
  277. /**
  278. * trace_seq_printf - sequence printing of trace information
  279. * @s: trace sequence descriptor
  280. * @fmt: printf format string
  281. *
  282. * The tracer may use either sequence operations or its own
  283. * copy to user routines. To simplify formating of a trace
  284. * trace_seq_printf is used to store strings into a special
  285. * buffer (@s). Then the output may be either used by
  286. * the sequencer or pulled into another buffer.
  287. */
  288. int
  289. trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
  290. {
  291. int len = (PAGE_SIZE - 1) - s->len;
  292. va_list ap;
  293. int ret;
  294. if (!len)
  295. return 0;
  296. va_start(ap, fmt);
  297. ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
  298. va_end(ap);
  299. /* If we can't write it all, don't bother writing anything */
  300. if (ret >= len)
  301. return 0;
  302. s->len += ret;
  303. return len;
  304. }
  305. /**
  306. * trace_seq_puts - trace sequence printing of simple string
  307. * @s: trace sequence descriptor
  308. * @str: simple string to record
  309. *
  310. * The tracer may use either the sequence operations or its own
  311. * copy to user routines. This function records a simple string
  312. * into a special buffer (@s) for later retrieval by a sequencer
  313. * or other mechanism.
  314. */
  315. static int
  316. trace_seq_puts(struct trace_seq *s, const char *str)
  317. {
  318. int len = strlen(str);
  319. if (len > ((PAGE_SIZE - 1) - s->len))
  320. return 0;
  321. memcpy(s->buffer + s->len, str, len);
  322. s->len += len;
  323. return len;
  324. }
  325. static int
  326. trace_seq_putc(struct trace_seq *s, unsigned char c)
  327. {
  328. if (s->len >= (PAGE_SIZE - 1))
  329. return 0;
  330. s->buffer[s->len++] = c;
  331. return 1;
  332. }
  333. static int
  334. trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
  335. {
  336. if (len > ((PAGE_SIZE - 1) - s->len))
  337. return 0;
  338. memcpy(s->buffer + s->len, mem, len);
  339. s->len += len;
  340. return len;
  341. }
  342. #define MAX_MEMHEX_BYTES 8
  343. #define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
  344. static int
  345. trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
  346. {
  347. unsigned char hex[HEX_CHARS];
  348. unsigned char *data = mem;
  349. int i, j;
  350. #ifdef __BIG_ENDIAN
  351. for (i = 0, j = 0; i < len; i++) {
  352. #else
  353. for (i = len-1, j = 0; i >= 0; i--) {
  354. #endif
  355. hex[j++] = hex_asc_hi(data[i]);
  356. hex[j++] = hex_asc_lo(data[i]);
  357. }
  358. hex[j++] = ' ';
  359. return trace_seq_putmem(s, hex, j);
  360. }
  361. static int
  362. trace_seq_path(struct trace_seq *s, struct path *path)
  363. {
  364. unsigned char *p;
  365. if (s->len >= (PAGE_SIZE - 1))
  366. return 0;
  367. p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len);
  368. if (!IS_ERR(p)) {
  369. p = mangle_path(s->buffer + s->len, p, "\n");
  370. if (p) {
  371. s->len = p - s->buffer;
  372. return 1;
  373. }
  374. } else {
  375. s->buffer[s->len++] = '?';
  376. return 1;
  377. }
  378. return 0;
  379. }
  380. static void
  381. trace_seq_reset(struct trace_seq *s)
  382. {
  383. s->len = 0;
  384. s->readpos = 0;
  385. }
  386. ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
  387. {
  388. int len;
  389. int ret;
  390. if (s->len <= s->readpos)
  391. return -EBUSY;
  392. len = s->len - s->readpos;
  393. if (cnt > len)
  394. cnt = len;
  395. ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
  396. if (ret)
  397. return -EFAULT;
  398. s->readpos += len;
  399. return cnt;
  400. }
  401. static void
  402. trace_print_seq(struct seq_file *m, struct trace_seq *s)
  403. {
  404. int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
  405. s->buffer[len] = 0;
  406. seq_puts(m, s->buffer);
  407. trace_seq_reset(s);
  408. }
  409. /**
  410. * update_max_tr - snapshot all trace buffers from global_trace to max_tr
  411. * @tr: tracer
  412. * @tsk: the task with the latency
  413. * @cpu: The cpu that initiated the trace.
  414. *
  415. * Flip the buffers between the @tr and the max_tr and record information
  416. * about which task was the cause of this latency.
  417. */
  418. void
  419. update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
  420. {
  421. struct ring_buffer *buf = tr->buffer;
  422. WARN_ON_ONCE(!irqs_disabled());
  423. __raw_spin_lock(&ftrace_max_lock);
  424. tr->buffer = max_tr.buffer;
  425. max_tr.buffer = buf;
  426. ftrace_disable_cpu();
  427. ring_buffer_reset(tr->buffer);
  428. ftrace_enable_cpu();
  429. __update_max_tr(tr, tsk, cpu);
  430. __raw_spin_unlock(&ftrace_max_lock);
  431. }
  432. /**
  433. * update_max_tr_single - only copy one trace over, and reset the rest
  434. * @tr - tracer
  435. * @tsk - task with the latency
  436. * @cpu - the cpu of the buffer to copy.
  437. *
  438. * Flip the trace of a single CPU buffer between the @tr and the max_tr.
  439. */
  440. void
  441. update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
  442. {
  443. int ret;
  444. WARN_ON_ONCE(!irqs_disabled());
  445. __raw_spin_lock(&ftrace_max_lock);
  446. ftrace_disable_cpu();
  447. ring_buffer_reset(max_tr.buffer);
  448. ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
  449. ftrace_enable_cpu();
  450. WARN_ON_ONCE(ret);
  451. __update_max_tr(tr, tsk, cpu);
  452. __raw_spin_unlock(&ftrace_max_lock);
  453. }
  454. /**
  455. * register_tracer - register a tracer with the ftrace system.
  456. * @type - the plugin for the tracer
  457. *
  458. * Register a new plugin tracer.
  459. */
  460. int register_tracer(struct tracer *type)
  461. {
  462. struct tracer *t;
  463. int len;
  464. int ret = 0;
  465. if (!type->name) {
  466. pr_info("Tracer must have a name\n");
  467. return -1;
  468. }
  469. /*
  470. * When this gets called we hold the BKL which means that
  471. * preemption is disabled. Various trace selftests however
  472. * need to disable and enable preemption for successful tests.
  473. * So we drop the BKL here and grab it after the tests again.
  474. */
  475. unlock_kernel();
  476. mutex_lock(&trace_types_lock);
  477. for (t = trace_types; t; t = t->next) {
  478. if (strcmp(type->name, t->name) == 0) {
  479. /* already found */
  480. pr_info("Trace %s already registered\n",
  481. type->name);
  482. ret = -1;
  483. goto out;
  484. }
  485. }
  486. if (!type->set_flag)
  487. type->set_flag = &dummy_set_flag;
  488. if (!type->flags)
  489. type->flags = &dummy_tracer_flags;
  490. else
  491. if (!type->flags->opts)
  492. type->flags->opts = dummy_tracer_opt;
  493. #ifdef CONFIG_FTRACE_STARTUP_TEST
  494. if (type->selftest) {
  495. struct tracer *saved_tracer = current_trace;
  496. struct trace_array *tr = &global_trace;
  497. int i;
  498. /*
  499. * Run a selftest on this tracer.
  500. * Here we reset the trace buffer, and set the current
  501. * tracer to be this tracer. The tracer can then run some
  502. * internal tracing to verify that everything is in order.
  503. * If we fail, we do not register this tracer.
  504. */
  505. for_each_tracing_cpu(i)
  506. tracing_reset(tr, i);
  507. current_trace = type;
  508. /* the test is responsible for initializing and enabling */
  509. pr_info("Testing tracer %s: ", type->name);
  510. ret = type->selftest(type, tr);
  511. /* the test is responsible for resetting too */
  512. current_trace = saved_tracer;
  513. if (ret) {
  514. printk(KERN_CONT "FAILED!\n");
  515. goto out;
  516. }
  517. /* Only reset on passing, to avoid touching corrupted buffers */
  518. for_each_tracing_cpu(i)
  519. tracing_reset(tr, i);
  520. printk(KERN_CONT "PASSED\n");
  521. }
  522. #endif
  523. type->next = trace_types;
  524. trace_types = type;
  525. len = strlen(type->name);
  526. if (len > max_tracer_type_len)
  527. max_tracer_type_len = len;
  528. out:
  529. mutex_unlock(&trace_types_lock);
  530. lock_kernel();
  531. return ret;
  532. }
  533. void unregister_tracer(struct tracer *type)
  534. {
  535. struct tracer **t;
  536. int len;
  537. mutex_lock(&trace_types_lock);
  538. for (t = &trace_types; *t; t = &(*t)->next) {
  539. if (*t == type)
  540. goto found;
  541. }
  542. pr_info("Trace %s not registered\n", type->name);
  543. goto out;
  544. found:
  545. *t = (*t)->next;
  546. if (strlen(type->name) != max_tracer_type_len)
  547. goto out;
  548. max_tracer_type_len = 0;
  549. for (t = &trace_types; *t; t = &(*t)->next) {
  550. len = strlen((*t)->name);
  551. if (len > max_tracer_type_len)
  552. max_tracer_type_len = len;
  553. }
  554. out:
  555. mutex_unlock(&trace_types_lock);
  556. }
  557. void tracing_reset(struct trace_array *tr, int cpu)
  558. {
  559. ftrace_disable_cpu();
  560. ring_buffer_reset_cpu(tr->buffer, cpu);
  561. ftrace_enable_cpu();
  562. }
  563. #define SAVED_CMDLINES 128
  564. static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
  565. static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
  566. static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
  567. static int cmdline_idx;
  568. static DEFINE_SPINLOCK(trace_cmdline_lock);
  569. /* temporary disable recording */
  570. atomic_t trace_record_cmdline_disabled __read_mostly;
  571. static void trace_init_cmdlines(void)
  572. {
  573. memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
  574. memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
  575. cmdline_idx = 0;
  576. }
  577. static int trace_stop_count;
  578. static DEFINE_SPINLOCK(tracing_start_lock);
  579. /**
  580. * ftrace_off_permanent - disable all ftrace code permanently
  581. *
  582. * This should only be called when a serious anomally has
  583. * been detected. This will turn off the function tracing,
  584. * ring buffers, and other tracing utilites. It takes no
  585. * locks and can be called from any context.
  586. */
  587. void ftrace_off_permanent(void)
  588. {
  589. tracing_disabled = 1;
  590. ftrace_stop();
  591. tracing_off_permanent();
  592. }
  593. /**
  594. * tracing_start - quick start of the tracer
  595. *
  596. * If tracing is enabled but was stopped by tracing_stop,
  597. * this will start the tracer back up.
  598. */
  599. void tracing_start(void)
  600. {
  601. struct ring_buffer *buffer;
  602. unsigned long flags;
  603. if (tracing_disabled)
  604. return;
  605. spin_lock_irqsave(&tracing_start_lock, flags);
  606. if (--trace_stop_count)
  607. goto out;
  608. if (trace_stop_count < 0) {
  609. /* Someone screwed up their debugging */
  610. WARN_ON_ONCE(1);
  611. trace_stop_count = 0;
  612. goto out;
  613. }
  614. buffer = global_trace.buffer;
  615. if (buffer)
  616. ring_buffer_record_enable(buffer);
  617. buffer = max_tr.buffer;
  618. if (buffer)
  619. ring_buffer_record_enable(buffer);
  620. ftrace_start();
  621. out:
  622. spin_unlock_irqrestore(&tracing_start_lock, flags);
  623. }
  624. /**
  625. * tracing_stop - quick stop of the tracer
  626. *
  627. * Light weight way to stop tracing. Use in conjunction with
  628. * tracing_start.
  629. */
  630. void tracing_stop(void)
  631. {
  632. struct ring_buffer *buffer;
  633. unsigned long flags;
  634. ftrace_stop();
  635. spin_lock_irqsave(&tracing_start_lock, flags);
  636. if (trace_stop_count++)
  637. goto out;
  638. buffer = global_trace.buffer;
  639. if (buffer)
  640. ring_buffer_record_disable(buffer);
  641. buffer = max_tr.buffer;
  642. if (buffer)
  643. ring_buffer_record_disable(buffer);
  644. out:
  645. spin_unlock_irqrestore(&tracing_start_lock, flags);
  646. }
  647. void trace_stop_cmdline_recording(void);
  648. static void trace_save_cmdline(struct task_struct *tsk)
  649. {
  650. unsigned map;
  651. unsigned idx;
  652. if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
  653. return;
  654. /*
  655. * It's not the end of the world if we don't get
  656. * the lock, but we also don't want to spin
  657. * nor do we want to disable interrupts,
  658. * so if we miss here, then better luck next time.
  659. */
  660. if (!spin_trylock(&trace_cmdline_lock))
  661. return;
  662. idx = map_pid_to_cmdline[tsk->pid];
  663. if (idx >= SAVED_CMDLINES) {
  664. idx = (cmdline_idx + 1) % SAVED_CMDLINES;
  665. map = map_cmdline_to_pid[idx];
  666. if (map <= PID_MAX_DEFAULT)
  667. map_pid_to_cmdline[map] = (unsigned)-1;
  668. map_pid_to_cmdline[tsk->pid] = idx;
  669. cmdline_idx = idx;
  670. }
  671. memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
  672. spin_unlock(&trace_cmdline_lock);
  673. }
  674. char *trace_find_cmdline(int pid)
  675. {
  676. char *cmdline = "<...>";
  677. unsigned map;
  678. if (!pid)
  679. return "<idle>";
  680. if (pid > PID_MAX_DEFAULT)
  681. goto out;
  682. map = map_pid_to_cmdline[pid];
  683. if (map >= SAVED_CMDLINES)
  684. goto out;
  685. cmdline = saved_cmdlines[map];
  686. out:
  687. return cmdline;
  688. }
  689. void tracing_record_cmdline(struct task_struct *tsk)
  690. {
  691. if (atomic_read(&trace_record_cmdline_disabled))
  692. return;
  693. trace_save_cmdline(tsk);
  694. }
  695. void
  696. tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
  697. int pc)
  698. {
  699. struct task_struct *tsk = current;
  700. entry->preempt_count = pc & 0xff;
  701. entry->pid = (tsk) ? tsk->pid : 0;
  702. entry->tgid = (tsk) ? tsk->tgid : 0;
  703. entry->flags =
  704. #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
  705. (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
  706. #else
  707. TRACE_FLAG_IRQS_NOSUPPORT |
  708. #endif
  709. ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
  710. ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
  711. (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
  712. }
  713. void
  714. trace_function(struct trace_array *tr, struct trace_array_cpu *data,
  715. unsigned long ip, unsigned long parent_ip, unsigned long flags,
  716. int pc)
  717. {
  718. struct ring_buffer_event *event;
  719. struct ftrace_entry *entry;
  720. unsigned long irq_flags;
  721. /* If we are reading the ring buffer, don't trace */
  722. if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
  723. return;
  724. event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
  725. &irq_flags);
  726. if (!event)
  727. return;
  728. entry = ring_buffer_event_data(event);
  729. tracing_generic_entry_update(&entry->ent, flags, pc);
  730. entry->ent.type = TRACE_FN;
  731. entry->ip = ip;
  732. entry->parent_ip = parent_ip;
  733. ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
  734. }
  735. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  736. static void __trace_graph_entry(struct trace_array *tr,
  737. struct trace_array_cpu *data,
  738. struct ftrace_graph_ent *trace,
  739. unsigned long flags,
  740. int pc)
  741. {
  742. struct ring_buffer_event *event;
  743. struct ftrace_graph_ent_entry *entry;
  744. unsigned long irq_flags;
  745. if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
  746. return;
  747. event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry),
  748. &irq_flags);
  749. if (!event)
  750. return;
  751. entry = ring_buffer_event_data(event);
  752. tracing_generic_entry_update(&entry->ent, flags, pc);
  753. entry->ent.type = TRACE_GRAPH_ENT;
  754. entry->graph_ent = *trace;
  755. ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags);
  756. }
  757. static void __trace_graph_return(struct trace_array *tr,
  758. struct trace_array_cpu *data,
  759. struct ftrace_graph_ret *trace,
  760. unsigned long flags,
  761. int pc)
  762. {
  763. struct ring_buffer_event *event;
  764. struct ftrace_graph_ret_entry *entry;
  765. unsigned long irq_flags;
  766. if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
  767. return;
  768. event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry),
  769. &irq_flags);
  770. if (!event)
  771. return;
  772. entry = ring_buffer_event_data(event);
  773. tracing_generic_entry_update(&entry->ent, flags, pc);
  774. entry->ent.type = TRACE_GRAPH_RET;
  775. entry->ret = *trace;
  776. ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags);
  777. }
  778. #endif
  779. void
  780. ftrace(struct trace_array *tr, struct trace_array_cpu *data,
  781. unsigned long ip, unsigned long parent_ip, unsigned long flags,
  782. int pc)
  783. {
  784. if (likely(!atomic_read(&data->disabled)))
  785. trace_function(tr, data, ip, parent_ip, flags, pc);
  786. }
  787. static void ftrace_trace_stack(struct trace_array *tr,
  788. struct trace_array_cpu *data,
  789. unsigned long flags,
  790. int skip, int pc)
  791. {
  792. #ifdef CONFIG_STACKTRACE
  793. struct ring_buffer_event *event;
  794. struct stack_entry *entry;
  795. struct stack_trace trace;
  796. unsigned long irq_flags;
  797. if (!(trace_flags & TRACE_ITER_STACKTRACE))
  798. return;
  799. event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
  800. &irq_flags);
  801. if (!event)
  802. return;
  803. entry = ring_buffer_event_data(event);
  804. tracing_generic_entry_update(&entry->ent, flags, pc);
  805. entry->ent.type = TRACE_STACK;
  806. memset(&entry->caller, 0, sizeof(entry->caller));
  807. trace.nr_entries = 0;
  808. trace.max_entries = FTRACE_STACK_ENTRIES;
  809. trace.skip = skip;
  810. trace.entries = entry->caller;
  811. save_stack_trace(&trace);
  812. ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
  813. #endif
  814. }
  815. void __trace_stack(struct trace_array *tr,
  816. struct trace_array_cpu *data,
  817. unsigned long flags,
  818. int skip)
  819. {
  820. ftrace_trace_stack(tr, data, flags, skip, preempt_count());
  821. }
  822. static void ftrace_trace_userstack(struct trace_array *tr,
  823. struct trace_array_cpu *data,
  824. unsigned long flags, int pc)
  825. {
  826. #ifdef CONFIG_STACKTRACE
  827. struct ring_buffer_event *event;
  828. struct userstack_entry *entry;
  829. struct stack_trace trace;
  830. unsigned long irq_flags;
  831. if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
  832. return;
  833. event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
  834. &irq_flags);
  835. if (!event)
  836. return;
  837. entry = ring_buffer_event_data(event);
  838. tracing_generic_entry_update(&entry->ent, flags, pc);
  839. entry->ent.type = TRACE_USER_STACK;
  840. memset(&entry->caller, 0, sizeof(entry->caller));
  841. trace.nr_entries = 0;
  842. trace.max_entries = FTRACE_STACK_ENTRIES;
  843. trace.skip = 0;
  844. trace.entries = entry->caller;
  845. save_stack_trace_user(&trace);
  846. ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
  847. #endif
  848. }
  849. void __trace_userstack(struct trace_array *tr,
  850. struct trace_array_cpu *data,
  851. unsigned long flags)
  852. {
  853. ftrace_trace_userstack(tr, data, flags, preempt_count());
  854. }
  855. static void
  856. ftrace_trace_special(void *__tr, void *__data,
  857. unsigned long arg1, unsigned long arg2, unsigned long arg3,
  858. int pc)
  859. {
  860. struct ring_buffer_event *event;
  861. struct trace_array_cpu *data = __data;
  862. struct trace_array *tr = __tr;
  863. struct special_entry *entry;
  864. unsigned long irq_flags;
  865. event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
  866. &irq_flags);
  867. if (!event)
  868. return;
  869. entry = ring_buffer_event_data(event);
  870. tracing_generic_entry_update(&entry->ent, 0, pc);
  871. entry->ent.type = TRACE_SPECIAL;
  872. entry->arg1 = arg1;
  873. entry->arg2 = arg2;
  874. entry->arg3 = arg3;
  875. ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
  876. ftrace_trace_stack(tr, data, irq_flags, 4, pc);
  877. ftrace_trace_userstack(tr, data, irq_flags, pc);
  878. trace_wake_up();
  879. }
  880. void
  881. __trace_special(void *__tr, void *__data,
  882. unsigned long arg1, unsigned long arg2, unsigned long arg3)
  883. {
  884. ftrace_trace_special(__tr, __data, arg1, arg2, arg3, preempt_count());
  885. }
  886. void
  887. tracing_sched_switch_trace(struct trace_array *tr,
  888. struct trace_array_cpu *data,
  889. struct task_struct *prev,
  890. struct task_struct *next,
  891. unsigned long flags, int pc)
  892. {
  893. struct ring_buffer_event *event;
  894. struct ctx_switch_entry *entry;
  895. unsigned long irq_flags;
  896. event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
  897. &irq_flags);
  898. if (!event)
  899. return;
  900. entry = ring_buffer_event_data(event);
  901. tracing_generic_entry_update(&entry->ent, flags, pc);
  902. entry->ent.type = TRACE_CTX;
  903. entry->prev_pid = prev->pid;
  904. entry->prev_prio = prev->prio;
  905. entry->prev_state = prev->state;
  906. entry->next_pid = next->pid;
  907. entry->next_prio = next->prio;
  908. entry->next_state = next->state;
  909. entry->next_cpu = task_cpu(next);
  910. ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
  911. ftrace_trace_stack(tr, data, flags, 5, pc);
  912. ftrace_trace_userstack(tr, data, flags, pc);
  913. }
  914. void
  915. tracing_sched_wakeup_trace(struct trace_array *tr,
  916. struct trace_array_cpu *data,
  917. struct task_struct *wakee,
  918. struct task_struct *curr,
  919. unsigned long flags, int pc)
  920. {
  921. struct ring_buffer_event *event;
  922. struct ctx_switch_entry *entry;
  923. unsigned long irq_flags;
  924. event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
  925. &irq_flags);
  926. if (!event)
  927. return;
  928. entry = ring_buffer_event_data(event);
  929. tracing_generic_entry_update(&entry->ent, flags, pc);
  930. entry->ent.type = TRACE_WAKE;
  931. entry->prev_pid = curr->pid;
  932. entry->prev_prio = curr->prio;
  933. entry->prev_state = curr->state;
  934. entry->next_pid = wakee->pid;
  935. entry->next_prio = wakee->prio;
  936. entry->next_state = wakee->state;
  937. entry->next_cpu = task_cpu(wakee);
  938. ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
  939. ftrace_trace_stack(tr, data, flags, 6, pc);
  940. ftrace_trace_userstack(tr, data, flags, pc);
  941. trace_wake_up();
  942. }
  943. void
  944. ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
  945. {
  946. struct trace_array *tr = &global_trace;
  947. struct trace_array_cpu *data;
  948. unsigned long flags;
  949. int cpu;
  950. int pc;
  951. if (tracing_disabled)
  952. return;
  953. pc = preempt_count();
  954. local_irq_save(flags);
  955. cpu = raw_smp_processor_id();
  956. data = tr->data[cpu];
  957. if (likely(atomic_inc_return(&data->disabled) == 1))
  958. ftrace_trace_special(tr, data, arg1, arg2, arg3, pc);
  959. atomic_dec(&data->disabled);
  960. local_irq_restore(flags);
  961. }
  962. #ifdef CONFIG_FUNCTION_TRACER
  963. static void
  964. function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
  965. {
  966. struct trace_array *tr = &global_trace;
  967. struct trace_array_cpu *data;
  968. unsigned long flags;
  969. long disabled;
  970. int cpu, resched;
  971. int pc;
  972. if (unlikely(!ftrace_function_enabled))
  973. return;
  974. pc = preempt_count();
  975. resched = ftrace_preempt_disable();
  976. local_save_flags(flags);
  977. cpu = raw_smp_processor_id();
  978. data = tr->data[cpu];
  979. disabled = atomic_inc_return(&data->disabled);
  980. if (likely(disabled == 1))
  981. trace_function(tr, data, ip, parent_ip, flags, pc);
  982. atomic_dec(&data->disabled);
  983. ftrace_preempt_enable(resched);
  984. }
  985. static void
  986. function_trace_call(unsigned long ip, unsigned long parent_ip)
  987. {
  988. struct trace_array *tr = &global_trace;
  989. struct trace_array_cpu *data;
  990. unsigned long flags;
  991. long disabled;
  992. int cpu;
  993. int pc;
  994. if (unlikely(!ftrace_function_enabled))
  995. return;
  996. /*
  997. * Need to use raw, since this must be called before the
  998. * recursive protection is performed.
  999. */
  1000. local_irq_save(flags);
  1001. cpu = raw_smp_processor_id();
  1002. data = tr->data[cpu];
  1003. disabled = atomic_inc_return(&data->disabled);
  1004. if (likely(disabled == 1)) {
  1005. pc = preempt_count();
  1006. trace_function(tr, data, ip, parent_ip, flags, pc);
  1007. }
  1008. atomic_dec(&data->disabled);
  1009. local_irq_restore(flags);
  1010. }
  1011. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  1012. void trace_graph_entry(struct ftrace_graph_ent *trace)
  1013. {
  1014. struct trace_array *tr = &global_trace;
  1015. struct trace_array_cpu *data;
  1016. unsigned long flags;
  1017. long disabled;
  1018. int cpu;
  1019. int pc;
  1020. raw_local_irq_save(flags);
  1021. cpu = raw_smp_processor_id();
  1022. data = tr->data[cpu];
  1023. disabled = atomic_inc_return(&data->disabled);
  1024. if (likely(disabled == 1)) {
  1025. pc = preempt_count();
  1026. __trace_graph_entry(tr, data, trace, flags, pc);
  1027. }
  1028. atomic_dec(&data->disabled);
  1029. raw_local_irq_restore(flags);
  1030. }
  1031. void trace_graph_return(struct ftrace_graph_ret *trace)
  1032. {
  1033. struct trace_array *tr = &global_trace;
  1034. struct trace_array_cpu *data;
  1035. unsigned long flags;
  1036. long disabled;
  1037. int cpu;
  1038. int pc;
  1039. raw_local_irq_save(flags);
  1040. cpu = raw_smp_processor_id();
  1041. data = tr->data[cpu];
  1042. disabled = atomic_inc_return(&data->disabled);
  1043. if (likely(disabled == 1)) {
  1044. pc = preempt_count();
  1045. __trace_graph_return(tr, data, trace, flags, pc);
  1046. }
  1047. atomic_dec(&data->disabled);
  1048. raw_local_irq_restore(flags);
  1049. }
  1050. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  1051. static struct ftrace_ops trace_ops __read_mostly =
  1052. {
  1053. .func = function_trace_call,
  1054. };
  1055. void tracing_start_function_trace(void)
  1056. {
  1057. ftrace_function_enabled = 0;
  1058. if (trace_flags & TRACE_ITER_PREEMPTONLY)
  1059. trace_ops.func = function_trace_call_preempt_only;
  1060. else
  1061. trace_ops.func = function_trace_call;
  1062. register_ftrace_function(&trace_ops);
  1063. ftrace_function_enabled = 1;
  1064. }
  1065. void tracing_stop_function_trace(void)
  1066. {
  1067. ftrace_function_enabled = 0;
  1068. unregister_ftrace_function(&trace_ops);
  1069. }
  1070. #endif
  1071. enum trace_file_type {
  1072. TRACE_FILE_LAT_FMT = 1,
  1073. TRACE_FILE_ANNOTATE = 2,
  1074. };
  1075. static void trace_iterator_increment(struct trace_iterator *iter, int cpu)
  1076. {
  1077. /* Don't allow ftrace to trace into the ring buffers */
  1078. ftrace_disable_cpu();
  1079. iter->idx++;
  1080. if (iter->buffer_iter[iter->cpu])
  1081. ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
  1082. ftrace_enable_cpu();
  1083. }
  1084. static struct trace_entry *
  1085. peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
  1086. {
  1087. struct ring_buffer_event *event;
  1088. struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
  1089. /* Don't allow ftrace to trace into the ring buffers */
  1090. ftrace_disable_cpu();
  1091. if (buf_iter)
  1092. event = ring_buffer_iter_peek(buf_iter, ts);
  1093. else
  1094. event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
  1095. ftrace_enable_cpu();
  1096. return event ? ring_buffer_event_data(event) : NULL;
  1097. }
  1098. static struct trace_entry *
  1099. __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
  1100. {
  1101. struct ring_buffer *buffer = iter->tr->buffer;
  1102. struct trace_entry *ent, *next = NULL;
  1103. u64 next_ts = 0, ts;
  1104. int next_cpu = -1;
  1105. int cpu;
  1106. for_each_tracing_cpu(cpu) {
  1107. if (ring_buffer_empty_cpu(buffer, cpu))
  1108. continue;
  1109. ent = peek_next_entry(iter, cpu, &ts);
  1110. /*
  1111. * Pick the entry with the smallest timestamp:
  1112. */
  1113. if (ent && (!next || ts < next_ts)) {
  1114. next = ent;
  1115. next_cpu = cpu;
  1116. next_ts = ts;
  1117. }
  1118. }
  1119. if (ent_cpu)
  1120. *ent_cpu = next_cpu;
  1121. if (ent_ts)
  1122. *ent_ts = next_ts;
  1123. return next;
  1124. }
  1125. /* Find the next real entry, without updating the iterator itself */
  1126. static struct trace_entry *
  1127. find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
  1128. {
  1129. return __find_next_entry(iter, ent_cpu, ent_ts);
  1130. }
  1131. /* Find the next real entry, and increment the iterator to the next entry */
  1132. static void *find_next_entry_inc(struct trace_iterator *iter)
  1133. {
  1134. iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
  1135. if (iter->ent)
  1136. trace_iterator_increment(iter, iter->cpu);
  1137. return iter->ent ? iter : NULL;
  1138. }
  1139. static void trace_consume(struct trace_iterator *iter)
  1140. {
  1141. /* Don't allow ftrace to trace into the ring buffers */
  1142. ftrace_disable_cpu();
  1143. ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
  1144. ftrace_enable_cpu();
  1145. }
  1146. static void *s_next(struct seq_file *m, void *v, loff_t *pos)
  1147. {
  1148. struct trace_iterator *iter = m->private;
  1149. int i = (int)*pos;
  1150. void *ent;
  1151. (*pos)++;
  1152. /* can't go backwards */
  1153. if (iter->idx > i)
  1154. return NULL;
  1155. if (iter->idx < 0)
  1156. ent = find_next_entry_inc(iter);
  1157. else
  1158. ent = iter;
  1159. while (ent && iter->idx < i)
  1160. ent = find_next_entry_inc(iter);
  1161. iter->pos = *pos;
  1162. return ent;
  1163. }
  1164. static void *s_start(struct seq_file *m, loff_t *pos)
  1165. {
  1166. struct trace_iterator *iter = m->private;
  1167. void *p = NULL;
  1168. loff_t l = 0;
  1169. int cpu;
  1170. mutex_lock(&trace_types_lock);
  1171. if (!current_trace || current_trace != iter->trace) {
  1172. mutex_unlock(&trace_types_lock);
  1173. return NULL;
  1174. }
  1175. atomic_inc(&trace_record_cmdline_disabled);
  1176. if (*pos != iter->pos) {
  1177. iter->ent = NULL;
  1178. iter->cpu = 0;
  1179. iter->idx = -1;
  1180. ftrace_disable_cpu();
  1181. for_each_tracing_cpu(cpu) {
  1182. ring_buffer_iter_reset(iter->buffer_iter[cpu]);
  1183. }
  1184. ftrace_enable_cpu();
  1185. for (p = iter; p && l < *pos; p = s_next(m, p, &l))
  1186. ;
  1187. } else {
  1188. l = *pos - 1;
  1189. p = s_next(m, p, &l);
  1190. }
  1191. return p;
  1192. }
  1193. static void s_stop(struct seq_file *m, void *p)
  1194. {
  1195. atomic_dec(&trace_record_cmdline_disabled);
  1196. mutex_unlock(&trace_types_lock);
  1197. }
  1198. #ifdef CONFIG_KRETPROBES
  1199. static inline const char *kretprobed(const char *name)
  1200. {
  1201. static const char tramp_name[] = "kretprobe_trampoline";
  1202. int size = sizeof(tramp_name);
  1203. if (strncmp(tramp_name, name, size) == 0)
  1204. return "[unknown/kretprobe'd]";
  1205. return name;
  1206. }
  1207. #else
  1208. static inline const char *kretprobed(const char *name)
  1209. {
  1210. return name;
  1211. }
  1212. #endif /* CONFIG_KRETPROBES */
  1213. static int
  1214. seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
  1215. {
  1216. #ifdef CONFIG_KALLSYMS
  1217. char str[KSYM_SYMBOL_LEN];
  1218. const char *name;
  1219. kallsyms_lookup(address, NULL, NULL, NULL, str);
  1220. name = kretprobed(str);
  1221. return trace_seq_printf(s, fmt, name);
  1222. #endif
  1223. return 1;
  1224. }
  1225. static int
  1226. seq_print_sym_offset(struct trace_seq *s, const char *fmt,
  1227. unsigned long address)
  1228. {
  1229. #ifdef CONFIG_KALLSYMS
  1230. char str[KSYM_SYMBOL_LEN];
  1231. const char *name;
  1232. sprint_symbol(str, address);
  1233. name = kretprobed(str);
  1234. return trace_seq_printf(s, fmt, name);
  1235. #endif
  1236. return 1;
  1237. }
  1238. #ifndef CONFIG_64BIT
  1239. # define IP_FMT "%08lx"
  1240. #else
  1241. # define IP_FMT "%016lx"
  1242. #endif
  1243. int
  1244. seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
  1245. {
  1246. int ret;
  1247. if (!ip)
  1248. return trace_seq_printf(s, "0");
  1249. if (sym_flags & TRACE_ITER_SYM_OFFSET)
  1250. ret = seq_print_sym_offset(s, "%s", ip);
  1251. else
  1252. ret = seq_print_sym_short(s, "%s", ip);
  1253. if (!ret)
  1254. return 0;
  1255. if (sym_flags & TRACE_ITER_SYM_ADDR)
  1256. ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
  1257. return ret;
  1258. }
  1259. static inline int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
  1260. unsigned long ip, unsigned long sym_flags)
  1261. {
  1262. struct file *file = NULL;
  1263. unsigned long vmstart = 0;
  1264. int ret = 1;
  1265. if (mm) {
  1266. const struct vm_area_struct *vma;
  1267. down_read(&mm->mmap_sem);
  1268. vma = find_vma(mm, ip);
  1269. if (vma) {
  1270. file = vma->vm_file;
  1271. vmstart = vma->vm_start;
  1272. }
  1273. if (file) {
  1274. ret = trace_seq_path(s, &file->f_path);
  1275. if (ret)
  1276. ret = trace_seq_printf(s, "[+0x%lx]", ip - vmstart);
  1277. }
  1278. up_read(&mm->mmap_sem);
  1279. }
  1280. if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
  1281. ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
  1282. return ret;
  1283. }
  1284. static int
  1285. seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
  1286. unsigned long sym_flags)
  1287. {
  1288. struct mm_struct *mm = NULL;
  1289. int ret = 1;
  1290. unsigned int i;
  1291. if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
  1292. struct task_struct *task;
  1293. /*
  1294. * we do the lookup on the thread group leader,
  1295. * since individual threads might have already quit!
  1296. */
  1297. rcu_read_lock();
  1298. task = find_task_by_vpid(entry->ent.tgid);
  1299. if (task)
  1300. mm = get_task_mm(task);
  1301. rcu_read_unlock();
  1302. }
  1303. for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
  1304. unsigned long ip = entry->caller[i];
  1305. if (ip == ULONG_MAX || !ret)
  1306. break;
  1307. if (i && ret)
  1308. ret = trace_seq_puts(s, " <- ");
  1309. if (!ip) {
  1310. if (ret)
  1311. ret = trace_seq_puts(s, "??");
  1312. continue;
  1313. }
  1314. if (!ret)
  1315. break;
  1316. if (ret)
  1317. ret = seq_print_user_ip(s, mm, ip, sym_flags);
  1318. }
  1319. if (mm)
  1320. mmput(mm);
  1321. return ret;
  1322. }
  1323. static void print_lat_help_header(struct seq_file *m)
  1324. {
  1325. seq_puts(m, "# _------=> CPU# \n");
  1326. seq_puts(m, "# / _-----=> irqs-off \n");
  1327. seq_puts(m, "# | / _----=> need-resched \n");
  1328. seq_puts(m, "# || / _---=> hardirq/softirq \n");
  1329. seq_puts(m, "# ||| / _--=> preempt-depth \n");
  1330. seq_puts(m, "# |||| / \n");
  1331. seq_puts(m, "# ||||| delay \n");
  1332. seq_puts(m, "# cmd pid ||||| time | caller \n");
  1333. seq_puts(m, "# \\ / ||||| \\ | / \n");
  1334. }
  1335. static void print_func_help_header(struct seq_file *m)
  1336. {
  1337. seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
  1338. seq_puts(m, "# | | | | |\n");
  1339. }
  1340. static void
  1341. print_trace_header(struct seq_file *m, struct trace_iterator *iter)
  1342. {
  1343. unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
  1344. struct trace_array *tr = iter->tr;
  1345. struct trace_array_cpu *data = tr->data[tr->cpu];
  1346. struct tracer *type = current_trace;
  1347. unsigned long total;
  1348. unsigned long entries;
  1349. const char *name = "preemption";
  1350. if (type)
  1351. name = type->name;
  1352. entries = ring_buffer_entries(iter->tr->buffer);
  1353. total = entries +
  1354. ring_buffer_overruns(iter->tr->buffer);
  1355. seq_printf(m, "%s latency trace v1.1.5 on %s\n",
  1356. name, UTS_RELEASE);
  1357. seq_puts(m, "-----------------------------------"
  1358. "---------------------------------\n");
  1359. seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
  1360. " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
  1361. nsecs_to_usecs(data->saved_latency),
  1362. entries,
  1363. total,
  1364. tr->cpu,
  1365. #if defined(CONFIG_PREEMPT_NONE)
  1366. "server",
  1367. #elif defined(CONFIG_PREEMPT_VOLUNTARY)
  1368. "desktop",
  1369. #elif defined(CONFIG_PREEMPT)
  1370. "preempt",
  1371. #else
  1372. "unknown",
  1373. #endif
  1374. /* These are reserved for later use */
  1375. 0, 0, 0, 0);
  1376. #ifdef CONFIG_SMP
  1377. seq_printf(m, " #P:%d)\n", num_online_cpus());
  1378. #else
  1379. seq_puts(m, ")\n");
  1380. #endif
  1381. seq_puts(m, " -----------------\n");
  1382. seq_printf(m, " | task: %.16s-%d "
  1383. "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
  1384. data->comm, data->pid, data->uid, data->nice,
  1385. data->policy, data->rt_priority);
  1386. seq_puts(m, " -----------------\n");
  1387. if (data->critical_start) {
  1388. seq_puts(m, " => started at: ");
  1389. seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
  1390. trace_print_seq(m, &iter->seq);
  1391. seq_puts(m, "\n => ended at: ");
  1392. seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
  1393. trace_print_seq(m, &iter->seq);
  1394. seq_puts(m, "\n");
  1395. }
  1396. seq_puts(m, "\n");
  1397. }
  1398. static void
  1399. lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
  1400. {
  1401. int hardirq, softirq;
  1402. char *comm;
  1403. comm = trace_find_cmdline(entry->pid);
  1404. trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
  1405. trace_seq_printf(s, "%3d", cpu);
  1406. trace_seq_printf(s, "%c%c",
  1407. (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
  1408. (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' : '.',
  1409. ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
  1410. hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
  1411. softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
  1412. if (hardirq && softirq) {
  1413. trace_seq_putc(s, 'H');
  1414. } else {
  1415. if (hardirq) {
  1416. trace_seq_putc(s, 'h');
  1417. } else {
  1418. if (softirq)
  1419. trace_seq_putc(s, 's');
  1420. else
  1421. trace_seq_putc(s, '.');
  1422. }
  1423. }
  1424. if (entry->preempt_count)
  1425. trace_seq_printf(s, "%x", entry->preempt_count);
  1426. else
  1427. trace_seq_puts(s, ".");
  1428. }
  1429. unsigned long preempt_mark_thresh = 100;
  1430. static void
  1431. lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
  1432. unsigned long rel_usecs)
  1433. {
  1434. trace_seq_printf(s, " %4lldus", abs_usecs);
  1435. if (rel_usecs > preempt_mark_thresh)
  1436. trace_seq_puts(s, "!: ");
  1437. else if (rel_usecs > 1)
  1438. trace_seq_puts(s, "+: ");
  1439. else
  1440. trace_seq_puts(s, " : ");
  1441. }
  1442. static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
  1443. /*
  1444. * The message is supposed to contain an ending newline.
  1445. * If the printing stops prematurely, try to add a newline of our own.
  1446. */
  1447. void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
  1448. {
  1449. struct trace_entry *ent;
  1450. struct trace_field_cont *cont;
  1451. bool ok = true;
  1452. ent = peek_next_entry(iter, iter->cpu, NULL);
  1453. if (!ent || ent->type != TRACE_CONT) {
  1454. trace_seq_putc(s, '\n');
  1455. return;
  1456. }
  1457. do {
  1458. cont = (struct trace_field_cont *)ent;
  1459. if (ok)
  1460. ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
  1461. ftrace_disable_cpu();
  1462. if (iter->buffer_iter[iter->cpu])
  1463. ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
  1464. else
  1465. ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
  1466. ftrace_enable_cpu();
  1467. ent = peek_next_entry(iter, iter->cpu, NULL);
  1468. } while (ent && ent->type == TRACE_CONT);
  1469. if (!ok)
  1470. trace_seq_putc(s, '\n');
  1471. }
  1472. static void test_cpu_buff_start(struct trace_iterator *iter)
  1473. {
  1474. struct trace_seq *s = &iter->seq;
  1475. if (!(trace_flags & TRACE_ITER_ANNOTATE))
  1476. return;
  1477. if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
  1478. return;
  1479. if (cpu_isset(iter->cpu, iter->started))
  1480. return;
  1481. cpu_set(iter->cpu, iter->started);
  1482. trace_seq_printf(s, "##### CPU %u buffer started ####\n", iter->cpu);
  1483. }
  1484. static enum print_line_t
  1485. print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
  1486. {
  1487. struct trace_seq *s = &iter->seq;
  1488. unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
  1489. struct trace_entry *next_entry;
  1490. unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
  1491. struct trace_entry *entry = iter->ent;
  1492. unsigned long abs_usecs;
  1493. unsigned long rel_usecs;
  1494. u64 next_ts;
  1495. char *comm;
  1496. int S, T;
  1497. int i;
  1498. unsigned state;
  1499. if (entry->type == TRACE_CONT)
  1500. return TRACE_TYPE_HANDLED;
  1501. test_cpu_buff_start(iter);
  1502. next_entry = find_next_entry(iter, NULL, &next_ts);
  1503. if (!next_entry)
  1504. next_ts = iter->ts;
  1505. rel_usecs = ns2usecs(next_ts - iter->ts);
  1506. abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
  1507. if (verbose) {
  1508. comm = trace_find_cmdline(entry->pid);
  1509. trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
  1510. " %ld.%03ldms (+%ld.%03ldms): ",
  1511. comm,
  1512. entry->pid, cpu, entry->flags,
  1513. entry->preempt_count, trace_idx,
  1514. ns2usecs(iter->ts),
  1515. abs_usecs/1000,
  1516. abs_usecs % 1000, rel_usecs/1000,
  1517. rel_usecs % 1000);
  1518. } else {
  1519. lat_print_generic(s, entry, cpu);
  1520. lat_print_timestamp(s, abs_usecs, rel_usecs);
  1521. }
  1522. switch (entry->type) {
  1523. case TRACE_FN: {
  1524. struct ftrace_entry *field;
  1525. trace_assign_type(field, entry);
  1526. seq_print_ip_sym(s, field->ip, sym_flags);
  1527. trace_seq_puts(s, " (");
  1528. seq_print_ip_sym(s, field->parent_ip, sym_flags);
  1529. trace_seq_puts(s, ")\n");
  1530. break;
  1531. }
  1532. case TRACE_CTX:
  1533. case TRACE_WAKE: {
  1534. struct ctx_switch_entry *field;
  1535. trace_assign_type(field, entry);
  1536. T = field->next_state < sizeof(state_to_char) ?
  1537. state_to_char[field->next_state] : 'X';
  1538. state = field->prev_state ?
  1539. __ffs(field->prev_state) + 1 : 0;
  1540. S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
  1541. comm = trace_find_cmdline(field->next_pid);
  1542. trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
  1543. field->prev_pid,
  1544. field->prev_prio,
  1545. S, entry->type == TRACE_CTX ? "==>" : " +",
  1546. field->next_cpu,
  1547. field->next_pid,
  1548. field->next_prio,
  1549. T, comm);
  1550. break;
  1551. }
  1552. case TRACE_SPECIAL: {
  1553. struct special_entry *field;
  1554. trace_assign_type(field, entry);
  1555. trace_seq_printf(s, "# %ld %ld %ld\n",
  1556. field->arg1,
  1557. field->arg2,
  1558. field->arg3);
  1559. break;
  1560. }
  1561. case TRACE_STACK: {
  1562. struct stack_entry *field;
  1563. trace_assign_type(field, entry);
  1564. for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
  1565. if (i)
  1566. trace_seq_puts(s, " <= ");
  1567. seq_print_ip_sym(s, field->caller[i], sym_flags);
  1568. }
  1569. trace_seq_puts(s, "\n");
  1570. break;
  1571. }
  1572. case TRACE_PRINT: {
  1573. struct print_entry *field;
  1574. trace_assign_type(field, entry);
  1575. seq_print_ip_sym(s, field->ip, sym_flags);
  1576. trace_seq_printf(s, ": %s", field->buf);
  1577. if (entry->flags & TRACE_FLAG_CONT)
  1578. trace_seq_print_cont(s, iter);
  1579. break;
  1580. }
  1581. case TRACE_BRANCH: {
  1582. struct trace_branch *field;
  1583. trace_assign_type(field, entry);
  1584. trace_seq_printf(s, "[%s] %s:%s:%d\n",
  1585. field->correct ? " ok " : " MISS ",
  1586. field->func,
  1587. field->file,
  1588. field->line);
  1589. break;
  1590. }
  1591. case TRACE_USER_STACK: {
  1592. struct userstack_entry *field;
  1593. trace_assign_type(field, entry);
  1594. seq_print_userip_objs(field, s, sym_flags);
  1595. trace_seq_putc(s, '\n');
  1596. break;
  1597. }
  1598. default:
  1599. trace_seq_printf(s, "Unknown type %d\n", entry->type);
  1600. }
  1601. return TRACE_TYPE_HANDLED;
  1602. }
  1603. static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
  1604. {
  1605. struct trace_seq *s = &iter->seq;
  1606. unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
  1607. struct trace_entry *entry;
  1608. unsigned long usec_rem;
  1609. unsigned long long t;
  1610. unsigned long secs;
  1611. char *comm;
  1612. int ret;
  1613. int S, T;
  1614. int i;
  1615. entry = iter->ent;
  1616. if (entry->type == TRACE_CONT)
  1617. return TRACE_TYPE_HANDLED;
  1618. test_cpu_buff_start(iter);
  1619. comm = trace_find_cmdline(iter->ent->pid);
  1620. t = ns2usecs(iter->ts);
  1621. usec_rem = do_div(t, 1000000ULL);
  1622. secs = (unsigned long)t;
  1623. ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
  1624. if (!ret)
  1625. return TRACE_TYPE_PARTIAL_LINE;
  1626. ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
  1627. if (!ret)
  1628. return TRACE_TYPE_PARTIAL_LINE;
  1629. ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
  1630. if (!ret)
  1631. return TRACE_TYPE_PARTIAL_LINE;
  1632. switch (entry->type) {
  1633. case TRACE_FN: {
  1634. struct ftrace_entry *field;
  1635. trace_assign_type(field, entry);
  1636. ret = seq_print_ip_sym(s, field->ip, sym_flags);
  1637. if (!ret)
  1638. return TRACE_TYPE_PARTIAL_LINE;
  1639. if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
  1640. field->parent_ip) {
  1641. ret = trace_seq_printf(s, " <-");
  1642. if (!ret)
  1643. return TRACE_TYPE_PARTIAL_LINE;
  1644. ret = seq_print_ip_sym(s,
  1645. field->parent_ip,
  1646. sym_flags);
  1647. if (!ret)
  1648. return TRACE_TYPE_PARTIAL_LINE;
  1649. }
  1650. ret = trace_seq_printf(s, "\n");
  1651. if (!ret)
  1652. return TRACE_TYPE_PARTIAL_LINE;
  1653. break;
  1654. }
  1655. case TRACE_CTX:
  1656. case TRACE_WAKE: {
  1657. struct ctx_switch_entry *field;
  1658. trace_assign_type(field, entry);
  1659. S = field->prev_state < sizeof(state_to_char) ?
  1660. state_to_char[field->prev_state] : 'X';
  1661. T = field->next_state < sizeof(state_to_char) ?
  1662. state_to_char[field->next_state] : 'X';
  1663. ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
  1664. field->prev_pid,
  1665. field->prev_prio,
  1666. S,
  1667. entry->type == TRACE_CTX ? "==>" : " +",
  1668. field->next_cpu,
  1669. field->next_pid,
  1670. field->next_prio,
  1671. T);
  1672. if (!ret)
  1673. return TRACE_TYPE_PARTIAL_LINE;
  1674. break;
  1675. }
  1676. case TRACE_SPECIAL: {
  1677. struct special_entry *field;
  1678. trace_assign_type(field, entry);
  1679. ret = trace_seq_printf(s, "# %ld %ld %ld\n",
  1680. field->arg1,
  1681. field->arg2,
  1682. field->arg3);
  1683. if (!ret)
  1684. return TRACE_TYPE_PARTIAL_LINE;
  1685. break;
  1686. }
  1687. case TRACE_STACK: {
  1688. struct stack_entry *field;
  1689. trace_assign_type(field, entry);
  1690. for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
  1691. if (i) {
  1692. ret = trace_seq_puts(s, " <= ");
  1693. if (!ret)
  1694. return TRACE_TYPE_PARTIAL_LINE;
  1695. }
  1696. ret = seq_print_ip_sym(s, field->caller[i],
  1697. sym_flags);
  1698. if (!ret)
  1699. return TRACE_TYPE_PARTIAL_LINE;
  1700. }
  1701. ret = trace_seq_puts(s, "\n");
  1702. if (!ret)
  1703. return TRACE_TYPE_PARTIAL_LINE;
  1704. break;
  1705. }
  1706. case TRACE_PRINT: {
  1707. struct print_entry *field;
  1708. trace_assign_type(field, entry);
  1709. seq_print_ip_sym(s, field->ip, sym_flags);
  1710. trace_seq_printf(s, ": %s", field->buf);
  1711. if (entry->flags & TRACE_FLAG_CONT)
  1712. trace_seq_print_cont(s, iter);
  1713. break;
  1714. }
  1715. case TRACE_GRAPH_RET: {
  1716. return print_graph_function(iter);
  1717. }
  1718. case TRACE_GRAPH_ENT: {
  1719. return print_graph_function(iter);
  1720. }
  1721. case TRACE_BRANCH: {
  1722. struct trace_branch *field;
  1723. trace_assign_type(field, entry);
  1724. trace_seq_printf(s, "[%s] %s:%s:%d\n",
  1725. field->correct ? " ok " : " MISS ",
  1726. field->func,
  1727. field->file,
  1728. field->line);
  1729. break;
  1730. }
  1731. case TRACE_USER_STACK: {
  1732. struct userstack_entry *field;
  1733. trace_assign_type(field, entry);
  1734. ret = seq_print_userip_objs(field, s, sym_flags);
  1735. if (!ret)
  1736. return TRACE_TYPE_PARTIAL_LINE;
  1737. ret = trace_seq_putc(s, '\n');
  1738. if (!ret)
  1739. return TRACE_TYPE_PARTIAL_LINE;
  1740. break;
  1741. }
  1742. }
  1743. return TRACE_TYPE_HANDLED;
  1744. }
  1745. static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
  1746. {
  1747. struct trace_seq *s = &iter->seq;
  1748. struct trace_entry *entry;
  1749. int ret;
  1750. int S, T;
  1751. entry = iter->ent;
  1752. if (entry->type == TRACE_CONT)
  1753. return TRACE_TYPE_HANDLED;
  1754. ret = trace_seq_printf(s, "%d %d %llu ",
  1755. entry->pid, iter->cpu, iter->ts);
  1756. if (!ret)
  1757. return TRACE_TYPE_PARTIAL_LINE;
  1758. switch (entry->type) {
  1759. case TRACE_FN: {
  1760. struct ftrace_entry *field;
  1761. trace_assign_type(field, entry);
  1762. ret = trace_seq_printf(s, "%x %x\n",
  1763. field->ip,
  1764. field->parent_ip);
  1765. if (!ret)
  1766. return TRACE_TYPE_PARTIAL_LINE;
  1767. break;
  1768. }
  1769. case TRACE_CTX:
  1770. case TRACE_WAKE: {
  1771. struct ctx_switch_entry *field;
  1772. trace_assign_type(field, entry);
  1773. S = field->prev_state < sizeof(state_to_char) ?
  1774. state_to_char[field->prev_state] : 'X';
  1775. T = field->next_state < sizeof(state_to_char) ?
  1776. state_to_char[field->next_state] : 'X';
  1777. if (entry->type == TRACE_WAKE)
  1778. S = '+';
  1779. ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
  1780. field->prev_pid,
  1781. field->prev_prio,
  1782. S,
  1783. field->next_cpu,
  1784. field->next_pid,
  1785. field->next_prio,
  1786. T);
  1787. if (!ret)
  1788. return TRACE_TYPE_PARTIAL_LINE;
  1789. break;
  1790. }
  1791. case TRACE_SPECIAL:
  1792. case TRACE_USER_STACK:
  1793. case TRACE_STACK: {
  1794. struct special_entry *field;
  1795. trace_assign_type(field, entry);
  1796. ret = trace_seq_printf(s, "# %ld %ld %ld\n",
  1797. field->arg1,
  1798. field->arg2,
  1799. field->arg3);
  1800. if (!ret)
  1801. return TRACE_TYPE_PARTIAL_LINE;
  1802. break;
  1803. }
  1804. case TRACE_PRINT: {
  1805. struct print_entry *field;
  1806. trace_assign_type(field, entry);
  1807. trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
  1808. if (entry->flags & TRACE_FLAG_CONT)
  1809. trace_seq_print_cont(s, iter);
  1810. break;
  1811. }
  1812. }
  1813. return TRACE_TYPE_HANDLED;
  1814. }
  1815. #define SEQ_PUT_FIELD_RET(s, x) \
  1816. do { \
  1817. if (!trace_seq_putmem(s, &(x), sizeof(x))) \
  1818. return 0; \
  1819. } while (0)
  1820. #define SEQ_PUT_HEX_FIELD_RET(s, x) \
  1821. do { \
  1822. BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
  1823. if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
  1824. return 0; \
  1825. } while (0)
  1826. static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
  1827. {
  1828. struct trace_seq *s = &iter->seq;
  1829. unsigned char newline = '\n';
  1830. struct trace_entry *entry;
  1831. int S, T;
  1832. entry = iter->ent;
  1833. if (entry->type == TRACE_CONT)
  1834. return TRACE_TYPE_HANDLED;
  1835. SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
  1836. SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
  1837. SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
  1838. switch (entry->type) {
  1839. case TRACE_FN: {
  1840. struct ftrace_entry *field;
  1841. trace_assign_type(field, entry);
  1842. SEQ_PUT_HEX_FIELD_RET(s, field->ip);
  1843. SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
  1844. break;
  1845. }
  1846. case TRACE_CTX:
  1847. case TRACE_WAKE: {
  1848. struct ctx_switch_entry *field;
  1849. trace_assign_type(field, entry);
  1850. S = field->prev_state < sizeof(state_to_char) ?
  1851. state_to_char[field->prev_state] : 'X';
  1852. T = field->next_state < sizeof(state_to_char) ?
  1853. state_to_char[field->next_state] : 'X';
  1854. if (entry->type == TRACE_WAKE)
  1855. S = '+';
  1856. SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
  1857. SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
  1858. SEQ_PUT_HEX_FIELD_RET(s, S);
  1859. SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
  1860. SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
  1861. SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
  1862. SEQ_PUT_HEX_FIELD_RET(s, T);
  1863. break;
  1864. }
  1865. case TRACE_SPECIAL:
  1866. case TRACE_USER_STACK:
  1867. case TRACE_STACK: {
  1868. struct special_entry *field;
  1869. trace_assign_type(field, entry);
  1870. SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
  1871. SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
  1872. SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
  1873. break;
  1874. }
  1875. }
  1876. SEQ_PUT_FIELD_RET(s, newline);
  1877. return TRACE_TYPE_HANDLED;
  1878. }
  1879. static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
  1880. {
  1881. struct trace_seq *s = &iter->seq;
  1882. struct trace_entry *entry;
  1883. entry = iter->ent;
  1884. if (entry->type == TRACE_CONT)
  1885. return TRACE_TYPE_HANDLED;
  1886. SEQ_PUT_FIELD_RET(s, entry->pid);
  1887. SEQ_PUT_FIELD_RET(s, entry->cpu);
  1888. SEQ_PUT_FIELD_RET(s, iter->ts);
  1889. switch (entry->type) {
  1890. case TRACE_FN: {
  1891. struct ftrace_entry *field;
  1892. trace_assign_type(field, entry);
  1893. SEQ_PUT_FIELD_RET(s, field->ip);
  1894. SEQ_PUT_FIELD_RET(s, field->parent_ip);
  1895. break;
  1896. }
  1897. case TRACE_CTX: {
  1898. struct ctx_switch_entry *field;
  1899. trace_assign_type(field, entry);
  1900. SEQ_PUT_FIELD_RET(s, field->prev_pid);
  1901. SEQ_PUT_FIELD_RET(s, field->prev_prio);
  1902. SEQ_PUT_FIELD_RET(s, field->prev_state);
  1903. SEQ_PUT_FIELD_RET(s, field->next_pid);
  1904. SEQ_PUT_FIELD_RET(s, field->next_prio);
  1905. SEQ_PUT_FIELD_RET(s, field->next_state);
  1906. break;
  1907. }
  1908. case TRACE_SPECIAL:
  1909. case TRACE_USER_STACK:
  1910. case TRACE_STACK: {
  1911. struct special_entry *field;
  1912. trace_assign_type(field, entry);
  1913. SEQ_PUT_FIELD_RET(s, field->arg1);
  1914. SEQ_PUT_FIELD_RET(s, field->arg2);
  1915. SEQ_PUT_FIELD_RET(s, field->arg3);
  1916. break;
  1917. }
  1918. }
  1919. return 1;
  1920. }
  1921. static int trace_empty(struct trace_iterator *iter)
  1922. {
  1923. int cpu;
  1924. for_each_tracing_cpu(cpu) {
  1925. if (iter->buffer_iter[cpu]) {
  1926. if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
  1927. return 0;
  1928. } else {
  1929. if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
  1930. return 0;
  1931. }
  1932. }
  1933. return 1;
  1934. }
  1935. static enum print_line_t print_trace_line(struct trace_iterator *iter)
  1936. {
  1937. enum print_line_t ret;
  1938. if (iter->trace && iter->trace->print_line) {
  1939. ret = iter->trace->print_line(iter);
  1940. if (ret != TRACE_TYPE_UNHANDLED)
  1941. return ret;
  1942. }
  1943. if (trace_flags & TRACE_ITER_BIN)
  1944. return print_bin_fmt(iter);
  1945. if (trace_flags & TRACE_ITER_HEX)
  1946. return print_hex_fmt(iter);
  1947. if (trace_flags & TRACE_ITER_RAW)
  1948. return print_raw_fmt(iter);
  1949. if (iter->iter_flags & TRACE_FILE_LAT_FMT)
  1950. return print_lat_fmt(iter, iter->idx, iter->cpu);
  1951. return print_trace_fmt(iter);
  1952. }
  1953. static int s_show(struct seq_file *m, void *v)
  1954. {
  1955. struct trace_iterator *iter = v;
  1956. if (iter->ent == NULL) {
  1957. if (iter->tr) {
  1958. seq_printf(m, "# tracer: %s\n", iter->trace->name);
  1959. seq_puts(m, "#\n");
  1960. }
  1961. if (iter->trace && iter->trace->print_header)
  1962. iter->trace->print_header(m);
  1963. else if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
  1964. /* print nothing if the buffers are empty */
  1965. if (trace_empty(iter))
  1966. return 0;
  1967. print_trace_header(m, iter);
  1968. if (!(trace_flags & TRACE_ITER_VERBOSE))
  1969. print_lat_help_header(m);
  1970. } else {
  1971. if (!(trace_flags & TRACE_ITER_VERBOSE))
  1972. print_func_help_header(m);
  1973. }
  1974. } else {
  1975. print_trace_line(iter);
  1976. trace_print_seq(m, &iter->seq);
  1977. }
  1978. return 0;
  1979. }
  1980. static struct seq_operations tracer_seq_ops = {
  1981. .start = s_start,
  1982. .next = s_next,
  1983. .stop = s_stop,
  1984. .show = s_show,
  1985. };
  1986. static struct trace_iterator *
  1987. __tracing_open(struct inode *inode, struct file *file, int *ret)
  1988. {
  1989. struct trace_iterator *iter;
  1990. struct seq_file *m;
  1991. int cpu;
  1992. if (tracing_disabled) {
  1993. *ret = -ENODEV;
  1994. return NULL;
  1995. }
  1996. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  1997. if (!iter) {
  1998. *ret = -ENOMEM;
  1999. goto out;
  2000. }
  2001. mutex_lock(&trace_types_lock);
  2002. if (current_trace && current_trace->print_max)
  2003. iter->tr = &max_tr;
  2004. else
  2005. iter->tr = inode->i_private;
  2006. iter->trace = current_trace;
  2007. iter->pos = -1;
  2008. /* Notify the tracer early; before we stop tracing. */
  2009. if (iter->trace && iter->trace->open)
  2010. iter->trace->open(iter);
  2011. /* Annotate start of buffers if we had overruns */
  2012. if (ring_buffer_overruns(iter->tr->buffer))
  2013. iter->iter_flags |= TRACE_FILE_ANNOTATE;
  2014. for_each_tracing_cpu(cpu) {
  2015. iter->buffer_iter[cpu] =
  2016. ring_buffer_read_start(iter->tr->buffer, cpu);
  2017. if (!iter->buffer_iter[cpu])
  2018. goto fail_buffer;
  2019. }
  2020. /* TODO stop tracer */
  2021. *ret = seq_open(file, &tracer_seq_ops);
  2022. if (*ret)
  2023. goto fail_buffer;
  2024. m = file->private_data;
  2025. m->private = iter;
  2026. /* stop the trace while dumping */
  2027. tracing_stop();
  2028. mutex_unlock(&trace_types_lock);
  2029. out:
  2030. return iter;
  2031. fail_buffer:
  2032. for_each_tracing_cpu(cpu) {
  2033. if (iter->buffer_iter[cpu])
  2034. ring_buffer_read_finish(iter->buffer_iter[cpu]);
  2035. }
  2036. mutex_unlock(&trace_types_lock);
  2037. kfree(iter);
  2038. return ERR_PTR(-ENOMEM);
  2039. }
  2040. int tracing_open_generic(struct inode *inode, struct file *filp)
  2041. {
  2042. if (tracing_disabled)
  2043. return -ENODEV;
  2044. filp->private_data = inode->i_private;
  2045. return 0;
  2046. }
  2047. int tracing_release(struct inode *inode, struct file *file)
  2048. {
  2049. struct seq_file *m = (struct seq_file *)file->private_data;
  2050. struct trace_iterator *iter = m->private;
  2051. int cpu;
  2052. mutex_lock(&trace_types_lock);
  2053. for_each_tracing_cpu(cpu) {
  2054. if (iter->buffer_iter[cpu])
  2055. ring_buffer_read_finish(iter->buffer_iter[cpu]);
  2056. }
  2057. if (iter->trace && iter->trace->close)
  2058. iter->trace->close(iter);
  2059. /* reenable tracing if it was previously enabled */
  2060. tracing_start();
  2061. mutex_unlock(&trace_types_lock);
  2062. seq_release(inode, file);
  2063. kfree(iter);
  2064. return 0;
  2065. }
  2066. static int tracing_open(struct inode *inode, struct file *file)
  2067. {
  2068. int ret;
  2069. __tracing_open(inode, file, &ret);
  2070. return ret;
  2071. }
  2072. static int tracing_lt_open(struct inode *inode, struct file *file)
  2073. {
  2074. struct trace_iterator *iter;
  2075. int ret;
  2076. iter = __tracing_open(inode, file, &ret);
  2077. if (!ret)
  2078. iter->iter_flags |= TRACE_FILE_LAT_FMT;
  2079. return ret;
  2080. }
  2081. static void *
  2082. t_next(struct seq_file *m, void *v, loff_t *pos)
  2083. {
  2084. struct tracer *t = m->private;
  2085. (*pos)++;
  2086. if (t)
  2087. t = t->next;
  2088. m->private = t;
  2089. return t;
  2090. }
  2091. static void *t_start(struct seq_file *m, loff_t *pos)
  2092. {
  2093. struct tracer *t = m->private;
  2094. loff_t l = 0;
  2095. mutex_lock(&trace_types_lock);
  2096. for (; t && l < *pos; t = t_next(m, t, &l))
  2097. ;
  2098. return t;
  2099. }
  2100. static void t_stop(struct seq_file *m, void *p)
  2101. {
  2102. mutex_unlock(&trace_types_lock);
  2103. }
  2104. static int t_show(struct seq_file *m, void *v)
  2105. {
  2106. struct tracer *t = v;
  2107. if (!t)
  2108. return 0;
  2109. seq_printf(m, "%s", t->name);
  2110. if (t->next)
  2111. seq_putc(m, ' ');
  2112. else
  2113. seq_putc(m, '\n');
  2114. return 0;
  2115. }
  2116. static struct seq_operations show_traces_seq_ops = {
  2117. .start = t_start,
  2118. .next = t_next,
  2119. .stop = t_stop,
  2120. .show = t_show,
  2121. };
  2122. static int show_traces_open(struct inode *inode, struct file *file)
  2123. {
  2124. int ret;
  2125. if (tracing_disabled)
  2126. return -ENODEV;
  2127. ret = seq_open(file, &show_traces_seq_ops);
  2128. if (!ret) {
  2129. struct seq_file *m = file->private_data;
  2130. m->private = trace_types;
  2131. }
  2132. return ret;
  2133. }
  2134. static struct file_operations tracing_fops = {
  2135. .open = tracing_open,
  2136. .read = seq_read,
  2137. .llseek = seq_lseek,
  2138. .release = tracing_release,
  2139. };
  2140. static struct file_operations tracing_lt_fops = {
  2141. .open = tracing_lt_open,
  2142. .read = seq_read,
  2143. .llseek = seq_lseek,
  2144. .release = tracing_release,
  2145. };
  2146. static struct file_operations show_traces_fops = {
  2147. .open = show_traces_open,
  2148. .read = seq_read,
  2149. .release = seq_release,
  2150. };
  2151. /*
  2152. * Only trace on a CPU if the bitmask is set:
  2153. */
  2154. static cpumask_t tracing_cpumask = CPU_MASK_ALL;
  2155. /*
  2156. * When tracing/tracing_cpu_mask is modified then this holds
  2157. * the new bitmask we are about to install:
  2158. */
  2159. static cpumask_t tracing_cpumask_new;
  2160. /*
  2161. * The tracer itself will not take this lock, but still we want
  2162. * to provide a consistent cpumask to user-space:
  2163. */
  2164. static DEFINE_MUTEX(tracing_cpumask_update_lock);
  2165. /*
  2166. * Temporary storage for the character representation of the
  2167. * CPU bitmask (and one more byte for the newline):
  2168. */
  2169. static char mask_str[NR_CPUS + 1];
  2170. static ssize_t
  2171. tracing_cpumask_read(struct file *filp, char __user *ubuf,
  2172. size_t count, loff_t *ppos)
  2173. {
  2174. int len;
  2175. mutex_lock(&tracing_cpumask_update_lock);
  2176. len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
  2177. if (count - len < 2) {
  2178. count = -EINVAL;
  2179. goto out_err;
  2180. }
  2181. len += sprintf(mask_str + len, "\n");
  2182. count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
  2183. out_err:
  2184. mutex_unlock(&tracing_cpumask_update_lock);
  2185. return count;
  2186. }
  2187. static ssize_t
  2188. tracing_cpumask_write(struct file *filp, const char __user *ubuf,
  2189. size_t count, loff_t *ppos)
  2190. {
  2191. int err, cpu;
  2192. mutex_lock(&tracing_cpumask_update_lock);
  2193. err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
  2194. if (err)
  2195. goto err_unlock;
  2196. raw_local_irq_disable();
  2197. __raw_spin_lock(&ftrace_max_lock);
  2198. for_each_tracing_cpu(cpu) {
  2199. /*
  2200. * Increase/decrease the disabled counter if we are
  2201. * about to flip a bit in the cpumask:
  2202. */
  2203. if (cpu_isset(cpu, tracing_cpumask) &&
  2204. !cpu_isset(cpu, tracing_cpumask_new)) {
  2205. atomic_inc(&global_trace.data[cpu]->disabled);
  2206. }
  2207. if (!cpu_isset(cpu, tracing_cpumask) &&
  2208. cpu_isset(cpu, tracing_cpumask_new)) {
  2209. atomic_dec(&global_trace.data[cpu]->disabled);
  2210. }
  2211. }
  2212. __raw_spin_unlock(&ftrace_max_lock);
  2213. raw_local_irq_enable();
  2214. tracing_cpumask = tracing_cpumask_new;
  2215. mutex_unlock(&tracing_cpumask_update_lock);
  2216. return count;
  2217. err_unlock:
  2218. mutex_unlock(&tracing_cpumask_update_lock);
  2219. return err;
  2220. }
  2221. static struct file_operations tracing_cpumask_fops = {
  2222. .open = tracing_open_generic,
  2223. .read = tracing_cpumask_read,
  2224. .write = tracing_cpumask_write,
  2225. };
  2226. static ssize_t
  2227. tracing_trace_options_read(struct file *filp, char __user *ubuf,
  2228. size_t cnt, loff_t *ppos)
  2229. {
  2230. int i;
  2231. char *buf;
  2232. int r = 0;
  2233. int len = 0;
  2234. u32 tracer_flags = current_trace->flags->val;
  2235. struct tracer_opt *trace_opts = current_trace->flags->opts;
  2236. /* calulate max size */
  2237. for (i = 0; trace_options[i]; i++) {
  2238. len += strlen(trace_options[i]);
  2239. len += 3; /* "no" and space */
  2240. }
  2241. /*
  2242. * Increase the size with names of options specific
  2243. * of the current tracer.
  2244. */
  2245. for (i = 0; trace_opts[i].name; i++) {
  2246. len += strlen(trace_opts[i].name);
  2247. len += 3; /* "no" and space */
  2248. }
  2249. /* +2 for \n and \0 */
  2250. buf = kmalloc(len + 2, GFP_KERNEL);
  2251. if (!buf)
  2252. return -ENOMEM;
  2253. for (i = 0; trace_options[i]; i++) {
  2254. if (trace_flags & (1 << i))
  2255. r += sprintf(buf + r, "%s ", trace_options[i]);
  2256. else
  2257. r += sprintf(buf + r, "no%s ", trace_options[i]);
  2258. }
  2259. for (i = 0; trace_opts[i].name; i++) {
  2260. if (tracer_flags & trace_opts[i].bit)
  2261. r += sprintf(buf + r, "%s ",
  2262. trace_opts[i].name);
  2263. else
  2264. r += sprintf(buf + r, "no%s ",
  2265. trace_opts[i].name);
  2266. }
  2267. r += sprintf(buf + r, "\n");
  2268. WARN_ON(r >= len + 2);
  2269. r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2270. kfree(buf);
  2271. return r;
  2272. }
  2273. /* Try to assign a tracer specific option */
  2274. static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
  2275. {
  2276. struct tracer_flags *trace_flags = trace->flags;
  2277. struct tracer_opt *opts = NULL;
  2278. int ret = 0, i = 0;
  2279. int len;
  2280. for (i = 0; trace_flags->opts[i].name; i++) {
  2281. opts = &trace_flags->opts[i];
  2282. len = strlen(opts->name);
  2283. if (strncmp(cmp, opts->name, len) == 0) {
  2284. ret = trace->set_flag(trace_flags->val,
  2285. opts->bit, !neg);
  2286. break;
  2287. }
  2288. }
  2289. /* Not found */
  2290. if (!trace_flags->opts[i].name)
  2291. return -EINVAL;
  2292. /* Refused to handle */
  2293. if (ret)
  2294. return ret;
  2295. if (neg)
  2296. trace_flags->val &= ~opts->bit;
  2297. else
  2298. trace_flags->val |= opts->bit;
  2299. return 0;
  2300. }
  2301. static ssize_t
  2302. tracing_trace_options_write(struct file *filp, const char __user *ubuf,
  2303. size_t cnt, loff_t *ppos)
  2304. {
  2305. char buf[64];
  2306. char *cmp = buf;
  2307. int neg = 0;
  2308. int ret;
  2309. int i;
  2310. if (cnt >= sizeof(buf))
  2311. return -EINVAL;
  2312. if (copy_from_user(&buf, ubuf, cnt))
  2313. return -EFAULT;
  2314. buf[cnt] = 0;
  2315. if (strncmp(buf, "no", 2) == 0) {
  2316. neg = 1;
  2317. cmp += 2;
  2318. }
  2319. for (i = 0; trace_options[i]; i++) {
  2320. int len = strlen(trace_options[i]);
  2321. if (strncmp(cmp, trace_options[i], len) == 0) {
  2322. if (neg)
  2323. trace_flags &= ~(1 << i);
  2324. else
  2325. trace_flags |= (1 << i);
  2326. break;
  2327. }
  2328. }
  2329. /* If no option could be set, test the specific tracer options */
  2330. if (!trace_options[i]) {
  2331. ret = set_tracer_option(current_trace, cmp, neg);
  2332. if (ret)
  2333. return ret;
  2334. }
  2335. filp->f_pos += cnt;
  2336. return cnt;
  2337. }
  2338. static struct file_operations tracing_iter_fops = {
  2339. .open = tracing_open_generic,
  2340. .read = tracing_trace_options_read,
  2341. .write = tracing_trace_options_write,
  2342. };
  2343. static const char readme_msg[] =
  2344. "tracing mini-HOWTO:\n\n"
  2345. "# mkdir /debug\n"
  2346. "# mount -t debugfs nodev /debug\n\n"
  2347. "# cat /debug/tracing/available_tracers\n"
  2348. "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
  2349. "# cat /debug/tracing/current_tracer\n"
  2350. "none\n"
  2351. "# echo sched_switch > /debug/tracing/current_tracer\n"
  2352. "# cat /debug/tracing/current_tracer\n"
  2353. "sched_switch\n"
  2354. "# cat /debug/tracing/trace_options\n"
  2355. "noprint-parent nosym-offset nosym-addr noverbose\n"
  2356. "# echo print-parent > /debug/tracing/trace_options\n"
  2357. "# echo 1 > /debug/tracing/tracing_enabled\n"
  2358. "# cat /debug/tracing/trace > /tmp/trace.txt\n"
  2359. "echo 0 > /debug/tracing/tracing_enabled\n"
  2360. ;
  2361. static ssize_t
  2362. tracing_readme_read(struct file *filp, char __user *ubuf,
  2363. size_t cnt, loff_t *ppos)
  2364. {
  2365. return simple_read_from_buffer(ubuf, cnt, ppos,
  2366. readme_msg, strlen(readme_msg));
  2367. }
  2368. static struct file_operations tracing_readme_fops = {
  2369. .open = tracing_open_generic,
  2370. .read = tracing_readme_read,
  2371. };
  2372. static ssize_t
  2373. tracing_ctrl_read(struct file *filp, char __user *ubuf,
  2374. size_t cnt, loff_t *ppos)
  2375. {
  2376. char buf[64];
  2377. int r;
  2378. r = sprintf(buf, "%u\n", tracer_enabled);
  2379. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2380. }
  2381. static ssize_t
  2382. tracing_ctrl_write(struct file *filp, const char __user *ubuf,
  2383. size_t cnt, loff_t *ppos)
  2384. {
  2385. struct trace_array *tr = filp->private_data;
  2386. char buf[64];
  2387. long val;
  2388. int ret;
  2389. if (cnt >= sizeof(buf))
  2390. return -EINVAL;
  2391. if (copy_from_user(&buf, ubuf, cnt))
  2392. return -EFAULT;
  2393. buf[cnt] = 0;
  2394. ret = strict_strtoul(buf, 10, &val);
  2395. if (ret < 0)
  2396. return ret;
  2397. val = !!val;
  2398. mutex_lock(&trace_types_lock);
  2399. if (tracer_enabled ^ val) {
  2400. if (val) {
  2401. tracer_enabled = 1;
  2402. if (current_trace->start)
  2403. current_trace->start(tr);
  2404. tracing_start();
  2405. } else {
  2406. tracer_enabled = 0;
  2407. tracing_stop();
  2408. if (current_trace->stop)
  2409. current_trace->stop(tr);
  2410. }
  2411. }
  2412. mutex_unlock(&trace_types_lock);
  2413. filp->f_pos += cnt;
  2414. return cnt;
  2415. }
  2416. static ssize_t
  2417. tracing_set_trace_read(struct file *filp, char __user *ubuf,
  2418. size_t cnt, loff_t *ppos)
  2419. {
  2420. char buf[max_tracer_type_len+2];
  2421. int r;
  2422. mutex_lock(&trace_types_lock);
  2423. if (current_trace)
  2424. r = sprintf(buf, "%s\n", current_trace->name);
  2425. else
  2426. r = sprintf(buf, "\n");
  2427. mutex_unlock(&trace_types_lock);
  2428. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2429. }
  2430. static int tracing_set_tracer(char *buf)
  2431. {
  2432. struct trace_array *tr = &global_trace;
  2433. struct tracer *t;
  2434. int ret = 0;
  2435. mutex_lock(&trace_types_lock);
  2436. for (t = trace_types; t; t = t->next) {
  2437. if (strcmp(t->name, buf) == 0)
  2438. break;
  2439. }
  2440. if (!t) {
  2441. ret = -EINVAL;
  2442. goto out;
  2443. }
  2444. if (t == current_trace)
  2445. goto out;
  2446. trace_branch_disable();
  2447. if (current_trace && current_trace->reset)
  2448. current_trace->reset(tr);
  2449. current_trace = t;
  2450. if (t->init) {
  2451. ret = t->init(tr);
  2452. if (ret)
  2453. goto out;
  2454. }
  2455. trace_branch_enable(tr);
  2456. out:
  2457. mutex_unlock(&trace_types_lock);
  2458. return ret;
  2459. }
  2460. static ssize_t
  2461. tracing_set_trace_write(struct file *filp, const char __user *ubuf,
  2462. size_t cnt, loff_t *ppos)
  2463. {
  2464. char buf[max_tracer_type_len+1];
  2465. int i;
  2466. size_t ret;
  2467. int err;
  2468. ret = cnt;
  2469. if (cnt > max_tracer_type_len)
  2470. cnt = max_tracer_type_len;
  2471. if (copy_from_user(&buf, ubuf, cnt))
  2472. return -EFAULT;
  2473. buf[cnt] = 0;
  2474. /* strip ending whitespace. */
  2475. for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
  2476. buf[i] = 0;
  2477. err = tracing_set_tracer(buf);
  2478. if (err)
  2479. return err;
  2480. filp->f_pos += ret;
  2481. return ret;
  2482. }
  2483. static ssize_t
  2484. tracing_max_lat_read(struct file *filp, char __user *ubuf,
  2485. size_t cnt, loff_t *ppos)
  2486. {
  2487. unsigned long *ptr = filp->private_data;
  2488. char buf[64];
  2489. int r;
  2490. r = snprintf(buf, sizeof(buf), "%ld\n",
  2491. *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
  2492. if (r > sizeof(buf))
  2493. r = sizeof(buf);
  2494. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2495. }
  2496. static ssize_t
  2497. tracing_max_lat_write(struct file *filp, const char __user *ubuf,
  2498. size_t cnt, loff_t *ppos)
  2499. {
  2500. long *ptr = filp->private_data;
  2501. char buf[64];
  2502. long val;
  2503. int ret;
  2504. if (cnt >= sizeof(buf))
  2505. return -EINVAL;
  2506. if (copy_from_user(&buf, ubuf, cnt))
  2507. return -EFAULT;
  2508. buf[cnt] = 0;
  2509. ret = strict_strtoul(buf, 10, &val);
  2510. if (ret < 0)
  2511. return ret;
  2512. *ptr = val * 1000;
  2513. return cnt;
  2514. }
  2515. static atomic_t tracing_reader;
  2516. static int tracing_open_pipe(struct inode *inode, struct file *filp)
  2517. {
  2518. struct trace_iterator *iter;
  2519. if (tracing_disabled)
  2520. return -ENODEV;
  2521. /* We only allow for reader of the pipe */
  2522. if (atomic_inc_return(&tracing_reader) != 1) {
  2523. atomic_dec(&tracing_reader);
  2524. return -EBUSY;
  2525. }
  2526. /* create a buffer to store the information to pass to userspace */
  2527. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  2528. if (!iter)
  2529. return -ENOMEM;
  2530. mutex_lock(&trace_types_lock);
  2531. /* trace pipe does not show start of buffer */
  2532. cpus_setall(iter->started);
  2533. iter->tr = &global_trace;
  2534. iter->trace = current_trace;
  2535. filp->private_data = iter;
  2536. if (iter->trace->pipe_open)
  2537. iter->trace->pipe_open(iter);
  2538. mutex_unlock(&trace_types_lock);
  2539. return 0;
  2540. }
  2541. static int tracing_release_pipe(struct inode *inode, struct file *file)
  2542. {
  2543. struct trace_iterator *iter = file->private_data;
  2544. kfree(iter);
  2545. atomic_dec(&tracing_reader);
  2546. return 0;
  2547. }
  2548. static unsigned int
  2549. tracing_poll_pipe(struct file *filp, poll_table *poll_table)
  2550. {
  2551. struct trace_iterator *iter = filp->private_data;
  2552. if (trace_flags & TRACE_ITER_BLOCK) {
  2553. /*
  2554. * Always select as readable when in blocking mode
  2555. */
  2556. return POLLIN | POLLRDNORM;
  2557. } else {
  2558. if (!trace_empty(iter))
  2559. return POLLIN | POLLRDNORM;
  2560. poll_wait(filp, &trace_wait, poll_table);
  2561. if (!trace_empty(iter))
  2562. return POLLIN | POLLRDNORM;
  2563. return 0;
  2564. }
  2565. }
  2566. /*
  2567. * Consumer reader.
  2568. */
  2569. static ssize_t
  2570. tracing_read_pipe(struct file *filp, char __user *ubuf,
  2571. size_t cnt, loff_t *ppos)
  2572. {
  2573. struct trace_iterator *iter = filp->private_data;
  2574. ssize_t sret;
  2575. /* return any leftover data */
  2576. sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
  2577. if (sret != -EBUSY)
  2578. return sret;
  2579. trace_seq_reset(&iter->seq);
  2580. mutex_lock(&trace_types_lock);
  2581. if (iter->trace->read) {
  2582. sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
  2583. if (sret)
  2584. goto out;
  2585. }
  2586. waitagain:
  2587. sret = 0;
  2588. while (trace_empty(iter)) {
  2589. if ((filp->f_flags & O_NONBLOCK)) {
  2590. sret = -EAGAIN;
  2591. goto out;
  2592. }
  2593. /*
  2594. * This is a make-shift waitqueue. The reason we don't use
  2595. * an actual wait queue is because:
  2596. * 1) we only ever have one waiter
  2597. * 2) the tracing, traces all functions, we don't want
  2598. * the overhead of calling wake_up and friends
  2599. * (and tracing them too)
  2600. * Anyway, this is really very primitive wakeup.
  2601. */
  2602. set_current_state(TASK_INTERRUPTIBLE);
  2603. iter->tr->waiter = current;
  2604. mutex_unlock(&trace_types_lock);
  2605. /* sleep for 100 msecs, and try again. */
  2606. schedule_timeout(HZ/10);
  2607. mutex_lock(&trace_types_lock);
  2608. iter->tr->waiter = NULL;
  2609. if (signal_pending(current)) {
  2610. sret = -EINTR;
  2611. goto out;
  2612. }
  2613. if (iter->trace != current_trace)
  2614. goto out;
  2615. /*
  2616. * We block until we read something and tracing is disabled.
  2617. * We still block if tracing is disabled, but we have never
  2618. * read anything. This allows a user to cat this file, and
  2619. * then enable tracing. But after we have read something,
  2620. * we give an EOF when tracing is again disabled.
  2621. *
  2622. * iter->pos will be 0 if we haven't read anything.
  2623. */
  2624. if (!tracer_enabled && iter->pos)
  2625. break;
  2626. continue;
  2627. }
  2628. /* stop when tracing is finished */
  2629. if (trace_empty(iter))
  2630. goto out;
  2631. if (cnt >= PAGE_SIZE)
  2632. cnt = PAGE_SIZE - 1;
  2633. /* reset all but tr, trace, and overruns */
  2634. memset(&iter->seq, 0,
  2635. sizeof(struct trace_iterator) -
  2636. offsetof(struct trace_iterator, seq));
  2637. iter->pos = -1;
  2638. while (find_next_entry_inc(iter) != NULL) {
  2639. enum print_line_t ret;
  2640. int len = iter->seq.len;
  2641. ret = print_trace_line(iter);
  2642. if (ret == TRACE_TYPE_PARTIAL_LINE) {
  2643. /* don't print partial lines */
  2644. iter->seq.len = len;
  2645. break;
  2646. }
  2647. trace_consume(iter);
  2648. if (iter->seq.len >= cnt)
  2649. break;
  2650. }
  2651. /* Now copy what we have to the user */
  2652. sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
  2653. if (iter->seq.readpos >= iter->seq.len)
  2654. trace_seq_reset(&iter->seq);
  2655. /*
  2656. * If there was nothing to send to user, inspite of consuming trace
  2657. * entries, go back to wait for more entries.
  2658. */
  2659. if (sret == -EBUSY)
  2660. goto waitagain;
  2661. out:
  2662. mutex_unlock(&trace_types_lock);
  2663. return sret;
  2664. }
  2665. static ssize_t
  2666. tracing_entries_read(struct file *filp, char __user *ubuf,
  2667. size_t cnt, loff_t *ppos)
  2668. {
  2669. struct trace_array *tr = filp->private_data;
  2670. char buf[64];
  2671. int r;
  2672. r = sprintf(buf, "%lu\n", tr->entries >> 10);
  2673. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2674. }
  2675. static ssize_t
  2676. tracing_entries_write(struct file *filp, const char __user *ubuf,
  2677. size_t cnt, loff_t *ppos)
  2678. {
  2679. unsigned long val;
  2680. char buf[64];
  2681. int ret, cpu;
  2682. if (cnt >= sizeof(buf))
  2683. return -EINVAL;
  2684. if (copy_from_user(&buf, ubuf, cnt))
  2685. return -EFAULT;
  2686. buf[cnt] = 0;
  2687. ret = strict_strtoul(buf, 10, &val);
  2688. if (ret < 0)
  2689. return ret;
  2690. /* must have at least 1 entry */
  2691. if (!val)
  2692. return -EINVAL;
  2693. mutex_lock(&trace_types_lock);
  2694. tracing_stop();
  2695. /* disable all cpu buffers */
  2696. for_each_tracing_cpu(cpu) {
  2697. if (global_trace.data[cpu])
  2698. atomic_inc(&global_trace.data[cpu]->disabled);
  2699. if (max_tr.data[cpu])
  2700. atomic_inc(&max_tr.data[cpu]->disabled);
  2701. }
  2702. /* value is in KB */
  2703. val <<= 10;
  2704. if (val != global_trace.entries) {
  2705. ret = ring_buffer_resize(global_trace.buffer, val);
  2706. if (ret < 0) {
  2707. cnt = ret;
  2708. goto out;
  2709. }
  2710. ret = ring_buffer_resize(max_tr.buffer, val);
  2711. if (ret < 0) {
  2712. int r;
  2713. cnt = ret;
  2714. r = ring_buffer_resize(global_trace.buffer,
  2715. global_trace.entries);
  2716. if (r < 0) {
  2717. /* AARGH! We are left with different
  2718. * size max buffer!!!! */
  2719. WARN_ON(1);
  2720. tracing_disabled = 1;
  2721. }
  2722. goto out;
  2723. }
  2724. global_trace.entries = val;
  2725. }
  2726. filp->f_pos += cnt;
  2727. /* If check pages failed, return ENOMEM */
  2728. if (tracing_disabled)
  2729. cnt = -ENOMEM;
  2730. out:
  2731. for_each_tracing_cpu(cpu) {
  2732. if (global_trace.data[cpu])
  2733. atomic_dec(&global_trace.data[cpu]->disabled);
  2734. if (max_tr.data[cpu])
  2735. atomic_dec(&max_tr.data[cpu]->disabled);
  2736. }
  2737. tracing_start();
  2738. max_tr.entries = global_trace.entries;
  2739. mutex_unlock(&trace_types_lock);
  2740. return cnt;
  2741. }
  2742. static int mark_printk(const char *fmt, ...)
  2743. {
  2744. int ret;
  2745. va_list args;
  2746. va_start(args, fmt);
  2747. ret = trace_vprintk(0, fmt, args);
  2748. va_end(args);
  2749. return ret;
  2750. }
  2751. static ssize_t
  2752. tracing_mark_write(struct file *filp, const char __user *ubuf,
  2753. size_t cnt, loff_t *fpos)
  2754. {
  2755. char *buf;
  2756. char *end;
  2757. if (tracing_disabled)
  2758. return -EINVAL;
  2759. if (cnt > TRACE_BUF_SIZE)
  2760. cnt = TRACE_BUF_SIZE;
  2761. buf = kmalloc(cnt + 1, GFP_KERNEL);
  2762. if (buf == NULL)
  2763. return -ENOMEM;
  2764. if (copy_from_user(buf, ubuf, cnt)) {
  2765. kfree(buf);
  2766. return -EFAULT;
  2767. }
  2768. /* Cut from the first nil or newline. */
  2769. buf[cnt] = '\0';
  2770. end = strchr(buf, '\n');
  2771. if (end)
  2772. *end = '\0';
  2773. cnt = mark_printk("%s\n", buf);
  2774. kfree(buf);
  2775. *fpos += cnt;
  2776. return cnt;
  2777. }
  2778. static struct file_operations tracing_max_lat_fops = {
  2779. .open = tracing_open_generic,
  2780. .read = tracing_max_lat_read,
  2781. .write = tracing_max_lat_write,
  2782. };
  2783. static struct file_operations tracing_ctrl_fops = {
  2784. .open = tracing_open_generic,
  2785. .read = tracing_ctrl_read,
  2786. .write = tracing_ctrl_write,
  2787. };
  2788. static struct file_operations set_tracer_fops = {
  2789. .open = tracing_open_generic,
  2790. .read = tracing_set_trace_read,
  2791. .write = tracing_set_trace_write,
  2792. };
  2793. static struct file_operations tracing_pipe_fops = {
  2794. .open = tracing_open_pipe,
  2795. .poll = tracing_poll_pipe,
  2796. .read = tracing_read_pipe,
  2797. .release = tracing_release_pipe,
  2798. };
  2799. static struct file_operations tracing_entries_fops = {
  2800. .open = tracing_open_generic,
  2801. .read = tracing_entries_read,
  2802. .write = tracing_entries_write,
  2803. };
  2804. static struct file_operations tracing_mark_fops = {
  2805. .open = tracing_open_generic,
  2806. .write = tracing_mark_write,
  2807. };
  2808. #ifdef CONFIG_DYNAMIC_FTRACE
  2809. int __weak ftrace_arch_read_dyn_info(char *buf, int size)
  2810. {
  2811. return 0;
  2812. }
  2813. static ssize_t
  2814. tracing_read_dyn_info(struct file *filp, char __user *ubuf,
  2815. size_t cnt, loff_t *ppos)
  2816. {
  2817. static char ftrace_dyn_info_buffer[1024];
  2818. static DEFINE_MUTEX(dyn_info_mutex);
  2819. unsigned long *p = filp->private_data;
  2820. char *buf = ftrace_dyn_info_buffer;
  2821. int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
  2822. int r;
  2823. mutex_lock(&dyn_info_mutex);
  2824. r = sprintf(buf, "%ld ", *p);
  2825. r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
  2826. buf[r++] = '\n';
  2827. r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2828. mutex_unlock(&dyn_info_mutex);
  2829. return r;
  2830. }
  2831. static struct file_operations tracing_dyn_info_fops = {
  2832. .open = tracing_open_generic,
  2833. .read = tracing_read_dyn_info,
  2834. };
  2835. #endif
  2836. static struct dentry *d_tracer;
  2837. struct dentry *tracing_init_dentry(void)
  2838. {
  2839. static int once;
  2840. if (d_tracer)
  2841. return d_tracer;
  2842. d_tracer = debugfs_create_dir("tracing", NULL);
  2843. if (!d_tracer && !once) {
  2844. once = 1;
  2845. pr_warning("Could not create debugfs directory 'tracing'\n");
  2846. return NULL;
  2847. }
  2848. return d_tracer;
  2849. }
  2850. #ifdef CONFIG_FTRACE_SELFTEST
  2851. /* Let selftest have access to static functions in this file */
  2852. #include "trace_selftest.c"
  2853. #endif
  2854. static __init int tracer_init_debugfs(void)
  2855. {
  2856. struct dentry *d_tracer;
  2857. struct dentry *entry;
  2858. d_tracer = tracing_init_dentry();
  2859. entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
  2860. &global_trace, &tracing_ctrl_fops);
  2861. if (!entry)
  2862. pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
  2863. entry = debugfs_create_file("trace_options", 0644, d_tracer,
  2864. NULL, &tracing_iter_fops);
  2865. if (!entry)
  2866. pr_warning("Could not create debugfs 'trace_options' entry\n");
  2867. entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
  2868. NULL, &tracing_cpumask_fops);
  2869. if (!entry)
  2870. pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
  2871. entry = debugfs_create_file("latency_trace", 0444, d_tracer,
  2872. &global_trace, &tracing_lt_fops);
  2873. if (!entry)
  2874. pr_warning("Could not create debugfs 'latency_trace' entry\n");
  2875. entry = debugfs_create_file("trace", 0444, d_tracer,
  2876. &global_trace, &tracing_fops);
  2877. if (!entry)
  2878. pr_warning("Could not create debugfs 'trace' entry\n");
  2879. entry = debugfs_create_file("available_tracers", 0444, d_tracer,
  2880. &global_trace, &show_traces_fops);
  2881. if (!entry)
  2882. pr_warning("Could not create debugfs 'available_tracers' entry\n");
  2883. entry = debugfs_create_file("current_tracer", 0444, d_tracer,
  2884. &global_trace, &set_tracer_fops);
  2885. if (!entry)
  2886. pr_warning("Could not create debugfs 'current_tracer' entry\n");
  2887. entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
  2888. &tracing_max_latency,
  2889. &tracing_max_lat_fops);
  2890. if (!entry)
  2891. pr_warning("Could not create debugfs "
  2892. "'tracing_max_latency' entry\n");
  2893. entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
  2894. &tracing_thresh, &tracing_max_lat_fops);
  2895. if (!entry)
  2896. pr_warning("Could not create debugfs "
  2897. "'tracing_thresh' entry\n");
  2898. entry = debugfs_create_file("README", 0644, d_tracer,
  2899. NULL, &tracing_readme_fops);
  2900. if (!entry)
  2901. pr_warning("Could not create debugfs 'README' entry\n");
  2902. entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
  2903. NULL, &tracing_pipe_fops);
  2904. if (!entry)
  2905. pr_warning("Could not create debugfs "
  2906. "'trace_pipe' entry\n");
  2907. entry = debugfs_create_file("buffer_size_kb", 0644, d_tracer,
  2908. &global_trace, &tracing_entries_fops);
  2909. if (!entry)
  2910. pr_warning("Could not create debugfs "
  2911. "'buffer_size_kb' entry\n");
  2912. entry = debugfs_create_file("trace_marker", 0220, d_tracer,
  2913. NULL, &tracing_mark_fops);
  2914. if (!entry)
  2915. pr_warning("Could not create debugfs "
  2916. "'trace_marker' entry\n");
  2917. #ifdef CONFIG_DYNAMIC_FTRACE
  2918. entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
  2919. &ftrace_update_tot_cnt,
  2920. &tracing_dyn_info_fops);
  2921. if (!entry)
  2922. pr_warning("Could not create debugfs "
  2923. "'dyn_ftrace_total_info' entry\n");
  2924. #endif
  2925. #ifdef CONFIG_SYSPROF_TRACER
  2926. init_tracer_sysprof_debugfs(d_tracer);
  2927. #endif
  2928. return 0;
  2929. }
  2930. int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
  2931. {
  2932. static DEFINE_SPINLOCK(trace_buf_lock);
  2933. static char trace_buf[TRACE_BUF_SIZE];
  2934. struct ring_buffer_event *event;
  2935. struct trace_array *tr = &global_trace;
  2936. struct trace_array_cpu *data;
  2937. struct print_entry *entry;
  2938. unsigned long flags, irq_flags;
  2939. int cpu, len = 0, size, pc;
  2940. if (tracing_disabled)
  2941. return 0;
  2942. pc = preempt_count();
  2943. preempt_disable_notrace();
  2944. cpu = raw_smp_processor_id();
  2945. data = tr->data[cpu];
  2946. if (unlikely(atomic_read(&data->disabled)))
  2947. goto out;
  2948. spin_lock_irqsave(&trace_buf_lock, flags);
  2949. len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
  2950. len = min(len, TRACE_BUF_SIZE-1);
  2951. trace_buf[len] = 0;
  2952. size = sizeof(*entry) + len + 1;
  2953. event = ring_buffer_lock_reserve(tr->buffer, size, &irq_flags);
  2954. if (!event)
  2955. goto out_unlock;
  2956. entry = ring_buffer_event_data(event);
  2957. tracing_generic_entry_update(&entry->ent, flags, pc);
  2958. entry->ent.type = TRACE_PRINT;
  2959. entry->ip = ip;
  2960. memcpy(&entry->buf, trace_buf, len);
  2961. entry->buf[len] = 0;
  2962. ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
  2963. out_unlock:
  2964. spin_unlock_irqrestore(&trace_buf_lock, flags);
  2965. out:
  2966. preempt_enable_notrace();
  2967. return len;
  2968. }
  2969. EXPORT_SYMBOL_GPL(trace_vprintk);
  2970. int __ftrace_printk(unsigned long ip, const char *fmt, ...)
  2971. {
  2972. int ret;
  2973. va_list ap;
  2974. if (!(trace_flags & TRACE_ITER_PRINTK))
  2975. return 0;
  2976. va_start(ap, fmt);
  2977. ret = trace_vprintk(ip, fmt, ap);
  2978. va_end(ap);
  2979. return ret;
  2980. }
  2981. EXPORT_SYMBOL_GPL(__ftrace_printk);
  2982. static int trace_panic_handler(struct notifier_block *this,
  2983. unsigned long event, void *unused)
  2984. {
  2985. if (ftrace_dump_on_oops)
  2986. ftrace_dump();
  2987. return NOTIFY_OK;
  2988. }
  2989. static struct notifier_block trace_panic_notifier = {
  2990. .notifier_call = trace_panic_handler,
  2991. .next = NULL,
  2992. .priority = 150 /* priority: INT_MAX >= x >= 0 */
  2993. };
  2994. static int trace_die_handler(struct notifier_block *self,
  2995. unsigned long val,
  2996. void *data)
  2997. {
  2998. switch (val) {
  2999. case DIE_OOPS:
  3000. if (ftrace_dump_on_oops)
  3001. ftrace_dump();
  3002. break;
  3003. default:
  3004. break;
  3005. }
  3006. return NOTIFY_OK;
  3007. }
  3008. static struct notifier_block trace_die_notifier = {
  3009. .notifier_call = trace_die_handler,
  3010. .priority = 200
  3011. };
  3012. /*
  3013. * printk is set to max of 1024, we really don't need it that big.
  3014. * Nothing should be printing 1000 characters anyway.
  3015. */
  3016. #define TRACE_MAX_PRINT 1000
  3017. /*
  3018. * Define here KERN_TRACE so that we have one place to modify
  3019. * it if we decide to change what log level the ftrace dump
  3020. * should be at.
  3021. */
  3022. #define KERN_TRACE KERN_INFO
  3023. static void
  3024. trace_printk_seq(struct trace_seq *s)
  3025. {
  3026. /* Probably should print a warning here. */
  3027. if (s->len >= 1000)
  3028. s->len = 1000;
  3029. /* should be zero ended, but we are paranoid. */
  3030. s->buffer[s->len] = 0;
  3031. printk(KERN_TRACE "%s", s->buffer);
  3032. trace_seq_reset(s);
  3033. }
  3034. void ftrace_dump(void)
  3035. {
  3036. static DEFINE_SPINLOCK(ftrace_dump_lock);
  3037. /* use static because iter can be a bit big for the stack */
  3038. static struct trace_iterator iter;
  3039. static cpumask_t mask;
  3040. static int dump_ran;
  3041. unsigned long flags;
  3042. int cnt = 0, cpu;
  3043. /* only one dump */
  3044. spin_lock_irqsave(&ftrace_dump_lock, flags);
  3045. if (dump_ran)
  3046. goto out;
  3047. dump_ran = 1;
  3048. /* No turning back! */
  3049. ftrace_kill();
  3050. for_each_tracing_cpu(cpu) {
  3051. atomic_inc(&global_trace.data[cpu]->disabled);
  3052. }
  3053. /* don't look at user memory in panic mode */
  3054. trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
  3055. printk(KERN_TRACE "Dumping ftrace buffer:\n");
  3056. iter.tr = &global_trace;
  3057. iter.trace = current_trace;
  3058. /*
  3059. * We need to stop all tracing on all CPUS to read the
  3060. * the next buffer. This is a bit expensive, but is
  3061. * not done often. We fill all what we can read,
  3062. * and then release the locks again.
  3063. */
  3064. cpus_clear(mask);
  3065. while (!trace_empty(&iter)) {
  3066. if (!cnt)
  3067. printk(KERN_TRACE "---------------------------------\n");
  3068. cnt++;
  3069. /* reset all but tr, trace, and overruns */
  3070. memset(&iter.seq, 0,
  3071. sizeof(struct trace_iterator) -
  3072. offsetof(struct trace_iterator, seq));
  3073. iter.iter_flags |= TRACE_FILE_LAT_FMT;
  3074. iter.pos = -1;
  3075. if (find_next_entry_inc(&iter) != NULL) {
  3076. print_trace_line(&iter);
  3077. trace_consume(&iter);
  3078. }
  3079. trace_printk_seq(&iter.seq);
  3080. }
  3081. if (!cnt)
  3082. printk(KERN_TRACE " (ftrace buffer empty)\n");
  3083. else
  3084. printk(KERN_TRACE "---------------------------------\n");
  3085. out:
  3086. spin_unlock_irqrestore(&ftrace_dump_lock, flags);
  3087. }
  3088. __init static int tracer_alloc_buffers(void)
  3089. {
  3090. struct trace_array_cpu *data;
  3091. int i;
  3092. /* TODO: make the number of buffers hot pluggable with CPUS */
  3093. tracing_buffer_mask = cpu_possible_map;
  3094. global_trace.buffer = ring_buffer_alloc(trace_buf_size,
  3095. TRACE_BUFFER_FLAGS);
  3096. if (!global_trace.buffer) {
  3097. printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
  3098. WARN_ON(1);
  3099. return 0;
  3100. }
  3101. global_trace.entries = ring_buffer_size(global_trace.buffer);
  3102. #ifdef CONFIG_TRACER_MAX_TRACE
  3103. max_tr.buffer = ring_buffer_alloc(trace_buf_size,
  3104. TRACE_BUFFER_FLAGS);
  3105. if (!max_tr.buffer) {
  3106. printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
  3107. WARN_ON(1);
  3108. ring_buffer_free(global_trace.buffer);
  3109. return 0;
  3110. }
  3111. max_tr.entries = ring_buffer_size(max_tr.buffer);
  3112. WARN_ON(max_tr.entries != global_trace.entries);
  3113. #endif
  3114. /* Allocate the first page for all buffers */
  3115. for_each_tracing_cpu(i) {
  3116. data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
  3117. max_tr.data[i] = &per_cpu(max_data, i);
  3118. }
  3119. trace_init_cmdlines();
  3120. register_tracer(&nop_trace);
  3121. #ifdef CONFIG_BOOT_TRACER
  3122. register_tracer(&boot_tracer);
  3123. current_trace = &boot_tracer;
  3124. current_trace->init(&global_trace);
  3125. #else
  3126. current_trace = &nop_trace;
  3127. #endif
  3128. /* All seems OK, enable tracing */
  3129. tracing_disabled = 0;
  3130. atomic_notifier_chain_register(&panic_notifier_list,
  3131. &trace_panic_notifier);
  3132. register_die_notifier(&trace_die_notifier);
  3133. return 0;
  3134. }
  3135. early_initcall(tracer_alloc_buffers);
  3136. fs_initcall(tracer_init_debugfs);