trace.c 100 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485
  1. /*
  2. * ring buffer based function tracer
  3. *
  4. * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  5. * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
  6. *
  7. * Originally taken from the RT patch by:
  8. * Arnaldo Carvalho de Melo <acme@redhat.com>
  9. *
  10. * Based on code from the latency_tracer, that is:
  11. * Copyright (C) 2004-2006 Ingo Molnar
  12. * Copyright (C) 2004 William Lee Irwin III
  13. */
  14. #include <linux/ring_buffer.h>
  15. #include <generated/utsrelease.h>
  16. #include <linux/stacktrace.h>
  17. #include <linux/writeback.h>
  18. #include <linux/kallsyms.h>
  19. #include <linux/seq_file.h>
  20. #include <linux/smp_lock.h>
  21. #include <linux/notifier.h>
  22. #include <linux/irqflags.h>
  23. #include <linux/debugfs.h>
  24. #include <linux/pagemap.h>
  25. #include <linux/hardirq.h>
  26. #include <linux/linkage.h>
  27. #include <linux/uaccess.h>
  28. #include <linux/kprobes.h>
  29. #include <linux/ftrace.h>
  30. #include <linux/module.h>
  31. #include <linux/percpu.h>
  32. #include <linux/splice.h>
  33. #include <linux/kdebug.h>
  34. #include <linux/string.h>
  35. #include <linux/ctype.h>
  36. #include <linux/init.h>
  37. #include <linux/poll.h>
  38. #include <linux/gfp.h>
  39. #include <linux/fs.h>
  40. #include "trace.h"
  41. #include "trace_output.h"
  42. #define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
  43. /*
  44. * On boot up, the ring buffer is set to the minimum size, so that
  45. * we do not waste memory on systems that are not using tracing.
  46. */
  47. int ring_buffer_expanded;
  48. /*
  49. * We need to change this state when a selftest is running.
  50. * A selftest will lurk into the ring-buffer to count the
  51. * entries inserted during the selftest although some concurrent
  52. * insertions into the ring-buffer such as trace_printk could occurred
  53. * at the same time, giving false positive or negative results.
  54. */
  55. static bool __read_mostly tracing_selftest_running;
  56. /*
  57. * If a tracer is running, we do not want to run SELFTEST.
  58. */
  59. bool __read_mostly tracing_selftest_disabled;
  60. /* For tracers that don't implement custom flags */
  61. static struct tracer_opt dummy_tracer_opt[] = {
  62. { }
  63. };
  64. static struct tracer_flags dummy_tracer_flags = {
  65. .val = 0,
  66. .opts = dummy_tracer_opt
  67. };
  68. static int dummy_set_flag(u32 old_flags, u32 bit, int set)
  69. {
  70. return 0;
  71. }
  72. /*
  73. * Kill all tracing for good (never come back).
  74. * It is initialized to 1 but will turn to zero if the initialization
  75. * of the tracer is successful. But that is the only place that sets
  76. * this back to zero.
  77. */
  78. static int tracing_disabled = 1;
  79. DEFINE_PER_CPU(int, ftrace_cpu_disabled);
  80. static inline void ftrace_disable_cpu(void)
  81. {
  82. preempt_disable();
  83. __this_cpu_inc(per_cpu_var(ftrace_cpu_disabled));
  84. }
  85. static inline void ftrace_enable_cpu(void)
  86. {
  87. __this_cpu_dec(per_cpu_var(ftrace_cpu_disabled));
  88. preempt_enable();
  89. }
  90. static cpumask_var_t __read_mostly tracing_buffer_mask;
  91. /* Define which cpu buffers are currently read in trace_pipe */
  92. static cpumask_var_t tracing_reader_cpumask;
  93. #define for_each_tracing_cpu(cpu) \
  94. for_each_cpu(cpu, tracing_buffer_mask)
  95. /*
  96. * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
  97. *
  98. * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
  99. * is set, then ftrace_dump is called. This will output the contents
  100. * of the ftrace buffers to the console. This is very useful for
  101. * capturing traces that lead to crashes and outputing it to a
  102. * serial console.
  103. *
  104. * It is default off, but you can enable it with either specifying
  105. * "ftrace_dump_on_oops" in the kernel command line, or setting
  106. * /proc/sys/kernel/ftrace_dump_on_oops to true.
  107. */
  108. int ftrace_dump_on_oops;
  109. static int tracing_set_tracer(const char *buf);
  110. #define MAX_TRACER_SIZE 100
  111. static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
  112. static char *default_bootup_tracer;
  113. static int __init set_cmdline_ftrace(char *str)
  114. {
  115. strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
  116. default_bootup_tracer = bootup_tracer_buf;
  117. /* We are using ftrace early, expand it */
  118. ring_buffer_expanded = 1;
  119. return 1;
  120. }
  121. __setup("ftrace=", set_cmdline_ftrace);
  122. static int __init set_ftrace_dump_on_oops(char *str)
  123. {
  124. ftrace_dump_on_oops = 1;
  125. return 1;
  126. }
  127. __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
  128. unsigned long long ns2usecs(cycle_t nsec)
  129. {
  130. nsec += 500;
  131. do_div(nsec, 1000);
  132. return nsec;
  133. }
  134. /*
  135. * The global_trace is the descriptor that holds the tracing
  136. * buffers for the live tracing. For each CPU, it contains
  137. * a link list of pages that will store trace entries. The
  138. * page descriptor of the pages in the memory is used to hold
  139. * the link list by linking the lru item in the page descriptor
  140. * to each of the pages in the buffer per CPU.
  141. *
  142. * For each active CPU there is a data field that holds the
  143. * pages for the buffer for that CPU. Each CPU has the same number
  144. * of pages allocated for its buffer.
  145. */
  146. static struct trace_array global_trace;
  147. static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
  148. int filter_current_check_discard(struct ring_buffer *buffer,
  149. struct ftrace_event_call *call, void *rec,
  150. struct ring_buffer_event *event)
  151. {
  152. return filter_check_discard(call, rec, buffer, event);
  153. }
  154. EXPORT_SYMBOL_GPL(filter_current_check_discard);
  155. cycle_t ftrace_now(int cpu)
  156. {
  157. u64 ts;
  158. /* Early boot up does not have a buffer yet */
  159. if (!global_trace.buffer)
  160. return trace_clock_local();
  161. ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
  162. ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
  163. return ts;
  164. }
  165. /*
  166. * The max_tr is used to snapshot the global_trace when a maximum
  167. * latency is reached. Some tracers will use this to store a maximum
  168. * trace while it continues examining live traces.
  169. *
  170. * The buffers for the max_tr are set up the same as the global_trace.
  171. * When a snapshot is taken, the link list of the max_tr is swapped
  172. * with the link list of the global_trace and the buffers are reset for
  173. * the global_trace so the tracing can continue.
  174. */
  175. static struct trace_array max_tr;
  176. static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
  177. /* tracer_enabled is used to toggle activation of a tracer */
  178. static int tracer_enabled = 1;
  179. /**
  180. * tracing_is_enabled - return tracer_enabled status
  181. *
  182. * This function is used by other tracers to know the status
  183. * of the tracer_enabled flag. Tracers may use this function
  184. * to know if it should enable their features when starting
  185. * up. See irqsoff tracer for an example (start_irqsoff_tracer).
  186. */
  187. int tracing_is_enabled(void)
  188. {
  189. return tracer_enabled;
  190. }
  191. /*
  192. * trace_buf_size is the size in bytes that is allocated
  193. * for a buffer. Note, the number of bytes is always rounded
  194. * to page size.
  195. *
  196. * This number is purposely set to a low number of 16384.
  197. * If the dump on oops happens, it will be much appreciated
  198. * to not have to wait for all that output. Anyway this can be
  199. * boot time and run time configurable.
  200. */
  201. #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
  202. static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
  203. /* trace_types holds a link list of available tracers. */
  204. static struct tracer *trace_types __read_mostly;
  205. /* current_trace points to the tracer that is currently active */
  206. static struct tracer *current_trace __read_mostly;
  207. /*
  208. * trace_types_lock is used to protect the trace_types list.
  209. * This lock is also used to keep user access serialized.
  210. * Accesses from userspace will grab this lock while userspace
  211. * activities happen inside the kernel.
  212. */
  213. static DEFINE_MUTEX(trace_types_lock);
  214. /* trace_wait is a waitqueue for tasks blocked on trace_poll */
  215. static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
  216. /* trace_flags holds trace_options default values */
  217. unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
  218. TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
  219. TRACE_ITER_GRAPH_TIME;
  220. static int trace_stop_count;
  221. static DEFINE_SPINLOCK(tracing_start_lock);
  222. /**
  223. * trace_wake_up - wake up tasks waiting for trace input
  224. *
  225. * Simply wakes up any task that is blocked on the trace_wait
  226. * queue. These is used with trace_poll for tasks polling the trace.
  227. */
  228. void trace_wake_up(void)
  229. {
  230. int cpu;
  231. if (trace_flags & TRACE_ITER_BLOCK)
  232. return;
  233. /*
  234. * The runqueue_is_locked() can fail, but this is the best we
  235. * have for now:
  236. */
  237. cpu = get_cpu();
  238. if (!runqueue_is_locked(cpu))
  239. wake_up(&trace_wait);
  240. put_cpu();
  241. }
  242. static int __init set_buf_size(char *str)
  243. {
  244. unsigned long buf_size;
  245. if (!str)
  246. return 0;
  247. buf_size = memparse(str, &str);
  248. /* nr_entries can not be zero */
  249. if (buf_size == 0)
  250. return 0;
  251. trace_buf_size = buf_size;
  252. return 1;
  253. }
  254. __setup("trace_buf_size=", set_buf_size);
  255. unsigned long nsecs_to_usecs(unsigned long nsecs)
  256. {
  257. return nsecs / 1000;
  258. }
  259. /* These must match the bit postions in trace_iterator_flags */
  260. static const char *trace_options[] = {
  261. "print-parent",
  262. "sym-offset",
  263. "sym-addr",
  264. "verbose",
  265. "raw",
  266. "hex",
  267. "bin",
  268. "block",
  269. "stacktrace",
  270. "trace_printk",
  271. "ftrace_preempt",
  272. "branch",
  273. "annotate",
  274. "userstacktrace",
  275. "sym-userobj",
  276. "printk-msg-only",
  277. "context-info",
  278. "latency-format",
  279. "sleep-time",
  280. "graph-time",
  281. NULL
  282. };
  283. static struct {
  284. u64 (*func)(void);
  285. const char *name;
  286. } trace_clocks[] = {
  287. { trace_clock_local, "local" },
  288. { trace_clock_global, "global" },
  289. };
  290. int trace_clock_id;
  291. /*
  292. * trace_parser_get_init - gets the buffer for trace parser
  293. */
  294. int trace_parser_get_init(struct trace_parser *parser, int size)
  295. {
  296. memset(parser, 0, sizeof(*parser));
  297. parser->buffer = kmalloc(size, GFP_KERNEL);
  298. if (!parser->buffer)
  299. return 1;
  300. parser->size = size;
  301. return 0;
  302. }
  303. /*
  304. * trace_parser_put - frees the buffer for trace parser
  305. */
  306. void trace_parser_put(struct trace_parser *parser)
  307. {
  308. kfree(parser->buffer);
  309. }
  310. /*
  311. * trace_get_user - reads the user input string separated by space
  312. * (matched by isspace(ch))
  313. *
  314. * For each string found the 'struct trace_parser' is updated,
  315. * and the function returns.
  316. *
  317. * Returns number of bytes read.
  318. *
  319. * See kernel/trace/trace.h for 'struct trace_parser' details.
  320. */
  321. int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
  322. size_t cnt, loff_t *ppos)
  323. {
  324. char ch;
  325. size_t read = 0;
  326. ssize_t ret;
  327. if (!*ppos)
  328. trace_parser_clear(parser);
  329. ret = get_user(ch, ubuf++);
  330. if (ret)
  331. goto out;
  332. read++;
  333. cnt--;
  334. /*
  335. * The parser is not finished with the last write,
  336. * continue reading the user input without skipping spaces.
  337. */
  338. if (!parser->cont) {
  339. /* skip white space */
  340. while (cnt && isspace(ch)) {
  341. ret = get_user(ch, ubuf++);
  342. if (ret)
  343. goto out;
  344. read++;
  345. cnt--;
  346. }
  347. /* only spaces were written */
  348. if (isspace(ch)) {
  349. *ppos += read;
  350. ret = read;
  351. goto out;
  352. }
  353. parser->idx = 0;
  354. }
  355. /* read the non-space input */
  356. while (cnt && !isspace(ch)) {
  357. if (parser->idx < parser->size - 1)
  358. parser->buffer[parser->idx++] = ch;
  359. else {
  360. ret = -EINVAL;
  361. goto out;
  362. }
  363. ret = get_user(ch, ubuf++);
  364. if (ret)
  365. goto out;
  366. read++;
  367. cnt--;
  368. }
  369. /* We either got finished input or we have to wait for another call. */
  370. if (isspace(ch)) {
  371. parser->buffer[parser->idx] = 0;
  372. parser->cont = false;
  373. } else {
  374. parser->cont = true;
  375. parser->buffer[parser->idx++] = ch;
  376. }
  377. *ppos += read;
  378. ret = read;
  379. out:
  380. return ret;
  381. }
  382. ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
  383. {
  384. int len;
  385. int ret;
  386. if (!cnt)
  387. return 0;
  388. if (s->len <= s->readpos)
  389. return -EBUSY;
  390. len = s->len - s->readpos;
  391. if (cnt > len)
  392. cnt = len;
  393. ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
  394. if (ret == cnt)
  395. return -EFAULT;
  396. cnt -= ret;
  397. s->readpos += cnt;
  398. return cnt;
  399. }
  400. static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
  401. {
  402. int len;
  403. void *ret;
  404. if (s->len <= s->readpos)
  405. return -EBUSY;
  406. len = s->len - s->readpos;
  407. if (cnt > len)
  408. cnt = len;
  409. ret = memcpy(buf, s->buffer + s->readpos, cnt);
  410. if (!ret)
  411. return -EFAULT;
  412. s->readpos += cnt;
  413. return cnt;
  414. }
  415. /*
  416. * ftrace_max_lock is used to protect the swapping of buffers
  417. * when taking a max snapshot. The buffers themselves are
  418. * protected by per_cpu spinlocks. But the action of the swap
  419. * needs its own lock.
  420. *
  421. * This is defined as a arch_spinlock_t in order to help
  422. * with performance when lockdep debugging is enabled.
  423. *
  424. * It is also used in other places outside the update_max_tr
  425. * so it needs to be defined outside of the
  426. * CONFIG_TRACER_MAX_TRACE.
  427. */
  428. static arch_spinlock_t ftrace_max_lock =
  429. (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  430. #ifdef CONFIG_TRACER_MAX_TRACE
  431. unsigned long __read_mostly tracing_max_latency;
  432. unsigned long __read_mostly tracing_thresh;
  433. /*
  434. * Copy the new maximum trace into the separate maximum-trace
  435. * structure. (this way the maximum trace is permanently saved,
  436. * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
  437. */
  438. static void
  439. __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
  440. {
  441. struct trace_array_cpu *data = tr->data[cpu];
  442. struct trace_array_cpu *max_data = tr->data[cpu];
  443. max_tr.cpu = cpu;
  444. max_tr.time_start = data->preempt_timestamp;
  445. max_data = max_tr.data[cpu];
  446. max_data->saved_latency = tracing_max_latency;
  447. max_data->critical_start = data->critical_start;
  448. max_data->critical_end = data->critical_end;
  449. memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
  450. max_data->pid = tsk->pid;
  451. max_data->uid = task_uid(tsk);
  452. max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
  453. max_data->policy = tsk->policy;
  454. max_data->rt_priority = tsk->rt_priority;
  455. /* record this tasks comm */
  456. tracing_record_cmdline(tsk);
  457. }
  458. /**
  459. * update_max_tr - snapshot all trace buffers from global_trace to max_tr
  460. * @tr: tracer
  461. * @tsk: the task with the latency
  462. * @cpu: The cpu that initiated the trace.
  463. *
  464. * Flip the buffers between the @tr and the max_tr and record information
  465. * about which task was the cause of this latency.
  466. */
  467. void
  468. update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
  469. {
  470. struct ring_buffer *buf = tr->buffer;
  471. if (trace_stop_count)
  472. return;
  473. WARN_ON_ONCE(!irqs_disabled());
  474. arch_spin_lock(&ftrace_max_lock);
  475. tr->buffer = max_tr.buffer;
  476. max_tr.buffer = buf;
  477. __update_max_tr(tr, tsk, cpu);
  478. arch_spin_unlock(&ftrace_max_lock);
  479. }
  480. /**
  481. * update_max_tr_single - only copy one trace over, and reset the rest
  482. * @tr - tracer
  483. * @tsk - task with the latency
  484. * @cpu - the cpu of the buffer to copy.
  485. *
  486. * Flip the trace of a single CPU buffer between the @tr and the max_tr.
  487. */
  488. void
  489. update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
  490. {
  491. int ret;
  492. if (trace_stop_count)
  493. return;
  494. WARN_ON_ONCE(!irqs_disabled());
  495. arch_spin_lock(&ftrace_max_lock);
  496. ftrace_disable_cpu();
  497. ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
  498. if (ret == -EBUSY) {
  499. /*
  500. * We failed to swap the buffer due to a commit taking
  501. * place on this CPU. We fail to record, but we reset
  502. * the max trace buffer (no one writes directly to it)
  503. * and flag that it failed.
  504. */
  505. trace_array_printk(&max_tr, _THIS_IP_,
  506. "Failed to swap buffers due to commit in progress\n");
  507. }
  508. ftrace_enable_cpu();
  509. WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
  510. __update_max_tr(tr, tsk, cpu);
  511. arch_spin_unlock(&ftrace_max_lock);
  512. }
  513. #endif /* CONFIG_TRACER_MAX_TRACE */
  514. /**
  515. * register_tracer - register a tracer with the ftrace system.
  516. * @type - the plugin for the tracer
  517. *
  518. * Register a new plugin tracer.
  519. */
  520. int register_tracer(struct tracer *type)
  521. __releases(kernel_lock)
  522. __acquires(kernel_lock)
  523. {
  524. struct tracer *t;
  525. int ret = 0;
  526. if (!type->name) {
  527. pr_info("Tracer must have a name\n");
  528. return -1;
  529. }
  530. if (strlen(type->name) > MAX_TRACER_SIZE) {
  531. pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
  532. return -1;
  533. }
  534. /*
  535. * When this gets called we hold the BKL which means that
  536. * preemption is disabled. Various trace selftests however
  537. * need to disable and enable preemption for successful tests.
  538. * So we drop the BKL here and grab it after the tests again.
  539. */
  540. unlock_kernel();
  541. mutex_lock(&trace_types_lock);
  542. tracing_selftest_running = true;
  543. for (t = trace_types; t; t = t->next) {
  544. if (strcmp(type->name, t->name) == 0) {
  545. /* already found */
  546. pr_info("Tracer %s already registered\n",
  547. type->name);
  548. ret = -1;
  549. goto out;
  550. }
  551. }
  552. if (!type->set_flag)
  553. type->set_flag = &dummy_set_flag;
  554. if (!type->flags)
  555. type->flags = &dummy_tracer_flags;
  556. else
  557. if (!type->flags->opts)
  558. type->flags->opts = dummy_tracer_opt;
  559. if (!type->wait_pipe)
  560. type->wait_pipe = default_wait_pipe;
  561. #ifdef CONFIG_FTRACE_STARTUP_TEST
  562. if (type->selftest && !tracing_selftest_disabled) {
  563. struct tracer *saved_tracer = current_trace;
  564. struct trace_array *tr = &global_trace;
  565. /*
  566. * Run a selftest on this tracer.
  567. * Here we reset the trace buffer, and set the current
  568. * tracer to be this tracer. The tracer can then run some
  569. * internal tracing to verify that everything is in order.
  570. * If we fail, we do not register this tracer.
  571. */
  572. tracing_reset_online_cpus(tr);
  573. current_trace = type;
  574. /* the test is responsible for initializing and enabling */
  575. pr_info("Testing tracer %s: ", type->name);
  576. ret = type->selftest(type, tr);
  577. /* the test is responsible for resetting too */
  578. current_trace = saved_tracer;
  579. if (ret) {
  580. printk(KERN_CONT "FAILED!\n");
  581. goto out;
  582. }
  583. /* Only reset on passing, to avoid touching corrupted buffers */
  584. tracing_reset_online_cpus(tr);
  585. printk(KERN_CONT "PASSED\n");
  586. }
  587. #endif
  588. type->next = trace_types;
  589. trace_types = type;
  590. out:
  591. tracing_selftest_running = false;
  592. mutex_unlock(&trace_types_lock);
  593. if (ret || !default_bootup_tracer)
  594. goto out_unlock;
  595. if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
  596. goto out_unlock;
  597. printk(KERN_INFO "Starting tracer '%s'\n", type->name);
  598. /* Do we want this tracer to start on bootup? */
  599. tracing_set_tracer(type->name);
  600. default_bootup_tracer = NULL;
  601. /* disable other selftests, since this will break it. */
  602. tracing_selftest_disabled = 1;
  603. #ifdef CONFIG_FTRACE_STARTUP_TEST
  604. printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
  605. type->name);
  606. #endif
  607. out_unlock:
  608. lock_kernel();
  609. return ret;
  610. }
  611. void unregister_tracer(struct tracer *type)
  612. {
  613. struct tracer **t;
  614. mutex_lock(&trace_types_lock);
  615. for (t = &trace_types; *t; t = &(*t)->next) {
  616. if (*t == type)
  617. goto found;
  618. }
  619. pr_info("Tracer %s not registered\n", type->name);
  620. goto out;
  621. found:
  622. *t = (*t)->next;
  623. if (type == current_trace && tracer_enabled) {
  624. tracer_enabled = 0;
  625. tracing_stop();
  626. if (current_trace->stop)
  627. current_trace->stop(&global_trace);
  628. current_trace = &nop_trace;
  629. }
  630. out:
  631. mutex_unlock(&trace_types_lock);
  632. }
  633. static void __tracing_reset(struct trace_array *tr, int cpu)
  634. {
  635. ftrace_disable_cpu();
  636. ring_buffer_reset_cpu(tr->buffer, cpu);
  637. ftrace_enable_cpu();
  638. }
  639. void tracing_reset(struct trace_array *tr, int cpu)
  640. {
  641. struct ring_buffer *buffer = tr->buffer;
  642. ring_buffer_record_disable(buffer);
  643. /* Make sure all commits have finished */
  644. synchronize_sched();
  645. __tracing_reset(tr, cpu);
  646. ring_buffer_record_enable(buffer);
  647. }
  648. void tracing_reset_online_cpus(struct trace_array *tr)
  649. {
  650. struct ring_buffer *buffer = tr->buffer;
  651. int cpu;
  652. ring_buffer_record_disable(buffer);
  653. /* Make sure all commits have finished */
  654. synchronize_sched();
  655. tr->time_start = ftrace_now(tr->cpu);
  656. for_each_online_cpu(cpu)
  657. __tracing_reset(tr, cpu);
  658. ring_buffer_record_enable(buffer);
  659. }
  660. void tracing_reset_current(int cpu)
  661. {
  662. tracing_reset(&global_trace, cpu);
  663. }
  664. void tracing_reset_current_online_cpus(void)
  665. {
  666. tracing_reset_online_cpus(&global_trace);
  667. }
  668. #define SAVED_CMDLINES 128
  669. #define NO_CMDLINE_MAP UINT_MAX
  670. static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
  671. static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
  672. static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
  673. static int cmdline_idx;
  674. static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
  675. /* temporary disable recording */
  676. static atomic_t trace_record_cmdline_disabled __read_mostly;
  677. static void trace_init_cmdlines(void)
  678. {
  679. memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
  680. memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
  681. cmdline_idx = 0;
  682. }
  683. int is_tracing_stopped(void)
  684. {
  685. return trace_stop_count;
  686. }
  687. /**
  688. * ftrace_off_permanent - disable all ftrace code permanently
  689. *
  690. * This should only be called when a serious anomally has
  691. * been detected. This will turn off the function tracing,
  692. * ring buffers, and other tracing utilites. It takes no
  693. * locks and can be called from any context.
  694. */
  695. void ftrace_off_permanent(void)
  696. {
  697. tracing_disabled = 1;
  698. ftrace_stop();
  699. tracing_off_permanent();
  700. }
  701. /**
  702. * tracing_start - quick start of the tracer
  703. *
  704. * If tracing is enabled but was stopped by tracing_stop,
  705. * this will start the tracer back up.
  706. */
  707. void tracing_start(void)
  708. {
  709. struct ring_buffer *buffer;
  710. unsigned long flags;
  711. if (tracing_disabled)
  712. return;
  713. spin_lock_irqsave(&tracing_start_lock, flags);
  714. if (--trace_stop_count) {
  715. if (trace_stop_count < 0) {
  716. /* Someone screwed up their debugging */
  717. WARN_ON_ONCE(1);
  718. trace_stop_count = 0;
  719. }
  720. goto out;
  721. }
  722. buffer = global_trace.buffer;
  723. if (buffer)
  724. ring_buffer_record_enable(buffer);
  725. buffer = max_tr.buffer;
  726. if (buffer)
  727. ring_buffer_record_enable(buffer);
  728. ftrace_start();
  729. out:
  730. spin_unlock_irqrestore(&tracing_start_lock, flags);
  731. }
  732. /**
  733. * tracing_stop - quick stop of the tracer
  734. *
  735. * Light weight way to stop tracing. Use in conjunction with
  736. * tracing_start.
  737. */
  738. void tracing_stop(void)
  739. {
  740. struct ring_buffer *buffer;
  741. unsigned long flags;
  742. ftrace_stop();
  743. spin_lock_irqsave(&tracing_start_lock, flags);
  744. if (trace_stop_count++)
  745. goto out;
  746. buffer = global_trace.buffer;
  747. if (buffer)
  748. ring_buffer_record_disable(buffer);
  749. buffer = max_tr.buffer;
  750. if (buffer)
  751. ring_buffer_record_disable(buffer);
  752. out:
  753. spin_unlock_irqrestore(&tracing_start_lock, flags);
  754. }
  755. void trace_stop_cmdline_recording(void);
  756. static void trace_save_cmdline(struct task_struct *tsk)
  757. {
  758. unsigned pid, idx;
  759. if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
  760. return;
  761. /*
  762. * It's not the end of the world if we don't get
  763. * the lock, but we also don't want to spin
  764. * nor do we want to disable interrupts,
  765. * so if we miss here, then better luck next time.
  766. */
  767. if (!arch_spin_trylock(&trace_cmdline_lock))
  768. return;
  769. idx = map_pid_to_cmdline[tsk->pid];
  770. if (idx == NO_CMDLINE_MAP) {
  771. idx = (cmdline_idx + 1) % SAVED_CMDLINES;
  772. /*
  773. * Check whether the cmdline buffer at idx has a pid
  774. * mapped. We are going to overwrite that entry so we
  775. * need to clear the map_pid_to_cmdline. Otherwise we
  776. * would read the new comm for the old pid.
  777. */
  778. pid = map_cmdline_to_pid[idx];
  779. if (pid != NO_CMDLINE_MAP)
  780. map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
  781. map_cmdline_to_pid[idx] = tsk->pid;
  782. map_pid_to_cmdline[tsk->pid] = idx;
  783. cmdline_idx = idx;
  784. }
  785. memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
  786. arch_spin_unlock(&trace_cmdline_lock);
  787. }
  788. void trace_find_cmdline(int pid, char comm[])
  789. {
  790. unsigned map;
  791. if (!pid) {
  792. strcpy(comm, "<idle>");
  793. return;
  794. }
  795. if (WARN_ON_ONCE(pid < 0)) {
  796. strcpy(comm, "<XXX>");
  797. return;
  798. }
  799. if (pid > PID_MAX_DEFAULT) {
  800. strcpy(comm, "<...>");
  801. return;
  802. }
  803. preempt_disable();
  804. arch_spin_lock(&trace_cmdline_lock);
  805. map = map_pid_to_cmdline[pid];
  806. if (map != NO_CMDLINE_MAP)
  807. strcpy(comm, saved_cmdlines[map]);
  808. else
  809. strcpy(comm, "<...>");
  810. arch_spin_unlock(&trace_cmdline_lock);
  811. preempt_enable();
  812. }
  813. void tracing_record_cmdline(struct task_struct *tsk)
  814. {
  815. if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
  816. !tracing_is_on())
  817. return;
  818. trace_save_cmdline(tsk);
  819. }
  820. void
  821. tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
  822. int pc)
  823. {
  824. struct task_struct *tsk = current;
  825. entry->preempt_count = pc & 0xff;
  826. entry->pid = (tsk) ? tsk->pid : 0;
  827. entry->lock_depth = (tsk) ? tsk->lock_depth : 0;
  828. entry->flags =
  829. #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
  830. (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
  831. #else
  832. TRACE_FLAG_IRQS_NOSUPPORT |
  833. #endif
  834. ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
  835. ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
  836. (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
  837. }
  838. EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
  839. struct ring_buffer_event *
  840. trace_buffer_lock_reserve(struct ring_buffer *buffer,
  841. int type,
  842. unsigned long len,
  843. unsigned long flags, int pc)
  844. {
  845. struct ring_buffer_event *event;
  846. event = ring_buffer_lock_reserve(buffer, len);
  847. if (event != NULL) {
  848. struct trace_entry *ent = ring_buffer_event_data(event);
  849. tracing_generic_entry_update(ent, flags, pc);
  850. ent->type = type;
  851. }
  852. return event;
  853. }
  854. static inline void
  855. __trace_buffer_unlock_commit(struct ring_buffer *buffer,
  856. struct ring_buffer_event *event,
  857. unsigned long flags, int pc,
  858. int wake)
  859. {
  860. ring_buffer_unlock_commit(buffer, event);
  861. ftrace_trace_stack(buffer, flags, 6, pc);
  862. ftrace_trace_userstack(buffer, flags, pc);
  863. if (wake)
  864. trace_wake_up();
  865. }
  866. void trace_buffer_unlock_commit(struct ring_buffer *buffer,
  867. struct ring_buffer_event *event,
  868. unsigned long flags, int pc)
  869. {
  870. __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
  871. }
  872. struct ring_buffer_event *
  873. trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
  874. int type, unsigned long len,
  875. unsigned long flags, int pc)
  876. {
  877. *current_rb = global_trace.buffer;
  878. return trace_buffer_lock_reserve(*current_rb,
  879. type, len, flags, pc);
  880. }
  881. EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
  882. void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
  883. struct ring_buffer_event *event,
  884. unsigned long flags, int pc)
  885. {
  886. __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
  887. }
  888. EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
  889. void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
  890. struct ring_buffer_event *event,
  891. unsigned long flags, int pc)
  892. {
  893. __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
  894. }
  895. EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
  896. void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
  897. struct ring_buffer_event *event)
  898. {
  899. ring_buffer_discard_commit(buffer, event);
  900. }
  901. EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
  902. void
  903. trace_function(struct trace_array *tr,
  904. unsigned long ip, unsigned long parent_ip, unsigned long flags,
  905. int pc)
  906. {
  907. struct ftrace_event_call *call = &event_function;
  908. struct ring_buffer *buffer = tr->buffer;
  909. struct ring_buffer_event *event;
  910. struct ftrace_entry *entry;
  911. /* If we are reading the ring buffer, don't trace */
  912. if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled))))
  913. return;
  914. event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
  915. flags, pc);
  916. if (!event)
  917. return;
  918. entry = ring_buffer_event_data(event);
  919. entry->ip = ip;
  920. entry->parent_ip = parent_ip;
  921. if (!filter_check_discard(call, entry, buffer, event))
  922. ring_buffer_unlock_commit(buffer, event);
  923. }
  924. void
  925. ftrace(struct trace_array *tr, struct trace_array_cpu *data,
  926. unsigned long ip, unsigned long parent_ip, unsigned long flags,
  927. int pc)
  928. {
  929. if (likely(!atomic_read(&data->disabled)))
  930. trace_function(tr, ip, parent_ip, flags, pc);
  931. }
  932. #ifdef CONFIG_STACKTRACE
  933. static void __ftrace_trace_stack(struct ring_buffer *buffer,
  934. unsigned long flags,
  935. int skip, int pc)
  936. {
  937. struct ftrace_event_call *call = &event_kernel_stack;
  938. struct ring_buffer_event *event;
  939. struct stack_entry *entry;
  940. struct stack_trace trace;
  941. event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
  942. sizeof(*entry), flags, pc);
  943. if (!event)
  944. return;
  945. entry = ring_buffer_event_data(event);
  946. memset(&entry->caller, 0, sizeof(entry->caller));
  947. trace.nr_entries = 0;
  948. trace.max_entries = FTRACE_STACK_ENTRIES;
  949. trace.skip = skip;
  950. trace.entries = entry->caller;
  951. save_stack_trace(&trace);
  952. if (!filter_check_discard(call, entry, buffer, event))
  953. ring_buffer_unlock_commit(buffer, event);
  954. }
  955. void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
  956. int skip, int pc)
  957. {
  958. if (!(trace_flags & TRACE_ITER_STACKTRACE))
  959. return;
  960. __ftrace_trace_stack(buffer, flags, skip, pc);
  961. }
  962. void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
  963. int pc)
  964. {
  965. __ftrace_trace_stack(tr->buffer, flags, skip, pc);
  966. }
  967. /**
  968. * trace_dump_stack - record a stack back trace in the trace buffer
  969. */
  970. void trace_dump_stack(void)
  971. {
  972. unsigned long flags;
  973. if (tracing_disabled || tracing_selftest_running)
  974. return;
  975. local_save_flags(flags);
  976. /* skipping 3 traces, seems to get us at the caller of this function */
  977. __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count());
  978. }
  979. void
  980. ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
  981. {
  982. struct ftrace_event_call *call = &event_user_stack;
  983. struct ring_buffer_event *event;
  984. struct userstack_entry *entry;
  985. struct stack_trace trace;
  986. if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
  987. return;
  988. event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
  989. sizeof(*entry), flags, pc);
  990. if (!event)
  991. return;
  992. entry = ring_buffer_event_data(event);
  993. entry->tgid = current->tgid;
  994. memset(&entry->caller, 0, sizeof(entry->caller));
  995. trace.nr_entries = 0;
  996. trace.max_entries = FTRACE_STACK_ENTRIES;
  997. trace.skip = 0;
  998. trace.entries = entry->caller;
  999. save_stack_trace_user(&trace);
  1000. if (!filter_check_discard(call, entry, buffer, event))
  1001. ring_buffer_unlock_commit(buffer, event);
  1002. }
  1003. #ifdef UNUSED
  1004. static void __trace_userstack(struct trace_array *tr, unsigned long flags)
  1005. {
  1006. ftrace_trace_userstack(tr, flags, preempt_count());
  1007. }
  1008. #endif /* UNUSED */
  1009. #endif /* CONFIG_STACKTRACE */
  1010. static void
  1011. ftrace_trace_special(void *__tr,
  1012. unsigned long arg1, unsigned long arg2, unsigned long arg3,
  1013. int pc)
  1014. {
  1015. struct ftrace_event_call *call = &event_special;
  1016. struct ring_buffer_event *event;
  1017. struct trace_array *tr = __tr;
  1018. struct ring_buffer *buffer = tr->buffer;
  1019. struct special_entry *entry;
  1020. event = trace_buffer_lock_reserve(buffer, TRACE_SPECIAL,
  1021. sizeof(*entry), 0, pc);
  1022. if (!event)
  1023. return;
  1024. entry = ring_buffer_event_data(event);
  1025. entry->arg1 = arg1;
  1026. entry->arg2 = arg2;
  1027. entry->arg3 = arg3;
  1028. if (!filter_check_discard(call, entry, buffer, event))
  1029. trace_buffer_unlock_commit(buffer, event, 0, pc);
  1030. }
  1031. void
  1032. __trace_special(void *__tr, void *__data,
  1033. unsigned long arg1, unsigned long arg2, unsigned long arg3)
  1034. {
  1035. ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
  1036. }
  1037. void
  1038. ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
  1039. {
  1040. struct trace_array *tr = &global_trace;
  1041. struct trace_array_cpu *data;
  1042. unsigned long flags;
  1043. int cpu;
  1044. int pc;
  1045. if (tracing_disabled)
  1046. return;
  1047. pc = preempt_count();
  1048. local_irq_save(flags);
  1049. cpu = raw_smp_processor_id();
  1050. data = tr->data[cpu];
  1051. if (likely(atomic_inc_return(&data->disabled) == 1))
  1052. ftrace_trace_special(tr, arg1, arg2, arg3, pc);
  1053. atomic_dec(&data->disabled);
  1054. local_irq_restore(flags);
  1055. }
  1056. /**
  1057. * trace_vbprintk - write binary msg to tracing buffer
  1058. *
  1059. */
  1060. int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
  1061. {
  1062. static arch_spinlock_t trace_buf_lock =
  1063. (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  1064. static u32 trace_buf[TRACE_BUF_SIZE];
  1065. struct ftrace_event_call *call = &event_bprint;
  1066. struct ring_buffer_event *event;
  1067. struct ring_buffer *buffer;
  1068. struct trace_array *tr = &global_trace;
  1069. struct trace_array_cpu *data;
  1070. struct bprint_entry *entry;
  1071. unsigned long flags;
  1072. int disable;
  1073. int resched;
  1074. int cpu, len = 0, size, pc;
  1075. if (unlikely(tracing_selftest_running || tracing_disabled))
  1076. return 0;
  1077. /* Don't pollute graph traces with trace_vprintk internals */
  1078. pause_graph_tracing();
  1079. pc = preempt_count();
  1080. resched = ftrace_preempt_disable();
  1081. cpu = raw_smp_processor_id();
  1082. data = tr->data[cpu];
  1083. disable = atomic_inc_return(&data->disabled);
  1084. if (unlikely(disable != 1))
  1085. goto out;
  1086. /* Lockdep uses trace_printk for lock tracing */
  1087. local_irq_save(flags);
  1088. arch_spin_lock(&trace_buf_lock);
  1089. len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
  1090. if (len > TRACE_BUF_SIZE || len < 0)
  1091. goto out_unlock;
  1092. size = sizeof(*entry) + sizeof(u32) * len;
  1093. buffer = tr->buffer;
  1094. event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
  1095. flags, pc);
  1096. if (!event)
  1097. goto out_unlock;
  1098. entry = ring_buffer_event_data(event);
  1099. entry->ip = ip;
  1100. entry->fmt = fmt;
  1101. memcpy(entry->buf, trace_buf, sizeof(u32) * len);
  1102. if (!filter_check_discard(call, entry, buffer, event))
  1103. ring_buffer_unlock_commit(buffer, event);
  1104. out_unlock:
  1105. arch_spin_unlock(&trace_buf_lock);
  1106. local_irq_restore(flags);
  1107. out:
  1108. atomic_dec_return(&data->disabled);
  1109. ftrace_preempt_enable(resched);
  1110. unpause_graph_tracing();
  1111. return len;
  1112. }
  1113. EXPORT_SYMBOL_GPL(trace_vbprintk);
  1114. int trace_array_printk(struct trace_array *tr,
  1115. unsigned long ip, const char *fmt, ...)
  1116. {
  1117. int ret;
  1118. va_list ap;
  1119. if (!(trace_flags & TRACE_ITER_PRINTK))
  1120. return 0;
  1121. va_start(ap, fmt);
  1122. ret = trace_array_vprintk(tr, ip, fmt, ap);
  1123. va_end(ap);
  1124. return ret;
  1125. }
  1126. int trace_array_vprintk(struct trace_array *tr,
  1127. unsigned long ip, const char *fmt, va_list args)
  1128. {
  1129. static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
  1130. static char trace_buf[TRACE_BUF_SIZE];
  1131. struct ftrace_event_call *call = &event_print;
  1132. struct ring_buffer_event *event;
  1133. struct ring_buffer *buffer;
  1134. struct trace_array_cpu *data;
  1135. int cpu, len = 0, size, pc;
  1136. struct print_entry *entry;
  1137. unsigned long irq_flags;
  1138. int disable;
  1139. if (tracing_disabled || tracing_selftest_running)
  1140. return 0;
  1141. pc = preempt_count();
  1142. preempt_disable_notrace();
  1143. cpu = raw_smp_processor_id();
  1144. data = tr->data[cpu];
  1145. disable = atomic_inc_return(&data->disabled);
  1146. if (unlikely(disable != 1))
  1147. goto out;
  1148. pause_graph_tracing();
  1149. raw_local_irq_save(irq_flags);
  1150. arch_spin_lock(&trace_buf_lock);
  1151. len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
  1152. size = sizeof(*entry) + len + 1;
  1153. buffer = tr->buffer;
  1154. event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
  1155. irq_flags, pc);
  1156. if (!event)
  1157. goto out_unlock;
  1158. entry = ring_buffer_event_data(event);
  1159. entry->ip = ip;
  1160. memcpy(&entry->buf, trace_buf, len);
  1161. entry->buf[len] = '\0';
  1162. if (!filter_check_discard(call, entry, buffer, event))
  1163. ring_buffer_unlock_commit(buffer, event);
  1164. out_unlock:
  1165. arch_spin_unlock(&trace_buf_lock);
  1166. raw_local_irq_restore(irq_flags);
  1167. unpause_graph_tracing();
  1168. out:
  1169. atomic_dec_return(&data->disabled);
  1170. preempt_enable_notrace();
  1171. return len;
  1172. }
  1173. int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
  1174. {
  1175. return trace_array_vprintk(&global_trace, ip, fmt, args);
  1176. }
  1177. EXPORT_SYMBOL_GPL(trace_vprintk);
  1178. enum trace_file_type {
  1179. TRACE_FILE_LAT_FMT = 1,
  1180. TRACE_FILE_ANNOTATE = 2,
  1181. };
  1182. static void trace_iterator_increment(struct trace_iterator *iter)
  1183. {
  1184. /* Don't allow ftrace to trace into the ring buffers */
  1185. ftrace_disable_cpu();
  1186. iter->idx++;
  1187. if (iter->buffer_iter[iter->cpu])
  1188. ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
  1189. ftrace_enable_cpu();
  1190. }
  1191. static struct trace_entry *
  1192. peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
  1193. {
  1194. struct ring_buffer_event *event;
  1195. struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
  1196. /* Don't allow ftrace to trace into the ring buffers */
  1197. ftrace_disable_cpu();
  1198. if (buf_iter)
  1199. event = ring_buffer_iter_peek(buf_iter, ts);
  1200. else
  1201. event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
  1202. ftrace_enable_cpu();
  1203. return event ? ring_buffer_event_data(event) : NULL;
  1204. }
  1205. static struct trace_entry *
  1206. __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
  1207. {
  1208. struct ring_buffer *buffer = iter->tr->buffer;
  1209. struct trace_entry *ent, *next = NULL;
  1210. int cpu_file = iter->cpu_file;
  1211. u64 next_ts = 0, ts;
  1212. int next_cpu = -1;
  1213. int cpu;
  1214. /*
  1215. * If we are in a per_cpu trace file, don't bother by iterating over
  1216. * all cpu and peek directly.
  1217. */
  1218. if (cpu_file > TRACE_PIPE_ALL_CPU) {
  1219. if (ring_buffer_empty_cpu(buffer, cpu_file))
  1220. return NULL;
  1221. ent = peek_next_entry(iter, cpu_file, ent_ts);
  1222. if (ent_cpu)
  1223. *ent_cpu = cpu_file;
  1224. return ent;
  1225. }
  1226. for_each_tracing_cpu(cpu) {
  1227. if (ring_buffer_empty_cpu(buffer, cpu))
  1228. continue;
  1229. ent = peek_next_entry(iter, cpu, &ts);
  1230. /*
  1231. * Pick the entry with the smallest timestamp:
  1232. */
  1233. if (ent && (!next || ts < next_ts)) {
  1234. next = ent;
  1235. next_cpu = cpu;
  1236. next_ts = ts;
  1237. }
  1238. }
  1239. if (ent_cpu)
  1240. *ent_cpu = next_cpu;
  1241. if (ent_ts)
  1242. *ent_ts = next_ts;
  1243. return next;
  1244. }
  1245. /* Find the next real entry, without updating the iterator itself */
  1246. struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
  1247. int *ent_cpu, u64 *ent_ts)
  1248. {
  1249. return __find_next_entry(iter, ent_cpu, ent_ts);
  1250. }
  1251. /* Find the next real entry, and increment the iterator to the next entry */
  1252. static void *find_next_entry_inc(struct trace_iterator *iter)
  1253. {
  1254. iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
  1255. if (iter->ent)
  1256. trace_iterator_increment(iter);
  1257. return iter->ent ? iter : NULL;
  1258. }
  1259. static void trace_consume(struct trace_iterator *iter)
  1260. {
  1261. /* Don't allow ftrace to trace into the ring buffers */
  1262. ftrace_disable_cpu();
  1263. ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
  1264. ftrace_enable_cpu();
  1265. }
  1266. static void *s_next(struct seq_file *m, void *v, loff_t *pos)
  1267. {
  1268. struct trace_iterator *iter = m->private;
  1269. int i = (int)*pos;
  1270. void *ent;
  1271. WARN_ON_ONCE(iter->leftover);
  1272. (*pos)++;
  1273. /* can't go backwards */
  1274. if (iter->idx > i)
  1275. return NULL;
  1276. if (iter->idx < 0)
  1277. ent = find_next_entry_inc(iter);
  1278. else
  1279. ent = iter;
  1280. while (ent && iter->idx < i)
  1281. ent = find_next_entry_inc(iter);
  1282. iter->pos = *pos;
  1283. return ent;
  1284. }
  1285. static void tracing_iter_reset(struct trace_iterator *iter, int cpu)
  1286. {
  1287. struct trace_array *tr = iter->tr;
  1288. struct ring_buffer_event *event;
  1289. struct ring_buffer_iter *buf_iter;
  1290. unsigned long entries = 0;
  1291. u64 ts;
  1292. tr->data[cpu]->skipped_entries = 0;
  1293. if (!iter->buffer_iter[cpu])
  1294. return;
  1295. buf_iter = iter->buffer_iter[cpu];
  1296. ring_buffer_iter_reset(buf_iter);
  1297. /*
  1298. * We could have the case with the max latency tracers
  1299. * that a reset never took place on a cpu. This is evident
  1300. * by the timestamp being before the start of the buffer.
  1301. */
  1302. while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
  1303. if (ts >= iter->tr->time_start)
  1304. break;
  1305. entries++;
  1306. ring_buffer_read(buf_iter, NULL);
  1307. }
  1308. tr->data[cpu]->skipped_entries = entries;
  1309. }
  1310. /*
  1311. * No necessary locking here. The worst thing which can
  1312. * happen is loosing events consumed at the same time
  1313. * by a trace_pipe reader.
  1314. * Other than that, we don't risk to crash the ring buffer
  1315. * because it serializes the readers.
  1316. *
  1317. * The current tracer is copied to avoid a global locking
  1318. * all around.
  1319. */
  1320. static void *s_start(struct seq_file *m, loff_t *pos)
  1321. {
  1322. struct trace_iterator *iter = m->private;
  1323. static struct tracer *old_tracer;
  1324. int cpu_file = iter->cpu_file;
  1325. void *p = NULL;
  1326. loff_t l = 0;
  1327. int cpu;
  1328. /* copy the tracer to avoid using a global lock all around */
  1329. mutex_lock(&trace_types_lock);
  1330. if (unlikely(old_tracer != current_trace && current_trace)) {
  1331. old_tracer = current_trace;
  1332. *iter->trace = *current_trace;
  1333. }
  1334. mutex_unlock(&trace_types_lock);
  1335. atomic_inc(&trace_record_cmdline_disabled);
  1336. if (*pos != iter->pos) {
  1337. iter->ent = NULL;
  1338. iter->cpu = 0;
  1339. iter->idx = -1;
  1340. ftrace_disable_cpu();
  1341. if (cpu_file == TRACE_PIPE_ALL_CPU) {
  1342. for_each_tracing_cpu(cpu)
  1343. tracing_iter_reset(iter, cpu);
  1344. } else
  1345. tracing_iter_reset(iter, cpu_file);
  1346. ftrace_enable_cpu();
  1347. for (p = iter; p && l < *pos; p = s_next(m, p, &l))
  1348. ;
  1349. } else {
  1350. /*
  1351. * If we overflowed the seq_file before, then we want
  1352. * to just reuse the trace_seq buffer again.
  1353. */
  1354. if (iter->leftover)
  1355. p = iter;
  1356. else {
  1357. l = *pos - 1;
  1358. p = s_next(m, p, &l);
  1359. }
  1360. }
  1361. trace_event_read_lock();
  1362. return p;
  1363. }
  1364. static void s_stop(struct seq_file *m, void *p)
  1365. {
  1366. atomic_dec(&trace_record_cmdline_disabled);
  1367. trace_event_read_unlock();
  1368. }
  1369. static void print_lat_help_header(struct seq_file *m)
  1370. {
  1371. seq_puts(m, "# _------=> CPU# \n");
  1372. seq_puts(m, "# / _-----=> irqs-off \n");
  1373. seq_puts(m, "# | / _----=> need-resched \n");
  1374. seq_puts(m, "# || / _---=> hardirq/softirq \n");
  1375. seq_puts(m, "# ||| / _--=> preempt-depth \n");
  1376. seq_puts(m, "# |||| /_--=> lock-depth \n");
  1377. seq_puts(m, "# |||||/ delay \n");
  1378. seq_puts(m, "# cmd pid |||||| time | caller \n");
  1379. seq_puts(m, "# \\ / |||||| \\ | / \n");
  1380. }
  1381. static void print_func_help_header(struct seq_file *m)
  1382. {
  1383. seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
  1384. seq_puts(m, "# | | | | |\n");
  1385. }
  1386. static void
  1387. print_trace_header(struct seq_file *m, struct trace_iterator *iter)
  1388. {
  1389. unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
  1390. struct trace_array *tr = iter->tr;
  1391. struct trace_array_cpu *data = tr->data[tr->cpu];
  1392. struct tracer *type = current_trace;
  1393. unsigned long entries = 0;
  1394. unsigned long total = 0;
  1395. unsigned long count;
  1396. const char *name = "preemption";
  1397. int cpu;
  1398. if (type)
  1399. name = type->name;
  1400. for_each_tracing_cpu(cpu) {
  1401. count = ring_buffer_entries_cpu(tr->buffer, cpu);
  1402. /*
  1403. * If this buffer has skipped entries, then we hold all
  1404. * entries for the trace and we need to ignore the
  1405. * ones before the time stamp.
  1406. */
  1407. if (tr->data[cpu]->skipped_entries) {
  1408. count -= tr->data[cpu]->skipped_entries;
  1409. /* total is the same as the entries */
  1410. total += count;
  1411. } else
  1412. total += count +
  1413. ring_buffer_overrun_cpu(tr->buffer, cpu);
  1414. entries += count;
  1415. }
  1416. seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
  1417. name, UTS_RELEASE);
  1418. seq_puts(m, "# -----------------------------------"
  1419. "---------------------------------\n");
  1420. seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
  1421. " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
  1422. nsecs_to_usecs(data->saved_latency),
  1423. entries,
  1424. total,
  1425. tr->cpu,
  1426. #if defined(CONFIG_PREEMPT_NONE)
  1427. "server",
  1428. #elif defined(CONFIG_PREEMPT_VOLUNTARY)
  1429. "desktop",
  1430. #elif defined(CONFIG_PREEMPT)
  1431. "preempt",
  1432. #else
  1433. "unknown",
  1434. #endif
  1435. /* These are reserved for later use */
  1436. 0, 0, 0, 0);
  1437. #ifdef CONFIG_SMP
  1438. seq_printf(m, " #P:%d)\n", num_online_cpus());
  1439. #else
  1440. seq_puts(m, ")\n");
  1441. #endif
  1442. seq_puts(m, "# -----------------\n");
  1443. seq_printf(m, "# | task: %.16s-%d "
  1444. "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
  1445. data->comm, data->pid, data->uid, data->nice,
  1446. data->policy, data->rt_priority);
  1447. seq_puts(m, "# -----------------\n");
  1448. if (data->critical_start) {
  1449. seq_puts(m, "# => started at: ");
  1450. seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
  1451. trace_print_seq(m, &iter->seq);
  1452. seq_puts(m, "\n# => ended at: ");
  1453. seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
  1454. trace_print_seq(m, &iter->seq);
  1455. seq_puts(m, "\n#\n");
  1456. }
  1457. seq_puts(m, "#\n");
  1458. }
  1459. static void test_cpu_buff_start(struct trace_iterator *iter)
  1460. {
  1461. struct trace_seq *s = &iter->seq;
  1462. if (!(trace_flags & TRACE_ITER_ANNOTATE))
  1463. return;
  1464. if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
  1465. return;
  1466. if (cpumask_test_cpu(iter->cpu, iter->started))
  1467. return;
  1468. if (iter->tr->data[iter->cpu]->skipped_entries)
  1469. return;
  1470. cpumask_set_cpu(iter->cpu, iter->started);
  1471. /* Don't print started cpu buffer for the first entry of the trace */
  1472. if (iter->idx > 1)
  1473. trace_seq_printf(s, "##### CPU %u buffer started ####\n",
  1474. iter->cpu);
  1475. }
  1476. static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
  1477. {
  1478. struct trace_seq *s = &iter->seq;
  1479. unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
  1480. struct trace_entry *entry;
  1481. struct trace_event *event;
  1482. entry = iter->ent;
  1483. test_cpu_buff_start(iter);
  1484. event = ftrace_find_event(entry->type);
  1485. if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
  1486. if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
  1487. if (!trace_print_lat_context(iter))
  1488. goto partial;
  1489. } else {
  1490. if (!trace_print_context(iter))
  1491. goto partial;
  1492. }
  1493. }
  1494. if (event)
  1495. return event->trace(iter, sym_flags);
  1496. if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
  1497. goto partial;
  1498. return TRACE_TYPE_HANDLED;
  1499. partial:
  1500. return TRACE_TYPE_PARTIAL_LINE;
  1501. }
  1502. static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
  1503. {
  1504. struct trace_seq *s = &iter->seq;
  1505. struct trace_entry *entry;
  1506. struct trace_event *event;
  1507. entry = iter->ent;
  1508. if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
  1509. if (!trace_seq_printf(s, "%d %d %llu ",
  1510. entry->pid, iter->cpu, iter->ts))
  1511. goto partial;
  1512. }
  1513. event = ftrace_find_event(entry->type);
  1514. if (event)
  1515. return event->raw(iter, 0);
  1516. if (!trace_seq_printf(s, "%d ?\n", entry->type))
  1517. goto partial;
  1518. return TRACE_TYPE_HANDLED;
  1519. partial:
  1520. return TRACE_TYPE_PARTIAL_LINE;
  1521. }
  1522. static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
  1523. {
  1524. struct trace_seq *s = &iter->seq;
  1525. unsigned char newline = '\n';
  1526. struct trace_entry *entry;
  1527. struct trace_event *event;
  1528. entry = iter->ent;
  1529. if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
  1530. SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
  1531. SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
  1532. SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
  1533. }
  1534. event = ftrace_find_event(entry->type);
  1535. if (event) {
  1536. enum print_line_t ret = event->hex(iter, 0);
  1537. if (ret != TRACE_TYPE_HANDLED)
  1538. return ret;
  1539. }
  1540. SEQ_PUT_FIELD_RET(s, newline);
  1541. return TRACE_TYPE_HANDLED;
  1542. }
  1543. static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
  1544. {
  1545. struct trace_seq *s = &iter->seq;
  1546. struct trace_entry *entry;
  1547. struct trace_event *event;
  1548. entry = iter->ent;
  1549. if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
  1550. SEQ_PUT_FIELD_RET(s, entry->pid);
  1551. SEQ_PUT_FIELD_RET(s, iter->cpu);
  1552. SEQ_PUT_FIELD_RET(s, iter->ts);
  1553. }
  1554. event = ftrace_find_event(entry->type);
  1555. return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
  1556. }
  1557. static int trace_empty(struct trace_iterator *iter)
  1558. {
  1559. int cpu;
  1560. /* If we are looking at one CPU buffer, only check that one */
  1561. if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
  1562. cpu = iter->cpu_file;
  1563. if (iter->buffer_iter[cpu]) {
  1564. if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
  1565. return 0;
  1566. } else {
  1567. if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
  1568. return 0;
  1569. }
  1570. return 1;
  1571. }
  1572. for_each_tracing_cpu(cpu) {
  1573. if (iter->buffer_iter[cpu]) {
  1574. if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
  1575. return 0;
  1576. } else {
  1577. if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
  1578. return 0;
  1579. }
  1580. }
  1581. return 1;
  1582. }
  1583. /* Called with trace_event_read_lock() held. */
  1584. static enum print_line_t print_trace_line(struct trace_iterator *iter)
  1585. {
  1586. enum print_line_t ret;
  1587. if (iter->trace && iter->trace->print_line) {
  1588. ret = iter->trace->print_line(iter);
  1589. if (ret != TRACE_TYPE_UNHANDLED)
  1590. return ret;
  1591. }
  1592. if (iter->ent->type == TRACE_BPRINT &&
  1593. trace_flags & TRACE_ITER_PRINTK &&
  1594. trace_flags & TRACE_ITER_PRINTK_MSGONLY)
  1595. return trace_print_bprintk_msg_only(iter);
  1596. if (iter->ent->type == TRACE_PRINT &&
  1597. trace_flags & TRACE_ITER_PRINTK &&
  1598. trace_flags & TRACE_ITER_PRINTK_MSGONLY)
  1599. return trace_print_printk_msg_only(iter);
  1600. if (trace_flags & TRACE_ITER_BIN)
  1601. return print_bin_fmt(iter);
  1602. if (trace_flags & TRACE_ITER_HEX)
  1603. return print_hex_fmt(iter);
  1604. if (trace_flags & TRACE_ITER_RAW)
  1605. return print_raw_fmt(iter);
  1606. return print_trace_fmt(iter);
  1607. }
  1608. static int s_show(struct seq_file *m, void *v)
  1609. {
  1610. struct trace_iterator *iter = v;
  1611. int ret;
  1612. if (iter->ent == NULL) {
  1613. if (iter->tr) {
  1614. seq_printf(m, "# tracer: %s\n", iter->trace->name);
  1615. seq_puts(m, "#\n");
  1616. }
  1617. if (iter->trace && iter->trace->print_header)
  1618. iter->trace->print_header(m);
  1619. else if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
  1620. /* print nothing if the buffers are empty */
  1621. if (trace_empty(iter))
  1622. return 0;
  1623. print_trace_header(m, iter);
  1624. if (!(trace_flags & TRACE_ITER_VERBOSE))
  1625. print_lat_help_header(m);
  1626. } else {
  1627. if (!(trace_flags & TRACE_ITER_VERBOSE))
  1628. print_func_help_header(m);
  1629. }
  1630. } else if (iter->leftover) {
  1631. /*
  1632. * If we filled the seq_file buffer earlier, we
  1633. * want to just show it now.
  1634. */
  1635. ret = trace_print_seq(m, &iter->seq);
  1636. /* ret should this time be zero, but you never know */
  1637. iter->leftover = ret;
  1638. } else {
  1639. print_trace_line(iter);
  1640. ret = trace_print_seq(m, &iter->seq);
  1641. /*
  1642. * If we overflow the seq_file buffer, then it will
  1643. * ask us for this data again at start up.
  1644. * Use that instead.
  1645. * ret is 0 if seq_file write succeeded.
  1646. * -1 otherwise.
  1647. */
  1648. iter->leftover = ret;
  1649. }
  1650. return 0;
  1651. }
  1652. static const struct seq_operations tracer_seq_ops = {
  1653. .start = s_start,
  1654. .next = s_next,
  1655. .stop = s_stop,
  1656. .show = s_show,
  1657. };
  1658. static struct trace_iterator *
  1659. __tracing_open(struct inode *inode, struct file *file)
  1660. {
  1661. long cpu_file = (long) inode->i_private;
  1662. void *fail_ret = ERR_PTR(-ENOMEM);
  1663. struct trace_iterator *iter;
  1664. struct seq_file *m;
  1665. int cpu, ret;
  1666. if (tracing_disabled)
  1667. return ERR_PTR(-ENODEV);
  1668. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  1669. if (!iter)
  1670. return ERR_PTR(-ENOMEM);
  1671. /*
  1672. * We make a copy of the current tracer to avoid concurrent
  1673. * changes on it while we are reading.
  1674. */
  1675. mutex_lock(&trace_types_lock);
  1676. iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
  1677. if (!iter->trace)
  1678. goto fail;
  1679. if (current_trace)
  1680. *iter->trace = *current_trace;
  1681. if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
  1682. goto fail;
  1683. if (current_trace && current_trace->print_max)
  1684. iter->tr = &max_tr;
  1685. else
  1686. iter->tr = &global_trace;
  1687. iter->pos = -1;
  1688. mutex_init(&iter->mutex);
  1689. iter->cpu_file = cpu_file;
  1690. /* Notify the tracer early; before we stop tracing. */
  1691. if (iter->trace && iter->trace->open)
  1692. iter->trace->open(iter);
  1693. /* Annotate start of buffers if we had overruns */
  1694. if (ring_buffer_overruns(iter->tr->buffer))
  1695. iter->iter_flags |= TRACE_FILE_ANNOTATE;
  1696. /* stop the trace while dumping */
  1697. tracing_stop();
  1698. if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
  1699. for_each_tracing_cpu(cpu) {
  1700. iter->buffer_iter[cpu] =
  1701. ring_buffer_read_start(iter->tr->buffer, cpu);
  1702. tracing_iter_reset(iter, cpu);
  1703. }
  1704. } else {
  1705. cpu = iter->cpu_file;
  1706. iter->buffer_iter[cpu] =
  1707. ring_buffer_read_start(iter->tr->buffer, cpu);
  1708. tracing_iter_reset(iter, cpu);
  1709. }
  1710. ret = seq_open(file, &tracer_seq_ops);
  1711. if (ret < 0) {
  1712. fail_ret = ERR_PTR(ret);
  1713. goto fail_buffer;
  1714. }
  1715. m = file->private_data;
  1716. m->private = iter;
  1717. mutex_unlock(&trace_types_lock);
  1718. return iter;
  1719. fail_buffer:
  1720. for_each_tracing_cpu(cpu) {
  1721. if (iter->buffer_iter[cpu])
  1722. ring_buffer_read_finish(iter->buffer_iter[cpu]);
  1723. }
  1724. free_cpumask_var(iter->started);
  1725. tracing_start();
  1726. fail:
  1727. mutex_unlock(&trace_types_lock);
  1728. kfree(iter->trace);
  1729. kfree(iter);
  1730. return fail_ret;
  1731. }
  1732. int tracing_open_generic(struct inode *inode, struct file *filp)
  1733. {
  1734. if (tracing_disabled)
  1735. return -ENODEV;
  1736. filp->private_data = inode->i_private;
  1737. return 0;
  1738. }
  1739. static int tracing_release(struct inode *inode, struct file *file)
  1740. {
  1741. struct seq_file *m = (struct seq_file *)file->private_data;
  1742. struct trace_iterator *iter;
  1743. int cpu;
  1744. if (!(file->f_mode & FMODE_READ))
  1745. return 0;
  1746. iter = m->private;
  1747. mutex_lock(&trace_types_lock);
  1748. for_each_tracing_cpu(cpu) {
  1749. if (iter->buffer_iter[cpu])
  1750. ring_buffer_read_finish(iter->buffer_iter[cpu]);
  1751. }
  1752. if (iter->trace && iter->trace->close)
  1753. iter->trace->close(iter);
  1754. /* reenable tracing if it was previously enabled */
  1755. tracing_start();
  1756. mutex_unlock(&trace_types_lock);
  1757. seq_release(inode, file);
  1758. mutex_destroy(&iter->mutex);
  1759. free_cpumask_var(iter->started);
  1760. kfree(iter->trace);
  1761. kfree(iter);
  1762. return 0;
  1763. }
  1764. static int tracing_open(struct inode *inode, struct file *file)
  1765. {
  1766. struct trace_iterator *iter;
  1767. int ret = 0;
  1768. /* If this file was open for write, then erase contents */
  1769. if ((file->f_mode & FMODE_WRITE) &&
  1770. (file->f_flags & O_TRUNC)) {
  1771. long cpu = (long) inode->i_private;
  1772. if (cpu == TRACE_PIPE_ALL_CPU)
  1773. tracing_reset_online_cpus(&global_trace);
  1774. else
  1775. tracing_reset(&global_trace, cpu);
  1776. }
  1777. if (file->f_mode & FMODE_READ) {
  1778. iter = __tracing_open(inode, file);
  1779. if (IS_ERR(iter))
  1780. ret = PTR_ERR(iter);
  1781. else if (trace_flags & TRACE_ITER_LATENCY_FMT)
  1782. iter->iter_flags |= TRACE_FILE_LAT_FMT;
  1783. }
  1784. return ret;
  1785. }
  1786. static void *
  1787. t_next(struct seq_file *m, void *v, loff_t *pos)
  1788. {
  1789. struct tracer *t = v;
  1790. (*pos)++;
  1791. if (t)
  1792. t = t->next;
  1793. return t;
  1794. }
  1795. static void *t_start(struct seq_file *m, loff_t *pos)
  1796. {
  1797. struct tracer *t;
  1798. loff_t l = 0;
  1799. mutex_lock(&trace_types_lock);
  1800. for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
  1801. ;
  1802. return t;
  1803. }
  1804. static void t_stop(struct seq_file *m, void *p)
  1805. {
  1806. mutex_unlock(&trace_types_lock);
  1807. }
  1808. static int t_show(struct seq_file *m, void *v)
  1809. {
  1810. struct tracer *t = v;
  1811. if (!t)
  1812. return 0;
  1813. seq_printf(m, "%s", t->name);
  1814. if (t->next)
  1815. seq_putc(m, ' ');
  1816. else
  1817. seq_putc(m, '\n');
  1818. return 0;
  1819. }
  1820. static const struct seq_operations show_traces_seq_ops = {
  1821. .start = t_start,
  1822. .next = t_next,
  1823. .stop = t_stop,
  1824. .show = t_show,
  1825. };
  1826. static int show_traces_open(struct inode *inode, struct file *file)
  1827. {
  1828. if (tracing_disabled)
  1829. return -ENODEV;
  1830. return seq_open(file, &show_traces_seq_ops);
  1831. }
  1832. static ssize_t
  1833. tracing_write_stub(struct file *filp, const char __user *ubuf,
  1834. size_t count, loff_t *ppos)
  1835. {
  1836. return count;
  1837. }
  1838. static const struct file_operations tracing_fops = {
  1839. .open = tracing_open,
  1840. .read = seq_read,
  1841. .write = tracing_write_stub,
  1842. .llseek = seq_lseek,
  1843. .release = tracing_release,
  1844. };
  1845. static const struct file_operations show_traces_fops = {
  1846. .open = show_traces_open,
  1847. .read = seq_read,
  1848. .release = seq_release,
  1849. };
  1850. /*
  1851. * Only trace on a CPU if the bitmask is set:
  1852. */
  1853. static cpumask_var_t tracing_cpumask;
  1854. /*
  1855. * The tracer itself will not take this lock, but still we want
  1856. * to provide a consistent cpumask to user-space:
  1857. */
  1858. static DEFINE_MUTEX(tracing_cpumask_update_lock);
  1859. /*
  1860. * Temporary storage for the character representation of the
  1861. * CPU bitmask (and one more byte for the newline):
  1862. */
  1863. static char mask_str[NR_CPUS + 1];
  1864. static ssize_t
  1865. tracing_cpumask_read(struct file *filp, char __user *ubuf,
  1866. size_t count, loff_t *ppos)
  1867. {
  1868. int len;
  1869. mutex_lock(&tracing_cpumask_update_lock);
  1870. len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
  1871. if (count - len < 2) {
  1872. count = -EINVAL;
  1873. goto out_err;
  1874. }
  1875. len += sprintf(mask_str + len, "\n");
  1876. count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
  1877. out_err:
  1878. mutex_unlock(&tracing_cpumask_update_lock);
  1879. return count;
  1880. }
  1881. static ssize_t
  1882. tracing_cpumask_write(struct file *filp, const char __user *ubuf,
  1883. size_t count, loff_t *ppos)
  1884. {
  1885. int err, cpu;
  1886. cpumask_var_t tracing_cpumask_new;
  1887. if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
  1888. return -ENOMEM;
  1889. err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
  1890. if (err)
  1891. goto err_unlock;
  1892. mutex_lock(&tracing_cpumask_update_lock);
  1893. local_irq_disable();
  1894. arch_spin_lock(&ftrace_max_lock);
  1895. for_each_tracing_cpu(cpu) {
  1896. /*
  1897. * Increase/decrease the disabled counter if we are
  1898. * about to flip a bit in the cpumask:
  1899. */
  1900. if (cpumask_test_cpu(cpu, tracing_cpumask) &&
  1901. !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
  1902. atomic_inc(&global_trace.data[cpu]->disabled);
  1903. }
  1904. if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
  1905. cpumask_test_cpu(cpu, tracing_cpumask_new)) {
  1906. atomic_dec(&global_trace.data[cpu]->disabled);
  1907. }
  1908. }
  1909. arch_spin_unlock(&ftrace_max_lock);
  1910. local_irq_enable();
  1911. cpumask_copy(tracing_cpumask, tracing_cpumask_new);
  1912. mutex_unlock(&tracing_cpumask_update_lock);
  1913. free_cpumask_var(tracing_cpumask_new);
  1914. return count;
  1915. err_unlock:
  1916. free_cpumask_var(tracing_cpumask_new);
  1917. return err;
  1918. }
  1919. static const struct file_operations tracing_cpumask_fops = {
  1920. .open = tracing_open_generic,
  1921. .read = tracing_cpumask_read,
  1922. .write = tracing_cpumask_write,
  1923. };
  1924. static int tracing_trace_options_show(struct seq_file *m, void *v)
  1925. {
  1926. struct tracer_opt *trace_opts;
  1927. u32 tracer_flags;
  1928. int i;
  1929. mutex_lock(&trace_types_lock);
  1930. tracer_flags = current_trace->flags->val;
  1931. trace_opts = current_trace->flags->opts;
  1932. for (i = 0; trace_options[i]; i++) {
  1933. if (trace_flags & (1 << i))
  1934. seq_printf(m, "%s\n", trace_options[i]);
  1935. else
  1936. seq_printf(m, "no%s\n", trace_options[i]);
  1937. }
  1938. for (i = 0; trace_opts[i].name; i++) {
  1939. if (tracer_flags & trace_opts[i].bit)
  1940. seq_printf(m, "%s\n", trace_opts[i].name);
  1941. else
  1942. seq_printf(m, "no%s\n", trace_opts[i].name);
  1943. }
  1944. mutex_unlock(&trace_types_lock);
  1945. return 0;
  1946. }
  1947. static int __set_tracer_option(struct tracer *trace,
  1948. struct tracer_flags *tracer_flags,
  1949. struct tracer_opt *opts, int neg)
  1950. {
  1951. int ret;
  1952. ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
  1953. if (ret)
  1954. return ret;
  1955. if (neg)
  1956. tracer_flags->val &= ~opts->bit;
  1957. else
  1958. tracer_flags->val |= opts->bit;
  1959. return 0;
  1960. }
  1961. /* Try to assign a tracer specific option */
  1962. static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
  1963. {
  1964. struct tracer_flags *tracer_flags = trace->flags;
  1965. struct tracer_opt *opts = NULL;
  1966. int i;
  1967. for (i = 0; tracer_flags->opts[i].name; i++) {
  1968. opts = &tracer_flags->opts[i];
  1969. if (strcmp(cmp, opts->name) == 0)
  1970. return __set_tracer_option(trace, trace->flags,
  1971. opts, neg);
  1972. }
  1973. return -EINVAL;
  1974. }
  1975. static void set_tracer_flags(unsigned int mask, int enabled)
  1976. {
  1977. /* do nothing if flag is already set */
  1978. if (!!(trace_flags & mask) == !!enabled)
  1979. return;
  1980. if (enabled)
  1981. trace_flags |= mask;
  1982. else
  1983. trace_flags &= ~mask;
  1984. }
  1985. static ssize_t
  1986. tracing_trace_options_write(struct file *filp, const char __user *ubuf,
  1987. size_t cnt, loff_t *ppos)
  1988. {
  1989. char buf[64];
  1990. char *cmp;
  1991. int neg = 0;
  1992. int ret;
  1993. int i;
  1994. if (cnt >= sizeof(buf))
  1995. return -EINVAL;
  1996. if (copy_from_user(&buf, ubuf, cnt))
  1997. return -EFAULT;
  1998. buf[cnt] = 0;
  1999. cmp = strstrip(buf);
  2000. if (strncmp(cmp, "no", 2) == 0) {
  2001. neg = 1;
  2002. cmp += 2;
  2003. }
  2004. for (i = 0; trace_options[i]; i++) {
  2005. if (strcmp(cmp, trace_options[i]) == 0) {
  2006. set_tracer_flags(1 << i, !neg);
  2007. break;
  2008. }
  2009. }
  2010. /* If no option could be set, test the specific tracer options */
  2011. if (!trace_options[i]) {
  2012. mutex_lock(&trace_types_lock);
  2013. ret = set_tracer_option(current_trace, cmp, neg);
  2014. mutex_unlock(&trace_types_lock);
  2015. if (ret)
  2016. return ret;
  2017. }
  2018. *ppos += cnt;
  2019. return cnt;
  2020. }
  2021. static int tracing_trace_options_open(struct inode *inode, struct file *file)
  2022. {
  2023. if (tracing_disabled)
  2024. return -ENODEV;
  2025. return single_open(file, tracing_trace_options_show, NULL);
  2026. }
  2027. static const struct file_operations tracing_iter_fops = {
  2028. .open = tracing_trace_options_open,
  2029. .read = seq_read,
  2030. .llseek = seq_lseek,
  2031. .release = single_release,
  2032. .write = tracing_trace_options_write,
  2033. };
  2034. static const char readme_msg[] =
  2035. "tracing mini-HOWTO:\n\n"
  2036. "# mount -t debugfs nodev /sys/kernel/debug\n\n"
  2037. "# cat /sys/kernel/debug/tracing/available_tracers\n"
  2038. "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
  2039. "# cat /sys/kernel/debug/tracing/current_tracer\n"
  2040. "nop\n"
  2041. "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
  2042. "# cat /sys/kernel/debug/tracing/current_tracer\n"
  2043. "sched_switch\n"
  2044. "# cat /sys/kernel/debug/tracing/trace_options\n"
  2045. "noprint-parent nosym-offset nosym-addr noverbose\n"
  2046. "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
  2047. "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
  2048. "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
  2049. "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
  2050. ;
  2051. static ssize_t
  2052. tracing_readme_read(struct file *filp, char __user *ubuf,
  2053. size_t cnt, loff_t *ppos)
  2054. {
  2055. return simple_read_from_buffer(ubuf, cnt, ppos,
  2056. readme_msg, strlen(readme_msg));
  2057. }
  2058. static const struct file_operations tracing_readme_fops = {
  2059. .open = tracing_open_generic,
  2060. .read = tracing_readme_read,
  2061. };
  2062. static ssize_t
  2063. tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
  2064. size_t cnt, loff_t *ppos)
  2065. {
  2066. char *buf_comm;
  2067. char *file_buf;
  2068. char *buf;
  2069. int len = 0;
  2070. int pid;
  2071. int i;
  2072. file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
  2073. if (!file_buf)
  2074. return -ENOMEM;
  2075. buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
  2076. if (!buf_comm) {
  2077. kfree(file_buf);
  2078. return -ENOMEM;
  2079. }
  2080. buf = file_buf;
  2081. for (i = 0; i < SAVED_CMDLINES; i++) {
  2082. int r;
  2083. pid = map_cmdline_to_pid[i];
  2084. if (pid == -1 || pid == NO_CMDLINE_MAP)
  2085. continue;
  2086. trace_find_cmdline(pid, buf_comm);
  2087. r = sprintf(buf, "%d %s\n", pid, buf_comm);
  2088. buf += r;
  2089. len += r;
  2090. }
  2091. len = simple_read_from_buffer(ubuf, cnt, ppos,
  2092. file_buf, len);
  2093. kfree(file_buf);
  2094. kfree(buf_comm);
  2095. return len;
  2096. }
  2097. static const struct file_operations tracing_saved_cmdlines_fops = {
  2098. .open = tracing_open_generic,
  2099. .read = tracing_saved_cmdlines_read,
  2100. };
  2101. static ssize_t
  2102. tracing_ctrl_read(struct file *filp, char __user *ubuf,
  2103. size_t cnt, loff_t *ppos)
  2104. {
  2105. char buf[64];
  2106. int r;
  2107. r = sprintf(buf, "%u\n", tracer_enabled);
  2108. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2109. }
  2110. static ssize_t
  2111. tracing_ctrl_write(struct file *filp, const char __user *ubuf,
  2112. size_t cnt, loff_t *ppos)
  2113. {
  2114. struct trace_array *tr = filp->private_data;
  2115. char buf[64];
  2116. unsigned long val;
  2117. int ret;
  2118. if (cnt >= sizeof(buf))
  2119. return -EINVAL;
  2120. if (copy_from_user(&buf, ubuf, cnt))
  2121. return -EFAULT;
  2122. buf[cnt] = 0;
  2123. ret = strict_strtoul(buf, 10, &val);
  2124. if (ret < 0)
  2125. return ret;
  2126. val = !!val;
  2127. mutex_lock(&trace_types_lock);
  2128. if (tracer_enabled ^ val) {
  2129. if (val) {
  2130. tracer_enabled = 1;
  2131. if (current_trace->start)
  2132. current_trace->start(tr);
  2133. tracing_start();
  2134. } else {
  2135. tracer_enabled = 0;
  2136. tracing_stop();
  2137. if (current_trace->stop)
  2138. current_trace->stop(tr);
  2139. }
  2140. }
  2141. mutex_unlock(&trace_types_lock);
  2142. *ppos += cnt;
  2143. return cnt;
  2144. }
  2145. static ssize_t
  2146. tracing_set_trace_read(struct file *filp, char __user *ubuf,
  2147. size_t cnt, loff_t *ppos)
  2148. {
  2149. char buf[MAX_TRACER_SIZE+2];
  2150. int r;
  2151. mutex_lock(&trace_types_lock);
  2152. if (current_trace)
  2153. r = sprintf(buf, "%s\n", current_trace->name);
  2154. else
  2155. r = sprintf(buf, "\n");
  2156. mutex_unlock(&trace_types_lock);
  2157. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2158. }
  2159. int tracer_init(struct tracer *t, struct trace_array *tr)
  2160. {
  2161. tracing_reset_online_cpus(tr);
  2162. return t->init(tr);
  2163. }
  2164. static int tracing_resize_ring_buffer(unsigned long size)
  2165. {
  2166. int ret;
  2167. /*
  2168. * If kernel or user changes the size of the ring buffer
  2169. * we use the size that was given, and we can forget about
  2170. * expanding it later.
  2171. */
  2172. ring_buffer_expanded = 1;
  2173. ret = ring_buffer_resize(global_trace.buffer, size);
  2174. if (ret < 0)
  2175. return ret;
  2176. ret = ring_buffer_resize(max_tr.buffer, size);
  2177. if (ret < 0) {
  2178. int r;
  2179. r = ring_buffer_resize(global_trace.buffer,
  2180. global_trace.entries);
  2181. if (r < 0) {
  2182. /*
  2183. * AARGH! We are left with different
  2184. * size max buffer!!!!
  2185. * The max buffer is our "snapshot" buffer.
  2186. * When a tracer needs a snapshot (one of the
  2187. * latency tracers), it swaps the max buffer
  2188. * with the saved snap shot. We succeeded to
  2189. * update the size of the main buffer, but failed to
  2190. * update the size of the max buffer. But when we tried
  2191. * to reset the main buffer to the original size, we
  2192. * failed there too. This is very unlikely to
  2193. * happen, but if it does, warn and kill all
  2194. * tracing.
  2195. */
  2196. WARN_ON(1);
  2197. tracing_disabled = 1;
  2198. }
  2199. return ret;
  2200. }
  2201. global_trace.entries = size;
  2202. return ret;
  2203. }
  2204. /**
  2205. * tracing_update_buffers - used by tracing facility to expand ring buffers
  2206. *
  2207. * To save on memory when the tracing is never used on a system with it
  2208. * configured in. The ring buffers are set to a minimum size. But once
  2209. * a user starts to use the tracing facility, then they need to grow
  2210. * to their default size.
  2211. *
  2212. * This function is to be called when a tracer is about to be used.
  2213. */
  2214. int tracing_update_buffers(void)
  2215. {
  2216. int ret = 0;
  2217. mutex_lock(&trace_types_lock);
  2218. if (!ring_buffer_expanded)
  2219. ret = tracing_resize_ring_buffer(trace_buf_size);
  2220. mutex_unlock(&trace_types_lock);
  2221. return ret;
  2222. }
  2223. struct trace_option_dentry;
  2224. static struct trace_option_dentry *
  2225. create_trace_option_files(struct tracer *tracer);
  2226. static void
  2227. destroy_trace_option_files(struct trace_option_dentry *topts);
  2228. static int tracing_set_tracer(const char *buf)
  2229. {
  2230. static struct trace_option_dentry *topts;
  2231. struct trace_array *tr = &global_trace;
  2232. struct tracer *t;
  2233. int ret = 0;
  2234. mutex_lock(&trace_types_lock);
  2235. if (!ring_buffer_expanded) {
  2236. ret = tracing_resize_ring_buffer(trace_buf_size);
  2237. if (ret < 0)
  2238. goto out;
  2239. ret = 0;
  2240. }
  2241. for (t = trace_types; t; t = t->next) {
  2242. if (strcmp(t->name, buf) == 0)
  2243. break;
  2244. }
  2245. if (!t) {
  2246. ret = -EINVAL;
  2247. goto out;
  2248. }
  2249. if (t == current_trace)
  2250. goto out;
  2251. trace_branch_disable();
  2252. if (current_trace && current_trace->reset)
  2253. current_trace->reset(tr);
  2254. destroy_trace_option_files(topts);
  2255. current_trace = t;
  2256. topts = create_trace_option_files(current_trace);
  2257. if (t->init) {
  2258. ret = tracer_init(t, tr);
  2259. if (ret)
  2260. goto out;
  2261. }
  2262. trace_branch_enable(tr);
  2263. out:
  2264. mutex_unlock(&trace_types_lock);
  2265. return ret;
  2266. }
  2267. static ssize_t
  2268. tracing_set_trace_write(struct file *filp, const char __user *ubuf,
  2269. size_t cnt, loff_t *ppos)
  2270. {
  2271. char buf[MAX_TRACER_SIZE+1];
  2272. int i;
  2273. size_t ret;
  2274. int err;
  2275. ret = cnt;
  2276. if (cnt > MAX_TRACER_SIZE)
  2277. cnt = MAX_TRACER_SIZE;
  2278. if (copy_from_user(&buf, ubuf, cnt))
  2279. return -EFAULT;
  2280. buf[cnt] = 0;
  2281. /* strip ending whitespace. */
  2282. for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
  2283. buf[i] = 0;
  2284. err = tracing_set_tracer(buf);
  2285. if (err)
  2286. return err;
  2287. *ppos += ret;
  2288. return ret;
  2289. }
  2290. static ssize_t
  2291. tracing_max_lat_read(struct file *filp, char __user *ubuf,
  2292. size_t cnt, loff_t *ppos)
  2293. {
  2294. unsigned long *ptr = filp->private_data;
  2295. char buf[64];
  2296. int r;
  2297. r = snprintf(buf, sizeof(buf), "%ld\n",
  2298. *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
  2299. if (r > sizeof(buf))
  2300. r = sizeof(buf);
  2301. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2302. }
  2303. static ssize_t
  2304. tracing_max_lat_write(struct file *filp, const char __user *ubuf,
  2305. size_t cnt, loff_t *ppos)
  2306. {
  2307. unsigned long *ptr = filp->private_data;
  2308. char buf[64];
  2309. unsigned long val;
  2310. int ret;
  2311. if (cnt >= sizeof(buf))
  2312. return -EINVAL;
  2313. if (copy_from_user(&buf, ubuf, cnt))
  2314. return -EFAULT;
  2315. buf[cnt] = 0;
  2316. ret = strict_strtoul(buf, 10, &val);
  2317. if (ret < 0)
  2318. return ret;
  2319. *ptr = val * 1000;
  2320. return cnt;
  2321. }
  2322. static int tracing_open_pipe(struct inode *inode, struct file *filp)
  2323. {
  2324. long cpu_file = (long) inode->i_private;
  2325. struct trace_iterator *iter;
  2326. int ret = 0;
  2327. if (tracing_disabled)
  2328. return -ENODEV;
  2329. mutex_lock(&trace_types_lock);
  2330. /* We only allow one reader per cpu */
  2331. if (cpu_file == TRACE_PIPE_ALL_CPU) {
  2332. if (!cpumask_empty(tracing_reader_cpumask)) {
  2333. ret = -EBUSY;
  2334. goto out;
  2335. }
  2336. cpumask_setall(tracing_reader_cpumask);
  2337. } else {
  2338. if (!cpumask_test_cpu(cpu_file, tracing_reader_cpumask))
  2339. cpumask_set_cpu(cpu_file, tracing_reader_cpumask);
  2340. else {
  2341. ret = -EBUSY;
  2342. goto out;
  2343. }
  2344. }
  2345. /* create a buffer to store the information to pass to userspace */
  2346. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  2347. if (!iter) {
  2348. ret = -ENOMEM;
  2349. goto out;
  2350. }
  2351. /*
  2352. * We make a copy of the current tracer to avoid concurrent
  2353. * changes on it while we are reading.
  2354. */
  2355. iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
  2356. if (!iter->trace) {
  2357. ret = -ENOMEM;
  2358. goto fail;
  2359. }
  2360. if (current_trace)
  2361. *iter->trace = *current_trace;
  2362. if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
  2363. ret = -ENOMEM;
  2364. goto fail;
  2365. }
  2366. /* trace pipe does not show start of buffer */
  2367. cpumask_setall(iter->started);
  2368. if (trace_flags & TRACE_ITER_LATENCY_FMT)
  2369. iter->iter_flags |= TRACE_FILE_LAT_FMT;
  2370. iter->cpu_file = cpu_file;
  2371. iter->tr = &global_trace;
  2372. mutex_init(&iter->mutex);
  2373. filp->private_data = iter;
  2374. if (iter->trace->pipe_open)
  2375. iter->trace->pipe_open(iter);
  2376. out:
  2377. mutex_unlock(&trace_types_lock);
  2378. return ret;
  2379. fail:
  2380. kfree(iter->trace);
  2381. kfree(iter);
  2382. mutex_unlock(&trace_types_lock);
  2383. return ret;
  2384. }
  2385. static int tracing_release_pipe(struct inode *inode, struct file *file)
  2386. {
  2387. struct trace_iterator *iter = file->private_data;
  2388. mutex_lock(&trace_types_lock);
  2389. if (iter->cpu_file == TRACE_PIPE_ALL_CPU)
  2390. cpumask_clear(tracing_reader_cpumask);
  2391. else
  2392. cpumask_clear_cpu(iter->cpu_file, tracing_reader_cpumask);
  2393. if (iter->trace->pipe_close)
  2394. iter->trace->pipe_close(iter);
  2395. mutex_unlock(&trace_types_lock);
  2396. free_cpumask_var(iter->started);
  2397. mutex_destroy(&iter->mutex);
  2398. kfree(iter->trace);
  2399. kfree(iter);
  2400. return 0;
  2401. }
  2402. static unsigned int
  2403. tracing_poll_pipe(struct file *filp, poll_table *poll_table)
  2404. {
  2405. struct trace_iterator *iter = filp->private_data;
  2406. if (trace_flags & TRACE_ITER_BLOCK) {
  2407. /*
  2408. * Always select as readable when in blocking mode
  2409. */
  2410. return POLLIN | POLLRDNORM;
  2411. } else {
  2412. if (!trace_empty(iter))
  2413. return POLLIN | POLLRDNORM;
  2414. poll_wait(filp, &trace_wait, poll_table);
  2415. if (!trace_empty(iter))
  2416. return POLLIN | POLLRDNORM;
  2417. return 0;
  2418. }
  2419. }
  2420. void default_wait_pipe(struct trace_iterator *iter)
  2421. {
  2422. DEFINE_WAIT(wait);
  2423. prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
  2424. if (trace_empty(iter))
  2425. schedule();
  2426. finish_wait(&trace_wait, &wait);
  2427. }
  2428. /*
  2429. * This is a make-shift waitqueue.
  2430. * A tracer might use this callback on some rare cases:
  2431. *
  2432. * 1) the current tracer might hold the runqueue lock when it wakes up
  2433. * a reader, hence a deadlock (sched, function, and function graph tracers)
  2434. * 2) the function tracers, trace all functions, we don't want
  2435. * the overhead of calling wake_up and friends
  2436. * (and tracing them too)
  2437. *
  2438. * Anyway, this is really very primitive wakeup.
  2439. */
  2440. void poll_wait_pipe(struct trace_iterator *iter)
  2441. {
  2442. set_current_state(TASK_INTERRUPTIBLE);
  2443. /* sleep for 100 msecs, and try again. */
  2444. schedule_timeout(HZ / 10);
  2445. }
  2446. /* Must be called with trace_types_lock mutex held. */
  2447. static int tracing_wait_pipe(struct file *filp)
  2448. {
  2449. struct trace_iterator *iter = filp->private_data;
  2450. while (trace_empty(iter)) {
  2451. if ((filp->f_flags & O_NONBLOCK)) {
  2452. return -EAGAIN;
  2453. }
  2454. mutex_unlock(&iter->mutex);
  2455. iter->trace->wait_pipe(iter);
  2456. mutex_lock(&iter->mutex);
  2457. if (signal_pending(current))
  2458. return -EINTR;
  2459. /*
  2460. * We block until we read something and tracing is disabled.
  2461. * We still block if tracing is disabled, but we have never
  2462. * read anything. This allows a user to cat this file, and
  2463. * then enable tracing. But after we have read something,
  2464. * we give an EOF when tracing is again disabled.
  2465. *
  2466. * iter->pos will be 0 if we haven't read anything.
  2467. */
  2468. if (!tracer_enabled && iter->pos)
  2469. break;
  2470. }
  2471. return 1;
  2472. }
  2473. /*
  2474. * Consumer reader.
  2475. */
  2476. static ssize_t
  2477. tracing_read_pipe(struct file *filp, char __user *ubuf,
  2478. size_t cnt, loff_t *ppos)
  2479. {
  2480. struct trace_iterator *iter = filp->private_data;
  2481. static struct tracer *old_tracer;
  2482. ssize_t sret;
  2483. /* return any leftover data */
  2484. sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
  2485. if (sret != -EBUSY)
  2486. return sret;
  2487. trace_seq_init(&iter->seq);
  2488. /* copy the tracer to avoid using a global lock all around */
  2489. mutex_lock(&trace_types_lock);
  2490. if (unlikely(old_tracer != current_trace && current_trace)) {
  2491. old_tracer = current_trace;
  2492. *iter->trace = *current_trace;
  2493. }
  2494. mutex_unlock(&trace_types_lock);
  2495. /*
  2496. * Avoid more than one consumer on a single file descriptor
  2497. * This is just a matter of traces coherency, the ring buffer itself
  2498. * is protected.
  2499. */
  2500. mutex_lock(&iter->mutex);
  2501. if (iter->trace->read) {
  2502. sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
  2503. if (sret)
  2504. goto out;
  2505. }
  2506. waitagain:
  2507. sret = tracing_wait_pipe(filp);
  2508. if (sret <= 0)
  2509. goto out;
  2510. /* stop when tracing is finished */
  2511. if (trace_empty(iter)) {
  2512. sret = 0;
  2513. goto out;
  2514. }
  2515. if (cnt >= PAGE_SIZE)
  2516. cnt = PAGE_SIZE - 1;
  2517. /* reset all but tr, trace, and overruns */
  2518. memset(&iter->seq, 0,
  2519. sizeof(struct trace_iterator) -
  2520. offsetof(struct trace_iterator, seq));
  2521. iter->pos = -1;
  2522. trace_event_read_lock();
  2523. while (find_next_entry_inc(iter) != NULL) {
  2524. enum print_line_t ret;
  2525. int len = iter->seq.len;
  2526. ret = print_trace_line(iter);
  2527. if (ret == TRACE_TYPE_PARTIAL_LINE) {
  2528. /* don't print partial lines */
  2529. iter->seq.len = len;
  2530. break;
  2531. }
  2532. if (ret != TRACE_TYPE_NO_CONSUME)
  2533. trace_consume(iter);
  2534. if (iter->seq.len >= cnt)
  2535. break;
  2536. }
  2537. trace_event_read_unlock();
  2538. /* Now copy what we have to the user */
  2539. sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
  2540. if (iter->seq.readpos >= iter->seq.len)
  2541. trace_seq_init(&iter->seq);
  2542. /*
  2543. * If there was nothing to send to user, inspite of consuming trace
  2544. * entries, go back to wait for more entries.
  2545. */
  2546. if (sret == -EBUSY)
  2547. goto waitagain;
  2548. out:
  2549. mutex_unlock(&iter->mutex);
  2550. return sret;
  2551. }
  2552. static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
  2553. struct pipe_buffer *buf)
  2554. {
  2555. __free_page(buf->page);
  2556. }
  2557. static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
  2558. unsigned int idx)
  2559. {
  2560. __free_page(spd->pages[idx]);
  2561. }
  2562. static const struct pipe_buf_operations tracing_pipe_buf_ops = {
  2563. .can_merge = 0,
  2564. .map = generic_pipe_buf_map,
  2565. .unmap = generic_pipe_buf_unmap,
  2566. .confirm = generic_pipe_buf_confirm,
  2567. .release = tracing_pipe_buf_release,
  2568. .steal = generic_pipe_buf_steal,
  2569. .get = generic_pipe_buf_get,
  2570. };
  2571. static size_t
  2572. tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
  2573. {
  2574. size_t count;
  2575. int ret;
  2576. /* Seq buffer is page-sized, exactly what we need. */
  2577. for (;;) {
  2578. count = iter->seq.len;
  2579. ret = print_trace_line(iter);
  2580. count = iter->seq.len - count;
  2581. if (rem < count) {
  2582. rem = 0;
  2583. iter->seq.len -= count;
  2584. break;
  2585. }
  2586. if (ret == TRACE_TYPE_PARTIAL_LINE) {
  2587. iter->seq.len -= count;
  2588. break;
  2589. }
  2590. if (ret != TRACE_TYPE_NO_CONSUME)
  2591. trace_consume(iter);
  2592. rem -= count;
  2593. if (!find_next_entry_inc(iter)) {
  2594. rem = 0;
  2595. iter->ent = NULL;
  2596. break;
  2597. }
  2598. }
  2599. return rem;
  2600. }
  2601. static ssize_t tracing_splice_read_pipe(struct file *filp,
  2602. loff_t *ppos,
  2603. struct pipe_inode_info *pipe,
  2604. size_t len,
  2605. unsigned int flags)
  2606. {
  2607. struct page *pages[PIPE_BUFFERS];
  2608. struct partial_page partial[PIPE_BUFFERS];
  2609. struct trace_iterator *iter = filp->private_data;
  2610. struct splice_pipe_desc spd = {
  2611. .pages = pages,
  2612. .partial = partial,
  2613. .nr_pages = 0, /* This gets updated below. */
  2614. .flags = flags,
  2615. .ops = &tracing_pipe_buf_ops,
  2616. .spd_release = tracing_spd_release_pipe,
  2617. };
  2618. static struct tracer *old_tracer;
  2619. ssize_t ret;
  2620. size_t rem;
  2621. unsigned int i;
  2622. /* copy the tracer to avoid using a global lock all around */
  2623. mutex_lock(&trace_types_lock);
  2624. if (unlikely(old_tracer != current_trace && current_trace)) {
  2625. old_tracer = current_trace;
  2626. *iter->trace = *current_trace;
  2627. }
  2628. mutex_unlock(&trace_types_lock);
  2629. mutex_lock(&iter->mutex);
  2630. if (iter->trace->splice_read) {
  2631. ret = iter->trace->splice_read(iter, filp,
  2632. ppos, pipe, len, flags);
  2633. if (ret)
  2634. goto out_err;
  2635. }
  2636. ret = tracing_wait_pipe(filp);
  2637. if (ret <= 0)
  2638. goto out_err;
  2639. if (!iter->ent && !find_next_entry_inc(iter)) {
  2640. ret = -EFAULT;
  2641. goto out_err;
  2642. }
  2643. trace_event_read_lock();
  2644. /* Fill as many pages as possible. */
  2645. for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
  2646. pages[i] = alloc_page(GFP_KERNEL);
  2647. if (!pages[i])
  2648. break;
  2649. rem = tracing_fill_pipe_page(rem, iter);
  2650. /* Copy the data into the page, so we can start over. */
  2651. ret = trace_seq_to_buffer(&iter->seq,
  2652. page_address(pages[i]),
  2653. iter->seq.len);
  2654. if (ret < 0) {
  2655. __free_page(pages[i]);
  2656. break;
  2657. }
  2658. partial[i].offset = 0;
  2659. partial[i].len = iter->seq.len;
  2660. trace_seq_init(&iter->seq);
  2661. }
  2662. trace_event_read_unlock();
  2663. mutex_unlock(&iter->mutex);
  2664. spd.nr_pages = i;
  2665. return splice_to_pipe(pipe, &spd);
  2666. out_err:
  2667. mutex_unlock(&iter->mutex);
  2668. return ret;
  2669. }
  2670. static ssize_t
  2671. tracing_entries_read(struct file *filp, char __user *ubuf,
  2672. size_t cnt, loff_t *ppos)
  2673. {
  2674. struct trace_array *tr = filp->private_data;
  2675. char buf[96];
  2676. int r;
  2677. mutex_lock(&trace_types_lock);
  2678. if (!ring_buffer_expanded)
  2679. r = sprintf(buf, "%lu (expanded: %lu)\n",
  2680. tr->entries >> 10,
  2681. trace_buf_size >> 10);
  2682. else
  2683. r = sprintf(buf, "%lu\n", tr->entries >> 10);
  2684. mutex_unlock(&trace_types_lock);
  2685. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  2686. }
  2687. static ssize_t
  2688. tracing_entries_write(struct file *filp, const char __user *ubuf,
  2689. size_t cnt, loff_t *ppos)
  2690. {
  2691. unsigned long val;
  2692. char buf[64];
  2693. int ret, cpu;
  2694. if (cnt >= sizeof(buf))
  2695. return -EINVAL;
  2696. if (copy_from_user(&buf, ubuf, cnt))
  2697. return -EFAULT;
  2698. buf[cnt] = 0;
  2699. ret = strict_strtoul(buf, 10, &val);
  2700. if (ret < 0)
  2701. return ret;
  2702. /* must have at least 1 entry */
  2703. if (!val)
  2704. return -EINVAL;
  2705. mutex_lock(&trace_types_lock);
  2706. tracing_stop();
  2707. /* disable all cpu buffers */
  2708. for_each_tracing_cpu(cpu) {
  2709. if (global_trace.data[cpu])
  2710. atomic_inc(&global_trace.data[cpu]->disabled);
  2711. if (max_tr.data[cpu])
  2712. atomic_inc(&max_tr.data[cpu]->disabled);
  2713. }
  2714. /* value is in KB */
  2715. val <<= 10;
  2716. if (val != global_trace.entries) {
  2717. ret = tracing_resize_ring_buffer(val);
  2718. if (ret < 0) {
  2719. cnt = ret;
  2720. goto out;
  2721. }
  2722. }
  2723. *ppos += cnt;
  2724. /* If check pages failed, return ENOMEM */
  2725. if (tracing_disabled)
  2726. cnt = -ENOMEM;
  2727. out:
  2728. for_each_tracing_cpu(cpu) {
  2729. if (global_trace.data[cpu])
  2730. atomic_dec(&global_trace.data[cpu]->disabled);
  2731. if (max_tr.data[cpu])
  2732. atomic_dec(&max_tr.data[cpu]->disabled);
  2733. }
  2734. tracing_start();
  2735. max_tr.entries = global_trace.entries;
  2736. mutex_unlock(&trace_types_lock);
  2737. return cnt;
  2738. }
  2739. static int mark_printk(const char *fmt, ...)
  2740. {
  2741. int ret;
  2742. va_list args;
  2743. va_start(args, fmt);
  2744. ret = trace_vprintk(0, fmt, args);
  2745. va_end(args);
  2746. return ret;
  2747. }
  2748. static ssize_t
  2749. tracing_mark_write(struct file *filp, const char __user *ubuf,
  2750. size_t cnt, loff_t *fpos)
  2751. {
  2752. char *buf;
  2753. if (tracing_disabled)
  2754. return -EINVAL;
  2755. if (cnt > TRACE_BUF_SIZE)
  2756. cnt = TRACE_BUF_SIZE;
  2757. buf = kmalloc(cnt + 2, GFP_KERNEL);
  2758. if (buf == NULL)
  2759. return -ENOMEM;
  2760. if (copy_from_user(buf, ubuf, cnt)) {
  2761. kfree(buf);
  2762. return -EFAULT;
  2763. }
  2764. if (buf[cnt-1] != '\n') {
  2765. buf[cnt] = '\n';
  2766. buf[cnt+1] = '\0';
  2767. } else
  2768. buf[cnt] = '\0';
  2769. cnt = mark_printk("%s", buf);
  2770. kfree(buf);
  2771. *fpos += cnt;
  2772. return cnt;
  2773. }
  2774. static int tracing_clock_show(struct seq_file *m, void *v)
  2775. {
  2776. int i;
  2777. for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
  2778. seq_printf(m,
  2779. "%s%s%s%s", i ? " " : "",
  2780. i == trace_clock_id ? "[" : "", trace_clocks[i].name,
  2781. i == trace_clock_id ? "]" : "");
  2782. seq_putc(m, '\n');
  2783. return 0;
  2784. }
  2785. static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
  2786. size_t cnt, loff_t *fpos)
  2787. {
  2788. char buf[64];
  2789. const char *clockstr;
  2790. int i;
  2791. if (cnt >= sizeof(buf))
  2792. return -EINVAL;
  2793. if (copy_from_user(&buf, ubuf, cnt))
  2794. return -EFAULT;
  2795. buf[cnt] = 0;
  2796. clockstr = strstrip(buf);
  2797. for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
  2798. if (strcmp(trace_clocks[i].name, clockstr) == 0)
  2799. break;
  2800. }
  2801. if (i == ARRAY_SIZE(trace_clocks))
  2802. return -EINVAL;
  2803. trace_clock_id = i;
  2804. mutex_lock(&trace_types_lock);
  2805. ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
  2806. if (max_tr.buffer)
  2807. ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
  2808. mutex_unlock(&trace_types_lock);
  2809. *fpos += cnt;
  2810. return cnt;
  2811. }
  2812. static int tracing_clock_open(struct inode *inode, struct file *file)
  2813. {
  2814. if (tracing_disabled)
  2815. return -ENODEV;
  2816. return single_open(file, tracing_clock_show, NULL);
  2817. }
  2818. static const struct file_operations tracing_max_lat_fops = {
  2819. .open = tracing_open_generic,
  2820. .read = tracing_max_lat_read,
  2821. .write = tracing_max_lat_write,
  2822. };
  2823. static const struct file_operations tracing_ctrl_fops = {
  2824. .open = tracing_open_generic,
  2825. .read = tracing_ctrl_read,
  2826. .write = tracing_ctrl_write,
  2827. };
  2828. static const struct file_operations set_tracer_fops = {
  2829. .open = tracing_open_generic,
  2830. .read = tracing_set_trace_read,
  2831. .write = tracing_set_trace_write,
  2832. };
  2833. static const struct file_operations tracing_pipe_fops = {
  2834. .open = tracing_open_pipe,
  2835. .poll = tracing_poll_pipe,
  2836. .read = tracing_read_pipe,
  2837. .splice_read = tracing_splice_read_pipe,
  2838. .release = tracing_release_pipe,
  2839. };
  2840. static const struct file_operations tracing_entries_fops = {
  2841. .open = tracing_open_generic,
  2842. .read = tracing_entries_read,
  2843. .write = tracing_entries_write,
  2844. };
  2845. static const struct file_operations tracing_mark_fops = {
  2846. .open = tracing_open_generic,
  2847. .write = tracing_mark_write,
  2848. };
  2849. static const struct file_operations trace_clock_fops = {
  2850. .open = tracing_clock_open,
  2851. .read = seq_read,
  2852. .llseek = seq_lseek,
  2853. .release = single_release,
  2854. .write = tracing_clock_write,
  2855. };
  2856. struct ftrace_buffer_info {
  2857. struct trace_array *tr;
  2858. void *spare;
  2859. int cpu;
  2860. unsigned int read;
  2861. };
  2862. static int tracing_buffers_open(struct inode *inode, struct file *filp)
  2863. {
  2864. int cpu = (int)(long)inode->i_private;
  2865. struct ftrace_buffer_info *info;
  2866. if (tracing_disabled)
  2867. return -ENODEV;
  2868. info = kzalloc(sizeof(*info), GFP_KERNEL);
  2869. if (!info)
  2870. return -ENOMEM;
  2871. info->tr = &global_trace;
  2872. info->cpu = cpu;
  2873. info->spare = NULL;
  2874. /* Force reading ring buffer for first read */
  2875. info->read = (unsigned int)-1;
  2876. filp->private_data = info;
  2877. return nonseekable_open(inode, filp);
  2878. }
  2879. static ssize_t
  2880. tracing_buffers_read(struct file *filp, char __user *ubuf,
  2881. size_t count, loff_t *ppos)
  2882. {
  2883. struct ftrace_buffer_info *info = filp->private_data;
  2884. unsigned int pos;
  2885. ssize_t ret;
  2886. size_t size;
  2887. if (!count)
  2888. return 0;
  2889. if (!info->spare)
  2890. info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
  2891. if (!info->spare)
  2892. return -ENOMEM;
  2893. /* Do we have previous read data to read? */
  2894. if (info->read < PAGE_SIZE)
  2895. goto read;
  2896. info->read = 0;
  2897. ret = ring_buffer_read_page(info->tr->buffer,
  2898. &info->spare,
  2899. count,
  2900. info->cpu, 0);
  2901. if (ret < 0)
  2902. return 0;
  2903. pos = ring_buffer_page_len(info->spare);
  2904. if (pos < PAGE_SIZE)
  2905. memset(info->spare + pos, 0, PAGE_SIZE - pos);
  2906. read:
  2907. size = PAGE_SIZE - info->read;
  2908. if (size > count)
  2909. size = count;
  2910. ret = copy_to_user(ubuf, info->spare + info->read, size);
  2911. if (ret == size)
  2912. return -EFAULT;
  2913. size -= ret;
  2914. *ppos += size;
  2915. info->read += size;
  2916. return size;
  2917. }
  2918. static int tracing_buffers_release(struct inode *inode, struct file *file)
  2919. {
  2920. struct ftrace_buffer_info *info = file->private_data;
  2921. if (info->spare)
  2922. ring_buffer_free_read_page(info->tr->buffer, info->spare);
  2923. kfree(info);
  2924. return 0;
  2925. }
  2926. struct buffer_ref {
  2927. struct ring_buffer *buffer;
  2928. void *page;
  2929. int ref;
  2930. };
  2931. static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
  2932. struct pipe_buffer *buf)
  2933. {
  2934. struct buffer_ref *ref = (struct buffer_ref *)buf->private;
  2935. if (--ref->ref)
  2936. return;
  2937. ring_buffer_free_read_page(ref->buffer, ref->page);
  2938. kfree(ref);
  2939. buf->private = 0;
  2940. }
  2941. static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
  2942. struct pipe_buffer *buf)
  2943. {
  2944. return 1;
  2945. }
  2946. static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
  2947. struct pipe_buffer *buf)
  2948. {
  2949. struct buffer_ref *ref = (struct buffer_ref *)buf->private;
  2950. ref->ref++;
  2951. }
  2952. /* Pipe buffer operations for a buffer. */
  2953. static const struct pipe_buf_operations buffer_pipe_buf_ops = {
  2954. .can_merge = 0,
  2955. .map = generic_pipe_buf_map,
  2956. .unmap = generic_pipe_buf_unmap,
  2957. .confirm = generic_pipe_buf_confirm,
  2958. .release = buffer_pipe_buf_release,
  2959. .steal = buffer_pipe_buf_steal,
  2960. .get = buffer_pipe_buf_get,
  2961. };
  2962. /*
  2963. * Callback from splice_to_pipe(), if we need to release some pages
  2964. * at the end of the spd in case we error'ed out in filling the pipe.
  2965. */
  2966. static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
  2967. {
  2968. struct buffer_ref *ref =
  2969. (struct buffer_ref *)spd->partial[i].private;
  2970. if (--ref->ref)
  2971. return;
  2972. ring_buffer_free_read_page(ref->buffer, ref->page);
  2973. kfree(ref);
  2974. spd->partial[i].private = 0;
  2975. }
  2976. static ssize_t
  2977. tracing_buffers_splice_read(struct file *file, loff_t *ppos,
  2978. struct pipe_inode_info *pipe, size_t len,
  2979. unsigned int flags)
  2980. {
  2981. struct ftrace_buffer_info *info = file->private_data;
  2982. struct partial_page partial[PIPE_BUFFERS];
  2983. struct page *pages[PIPE_BUFFERS];
  2984. struct splice_pipe_desc spd = {
  2985. .pages = pages,
  2986. .partial = partial,
  2987. .flags = flags,
  2988. .ops = &buffer_pipe_buf_ops,
  2989. .spd_release = buffer_spd_release,
  2990. };
  2991. struct buffer_ref *ref;
  2992. int entries, size, i;
  2993. size_t ret;
  2994. if (*ppos & (PAGE_SIZE - 1)) {
  2995. WARN_ONCE(1, "Ftrace: previous read must page-align\n");
  2996. return -EINVAL;
  2997. }
  2998. if (len & (PAGE_SIZE - 1)) {
  2999. WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
  3000. if (len < PAGE_SIZE)
  3001. return -EINVAL;
  3002. len &= PAGE_MASK;
  3003. }
  3004. entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
  3005. for (i = 0; i < PIPE_BUFFERS && len && entries; i++, len -= PAGE_SIZE) {
  3006. struct page *page;
  3007. int r;
  3008. ref = kzalloc(sizeof(*ref), GFP_KERNEL);
  3009. if (!ref)
  3010. break;
  3011. ref->ref = 1;
  3012. ref->buffer = info->tr->buffer;
  3013. ref->page = ring_buffer_alloc_read_page(ref->buffer);
  3014. if (!ref->page) {
  3015. kfree(ref);
  3016. break;
  3017. }
  3018. r = ring_buffer_read_page(ref->buffer, &ref->page,
  3019. len, info->cpu, 1);
  3020. if (r < 0) {
  3021. ring_buffer_free_read_page(ref->buffer,
  3022. ref->page);
  3023. kfree(ref);
  3024. break;
  3025. }
  3026. /*
  3027. * zero out any left over data, this is going to
  3028. * user land.
  3029. */
  3030. size = ring_buffer_page_len(ref->page);
  3031. if (size < PAGE_SIZE)
  3032. memset(ref->page + size, 0, PAGE_SIZE - size);
  3033. page = virt_to_page(ref->page);
  3034. spd.pages[i] = page;
  3035. spd.partial[i].len = PAGE_SIZE;
  3036. spd.partial[i].offset = 0;
  3037. spd.partial[i].private = (unsigned long)ref;
  3038. spd.nr_pages++;
  3039. *ppos += PAGE_SIZE;
  3040. entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
  3041. }
  3042. spd.nr_pages = i;
  3043. /* did we read anything? */
  3044. if (!spd.nr_pages) {
  3045. if (flags & SPLICE_F_NONBLOCK)
  3046. ret = -EAGAIN;
  3047. else
  3048. ret = 0;
  3049. /* TODO: block */
  3050. return ret;
  3051. }
  3052. ret = splice_to_pipe(pipe, &spd);
  3053. return ret;
  3054. }
  3055. static const struct file_operations tracing_buffers_fops = {
  3056. .open = tracing_buffers_open,
  3057. .read = tracing_buffers_read,
  3058. .release = tracing_buffers_release,
  3059. .splice_read = tracing_buffers_splice_read,
  3060. .llseek = no_llseek,
  3061. };
  3062. static ssize_t
  3063. tracing_stats_read(struct file *filp, char __user *ubuf,
  3064. size_t count, loff_t *ppos)
  3065. {
  3066. unsigned long cpu = (unsigned long)filp->private_data;
  3067. struct trace_array *tr = &global_trace;
  3068. struct trace_seq *s;
  3069. unsigned long cnt;
  3070. s = kmalloc(sizeof(*s), GFP_KERNEL);
  3071. if (!s)
  3072. return -ENOMEM;
  3073. trace_seq_init(s);
  3074. cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
  3075. trace_seq_printf(s, "entries: %ld\n", cnt);
  3076. cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
  3077. trace_seq_printf(s, "overrun: %ld\n", cnt);
  3078. cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
  3079. trace_seq_printf(s, "commit overrun: %ld\n", cnt);
  3080. count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
  3081. kfree(s);
  3082. return count;
  3083. }
  3084. static const struct file_operations tracing_stats_fops = {
  3085. .open = tracing_open_generic,
  3086. .read = tracing_stats_read,
  3087. };
  3088. #ifdef CONFIG_DYNAMIC_FTRACE
  3089. int __weak ftrace_arch_read_dyn_info(char *buf, int size)
  3090. {
  3091. return 0;
  3092. }
  3093. static ssize_t
  3094. tracing_read_dyn_info(struct file *filp, char __user *ubuf,
  3095. size_t cnt, loff_t *ppos)
  3096. {
  3097. static char ftrace_dyn_info_buffer[1024];
  3098. static DEFINE_MUTEX(dyn_info_mutex);
  3099. unsigned long *p = filp->private_data;
  3100. char *buf = ftrace_dyn_info_buffer;
  3101. int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
  3102. int r;
  3103. mutex_lock(&dyn_info_mutex);
  3104. r = sprintf(buf, "%ld ", *p);
  3105. r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
  3106. buf[r++] = '\n';
  3107. r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  3108. mutex_unlock(&dyn_info_mutex);
  3109. return r;
  3110. }
  3111. static const struct file_operations tracing_dyn_info_fops = {
  3112. .open = tracing_open_generic,
  3113. .read = tracing_read_dyn_info,
  3114. };
  3115. #endif
  3116. static struct dentry *d_tracer;
  3117. struct dentry *tracing_init_dentry(void)
  3118. {
  3119. static int once;
  3120. if (d_tracer)
  3121. return d_tracer;
  3122. if (!debugfs_initialized())
  3123. return NULL;
  3124. d_tracer = debugfs_create_dir("tracing", NULL);
  3125. if (!d_tracer && !once) {
  3126. once = 1;
  3127. pr_warning("Could not create debugfs directory 'tracing'\n");
  3128. return NULL;
  3129. }
  3130. return d_tracer;
  3131. }
  3132. static struct dentry *d_percpu;
  3133. struct dentry *tracing_dentry_percpu(void)
  3134. {
  3135. static int once;
  3136. struct dentry *d_tracer;
  3137. if (d_percpu)
  3138. return d_percpu;
  3139. d_tracer = tracing_init_dentry();
  3140. if (!d_tracer)
  3141. return NULL;
  3142. d_percpu = debugfs_create_dir("per_cpu", d_tracer);
  3143. if (!d_percpu && !once) {
  3144. once = 1;
  3145. pr_warning("Could not create debugfs directory 'per_cpu'\n");
  3146. return NULL;
  3147. }
  3148. return d_percpu;
  3149. }
  3150. static void tracing_init_debugfs_percpu(long cpu)
  3151. {
  3152. struct dentry *d_percpu = tracing_dentry_percpu();
  3153. struct dentry *d_cpu;
  3154. /* strlen(cpu) + MAX(log10(cpu)) + '\0' */
  3155. char cpu_dir[7];
  3156. if (cpu > 999 || cpu < 0)
  3157. return;
  3158. sprintf(cpu_dir, "cpu%ld", cpu);
  3159. d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
  3160. if (!d_cpu) {
  3161. pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
  3162. return;
  3163. }
  3164. /* per cpu trace_pipe */
  3165. trace_create_file("trace_pipe", 0444, d_cpu,
  3166. (void *) cpu, &tracing_pipe_fops);
  3167. /* per cpu trace */
  3168. trace_create_file("trace", 0644, d_cpu,
  3169. (void *) cpu, &tracing_fops);
  3170. trace_create_file("trace_pipe_raw", 0444, d_cpu,
  3171. (void *) cpu, &tracing_buffers_fops);
  3172. trace_create_file("stats", 0444, d_cpu,
  3173. (void *) cpu, &tracing_stats_fops);
  3174. }
  3175. #ifdef CONFIG_FTRACE_SELFTEST
  3176. /* Let selftest have access to static functions in this file */
  3177. #include "trace_selftest.c"
  3178. #endif
  3179. struct trace_option_dentry {
  3180. struct tracer_opt *opt;
  3181. struct tracer_flags *flags;
  3182. struct dentry *entry;
  3183. };
  3184. static ssize_t
  3185. trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
  3186. loff_t *ppos)
  3187. {
  3188. struct trace_option_dentry *topt = filp->private_data;
  3189. char *buf;
  3190. if (topt->flags->val & topt->opt->bit)
  3191. buf = "1\n";
  3192. else
  3193. buf = "0\n";
  3194. return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
  3195. }
  3196. static ssize_t
  3197. trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
  3198. loff_t *ppos)
  3199. {
  3200. struct trace_option_dentry *topt = filp->private_data;
  3201. unsigned long val;
  3202. char buf[64];
  3203. int ret;
  3204. if (cnt >= sizeof(buf))
  3205. return -EINVAL;
  3206. if (copy_from_user(&buf, ubuf, cnt))
  3207. return -EFAULT;
  3208. buf[cnt] = 0;
  3209. ret = strict_strtoul(buf, 10, &val);
  3210. if (ret < 0)
  3211. return ret;
  3212. if (val != 0 && val != 1)
  3213. return -EINVAL;
  3214. if (!!(topt->flags->val & topt->opt->bit) != val) {
  3215. mutex_lock(&trace_types_lock);
  3216. ret = __set_tracer_option(current_trace, topt->flags,
  3217. topt->opt, !val);
  3218. mutex_unlock(&trace_types_lock);
  3219. if (ret)
  3220. return ret;
  3221. }
  3222. *ppos += cnt;
  3223. return cnt;
  3224. }
  3225. static const struct file_operations trace_options_fops = {
  3226. .open = tracing_open_generic,
  3227. .read = trace_options_read,
  3228. .write = trace_options_write,
  3229. };
  3230. static ssize_t
  3231. trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
  3232. loff_t *ppos)
  3233. {
  3234. long index = (long)filp->private_data;
  3235. char *buf;
  3236. if (trace_flags & (1 << index))
  3237. buf = "1\n";
  3238. else
  3239. buf = "0\n";
  3240. return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
  3241. }
  3242. static ssize_t
  3243. trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
  3244. loff_t *ppos)
  3245. {
  3246. long index = (long)filp->private_data;
  3247. char buf[64];
  3248. unsigned long val;
  3249. int ret;
  3250. if (cnt >= sizeof(buf))
  3251. return -EINVAL;
  3252. if (copy_from_user(&buf, ubuf, cnt))
  3253. return -EFAULT;
  3254. buf[cnt] = 0;
  3255. ret = strict_strtoul(buf, 10, &val);
  3256. if (ret < 0)
  3257. return ret;
  3258. if (val != 0 && val != 1)
  3259. return -EINVAL;
  3260. set_tracer_flags(1 << index, val);
  3261. *ppos += cnt;
  3262. return cnt;
  3263. }
  3264. static const struct file_operations trace_options_core_fops = {
  3265. .open = tracing_open_generic,
  3266. .read = trace_options_core_read,
  3267. .write = trace_options_core_write,
  3268. };
  3269. struct dentry *trace_create_file(const char *name,
  3270. mode_t mode,
  3271. struct dentry *parent,
  3272. void *data,
  3273. const struct file_operations *fops)
  3274. {
  3275. struct dentry *ret;
  3276. ret = debugfs_create_file(name, mode, parent, data, fops);
  3277. if (!ret)
  3278. pr_warning("Could not create debugfs '%s' entry\n", name);
  3279. return ret;
  3280. }
  3281. static struct dentry *trace_options_init_dentry(void)
  3282. {
  3283. struct dentry *d_tracer;
  3284. static struct dentry *t_options;
  3285. if (t_options)
  3286. return t_options;
  3287. d_tracer = tracing_init_dentry();
  3288. if (!d_tracer)
  3289. return NULL;
  3290. t_options = debugfs_create_dir("options", d_tracer);
  3291. if (!t_options) {
  3292. pr_warning("Could not create debugfs directory 'options'\n");
  3293. return NULL;
  3294. }
  3295. return t_options;
  3296. }
  3297. static void
  3298. create_trace_option_file(struct trace_option_dentry *topt,
  3299. struct tracer_flags *flags,
  3300. struct tracer_opt *opt)
  3301. {
  3302. struct dentry *t_options;
  3303. t_options = trace_options_init_dentry();
  3304. if (!t_options)
  3305. return;
  3306. topt->flags = flags;
  3307. topt->opt = opt;
  3308. topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
  3309. &trace_options_fops);
  3310. }
  3311. static struct trace_option_dentry *
  3312. create_trace_option_files(struct tracer *tracer)
  3313. {
  3314. struct trace_option_dentry *topts;
  3315. struct tracer_flags *flags;
  3316. struct tracer_opt *opts;
  3317. int cnt;
  3318. if (!tracer)
  3319. return NULL;
  3320. flags = tracer->flags;
  3321. if (!flags || !flags->opts)
  3322. return NULL;
  3323. opts = flags->opts;
  3324. for (cnt = 0; opts[cnt].name; cnt++)
  3325. ;
  3326. topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
  3327. if (!topts)
  3328. return NULL;
  3329. for (cnt = 0; opts[cnt].name; cnt++)
  3330. create_trace_option_file(&topts[cnt], flags,
  3331. &opts[cnt]);
  3332. return topts;
  3333. }
  3334. static void
  3335. destroy_trace_option_files(struct trace_option_dentry *topts)
  3336. {
  3337. int cnt;
  3338. if (!topts)
  3339. return;
  3340. for (cnt = 0; topts[cnt].opt; cnt++) {
  3341. if (topts[cnt].entry)
  3342. debugfs_remove(topts[cnt].entry);
  3343. }
  3344. kfree(topts);
  3345. }
  3346. static struct dentry *
  3347. create_trace_option_core_file(const char *option, long index)
  3348. {
  3349. struct dentry *t_options;
  3350. t_options = trace_options_init_dentry();
  3351. if (!t_options)
  3352. return NULL;
  3353. return trace_create_file(option, 0644, t_options, (void *)index,
  3354. &trace_options_core_fops);
  3355. }
  3356. static __init void create_trace_options_dir(void)
  3357. {
  3358. struct dentry *t_options;
  3359. int i;
  3360. t_options = trace_options_init_dentry();
  3361. if (!t_options)
  3362. return;
  3363. for (i = 0; trace_options[i]; i++)
  3364. create_trace_option_core_file(trace_options[i], i);
  3365. }
  3366. static __init int tracer_init_debugfs(void)
  3367. {
  3368. struct dentry *d_tracer;
  3369. int cpu;
  3370. d_tracer = tracing_init_dentry();
  3371. trace_create_file("tracing_enabled", 0644, d_tracer,
  3372. &global_trace, &tracing_ctrl_fops);
  3373. trace_create_file("trace_options", 0644, d_tracer,
  3374. NULL, &tracing_iter_fops);
  3375. trace_create_file("tracing_cpumask", 0644, d_tracer,
  3376. NULL, &tracing_cpumask_fops);
  3377. trace_create_file("trace", 0644, d_tracer,
  3378. (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
  3379. trace_create_file("available_tracers", 0444, d_tracer,
  3380. &global_trace, &show_traces_fops);
  3381. trace_create_file("current_tracer", 0644, d_tracer,
  3382. &global_trace, &set_tracer_fops);
  3383. #ifdef CONFIG_TRACER_MAX_TRACE
  3384. trace_create_file("tracing_max_latency", 0644, d_tracer,
  3385. &tracing_max_latency, &tracing_max_lat_fops);
  3386. trace_create_file("tracing_thresh", 0644, d_tracer,
  3387. &tracing_thresh, &tracing_max_lat_fops);
  3388. #endif
  3389. trace_create_file("README", 0444, d_tracer,
  3390. NULL, &tracing_readme_fops);
  3391. trace_create_file("trace_pipe", 0444, d_tracer,
  3392. (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
  3393. trace_create_file("buffer_size_kb", 0644, d_tracer,
  3394. &global_trace, &tracing_entries_fops);
  3395. trace_create_file("trace_marker", 0220, d_tracer,
  3396. NULL, &tracing_mark_fops);
  3397. trace_create_file("saved_cmdlines", 0444, d_tracer,
  3398. NULL, &tracing_saved_cmdlines_fops);
  3399. trace_create_file("trace_clock", 0644, d_tracer, NULL,
  3400. &trace_clock_fops);
  3401. #ifdef CONFIG_DYNAMIC_FTRACE
  3402. trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
  3403. &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
  3404. #endif
  3405. #ifdef CONFIG_SYSPROF_TRACER
  3406. init_tracer_sysprof_debugfs(d_tracer);
  3407. #endif
  3408. create_trace_options_dir();
  3409. for_each_tracing_cpu(cpu)
  3410. tracing_init_debugfs_percpu(cpu);
  3411. return 0;
  3412. }
  3413. static int trace_panic_handler(struct notifier_block *this,
  3414. unsigned long event, void *unused)
  3415. {
  3416. if (ftrace_dump_on_oops)
  3417. ftrace_dump();
  3418. return NOTIFY_OK;
  3419. }
  3420. static struct notifier_block trace_panic_notifier = {
  3421. .notifier_call = trace_panic_handler,
  3422. .next = NULL,
  3423. .priority = 150 /* priority: INT_MAX >= x >= 0 */
  3424. };
  3425. static int trace_die_handler(struct notifier_block *self,
  3426. unsigned long val,
  3427. void *data)
  3428. {
  3429. switch (val) {
  3430. case DIE_OOPS:
  3431. if (ftrace_dump_on_oops)
  3432. ftrace_dump();
  3433. break;
  3434. default:
  3435. break;
  3436. }
  3437. return NOTIFY_OK;
  3438. }
  3439. static struct notifier_block trace_die_notifier = {
  3440. .notifier_call = trace_die_handler,
  3441. .priority = 200
  3442. };
  3443. /*
  3444. * printk is set to max of 1024, we really don't need it that big.
  3445. * Nothing should be printing 1000 characters anyway.
  3446. */
  3447. #define TRACE_MAX_PRINT 1000
  3448. /*
  3449. * Define here KERN_TRACE so that we have one place to modify
  3450. * it if we decide to change what log level the ftrace dump
  3451. * should be at.
  3452. */
  3453. #define KERN_TRACE KERN_EMERG
  3454. static void
  3455. trace_printk_seq(struct trace_seq *s)
  3456. {
  3457. /* Probably should print a warning here. */
  3458. if (s->len >= 1000)
  3459. s->len = 1000;
  3460. /* should be zero ended, but we are paranoid. */
  3461. s->buffer[s->len] = 0;
  3462. printk(KERN_TRACE "%s", s->buffer);
  3463. trace_seq_init(s);
  3464. }
  3465. static void __ftrace_dump(bool disable_tracing)
  3466. {
  3467. static arch_spinlock_t ftrace_dump_lock =
  3468. (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  3469. /* use static because iter can be a bit big for the stack */
  3470. static struct trace_iterator iter;
  3471. unsigned int old_userobj;
  3472. static int dump_ran;
  3473. unsigned long flags;
  3474. int cnt = 0, cpu;
  3475. /* only one dump */
  3476. local_irq_save(flags);
  3477. arch_spin_lock(&ftrace_dump_lock);
  3478. if (dump_ran)
  3479. goto out;
  3480. dump_ran = 1;
  3481. tracing_off();
  3482. if (disable_tracing)
  3483. ftrace_kill();
  3484. for_each_tracing_cpu(cpu) {
  3485. atomic_inc(&global_trace.data[cpu]->disabled);
  3486. }
  3487. old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
  3488. /* don't look at user memory in panic mode */
  3489. trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
  3490. printk(KERN_TRACE "Dumping ftrace buffer:\n");
  3491. /* Simulate the iterator */
  3492. iter.tr = &global_trace;
  3493. iter.trace = current_trace;
  3494. iter.cpu_file = TRACE_PIPE_ALL_CPU;
  3495. /*
  3496. * We need to stop all tracing on all CPUS to read the
  3497. * the next buffer. This is a bit expensive, but is
  3498. * not done often. We fill all what we can read,
  3499. * and then release the locks again.
  3500. */
  3501. while (!trace_empty(&iter)) {
  3502. if (!cnt)
  3503. printk(KERN_TRACE "---------------------------------\n");
  3504. cnt++;
  3505. /* reset all but tr, trace, and overruns */
  3506. memset(&iter.seq, 0,
  3507. sizeof(struct trace_iterator) -
  3508. offsetof(struct trace_iterator, seq));
  3509. iter.iter_flags |= TRACE_FILE_LAT_FMT;
  3510. iter.pos = -1;
  3511. if (find_next_entry_inc(&iter) != NULL) {
  3512. int ret;
  3513. ret = print_trace_line(&iter);
  3514. if (ret != TRACE_TYPE_NO_CONSUME)
  3515. trace_consume(&iter);
  3516. }
  3517. trace_printk_seq(&iter.seq);
  3518. }
  3519. if (!cnt)
  3520. printk(KERN_TRACE " (ftrace buffer empty)\n");
  3521. else
  3522. printk(KERN_TRACE "---------------------------------\n");
  3523. /* Re-enable tracing if requested */
  3524. if (!disable_tracing) {
  3525. trace_flags |= old_userobj;
  3526. for_each_tracing_cpu(cpu) {
  3527. atomic_dec(&global_trace.data[cpu]->disabled);
  3528. }
  3529. tracing_on();
  3530. }
  3531. out:
  3532. arch_spin_unlock(&ftrace_dump_lock);
  3533. local_irq_restore(flags);
  3534. }
  3535. /* By default: disable tracing after the dump */
  3536. void ftrace_dump(void)
  3537. {
  3538. __ftrace_dump(true);
  3539. }
  3540. __init static int tracer_alloc_buffers(void)
  3541. {
  3542. int ring_buf_size;
  3543. int i;
  3544. int ret = -ENOMEM;
  3545. if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
  3546. goto out;
  3547. if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
  3548. goto out_free_buffer_mask;
  3549. if (!zalloc_cpumask_var(&tracing_reader_cpumask, GFP_KERNEL))
  3550. goto out_free_tracing_cpumask;
  3551. /* To save memory, keep the ring buffer size to its minimum */
  3552. if (ring_buffer_expanded)
  3553. ring_buf_size = trace_buf_size;
  3554. else
  3555. ring_buf_size = 1;
  3556. cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
  3557. cpumask_copy(tracing_cpumask, cpu_all_mask);
  3558. /* TODO: make the number of buffers hot pluggable with CPUS */
  3559. global_trace.buffer = ring_buffer_alloc(ring_buf_size,
  3560. TRACE_BUFFER_FLAGS);
  3561. if (!global_trace.buffer) {
  3562. printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
  3563. WARN_ON(1);
  3564. goto out_free_cpumask;
  3565. }
  3566. global_trace.entries = ring_buffer_size(global_trace.buffer);
  3567. #ifdef CONFIG_TRACER_MAX_TRACE
  3568. max_tr.buffer = ring_buffer_alloc(ring_buf_size,
  3569. TRACE_BUFFER_FLAGS);
  3570. if (!max_tr.buffer) {
  3571. printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
  3572. WARN_ON(1);
  3573. ring_buffer_free(global_trace.buffer);
  3574. goto out_free_cpumask;
  3575. }
  3576. max_tr.entries = ring_buffer_size(max_tr.buffer);
  3577. WARN_ON(max_tr.entries != global_trace.entries);
  3578. #endif
  3579. /* Allocate the first page for all buffers */
  3580. for_each_tracing_cpu(i) {
  3581. global_trace.data[i] = &per_cpu(global_trace_cpu, i);
  3582. max_tr.data[i] = &per_cpu(max_tr_data, i);
  3583. }
  3584. trace_init_cmdlines();
  3585. register_tracer(&nop_trace);
  3586. current_trace = &nop_trace;
  3587. #ifdef CONFIG_BOOT_TRACER
  3588. register_tracer(&boot_tracer);
  3589. #endif
  3590. /* All seems OK, enable tracing */
  3591. tracing_disabled = 0;
  3592. atomic_notifier_chain_register(&panic_notifier_list,
  3593. &trace_panic_notifier);
  3594. register_die_notifier(&trace_die_notifier);
  3595. return 0;
  3596. out_free_cpumask:
  3597. free_cpumask_var(tracing_reader_cpumask);
  3598. out_free_tracing_cpumask:
  3599. free_cpumask_var(tracing_cpumask);
  3600. out_free_buffer_mask:
  3601. free_cpumask_var(tracing_buffer_mask);
  3602. out:
  3603. return ret;
  3604. }
  3605. __init static int clear_boot_tracer(void)
  3606. {
  3607. /*
  3608. * The default tracer at boot buffer is an init section.
  3609. * This function is called in lateinit. If we did not
  3610. * find the boot tracer, then clear it out, to prevent
  3611. * later registration from accessing the buffer that is
  3612. * about to be freed.
  3613. */
  3614. if (!default_bootup_tracer)
  3615. return 0;
  3616. printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
  3617. default_bootup_tracer);
  3618. default_bootup_tracer = NULL;
  3619. return 0;
  3620. }
  3621. early_initcall(tracer_alloc_buffers);
  3622. fs_initcall(tracer_init_debugfs);
  3623. late_initcall(clear_boot_tracer);