ftrace.c 96 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531
  1. /*
  2. * Infrastructure for profiling code inserted by 'gcc -pg'.
  3. *
  4. * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  5. * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
  6. *
  7. * Originally ported from the -rt patch by:
  8. * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
  9. *
  10. * Based on code in the latency_tracer, that is:
  11. *
  12. * Copyright (C) 2004-2006 Ingo Molnar
  13. * Copyright (C) 2004 William Lee Irwin III
  14. */
  15. #include <linux/stop_machine.h>
  16. #include <linux/clocksource.h>
  17. #include <linux/kallsyms.h>
  18. #include <linux/seq_file.h>
  19. #include <linux/suspend.h>
  20. #include <linux/debugfs.h>
  21. #include <linux/hardirq.h>
  22. #include <linux/kthread.h>
  23. #include <linux/uaccess.h>
  24. #include <linux/module.h>
  25. #include <linux/ftrace.h>
  26. #include <linux/sysctl.h>
  27. #include <linux/slab.h>
  28. #include <linux/ctype.h>
  29. #include <linux/sort.h>
  30. #include <linux/list.h>
  31. #include <linux/hash.h>
  32. #include <linux/rcupdate.h>
  33. #include <trace/events/sched.h>
  34. #include <asm/setup.h>
  35. #include "trace_output.h"
  36. #include "trace_stat.h"
  37. #define FTRACE_WARN_ON(cond) \
  38. ({ \
  39. int ___r = cond; \
  40. if (WARN_ON(___r)) \
  41. ftrace_kill(); \
  42. ___r; \
  43. })
  44. #define FTRACE_WARN_ON_ONCE(cond) \
  45. ({ \
  46. int ___r = cond; \
  47. if (WARN_ON_ONCE(___r)) \
  48. ftrace_kill(); \
  49. ___r; \
  50. })
  51. /* hash bits for specific function selection */
  52. #define FTRACE_HASH_BITS 7
  53. #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  54. #define FTRACE_HASH_DEFAULT_BITS 10
  55. #define FTRACE_HASH_MAX_BITS 12
  56. /* ftrace_enabled is a method to turn ftrace on or off */
  57. int ftrace_enabled __read_mostly;
  58. static int last_ftrace_enabled;
  59. /* Quick disabling of function tracer. */
  60. int function_trace_stop;
  61. /* List for set_ftrace_pid's pids. */
  62. LIST_HEAD(ftrace_pids);
  63. struct ftrace_pid {
  64. struct list_head list;
  65. struct pid *pid;
  66. };
  67. /*
  68. * ftrace_disabled is set when an anomaly is discovered.
  69. * ftrace_disabled is much stronger than ftrace_enabled.
  70. */
  71. static int ftrace_disabled __read_mostly;
  72. static DEFINE_MUTEX(ftrace_lock);
  73. static struct ftrace_ops ftrace_list_end __read_mostly = {
  74. .func = ftrace_stub,
  75. };
  76. static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
  77. static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
  78. ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  79. static ftrace_func_t __ftrace_trace_function_delay __read_mostly = ftrace_stub;
  80. ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  81. ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  82. static struct ftrace_ops global_ops;
  83. static void
  84. ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);
  85. /*
  86. * Traverse the ftrace_global_list, invoking all entries. The reason that we
  87. * can use rcu_dereference_raw() is that elements removed from this list
  88. * are simply leaked, so there is no need to interact with a grace-period
  89. * mechanism. The rcu_dereference_raw() calls are needed to handle
  90. * concurrent insertions into the ftrace_global_list.
  91. *
  92. * Silly Alpha and silly pointer-speculation compiler optimizations!
  93. */
  94. static void ftrace_global_list_func(unsigned long ip,
  95. unsigned long parent_ip)
  96. {
  97. struct ftrace_ops *op;
  98. if (unlikely(trace_recursion_test(TRACE_GLOBAL_BIT)))
  99. return;
  100. trace_recursion_set(TRACE_GLOBAL_BIT);
  101. op = rcu_dereference_raw(ftrace_global_list); /*see above*/
  102. while (op != &ftrace_list_end) {
  103. op->func(ip, parent_ip);
  104. op = rcu_dereference_raw(op->next); /*see above*/
  105. };
  106. trace_recursion_clear(TRACE_GLOBAL_BIT);
  107. }
  108. static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
  109. {
  110. if (!test_tsk_trace_trace(current))
  111. return;
  112. ftrace_pid_function(ip, parent_ip);
  113. }
  114. static void set_ftrace_pid_function(ftrace_func_t func)
  115. {
  116. /* do not set ftrace_pid_function to itself! */
  117. if (func != ftrace_pid_func)
  118. ftrace_pid_function = func;
  119. }
  120. /**
  121. * clear_ftrace_function - reset the ftrace function
  122. *
  123. * This NULLs the ftrace function and in essence stops
  124. * tracing. There may be lag
  125. */
  126. void clear_ftrace_function(void)
  127. {
  128. ftrace_trace_function = ftrace_stub;
  129. __ftrace_trace_function = ftrace_stub;
  130. __ftrace_trace_function_delay = ftrace_stub;
  131. ftrace_pid_function = ftrace_stub;
  132. }
  133. #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
  134. /*
  135. * For those archs that do not test ftrace_trace_stop in their
  136. * mcount call site, we need to do it from C.
  137. */
  138. static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
  139. {
  140. if (function_trace_stop)
  141. return;
  142. __ftrace_trace_function(ip, parent_ip);
  143. }
  144. #endif
  145. static void update_global_ops(void)
  146. {
  147. ftrace_func_t func;
  148. /*
  149. * If there's only one function registered, then call that
  150. * function directly. Otherwise, we need to iterate over the
  151. * registered callers.
  152. */
  153. if (ftrace_global_list == &ftrace_list_end ||
  154. ftrace_global_list->next == &ftrace_list_end)
  155. func = ftrace_global_list->func;
  156. else
  157. func = ftrace_global_list_func;
  158. /* If we filter on pids, update to use the pid function */
  159. if (!list_empty(&ftrace_pids)) {
  160. set_ftrace_pid_function(func);
  161. func = ftrace_pid_func;
  162. }
  163. global_ops.func = func;
  164. }
  165. static void update_ftrace_function(void)
  166. {
  167. ftrace_func_t func;
  168. update_global_ops();
  169. /*
  170. * If we are at the end of the list and this ops is
  171. * not dynamic, then have the mcount trampoline call
  172. * the function directly
  173. */
  174. if (ftrace_ops_list == &ftrace_list_end ||
  175. (ftrace_ops_list->next == &ftrace_list_end &&
  176. !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC)))
  177. func = ftrace_ops_list->func;
  178. else
  179. func = ftrace_ops_list_func;
  180. #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
  181. ftrace_trace_function = func;
  182. #else
  183. #ifdef CONFIG_DYNAMIC_FTRACE
  184. /* do not update till all functions have been modified */
  185. __ftrace_trace_function_delay = func;
  186. #else
  187. __ftrace_trace_function = func;
  188. #endif
  189. ftrace_trace_function = ftrace_test_stop_func;
  190. #endif
  191. }
  192. static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
  193. {
  194. ops->next = *list;
  195. /*
  196. * We are entering ops into the list but another
  197. * CPU might be walking that list. We need to make sure
  198. * the ops->next pointer is valid before another CPU sees
  199. * the ops pointer included into the list.
  200. */
  201. rcu_assign_pointer(*list, ops);
  202. }
  203. static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
  204. {
  205. struct ftrace_ops **p;
  206. /*
  207. * If we are removing the last function, then simply point
  208. * to the ftrace_stub.
  209. */
  210. if (*list == ops && ops->next == &ftrace_list_end) {
  211. *list = &ftrace_list_end;
  212. return 0;
  213. }
  214. for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
  215. if (*p == ops)
  216. break;
  217. if (*p != ops)
  218. return -1;
  219. *p = (*p)->next;
  220. return 0;
  221. }
  222. static int __register_ftrace_function(struct ftrace_ops *ops)
  223. {
  224. if (ftrace_disabled)
  225. return -ENODEV;
  226. if (FTRACE_WARN_ON(ops == &global_ops))
  227. return -EINVAL;
  228. if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
  229. return -EBUSY;
  230. if (!core_kernel_data((unsigned long)ops))
  231. ops->flags |= FTRACE_OPS_FL_DYNAMIC;
  232. if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
  233. int first = ftrace_global_list == &ftrace_list_end;
  234. add_ftrace_ops(&ftrace_global_list, ops);
  235. ops->flags |= FTRACE_OPS_FL_ENABLED;
  236. if (first)
  237. add_ftrace_ops(&ftrace_ops_list, &global_ops);
  238. } else
  239. add_ftrace_ops(&ftrace_ops_list, ops);
  240. if (ftrace_enabled)
  241. update_ftrace_function();
  242. return 0;
  243. }
  244. static int __unregister_ftrace_function(struct ftrace_ops *ops)
  245. {
  246. int ret;
  247. if (ftrace_disabled)
  248. return -ENODEV;
  249. if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
  250. return -EBUSY;
  251. if (FTRACE_WARN_ON(ops == &global_ops))
  252. return -EINVAL;
  253. if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
  254. ret = remove_ftrace_ops(&ftrace_global_list, ops);
  255. if (!ret && ftrace_global_list == &ftrace_list_end)
  256. ret = remove_ftrace_ops(&ftrace_ops_list, &global_ops);
  257. if (!ret)
  258. ops->flags &= ~FTRACE_OPS_FL_ENABLED;
  259. } else
  260. ret = remove_ftrace_ops(&ftrace_ops_list, ops);
  261. if (ret < 0)
  262. return ret;
  263. if (ftrace_enabled)
  264. update_ftrace_function();
  265. /*
  266. * Dynamic ops may be freed, we must make sure that all
  267. * callers are done before leaving this function.
  268. */
  269. if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
  270. synchronize_sched();
  271. return 0;
  272. }
  273. static void ftrace_update_pid_func(void)
  274. {
  275. /* Only do something if we are tracing something */
  276. if (ftrace_trace_function == ftrace_stub)
  277. return;
  278. update_ftrace_function();
  279. }
  280. #ifdef CONFIG_FUNCTION_PROFILER
  281. struct ftrace_profile {
  282. struct hlist_node node;
  283. unsigned long ip;
  284. unsigned long counter;
  285. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  286. unsigned long long time;
  287. unsigned long long time_squared;
  288. #endif
  289. };
  290. struct ftrace_profile_page {
  291. struct ftrace_profile_page *next;
  292. unsigned long index;
  293. struct ftrace_profile records[];
  294. };
  295. struct ftrace_profile_stat {
  296. atomic_t disabled;
  297. struct hlist_head *hash;
  298. struct ftrace_profile_page *pages;
  299. struct ftrace_profile_page *start;
  300. struct tracer_stat stat;
  301. };
  302. #define PROFILE_RECORDS_SIZE \
  303. (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
  304. #define PROFILES_PER_PAGE \
  305. (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
  306. static int ftrace_profile_bits __read_mostly;
  307. static int ftrace_profile_enabled __read_mostly;
  308. /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
  309. static DEFINE_MUTEX(ftrace_profile_lock);
  310. static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
  311. #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
  312. static void *
  313. function_stat_next(void *v, int idx)
  314. {
  315. struct ftrace_profile *rec = v;
  316. struct ftrace_profile_page *pg;
  317. pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
  318. again:
  319. if (idx != 0)
  320. rec++;
  321. if ((void *)rec >= (void *)&pg->records[pg->index]) {
  322. pg = pg->next;
  323. if (!pg)
  324. return NULL;
  325. rec = &pg->records[0];
  326. if (!rec->counter)
  327. goto again;
  328. }
  329. return rec;
  330. }
  331. static void *function_stat_start(struct tracer_stat *trace)
  332. {
  333. struct ftrace_profile_stat *stat =
  334. container_of(trace, struct ftrace_profile_stat, stat);
  335. if (!stat || !stat->start)
  336. return NULL;
  337. return function_stat_next(&stat->start->records[0], 0);
  338. }
  339. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  340. /* function graph compares on total time */
  341. static int function_stat_cmp(void *p1, void *p2)
  342. {
  343. struct ftrace_profile *a = p1;
  344. struct ftrace_profile *b = p2;
  345. if (a->time < b->time)
  346. return -1;
  347. if (a->time > b->time)
  348. return 1;
  349. else
  350. return 0;
  351. }
  352. #else
  353. /* not function graph compares against hits */
  354. static int function_stat_cmp(void *p1, void *p2)
  355. {
  356. struct ftrace_profile *a = p1;
  357. struct ftrace_profile *b = p2;
  358. if (a->counter < b->counter)
  359. return -1;
  360. if (a->counter > b->counter)
  361. return 1;
  362. else
  363. return 0;
  364. }
  365. #endif
  366. static int function_stat_headers(struct seq_file *m)
  367. {
  368. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  369. seq_printf(m, " Function "
  370. "Hit Time Avg s^2\n"
  371. " -------- "
  372. "--- ---- --- ---\n");
  373. #else
  374. seq_printf(m, " Function Hit\n"
  375. " -------- ---\n");
  376. #endif
  377. return 0;
  378. }
  379. static int function_stat_show(struct seq_file *m, void *v)
  380. {
  381. struct ftrace_profile *rec = v;
  382. char str[KSYM_SYMBOL_LEN];
  383. int ret = 0;
  384. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  385. static struct trace_seq s;
  386. unsigned long long avg;
  387. unsigned long long stddev;
  388. #endif
  389. mutex_lock(&ftrace_profile_lock);
  390. /* we raced with function_profile_reset() */
  391. if (unlikely(rec->counter == 0)) {
  392. ret = -EBUSY;
  393. goto out;
  394. }
  395. kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
  396. seq_printf(m, " %-30.30s %10lu", str, rec->counter);
  397. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  398. seq_printf(m, " ");
  399. avg = rec->time;
  400. do_div(avg, rec->counter);
  401. /* Sample standard deviation (s^2) */
  402. if (rec->counter <= 1)
  403. stddev = 0;
  404. else {
  405. stddev = rec->time_squared - rec->counter * avg * avg;
  406. /*
  407. * Divide only 1000 for ns^2 -> us^2 conversion.
  408. * trace_print_graph_duration will divide 1000 again.
  409. */
  410. do_div(stddev, (rec->counter - 1) * 1000);
  411. }
  412. trace_seq_init(&s);
  413. trace_print_graph_duration(rec->time, &s);
  414. trace_seq_puts(&s, " ");
  415. trace_print_graph_duration(avg, &s);
  416. trace_seq_puts(&s, " ");
  417. trace_print_graph_duration(stddev, &s);
  418. trace_print_seq(m, &s);
  419. #endif
  420. seq_putc(m, '\n');
  421. out:
  422. mutex_unlock(&ftrace_profile_lock);
  423. return ret;
  424. }
  425. static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
  426. {
  427. struct ftrace_profile_page *pg;
  428. pg = stat->pages = stat->start;
  429. while (pg) {
  430. memset(pg->records, 0, PROFILE_RECORDS_SIZE);
  431. pg->index = 0;
  432. pg = pg->next;
  433. }
  434. memset(stat->hash, 0,
  435. FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
  436. }
  437. int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
  438. {
  439. struct ftrace_profile_page *pg;
  440. int functions;
  441. int pages;
  442. int i;
  443. /* If we already allocated, do nothing */
  444. if (stat->pages)
  445. return 0;
  446. stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
  447. if (!stat->pages)
  448. return -ENOMEM;
  449. #ifdef CONFIG_DYNAMIC_FTRACE
  450. functions = ftrace_update_tot_cnt;
  451. #else
  452. /*
  453. * We do not know the number of functions that exist because
  454. * dynamic tracing is what counts them. With past experience
  455. * we have around 20K functions. That should be more than enough.
  456. * It is highly unlikely we will execute every function in
  457. * the kernel.
  458. */
  459. functions = 20000;
  460. #endif
  461. pg = stat->start = stat->pages;
  462. pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
  463. for (i = 0; i < pages; i++) {
  464. pg->next = (void *)get_zeroed_page(GFP_KERNEL);
  465. if (!pg->next)
  466. goto out_free;
  467. pg = pg->next;
  468. }
  469. return 0;
  470. out_free:
  471. pg = stat->start;
  472. while (pg) {
  473. unsigned long tmp = (unsigned long)pg;
  474. pg = pg->next;
  475. free_page(tmp);
  476. }
  477. free_page((unsigned long)stat->pages);
  478. stat->pages = NULL;
  479. stat->start = NULL;
  480. return -ENOMEM;
  481. }
  482. static int ftrace_profile_init_cpu(int cpu)
  483. {
  484. struct ftrace_profile_stat *stat;
  485. int size;
  486. stat = &per_cpu(ftrace_profile_stats, cpu);
  487. if (stat->hash) {
  488. /* If the profile is already created, simply reset it */
  489. ftrace_profile_reset(stat);
  490. return 0;
  491. }
  492. /*
  493. * We are profiling all functions, but usually only a few thousand
  494. * functions are hit. We'll make a hash of 1024 items.
  495. */
  496. size = FTRACE_PROFILE_HASH_SIZE;
  497. stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
  498. if (!stat->hash)
  499. return -ENOMEM;
  500. if (!ftrace_profile_bits) {
  501. size--;
  502. for (; size; size >>= 1)
  503. ftrace_profile_bits++;
  504. }
  505. /* Preallocate the function profiling pages */
  506. if (ftrace_profile_pages_init(stat) < 0) {
  507. kfree(stat->hash);
  508. stat->hash = NULL;
  509. return -ENOMEM;
  510. }
  511. return 0;
  512. }
  513. static int ftrace_profile_init(void)
  514. {
  515. int cpu;
  516. int ret = 0;
  517. for_each_online_cpu(cpu) {
  518. ret = ftrace_profile_init_cpu(cpu);
  519. if (ret)
  520. break;
  521. }
  522. return ret;
  523. }
  524. /* interrupts must be disabled */
  525. static struct ftrace_profile *
  526. ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
  527. {
  528. struct ftrace_profile *rec;
  529. struct hlist_head *hhd;
  530. struct hlist_node *n;
  531. unsigned long key;
  532. key = hash_long(ip, ftrace_profile_bits);
  533. hhd = &stat->hash[key];
  534. if (hlist_empty(hhd))
  535. return NULL;
  536. hlist_for_each_entry_rcu(rec, n, hhd, node) {
  537. if (rec->ip == ip)
  538. return rec;
  539. }
  540. return NULL;
  541. }
  542. static void ftrace_add_profile(struct ftrace_profile_stat *stat,
  543. struct ftrace_profile *rec)
  544. {
  545. unsigned long key;
  546. key = hash_long(rec->ip, ftrace_profile_bits);
  547. hlist_add_head_rcu(&rec->node, &stat->hash[key]);
  548. }
  549. /*
  550. * The memory is already allocated, this simply finds a new record to use.
  551. */
  552. static struct ftrace_profile *
  553. ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
  554. {
  555. struct ftrace_profile *rec = NULL;
  556. /* prevent recursion (from NMIs) */
  557. if (atomic_inc_return(&stat->disabled) != 1)
  558. goto out;
  559. /*
  560. * Try to find the function again since an NMI
  561. * could have added it
  562. */
  563. rec = ftrace_find_profiled_func(stat, ip);
  564. if (rec)
  565. goto out;
  566. if (stat->pages->index == PROFILES_PER_PAGE) {
  567. if (!stat->pages->next)
  568. goto out;
  569. stat->pages = stat->pages->next;
  570. }
  571. rec = &stat->pages->records[stat->pages->index++];
  572. rec->ip = ip;
  573. ftrace_add_profile(stat, rec);
  574. out:
  575. atomic_dec(&stat->disabled);
  576. return rec;
  577. }
  578. static void
  579. function_profile_call(unsigned long ip, unsigned long parent_ip)
  580. {
  581. struct ftrace_profile_stat *stat;
  582. struct ftrace_profile *rec;
  583. unsigned long flags;
  584. if (!ftrace_profile_enabled)
  585. return;
  586. local_irq_save(flags);
  587. stat = &__get_cpu_var(ftrace_profile_stats);
  588. if (!stat->hash || !ftrace_profile_enabled)
  589. goto out;
  590. rec = ftrace_find_profiled_func(stat, ip);
  591. if (!rec) {
  592. rec = ftrace_profile_alloc(stat, ip);
  593. if (!rec)
  594. goto out;
  595. }
  596. rec->counter++;
  597. out:
  598. local_irq_restore(flags);
  599. }
  600. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  601. static int profile_graph_entry(struct ftrace_graph_ent *trace)
  602. {
  603. function_profile_call(trace->func, 0);
  604. return 1;
  605. }
  606. static void profile_graph_return(struct ftrace_graph_ret *trace)
  607. {
  608. struct ftrace_profile_stat *stat;
  609. unsigned long long calltime;
  610. struct ftrace_profile *rec;
  611. unsigned long flags;
  612. local_irq_save(flags);
  613. stat = &__get_cpu_var(ftrace_profile_stats);
  614. if (!stat->hash || !ftrace_profile_enabled)
  615. goto out;
  616. /* If the calltime was zero'd ignore it */
  617. if (!trace->calltime)
  618. goto out;
  619. calltime = trace->rettime - trace->calltime;
  620. if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
  621. int index;
  622. index = trace->depth;
  623. /* Append this call time to the parent time to subtract */
  624. if (index)
  625. current->ret_stack[index - 1].subtime += calltime;
  626. if (current->ret_stack[index].subtime < calltime)
  627. calltime -= current->ret_stack[index].subtime;
  628. else
  629. calltime = 0;
  630. }
  631. rec = ftrace_find_profiled_func(stat, trace->func);
  632. if (rec) {
  633. rec->time += calltime;
  634. rec->time_squared += calltime * calltime;
  635. }
  636. out:
  637. local_irq_restore(flags);
  638. }
  639. static int register_ftrace_profiler(void)
  640. {
  641. return register_ftrace_graph(&profile_graph_return,
  642. &profile_graph_entry);
  643. }
  644. static void unregister_ftrace_profiler(void)
  645. {
  646. unregister_ftrace_graph();
  647. }
  648. #else
  649. static struct ftrace_ops ftrace_profile_ops __read_mostly = {
  650. .func = function_profile_call,
  651. };
  652. static int register_ftrace_profiler(void)
  653. {
  654. return register_ftrace_function(&ftrace_profile_ops);
  655. }
  656. static void unregister_ftrace_profiler(void)
  657. {
  658. unregister_ftrace_function(&ftrace_profile_ops);
  659. }
  660. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  661. static ssize_t
  662. ftrace_profile_write(struct file *filp, const char __user *ubuf,
  663. size_t cnt, loff_t *ppos)
  664. {
  665. unsigned long val;
  666. int ret;
  667. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  668. if (ret)
  669. return ret;
  670. val = !!val;
  671. mutex_lock(&ftrace_profile_lock);
  672. if (ftrace_profile_enabled ^ val) {
  673. if (val) {
  674. ret = ftrace_profile_init();
  675. if (ret < 0) {
  676. cnt = ret;
  677. goto out;
  678. }
  679. ret = register_ftrace_profiler();
  680. if (ret < 0) {
  681. cnt = ret;
  682. goto out;
  683. }
  684. ftrace_profile_enabled = 1;
  685. } else {
  686. ftrace_profile_enabled = 0;
  687. /*
  688. * unregister_ftrace_profiler calls stop_machine
  689. * so this acts like an synchronize_sched.
  690. */
  691. unregister_ftrace_profiler();
  692. }
  693. }
  694. out:
  695. mutex_unlock(&ftrace_profile_lock);
  696. *ppos += cnt;
  697. return cnt;
  698. }
  699. static ssize_t
  700. ftrace_profile_read(struct file *filp, char __user *ubuf,
  701. size_t cnt, loff_t *ppos)
  702. {
  703. char buf[64]; /* big enough to hold a number */
  704. int r;
  705. r = sprintf(buf, "%u\n", ftrace_profile_enabled);
  706. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  707. }
  708. static const struct file_operations ftrace_profile_fops = {
  709. .open = tracing_open_generic,
  710. .read = ftrace_profile_read,
  711. .write = ftrace_profile_write,
  712. .llseek = default_llseek,
  713. };
  714. /* used to initialize the real stat files */
  715. static struct tracer_stat function_stats __initdata = {
  716. .name = "functions",
  717. .stat_start = function_stat_start,
  718. .stat_next = function_stat_next,
  719. .stat_cmp = function_stat_cmp,
  720. .stat_headers = function_stat_headers,
  721. .stat_show = function_stat_show
  722. };
  723. static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
  724. {
  725. struct ftrace_profile_stat *stat;
  726. struct dentry *entry;
  727. char *name;
  728. int ret;
  729. int cpu;
  730. for_each_possible_cpu(cpu) {
  731. stat = &per_cpu(ftrace_profile_stats, cpu);
  732. /* allocate enough for function name + cpu number */
  733. name = kmalloc(32, GFP_KERNEL);
  734. if (!name) {
  735. /*
  736. * The files created are permanent, if something happens
  737. * we still do not free memory.
  738. */
  739. WARN(1,
  740. "Could not allocate stat file for cpu %d\n",
  741. cpu);
  742. return;
  743. }
  744. stat->stat = function_stats;
  745. snprintf(name, 32, "function%d", cpu);
  746. stat->stat.name = name;
  747. ret = register_stat_tracer(&stat->stat);
  748. if (ret) {
  749. WARN(1,
  750. "Could not register function stat for cpu %d\n",
  751. cpu);
  752. kfree(name);
  753. return;
  754. }
  755. }
  756. entry = debugfs_create_file("function_profile_enabled", 0644,
  757. d_tracer, NULL, &ftrace_profile_fops);
  758. if (!entry)
  759. pr_warning("Could not create debugfs "
  760. "'function_profile_enabled' entry\n");
  761. }
  762. #else /* CONFIG_FUNCTION_PROFILER */
  763. static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
  764. {
  765. }
  766. #endif /* CONFIG_FUNCTION_PROFILER */
  767. static struct pid * const ftrace_swapper_pid = &init_struct_pid;
  768. #ifdef CONFIG_DYNAMIC_FTRACE
  769. #ifndef CONFIG_FTRACE_MCOUNT_RECORD
  770. # error Dynamic ftrace depends on MCOUNT_RECORD
  771. #endif
  772. static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
  773. struct ftrace_func_probe {
  774. struct hlist_node node;
  775. struct ftrace_probe_ops *ops;
  776. unsigned long flags;
  777. unsigned long ip;
  778. void *data;
  779. struct rcu_head rcu;
  780. };
  781. struct ftrace_func_entry {
  782. struct hlist_node hlist;
  783. unsigned long ip;
  784. };
  785. struct ftrace_hash {
  786. unsigned long size_bits;
  787. struct hlist_head *buckets;
  788. unsigned long count;
  789. struct rcu_head rcu;
  790. };
  791. /*
  792. * We make these constant because no one should touch them,
  793. * but they are used as the default "empty hash", to avoid allocating
  794. * it all the time. These are in a read only section such that if
  795. * anyone does try to modify it, it will cause an exception.
  796. */
  797. static const struct hlist_head empty_buckets[1];
  798. static const struct ftrace_hash empty_hash = {
  799. .buckets = (struct hlist_head *)empty_buckets,
  800. };
  801. #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
  802. static struct ftrace_ops global_ops = {
  803. .func = ftrace_stub,
  804. .notrace_hash = EMPTY_HASH,
  805. .filter_hash = EMPTY_HASH,
  806. };
  807. static DEFINE_MUTEX(ftrace_regex_lock);
  808. struct ftrace_page {
  809. struct ftrace_page *next;
  810. struct dyn_ftrace *records;
  811. int index;
  812. int size;
  813. };
  814. static struct ftrace_page *ftrace_new_pgs;
  815. #define ENTRY_SIZE sizeof(struct dyn_ftrace)
  816. #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
  817. /* estimate from running different kernels */
  818. #define NR_TO_INIT 10000
  819. static struct ftrace_page *ftrace_pages_start;
  820. static struct ftrace_page *ftrace_pages;
  821. static struct ftrace_func_entry *
  822. ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
  823. {
  824. unsigned long key;
  825. struct ftrace_func_entry *entry;
  826. struct hlist_head *hhd;
  827. struct hlist_node *n;
  828. if (!hash->count)
  829. return NULL;
  830. if (hash->size_bits > 0)
  831. key = hash_long(ip, hash->size_bits);
  832. else
  833. key = 0;
  834. hhd = &hash->buckets[key];
  835. hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
  836. if (entry->ip == ip)
  837. return entry;
  838. }
  839. return NULL;
  840. }
  841. static void __add_hash_entry(struct ftrace_hash *hash,
  842. struct ftrace_func_entry *entry)
  843. {
  844. struct hlist_head *hhd;
  845. unsigned long key;
  846. if (hash->size_bits)
  847. key = hash_long(entry->ip, hash->size_bits);
  848. else
  849. key = 0;
  850. hhd = &hash->buckets[key];
  851. hlist_add_head(&entry->hlist, hhd);
  852. hash->count++;
  853. }
  854. static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
  855. {
  856. struct ftrace_func_entry *entry;
  857. entry = kmalloc(sizeof(*entry), GFP_KERNEL);
  858. if (!entry)
  859. return -ENOMEM;
  860. entry->ip = ip;
  861. __add_hash_entry(hash, entry);
  862. return 0;
  863. }
  864. static void
  865. free_hash_entry(struct ftrace_hash *hash,
  866. struct ftrace_func_entry *entry)
  867. {
  868. hlist_del(&entry->hlist);
  869. kfree(entry);
  870. hash->count--;
  871. }
  872. static void
  873. remove_hash_entry(struct ftrace_hash *hash,
  874. struct ftrace_func_entry *entry)
  875. {
  876. hlist_del(&entry->hlist);
  877. hash->count--;
  878. }
  879. static void ftrace_hash_clear(struct ftrace_hash *hash)
  880. {
  881. struct hlist_head *hhd;
  882. struct hlist_node *tp, *tn;
  883. struct ftrace_func_entry *entry;
  884. int size = 1 << hash->size_bits;
  885. int i;
  886. if (!hash->count)
  887. return;
  888. for (i = 0; i < size; i++) {
  889. hhd = &hash->buckets[i];
  890. hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
  891. free_hash_entry(hash, entry);
  892. }
  893. FTRACE_WARN_ON(hash->count);
  894. }
  895. static void free_ftrace_hash(struct ftrace_hash *hash)
  896. {
  897. if (!hash || hash == EMPTY_HASH)
  898. return;
  899. ftrace_hash_clear(hash);
  900. kfree(hash->buckets);
  901. kfree(hash);
  902. }
  903. static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
  904. {
  905. struct ftrace_hash *hash;
  906. hash = container_of(rcu, struct ftrace_hash, rcu);
  907. free_ftrace_hash(hash);
  908. }
  909. static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
  910. {
  911. if (!hash || hash == EMPTY_HASH)
  912. return;
  913. call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
  914. }
  915. static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
  916. {
  917. struct ftrace_hash *hash;
  918. int size;
  919. hash = kzalloc(sizeof(*hash), GFP_KERNEL);
  920. if (!hash)
  921. return NULL;
  922. size = 1 << size_bits;
  923. hash->buckets = kzalloc(sizeof(*hash->buckets) * size, GFP_KERNEL);
  924. if (!hash->buckets) {
  925. kfree(hash);
  926. return NULL;
  927. }
  928. hash->size_bits = size_bits;
  929. return hash;
  930. }
  931. static struct ftrace_hash *
  932. alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
  933. {
  934. struct ftrace_func_entry *entry;
  935. struct ftrace_hash *new_hash;
  936. struct hlist_node *tp;
  937. int size;
  938. int ret;
  939. int i;
  940. new_hash = alloc_ftrace_hash(size_bits);
  941. if (!new_hash)
  942. return NULL;
  943. /* Empty hash? */
  944. if (!hash || !hash->count)
  945. return new_hash;
  946. size = 1 << hash->size_bits;
  947. for (i = 0; i < size; i++) {
  948. hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
  949. ret = add_hash_entry(new_hash, entry->ip);
  950. if (ret < 0)
  951. goto free_hash;
  952. }
  953. }
  954. FTRACE_WARN_ON(new_hash->count != hash->count);
  955. return new_hash;
  956. free_hash:
  957. free_ftrace_hash(new_hash);
  958. return NULL;
  959. }
  960. static void
  961. ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
  962. static void
  963. ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
  964. static int
  965. ftrace_hash_move(struct ftrace_ops *ops, int enable,
  966. struct ftrace_hash **dst, struct ftrace_hash *src)
  967. {
  968. struct ftrace_func_entry *entry;
  969. struct hlist_node *tp, *tn;
  970. struct hlist_head *hhd;
  971. struct ftrace_hash *old_hash;
  972. struct ftrace_hash *new_hash;
  973. unsigned long key;
  974. int size = src->count;
  975. int bits = 0;
  976. int ret;
  977. int i;
  978. /*
  979. * Remove the current set, update the hash and add
  980. * them back.
  981. */
  982. ftrace_hash_rec_disable(ops, enable);
  983. /*
  984. * If the new source is empty, just free dst and assign it
  985. * the empty_hash.
  986. */
  987. if (!src->count) {
  988. free_ftrace_hash_rcu(*dst);
  989. rcu_assign_pointer(*dst, EMPTY_HASH);
  990. /* still need to update the function records */
  991. ret = 0;
  992. goto out;
  993. }
  994. /*
  995. * Make the hash size about 1/2 the # found
  996. */
  997. for (size /= 2; size; size >>= 1)
  998. bits++;
  999. /* Don't allocate too much */
  1000. if (bits > FTRACE_HASH_MAX_BITS)
  1001. bits = FTRACE_HASH_MAX_BITS;
  1002. ret = -ENOMEM;
  1003. new_hash = alloc_ftrace_hash(bits);
  1004. if (!new_hash)
  1005. goto out;
  1006. size = 1 << src->size_bits;
  1007. for (i = 0; i < size; i++) {
  1008. hhd = &src->buckets[i];
  1009. hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
  1010. if (bits > 0)
  1011. key = hash_long(entry->ip, bits);
  1012. else
  1013. key = 0;
  1014. remove_hash_entry(src, entry);
  1015. __add_hash_entry(new_hash, entry);
  1016. }
  1017. }
  1018. old_hash = *dst;
  1019. rcu_assign_pointer(*dst, new_hash);
  1020. free_ftrace_hash_rcu(old_hash);
  1021. ret = 0;
  1022. out:
  1023. /*
  1024. * Enable regardless of ret:
  1025. * On success, we enable the new hash.
  1026. * On failure, we re-enable the original hash.
  1027. */
  1028. ftrace_hash_rec_enable(ops, enable);
  1029. return ret;
  1030. }
  1031. /*
  1032. * Test the hashes for this ops to see if we want to call
  1033. * the ops->func or not.
  1034. *
  1035. * It's a match if the ip is in the ops->filter_hash or
  1036. * the filter_hash does not exist or is empty,
  1037. * AND
  1038. * the ip is not in the ops->notrace_hash.
  1039. *
  1040. * This needs to be called with preemption disabled as
  1041. * the hashes are freed with call_rcu_sched().
  1042. */
  1043. static int
  1044. ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
  1045. {
  1046. struct ftrace_hash *filter_hash;
  1047. struct ftrace_hash *notrace_hash;
  1048. int ret;
  1049. filter_hash = rcu_dereference_raw(ops->filter_hash);
  1050. notrace_hash = rcu_dereference_raw(ops->notrace_hash);
  1051. if ((!filter_hash || !filter_hash->count ||
  1052. ftrace_lookup_ip(filter_hash, ip)) &&
  1053. (!notrace_hash || !notrace_hash->count ||
  1054. !ftrace_lookup_ip(notrace_hash, ip)))
  1055. ret = 1;
  1056. else
  1057. ret = 0;
  1058. return ret;
  1059. }
  1060. /*
  1061. * This is a double for. Do not use 'break' to break out of the loop,
  1062. * you must use a goto.
  1063. */
  1064. #define do_for_each_ftrace_rec(pg, rec) \
  1065. for (pg = ftrace_pages_start; pg; pg = pg->next) { \
  1066. int _____i; \
  1067. for (_____i = 0; _____i < pg->index; _____i++) { \
  1068. rec = &pg->records[_____i];
  1069. #define while_for_each_ftrace_rec() \
  1070. } \
  1071. }
  1072. /**
  1073. * ftrace_location - return true if the ip giving is a traced location
  1074. * @ip: the instruction pointer to check
  1075. *
  1076. * Returns 1 if @ip given is a pointer to a ftrace location.
  1077. * That is, the instruction that is either a NOP or call to
  1078. * the function tracer. It checks the ftrace internal tables to
  1079. * determine if the address belongs or not.
  1080. */
  1081. int ftrace_location(unsigned long ip)
  1082. {
  1083. struct ftrace_page *pg;
  1084. struct dyn_ftrace *rec;
  1085. do_for_each_ftrace_rec(pg, rec) {
  1086. if (rec->ip == ip)
  1087. return 1;
  1088. } while_for_each_ftrace_rec();
  1089. return 0;
  1090. }
  1091. static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
  1092. int filter_hash,
  1093. bool inc)
  1094. {
  1095. struct ftrace_hash *hash;
  1096. struct ftrace_hash *other_hash;
  1097. struct ftrace_page *pg;
  1098. struct dyn_ftrace *rec;
  1099. int count = 0;
  1100. int all = 0;
  1101. /* Only update if the ops has been registered */
  1102. if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
  1103. return;
  1104. /*
  1105. * In the filter_hash case:
  1106. * If the count is zero, we update all records.
  1107. * Otherwise we just update the items in the hash.
  1108. *
  1109. * In the notrace_hash case:
  1110. * We enable the update in the hash.
  1111. * As disabling notrace means enabling the tracing,
  1112. * and enabling notrace means disabling, the inc variable
  1113. * gets inversed.
  1114. */
  1115. if (filter_hash) {
  1116. hash = ops->filter_hash;
  1117. other_hash = ops->notrace_hash;
  1118. if (!hash || !hash->count)
  1119. all = 1;
  1120. } else {
  1121. inc = !inc;
  1122. hash = ops->notrace_hash;
  1123. other_hash = ops->filter_hash;
  1124. /*
  1125. * If the notrace hash has no items,
  1126. * then there's nothing to do.
  1127. */
  1128. if (hash && !hash->count)
  1129. return;
  1130. }
  1131. do_for_each_ftrace_rec(pg, rec) {
  1132. int in_other_hash = 0;
  1133. int in_hash = 0;
  1134. int match = 0;
  1135. if (all) {
  1136. /*
  1137. * Only the filter_hash affects all records.
  1138. * Update if the record is not in the notrace hash.
  1139. */
  1140. if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
  1141. match = 1;
  1142. } else {
  1143. in_hash = hash && !!ftrace_lookup_ip(hash, rec->ip);
  1144. in_other_hash = other_hash && !!ftrace_lookup_ip(other_hash, rec->ip);
  1145. /*
  1146. *
  1147. */
  1148. if (filter_hash && in_hash && !in_other_hash)
  1149. match = 1;
  1150. else if (!filter_hash && in_hash &&
  1151. (in_other_hash || !other_hash->count))
  1152. match = 1;
  1153. }
  1154. if (!match)
  1155. continue;
  1156. if (inc) {
  1157. rec->flags++;
  1158. if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
  1159. return;
  1160. } else {
  1161. if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
  1162. return;
  1163. rec->flags--;
  1164. }
  1165. count++;
  1166. /* Shortcut, if we handled all records, we are done. */
  1167. if (!all && count == hash->count)
  1168. return;
  1169. } while_for_each_ftrace_rec();
  1170. }
  1171. static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
  1172. int filter_hash)
  1173. {
  1174. __ftrace_hash_rec_update(ops, filter_hash, 0);
  1175. }
  1176. static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
  1177. int filter_hash)
  1178. {
  1179. __ftrace_hash_rec_update(ops, filter_hash, 1);
  1180. }
  1181. static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
  1182. {
  1183. if (ftrace_pages->index == ftrace_pages->size) {
  1184. /* We should have allocated enough */
  1185. if (WARN_ON(!ftrace_pages->next))
  1186. return NULL;
  1187. ftrace_pages = ftrace_pages->next;
  1188. }
  1189. return &ftrace_pages->records[ftrace_pages->index++];
  1190. }
  1191. static struct dyn_ftrace *
  1192. ftrace_record_ip(unsigned long ip)
  1193. {
  1194. struct dyn_ftrace *rec;
  1195. if (ftrace_disabled)
  1196. return NULL;
  1197. rec = ftrace_alloc_dyn_node(ip);
  1198. if (!rec)
  1199. return NULL;
  1200. rec->ip = ip;
  1201. return rec;
  1202. }
  1203. static void print_ip_ins(const char *fmt, unsigned char *p)
  1204. {
  1205. int i;
  1206. printk(KERN_CONT "%s", fmt);
  1207. for (i = 0; i < MCOUNT_INSN_SIZE; i++)
  1208. printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
  1209. }
  1210. /**
  1211. * ftrace_bug - report and shutdown function tracer
  1212. * @failed: The failed type (EFAULT, EINVAL, EPERM)
  1213. * @ip: The address that failed
  1214. *
  1215. * The arch code that enables or disables the function tracing
  1216. * can call ftrace_bug() when it has detected a problem in
  1217. * modifying the code. @failed should be one of either:
  1218. * EFAULT - if the problem happens on reading the @ip address
  1219. * EINVAL - if what is read at @ip is not what was expected
  1220. * EPERM - if the problem happens on writting to the @ip address
  1221. */
  1222. void ftrace_bug(int failed, unsigned long ip)
  1223. {
  1224. switch (failed) {
  1225. case -EFAULT:
  1226. FTRACE_WARN_ON_ONCE(1);
  1227. pr_info("ftrace faulted on modifying ");
  1228. print_ip_sym(ip);
  1229. break;
  1230. case -EINVAL:
  1231. FTRACE_WARN_ON_ONCE(1);
  1232. pr_info("ftrace failed to modify ");
  1233. print_ip_sym(ip);
  1234. print_ip_ins(" actual: ", (unsigned char *)ip);
  1235. printk(KERN_CONT "\n");
  1236. break;
  1237. case -EPERM:
  1238. FTRACE_WARN_ON_ONCE(1);
  1239. pr_info("ftrace faulted on writing ");
  1240. print_ip_sym(ip);
  1241. break;
  1242. default:
  1243. FTRACE_WARN_ON_ONCE(1);
  1244. pr_info("ftrace faulted on unknown error ");
  1245. print_ip_sym(ip);
  1246. }
  1247. }
  1248. /* Return 1 if the address range is reserved for ftrace */
  1249. int ftrace_text_reserved(void *start, void *end)
  1250. {
  1251. struct dyn_ftrace *rec;
  1252. struct ftrace_page *pg;
  1253. do_for_each_ftrace_rec(pg, rec) {
  1254. if (rec->ip <= (unsigned long)end &&
  1255. rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
  1256. return 1;
  1257. } while_for_each_ftrace_rec();
  1258. return 0;
  1259. }
  1260. static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
  1261. {
  1262. unsigned long flag = 0UL;
  1263. /*
  1264. * If we are updating calls:
  1265. *
  1266. * If the record has a ref count, then we need to enable it
  1267. * because someone is using it.
  1268. *
  1269. * Otherwise we make sure its disabled.
  1270. *
  1271. * If we are disabling calls, then disable all records that
  1272. * are enabled.
  1273. */
  1274. if (enable && (rec->flags & ~FTRACE_FL_MASK))
  1275. flag = FTRACE_FL_ENABLED;
  1276. /* If the state of this record hasn't changed, then do nothing */
  1277. if ((rec->flags & FTRACE_FL_ENABLED) == flag)
  1278. return FTRACE_UPDATE_IGNORE;
  1279. if (flag) {
  1280. if (update)
  1281. rec->flags |= FTRACE_FL_ENABLED;
  1282. return FTRACE_UPDATE_MAKE_CALL;
  1283. }
  1284. if (update)
  1285. rec->flags &= ~FTRACE_FL_ENABLED;
  1286. return FTRACE_UPDATE_MAKE_NOP;
  1287. }
  1288. /**
  1289. * ftrace_update_record, set a record that now is tracing or not
  1290. * @rec: the record to update
  1291. * @enable: set to 1 if the record is tracing, zero to force disable
  1292. *
  1293. * The records that represent all functions that can be traced need
  1294. * to be updated when tracing has been enabled.
  1295. */
  1296. int ftrace_update_record(struct dyn_ftrace *rec, int enable)
  1297. {
  1298. return ftrace_check_record(rec, enable, 1);
  1299. }
  1300. /**
  1301. * ftrace_test_record, check if the record has been enabled or not
  1302. * @rec: the record to test
  1303. * @enable: set to 1 to check if enabled, 0 if it is disabled
  1304. *
  1305. * The arch code may need to test if a record is already set to
  1306. * tracing to determine how to modify the function code that it
  1307. * represents.
  1308. */
  1309. int ftrace_test_record(struct dyn_ftrace *rec, int enable)
  1310. {
  1311. return ftrace_check_record(rec, enable, 0);
  1312. }
  1313. static int
  1314. __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
  1315. {
  1316. unsigned long ftrace_addr;
  1317. int ret;
  1318. ftrace_addr = (unsigned long)FTRACE_ADDR;
  1319. ret = ftrace_update_record(rec, enable);
  1320. switch (ret) {
  1321. case FTRACE_UPDATE_IGNORE:
  1322. return 0;
  1323. case FTRACE_UPDATE_MAKE_CALL:
  1324. return ftrace_make_call(rec, ftrace_addr);
  1325. case FTRACE_UPDATE_MAKE_NOP:
  1326. return ftrace_make_nop(NULL, rec, ftrace_addr);
  1327. }
  1328. return -1; /* unknow ftrace bug */
  1329. }
  1330. static void ftrace_replace_code(int update)
  1331. {
  1332. struct dyn_ftrace *rec;
  1333. struct ftrace_page *pg;
  1334. int failed;
  1335. if (unlikely(ftrace_disabled))
  1336. return;
  1337. do_for_each_ftrace_rec(pg, rec) {
  1338. failed = __ftrace_replace_code(rec, update);
  1339. if (failed) {
  1340. ftrace_bug(failed, rec->ip);
  1341. /* Stop processing */
  1342. return;
  1343. }
  1344. } while_for_each_ftrace_rec();
  1345. }
  1346. struct ftrace_rec_iter {
  1347. struct ftrace_page *pg;
  1348. int index;
  1349. };
  1350. /**
  1351. * ftrace_rec_iter_start, start up iterating over traced functions
  1352. *
  1353. * Returns an iterator handle that is used to iterate over all
  1354. * the records that represent address locations where functions
  1355. * are traced.
  1356. *
  1357. * May return NULL if no records are available.
  1358. */
  1359. struct ftrace_rec_iter *ftrace_rec_iter_start(void)
  1360. {
  1361. /*
  1362. * We only use a single iterator.
  1363. * Protected by the ftrace_lock mutex.
  1364. */
  1365. static struct ftrace_rec_iter ftrace_rec_iter;
  1366. struct ftrace_rec_iter *iter = &ftrace_rec_iter;
  1367. iter->pg = ftrace_pages_start;
  1368. iter->index = 0;
  1369. /* Could have empty pages */
  1370. while (iter->pg && !iter->pg->index)
  1371. iter->pg = iter->pg->next;
  1372. if (!iter->pg)
  1373. return NULL;
  1374. return iter;
  1375. }
  1376. /**
  1377. * ftrace_rec_iter_next, get the next record to process.
  1378. * @iter: The handle to the iterator.
  1379. *
  1380. * Returns the next iterator after the given iterator @iter.
  1381. */
  1382. struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
  1383. {
  1384. iter->index++;
  1385. if (iter->index >= iter->pg->index) {
  1386. iter->pg = iter->pg->next;
  1387. iter->index = 0;
  1388. /* Could have empty pages */
  1389. while (iter->pg && !iter->pg->index)
  1390. iter->pg = iter->pg->next;
  1391. }
  1392. if (!iter->pg)
  1393. return NULL;
  1394. return iter;
  1395. }
  1396. /**
  1397. * ftrace_rec_iter_record, get the record at the iterator location
  1398. * @iter: The current iterator location
  1399. *
  1400. * Returns the record that the current @iter is at.
  1401. */
  1402. struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
  1403. {
  1404. return &iter->pg->records[iter->index];
  1405. }
  1406. static int
  1407. ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
  1408. {
  1409. unsigned long ip;
  1410. int ret;
  1411. ip = rec->ip;
  1412. if (unlikely(ftrace_disabled))
  1413. return 0;
  1414. ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
  1415. if (ret) {
  1416. ftrace_bug(ret, ip);
  1417. return 0;
  1418. }
  1419. return 1;
  1420. }
  1421. /*
  1422. * archs can override this function if they must do something
  1423. * before the modifying code is performed.
  1424. */
  1425. int __weak ftrace_arch_code_modify_prepare(void)
  1426. {
  1427. return 0;
  1428. }
  1429. /*
  1430. * archs can override this function if they must do something
  1431. * after the modifying code is performed.
  1432. */
  1433. int __weak ftrace_arch_code_modify_post_process(void)
  1434. {
  1435. return 0;
  1436. }
  1437. static int __ftrace_modify_code(void *data)
  1438. {
  1439. int *command = data;
  1440. if (*command & FTRACE_UPDATE_CALLS)
  1441. ftrace_replace_code(1);
  1442. else if (*command & FTRACE_DISABLE_CALLS)
  1443. ftrace_replace_code(0);
  1444. if (*command & FTRACE_UPDATE_TRACE_FUNC)
  1445. ftrace_update_ftrace_func(ftrace_trace_function);
  1446. if (*command & FTRACE_START_FUNC_RET)
  1447. ftrace_enable_ftrace_graph_caller();
  1448. else if (*command & FTRACE_STOP_FUNC_RET)
  1449. ftrace_disable_ftrace_graph_caller();
  1450. return 0;
  1451. }
  1452. /**
  1453. * ftrace_run_stop_machine, go back to the stop machine method
  1454. * @command: The command to tell ftrace what to do
  1455. *
  1456. * If an arch needs to fall back to the stop machine method, the
  1457. * it can call this function.
  1458. */
  1459. void ftrace_run_stop_machine(int command)
  1460. {
  1461. stop_machine(__ftrace_modify_code, &command, NULL);
  1462. }
  1463. /**
  1464. * arch_ftrace_update_code, modify the code to trace or not trace
  1465. * @command: The command that needs to be done
  1466. *
  1467. * Archs can override this function if it does not need to
  1468. * run stop_machine() to modify code.
  1469. */
  1470. void __weak arch_ftrace_update_code(int command)
  1471. {
  1472. ftrace_run_stop_machine(command);
  1473. }
  1474. static void ftrace_run_update_code(int command)
  1475. {
  1476. int ret;
  1477. ret = ftrace_arch_code_modify_prepare();
  1478. FTRACE_WARN_ON(ret);
  1479. if (ret)
  1480. return;
  1481. /*
  1482. * Do not call function tracer while we update the code.
  1483. * We are in stop machine.
  1484. */
  1485. function_trace_stop++;
  1486. /*
  1487. * By default we use stop_machine() to modify the code.
  1488. * But archs can do what ever they want as long as it
  1489. * is safe. The stop_machine() is the safest, but also
  1490. * produces the most overhead.
  1491. */
  1492. arch_ftrace_update_code(command);
  1493. #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
  1494. /*
  1495. * For archs that call ftrace_test_stop_func(), we must
  1496. * wait till after we update all the function callers
  1497. * before we update the callback. This keeps different
  1498. * ops that record different functions from corrupting
  1499. * each other.
  1500. */
  1501. __ftrace_trace_function = __ftrace_trace_function_delay;
  1502. #endif
  1503. function_trace_stop--;
  1504. ret = ftrace_arch_code_modify_post_process();
  1505. FTRACE_WARN_ON(ret);
  1506. }
  1507. static ftrace_func_t saved_ftrace_func;
  1508. static int ftrace_start_up;
  1509. static int global_start_up;
  1510. static void ftrace_startup_enable(int command)
  1511. {
  1512. if (saved_ftrace_func != ftrace_trace_function) {
  1513. saved_ftrace_func = ftrace_trace_function;
  1514. command |= FTRACE_UPDATE_TRACE_FUNC;
  1515. }
  1516. if (!command || !ftrace_enabled)
  1517. return;
  1518. ftrace_run_update_code(command);
  1519. }
  1520. static int ftrace_startup(struct ftrace_ops *ops, int command)
  1521. {
  1522. bool hash_enable = true;
  1523. if (unlikely(ftrace_disabled))
  1524. return -ENODEV;
  1525. ftrace_start_up++;
  1526. command |= FTRACE_UPDATE_CALLS;
  1527. /* ops marked global share the filter hashes */
  1528. if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
  1529. ops = &global_ops;
  1530. /* Don't update hash if global is already set */
  1531. if (global_start_up)
  1532. hash_enable = false;
  1533. global_start_up++;
  1534. }
  1535. ops->flags |= FTRACE_OPS_FL_ENABLED;
  1536. if (hash_enable)
  1537. ftrace_hash_rec_enable(ops, 1);
  1538. ftrace_startup_enable(command);
  1539. return 0;
  1540. }
  1541. static void ftrace_shutdown(struct ftrace_ops *ops, int command)
  1542. {
  1543. bool hash_disable = true;
  1544. if (unlikely(ftrace_disabled))
  1545. return;
  1546. ftrace_start_up--;
  1547. /*
  1548. * Just warn in case of unbalance, no need to kill ftrace, it's not
  1549. * critical but the ftrace_call callers may be never nopped again after
  1550. * further ftrace uses.
  1551. */
  1552. WARN_ON_ONCE(ftrace_start_up < 0);
  1553. if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
  1554. ops = &global_ops;
  1555. global_start_up--;
  1556. WARN_ON_ONCE(global_start_up < 0);
  1557. /* Don't update hash if global still has users */
  1558. if (global_start_up) {
  1559. WARN_ON_ONCE(!ftrace_start_up);
  1560. hash_disable = false;
  1561. }
  1562. }
  1563. if (hash_disable)
  1564. ftrace_hash_rec_disable(ops, 1);
  1565. if (ops != &global_ops || !global_start_up)
  1566. ops->flags &= ~FTRACE_OPS_FL_ENABLED;
  1567. command |= FTRACE_UPDATE_CALLS;
  1568. if (saved_ftrace_func != ftrace_trace_function) {
  1569. saved_ftrace_func = ftrace_trace_function;
  1570. command |= FTRACE_UPDATE_TRACE_FUNC;
  1571. }
  1572. if (!command || !ftrace_enabled)
  1573. return;
  1574. ftrace_run_update_code(command);
  1575. }
  1576. static void ftrace_startup_sysctl(void)
  1577. {
  1578. if (unlikely(ftrace_disabled))
  1579. return;
  1580. /* Force update next time */
  1581. saved_ftrace_func = NULL;
  1582. /* ftrace_start_up is true if we want ftrace running */
  1583. if (ftrace_start_up)
  1584. ftrace_run_update_code(FTRACE_UPDATE_CALLS);
  1585. }
  1586. static void ftrace_shutdown_sysctl(void)
  1587. {
  1588. if (unlikely(ftrace_disabled))
  1589. return;
  1590. /* ftrace_start_up is true if ftrace is running */
  1591. if (ftrace_start_up)
  1592. ftrace_run_update_code(FTRACE_DISABLE_CALLS);
  1593. }
  1594. static cycle_t ftrace_update_time;
  1595. static unsigned long ftrace_update_cnt;
  1596. unsigned long ftrace_update_tot_cnt;
  1597. static int ops_traces_mod(struct ftrace_ops *ops)
  1598. {
  1599. struct ftrace_hash *hash;
  1600. hash = ops->filter_hash;
  1601. return !!(!hash || !hash->count);
  1602. }
  1603. static int ftrace_update_code(struct module *mod)
  1604. {
  1605. struct ftrace_page *pg;
  1606. struct dyn_ftrace *p;
  1607. cycle_t start, stop;
  1608. unsigned long ref = 0;
  1609. int i;
  1610. /*
  1611. * When adding a module, we need to check if tracers are
  1612. * currently enabled and if they are set to trace all functions.
  1613. * If they are, we need to enable the module functions as well
  1614. * as update the reference counts for those function records.
  1615. */
  1616. if (mod) {
  1617. struct ftrace_ops *ops;
  1618. for (ops = ftrace_ops_list;
  1619. ops != &ftrace_list_end; ops = ops->next) {
  1620. if (ops->flags & FTRACE_OPS_FL_ENABLED &&
  1621. ops_traces_mod(ops))
  1622. ref++;
  1623. }
  1624. }
  1625. start = ftrace_now(raw_smp_processor_id());
  1626. ftrace_update_cnt = 0;
  1627. for (pg = ftrace_new_pgs; pg; pg = pg->next) {
  1628. for (i = 0; i < pg->index; i++) {
  1629. /* If something went wrong, bail without enabling anything */
  1630. if (unlikely(ftrace_disabled))
  1631. return -1;
  1632. p = &pg->records[i];
  1633. p->flags = ref;
  1634. /*
  1635. * Do the initial record conversion from mcount jump
  1636. * to the NOP instructions.
  1637. */
  1638. if (!ftrace_code_disable(mod, p))
  1639. break;
  1640. ftrace_update_cnt++;
  1641. /*
  1642. * If the tracing is enabled, go ahead and enable the record.
  1643. *
  1644. * The reason not to enable the record immediatelly is the
  1645. * inherent check of ftrace_make_nop/ftrace_make_call for
  1646. * correct previous instructions. Making first the NOP
  1647. * conversion puts the module to the correct state, thus
  1648. * passing the ftrace_make_call check.
  1649. */
  1650. if (ftrace_start_up && ref) {
  1651. int failed = __ftrace_replace_code(p, 1);
  1652. if (failed)
  1653. ftrace_bug(failed, p->ip);
  1654. }
  1655. }
  1656. }
  1657. ftrace_new_pgs = NULL;
  1658. stop = ftrace_now(raw_smp_processor_id());
  1659. ftrace_update_time = stop - start;
  1660. ftrace_update_tot_cnt += ftrace_update_cnt;
  1661. return 0;
  1662. }
  1663. static int ftrace_allocate_records(struct ftrace_page *pg, int count)
  1664. {
  1665. int order;
  1666. int cnt;
  1667. if (WARN_ON(!count))
  1668. return -EINVAL;
  1669. order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
  1670. /*
  1671. * We want to fill as much as possible. No more than a page
  1672. * may be empty.
  1673. */
  1674. while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
  1675. order--;
  1676. again:
  1677. pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
  1678. if (!pg->records) {
  1679. /* if we can't allocate this size, try something smaller */
  1680. if (!order)
  1681. return -ENOMEM;
  1682. order >>= 1;
  1683. goto again;
  1684. }
  1685. cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
  1686. pg->size = cnt;
  1687. if (cnt > count)
  1688. cnt = count;
  1689. return cnt;
  1690. }
  1691. static struct ftrace_page *
  1692. ftrace_allocate_pages(unsigned long num_to_init)
  1693. {
  1694. struct ftrace_page *start_pg;
  1695. struct ftrace_page *pg;
  1696. int order;
  1697. int cnt;
  1698. if (!num_to_init)
  1699. return 0;
  1700. start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
  1701. if (!pg)
  1702. return NULL;
  1703. /*
  1704. * Try to allocate as much as possible in one continues
  1705. * location that fills in all of the space. We want to
  1706. * waste as little space as possible.
  1707. */
  1708. for (;;) {
  1709. cnt = ftrace_allocate_records(pg, num_to_init);
  1710. if (cnt < 0)
  1711. goto free_pages;
  1712. num_to_init -= cnt;
  1713. if (!num_to_init)
  1714. break;
  1715. pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
  1716. if (!pg->next)
  1717. goto free_pages;
  1718. pg = pg->next;
  1719. }
  1720. return start_pg;
  1721. free_pages:
  1722. while (start_pg) {
  1723. order = get_count_order(pg->size / ENTRIES_PER_PAGE);
  1724. free_pages((unsigned long)pg->records, order);
  1725. start_pg = pg->next;
  1726. kfree(pg);
  1727. pg = start_pg;
  1728. }
  1729. pr_info("ftrace: FAILED to allocate memory for functions\n");
  1730. return NULL;
  1731. }
  1732. static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
  1733. {
  1734. int cnt;
  1735. if (!num_to_init) {
  1736. pr_info("ftrace: No functions to be traced?\n");
  1737. return -1;
  1738. }
  1739. cnt = num_to_init / ENTRIES_PER_PAGE;
  1740. pr_info("ftrace: allocating %ld entries in %d pages\n",
  1741. num_to_init, cnt + 1);
  1742. return 0;
  1743. }
  1744. enum {
  1745. FTRACE_ITER_FILTER = (1 << 0),
  1746. FTRACE_ITER_NOTRACE = (1 << 1),
  1747. FTRACE_ITER_PRINTALL = (1 << 2),
  1748. FTRACE_ITER_HASH = (1 << 3),
  1749. FTRACE_ITER_ENABLED = (1 << 4),
  1750. };
  1751. #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
  1752. struct ftrace_iterator {
  1753. loff_t pos;
  1754. loff_t func_pos;
  1755. struct ftrace_page *pg;
  1756. struct dyn_ftrace *func;
  1757. struct ftrace_func_probe *probe;
  1758. struct trace_parser parser;
  1759. struct ftrace_hash *hash;
  1760. struct ftrace_ops *ops;
  1761. int hidx;
  1762. int idx;
  1763. unsigned flags;
  1764. };
  1765. static void *
  1766. t_hash_next(struct seq_file *m, loff_t *pos)
  1767. {
  1768. struct ftrace_iterator *iter = m->private;
  1769. struct hlist_node *hnd = NULL;
  1770. struct hlist_head *hhd;
  1771. (*pos)++;
  1772. iter->pos = *pos;
  1773. if (iter->probe)
  1774. hnd = &iter->probe->node;
  1775. retry:
  1776. if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
  1777. return NULL;
  1778. hhd = &ftrace_func_hash[iter->hidx];
  1779. if (hlist_empty(hhd)) {
  1780. iter->hidx++;
  1781. hnd = NULL;
  1782. goto retry;
  1783. }
  1784. if (!hnd)
  1785. hnd = hhd->first;
  1786. else {
  1787. hnd = hnd->next;
  1788. if (!hnd) {
  1789. iter->hidx++;
  1790. goto retry;
  1791. }
  1792. }
  1793. if (WARN_ON_ONCE(!hnd))
  1794. return NULL;
  1795. iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
  1796. return iter;
  1797. }
  1798. static void *t_hash_start(struct seq_file *m, loff_t *pos)
  1799. {
  1800. struct ftrace_iterator *iter = m->private;
  1801. void *p = NULL;
  1802. loff_t l;
  1803. if (iter->func_pos > *pos)
  1804. return NULL;
  1805. iter->hidx = 0;
  1806. for (l = 0; l <= (*pos - iter->func_pos); ) {
  1807. p = t_hash_next(m, &l);
  1808. if (!p)
  1809. break;
  1810. }
  1811. if (!p)
  1812. return NULL;
  1813. /* Only set this if we have an item */
  1814. iter->flags |= FTRACE_ITER_HASH;
  1815. return iter;
  1816. }
  1817. static int
  1818. t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
  1819. {
  1820. struct ftrace_func_probe *rec;
  1821. rec = iter->probe;
  1822. if (WARN_ON_ONCE(!rec))
  1823. return -EIO;
  1824. if (rec->ops->print)
  1825. return rec->ops->print(m, rec->ip, rec->ops, rec->data);
  1826. seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
  1827. if (rec->data)
  1828. seq_printf(m, ":%p", rec->data);
  1829. seq_putc(m, '\n');
  1830. return 0;
  1831. }
  1832. static void *
  1833. t_next(struct seq_file *m, void *v, loff_t *pos)
  1834. {
  1835. struct ftrace_iterator *iter = m->private;
  1836. struct ftrace_ops *ops = &global_ops;
  1837. struct dyn_ftrace *rec = NULL;
  1838. if (unlikely(ftrace_disabled))
  1839. return NULL;
  1840. if (iter->flags & FTRACE_ITER_HASH)
  1841. return t_hash_next(m, pos);
  1842. (*pos)++;
  1843. iter->pos = iter->func_pos = *pos;
  1844. if (iter->flags & FTRACE_ITER_PRINTALL)
  1845. return t_hash_start(m, pos);
  1846. retry:
  1847. if (iter->idx >= iter->pg->index) {
  1848. if (iter->pg->next) {
  1849. iter->pg = iter->pg->next;
  1850. iter->idx = 0;
  1851. goto retry;
  1852. }
  1853. } else {
  1854. rec = &iter->pg->records[iter->idx++];
  1855. if (((iter->flags & FTRACE_ITER_FILTER) &&
  1856. !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
  1857. ((iter->flags & FTRACE_ITER_NOTRACE) &&
  1858. !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
  1859. ((iter->flags & FTRACE_ITER_ENABLED) &&
  1860. !(rec->flags & ~FTRACE_FL_MASK))) {
  1861. rec = NULL;
  1862. goto retry;
  1863. }
  1864. }
  1865. if (!rec)
  1866. return t_hash_start(m, pos);
  1867. iter->func = rec;
  1868. return iter;
  1869. }
  1870. static void reset_iter_read(struct ftrace_iterator *iter)
  1871. {
  1872. iter->pos = 0;
  1873. iter->func_pos = 0;
  1874. iter->flags &= ~(FTRACE_ITER_PRINTALL & FTRACE_ITER_HASH);
  1875. }
  1876. static void *t_start(struct seq_file *m, loff_t *pos)
  1877. {
  1878. struct ftrace_iterator *iter = m->private;
  1879. struct ftrace_ops *ops = &global_ops;
  1880. void *p = NULL;
  1881. loff_t l;
  1882. mutex_lock(&ftrace_lock);
  1883. if (unlikely(ftrace_disabled))
  1884. return NULL;
  1885. /*
  1886. * If an lseek was done, then reset and start from beginning.
  1887. */
  1888. if (*pos < iter->pos)
  1889. reset_iter_read(iter);
  1890. /*
  1891. * For set_ftrace_filter reading, if we have the filter
  1892. * off, we can short cut and just print out that all
  1893. * functions are enabled.
  1894. */
  1895. if (iter->flags & FTRACE_ITER_FILTER && !ops->filter_hash->count) {
  1896. if (*pos > 0)
  1897. return t_hash_start(m, pos);
  1898. iter->flags |= FTRACE_ITER_PRINTALL;
  1899. /* reset in case of seek/pread */
  1900. iter->flags &= ~FTRACE_ITER_HASH;
  1901. return iter;
  1902. }
  1903. if (iter->flags & FTRACE_ITER_HASH)
  1904. return t_hash_start(m, pos);
  1905. /*
  1906. * Unfortunately, we need to restart at ftrace_pages_start
  1907. * every time we let go of the ftrace_mutex. This is because
  1908. * those pointers can change without the lock.
  1909. */
  1910. iter->pg = ftrace_pages_start;
  1911. iter->idx = 0;
  1912. for (l = 0; l <= *pos; ) {
  1913. p = t_next(m, p, &l);
  1914. if (!p)
  1915. break;
  1916. }
  1917. if (!p) {
  1918. if (iter->flags & FTRACE_ITER_FILTER)
  1919. return t_hash_start(m, pos);
  1920. return NULL;
  1921. }
  1922. return iter;
  1923. }
  1924. static void t_stop(struct seq_file *m, void *p)
  1925. {
  1926. mutex_unlock(&ftrace_lock);
  1927. }
  1928. static int t_show(struct seq_file *m, void *v)
  1929. {
  1930. struct ftrace_iterator *iter = m->private;
  1931. struct dyn_ftrace *rec;
  1932. if (iter->flags & FTRACE_ITER_HASH)
  1933. return t_hash_show(m, iter);
  1934. if (iter->flags & FTRACE_ITER_PRINTALL) {
  1935. seq_printf(m, "#### all functions enabled ####\n");
  1936. return 0;
  1937. }
  1938. rec = iter->func;
  1939. if (!rec)
  1940. return 0;
  1941. seq_printf(m, "%ps", (void *)rec->ip);
  1942. if (iter->flags & FTRACE_ITER_ENABLED)
  1943. seq_printf(m, " (%ld)",
  1944. rec->flags & ~FTRACE_FL_MASK);
  1945. seq_printf(m, "\n");
  1946. return 0;
  1947. }
  1948. static const struct seq_operations show_ftrace_seq_ops = {
  1949. .start = t_start,
  1950. .next = t_next,
  1951. .stop = t_stop,
  1952. .show = t_show,
  1953. };
  1954. static int
  1955. ftrace_avail_open(struct inode *inode, struct file *file)
  1956. {
  1957. struct ftrace_iterator *iter;
  1958. int ret;
  1959. if (unlikely(ftrace_disabled))
  1960. return -ENODEV;
  1961. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  1962. if (!iter)
  1963. return -ENOMEM;
  1964. iter->pg = ftrace_pages_start;
  1965. ret = seq_open(file, &show_ftrace_seq_ops);
  1966. if (!ret) {
  1967. struct seq_file *m = file->private_data;
  1968. m->private = iter;
  1969. } else {
  1970. kfree(iter);
  1971. }
  1972. return ret;
  1973. }
  1974. static int
  1975. ftrace_enabled_open(struct inode *inode, struct file *file)
  1976. {
  1977. struct ftrace_iterator *iter;
  1978. int ret;
  1979. if (unlikely(ftrace_disabled))
  1980. return -ENODEV;
  1981. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  1982. if (!iter)
  1983. return -ENOMEM;
  1984. iter->pg = ftrace_pages_start;
  1985. iter->flags = FTRACE_ITER_ENABLED;
  1986. ret = seq_open(file, &show_ftrace_seq_ops);
  1987. if (!ret) {
  1988. struct seq_file *m = file->private_data;
  1989. m->private = iter;
  1990. } else {
  1991. kfree(iter);
  1992. }
  1993. return ret;
  1994. }
  1995. static void ftrace_filter_reset(struct ftrace_hash *hash)
  1996. {
  1997. mutex_lock(&ftrace_lock);
  1998. ftrace_hash_clear(hash);
  1999. mutex_unlock(&ftrace_lock);
  2000. }
  2001. static int
  2002. ftrace_regex_open(struct ftrace_ops *ops, int flag,
  2003. struct inode *inode, struct file *file)
  2004. {
  2005. struct ftrace_iterator *iter;
  2006. struct ftrace_hash *hash;
  2007. int ret = 0;
  2008. if (unlikely(ftrace_disabled))
  2009. return -ENODEV;
  2010. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  2011. if (!iter)
  2012. return -ENOMEM;
  2013. if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
  2014. kfree(iter);
  2015. return -ENOMEM;
  2016. }
  2017. if (flag & FTRACE_ITER_NOTRACE)
  2018. hash = ops->notrace_hash;
  2019. else
  2020. hash = ops->filter_hash;
  2021. iter->ops = ops;
  2022. iter->flags = flag;
  2023. if (file->f_mode & FMODE_WRITE) {
  2024. mutex_lock(&ftrace_lock);
  2025. iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
  2026. mutex_unlock(&ftrace_lock);
  2027. if (!iter->hash) {
  2028. trace_parser_put(&iter->parser);
  2029. kfree(iter);
  2030. return -ENOMEM;
  2031. }
  2032. }
  2033. mutex_lock(&ftrace_regex_lock);
  2034. if ((file->f_mode & FMODE_WRITE) &&
  2035. (file->f_flags & O_TRUNC))
  2036. ftrace_filter_reset(iter->hash);
  2037. if (file->f_mode & FMODE_READ) {
  2038. iter->pg = ftrace_pages_start;
  2039. ret = seq_open(file, &show_ftrace_seq_ops);
  2040. if (!ret) {
  2041. struct seq_file *m = file->private_data;
  2042. m->private = iter;
  2043. } else {
  2044. /* Failed */
  2045. free_ftrace_hash(iter->hash);
  2046. trace_parser_put(&iter->parser);
  2047. kfree(iter);
  2048. }
  2049. } else
  2050. file->private_data = iter;
  2051. mutex_unlock(&ftrace_regex_lock);
  2052. return ret;
  2053. }
  2054. static int
  2055. ftrace_filter_open(struct inode *inode, struct file *file)
  2056. {
  2057. return ftrace_regex_open(&global_ops, FTRACE_ITER_FILTER,
  2058. inode, file);
  2059. }
  2060. static int
  2061. ftrace_notrace_open(struct inode *inode, struct file *file)
  2062. {
  2063. return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
  2064. inode, file);
  2065. }
  2066. static loff_t
  2067. ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
  2068. {
  2069. loff_t ret;
  2070. if (file->f_mode & FMODE_READ)
  2071. ret = seq_lseek(file, offset, origin);
  2072. else
  2073. file->f_pos = ret = 1;
  2074. return ret;
  2075. }
  2076. static int ftrace_match(char *str, char *regex, int len, int type)
  2077. {
  2078. int matched = 0;
  2079. int slen;
  2080. switch (type) {
  2081. case MATCH_FULL:
  2082. if (strcmp(str, regex) == 0)
  2083. matched = 1;
  2084. break;
  2085. case MATCH_FRONT_ONLY:
  2086. if (strncmp(str, regex, len) == 0)
  2087. matched = 1;
  2088. break;
  2089. case MATCH_MIDDLE_ONLY:
  2090. if (strstr(str, regex))
  2091. matched = 1;
  2092. break;
  2093. case MATCH_END_ONLY:
  2094. slen = strlen(str);
  2095. if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
  2096. matched = 1;
  2097. break;
  2098. }
  2099. return matched;
  2100. }
  2101. static int
  2102. enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
  2103. {
  2104. struct ftrace_func_entry *entry;
  2105. int ret = 0;
  2106. entry = ftrace_lookup_ip(hash, rec->ip);
  2107. if (not) {
  2108. /* Do nothing if it doesn't exist */
  2109. if (!entry)
  2110. return 0;
  2111. free_hash_entry(hash, entry);
  2112. } else {
  2113. /* Do nothing if it exists */
  2114. if (entry)
  2115. return 0;
  2116. ret = add_hash_entry(hash, rec->ip);
  2117. }
  2118. return ret;
  2119. }
  2120. static int
  2121. ftrace_match_record(struct dyn_ftrace *rec, char *mod,
  2122. char *regex, int len, int type)
  2123. {
  2124. char str[KSYM_SYMBOL_LEN];
  2125. char *modname;
  2126. kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
  2127. if (mod) {
  2128. /* module lookup requires matching the module */
  2129. if (!modname || strcmp(modname, mod))
  2130. return 0;
  2131. /* blank search means to match all funcs in the mod */
  2132. if (!len)
  2133. return 1;
  2134. }
  2135. return ftrace_match(str, regex, len, type);
  2136. }
  2137. static int
  2138. match_records(struct ftrace_hash *hash, char *buff,
  2139. int len, char *mod, int not)
  2140. {
  2141. unsigned search_len = 0;
  2142. struct ftrace_page *pg;
  2143. struct dyn_ftrace *rec;
  2144. int type = MATCH_FULL;
  2145. char *search = buff;
  2146. int found = 0;
  2147. int ret;
  2148. if (len) {
  2149. type = filter_parse_regex(buff, len, &search, &not);
  2150. search_len = strlen(search);
  2151. }
  2152. mutex_lock(&ftrace_lock);
  2153. if (unlikely(ftrace_disabled))
  2154. goto out_unlock;
  2155. do_for_each_ftrace_rec(pg, rec) {
  2156. if (ftrace_match_record(rec, mod, search, search_len, type)) {
  2157. ret = enter_record(hash, rec, not);
  2158. if (ret < 0) {
  2159. found = ret;
  2160. goto out_unlock;
  2161. }
  2162. found = 1;
  2163. }
  2164. } while_for_each_ftrace_rec();
  2165. out_unlock:
  2166. mutex_unlock(&ftrace_lock);
  2167. return found;
  2168. }
  2169. static int
  2170. ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
  2171. {
  2172. return match_records(hash, buff, len, NULL, 0);
  2173. }
  2174. static int
  2175. ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
  2176. {
  2177. int not = 0;
  2178. /* blank or '*' mean the same */
  2179. if (strcmp(buff, "*") == 0)
  2180. buff[0] = 0;
  2181. /* handle the case of 'dont filter this module' */
  2182. if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
  2183. buff[0] = 0;
  2184. not = 1;
  2185. }
  2186. return match_records(hash, buff, strlen(buff), mod, not);
  2187. }
  2188. /*
  2189. * We register the module command as a template to show others how
  2190. * to register the a command as well.
  2191. */
  2192. static int
  2193. ftrace_mod_callback(struct ftrace_hash *hash,
  2194. char *func, char *cmd, char *param, int enable)
  2195. {
  2196. char *mod;
  2197. int ret = -EINVAL;
  2198. /*
  2199. * cmd == 'mod' because we only registered this func
  2200. * for the 'mod' ftrace_func_command.
  2201. * But if you register one func with multiple commands,
  2202. * you can tell which command was used by the cmd
  2203. * parameter.
  2204. */
  2205. /* we must have a module name */
  2206. if (!param)
  2207. return ret;
  2208. mod = strsep(&param, ":");
  2209. if (!strlen(mod))
  2210. return ret;
  2211. ret = ftrace_match_module_records(hash, func, mod);
  2212. if (!ret)
  2213. ret = -EINVAL;
  2214. if (ret < 0)
  2215. return ret;
  2216. return 0;
  2217. }
  2218. static struct ftrace_func_command ftrace_mod_cmd = {
  2219. .name = "mod",
  2220. .func = ftrace_mod_callback,
  2221. };
  2222. static int __init ftrace_mod_cmd_init(void)
  2223. {
  2224. return register_ftrace_command(&ftrace_mod_cmd);
  2225. }
  2226. device_initcall(ftrace_mod_cmd_init);
  2227. static void
  2228. function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
  2229. {
  2230. struct ftrace_func_probe *entry;
  2231. struct hlist_head *hhd;
  2232. struct hlist_node *n;
  2233. unsigned long key;
  2234. key = hash_long(ip, FTRACE_HASH_BITS);
  2235. hhd = &ftrace_func_hash[key];
  2236. if (hlist_empty(hhd))
  2237. return;
  2238. /*
  2239. * Disable preemption for these calls to prevent a RCU grace
  2240. * period. This syncs the hash iteration and freeing of items
  2241. * on the hash. rcu_read_lock is too dangerous here.
  2242. */
  2243. preempt_disable_notrace();
  2244. hlist_for_each_entry_rcu(entry, n, hhd, node) {
  2245. if (entry->ip == ip)
  2246. entry->ops->func(ip, parent_ip, &entry->data);
  2247. }
  2248. preempt_enable_notrace();
  2249. }
  2250. static struct ftrace_ops trace_probe_ops __read_mostly =
  2251. {
  2252. .func = function_trace_probe_call,
  2253. };
  2254. static int ftrace_probe_registered;
  2255. static void __enable_ftrace_function_probe(void)
  2256. {
  2257. int ret;
  2258. int i;
  2259. if (ftrace_probe_registered)
  2260. return;
  2261. for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
  2262. struct hlist_head *hhd = &ftrace_func_hash[i];
  2263. if (hhd->first)
  2264. break;
  2265. }
  2266. /* Nothing registered? */
  2267. if (i == FTRACE_FUNC_HASHSIZE)
  2268. return;
  2269. ret = __register_ftrace_function(&trace_probe_ops);
  2270. if (!ret)
  2271. ret = ftrace_startup(&trace_probe_ops, 0);
  2272. ftrace_probe_registered = 1;
  2273. }
  2274. static void __disable_ftrace_function_probe(void)
  2275. {
  2276. int ret;
  2277. int i;
  2278. if (!ftrace_probe_registered)
  2279. return;
  2280. for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
  2281. struct hlist_head *hhd = &ftrace_func_hash[i];
  2282. if (hhd->first)
  2283. return;
  2284. }
  2285. /* no more funcs left */
  2286. ret = __unregister_ftrace_function(&trace_probe_ops);
  2287. if (!ret)
  2288. ftrace_shutdown(&trace_probe_ops, 0);
  2289. ftrace_probe_registered = 0;
  2290. }
  2291. static void ftrace_free_entry_rcu(struct rcu_head *rhp)
  2292. {
  2293. struct ftrace_func_probe *entry =
  2294. container_of(rhp, struct ftrace_func_probe, rcu);
  2295. if (entry->ops->free)
  2296. entry->ops->free(&entry->data);
  2297. kfree(entry);
  2298. }
  2299. int
  2300. register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
  2301. void *data)
  2302. {
  2303. struct ftrace_func_probe *entry;
  2304. struct ftrace_page *pg;
  2305. struct dyn_ftrace *rec;
  2306. int type, len, not;
  2307. unsigned long key;
  2308. int count = 0;
  2309. char *search;
  2310. type = filter_parse_regex(glob, strlen(glob), &search, &not);
  2311. len = strlen(search);
  2312. /* we do not support '!' for function probes */
  2313. if (WARN_ON(not))
  2314. return -EINVAL;
  2315. mutex_lock(&ftrace_lock);
  2316. if (unlikely(ftrace_disabled))
  2317. goto out_unlock;
  2318. do_for_each_ftrace_rec(pg, rec) {
  2319. if (!ftrace_match_record(rec, NULL, search, len, type))
  2320. continue;
  2321. entry = kmalloc(sizeof(*entry), GFP_KERNEL);
  2322. if (!entry) {
  2323. /* If we did not process any, then return error */
  2324. if (!count)
  2325. count = -ENOMEM;
  2326. goto out_unlock;
  2327. }
  2328. count++;
  2329. entry->data = data;
  2330. /*
  2331. * The caller might want to do something special
  2332. * for each function we find. We call the callback
  2333. * to give the caller an opportunity to do so.
  2334. */
  2335. if (ops->callback) {
  2336. if (ops->callback(rec->ip, &entry->data) < 0) {
  2337. /* caller does not like this func */
  2338. kfree(entry);
  2339. continue;
  2340. }
  2341. }
  2342. entry->ops = ops;
  2343. entry->ip = rec->ip;
  2344. key = hash_long(entry->ip, FTRACE_HASH_BITS);
  2345. hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
  2346. } while_for_each_ftrace_rec();
  2347. __enable_ftrace_function_probe();
  2348. out_unlock:
  2349. mutex_unlock(&ftrace_lock);
  2350. return count;
  2351. }
  2352. enum {
  2353. PROBE_TEST_FUNC = 1,
  2354. PROBE_TEST_DATA = 2
  2355. };
  2356. static void
  2357. __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
  2358. void *data, int flags)
  2359. {
  2360. struct ftrace_func_probe *entry;
  2361. struct hlist_node *n, *tmp;
  2362. char str[KSYM_SYMBOL_LEN];
  2363. int type = MATCH_FULL;
  2364. int i, len = 0;
  2365. char *search;
  2366. if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
  2367. glob = NULL;
  2368. else if (glob) {
  2369. int not;
  2370. type = filter_parse_regex(glob, strlen(glob), &search, &not);
  2371. len = strlen(search);
  2372. /* we do not support '!' for function probes */
  2373. if (WARN_ON(not))
  2374. return;
  2375. }
  2376. mutex_lock(&ftrace_lock);
  2377. for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
  2378. struct hlist_head *hhd = &ftrace_func_hash[i];
  2379. hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
  2380. /* break up if statements for readability */
  2381. if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
  2382. continue;
  2383. if ((flags & PROBE_TEST_DATA) && entry->data != data)
  2384. continue;
  2385. /* do this last, since it is the most expensive */
  2386. if (glob) {
  2387. kallsyms_lookup(entry->ip, NULL, NULL,
  2388. NULL, str);
  2389. if (!ftrace_match(str, glob, len, type))
  2390. continue;
  2391. }
  2392. hlist_del(&entry->node);
  2393. call_rcu(&entry->rcu, ftrace_free_entry_rcu);
  2394. }
  2395. }
  2396. __disable_ftrace_function_probe();
  2397. mutex_unlock(&ftrace_lock);
  2398. }
  2399. void
  2400. unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
  2401. void *data)
  2402. {
  2403. __unregister_ftrace_function_probe(glob, ops, data,
  2404. PROBE_TEST_FUNC | PROBE_TEST_DATA);
  2405. }
  2406. void
  2407. unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
  2408. {
  2409. __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
  2410. }
  2411. void unregister_ftrace_function_probe_all(char *glob)
  2412. {
  2413. __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
  2414. }
  2415. static LIST_HEAD(ftrace_commands);
  2416. static DEFINE_MUTEX(ftrace_cmd_mutex);
  2417. int register_ftrace_command(struct ftrace_func_command *cmd)
  2418. {
  2419. struct ftrace_func_command *p;
  2420. int ret = 0;
  2421. mutex_lock(&ftrace_cmd_mutex);
  2422. list_for_each_entry(p, &ftrace_commands, list) {
  2423. if (strcmp(cmd->name, p->name) == 0) {
  2424. ret = -EBUSY;
  2425. goto out_unlock;
  2426. }
  2427. }
  2428. list_add(&cmd->list, &ftrace_commands);
  2429. out_unlock:
  2430. mutex_unlock(&ftrace_cmd_mutex);
  2431. return ret;
  2432. }
  2433. int unregister_ftrace_command(struct ftrace_func_command *cmd)
  2434. {
  2435. struct ftrace_func_command *p, *n;
  2436. int ret = -ENODEV;
  2437. mutex_lock(&ftrace_cmd_mutex);
  2438. list_for_each_entry_safe(p, n, &ftrace_commands, list) {
  2439. if (strcmp(cmd->name, p->name) == 0) {
  2440. ret = 0;
  2441. list_del_init(&p->list);
  2442. goto out_unlock;
  2443. }
  2444. }
  2445. out_unlock:
  2446. mutex_unlock(&ftrace_cmd_mutex);
  2447. return ret;
  2448. }
  2449. static int ftrace_process_regex(struct ftrace_hash *hash,
  2450. char *buff, int len, int enable)
  2451. {
  2452. char *func, *command, *next = buff;
  2453. struct ftrace_func_command *p;
  2454. int ret = -EINVAL;
  2455. func = strsep(&next, ":");
  2456. if (!next) {
  2457. ret = ftrace_match_records(hash, func, len);
  2458. if (!ret)
  2459. ret = -EINVAL;
  2460. if (ret < 0)
  2461. return ret;
  2462. return 0;
  2463. }
  2464. /* command found */
  2465. command = strsep(&next, ":");
  2466. mutex_lock(&ftrace_cmd_mutex);
  2467. list_for_each_entry(p, &ftrace_commands, list) {
  2468. if (strcmp(p->name, command) == 0) {
  2469. ret = p->func(hash, func, command, next, enable);
  2470. goto out_unlock;
  2471. }
  2472. }
  2473. out_unlock:
  2474. mutex_unlock(&ftrace_cmd_mutex);
  2475. return ret;
  2476. }
  2477. static ssize_t
  2478. ftrace_regex_write(struct file *file, const char __user *ubuf,
  2479. size_t cnt, loff_t *ppos, int enable)
  2480. {
  2481. struct ftrace_iterator *iter;
  2482. struct trace_parser *parser;
  2483. ssize_t ret, read;
  2484. if (!cnt)
  2485. return 0;
  2486. mutex_lock(&ftrace_regex_lock);
  2487. ret = -ENODEV;
  2488. if (unlikely(ftrace_disabled))
  2489. goto out_unlock;
  2490. if (file->f_mode & FMODE_READ) {
  2491. struct seq_file *m = file->private_data;
  2492. iter = m->private;
  2493. } else
  2494. iter = file->private_data;
  2495. parser = &iter->parser;
  2496. read = trace_get_user(parser, ubuf, cnt, ppos);
  2497. if (read >= 0 && trace_parser_loaded(parser) &&
  2498. !trace_parser_cont(parser)) {
  2499. ret = ftrace_process_regex(iter->hash, parser->buffer,
  2500. parser->idx, enable);
  2501. trace_parser_clear(parser);
  2502. if (ret)
  2503. goto out_unlock;
  2504. }
  2505. ret = read;
  2506. out_unlock:
  2507. mutex_unlock(&ftrace_regex_lock);
  2508. return ret;
  2509. }
  2510. static ssize_t
  2511. ftrace_filter_write(struct file *file, const char __user *ubuf,
  2512. size_t cnt, loff_t *ppos)
  2513. {
  2514. return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
  2515. }
  2516. static ssize_t
  2517. ftrace_notrace_write(struct file *file, const char __user *ubuf,
  2518. size_t cnt, loff_t *ppos)
  2519. {
  2520. return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
  2521. }
  2522. static int
  2523. ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
  2524. int reset, int enable)
  2525. {
  2526. struct ftrace_hash **orig_hash;
  2527. struct ftrace_hash *hash;
  2528. int ret;
  2529. /* All global ops uses the global ops filters */
  2530. if (ops->flags & FTRACE_OPS_FL_GLOBAL)
  2531. ops = &global_ops;
  2532. if (unlikely(ftrace_disabled))
  2533. return -ENODEV;
  2534. if (enable)
  2535. orig_hash = &ops->filter_hash;
  2536. else
  2537. orig_hash = &ops->notrace_hash;
  2538. hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
  2539. if (!hash)
  2540. return -ENOMEM;
  2541. mutex_lock(&ftrace_regex_lock);
  2542. if (reset)
  2543. ftrace_filter_reset(hash);
  2544. if (buf)
  2545. ftrace_match_records(hash, buf, len);
  2546. mutex_lock(&ftrace_lock);
  2547. ret = ftrace_hash_move(ops, enable, orig_hash, hash);
  2548. if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
  2549. && ftrace_enabled)
  2550. ftrace_run_update_code(FTRACE_UPDATE_CALLS);
  2551. mutex_unlock(&ftrace_lock);
  2552. mutex_unlock(&ftrace_regex_lock);
  2553. free_ftrace_hash(hash);
  2554. return ret;
  2555. }
  2556. /**
  2557. * ftrace_set_filter - set a function to filter on in ftrace
  2558. * @ops - the ops to set the filter with
  2559. * @buf - the string that holds the function filter text.
  2560. * @len - the length of the string.
  2561. * @reset - non zero to reset all filters before applying this filter.
  2562. *
  2563. * Filters denote which functions should be enabled when tracing is enabled.
  2564. * If @buf is NULL and reset is set, all functions will be enabled for tracing.
  2565. */
  2566. void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
  2567. int len, int reset)
  2568. {
  2569. ftrace_set_regex(ops, buf, len, reset, 1);
  2570. }
  2571. EXPORT_SYMBOL_GPL(ftrace_set_filter);
  2572. /**
  2573. * ftrace_set_notrace - set a function to not trace in ftrace
  2574. * @ops - the ops to set the notrace filter with
  2575. * @buf - the string that holds the function notrace text.
  2576. * @len - the length of the string.
  2577. * @reset - non zero to reset all filters before applying this filter.
  2578. *
  2579. * Notrace Filters denote which functions should not be enabled when tracing
  2580. * is enabled. If @buf is NULL and reset is set, all functions will be enabled
  2581. * for tracing.
  2582. */
  2583. void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
  2584. int len, int reset)
  2585. {
  2586. ftrace_set_regex(ops, buf, len, reset, 0);
  2587. }
  2588. EXPORT_SYMBOL_GPL(ftrace_set_notrace);
  2589. /**
  2590. * ftrace_set_filter - set a function to filter on in ftrace
  2591. * @ops - the ops to set the filter with
  2592. * @buf - the string that holds the function filter text.
  2593. * @len - the length of the string.
  2594. * @reset - non zero to reset all filters before applying this filter.
  2595. *
  2596. * Filters denote which functions should be enabled when tracing is enabled.
  2597. * If @buf is NULL and reset is set, all functions will be enabled for tracing.
  2598. */
  2599. void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
  2600. {
  2601. ftrace_set_regex(&global_ops, buf, len, reset, 1);
  2602. }
  2603. EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
  2604. /**
  2605. * ftrace_set_notrace - set a function to not trace in ftrace
  2606. * @ops - the ops to set the notrace filter with
  2607. * @buf - the string that holds the function notrace text.
  2608. * @len - the length of the string.
  2609. * @reset - non zero to reset all filters before applying this filter.
  2610. *
  2611. * Notrace Filters denote which functions should not be enabled when tracing
  2612. * is enabled. If @buf is NULL and reset is set, all functions will be enabled
  2613. * for tracing.
  2614. */
  2615. void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
  2616. {
  2617. ftrace_set_regex(&global_ops, buf, len, reset, 0);
  2618. }
  2619. EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
  2620. /*
  2621. * command line interface to allow users to set filters on boot up.
  2622. */
  2623. #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
  2624. static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
  2625. static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
  2626. static int __init set_ftrace_notrace(char *str)
  2627. {
  2628. strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
  2629. return 1;
  2630. }
  2631. __setup("ftrace_notrace=", set_ftrace_notrace);
  2632. static int __init set_ftrace_filter(char *str)
  2633. {
  2634. strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
  2635. return 1;
  2636. }
  2637. __setup("ftrace_filter=", set_ftrace_filter);
  2638. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  2639. static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
  2640. static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
  2641. static int __init set_graph_function(char *str)
  2642. {
  2643. strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
  2644. return 1;
  2645. }
  2646. __setup("ftrace_graph_filter=", set_graph_function);
  2647. static void __init set_ftrace_early_graph(char *buf)
  2648. {
  2649. int ret;
  2650. char *func;
  2651. while (buf) {
  2652. func = strsep(&buf, ",");
  2653. /* we allow only one expression at a time */
  2654. ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
  2655. func);
  2656. if (ret)
  2657. printk(KERN_DEBUG "ftrace: function %s not "
  2658. "traceable\n", func);
  2659. }
  2660. }
  2661. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  2662. static void __init
  2663. set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable)
  2664. {
  2665. char *func;
  2666. while (buf) {
  2667. func = strsep(&buf, ",");
  2668. ftrace_set_regex(ops, func, strlen(func), 0, enable);
  2669. }
  2670. }
  2671. static void __init set_ftrace_early_filters(void)
  2672. {
  2673. if (ftrace_filter_buf[0])
  2674. set_ftrace_early_filter(&global_ops, ftrace_filter_buf, 1);
  2675. if (ftrace_notrace_buf[0])
  2676. set_ftrace_early_filter(&global_ops, ftrace_notrace_buf, 0);
  2677. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  2678. if (ftrace_graph_buf[0])
  2679. set_ftrace_early_graph(ftrace_graph_buf);
  2680. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  2681. }
  2682. static int
  2683. ftrace_regex_release(struct inode *inode, struct file *file)
  2684. {
  2685. struct seq_file *m = (struct seq_file *)file->private_data;
  2686. struct ftrace_iterator *iter;
  2687. struct ftrace_hash **orig_hash;
  2688. struct trace_parser *parser;
  2689. int filter_hash;
  2690. int ret;
  2691. mutex_lock(&ftrace_regex_lock);
  2692. if (file->f_mode & FMODE_READ) {
  2693. iter = m->private;
  2694. seq_release(inode, file);
  2695. } else
  2696. iter = file->private_data;
  2697. parser = &iter->parser;
  2698. if (trace_parser_loaded(parser)) {
  2699. parser->buffer[parser->idx] = 0;
  2700. ftrace_match_records(iter->hash, parser->buffer, parser->idx);
  2701. }
  2702. trace_parser_put(parser);
  2703. if (file->f_mode & FMODE_WRITE) {
  2704. filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
  2705. if (filter_hash)
  2706. orig_hash = &iter->ops->filter_hash;
  2707. else
  2708. orig_hash = &iter->ops->notrace_hash;
  2709. mutex_lock(&ftrace_lock);
  2710. ret = ftrace_hash_move(iter->ops, filter_hash,
  2711. orig_hash, iter->hash);
  2712. if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
  2713. && ftrace_enabled)
  2714. ftrace_run_update_code(FTRACE_UPDATE_CALLS);
  2715. mutex_unlock(&ftrace_lock);
  2716. }
  2717. free_ftrace_hash(iter->hash);
  2718. kfree(iter);
  2719. mutex_unlock(&ftrace_regex_lock);
  2720. return 0;
  2721. }
  2722. static const struct file_operations ftrace_avail_fops = {
  2723. .open = ftrace_avail_open,
  2724. .read = seq_read,
  2725. .llseek = seq_lseek,
  2726. .release = seq_release_private,
  2727. };
  2728. static const struct file_operations ftrace_enabled_fops = {
  2729. .open = ftrace_enabled_open,
  2730. .read = seq_read,
  2731. .llseek = seq_lseek,
  2732. .release = seq_release_private,
  2733. };
  2734. static const struct file_operations ftrace_filter_fops = {
  2735. .open = ftrace_filter_open,
  2736. .read = seq_read,
  2737. .write = ftrace_filter_write,
  2738. .llseek = ftrace_regex_lseek,
  2739. .release = ftrace_regex_release,
  2740. };
  2741. static const struct file_operations ftrace_notrace_fops = {
  2742. .open = ftrace_notrace_open,
  2743. .read = seq_read,
  2744. .write = ftrace_notrace_write,
  2745. .llseek = ftrace_regex_lseek,
  2746. .release = ftrace_regex_release,
  2747. };
  2748. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  2749. static DEFINE_MUTEX(graph_lock);
  2750. int ftrace_graph_count;
  2751. int ftrace_graph_filter_enabled;
  2752. unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
  2753. static void *
  2754. __g_next(struct seq_file *m, loff_t *pos)
  2755. {
  2756. if (*pos >= ftrace_graph_count)
  2757. return NULL;
  2758. return &ftrace_graph_funcs[*pos];
  2759. }
  2760. static void *
  2761. g_next(struct seq_file *m, void *v, loff_t *pos)
  2762. {
  2763. (*pos)++;
  2764. return __g_next(m, pos);
  2765. }
  2766. static void *g_start(struct seq_file *m, loff_t *pos)
  2767. {
  2768. mutex_lock(&graph_lock);
  2769. /* Nothing, tell g_show to print all functions are enabled */
  2770. if (!ftrace_graph_filter_enabled && !*pos)
  2771. return (void *)1;
  2772. return __g_next(m, pos);
  2773. }
  2774. static void g_stop(struct seq_file *m, void *p)
  2775. {
  2776. mutex_unlock(&graph_lock);
  2777. }
  2778. static int g_show(struct seq_file *m, void *v)
  2779. {
  2780. unsigned long *ptr = v;
  2781. if (!ptr)
  2782. return 0;
  2783. if (ptr == (unsigned long *)1) {
  2784. seq_printf(m, "#### all functions enabled ####\n");
  2785. return 0;
  2786. }
  2787. seq_printf(m, "%ps\n", (void *)*ptr);
  2788. return 0;
  2789. }
  2790. static const struct seq_operations ftrace_graph_seq_ops = {
  2791. .start = g_start,
  2792. .next = g_next,
  2793. .stop = g_stop,
  2794. .show = g_show,
  2795. };
  2796. static int
  2797. ftrace_graph_open(struct inode *inode, struct file *file)
  2798. {
  2799. int ret = 0;
  2800. if (unlikely(ftrace_disabled))
  2801. return -ENODEV;
  2802. mutex_lock(&graph_lock);
  2803. if ((file->f_mode & FMODE_WRITE) &&
  2804. (file->f_flags & O_TRUNC)) {
  2805. ftrace_graph_filter_enabled = 0;
  2806. ftrace_graph_count = 0;
  2807. memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
  2808. }
  2809. mutex_unlock(&graph_lock);
  2810. if (file->f_mode & FMODE_READ)
  2811. ret = seq_open(file, &ftrace_graph_seq_ops);
  2812. return ret;
  2813. }
  2814. static int
  2815. ftrace_graph_release(struct inode *inode, struct file *file)
  2816. {
  2817. if (file->f_mode & FMODE_READ)
  2818. seq_release(inode, file);
  2819. return 0;
  2820. }
  2821. static int
  2822. ftrace_set_func(unsigned long *array, int *idx, char *buffer)
  2823. {
  2824. struct dyn_ftrace *rec;
  2825. struct ftrace_page *pg;
  2826. int search_len;
  2827. int fail = 1;
  2828. int type, not;
  2829. char *search;
  2830. bool exists;
  2831. int i;
  2832. /* decode regex */
  2833. type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
  2834. if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
  2835. return -EBUSY;
  2836. search_len = strlen(search);
  2837. mutex_lock(&ftrace_lock);
  2838. if (unlikely(ftrace_disabled)) {
  2839. mutex_unlock(&ftrace_lock);
  2840. return -ENODEV;
  2841. }
  2842. do_for_each_ftrace_rec(pg, rec) {
  2843. if (ftrace_match_record(rec, NULL, search, search_len, type)) {
  2844. /* if it is in the array */
  2845. exists = false;
  2846. for (i = 0; i < *idx; i++) {
  2847. if (array[i] == rec->ip) {
  2848. exists = true;
  2849. break;
  2850. }
  2851. }
  2852. if (!not) {
  2853. fail = 0;
  2854. if (!exists) {
  2855. array[(*idx)++] = rec->ip;
  2856. if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
  2857. goto out;
  2858. }
  2859. } else {
  2860. if (exists) {
  2861. array[i] = array[--(*idx)];
  2862. array[*idx] = 0;
  2863. fail = 0;
  2864. }
  2865. }
  2866. }
  2867. } while_for_each_ftrace_rec();
  2868. out:
  2869. mutex_unlock(&ftrace_lock);
  2870. if (fail)
  2871. return -EINVAL;
  2872. ftrace_graph_filter_enabled = 1;
  2873. return 0;
  2874. }
  2875. static ssize_t
  2876. ftrace_graph_write(struct file *file, const char __user *ubuf,
  2877. size_t cnt, loff_t *ppos)
  2878. {
  2879. struct trace_parser parser;
  2880. ssize_t read, ret;
  2881. if (!cnt)
  2882. return 0;
  2883. mutex_lock(&graph_lock);
  2884. if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
  2885. ret = -ENOMEM;
  2886. goto out_unlock;
  2887. }
  2888. read = trace_get_user(&parser, ubuf, cnt, ppos);
  2889. if (read >= 0 && trace_parser_loaded((&parser))) {
  2890. parser.buffer[parser.idx] = 0;
  2891. /* we allow only one expression at a time */
  2892. ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
  2893. parser.buffer);
  2894. if (ret)
  2895. goto out_free;
  2896. }
  2897. ret = read;
  2898. out_free:
  2899. trace_parser_put(&parser);
  2900. out_unlock:
  2901. mutex_unlock(&graph_lock);
  2902. return ret;
  2903. }
  2904. static const struct file_operations ftrace_graph_fops = {
  2905. .open = ftrace_graph_open,
  2906. .read = seq_read,
  2907. .write = ftrace_graph_write,
  2908. .release = ftrace_graph_release,
  2909. .llseek = seq_lseek,
  2910. };
  2911. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  2912. static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
  2913. {
  2914. trace_create_file("available_filter_functions", 0444,
  2915. d_tracer, NULL, &ftrace_avail_fops);
  2916. trace_create_file("enabled_functions", 0444,
  2917. d_tracer, NULL, &ftrace_enabled_fops);
  2918. trace_create_file("set_ftrace_filter", 0644, d_tracer,
  2919. NULL, &ftrace_filter_fops);
  2920. trace_create_file("set_ftrace_notrace", 0644, d_tracer,
  2921. NULL, &ftrace_notrace_fops);
  2922. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  2923. trace_create_file("set_graph_function", 0444, d_tracer,
  2924. NULL,
  2925. &ftrace_graph_fops);
  2926. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  2927. return 0;
  2928. }
  2929. static void ftrace_swap_recs(void *a, void *b, int size)
  2930. {
  2931. struct dyn_ftrace *reca = a;
  2932. struct dyn_ftrace *recb = b;
  2933. struct dyn_ftrace t;
  2934. t = *reca;
  2935. *reca = *recb;
  2936. *recb = t;
  2937. }
  2938. static int ftrace_cmp_recs(const void *a, const void *b)
  2939. {
  2940. const struct dyn_ftrace *reca = a;
  2941. const struct dyn_ftrace *recb = b;
  2942. if (reca->ip > recb->ip)
  2943. return 1;
  2944. if (reca->ip < recb->ip)
  2945. return -1;
  2946. return 0;
  2947. }
  2948. static int ftrace_process_locs(struct module *mod,
  2949. unsigned long *start,
  2950. unsigned long *end)
  2951. {
  2952. struct ftrace_page *pg;
  2953. unsigned long count;
  2954. unsigned long *p;
  2955. unsigned long addr;
  2956. unsigned long flags = 0; /* Shut up gcc */
  2957. int ret = -ENOMEM;
  2958. count = end - start;
  2959. if (!count)
  2960. return 0;
  2961. pg = ftrace_allocate_pages(count);
  2962. if (!pg)
  2963. return -ENOMEM;
  2964. mutex_lock(&ftrace_lock);
  2965. /*
  2966. * Core and each module needs their own pages, as
  2967. * modules will free them when they are removed.
  2968. * Force a new page to be allocated for modules.
  2969. */
  2970. if (!mod) {
  2971. WARN_ON(ftrace_pages || ftrace_pages_start);
  2972. /* First initialization */
  2973. ftrace_pages = ftrace_pages_start = pg;
  2974. } else {
  2975. if (!ftrace_pages)
  2976. goto out;
  2977. if (WARN_ON(ftrace_pages->next)) {
  2978. /* Hmm, we have free pages? */
  2979. while (ftrace_pages->next)
  2980. ftrace_pages = ftrace_pages->next;
  2981. }
  2982. ftrace_pages->next = pg;
  2983. ftrace_pages = pg;
  2984. }
  2985. p = start;
  2986. while (p < end) {
  2987. addr = ftrace_call_adjust(*p++);
  2988. /*
  2989. * Some architecture linkers will pad between
  2990. * the different mcount_loc sections of different
  2991. * object files to satisfy alignments.
  2992. * Skip any NULL pointers.
  2993. */
  2994. if (!addr)
  2995. continue;
  2996. if (!ftrace_record_ip(addr))
  2997. break;
  2998. }
  2999. /* These new locations need to be initialized */
  3000. ftrace_new_pgs = pg;
  3001. /* Make each individual set of pages sorted by ips */
  3002. for (; pg; pg = pg->next)
  3003. sort(pg->records, pg->index, sizeof(struct dyn_ftrace),
  3004. ftrace_cmp_recs, ftrace_swap_recs);
  3005. /*
  3006. * We only need to disable interrupts on start up
  3007. * because we are modifying code that an interrupt
  3008. * may execute, and the modification is not atomic.
  3009. * But for modules, nothing runs the code we modify
  3010. * until we are finished with it, and there's no
  3011. * reason to cause large interrupt latencies while we do it.
  3012. */
  3013. if (!mod)
  3014. local_irq_save(flags);
  3015. ftrace_update_code(mod);
  3016. if (!mod)
  3017. local_irq_restore(flags);
  3018. ret = 0;
  3019. out:
  3020. mutex_unlock(&ftrace_lock);
  3021. return ret;
  3022. }
  3023. #ifdef CONFIG_MODULES
  3024. #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
  3025. void ftrace_release_mod(struct module *mod)
  3026. {
  3027. struct dyn_ftrace *rec;
  3028. struct ftrace_page **last_pg;
  3029. struct ftrace_page *pg;
  3030. int order;
  3031. mutex_lock(&ftrace_lock);
  3032. if (ftrace_disabled)
  3033. goto out_unlock;
  3034. /*
  3035. * Each module has its own ftrace_pages, remove
  3036. * them from the list.
  3037. */
  3038. last_pg = &ftrace_pages_start;
  3039. for (pg = ftrace_pages_start; pg; pg = *last_pg) {
  3040. rec = &pg->records[0];
  3041. if (within_module_core(rec->ip, mod)) {
  3042. /*
  3043. * As core pages are first, the first
  3044. * page should never be a module page.
  3045. */
  3046. if (WARN_ON(pg == ftrace_pages_start))
  3047. goto out_unlock;
  3048. /* Check if we are deleting the last page */
  3049. if (pg == ftrace_pages)
  3050. ftrace_pages = next_to_ftrace_page(last_pg);
  3051. *last_pg = pg->next;
  3052. order = get_count_order(pg->size / ENTRIES_PER_PAGE);
  3053. free_pages((unsigned long)pg->records, order);
  3054. kfree(pg);
  3055. } else
  3056. last_pg = &pg->next;
  3057. }
  3058. out_unlock:
  3059. mutex_unlock(&ftrace_lock);
  3060. }
  3061. static void ftrace_init_module(struct module *mod,
  3062. unsigned long *start, unsigned long *end)
  3063. {
  3064. if (ftrace_disabled || start == end)
  3065. return;
  3066. ftrace_process_locs(mod, start, end);
  3067. }
  3068. static int ftrace_module_notify(struct notifier_block *self,
  3069. unsigned long val, void *data)
  3070. {
  3071. struct module *mod = data;
  3072. switch (val) {
  3073. case MODULE_STATE_COMING:
  3074. ftrace_init_module(mod, mod->ftrace_callsites,
  3075. mod->ftrace_callsites +
  3076. mod->num_ftrace_callsites);
  3077. break;
  3078. case MODULE_STATE_GOING:
  3079. ftrace_release_mod(mod);
  3080. break;
  3081. }
  3082. return 0;
  3083. }
  3084. #else
  3085. static int ftrace_module_notify(struct notifier_block *self,
  3086. unsigned long val, void *data)
  3087. {
  3088. return 0;
  3089. }
  3090. #endif /* CONFIG_MODULES */
  3091. struct notifier_block ftrace_module_nb = {
  3092. .notifier_call = ftrace_module_notify,
  3093. .priority = 0,
  3094. };
  3095. extern unsigned long __start_mcount_loc[];
  3096. extern unsigned long __stop_mcount_loc[];
  3097. void __init ftrace_init(void)
  3098. {
  3099. unsigned long count, addr, flags;
  3100. int ret;
  3101. /* Keep the ftrace pointer to the stub */
  3102. addr = (unsigned long)ftrace_stub;
  3103. local_irq_save(flags);
  3104. ftrace_dyn_arch_init(&addr);
  3105. local_irq_restore(flags);
  3106. /* ftrace_dyn_arch_init places the return code in addr */
  3107. if (addr)
  3108. goto failed;
  3109. count = __stop_mcount_loc - __start_mcount_loc;
  3110. ret = ftrace_dyn_table_alloc(count);
  3111. if (ret)
  3112. goto failed;
  3113. last_ftrace_enabled = ftrace_enabled = 1;
  3114. ret = ftrace_process_locs(NULL,
  3115. __start_mcount_loc,
  3116. __stop_mcount_loc);
  3117. ret = register_module_notifier(&ftrace_module_nb);
  3118. if (ret)
  3119. pr_warning("Failed to register trace ftrace module notifier\n");
  3120. set_ftrace_early_filters();
  3121. return;
  3122. failed:
  3123. ftrace_disabled = 1;
  3124. }
  3125. #else
  3126. static struct ftrace_ops global_ops = {
  3127. .func = ftrace_stub,
  3128. };
  3129. static int __init ftrace_nodyn_init(void)
  3130. {
  3131. ftrace_enabled = 1;
  3132. return 0;
  3133. }
  3134. device_initcall(ftrace_nodyn_init);
  3135. static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
  3136. static inline void ftrace_startup_enable(int command) { }
  3137. /* Keep as macros so we do not need to define the commands */
  3138. # define ftrace_startup(ops, command) \
  3139. ({ \
  3140. (ops)->flags |= FTRACE_OPS_FL_ENABLED; \
  3141. 0; \
  3142. })
  3143. # define ftrace_shutdown(ops, command) do { } while (0)
  3144. # define ftrace_startup_sysctl() do { } while (0)
  3145. # define ftrace_shutdown_sysctl() do { } while (0)
  3146. static inline int
  3147. ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
  3148. {
  3149. return 1;
  3150. }
  3151. #endif /* CONFIG_DYNAMIC_FTRACE */
  3152. static void
  3153. ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
  3154. {
  3155. struct ftrace_ops *op;
  3156. if (unlikely(trace_recursion_test(TRACE_INTERNAL_BIT)))
  3157. return;
  3158. trace_recursion_set(TRACE_INTERNAL_BIT);
  3159. /*
  3160. * Some of the ops may be dynamically allocated,
  3161. * they must be freed after a synchronize_sched().
  3162. */
  3163. preempt_disable_notrace();
  3164. op = rcu_dereference_raw(ftrace_ops_list);
  3165. while (op != &ftrace_list_end) {
  3166. if (ftrace_ops_test(op, ip))
  3167. op->func(ip, parent_ip);
  3168. op = rcu_dereference_raw(op->next);
  3169. };
  3170. preempt_enable_notrace();
  3171. trace_recursion_clear(TRACE_INTERNAL_BIT);
  3172. }
  3173. static void clear_ftrace_swapper(void)
  3174. {
  3175. struct task_struct *p;
  3176. int cpu;
  3177. get_online_cpus();
  3178. for_each_online_cpu(cpu) {
  3179. p = idle_task(cpu);
  3180. clear_tsk_trace_trace(p);
  3181. }
  3182. put_online_cpus();
  3183. }
  3184. static void set_ftrace_swapper(void)
  3185. {
  3186. struct task_struct *p;
  3187. int cpu;
  3188. get_online_cpus();
  3189. for_each_online_cpu(cpu) {
  3190. p = idle_task(cpu);
  3191. set_tsk_trace_trace(p);
  3192. }
  3193. put_online_cpus();
  3194. }
  3195. static void clear_ftrace_pid(struct pid *pid)
  3196. {
  3197. struct task_struct *p;
  3198. rcu_read_lock();
  3199. do_each_pid_task(pid, PIDTYPE_PID, p) {
  3200. clear_tsk_trace_trace(p);
  3201. } while_each_pid_task(pid, PIDTYPE_PID, p);
  3202. rcu_read_unlock();
  3203. put_pid(pid);
  3204. }
  3205. static void set_ftrace_pid(struct pid *pid)
  3206. {
  3207. struct task_struct *p;
  3208. rcu_read_lock();
  3209. do_each_pid_task(pid, PIDTYPE_PID, p) {
  3210. set_tsk_trace_trace(p);
  3211. } while_each_pid_task(pid, PIDTYPE_PID, p);
  3212. rcu_read_unlock();
  3213. }
  3214. static void clear_ftrace_pid_task(struct pid *pid)
  3215. {
  3216. if (pid == ftrace_swapper_pid)
  3217. clear_ftrace_swapper();
  3218. else
  3219. clear_ftrace_pid(pid);
  3220. }
  3221. static void set_ftrace_pid_task(struct pid *pid)
  3222. {
  3223. if (pid == ftrace_swapper_pid)
  3224. set_ftrace_swapper();
  3225. else
  3226. set_ftrace_pid(pid);
  3227. }
  3228. static int ftrace_pid_add(int p)
  3229. {
  3230. struct pid *pid;
  3231. struct ftrace_pid *fpid;
  3232. int ret = -EINVAL;
  3233. mutex_lock(&ftrace_lock);
  3234. if (!p)
  3235. pid = ftrace_swapper_pid;
  3236. else
  3237. pid = find_get_pid(p);
  3238. if (!pid)
  3239. goto out;
  3240. ret = 0;
  3241. list_for_each_entry(fpid, &ftrace_pids, list)
  3242. if (fpid->pid == pid)
  3243. goto out_put;
  3244. ret = -ENOMEM;
  3245. fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
  3246. if (!fpid)
  3247. goto out_put;
  3248. list_add(&fpid->list, &ftrace_pids);
  3249. fpid->pid = pid;
  3250. set_ftrace_pid_task(pid);
  3251. ftrace_update_pid_func();
  3252. ftrace_startup_enable(0);
  3253. mutex_unlock(&ftrace_lock);
  3254. return 0;
  3255. out_put:
  3256. if (pid != ftrace_swapper_pid)
  3257. put_pid(pid);
  3258. out:
  3259. mutex_unlock(&ftrace_lock);
  3260. return ret;
  3261. }
  3262. static void ftrace_pid_reset(void)
  3263. {
  3264. struct ftrace_pid *fpid, *safe;
  3265. mutex_lock(&ftrace_lock);
  3266. list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
  3267. struct pid *pid = fpid->pid;
  3268. clear_ftrace_pid_task(pid);
  3269. list_del(&fpid->list);
  3270. kfree(fpid);
  3271. }
  3272. ftrace_update_pid_func();
  3273. ftrace_startup_enable(0);
  3274. mutex_unlock(&ftrace_lock);
  3275. }
  3276. static void *fpid_start(struct seq_file *m, loff_t *pos)
  3277. {
  3278. mutex_lock(&ftrace_lock);
  3279. if (list_empty(&ftrace_pids) && (!*pos))
  3280. return (void *) 1;
  3281. return seq_list_start(&ftrace_pids, *pos);
  3282. }
  3283. static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
  3284. {
  3285. if (v == (void *)1)
  3286. return NULL;
  3287. return seq_list_next(v, &ftrace_pids, pos);
  3288. }
  3289. static void fpid_stop(struct seq_file *m, void *p)
  3290. {
  3291. mutex_unlock(&ftrace_lock);
  3292. }
  3293. static int fpid_show(struct seq_file *m, void *v)
  3294. {
  3295. const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
  3296. if (v == (void *)1) {
  3297. seq_printf(m, "no pid\n");
  3298. return 0;
  3299. }
  3300. if (fpid->pid == ftrace_swapper_pid)
  3301. seq_printf(m, "swapper tasks\n");
  3302. else
  3303. seq_printf(m, "%u\n", pid_vnr(fpid->pid));
  3304. return 0;
  3305. }
  3306. static const struct seq_operations ftrace_pid_sops = {
  3307. .start = fpid_start,
  3308. .next = fpid_next,
  3309. .stop = fpid_stop,
  3310. .show = fpid_show,
  3311. };
  3312. static int
  3313. ftrace_pid_open(struct inode *inode, struct file *file)
  3314. {
  3315. int ret = 0;
  3316. if ((file->f_mode & FMODE_WRITE) &&
  3317. (file->f_flags & O_TRUNC))
  3318. ftrace_pid_reset();
  3319. if (file->f_mode & FMODE_READ)
  3320. ret = seq_open(file, &ftrace_pid_sops);
  3321. return ret;
  3322. }
  3323. static ssize_t
  3324. ftrace_pid_write(struct file *filp, const char __user *ubuf,
  3325. size_t cnt, loff_t *ppos)
  3326. {
  3327. char buf[64], *tmp;
  3328. long val;
  3329. int ret;
  3330. if (cnt >= sizeof(buf))
  3331. return -EINVAL;
  3332. if (copy_from_user(&buf, ubuf, cnt))
  3333. return -EFAULT;
  3334. buf[cnt] = 0;
  3335. /*
  3336. * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
  3337. * to clean the filter quietly.
  3338. */
  3339. tmp = strstrip(buf);
  3340. if (strlen(tmp) == 0)
  3341. return 1;
  3342. ret = strict_strtol(tmp, 10, &val);
  3343. if (ret < 0)
  3344. return ret;
  3345. ret = ftrace_pid_add(val);
  3346. return ret ? ret : cnt;
  3347. }
  3348. static int
  3349. ftrace_pid_release(struct inode *inode, struct file *file)
  3350. {
  3351. if (file->f_mode & FMODE_READ)
  3352. seq_release(inode, file);
  3353. return 0;
  3354. }
  3355. static const struct file_operations ftrace_pid_fops = {
  3356. .open = ftrace_pid_open,
  3357. .write = ftrace_pid_write,
  3358. .read = seq_read,
  3359. .llseek = seq_lseek,
  3360. .release = ftrace_pid_release,
  3361. };
  3362. static __init int ftrace_init_debugfs(void)
  3363. {
  3364. struct dentry *d_tracer;
  3365. d_tracer = tracing_init_dentry();
  3366. if (!d_tracer)
  3367. return 0;
  3368. ftrace_init_dyn_debugfs(d_tracer);
  3369. trace_create_file("set_ftrace_pid", 0644, d_tracer,
  3370. NULL, &ftrace_pid_fops);
  3371. ftrace_profile_debugfs(d_tracer);
  3372. return 0;
  3373. }
  3374. fs_initcall(ftrace_init_debugfs);
  3375. /**
  3376. * ftrace_kill - kill ftrace
  3377. *
  3378. * This function should be used by panic code. It stops ftrace
  3379. * but in a not so nice way. If you need to simply kill ftrace
  3380. * from a non-atomic section, use ftrace_kill.
  3381. */
  3382. void ftrace_kill(void)
  3383. {
  3384. ftrace_disabled = 1;
  3385. ftrace_enabled = 0;
  3386. clear_ftrace_function();
  3387. }
  3388. /**
  3389. * Test if ftrace is dead or not.
  3390. */
  3391. int ftrace_is_dead(void)
  3392. {
  3393. return ftrace_disabled;
  3394. }
  3395. /**
  3396. * register_ftrace_function - register a function for profiling
  3397. * @ops - ops structure that holds the function for profiling.
  3398. *
  3399. * Register a function to be called by all functions in the
  3400. * kernel.
  3401. *
  3402. * Note: @ops->func and all the functions it calls must be labeled
  3403. * with "notrace", otherwise it will go into a
  3404. * recursive loop.
  3405. */
  3406. int register_ftrace_function(struct ftrace_ops *ops)
  3407. {
  3408. int ret = -1;
  3409. mutex_lock(&ftrace_lock);
  3410. if (unlikely(ftrace_disabled))
  3411. goto out_unlock;
  3412. ret = __register_ftrace_function(ops);
  3413. if (!ret)
  3414. ret = ftrace_startup(ops, 0);
  3415. out_unlock:
  3416. mutex_unlock(&ftrace_lock);
  3417. return ret;
  3418. }
  3419. EXPORT_SYMBOL_GPL(register_ftrace_function);
  3420. /**
  3421. * unregister_ftrace_function - unregister a function for profiling.
  3422. * @ops - ops structure that holds the function to unregister
  3423. *
  3424. * Unregister a function that was added to be called by ftrace profiling.
  3425. */
  3426. int unregister_ftrace_function(struct ftrace_ops *ops)
  3427. {
  3428. int ret;
  3429. mutex_lock(&ftrace_lock);
  3430. ret = __unregister_ftrace_function(ops);
  3431. if (!ret)
  3432. ftrace_shutdown(ops, 0);
  3433. mutex_unlock(&ftrace_lock);
  3434. return ret;
  3435. }
  3436. EXPORT_SYMBOL_GPL(unregister_ftrace_function);
  3437. int
  3438. ftrace_enable_sysctl(struct ctl_table *table, int write,
  3439. void __user *buffer, size_t *lenp,
  3440. loff_t *ppos)
  3441. {
  3442. int ret = -ENODEV;
  3443. mutex_lock(&ftrace_lock);
  3444. if (unlikely(ftrace_disabled))
  3445. goto out;
  3446. ret = proc_dointvec(table, write, buffer, lenp, ppos);
  3447. if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
  3448. goto out;
  3449. last_ftrace_enabled = !!ftrace_enabled;
  3450. if (ftrace_enabled) {
  3451. ftrace_startup_sysctl();
  3452. /* we are starting ftrace again */
  3453. if (ftrace_ops_list != &ftrace_list_end) {
  3454. if (ftrace_ops_list->next == &ftrace_list_end)
  3455. ftrace_trace_function = ftrace_ops_list->func;
  3456. else
  3457. ftrace_trace_function = ftrace_ops_list_func;
  3458. }
  3459. } else {
  3460. /* stopping ftrace calls (just send to ftrace_stub) */
  3461. ftrace_trace_function = ftrace_stub;
  3462. ftrace_shutdown_sysctl();
  3463. }
  3464. out:
  3465. mutex_unlock(&ftrace_lock);
  3466. return ret;
  3467. }
  3468. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  3469. static int ftrace_graph_active;
  3470. static struct notifier_block ftrace_suspend_notifier;
  3471. int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
  3472. {
  3473. return 0;
  3474. }
  3475. /* The callbacks that hook a function */
  3476. trace_func_graph_ret_t ftrace_graph_return =
  3477. (trace_func_graph_ret_t)ftrace_stub;
  3478. trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
  3479. /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
  3480. static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
  3481. {
  3482. int i;
  3483. int ret = 0;
  3484. unsigned long flags;
  3485. int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
  3486. struct task_struct *g, *t;
  3487. for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
  3488. ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
  3489. * sizeof(struct ftrace_ret_stack),
  3490. GFP_KERNEL);
  3491. if (!ret_stack_list[i]) {
  3492. start = 0;
  3493. end = i;
  3494. ret = -ENOMEM;
  3495. goto free;
  3496. }
  3497. }
  3498. read_lock_irqsave(&tasklist_lock, flags);
  3499. do_each_thread(g, t) {
  3500. if (start == end) {
  3501. ret = -EAGAIN;
  3502. goto unlock;
  3503. }
  3504. if (t->ret_stack == NULL) {
  3505. atomic_set(&t->tracing_graph_pause, 0);
  3506. atomic_set(&t->trace_overrun, 0);
  3507. t->curr_ret_stack = -1;
  3508. /* Make sure the tasks see the -1 first: */
  3509. smp_wmb();
  3510. t->ret_stack = ret_stack_list[start++];
  3511. }
  3512. } while_each_thread(g, t);
  3513. unlock:
  3514. read_unlock_irqrestore(&tasklist_lock, flags);
  3515. free:
  3516. for (i = start; i < end; i++)
  3517. kfree(ret_stack_list[i]);
  3518. return ret;
  3519. }
  3520. static void
  3521. ftrace_graph_probe_sched_switch(void *ignore,
  3522. struct task_struct *prev, struct task_struct *next)
  3523. {
  3524. unsigned long long timestamp;
  3525. int index;
  3526. /*
  3527. * Does the user want to count the time a function was asleep.
  3528. * If so, do not update the time stamps.
  3529. */
  3530. if (trace_flags & TRACE_ITER_SLEEP_TIME)
  3531. return;
  3532. timestamp = trace_clock_local();
  3533. prev->ftrace_timestamp = timestamp;
  3534. /* only process tasks that we timestamped */
  3535. if (!next->ftrace_timestamp)
  3536. return;
  3537. /*
  3538. * Update all the counters in next to make up for the
  3539. * time next was sleeping.
  3540. */
  3541. timestamp -= next->ftrace_timestamp;
  3542. for (index = next->curr_ret_stack; index >= 0; index--)
  3543. next->ret_stack[index].calltime += timestamp;
  3544. }
  3545. /* Allocate a return stack for each task */
  3546. static int start_graph_tracing(void)
  3547. {
  3548. struct ftrace_ret_stack **ret_stack_list;
  3549. int ret, cpu;
  3550. ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
  3551. sizeof(struct ftrace_ret_stack *),
  3552. GFP_KERNEL);
  3553. if (!ret_stack_list)
  3554. return -ENOMEM;
  3555. /* The cpu_boot init_task->ret_stack will never be freed */
  3556. for_each_online_cpu(cpu) {
  3557. if (!idle_task(cpu)->ret_stack)
  3558. ftrace_graph_init_idle_task(idle_task(cpu), cpu);
  3559. }
  3560. do {
  3561. ret = alloc_retstack_tasklist(ret_stack_list);
  3562. } while (ret == -EAGAIN);
  3563. if (!ret) {
  3564. ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
  3565. if (ret)
  3566. pr_info("ftrace_graph: Couldn't activate tracepoint"
  3567. " probe to kernel_sched_switch\n");
  3568. }
  3569. kfree(ret_stack_list);
  3570. return ret;
  3571. }
  3572. /*
  3573. * Hibernation protection.
  3574. * The state of the current task is too much unstable during
  3575. * suspend/restore to disk. We want to protect against that.
  3576. */
  3577. static int
  3578. ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
  3579. void *unused)
  3580. {
  3581. switch (state) {
  3582. case PM_HIBERNATION_PREPARE:
  3583. pause_graph_tracing();
  3584. break;
  3585. case PM_POST_HIBERNATION:
  3586. unpause_graph_tracing();
  3587. break;
  3588. }
  3589. return NOTIFY_DONE;
  3590. }
  3591. int register_ftrace_graph(trace_func_graph_ret_t retfunc,
  3592. trace_func_graph_ent_t entryfunc)
  3593. {
  3594. int ret = 0;
  3595. mutex_lock(&ftrace_lock);
  3596. /* we currently allow only one tracer registered at a time */
  3597. if (ftrace_graph_active) {
  3598. ret = -EBUSY;
  3599. goto out;
  3600. }
  3601. ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
  3602. register_pm_notifier(&ftrace_suspend_notifier);
  3603. ftrace_graph_active++;
  3604. ret = start_graph_tracing();
  3605. if (ret) {
  3606. ftrace_graph_active--;
  3607. goto out;
  3608. }
  3609. ftrace_graph_return = retfunc;
  3610. ftrace_graph_entry = entryfunc;
  3611. ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
  3612. out:
  3613. mutex_unlock(&ftrace_lock);
  3614. return ret;
  3615. }
  3616. void unregister_ftrace_graph(void)
  3617. {
  3618. mutex_lock(&ftrace_lock);
  3619. if (unlikely(!ftrace_graph_active))
  3620. goto out;
  3621. ftrace_graph_active--;
  3622. ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
  3623. ftrace_graph_entry = ftrace_graph_entry_stub;
  3624. ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
  3625. unregister_pm_notifier(&ftrace_suspend_notifier);
  3626. unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
  3627. out:
  3628. mutex_unlock(&ftrace_lock);
  3629. }
  3630. static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
  3631. static void
  3632. graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
  3633. {
  3634. atomic_set(&t->tracing_graph_pause, 0);
  3635. atomic_set(&t->trace_overrun, 0);
  3636. t->ftrace_timestamp = 0;
  3637. /* make curr_ret_stack visible before we add the ret_stack */
  3638. smp_wmb();
  3639. t->ret_stack = ret_stack;
  3640. }
  3641. /*
  3642. * Allocate a return stack for the idle task. May be the first
  3643. * time through, or it may be done by CPU hotplug online.
  3644. */
  3645. void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
  3646. {
  3647. t->curr_ret_stack = -1;
  3648. /*
  3649. * The idle task has no parent, it either has its own
  3650. * stack or no stack at all.
  3651. */
  3652. if (t->ret_stack)
  3653. WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
  3654. if (ftrace_graph_active) {
  3655. struct ftrace_ret_stack *ret_stack;
  3656. ret_stack = per_cpu(idle_ret_stack, cpu);
  3657. if (!ret_stack) {
  3658. ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
  3659. * sizeof(struct ftrace_ret_stack),
  3660. GFP_KERNEL);
  3661. if (!ret_stack)
  3662. return;
  3663. per_cpu(idle_ret_stack, cpu) = ret_stack;
  3664. }
  3665. graph_init_task(t, ret_stack);
  3666. }
  3667. }
  3668. /* Allocate a return stack for newly created task */
  3669. void ftrace_graph_init_task(struct task_struct *t)
  3670. {
  3671. /* Make sure we do not use the parent ret_stack */
  3672. t->ret_stack = NULL;
  3673. t->curr_ret_stack = -1;
  3674. if (ftrace_graph_active) {
  3675. struct ftrace_ret_stack *ret_stack;
  3676. ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
  3677. * sizeof(struct ftrace_ret_stack),
  3678. GFP_KERNEL);
  3679. if (!ret_stack)
  3680. return;
  3681. graph_init_task(t, ret_stack);
  3682. }
  3683. }
  3684. void ftrace_graph_exit_task(struct task_struct *t)
  3685. {
  3686. struct ftrace_ret_stack *ret_stack = t->ret_stack;
  3687. t->ret_stack = NULL;
  3688. /* NULL must become visible to IRQs before we free it: */
  3689. barrier();
  3690. kfree(ret_stack);
  3691. }
  3692. void ftrace_graph_stop(void)
  3693. {
  3694. ftrace_stop();
  3695. }
  3696. #endif