ring_buffer.c 119 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615
  1. /*
  2. * Generic ring buffer
  3. *
  4. * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
  5. */
  6. #include <linux/ftrace_event.h>
  7. #include <linux/ring_buffer.h>
  8. #include <linux/trace_clock.h>
  9. #include <linux/trace_seq.h>
  10. #include <linux/spinlock.h>
  11. #include <linux/irq_work.h>
  12. #include <linux/debugfs.h>
  13. #include <linux/uaccess.h>
  14. #include <linux/hardirq.h>
  15. #include <linux/kmemcheck.h>
  16. #include <linux/module.h>
  17. #include <linux/percpu.h>
  18. #include <linux/mutex.h>
  19. #include <linux/slab.h>
  20. #include <linux/init.h>
  21. #include <linux/hash.h>
  22. #include <linux/list.h>
  23. #include <linux/cpu.h>
  24. #include <linux/fs.h>
  25. #include <asm/local.h>
  26. static void update_pages_handler(struct work_struct *work);
  27. /*
  28. * The ring buffer header is special. We must manually up keep it.
  29. */
  30. int ring_buffer_print_entry_header(struct trace_seq *s)
  31. {
  32. int ret;
  33. ret = trace_seq_printf(s, "# compressed entry header\n");
  34. ret = trace_seq_printf(s, "\ttype_len : 5 bits\n");
  35. ret = trace_seq_printf(s, "\ttime_delta : 27 bits\n");
  36. ret = trace_seq_printf(s, "\tarray : 32 bits\n");
  37. ret = trace_seq_printf(s, "\n");
  38. ret = trace_seq_printf(s, "\tpadding : type == %d\n",
  39. RINGBUF_TYPE_PADDING);
  40. ret = trace_seq_printf(s, "\ttime_extend : type == %d\n",
  41. RINGBUF_TYPE_TIME_EXTEND);
  42. ret = trace_seq_printf(s, "\tdata max type_len == %d\n",
  43. RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
  44. return ret;
  45. }
  46. /*
  47. * The ring buffer is made up of a list of pages. A separate list of pages is
  48. * allocated for each CPU. A writer may only write to a buffer that is
  49. * associated with the CPU it is currently executing on. A reader may read
  50. * from any per cpu buffer.
  51. *
  52. * The reader is special. For each per cpu buffer, the reader has its own
  53. * reader page. When a reader has read the entire reader page, this reader
  54. * page is swapped with another page in the ring buffer.
  55. *
  56. * Now, as long as the writer is off the reader page, the reader can do what
  57. * ever it wants with that page. The writer will never write to that page
  58. * again (as long as it is out of the ring buffer).
  59. *
  60. * Here's some silly ASCII art.
  61. *
  62. * +------+
  63. * |reader| RING BUFFER
  64. * |page |
  65. * +------+ +---+ +---+ +---+
  66. * | |-->| |-->| |
  67. * +---+ +---+ +---+
  68. * ^ |
  69. * | |
  70. * +---------------+
  71. *
  72. *
  73. * +------+
  74. * |reader| RING BUFFER
  75. * |page |------------------v
  76. * +------+ +---+ +---+ +---+
  77. * | |-->| |-->| |
  78. * +---+ +---+ +---+
  79. * ^ |
  80. * | |
  81. * +---------------+
  82. *
  83. *
  84. * +------+
  85. * |reader| RING BUFFER
  86. * |page |------------------v
  87. * +------+ +---+ +---+ +---+
  88. * ^ | |-->| |-->| |
  89. * | +---+ +---+ +---+
  90. * | |
  91. * | |
  92. * +------------------------------+
  93. *
  94. *
  95. * +------+
  96. * |buffer| RING BUFFER
  97. * |page |------------------v
  98. * +------+ +---+ +---+ +---+
  99. * ^ | | | |-->| |
  100. * | New +---+ +---+ +---+
  101. * | Reader------^ |
  102. * | page |
  103. * +------------------------------+
  104. *
  105. *
  106. * After we make this swap, the reader can hand this page off to the splice
  107. * code and be done with it. It can even allocate a new page if it needs to
  108. * and swap that into the ring buffer.
  109. *
  110. * We will be using cmpxchg soon to make all this lockless.
  111. *
  112. */
  113. /*
  114. * A fast way to enable or disable all ring buffers is to
  115. * call tracing_on or tracing_off. Turning off the ring buffers
  116. * prevents all ring buffers from being recorded to.
  117. * Turning this switch on, makes it OK to write to the
  118. * ring buffer, if the ring buffer is enabled itself.
  119. *
  120. * There's three layers that must be on in order to write
  121. * to the ring buffer.
  122. *
  123. * 1) This global flag must be set.
  124. * 2) The ring buffer must be enabled for recording.
  125. * 3) The per cpu buffer must be enabled for recording.
  126. *
  127. * In case of an anomaly, this global flag has a bit set that
  128. * will permantly disable all ring buffers.
  129. */
  130. /*
  131. * Global flag to disable all recording to ring buffers
  132. * This has two bits: ON, DISABLED
  133. *
  134. * ON DISABLED
  135. * ---- ----------
  136. * 0 0 : ring buffers are off
  137. * 1 0 : ring buffers are on
  138. * X 1 : ring buffers are permanently disabled
  139. */
  140. enum {
  141. RB_BUFFERS_ON_BIT = 0,
  142. RB_BUFFERS_DISABLED_BIT = 1,
  143. };
  144. enum {
  145. RB_BUFFERS_ON = 1 << RB_BUFFERS_ON_BIT,
  146. RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT,
  147. };
  148. static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON;
  149. /* Used for individual buffers (after the counter) */
  150. #define RB_BUFFER_OFF (1 << 20)
  151. #define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)
  152. /**
  153. * tracing_off_permanent - permanently disable ring buffers
  154. *
  155. * This function, once called, will disable all ring buffers
  156. * permanently.
  157. */
  158. void tracing_off_permanent(void)
  159. {
  160. set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);
  161. }
  162. #define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
  163. #define RB_ALIGNMENT 4U
  164. #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
  165. #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */
  166. #if !defined(CONFIG_64BIT) || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
  167. # define RB_FORCE_8BYTE_ALIGNMENT 0
  168. # define RB_ARCH_ALIGNMENT RB_ALIGNMENT
  169. #else
  170. # define RB_FORCE_8BYTE_ALIGNMENT 1
  171. # define RB_ARCH_ALIGNMENT 8U
  172. #endif
  173. /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */
  174. #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX
  175. enum {
  176. RB_LEN_TIME_EXTEND = 8,
  177. RB_LEN_TIME_STAMP = 16,
  178. };
  179. #define skip_time_extend(event) \
  180. ((struct ring_buffer_event *)((char *)event + RB_LEN_TIME_EXTEND))
  181. static inline int rb_null_event(struct ring_buffer_event *event)
  182. {
  183. return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta;
  184. }
  185. static void rb_event_set_padding(struct ring_buffer_event *event)
  186. {
  187. /* padding has a NULL time_delta */
  188. event->type_len = RINGBUF_TYPE_PADDING;
  189. event->time_delta = 0;
  190. }
  191. static unsigned
  192. rb_event_data_length(struct ring_buffer_event *event)
  193. {
  194. unsigned length;
  195. if (event->type_len)
  196. length = event->type_len * RB_ALIGNMENT;
  197. else
  198. length = event->array[0];
  199. return length + RB_EVNT_HDR_SIZE;
  200. }
  201. /*
  202. * Return the length of the given event. Will return
  203. * the length of the time extend if the event is a
  204. * time extend.
  205. */
  206. static inline unsigned
  207. rb_event_length(struct ring_buffer_event *event)
  208. {
  209. switch (event->type_len) {
  210. case RINGBUF_TYPE_PADDING:
  211. if (rb_null_event(event))
  212. /* undefined */
  213. return -1;
  214. return event->array[0] + RB_EVNT_HDR_SIZE;
  215. case RINGBUF_TYPE_TIME_EXTEND:
  216. return RB_LEN_TIME_EXTEND;
  217. case RINGBUF_TYPE_TIME_STAMP:
  218. return RB_LEN_TIME_STAMP;
  219. case RINGBUF_TYPE_DATA:
  220. return rb_event_data_length(event);
  221. default:
  222. BUG();
  223. }
  224. /* not hit */
  225. return 0;
  226. }
  227. /*
  228. * Return total length of time extend and data,
  229. * or just the event length for all other events.
  230. */
  231. static inline unsigned
  232. rb_event_ts_length(struct ring_buffer_event *event)
  233. {
  234. unsigned len = 0;
  235. if (event->type_len == RINGBUF_TYPE_TIME_EXTEND) {
  236. /* time extends include the data event after it */
  237. len = RB_LEN_TIME_EXTEND;
  238. event = skip_time_extend(event);
  239. }
  240. return len + rb_event_length(event);
  241. }
  242. /**
  243. * ring_buffer_event_length - return the length of the event
  244. * @event: the event to get the length of
  245. *
  246. * Returns the size of the data load of a data event.
  247. * If the event is something other than a data event, it
  248. * returns the size of the event itself. With the exception
  249. * of a TIME EXTEND, where it still returns the size of the
  250. * data load of the data event after it.
  251. */
  252. unsigned ring_buffer_event_length(struct ring_buffer_event *event)
  253. {
  254. unsigned length;
  255. if (event->type_len == RINGBUF_TYPE_TIME_EXTEND)
  256. event = skip_time_extend(event);
  257. length = rb_event_length(event);
  258. if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
  259. return length;
  260. length -= RB_EVNT_HDR_SIZE;
  261. if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))
  262. length -= sizeof(event->array[0]);
  263. return length;
  264. }
  265. EXPORT_SYMBOL_GPL(ring_buffer_event_length);
  266. /* inline for ring buffer fast paths */
  267. static void *
  268. rb_event_data(struct ring_buffer_event *event)
  269. {
  270. if (event->type_len == RINGBUF_TYPE_TIME_EXTEND)
  271. event = skip_time_extend(event);
  272. BUG_ON(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
  273. /* If length is in len field, then array[0] has the data */
  274. if (event->type_len)
  275. return (void *)&event->array[0];
  276. /* Otherwise length is in array[0] and array[1] has the data */
  277. return (void *)&event->array[1];
  278. }
  279. /**
  280. * ring_buffer_event_data - return the data of the event
  281. * @event: the event to get the data from
  282. */
  283. void *ring_buffer_event_data(struct ring_buffer_event *event)
  284. {
  285. return rb_event_data(event);
  286. }
  287. EXPORT_SYMBOL_GPL(ring_buffer_event_data);
  288. #define for_each_buffer_cpu(buffer, cpu) \
  289. for_each_cpu(cpu, buffer->cpumask)
  290. #define TS_SHIFT 27
  291. #define TS_MASK ((1ULL << TS_SHIFT) - 1)
  292. #define TS_DELTA_TEST (~TS_MASK)
  293. /* Flag when events were overwritten */
  294. #define RB_MISSED_EVENTS (1 << 31)
  295. /* Missed count stored at end */
  296. #define RB_MISSED_STORED (1 << 30)
  297. struct buffer_data_page {
  298. u64 time_stamp; /* page time stamp */
  299. local_t commit; /* write committed index */
  300. unsigned char data[]; /* data of buffer page */
  301. };
  302. /*
  303. * Note, the buffer_page list must be first. The buffer pages
  304. * are allocated in cache lines, which means that each buffer
  305. * page will be at the beginning of a cache line, and thus
  306. * the least significant bits will be zero. We use this to
  307. * add flags in the list struct pointers, to make the ring buffer
  308. * lockless.
  309. */
  310. struct buffer_page {
  311. struct list_head list; /* list of buffer pages */
  312. local_t write; /* index for next write */
  313. unsigned read; /* index for next read */
  314. local_t entries; /* entries on this page */
  315. unsigned long real_end; /* real end of data */
  316. struct buffer_data_page *page; /* Actual data page */
  317. };
  318. /*
  319. * The buffer page counters, write and entries, must be reset
  320. * atomically when crossing page boundaries. To synchronize this
  321. * update, two counters are inserted into the number. One is
  322. * the actual counter for the write position or count on the page.
  323. *
  324. * The other is a counter of updaters. Before an update happens
  325. * the update partition of the counter is incremented. This will
  326. * allow the updater to update the counter atomically.
  327. *
  328. * The counter is 20 bits, and the state data is 12.
  329. */
  330. #define RB_WRITE_MASK 0xfffff
  331. #define RB_WRITE_INTCNT (1 << 20)
  332. static void rb_init_page(struct buffer_data_page *bpage)
  333. {
  334. local_set(&bpage->commit, 0);
  335. }
  336. /**
  337. * ring_buffer_page_len - the size of data on the page.
  338. * @page: The page to read
  339. *
  340. * Returns the amount of data on the page, including buffer page header.
  341. */
  342. size_t ring_buffer_page_len(void *page)
  343. {
  344. return local_read(&((struct buffer_data_page *)page)->commit)
  345. + BUF_PAGE_HDR_SIZE;
  346. }
  347. /*
  348. * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
  349. * this issue out.
  350. */
  351. static void free_buffer_page(struct buffer_page *bpage)
  352. {
  353. free_page((unsigned long)bpage->page);
  354. kfree(bpage);
  355. }
  356. /*
  357. * We need to fit the time_stamp delta into 27 bits.
  358. */
  359. static inline int test_time_stamp(u64 delta)
  360. {
  361. if (delta & TS_DELTA_TEST)
  362. return 1;
  363. return 0;
  364. }
  365. #define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
  366. /* Max payload is BUF_PAGE_SIZE - header (8bytes) */
  367. #define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2))
  368. int ring_buffer_print_page_header(struct trace_seq *s)
  369. {
  370. struct buffer_data_page field;
  371. int ret;
  372. ret = trace_seq_printf(s, "\tfield: u64 timestamp;\t"
  373. "offset:0;\tsize:%u;\tsigned:%u;\n",
  374. (unsigned int)sizeof(field.time_stamp),
  375. (unsigned int)is_signed_type(u64));
  376. ret = trace_seq_printf(s, "\tfield: local_t commit;\t"
  377. "offset:%u;\tsize:%u;\tsigned:%u;\n",
  378. (unsigned int)offsetof(typeof(field), commit),
  379. (unsigned int)sizeof(field.commit),
  380. (unsigned int)is_signed_type(long));
  381. ret = trace_seq_printf(s, "\tfield: int overwrite;\t"
  382. "offset:%u;\tsize:%u;\tsigned:%u;\n",
  383. (unsigned int)offsetof(typeof(field), commit),
  384. 1,
  385. (unsigned int)is_signed_type(long));
  386. ret = trace_seq_printf(s, "\tfield: char data;\t"
  387. "offset:%u;\tsize:%u;\tsigned:%u;\n",
  388. (unsigned int)offsetof(typeof(field), data),
  389. (unsigned int)BUF_PAGE_SIZE,
  390. (unsigned int)is_signed_type(char));
  391. return ret;
  392. }
  393. struct rb_irq_work {
  394. struct irq_work work;
  395. wait_queue_head_t waiters;
  396. bool waiters_pending;
  397. };
  398. /*
  399. * head_page == tail_page && head == tail then buffer is empty.
  400. */
  401. struct ring_buffer_per_cpu {
  402. int cpu;
  403. atomic_t record_disabled;
  404. struct ring_buffer *buffer;
  405. raw_spinlock_t reader_lock; /* serialize readers */
  406. arch_spinlock_t lock;
  407. struct lock_class_key lock_key;
  408. unsigned int nr_pages;
  409. struct list_head *pages;
  410. struct buffer_page *head_page; /* read from head */
  411. struct buffer_page *tail_page; /* write to tail */
  412. struct buffer_page *commit_page; /* committed pages */
  413. struct buffer_page *reader_page;
  414. unsigned long lost_events;
  415. unsigned long last_overrun;
  416. local_t entries_bytes;
  417. local_t entries;
  418. local_t overrun;
  419. local_t commit_overrun;
  420. local_t dropped_events;
  421. local_t committing;
  422. local_t commits;
  423. unsigned long read;
  424. unsigned long read_bytes;
  425. u64 write_stamp;
  426. u64 read_stamp;
  427. /* ring buffer pages to update, > 0 to add, < 0 to remove */
  428. int nr_pages_to_update;
  429. struct list_head new_pages; /* new pages to add */
  430. struct work_struct update_pages_work;
  431. struct completion update_done;
  432. struct rb_irq_work irq_work;
  433. };
  434. struct ring_buffer {
  435. unsigned flags;
  436. int cpus;
  437. atomic_t record_disabled;
  438. atomic_t resize_disabled;
  439. cpumask_var_t cpumask;
  440. struct lock_class_key *reader_lock_key;
  441. struct mutex mutex;
  442. struct ring_buffer_per_cpu **buffers;
  443. #ifdef CONFIG_HOTPLUG_CPU
  444. struct notifier_block cpu_notify;
  445. #endif
  446. u64 (*clock)(void);
  447. struct rb_irq_work irq_work;
  448. };
  449. struct ring_buffer_iter {
  450. struct ring_buffer_per_cpu *cpu_buffer;
  451. unsigned long head;
  452. struct buffer_page *head_page;
  453. struct buffer_page *cache_reader_page;
  454. unsigned long cache_read;
  455. u64 read_stamp;
  456. };
  457. /*
  458. * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
  459. *
  460. * Schedules a delayed work to wake up any task that is blocked on the
  461. * ring buffer waiters queue.
  462. */
  463. static void rb_wake_up_waiters(struct irq_work *work)
  464. {
  465. struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);
  466. wake_up_all(&rbwork->waiters);
  467. }
  468. /**
  469. * ring_buffer_wait - wait for input to the ring buffer
  470. * @buffer: buffer to wait on
  471. * @cpu: the cpu buffer to wait on
  472. *
  473. * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
  474. * as data is added to any of the @buffer's cpu buffers. Otherwise
  475. * it will wait for data to be added to a specific cpu buffer.
  476. */
  477. void ring_buffer_wait(struct ring_buffer *buffer, int cpu)
  478. {
  479. struct ring_buffer_per_cpu *cpu_buffer;
  480. DEFINE_WAIT(wait);
  481. struct rb_irq_work *work;
  482. /*
  483. * Depending on what the caller is waiting for, either any
  484. * data in any cpu buffer, or a specific buffer, put the
  485. * caller on the appropriate wait queue.
  486. */
  487. if (cpu == RING_BUFFER_ALL_CPUS)
  488. work = &buffer->irq_work;
  489. else {
  490. cpu_buffer = buffer->buffers[cpu];
  491. work = &cpu_buffer->irq_work;
  492. }
  493. prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE);
  494. /*
  495. * The events can happen in critical sections where
  496. * checking a work queue can cause deadlocks.
  497. * After adding a task to the queue, this flag is set
  498. * only to notify events to try to wake up the queue
  499. * using irq_work.
  500. *
  501. * We don't clear it even if the buffer is no longer
  502. * empty. The flag only causes the next event to run
  503. * irq_work to do the work queue wake up. The worse
  504. * that can happen if we race with !trace_empty() is that
  505. * an event will cause an irq_work to try to wake up
  506. * an empty queue.
  507. *
  508. * There's no reason to protect this flag either, as
  509. * the work queue and irq_work logic will do the necessary
  510. * synchronization for the wake ups. The only thing
  511. * that is necessary is that the wake up happens after
  512. * a task has been queued. It's OK for spurious wake ups.
  513. */
  514. work->waiters_pending = true;
  515. if ((cpu == RING_BUFFER_ALL_CPUS && ring_buffer_empty(buffer)) ||
  516. (cpu != RING_BUFFER_ALL_CPUS && ring_buffer_empty_cpu(buffer, cpu)))
  517. schedule();
  518. finish_wait(&work->waiters, &wait);
  519. }
  520. /**
  521. * ring_buffer_poll_wait - poll on buffer input
  522. * @buffer: buffer to wait on
  523. * @cpu: the cpu buffer to wait on
  524. * @filp: the file descriptor
  525. * @poll_table: The poll descriptor
  526. *
  527. * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
  528. * as data is added to any of the @buffer's cpu buffers. Otherwise
  529. * it will wait for data to be added to a specific cpu buffer.
  530. *
  531. * Returns POLLIN | POLLRDNORM if data exists in the buffers,
  532. * zero otherwise.
  533. */
  534. int ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu,
  535. struct file *filp, poll_table *poll_table)
  536. {
  537. struct ring_buffer_per_cpu *cpu_buffer;
  538. struct rb_irq_work *work;
  539. if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) ||
  540. (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))
  541. return POLLIN | POLLRDNORM;
  542. if (cpu == RING_BUFFER_ALL_CPUS)
  543. work = &buffer->irq_work;
  544. else {
  545. cpu_buffer = buffer->buffers[cpu];
  546. work = &cpu_buffer->irq_work;
  547. }
  548. work->waiters_pending = true;
  549. poll_wait(filp, &work->waiters, poll_table);
  550. if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) ||
  551. (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))
  552. return POLLIN | POLLRDNORM;
  553. return 0;
  554. }
  555. /* buffer may be either ring_buffer or ring_buffer_per_cpu */
  556. #define RB_WARN_ON(b, cond) \
  557. ({ \
  558. int _____ret = unlikely(cond); \
  559. if (_____ret) { \
  560. if (__same_type(*(b), struct ring_buffer_per_cpu)) { \
  561. struct ring_buffer_per_cpu *__b = \
  562. (void *)b; \
  563. atomic_inc(&__b->buffer->record_disabled); \
  564. } else \
  565. atomic_inc(&b->record_disabled); \
  566. WARN_ON(1); \
  567. } \
  568. _____ret; \
  569. })
  570. /* Up this if you want to test the TIME_EXTENTS and normalization */
  571. #define DEBUG_SHIFT 0
  572. static inline u64 rb_time_stamp(struct ring_buffer *buffer)
  573. {
  574. /* shift to debug/test normalization and TIME_EXTENTS */
  575. return buffer->clock() << DEBUG_SHIFT;
  576. }
  577. u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu)
  578. {
  579. u64 time;
  580. preempt_disable_notrace();
  581. time = rb_time_stamp(buffer);
  582. preempt_enable_no_resched_notrace();
  583. return time;
  584. }
  585. EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);
  586. void ring_buffer_normalize_time_stamp(struct ring_buffer *buffer,
  587. int cpu, u64 *ts)
  588. {
  589. /* Just stupid testing the normalize function and deltas */
  590. *ts >>= DEBUG_SHIFT;
  591. }
  592. EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
  593. /*
  594. * Making the ring buffer lockless makes things tricky.
  595. * Although writes only happen on the CPU that they are on,
  596. * and they only need to worry about interrupts. Reads can
  597. * happen on any CPU.
  598. *
  599. * The reader page is always off the ring buffer, but when the
  600. * reader finishes with a page, it needs to swap its page with
  601. * a new one from the buffer. The reader needs to take from
  602. * the head (writes go to the tail). But if a writer is in overwrite
  603. * mode and wraps, it must push the head page forward.
  604. *
  605. * Here lies the problem.
  606. *
  607. * The reader must be careful to replace only the head page, and
  608. * not another one. As described at the top of the file in the
  609. * ASCII art, the reader sets its old page to point to the next
  610. * page after head. It then sets the page after head to point to
  611. * the old reader page. But if the writer moves the head page
  612. * during this operation, the reader could end up with the tail.
  613. *
  614. * We use cmpxchg to help prevent this race. We also do something
  615. * special with the page before head. We set the LSB to 1.
  616. *
  617. * When the writer must push the page forward, it will clear the
  618. * bit that points to the head page, move the head, and then set
  619. * the bit that points to the new head page.
  620. *
  621. * We also don't want an interrupt coming in and moving the head
  622. * page on another writer. Thus we use the second LSB to catch
  623. * that too. Thus:
  624. *
  625. * head->list->prev->next bit 1 bit 0
  626. * ------- -------
  627. * Normal page 0 0
  628. * Points to head page 0 1
  629. * New head page 1 0
  630. *
  631. * Note we can not trust the prev pointer of the head page, because:
  632. *
  633. * +----+ +-----+ +-----+
  634. * | |------>| T |---X--->| N |
  635. * | |<------| | | |
  636. * +----+ +-----+ +-----+
  637. * ^ ^ |
  638. * | +-----+ | |
  639. * +----------| R |----------+ |
  640. * | |<-----------+
  641. * +-----+
  642. *
  643. * Key: ---X--> HEAD flag set in pointer
  644. * T Tail page
  645. * R Reader page
  646. * N Next page
  647. *
  648. * (see __rb_reserve_next() to see where this happens)
  649. *
  650. * What the above shows is that the reader just swapped out
  651. * the reader page with a page in the buffer, but before it
  652. * could make the new header point back to the new page added
  653. * it was preempted by a writer. The writer moved forward onto
  654. * the new page added by the reader and is about to move forward
  655. * again.
  656. *
  657. * You can see, it is legitimate for the previous pointer of
  658. * the head (or any page) not to point back to itself. But only
  659. * temporarially.
  660. */
  661. #define RB_PAGE_NORMAL 0UL
  662. #define RB_PAGE_HEAD 1UL
  663. #define RB_PAGE_UPDATE 2UL
  664. #define RB_FLAG_MASK 3UL
  665. /* PAGE_MOVED is not part of the mask */
  666. #define RB_PAGE_MOVED 4UL
  667. /*
  668. * rb_list_head - remove any bit
  669. */
  670. static struct list_head *rb_list_head(struct list_head *list)
  671. {
  672. unsigned long val = (unsigned long)list;
  673. return (struct list_head *)(val & ~RB_FLAG_MASK);
  674. }
  675. /*
  676. * rb_is_head_page - test if the given page is the head page
  677. *
  678. * Because the reader may move the head_page pointer, we can
  679. * not trust what the head page is (it may be pointing to
  680. * the reader page). But if the next page is a header page,
  681. * its flags will be non zero.
  682. */
  683. static inline int
  684. rb_is_head_page(struct ring_buffer_per_cpu *cpu_buffer,
  685. struct buffer_page *page, struct list_head *list)
  686. {
  687. unsigned long val;
  688. val = (unsigned long)list->next;
  689. if ((val & ~RB_FLAG_MASK) != (unsigned long)&page->list)
  690. return RB_PAGE_MOVED;
  691. return val & RB_FLAG_MASK;
  692. }
  693. /*
  694. * rb_is_reader_page
  695. *
  696. * The unique thing about the reader page, is that, if the
  697. * writer is ever on it, the previous pointer never points
  698. * back to the reader page.
  699. */
  700. static int rb_is_reader_page(struct buffer_page *page)
  701. {
  702. struct list_head *list = page->list.prev;
  703. return rb_list_head(list->next) != &page->list;
  704. }
  705. /*
  706. * rb_set_list_to_head - set a list_head to be pointing to head.
  707. */
  708. static void rb_set_list_to_head(struct ring_buffer_per_cpu *cpu_buffer,
  709. struct list_head *list)
  710. {
  711. unsigned long *ptr;
  712. ptr = (unsigned long *)&list->next;
  713. *ptr |= RB_PAGE_HEAD;
  714. *ptr &= ~RB_PAGE_UPDATE;
  715. }
  716. /*
  717. * rb_head_page_activate - sets up head page
  718. */
  719. static void rb_head_page_activate(struct ring_buffer_per_cpu *cpu_buffer)
  720. {
  721. struct buffer_page *head;
  722. head = cpu_buffer->head_page;
  723. if (!head)
  724. return;
  725. /*
  726. * Set the previous list pointer to have the HEAD flag.
  727. */
  728. rb_set_list_to_head(cpu_buffer, head->list.prev);
  729. }
  730. static void rb_list_head_clear(struct list_head *list)
  731. {
  732. unsigned long *ptr = (unsigned long *)&list->next;
  733. *ptr &= ~RB_FLAG_MASK;
  734. }
  735. /*
  736. * rb_head_page_dactivate - clears head page ptr (for free list)
  737. */
  738. static void
  739. rb_head_page_deactivate(struct ring_buffer_per_cpu *cpu_buffer)
  740. {
  741. struct list_head *hd;
  742. /* Go through the whole list and clear any pointers found. */
  743. rb_list_head_clear(cpu_buffer->pages);
  744. list_for_each(hd, cpu_buffer->pages)
  745. rb_list_head_clear(hd);
  746. }
  747. static int rb_head_page_set(struct ring_buffer_per_cpu *cpu_buffer,
  748. struct buffer_page *head,
  749. struct buffer_page *prev,
  750. int old_flag, int new_flag)
  751. {
  752. struct list_head *list;
  753. unsigned long val = (unsigned long)&head->list;
  754. unsigned long ret;
  755. list = &prev->list;
  756. val &= ~RB_FLAG_MASK;
  757. ret = cmpxchg((unsigned long *)&list->next,
  758. val | old_flag, val | new_flag);
  759. /* check if the reader took the page */
  760. if ((ret & ~RB_FLAG_MASK) != val)
  761. return RB_PAGE_MOVED;
  762. return ret & RB_FLAG_MASK;
  763. }
  764. static int rb_head_page_set_update(struct ring_buffer_per_cpu *cpu_buffer,
  765. struct buffer_page *head,
  766. struct buffer_page *prev,
  767. int old_flag)
  768. {
  769. return rb_head_page_set(cpu_buffer, head, prev,
  770. old_flag, RB_PAGE_UPDATE);
  771. }
  772. static int rb_head_page_set_head(struct ring_buffer_per_cpu *cpu_buffer,
  773. struct buffer_page *head,
  774. struct buffer_page *prev,
  775. int old_flag)
  776. {
  777. return rb_head_page_set(cpu_buffer, head, prev,
  778. old_flag, RB_PAGE_HEAD);
  779. }
  780. static int rb_head_page_set_normal(struct ring_buffer_per_cpu *cpu_buffer,
  781. struct buffer_page *head,
  782. struct buffer_page *prev,
  783. int old_flag)
  784. {
  785. return rb_head_page_set(cpu_buffer, head, prev,
  786. old_flag, RB_PAGE_NORMAL);
  787. }
  788. static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer,
  789. struct buffer_page **bpage)
  790. {
  791. struct list_head *p = rb_list_head((*bpage)->list.next);
  792. *bpage = list_entry(p, struct buffer_page, list);
  793. }
  794. static struct buffer_page *
  795. rb_set_head_page(struct ring_buffer_per_cpu *cpu_buffer)
  796. {
  797. struct buffer_page *head;
  798. struct buffer_page *page;
  799. struct list_head *list;
  800. int i;
  801. if (RB_WARN_ON(cpu_buffer, !cpu_buffer->head_page))
  802. return NULL;
  803. /* sanity check */
  804. list = cpu_buffer->pages;
  805. if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev->next) != list))
  806. return NULL;
  807. page = head = cpu_buffer->head_page;
  808. /*
  809. * It is possible that the writer moves the header behind
  810. * where we started, and we miss in one loop.
  811. * A second loop should grab the header, but we'll do
  812. * three loops just because I'm paranoid.
  813. */
  814. for (i = 0; i < 3; i++) {
  815. do {
  816. if (rb_is_head_page(cpu_buffer, page, page->list.prev)) {
  817. cpu_buffer->head_page = page;
  818. return page;
  819. }
  820. rb_inc_page(cpu_buffer, &page);
  821. } while (page != head);
  822. }
  823. RB_WARN_ON(cpu_buffer, 1);
  824. return NULL;
  825. }
  826. static int rb_head_page_replace(struct buffer_page *old,
  827. struct buffer_page *new)
  828. {
  829. unsigned long *ptr = (unsigned long *)&old->list.prev->next;
  830. unsigned long val;
  831. unsigned long ret;
  832. val = *ptr & ~RB_FLAG_MASK;
  833. val |= RB_PAGE_HEAD;
  834. ret = cmpxchg(ptr, val, (unsigned long)&new->list);
  835. return ret == val;
  836. }
  837. /*
  838. * rb_tail_page_update - move the tail page forward
  839. *
  840. * Returns 1 if moved tail page, 0 if someone else did.
  841. */
  842. static int rb_tail_page_update(struct ring_buffer_per_cpu *cpu_buffer,
  843. struct buffer_page *tail_page,
  844. struct buffer_page *next_page)
  845. {
  846. struct buffer_page *old_tail;
  847. unsigned long old_entries;
  848. unsigned long old_write;
  849. int ret = 0;
  850. /*
  851. * The tail page now needs to be moved forward.
  852. *
  853. * We need to reset the tail page, but without messing
  854. * with possible erasing of data brought in by interrupts
  855. * that have moved the tail page and are currently on it.
  856. *
  857. * We add a counter to the write field to denote this.
  858. */
  859. old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
  860. old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);
  861. /*
  862. * Just make sure we have seen our old_write and synchronize
  863. * with any interrupts that come in.
  864. */
  865. barrier();
  866. /*
  867. * If the tail page is still the same as what we think
  868. * it is, then it is up to us to update the tail
  869. * pointer.
  870. */
  871. if (tail_page == cpu_buffer->tail_page) {
  872. /* Zero the write counter */
  873. unsigned long val = old_write & ~RB_WRITE_MASK;
  874. unsigned long eval = old_entries & ~RB_WRITE_MASK;
  875. /*
  876. * This will only succeed if an interrupt did
  877. * not come in and change it. In which case, we
  878. * do not want to modify it.
  879. *
  880. * We add (void) to let the compiler know that we do not care
  881. * about the return value of these functions. We use the
  882. * cmpxchg to only update if an interrupt did not already
  883. * do it for us. If the cmpxchg fails, we don't care.
  884. */
  885. (void)local_cmpxchg(&next_page->write, old_write, val);
  886. (void)local_cmpxchg(&next_page->entries, old_entries, eval);
  887. /*
  888. * No need to worry about races with clearing out the commit.
  889. * it only can increment when a commit takes place. But that
  890. * only happens in the outer most nested commit.
  891. */
  892. local_set(&next_page->page->commit, 0);
  893. old_tail = cmpxchg(&cpu_buffer->tail_page,
  894. tail_page, next_page);
  895. if (old_tail == tail_page)
  896. ret = 1;
  897. }
  898. return ret;
  899. }
  900. static int rb_check_bpage(struct ring_buffer_per_cpu *cpu_buffer,
  901. struct buffer_page *bpage)
  902. {
  903. unsigned long val = (unsigned long)bpage;
  904. if (RB_WARN_ON(cpu_buffer, val & RB_FLAG_MASK))
  905. return 1;
  906. return 0;
  907. }
  908. /**
  909. * rb_check_list - make sure a pointer to a list has the last bits zero
  910. */
  911. static int rb_check_list(struct ring_buffer_per_cpu *cpu_buffer,
  912. struct list_head *list)
  913. {
  914. if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev) != list->prev))
  915. return 1;
  916. if (RB_WARN_ON(cpu_buffer, rb_list_head(list->next) != list->next))
  917. return 1;
  918. return 0;
  919. }
  920. /**
  921. * check_pages - integrity check of buffer pages
  922. * @cpu_buffer: CPU buffer with pages to test
  923. *
  924. * As a safety measure we check to make sure the data pages have not
  925. * been corrupted.
  926. */
  927. static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
  928. {
  929. struct list_head *head = cpu_buffer->pages;
  930. struct buffer_page *bpage, *tmp;
  931. /* Reset the head page if it exists */
  932. if (cpu_buffer->head_page)
  933. rb_set_head_page(cpu_buffer);
  934. rb_head_page_deactivate(cpu_buffer);
  935. if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
  936. return -1;
  937. if (RB_WARN_ON(cpu_buffer, head->prev->next != head))
  938. return -1;
  939. if (rb_check_list(cpu_buffer, head))
  940. return -1;
  941. list_for_each_entry_safe(bpage, tmp, head, list) {
  942. if (RB_WARN_ON(cpu_buffer,
  943. bpage->list.next->prev != &bpage->list))
  944. return -1;
  945. if (RB_WARN_ON(cpu_buffer,
  946. bpage->list.prev->next != &bpage->list))
  947. return -1;
  948. if (rb_check_list(cpu_buffer, &bpage->list))
  949. return -1;
  950. }
  951. rb_head_page_activate(cpu_buffer);
  952. return 0;
  953. }
  954. static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu)
  955. {
  956. int i;
  957. struct buffer_page *bpage, *tmp;
  958. for (i = 0; i < nr_pages; i++) {
  959. struct page *page;
  960. /*
  961. * __GFP_NORETRY flag makes sure that the allocation fails
  962. * gracefully without invoking oom-killer and the system is
  963. * not destabilized.
  964. */
  965. bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
  966. GFP_KERNEL | __GFP_NORETRY,
  967. cpu_to_node(cpu));
  968. if (!bpage)
  969. goto free_pages;
  970. list_add(&bpage->list, pages);
  971. page = alloc_pages_node(cpu_to_node(cpu),
  972. GFP_KERNEL | __GFP_NORETRY, 0);
  973. if (!page)
  974. goto free_pages;
  975. bpage->page = page_address(page);
  976. rb_init_page(bpage->page);
  977. }
  978. return 0;
  979. free_pages:
  980. list_for_each_entry_safe(bpage, tmp, pages, list) {
  981. list_del_init(&bpage->list);
  982. free_buffer_page(bpage);
  983. }
  984. return -ENOMEM;
  985. }
  986. static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
  987. unsigned nr_pages)
  988. {
  989. LIST_HEAD(pages);
  990. WARN_ON(!nr_pages);
  991. if (__rb_allocate_pages(nr_pages, &pages, cpu_buffer->cpu))
  992. return -ENOMEM;
  993. /*
  994. * The ring buffer page list is a circular list that does not
  995. * start and end with a list head. All page list items point to
  996. * other pages.
  997. */
  998. cpu_buffer->pages = pages.next;
  999. list_del(&pages);
  1000. cpu_buffer->nr_pages = nr_pages;
  1001. rb_check_pages(cpu_buffer);
  1002. return 0;
  1003. }
  1004. static struct ring_buffer_per_cpu *
  1005. rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu)
  1006. {
  1007. struct ring_buffer_per_cpu *cpu_buffer;
  1008. struct buffer_page *bpage;
  1009. struct page *page;
  1010. int ret;
  1011. cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()),
  1012. GFP_KERNEL, cpu_to_node(cpu));
  1013. if (!cpu_buffer)
  1014. return NULL;
  1015. cpu_buffer->cpu = cpu;
  1016. cpu_buffer->buffer = buffer;
  1017. raw_spin_lock_init(&cpu_buffer->reader_lock);
  1018. lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);
  1019. cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  1020. INIT_WORK(&cpu_buffer->update_pages_work, update_pages_handler);
  1021. init_completion(&cpu_buffer->update_done);
  1022. init_irq_work(&cpu_buffer->irq_work.work, rb_wake_up_waiters);
  1023. init_waitqueue_head(&cpu_buffer->irq_work.waiters);
  1024. bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
  1025. GFP_KERNEL, cpu_to_node(cpu));
  1026. if (!bpage)
  1027. goto fail_free_buffer;
  1028. rb_check_bpage(cpu_buffer, bpage);
  1029. cpu_buffer->reader_page = bpage;
  1030. page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0);
  1031. if (!page)
  1032. goto fail_free_reader;
  1033. bpage->page = page_address(page);
  1034. rb_init_page(bpage->page);
  1035. INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
  1036. INIT_LIST_HEAD(&cpu_buffer->new_pages);
  1037. ret = rb_allocate_pages(cpu_buffer, nr_pages);
  1038. if (ret < 0)
  1039. goto fail_free_reader;
  1040. cpu_buffer->head_page
  1041. = list_entry(cpu_buffer->pages, struct buffer_page, list);
  1042. cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
  1043. rb_head_page_activate(cpu_buffer);
  1044. return cpu_buffer;
  1045. fail_free_reader:
  1046. free_buffer_page(cpu_buffer->reader_page);
  1047. fail_free_buffer:
  1048. kfree(cpu_buffer);
  1049. return NULL;
  1050. }
  1051. static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
  1052. {
  1053. struct list_head *head = cpu_buffer->pages;
  1054. struct buffer_page *bpage, *tmp;
  1055. free_buffer_page(cpu_buffer->reader_page);
  1056. rb_head_page_deactivate(cpu_buffer);
  1057. if (head) {
  1058. list_for_each_entry_safe(bpage, tmp, head, list) {
  1059. list_del_init(&bpage->list);
  1060. free_buffer_page(bpage);
  1061. }
  1062. bpage = list_entry(head, struct buffer_page, list);
  1063. free_buffer_page(bpage);
  1064. }
  1065. kfree(cpu_buffer);
  1066. }
  1067. #ifdef CONFIG_HOTPLUG_CPU
  1068. static int rb_cpu_notify(struct notifier_block *self,
  1069. unsigned long action, void *hcpu);
  1070. #endif
  1071. /**
  1072. * ring_buffer_alloc - allocate a new ring_buffer
  1073. * @size: the size in bytes per cpu that is needed.
  1074. * @flags: attributes to set for the ring buffer.
  1075. *
  1076. * Currently the only flag that is available is the RB_FL_OVERWRITE
  1077. * flag. This flag means that the buffer will overwrite old data
  1078. * when the buffer wraps. If this flag is not set, the buffer will
  1079. * drop data when the tail hits the head.
  1080. */
  1081. struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
  1082. struct lock_class_key *key)
  1083. {
  1084. struct ring_buffer *buffer;
  1085. int bsize;
  1086. int cpu, nr_pages;
  1087. /* keep it in its own cache line */
  1088. buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
  1089. GFP_KERNEL);
  1090. if (!buffer)
  1091. return NULL;
  1092. if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
  1093. goto fail_free_buffer;
  1094. nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
  1095. buffer->flags = flags;
  1096. buffer->clock = trace_clock_local;
  1097. buffer->reader_lock_key = key;
  1098. init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters);
  1099. init_waitqueue_head(&buffer->irq_work.waiters);
  1100. /* need at least two pages */
  1101. if (nr_pages < 2)
  1102. nr_pages = 2;
  1103. /*
  1104. * In case of non-hotplug cpu, if the ring-buffer is allocated
  1105. * in early initcall, it will not be notified of secondary cpus.
  1106. * In that off case, we need to allocate for all possible cpus.
  1107. */
  1108. #ifdef CONFIG_HOTPLUG_CPU
  1109. get_online_cpus();
  1110. cpumask_copy(buffer->cpumask, cpu_online_mask);
  1111. #else
  1112. cpumask_copy(buffer->cpumask, cpu_possible_mask);
  1113. #endif
  1114. buffer->cpus = nr_cpu_ids;
  1115. bsize = sizeof(void *) * nr_cpu_ids;
  1116. buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),
  1117. GFP_KERNEL);
  1118. if (!buffer->buffers)
  1119. goto fail_free_cpumask;
  1120. for_each_buffer_cpu(buffer, cpu) {
  1121. buffer->buffers[cpu] =
  1122. rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
  1123. if (!buffer->buffers[cpu])
  1124. goto fail_free_buffers;
  1125. }
  1126. #ifdef CONFIG_HOTPLUG_CPU
  1127. buffer->cpu_notify.notifier_call = rb_cpu_notify;
  1128. buffer->cpu_notify.priority = 0;
  1129. register_cpu_notifier(&buffer->cpu_notify);
  1130. #endif
  1131. put_online_cpus();
  1132. mutex_init(&buffer->mutex);
  1133. return buffer;
  1134. fail_free_buffers:
  1135. for_each_buffer_cpu(buffer, cpu) {
  1136. if (buffer->buffers[cpu])
  1137. rb_free_cpu_buffer(buffer->buffers[cpu]);
  1138. }
  1139. kfree(buffer->buffers);
  1140. fail_free_cpumask:
  1141. free_cpumask_var(buffer->cpumask);
  1142. put_online_cpus();
  1143. fail_free_buffer:
  1144. kfree(buffer);
  1145. return NULL;
  1146. }
  1147. EXPORT_SYMBOL_GPL(__ring_buffer_alloc);
  1148. /**
  1149. * ring_buffer_free - free a ring buffer.
  1150. * @buffer: the buffer to free.
  1151. */
  1152. void
  1153. ring_buffer_free(struct ring_buffer *buffer)
  1154. {
  1155. int cpu;
  1156. get_online_cpus();
  1157. #ifdef CONFIG_HOTPLUG_CPU
  1158. unregister_cpu_notifier(&buffer->cpu_notify);
  1159. #endif
  1160. for_each_buffer_cpu(buffer, cpu)
  1161. rb_free_cpu_buffer(buffer->buffers[cpu]);
  1162. put_online_cpus();
  1163. kfree(buffer->buffers);
  1164. free_cpumask_var(buffer->cpumask);
  1165. kfree(buffer);
  1166. }
  1167. EXPORT_SYMBOL_GPL(ring_buffer_free);
  1168. void ring_buffer_set_clock(struct ring_buffer *buffer,
  1169. u64 (*clock)(void))
  1170. {
  1171. buffer->clock = clock;
  1172. }
  1173. static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
  1174. static inline unsigned long rb_page_entries(struct buffer_page *bpage)
  1175. {
  1176. return local_read(&bpage->entries) & RB_WRITE_MASK;
  1177. }
  1178. static inline unsigned long rb_page_write(struct buffer_page *bpage)
  1179. {
  1180. return local_read(&bpage->write) & RB_WRITE_MASK;
  1181. }
  1182. static int
  1183. rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned int nr_pages)
  1184. {
  1185. struct list_head *tail_page, *to_remove, *next_page;
  1186. struct buffer_page *to_remove_page, *tmp_iter_page;
  1187. struct buffer_page *last_page, *first_page;
  1188. unsigned int nr_removed;
  1189. unsigned long head_bit;
  1190. int page_entries;
  1191. head_bit = 0;
  1192. raw_spin_lock_irq(&cpu_buffer->reader_lock);
  1193. atomic_inc(&cpu_buffer->record_disabled);
  1194. /*
  1195. * We don't race with the readers since we have acquired the reader
  1196. * lock. We also don't race with writers after disabling recording.
  1197. * This makes it easy to figure out the first and the last page to be
  1198. * removed from the list. We unlink all the pages in between including
  1199. * the first and last pages. This is done in a busy loop so that we
  1200. * lose the least number of traces.
  1201. * The pages are freed after we restart recording and unlock readers.
  1202. */
  1203. tail_page = &cpu_buffer->tail_page->list;
  1204. /*
  1205. * tail page might be on reader page, we remove the next page
  1206. * from the ring buffer
  1207. */
  1208. if (cpu_buffer->tail_page == cpu_buffer->reader_page)
  1209. tail_page = rb_list_head(tail_page->next);
  1210. to_remove = tail_page;
  1211. /* start of pages to remove */
  1212. first_page = list_entry(rb_list_head(to_remove->next),
  1213. struct buffer_page, list);
  1214. for (nr_removed = 0; nr_removed < nr_pages; nr_removed++) {
  1215. to_remove = rb_list_head(to_remove)->next;
  1216. head_bit |= (unsigned long)to_remove & RB_PAGE_HEAD;
  1217. }
  1218. next_page = rb_list_head(to_remove)->next;
  1219. /*
  1220. * Now we remove all pages between tail_page and next_page.
  1221. * Make sure that we have head_bit value preserved for the
  1222. * next page
  1223. */
  1224. tail_page->next = (struct list_head *)((unsigned long)next_page |
  1225. head_bit);
  1226. next_page = rb_list_head(next_page);
  1227. next_page->prev = tail_page;
  1228. /* make sure pages points to a valid page in the ring buffer */
  1229. cpu_buffer->pages = next_page;
  1230. /* update head page */
  1231. if (head_bit)
  1232. cpu_buffer->head_page = list_entry(next_page,
  1233. struct buffer_page, list);
  1234. /*
  1235. * change read pointer to make sure any read iterators reset
  1236. * themselves
  1237. */
  1238. cpu_buffer->read = 0;
  1239. /* pages are removed, resume tracing and then free the pages */
  1240. atomic_dec(&cpu_buffer->record_disabled);
  1241. raw_spin_unlock_irq(&cpu_buffer->reader_lock);
  1242. RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages));
  1243. /* last buffer page to remove */
  1244. last_page = list_entry(rb_list_head(to_remove), struct buffer_page,
  1245. list);
  1246. tmp_iter_page = first_page;
  1247. do {
  1248. to_remove_page = tmp_iter_page;
  1249. rb_inc_page(cpu_buffer, &tmp_iter_page);
  1250. /* update the counters */
  1251. page_entries = rb_page_entries(to_remove_page);
  1252. if (page_entries) {
  1253. /*
  1254. * If something was added to this page, it was full
  1255. * since it is not the tail page. So we deduct the
  1256. * bytes consumed in ring buffer from here.
  1257. * Increment overrun to account for the lost events.
  1258. */
  1259. local_add(page_entries, &cpu_buffer->overrun);
  1260. local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
  1261. }
  1262. /*
  1263. * We have already removed references to this list item, just
  1264. * free up the buffer_page and its page
  1265. */
  1266. free_buffer_page(to_remove_page);
  1267. nr_removed--;
  1268. } while (to_remove_page != last_page);
  1269. RB_WARN_ON(cpu_buffer, nr_removed);
  1270. return nr_removed == 0;
  1271. }
  1272. static int
  1273. rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer)
  1274. {
  1275. struct list_head *pages = &cpu_buffer->new_pages;
  1276. int retries, success;
  1277. raw_spin_lock_irq(&cpu_buffer->reader_lock);
  1278. /*
  1279. * We are holding the reader lock, so the reader page won't be swapped
  1280. * in the ring buffer. Now we are racing with the writer trying to
  1281. * move head page and the tail page.
  1282. * We are going to adapt the reader page update process where:
  1283. * 1. We first splice the start and end of list of new pages between
  1284. * the head page and its previous page.
  1285. * 2. We cmpxchg the prev_page->next to point from head page to the
  1286. * start of new pages list.
  1287. * 3. Finally, we update the head->prev to the end of new list.
  1288. *
  1289. * We will try this process 10 times, to make sure that we don't keep
  1290. * spinning.
  1291. */
  1292. retries = 10;
  1293. success = 0;
  1294. while (retries--) {
  1295. struct list_head *head_page, *prev_page, *r;
  1296. struct list_head *last_page, *first_page;
  1297. struct list_head *head_page_with_bit;
  1298. head_page = &rb_set_head_page(cpu_buffer)->list;
  1299. if (!head_page)
  1300. break;
  1301. prev_page = head_page->prev;
  1302. first_page = pages->next;
  1303. last_page = pages->prev;
  1304. head_page_with_bit = (struct list_head *)
  1305. ((unsigned long)head_page | RB_PAGE_HEAD);
  1306. last_page->next = head_page_with_bit;
  1307. first_page->prev = prev_page;
  1308. r = cmpxchg(&prev_page->next, head_page_with_bit, first_page);
  1309. if (r == head_page_with_bit) {
  1310. /*
  1311. * yay, we replaced the page pointer to our new list,
  1312. * now, we just have to update to head page's prev
  1313. * pointer to point to end of list
  1314. */
  1315. head_page->prev = last_page;
  1316. success = 1;
  1317. break;
  1318. }
  1319. }
  1320. if (success)
  1321. INIT_LIST_HEAD(pages);
  1322. /*
  1323. * If we weren't successful in adding in new pages, warn and stop
  1324. * tracing
  1325. */
  1326. RB_WARN_ON(cpu_buffer, !success);
  1327. raw_spin_unlock_irq(&cpu_buffer->reader_lock);
  1328. /* free pages if they weren't inserted */
  1329. if (!success) {
  1330. struct buffer_page *bpage, *tmp;
  1331. list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
  1332. list) {
  1333. list_del_init(&bpage->list);
  1334. free_buffer_page(bpage);
  1335. }
  1336. }
  1337. return success;
  1338. }
  1339. static void rb_update_pages(struct ring_buffer_per_cpu *cpu_buffer)
  1340. {
  1341. int success;
  1342. if (cpu_buffer->nr_pages_to_update > 0)
  1343. success = rb_insert_pages(cpu_buffer);
  1344. else
  1345. success = rb_remove_pages(cpu_buffer,
  1346. -cpu_buffer->nr_pages_to_update);
  1347. if (success)
  1348. cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update;
  1349. }
  1350. static void update_pages_handler(struct work_struct *work)
  1351. {
  1352. struct ring_buffer_per_cpu *cpu_buffer = container_of(work,
  1353. struct ring_buffer_per_cpu, update_pages_work);
  1354. rb_update_pages(cpu_buffer);
  1355. complete(&cpu_buffer->update_done);
  1356. }
  1357. /**
  1358. * ring_buffer_resize - resize the ring buffer
  1359. * @buffer: the buffer to resize.
  1360. * @size: the new size.
  1361. *
  1362. * Minimum size is 2 * BUF_PAGE_SIZE.
  1363. *
  1364. * Returns 0 on success and < 0 on failure.
  1365. */
  1366. int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size,
  1367. int cpu_id)
  1368. {
  1369. struct ring_buffer_per_cpu *cpu_buffer;
  1370. unsigned nr_pages;
  1371. int cpu, err = 0;
  1372. /*
  1373. * Always succeed at resizing a non-existent buffer:
  1374. */
  1375. if (!buffer)
  1376. return size;
  1377. /* Make sure the requested buffer exists */
  1378. if (cpu_id != RING_BUFFER_ALL_CPUS &&
  1379. !cpumask_test_cpu(cpu_id, buffer->cpumask))
  1380. return size;
  1381. size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
  1382. size *= BUF_PAGE_SIZE;
  1383. /* we need a minimum of two pages */
  1384. if (size < BUF_PAGE_SIZE * 2)
  1385. size = BUF_PAGE_SIZE * 2;
  1386. nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
  1387. /*
  1388. * Don't succeed if resizing is disabled, as a reader might be
  1389. * manipulating the ring buffer and is expecting a sane state while
  1390. * this is true.
  1391. */
  1392. if (atomic_read(&buffer->resize_disabled))
  1393. return -EBUSY;
  1394. /* prevent another thread from changing buffer sizes */
  1395. mutex_lock(&buffer->mutex);
  1396. if (cpu_id == RING_BUFFER_ALL_CPUS) {
  1397. /* calculate the pages to update */
  1398. for_each_buffer_cpu(buffer, cpu) {
  1399. cpu_buffer = buffer->buffers[cpu];
  1400. cpu_buffer->nr_pages_to_update = nr_pages -
  1401. cpu_buffer->nr_pages;
  1402. /*
  1403. * nothing more to do for removing pages or no update
  1404. */
  1405. if (cpu_buffer->nr_pages_to_update <= 0)
  1406. continue;
  1407. /*
  1408. * to add pages, make sure all new pages can be
  1409. * allocated without receiving ENOMEM
  1410. */
  1411. INIT_LIST_HEAD(&cpu_buffer->new_pages);
  1412. if (__rb_allocate_pages(cpu_buffer->nr_pages_to_update,
  1413. &cpu_buffer->new_pages, cpu)) {
  1414. /* not enough memory for new pages */
  1415. err = -ENOMEM;
  1416. goto out_err;
  1417. }
  1418. }
  1419. get_online_cpus();
  1420. /*
  1421. * Fire off all the required work handlers
  1422. * We can't schedule on offline CPUs, but it's not necessary
  1423. * since we can change their buffer sizes without any race.
  1424. */
  1425. for_each_buffer_cpu(buffer, cpu) {
  1426. cpu_buffer = buffer->buffers[cpu];
  1427. if (!cpu_buffer->nr_pages_to_update)
  1428. continue;
  1429. if (cpu_online(cpu))
  1430. schedule_work_on(cpu,
  1431. &cpu_buffer->update_pages_work);
  1432. else
  1433. rb_update_pages(cpu_buffer);
  1434. }
  1435. /* wait for all the updates to complete */
  1436. for_each_buffer_cpu(buffer, cpu) {
  1437. cpu_buffer = buffer->buffers[cpu];
  1438. if (!cpu_buffer->nr_pages_to_update)
  1439. continue;
  1440. if (cpu_online(cpu))
  1441. wait_for_completion(&cpu_buffer->update_done);
  1442. cpu_buffer->nr_pages_to_update = 0;
  1443. }
  1444. put_online_cpus();
  1445. } else {
  1446. /* Make sure this CPU has been intitialized */
  1447. if (!cpumask_test_cpu(cpu_id, buffer->cpumask))
  1448. goto out;
  1449. cpu_buffer = buffer->buffers[cpu_id];
  1450. if (nr_pages == cpu_buffer->nr_pages)
  1451. goto out;
  1452. cpu_buffer->nr_pages_to_update = nr_pages -
  1453. cpu_buffer->nr_pages;
  1454. INIT_LIST_HEAD(&cpu_buffer->new_pages);
  1455. if (cpu_buffer->nr_pages_to_update > 0 &&
  1456. __rb_allocate_pages(cpu_buffer->nr_pages_to_update,
  1457. &cpu_buffer->new_pages, cpu_id)) {
  1458. err = -ENOMEM;
  1459. goto out_err;
  1460. }
  1461. get_online_cpus();
  1462. if (cpu_online(cpu_id)) {
  1463. schedule_work_on(cpu_id,
  1464. &cpu_buffer->update_pages_work);
  1465. wait_for_completion(&cpu_buffer->update_done);
  1466. } else
  1467. rb_update_pages(cpu_buffer);
  1468. cpu_buffer->nr_pages_to_update = 0;
  1469. put_online_cpus();
  1470. }
  1471. out:
  1472. /*
  1473. * The ring buffer resize can happen with the ring buffer
  1474. * enabled, so that the update disturbs the tracing as little
  1475. * as possible. But if the buffer is disabled, we do not need
  1476. * to worry about that, and we can take the time to verify
  1477. * that the buffer is not corrupt.
  1478. */
  1479. if (atomic_read(&buffer->record_disabled)) {
  1480. atomic_inc(&buffer->record_disabled);
  1481. /*
  1482. * Even though the buffer was disabled, we must make sure
  1483. * that it is truly disabled before calling rb_check_pages.
  1484. * There could have been a race between checking
  1485. * record_disable and incrementing it.
  1486. */
  1487. synchronize_sched();
  1488. for_each_buffer_cpu(buffer, cpu) {
  1489. cpu_buffer = buffer->buffers[cpu];
  1490. rb_check_pages(cpu_buffer);
  1491. }
  1492. atomic_dec(&buffer->record_disabled);
  1493. }
  1494. mutex_unlock(&buffer->mutex);
  1495. return size;
  1496. out_err:
  1497. for_each_buffer_cpu(buffer, cpu) {
  1498. struct buffer_page *bpage, *tmp;
  1499. cpu_buffer = buffer->buffers[cpu];
  1500. cpu_buffer->nr_pages_to_update = 0;
  1501. if (list_empty(&cpu_buffer->new_pages))
  1502. continue;
  1503. list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
  1504. list) {
  1505. list_del_init(&bpage->list);
  1506. free_buffer_page(bpage);
  1507. }
  1508. }
  1509. mutex_unlock(&buffer->mutex);
  1510. return err;
  1511. }
  1512. EXPORT_SYMBOL_GPL(ring_buffer_resize);
  1513. void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val)
  1514. {
  1515. mutex_lock(&buffer->mutex);
  1516. if (val)
  1517. buffer->flags |= RB_FL_OVERWRITE;
  1518. else
  1519. buffer->flags &= ~RB_FL_OVERWRITE;
  1520. mutex_unlock(&buffer->mutex);
  1521. }
  1522. EXPORT_SYMBOL_GPL(ring_buffer_change_overwrite);
  1523. static inline void *
  1524. __rb_data_page_index(struct buffer_data_page *bpage, unsigned index)
  1525. {
  1526. return bpage->data + index;
  1527. }
  1528. static inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)
  1529. {
  1530. return bpage->page->data + index;
  1531. }
  1532. static inline struct ring_buffer_event *
  1533. rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer)
  1534. {
  1535. return __rb_page_index(cpu_buffer->reader_page,
  1536. cpu_buffer->reader_page->read);
  1537. }
  1538. static inline struct ring_buffer_event *
  1539. rb_iter_head_event(struct ring_buffer_iter *iter)
  1540. {
  1541. return __rb_page_index(iter->head_page, iter->head);
  1542. }
  1543. static inline unsigned rb_page_commit(struct buffer_page *bpage)
  1544. {
  1545. return local_read(&bpage->page->commit);
  1546. }
  1547. /* Size is determined by what has been committed */
  1548. static inline unsigned rb_page_size(struct buffer_page *bpage)
  1549. {
  1550. return rb_page_commit(bpage);
  1551. }
  1552. static inline unsigned
  1553. rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer)
  1554. {
  1555. return rb_page_commit(cpu_buffer->commit_page);
  1556. }
  1557. static inline unsigned
  1558. rb_event_index(struct ring_buffer_event *event)
  1559. {
  1560. unsigned long addr = (unsigned long)event;
  1561. return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE;
  1562. }
  1563. static inline int
  1564. rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
  1565. struct ring_buffer_event *event)
  1566. {
  1567. unsigned long addr = (unsigned long)event;
  1568. unsigned long index;
  1569. index = rb_event_index(event);
  1570. addr &= PAGE_MASK;
  1571. return cpu_buffer->commit_page->page == (void *)addr &&
  1572. rb_commit_index(cpu_buffer) == index;
  1573. }
  1574. static void
  1575. rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
  1576. {
  1577. unsigned long max_count;
  1578. /*
  1579. * We only race with interrupts and NMIs on this CPU.
  1580. * If we own the commit event, then we can commit
  1581. * all others that interrupted us, since the interruptions
  1582. * are in stack format (they finish before they come
  1583. * back to us). This allows us to do a simple loop to
  1584. * assign the commit to the tail.
  1585. */
  1586. again:
  1587. max_count = cpu_buffer->nr_pages * 100;
  1588. while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
  1589. if (RB_WARN_ON(cpu_buffer, !(--max_count)))
  1590. return;
  1591. if (RB_WARN_ON(cpu_buffer,
  1592. rb_is_reader_page(cpu_buffer->tail_page)))
  1593. return;
  1594. local_set(&cpu_buffer->commit_page->page->commit,
  1595. rb_page_write(cpu_buffer->commit_page));
  1596. rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
  1597. cpu_buffer->write_stamp =
  1598. cpu_buffer->commit_page->page->time_stamp;
  1599. /* add barrier to keep gcc from optimizing too much */
  1600. barrier();
  1601. }
  1602. while (rb_commit_index(cpu_buffer) !=
  1603. rb_page_write(cpu_buffer->commit_page)) {
  1604. local_set(&cpu_buffer->commit_page->page->commit,
  1605. rb_page_write(cpu_buffer->commit_page));
  1606. RB_WARN_ON(cpu_buffer,
  1607. local_read(&cpu_buffer->commit_page->page->commit) &
  1608. ~RB_WRITE_MASK);
  1609. barrier();
  1610. }
  1611. /* again, keep gcc from optimizing */
  1612. barrier();
  1613. /*
  1614. * If an interrupt came in just after the first while loop
  1615. * and pushed the tail page forward, we will be left with
  1616. * a dangling commit that will never go forward.
  1617. */
  1618. if (unlikely(cpu_buffer->commit_page != cpu_buffer->tail_page))
  1619. goto again;
  1620. }
  1621. static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
  1622. {
  1623. cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;
  1624. cpu_buffer->reader_page->read = 0;
  1625. }
  1626. static void rb_inc_iter(struct ring_buffer_iter *iter)
  1627. {
  1628. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  1629. /*
  1630. * The iterator could be on the reader page (it starts there).
  1631. * But the head could have moved, since the reader was
  1632. * found. Check for this case and assign the iterator
  1633. * to the head page instead of next.
  1634. */
  1635. if (iter->head_page == cpu_buffer->reader_page)
  1636. iter->head_page = rb_set_head_page(cpu_buffer);
  1637. else
  1638. rb_inc_page(cpu_buffer, &iter->head_page);
  1639. iter->read_stamp = iter->head_page->page->time_stamp;
  1640. iter->head = 0;
  1641. }
  1642. /* Slow path, do not inline */
  1643. static noinline struct ring_buffer_event *
  1644. rb_add_time_stamp(struct ring_buffer_event *event, u64 delta)
  1645. {
  1646. event->type_len = RINGBUF_TYPE_TIME_EXTEND;
  1647. /* Not the first event on the page? */
  1648. if (rb_event_index(event)) {
  1649. event->time_delta = delta & TS_MASK;
  1650. event->array[0] = delta >> TS_SHIFT;
  1651. } else {
  1652. /* nope, just zero it */
  1653. event->time_delta = 0;
  1654. event->array[0] = 0;
  1655. }
  1656. return skip_time_extend(event);
  1657. }
  1658. /**
  1659. * rb_update_event - update event type and data
  1660. * @event: the even to update
  1661. * @type: the type of event
  1662. * @length: the size of the event field in the ring buffer
  1663. *
  1664. * Update the type and data fields of the event. The length
  1665. * is the actual size that is written to the ring buffer,
  1666. * and with this, we can determine what to place into the
  1667. * data field.
  1668. */
  1669. static void
  1670. rb_update_event(struct ring_buffer_per_cpu *cpu_buffer,
  1671. struct ring_buffer_event *event, unsigned length,
  1672. int add_timestamp, u64 delta)
  1673. {
  1674. /* Only a commit updates the timestamp */
  1675. if (unlikely(!rb_event_is_commit(cpu_buffer, event)))
  1676. delta = 0;
  1677. /*
  1678. * If we need to add a timestamp, then we
  1679. * add it to the start of the resevered space.
  1680. */
  1681. if (unlikely(add_timestamp)) {
  1682. event = rb_add_time_stamp(event, delta);
  1683. length -= RB_LEN_TIME_EXTEND;
  1684. delta = 0;
  1685. }
  1686. event->time_delta = delta;
  1687. length -= RB_EVNT_HDR_SIZE;
  1688. if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) {
  1689. event->type_len = 0;
  1690. event->array[0] = length;
  1691. } else
  1692. event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT);
  1693. }
  1694. /*
  1695. * rb_handle_head_page - writer hit the head page
  1696. *
  1697. * Returns: +1 to retry page
  1698. * 0 to continue
  1699. * -1 on error
  1700. */
  1701. static int
  1702. rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
  1703. struct buffer_page *tail_page,
  1704. struct buffer_page *next_page)
  1705. {
  1706. struct buffer_page *new_head;
  1707. int entries;
  1708. int type;
  1709. int ret;
  1710. entries = rb_page_entries(next_page);
  1711. /*
  1712. * The hard part is here. We need to move the head
  1713. * forward, and protect against both readers on
  1714. * other CPUs and writers coming in via interrupts.
  1715. */
  1716. type = rb_head_page_set_update(cpu_buffer, next_page, tail_page,
  1717. RB_PAGE_HEAD);
  1718. /*
  1719. * type can be one of four:
  1720. * NORMAL - an interrupt already moved it for us
  1721. * HEAD - we are the first to get here.
  1722. * UPDATE - we are the interrupt interrupting
  1723. * a current move.
  1724. * MOVED - a reader on another CPU moved the next
  1725. * pointer to its reader page. Give up
  1726. * and try again.
  1727. */
  1728. switch (type) {
  1729. case RB_PAGE_HEAD:
  1730. /*
  1731. * We changed the head to UPDATE, thus
  1732. * it is our responsibility to update
  1733. * the counters.
  1734. */
  1735. local_add(entries, &cpu_buffer->overrun);
  1736. local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
  1737. /*
  1738. * The entries will be zeroed out when we move the
  1739. * tail page.
  1740. */
  1741. /* still more to do */
  1742. break;
  1743. case RB_PAGE_UPDATE:
  1744. /*
  1745. * This is an interrupt that interrupt the
  1746. * previous update. Still more to do.
  1747. */
  1748. break;
  1749. case RB_PAGE_NORMAL:
  1750. /*
  1751. * An interrupt came in before the update
  1752. * and processed this for us.
  1753. * Nothing left to do.
  1754. */
  1755. return 1;
  1756. case RB_PAGE_MOVED:
  1757. /*
  1758. * The reader is on another CPU and just did
  1759. * a swap with our next_page.
  1760. * Try again.
  1761. */
  1762. return 1;
  1763. default:
  1764. RB_WARN_ON(cpu_buffer, 1); /* WTF??? */
  1765. return -1;
  1766. }
  1767. /*
  1768. * Now that we are here, the old head pointer is
  1769. * set to UPDATE. This will keep the reader from
  1770. * swapping the head page with the reader page.
  1771. * The reader (on another CPU) will spin till
  1772. * we are finished.
  1773. *
  1774. * We just need to protect against interrupts
  1775. * doing the job. We will set the next pointer
  1776. * to HEAD. After that, we set the old pointer
  1777. * to NORMAL, but only if it was HEAD before.
  1778. * otherwise we are an interrupt, and only
  1779. * want the outer most commit to reset it.
  1780. */
  1781. new_head = next_page;
  1782. rb_inc_page(cpu_buffer, &new_head);
  1783. ret = rb_head_page_set_head(cpu_buffer, new_head, next_page,
  1784. RB_PAGE_NORMAL);
  1785. /*
  1786. * Valid returns are:
  1787. * HEAD - an interrupt came in and already set it.
  1788. * NORMAL - One of two things:
  1789. * 1) We really set it.
  1790. * 2) A bunch of interrupts came in and moved
  1791. * the page forward again.
  1792. */
  1793. switch (ret) {
  1794. case RB_PAGE_HEAD:
  1795. case RB_PAGE_NORMAL:
  1796. /* OK */
  1797. break;
  1798. default:
  1799. RB_WARN_ON(cpu_buffer, 1);
  1800. return -1;
  1801. }
  1802. /*
  1803. * It is possible that an interrupt came in,
  1804. * set the head up, then more interrupts came in
  1805. * and moved it again. When we get back here,
  1806. * the page would have been set to NORMAL but we
  1807. * just set it back to HEAD.
  1808. *
  1809. * How do you detect this? Well, if that happened
  1810. * the tail page would have moved.
  1811. */
  1812. if (ret == RB_PAGE_NORMAL) {
  1813. /*
  1814. * If the tail had moved passed next, then we need
  1815. * to reset the pointer.
  1816. */
  1817. if (cpu_buffer->tail_page != tail_page &&
  1818. cpu_buffer->tail_page != next_page)
  1819. rb_head_page_set_normal(cpu_buffer, new_head,
  1820. next_page,
  1821. RB_PAGE_HEAD);
  1822. }
  1823. /*
  1824. * If this was the outer most commit (the one that
  1825. * changed the original pointer from HEAD to UPDATE),
  1826. * then it is up to us to reset it to NORMAL.
  1827. */
  1828. if (type == RB_PAGE_HEAD) {
  1829. ret = rb_head_page_set_normal(cpu_buffer, next_page,
  1830. tail_page,
  1831. RB_PAGE_UPDATE);
  1832. if (RB_WARN_ON(cpu_buffer,
  1833. ret != RB_PAGE_UPDATE))
  1834. return -1;
  1835. }
  1836. return 0;
  1837. }
  1838. static unsigned rb_calculate_event_length(unsigned length)
  1839. {
  1840. struct ring_buffer_event event; /* Used only for sizeof array */
  1841. /* zero length can cause confusions */
  1842. if (!length)
  1843. length = 1;
  1844. if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT)
  1845. length += sizeof(event.array[0]);
  1846. length += RB_EVNT_HDR_SIZE;
  1847. length = ALIGN(length, RB_ARCH_ALIGNMENT);
  1848. return length;
  1849. }
  1850. static inline void
  1851. rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
  1852. struct buffer_page *tail_page,
  1853. unsigned long tail, unsigned long length)
  1854. {
  1855. struct ring_buffer_event *event;
  1856. /*
  1857. * Only the event that crossed the page boundary
  1858. * must fill the old tail_page with padding.
  1859. */
  1860. if (tail >= BUF_PAGE_SIZE) {
  1861. /*
  1862. * If the page was filled, then we still need
  1863. * to update the real_end. Reset it to zero
  1864. * and the reader will ignore it.
  1865. */
  1866. if (tail == BUF_PAGE_SIZE)
  1867. tail_page->real_end = 0;
  1868. local_sub(length, &tail_page->write);
  1869. return;
  1870. }
  1871. event = __rb_page_index(tail_page, tail);
  1872. kmemcheck_annotate_bitfield(event, bitfield);
  1873. /* account for padding bytes */
  1874. local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes);
  1875. /*
  1876. * Save the original length to the meta data.
  1877. * This will be used by the reader to add lost event
  1878. * counter.
  1879. */
  1880. tail_page->real_end = tail;
  1881. /*
  1882. * If this event is bigger than the minimum size, then
  1883. * we need to be careful that we don't subtract the
  1884. * write counter enough to allow another writer to slip
  1885. * in on this page.
  1886. * We put in a discarded commit instead, to make sure
  1887. * that this space is not used again.
  1888. *
  1889. * If we are less than the minimum size, we don't need to
  1890. * worry about it.
  1891. */
  1892. if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) {
  1893. /* No room for any events */
  1894. /* Mark the rest of the page with padding */
  1895. rb_event_set_padding(event);
  1896. /* Set the write back to the previous setting */
  1897. local_sub(length, &tail_page->write);
  1898. return;
  1899. }
  1900. /* Put in a discarded event */
  1901. event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE;
  1902. event->type_len = RINGBUF_TYPE_PADDING;
  1903. /* time delta must be non zero */
  1904. event->time_delta = 1;
  1905. /* Set write to end of buffer */
  1906. length = (tail + length) - BUF_PAGE_SIZE;
  1907. local_sub(length, &tail_page->write);
  1908. }
  1909. /*
  1910. * This is the slow path, force gcc not to inline it.
  1911. */
  1912. static noinline struct ring_buffer_event *
  1913. rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
  1914. unsigned long length, unsigned long tail,
  1915. struct buffer_page *tail_page, u64 ts)
  1916. {
  1917. struct buffer_page *commit_page = cpu_buffer->commit_page;
  1918. struct ring_buffer *buffer = cpu_buffer->buffer;
  1919. struct buffer_page *next_page;
  1920. int ret;
  1921. next_page = tail_page;
  1922. rb_inc_page(cpu_buffer, &next_page);
  1923. /*
  1924. * If for some reason, we had an interrupt storm that made
  1925. * it all the way around the buffer, bail, and warn
  1926. * about it.
  1927. */
  1928. if (unlikely(next_page == commit_page)) {
  1929. local_inc(&cpu_buffer->commit_overrun);
  1930. goto out_reset;
  1931. }
  1932. /*
  1933. * This is where the fun begins!
  1934. *
  1935. * We are fighting against races between a reader that
  1936. * could be on another CPU trying to swap its reader
  1937. * page with the buffer head.
  1938. *
  1939. * We are also fighting against interrupts coming in and
  1940. * moving the head or tail on us as well.
  1941. *
  1942. * If the next page is the head page then we have filled
  1943. * the buffer, unless the commit page is still on the
  1944. * reader page.
  1945. */
  1946. if (rb_is_head_page(cpu_buffer, next_page, &tail_page->list)) {
  1947. /*
  1948. * If the commit is not on the reader page, then
  1949. * move the header page.
  1950. */
  1951. if (!rb_is_reader_page(cpu_buffer->commit_page)) {
  1952. /*
  1953. * If we are not in overwrite mode,
  1954. * this is easy, just stop here.
  1955. */
  1956. if (!(buffer->flags & RB_FL_OVERWRITE)) {
  1957. local_inc(&cpu_buffer->dropped_events);
  1958. goto out_reset;
  1959. }
  1960. ret = rb_handle_head_page(cpu_buffer,
  1961. tail_page,
  1962. next_page);
  1963. if (ret < 0)
  1964. goto out_reset;
  1965. if (ret)
  1966. goto out_again;
  1967. } else {
  1968. /*
  1969. * We need to be careful here too. The
  1970. * commit page could still be on the reader
  1971. * page. We could have a small buffer, and
  1972. * have filled up the buffer with events
  1973. * from interrupts and such, and wrapped.
  1974. *
  1975. * Note, if the tail page is also the on the
  1976. * reader_page, we let it move out.
  1977. */
  1978. if (unlikely((cpu_buffer->commit_page !=
  1979. cpu_buffer->tail_page) &&
  1980. (cpu_buffer->commit_page ==
  1981. cpu_buffer->reader_page))) {
  1982. local_inc(&cpu_buffer->commit_overrun);
  1983. goto out_reset;
  1984. }
  1985. }
  1986. }
  1987. ret = rb_tail_page_update(cpu_buffer, tail_page, next_page);
  1988. if (ret) {
  1989. /*
  1990. * Nested commits always have zero deltas, so
  1991. * just reread the time stamp
  1992. */
  1993. ts = rb_time_stamp(buffer);
  1994. next_page->page->time_stamp = ts;
  1995. }
  1996. out_again:
  1997. rb_reset_tail(cpu_buffer, tail_page, tail, length);
  1998. /* fail and let the caller try again */
  1999. return ERR_PTR(-EAGAIN);
  2000. out_reset:
  2001. /* reset write */
  2002. rb_reset_tail(cpu_buffer, tail_page, tail, length);
  2003. return NULL;
  2004. }
  2005. static struct ring_buffer_event *
  2006. __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
  2007. unsigned long length, u64 ts,
  2008. u64 delta, int add_timestamp)
  2009. {
  2010. struct buffer_page *tail_page;
  2011. struct ring_buffer_event *event;
  2012. unsigned long tail, write;
  2013. /*
  2014. * If the time delta since the last event is too big to
  2015. * hold in the time field of the event, then we append a
  2016. * TIME EXTEND event ahead of the data event.
  2017. */
  2018. if (unlikely(add_timestamp))
  2019. length += RB_LEN_TIME_EXTEND;
  2020. tail_page = cpu_buffer->tail_page;
  2021. write = local_add_return(length, &tail_page->write);
  2022. /* set write to only the index of the write */
  2023. write &= RB_WRITE_MASK;
  2024. tail = write - length;
  2025. /* See if we shot pass the end of this buffer page */
  2026. if (unlikely(write > BUF_PAGE_SIZE))
  2027. return rb_move_tail(cpu_buffer, length, tail,
  2028. tail_page, ts);
  2029. /* We reserved something on the buffer */
  2030. event = __rb_page_index(tail_page, tail);
  2031. kmemcheck_annotate_bitfield(event, bitfield);
  2032. rb_update_event(cpu_buffer, event, length, add_timestamp, delta);
  2033. local_inc(&tail_page->entries);
  2034. /*
  2035. * If this is the first commit on the page, then update
  2036. * its timestamp.
  2037. */
  2038. if (!tail)
  2039. tail_page->page->time_stamp = ts;
  2040. /* account for these added bytes */
  2041. local_add(length, &cpu_buffer->entries_bytes);
  2042. return event;
  2043. }
  2044. static inline int
  2045. rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
  2046. struct ring_buffer_event *event)
  2047. {
  2048. unsigned long new_index, old_index;
  2049. struct buffer_page *bpage;
  2050. unsigned long index;
  2051. unsigned long addr;
  2052. new_index = rb_event_index(event);
  2053. old_index = new_index + rb_event_ts_length(event);
  2054. addr = (unsigned long)event;
  2055. addr &= PAGE_MASK;
  2056. bpage = cpu_buffer->tail_page;
  2057. if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
  2058. unsigned long write_mask =
  2059. local_read(&bpage->write) & ~RB_WRITE_MASK;
  2060. unsigned long event_length = rb_event_length(event);
  2061. /*
  2062. * This is on the tail page. It is possible that
  2063. * a write could come in and move the tail page
  2064. * and write to the next page. That is fine
  2065. * because we just shorten what is on this page.
  2066. */
  2067. old_index += write_mask;
  2068. new_index += write_mask;
  2069. index = local_cmpxchg(&bpage->write, old_index, new_index);
  2070. if (index == old_index) {
  2071. /* update counters */
  2072. local_sub(event_length, &cpu_buffer->entries_bytes);
  2073. return 1;
  2074. }
  2075. }
  2076. /* could not discard */
  2077. return 0;
  2078. }
  2079. static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer)
  2080. {
  2081. local_inc(&cpu_buffer->committing);
  2082. local_inc(&cpu_buffer->commits);
  2083. }
  2084. static inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
  2085. {
  2086. unsigned long commits;
  2087. if (RB_WARN_ON(cpu_buffer,
  2088. !local_read(&cpu_buffer->committing)))
  2089. return;
  2090. again:
  2091. commits = local_read(&cpu_buffer->commits);
  2092. /* synchronize with interrupts */
  2093. barrier();
  2094. if (local_read(&cpu_buffer->committing) == 1)
  2095. rb_set_commit_to_write(cpu_buffer);
  2096. local_dec(&cpu_buffer->committing);
  2097. /* synchronize with interrupts */
  2098. barrier();
  2099. /*
  2100. * Need to account for interrupts coming in between the
  2101. * updating of the commit page and the clearing of the
  2102. * committing counter.
  2103. */
  2104. if (unlikely(local_read(&cpu_buffer->commits) != commits) &&
  2105. !local_read(&cpu_buffer->committing)) {
  2106. local_inc(&cpu_buffer->committing);
  2107. goto again;
  2108. }
  2109. }
  2110. static struct ring_buffer_event *
  2111. rb_reserve_next_event(struct ring_buffer *buffer,
  2112. struct ring_buffer_per_cpu *cpu_buffer,
  2113. unsigned long length)
  2114. {
  2115. struct ring_buffer_event *event;
  2116. u64 ts, delta;
  2117. int nr_loops = 0;
  2118. int add_timestamp;
  2119. u64 diff;
  2120. rb_start_commit(cpu_buffer);
  2121. #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
  2122. /*
  2123. * Due to the ability to swap a cpu buffer from a buffer
  2124. * it is possible it was swapped before we committed.
  2125. * (committing stops a swap). We check for it here and
  2126. * if it happened, we have to fail the write.
  2127. */
  2128. barrier();
  2129. if (unlikely(ACCESS_ONCE(cpu_buffer->buffer) != buffer)) {
  2130. local_dec(&cpu_buffer->committing);
  2131. local_dec(&cpu_buffer->commits);
  2132. return NULL;
  2133. }
  2134. #endif
  2135. length = rb_calculate_event_length(length);
  2136. again:
  2137. add_timestamp = 0;
  2138. delta = 0;
  2139. /*
  2140. * We allow for interrupts to reenter here and do a trace.
  2141. * If one does, it will cause this original code to loop
  2142. * back here. Even with heavy interrupts happening, this
  2143. * should only happen a few times in a row. If this happens
  2144. * 1000 times in a row, there must be either an interrupt
  2145. * storm or we have something buggy.
  2146. * Bail!
  2147. */
  2148. if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
  2149. goto out_fail;
  2150. ts = rb_time_stamp(cpu_buffer->buffer);
  2151. diff = ts - cpu_buffer->write_stamp;
  2152. /* make sure this diff is calculated here */
  2153. barrier();
  2154. /* Did the write stamp get updated already? */
  2155. if (likely(ts >= cpu_buffer->write_stamp)) {
  2156. delta = diff;
  2157. if (unlikely(test_time_stamp(delta))) {
  2158. int local_clock_stable = 1;
  2159. #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
  2160. local_clock_stable = sched_clock_stable;
  2161. #endif
  2162. WARN_ONCE(delta > (1ULL << 59),
  2163. KERN_WARNING "Delta way too big! %llu ts=%llu write stamp = %llu\n%s",
  2164. (unsigned long long)delta,
  2165. (unsigned long long)ts,
  2166. (unsigned long long)cpu_buffer->write_stamp,
  2167. local_clock_stable ? "" :
  2168. "If you just came from a suspend/resume,\n"
  2169. "please switch to the trace global clock:\n"
  2170. " echo global > /sys/kernel/debug/tracing/trace_clock\n");
  2171. add_timestamp = 1;
  2172. }
  2173. }
  2174. event = __rb_reserve_next(cpu_buffer, length, ts,
  2175. delta, add_timestamp);
  2176. if (unlikely(PTR_ERR(event) == -EAGAIN))
  2177. goto again;
  2178. if (!event)
  2179. goto out_fail;
  2180. return event;
  2181. out_fail:
  2182. rb_end_commit(cpu_buffer);
  2183. return NULL;
  2184. }
  2185. #ifdef CONFIG_TRACING
  2186. /*
  2187. * The lock and unlock are done within a preempt disable section.
  2188. * The current_context per_cpu variable can only be modified
  2189. * by the current task between lock and unlock. But it can
  2190. * be modified more than once via an interrupt. To pass this
  2191. * information from the lock to the unlock without having to
  2192. * access the 'in_interrupt()' functions again (which do show
  2193. * a bit of overhead in something as critical as function tracing,
  2194. * we use a bitmask trick.
  2195. *
  2196. * bit 0 = NMI context
  2197. * bit 1 = IRQ context
  2198. * bit 2 = SoftIRQ context
  2199. * bit 3 = normal context.
  2200. *
  2201. * This works because this is the order of contexts that can
  2202. * preempt other contexts. A SoftIRQ never preempts an IRQ
  2203. * context.
  2204. *
  2205. * When the context is determined, the corresponding bit is
  2206. * checked and set (if it was set, then a recursion of that context
  2207. * happened).
  2208. *
  2209. * On unlock, we need to clear this bit. To do so, just subtract
  2210. * 1 from the current_context and AND it to itself.
  2211. *
  2212. * (binary)
  2213. * 101 - 1 = 100
  2214. * 101 & 100 = 100 (clearing bit zero)
  2215. *
  2216. * 1010 - 1 = 1001
  2217. * 1010 & 1001 = 1000 (clearing bit 1)
  2218. *
  2219. * The least significant bit can be cleared this way, and it
  2220. * just so happens that it is the same bit corresponding to
  2221. * the current context.
  2222. */
  2223. static DEFINE_PER_CPU(unsigned int, current_context);
  2224. static __always_inline int trace_recursive_lock(void)
  2225. {
  2226. unsigned int val = this_cpu_read(current_context);
  2227. int bit;
  2228. if (in_interrupt()) {
  2229. if (in_nmi())
  2230. bit = 0;
  2231. else if (in_irq())
  2232. bit = 1;
  2233. else
  2234. bit = 2;
  2235. } else
  2236. bit = 3;
  2237. if (unlikely(val & (1 << bit)))
  2238. return 1;
  2239. val |= (1 << bit);
  2240. this_cpu_write(current_context, val);
  2241. return 0;
  2242. }
  2243. static __always_inline void trace_recursive_unlock(void)
  2244. {
  2245. unsigned int val = this_cpu_read(current_context);
  2246. val--;
  2247. val &= this_cpu_read(current_context);
  2248. this_cpu_write(current_context, val);
  2249. }
  2250. #else
  2251. #define trace_recursive_lock() (0)
  2252. #define trace_recursive_unlock() do { } while (0)
  2253. #endif
  2254. /**
  2255. * ring_buffer_lock_reserve - reserve a part of the buffer
  2256. * @buffer: the ring buffer to reserve from
  2257. * @length: the length of the data to reserve (excluding event header)
  2258. *
  2259. * Returns a reseverd event on the ring buffer to copy directly to.
  2260. * The user of this interface will need to get the body to write into
  2261. * and can use the ring_buffer_event_data() interface.
  2262. *
  2263. * The length is the length of the data needed, not the event length
  2264. * which also includes the event header.
  2265. *
  2266. * Must be paired with ring_buffer_unlock_commit, unless NULL is returned.
  2267. * If NULL is returned, then nothing has been allocated or locked.
  2268. */
  2269. struct ring_buffer_event *
  2270. ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
  2271. {
  2272. struct ring_buffer_per_cpu *cpu_buffer;
  2273. struct ring_buffer_event *event;
  2274. int cpu;
  2275. if (ring_buffer_flags != RB_BUFFERS_ON)
  2276. return NULL;
  2277. /* If we are tracing schedule, we don't want to recurse */
  2278. preempt_disable_notrace();
  2279. if (atomic_read(&buffer->record_disabled))
  2280. goto out_nocheck;
  2281. if (trace_recursive_lock())
  2282. goto out_nocheck;
  2283. cpu = raw_smp_processor_id();
  2284. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2285. goto out;
  2286. cpu_buffer = buffer->buffers[cpu];
  2287. if (atomic_read(&cpu_buffer->record_disabled))
  2288. goto out;
  2289. if (length > BUF_MAX_DATA_SIZE)
  2290. goto out;
  2291. event = rb_reserve_next_event(buffer, cpu_buffer, length);
  2292. if (!event)
  2293. goto out;
  2294. return event;
  2295. out:
  2296. trace_recursive_unlock();
  2297. out_nocheck:
  2298. preempt_enable_notrace();
  2299. return NULL;
  2300. }
  2301. EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
  2302. static void
  2303. rb_update_write_stamp(struct ring_buffer_per_cpu *cpu_buffer,
  2304. struct ring_buffer_event *event)
  2305. {
  2306. u64 delta;
  2307. /*
  2308. * The event first in the commit queue updates the
  2309. * time stamp.
  2310. */
  2311. if (rb_event_is_commit(cpu_buffer, event)) {
  2312. /*
  2313. * A commit event that is first on a page
  2314. * updates the write timestamp with the page stamp
  2315. */
  2316. if (!rb_event_index(event))
  2317. cpu_buffer->write_stamp =
  2318. cpu_buffer->commit_page->page->time_stamp;
  2319. else if (event->type_len == RINGBUF_TYPE_TIME_EXTEND) {
  2320. delta = event->array[0];
  2321. delta <<= TS_SHIFT;
  2322. delta += event->time_delta;
  2323. cpu_buffer->write_stamp += delta;
  2324. } else
  2325. cpu_buffer->write_stamp += event->time_delta;
  2326. }
  2327. }
  2328. static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
  2329. struct ring_buffer_event *event)
  2330. {
  2331. local_inc(&cpu_buffer->entries);
  2332. rb_update_write_stamp(cpu_buffer, event);
  2333. rb_end_commit(cpu_buffer);
  2334. }
  2335. static __always_inline void
  2336. rb_wakeups(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer)
  2337. {
  2338. if (buffer->irq_work.waiters_pending) {
  2339. buffer->irq_work.waiters_pending = false;
  2340. /* irq_work_queue() supplies it's own memory barriers */
  2341. irq_work_queue(&buffer->irq_work.work);
  2342. }
  2343. if (cpu_buffer->irq_work.waiters_pending) {
  2344. cpu_buffer->irq_work.waiters_pending = false;
  2345. /* irq_work_queue() supplies it's own memory barriers */
  2346. irq_work_queue(&cpu_buffer->irq_work.work);
  2347. }
  2348. }
  2349. /**
  2350. * ring_buffer_unlock_commit - commit a reserved
  2351. * @buffer: The buffer to commit to
  2352. * @event: The event pointer to commit.
  2353. *
  2354. * This commits the data to the ring buffer, and releases any locks held.
  2355. *
  2356. * Must be paired with ring_buffer_lock_reserve.
  2357. */
  2358. int ring_buffer_unlock_commit(struct ring_buffer *buffer,
  2359. struct ring_buffer_event *event)
  2360. {
  2361. struct ring_buffer_per_cpu *cpu_buffer;
  2362. int cpu = raw_smp_processor_id();
  2363. cpu_buffer = buffer->buffers[cpu];
  2364. rb_commit(cpu_buffer, event);
  2365. rb_wakeups(buffer, cpu_buffer);
  2366. trace_recursive_unlock();
  2367. preempt_enable_notrace();
  2368. return 0;
  2369. }
  2370. EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);
  2371. static inline void rb_event_discard(struct ring_buffer_event *event)
  2372. {
  2373. if (event->type_len == RINGBUF_TYPE_TIME_EXTEND)
  2374. event = skip_time_extend(event);
  2375. /* array[0] holds the actual length for the discarded event */
  2376. event->array[0] = rb_event_data_length(event) - RB_EVNT_HDR_SIZE;
  2377. event->type_len = RINGBUF_TYPE_PADDING;
  2378. /* time delta must be non zero */
  2379. if (!event->time_delta)
  2380. event->time_delta = 1;
  2381. }
  2382. /*
  2383. * Decrement the entries to the page that an event is on.
  2384. * The event does not even need to exist, only the pointer
  2385. * to the page it is on. This may only be called before the commit
  2386. * takes place.
  2387. */
  2388. static inline void
  2389. rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer,
  2390. struct ring_buffer_event *event)
  2391. {
  2392. unsigned long addr = (unsigned long)event;
  2393. struct buffer_page *bpage = cpu_buffer->commit_page;
  2394. struct buffer_page *start;
  2395. addr &= PAGE_MASK;
  2396. /* Do the likely case first */
  2397. if (likely(bpage->page == (void *)addr)) {
  2398. local_dec(&bpage->entries);
  2399. return;
  2400. }
  2401. /*
  2402. * Because the commit page may be on the reader page we
  2403. * start with the next page and check the end loop there.
  2404. */
  2405. rb_inc_page(cpu_buffer, &bpage);
  2406. start = bpage;
  2407. do {
  2408. if (bpage->page == (void *)addr) {
  2409. local_dec(&bpage->entries);
  2410. return;
  2411. }
  2412. rb_inc_page(cpu_buffer, &bpage);
  2413. } while (bpage != start);
  2414. /* commit not part of this buffer?? */
  2415. RB_WARN_ON(cpu_buffer, 1);
  2416. }
  2417. /**
  2418. * ring_buffer_commit_discard - discard an event that has not been committed
  2419. * @buffer: the ring buffer
  2420. * @event: non committed event to discard
  2421. *
  2422. * Sometimes an event that is in the ring buffer needs to be ignored.
  2423. * This function lets the user discard an event in the ring buffer
  2424. * and then that event will not be read later.
  2425. *
  2426. * This function only works if it is called before the the item has been
  2427. * committed. It will try to free the event from the ring buffer
  2428. * if another event has not been added behind it.
  2429. *
  2430. * If another event has been added behind it, it will set the event
  2431. * up as discarded, and perform the commit.
  2432. *
  2433. * If this function is called, do not call ring_buffer_unlock_commit on
  2434. * the event.
  2435. */
  2436. void ring_buffer_discard_commit(struct ring_buffer *buffer,
  2437. struct ring_buffer_event *event)
  2438. {
  2439. struct ring_buffer_per_cpu *cpu_buffer;
  2440. int cpu;
  2441. /* The event is discarded regardless */
  2442. rb_event_discard(event);
  2443. cpu = smp_processor_id();
  2444. cpu_buffer = buffer->buffers[cpu];
  2445. /*
  2446. * This must only be called if the event has not been
  2447. * committed yet. Thus we can assume that preemption
  2448. * is still disabled.
  2449. */
  2450. RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));
  2451. rb_decrement_entry(cpu_buffer, event);
  2452. if (rb_try_to_discard(cpu_buffer, event))
  2453. goto out;
  2454. /*
  2455. * The commit is still visible by the reader, so we
  2456. * must still update the timestamp.
  2457. */
  2458. rb_update_write_stamp(cpu_buffer, event);
  2459. out:
  2460. rb_end_commit(cpu_buffer);
  2461. trace_recursive_unlock();
  2462. preempt_enable_notrace();
  2463. }
  2464. EXPORT_SYMBOL_GPL(ring_buffer_discard_commit);
  2465. /**
  2466. * ring_buffer_write - write data to the buffer without reserving
  2467. * @buffer: The ring buffer to write to.
  2468. * @length: The length of the data being written (excluding the event header)
  2469. * @data: The data to write to the buffer.
  2470. *
  2471. * This is like ring_buffer_lock_reserve and ring_buffer_unlock_commit as
  2472. * one function. If you already have the data to write to the buffer, it
  2473. * may be easier to simply call this function.
  2474. *
  2475. * Note, like ring_buffer_lock_reserve, the length is the length of the data
  2476. * and not the length of the event which would hold the header.
  2477. */
  2478. int ring_buffer_write(struct ring_buffer *buffer,
  2479. unsigned long length,
  2480. void *data)
  2481. {
  2482. struct ring_buffer_per_cpu *cpu_buffer;
  2483. struct ring_buffer_event *event;
  2484. void *body;
  2485. int ret = -EBUSY;
  2486. int cpu;
  2487. if (ring_buffer_flags != RB_BUFFERS_ON)
  2488. return -EBUSY;
  2489. preempt_disable_notrace();
  2490. if (atomic_read(&buffer->record_disabled))
  2491. goto out;
  2492. cpu = raw_smp_processor_id();
  2493. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2494. goto out;
  2495. cpu_buffer = buffer->buffers[cpu];
  2496. if (atomic_read(&cpu_buffer->record_disabled))
  2497. goto out;
  2498. if (length > BUF_MAX_DATA_SIZE)
  2499. goto out;
  2500. event = rb_reserve_next_event(buffer, cpu_buffer, length);
  2501. if (!event)
  2502. goto out;
  2503. body = rb_event_data(event);
  2504. memcpy(body, data, length);
  2505. rb_commit(cpu_buffer, event);
  2506. rb_wakeups(buffer, cpu_buffer);
  2507. ret = 0;
  2508. out:
  2509. preempt_enable_notrace();
  2510. return ret;
  2511. }
  2512. EXPORT_SYMBOL_GPL(ring_buffer_write);
  2513. static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
  2514. {
  2515. struct buffer_page *reader = cpu_buffer->reader_page;
  2516. struct buffer_page *head = rb_set_head_page(cpu_buffer);
  2517. struct buffer_page *commit = cpu_buffer->commit_page;
  2518. /* In case of error, head will be NULL */
  2519. if (unlikely(!head))
  2520. return 1;
  2521. return reader->read == rb_page_commit(reader) &&
  2522. (commit == reader ||
  2523. (commit == head &&
  2524. head->read == rb_page_commit(commit)));
  2525. }
  2526. /**
  2527. * ring_buffer_record_disable - stop all writes into the buffer
  2528. * @buffer: The ring buffer to stop writes to.
  2529. *
  2530. * This prevents all writes to the buffer. Any attempt to write
  2531. * to the buffer after this will fail and return NULL.
  2532. *
  2533. * The caller should call synchronize_sched() after this.
  2534. */
  2535. void ring_buffer_record_disable(struct ring_buffer *buffer)
  2536. {
  2537. atomic_inc(&buffer->record_disabled);
  2538. }
  2539. EXPORT_SYMBOL_GPL(ring_buffer_record_disable);
  2540. /**
  2541. * ring_buffer_record_enable - enable writes to the buffer
  2542. * @buffer: The ring buffer to enable writes
  2543. *
  2544. * Note, multiple disables will need the same number of enables
  2545. * to truly enable the writing (much like preempt_disable).
  2546. */
  2547. void ring_buffer_record_enable(struct ring_buffer *buffer)
  2548. {
  2549. atomic_dec(&buffer->record_disabled);
  2550. }
  2551. EXPORT_SYMBOL_GPL(ring_buffer_record_enable);
  2552. /**
  2553. * ring_buffer_record_off - stop all writes into the buffer
  2554. * @buffer: The ring buffer to stop writes to.
  2555. *
  2556. * This prevents all writes to the buffer. Any attempt to write
  2557. * to the buffer after this will fail and return NULL.
  2558. *
  2559. * This is different than ring_buffer_record_disable() as
  2560. * it works like an on/off switch, where as the disable() version
  2561. * must be paired with a enable().
  2562. */
  2563. void ring_buffer_record_off(struct ring_buffer *buffer)
  2564. {
  2565. unsigned int rd;
  2566. unsigned int new_rd;
  2567. do {
  2568. rd = atomic_read(&buffer->record_disabled);
  2569. new_rd = rd | RB_BUFFER_OFF;
  2570. } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);
  2571. }
  2572. EXPORT_SYMBOL_GPL(ring_buffer_record_off);
  2573. /**
  2574. * ring_buffer_record_on - restart writes into the buffer
  2575. * @buffer: The ring buffer to start writes to.
  2576. *
  2577. * This enables all writes to the buffer that was disabled by
  2578. * ring_buffer_record_off().
  2579. *
  2580. * This is different than ring_buffer_record_enable() as
  2581. * it works like an on/off switch, where as the enable() version
  2582. * must be paired with a disable().
  2583. */
  2584. void ring_buffer_record_on(struct ring_buffer *buffer)
  2585. {
  2586. unsigned int rd;
  2587. unsigned int new_rd;
  2588. do {
  2589. rd = atomic_read(&buffer->record_disabled);
  2590. new_rd = rd & ~RB_BUFFER_OFF;
  2591. } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);
  2592. }
  2593. EXPORT_SYMBOL_GPL(ring_buffer_record_on);
  2594. /**
  2595. * ring_buffer_record_is_on - return true if the ring buffer can write
  2596. * @buffer: The ring buffer to see if write is enabled
  2597. *
  2598. * Returns true if the ring buffer is in a state that it accepts writes.
  2599. */
  2600. int ring_buffer_record_is_on(struct ring_buffer *buffer)
  2601. {
  2602. return !atomic_read(&buffer->record_disabled);
  2603. }
  2604. /**
  2605. * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
  2606. * @buffer: The ring buffer to stop writes to.
  2607. * @cpu: The CPU buffer to stop
  2608. *
  2609. * This prevents all writes to the buffer. Any attempt to write
  2610. * to the buffer after this will fail and return NULL.
  2611. *
  2612. * The caller should call synchronize_sched() after this.
  2613. */
  2614. void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
  2615. {
  2616. struct ring_buffer_per_cpu *cpu_buffer;
  2617. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2618. return;
  2619. cpu_buffer = buffer->buffers[cpu];
  2620. atomic_inc(&cpu_buffer->record_disabled);
  2621. }
  2622. EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu);
  2623. /**
  2624. * ring_buffer_record_enable_cpu - enable writes to the buffer
  2625. * @buffer: The ring buffer to enable writes
  2626. * @cpu: The CPU to enable.
  2627. *
  2628. * Note, multiple disables will need the same number of enables
  2629. * to truly enable the writing (much like preempt_disable).
  2630. */
  2631. void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
  2632. {
  2633. struct ring_buffer_per_cpu *cpu_buffer;
  2634. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2635. return;
  2636. cpu_buffer = buffer->buffers[cpu];
  2637. atomic_dec(&cpu_buffer->record_disabled);
  2638. }
  2639. EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu);
  2640. /*
  2641. * The total entries in the ring buffer is the running counter
  2642. * of entries entered into the ring buffer, minus the sum of
  2643. * the entries read from the ring buffer and the number of
  2644. * entries that were overwritten.
  2645. */
  2646. static inline unsigned long
  2647. rb_num_of_entries(struct ring_buffer_per_cpu *cpu_buffer)
  2648. {
  2649. return local_read(&cpu_buffer->entries) -
  2650. (local_read(&cpu_buffer->overrun) + cpu_buffer->read);
  2651. }
  2652. /**
  2653. * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer
  2654. * @buffer: The ring buffer
  2655. * @cpu: The per CPU buffer to read from.
  2656. */
  2657. u64 ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu)
  2658. {
  2659. unsigned long flags;
  2660. struct ring_buffer_per_cpu *cpu_buffer;
  2661. struct buffer_page *bpage;
  2662. u64 ret = 0;
  2663. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2664. return 0;
  2665. cpu_buffer = buffer->buffers[cpu];
  2666. raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  2667. /*
  2668. * if the tail is on reader_page, oldest time stamp is on the reader
  2669. * page
  2670. */
  2671. if (cpu_buffer->tail_page == cpu_buffer->reader_page)
  2672. bpage = cpu_buffer->reader_page;
  2673. else
  2674. bpage = rb_set_head_page(cpu_buffer);
  2675. if (bpage)
  2676. ret = bpage->page->time_stamp;
  2677. raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  2678. return ret;
  2679. }
  2680. EXPORT_SYMBOL_GPL(ring_buffer_oldest_event_ts);
  2681. /**
  2682. * ring_buffer_bytes_cpu - get the number of bytes consumed in a cpu buffer
  2683. * @buffer: The ring buffer
  2684. * @cpu: The per CPU buffer to read from.
  2685. */
  2686. unsigned long ring_buffer_bytes_cpu(struct ring_buffer *buffer, int cpu)
  2687. {
  2688. struct ring_buffer_per_cpu *cpu_buffer;
  2689. unsigned long ret;
  2690. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2691. return 0;
  2692. cpu_buffer = buffer->buffers[cpu];
  2693. ret = local_read(&cpu_buffer->entries_bytes) - cpu_buffer->read_bytes;
  2694. return ret;
  2695. }
  2696. EXPORT_SYMBOL_GPL(ring_buffer_bytes_cpu);
  2697. /**
  2698. * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
  2699. * @buffer: The ring buffer
  2700. * @cpu: The per CPU buffer to get the entries from.
  2701. */
  2702. unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
  2703. {
  2704. struct ring_buffer_per_cpu *cpu_buffer;
  2705. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2706. return 0;
  2707. cpu_buffer = buffer->buffers[cpu];
  2708. return rb_num_of_entries(cpu_buffer);
  2709. }
  2710. EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);
  2711. /**
  2712. * ring_buffer_overrun_cpu - get the number of overruns caused by the ring
  2713. * buffer wrapping around (only if RB_FL_OVERWRITE is on).
  2714. * @buffer: The ring buffer
  2715. * @cpu: The per CPU buffer to get the number of overruns from
  2716. */
  2717. unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
  2718. {
  2719. struct ring_buffer_per_cpu *cpu_buffer;
  2720. unsigned long ret;
  2721. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2722. return 0;
  2723. cpu_buffer = buffer->buffers[cpu];
  2724. ret = local_read(&cpu_buffer->overrun);
  2725. return ret;
  2726. }
  2727. EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
  2728. /**
  2729. * ring_buffer_commit_overrun_cpu - get the number of overruns caused by
  2730. * commits failing due to the buffer wrapping around while there are uncommitted
  2731. * events, such as during an interrupt storm.
  2732. * @buffer: The ring buffer
  2733. * @cpu: The per CPU buffer to get the number of overruns from
  2734. */
  2735. unsigned long
  2736. ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)
  2737. {
  2738. struct ring_buffer_per_cpu *cpu_buffer;
  2739. unsigned long ret;
  2740. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2741. return 0;
  2742. cpu_buffer = buffer->buffers[cpu];
  2743. ret = local_read(&cpu_buffer->commit_overrun);
  2744. return ret;
  2745. }
  2746. EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu);
  2747. /**
  2748. * ring_buffer_dropped_events_cpu - get the number of dropped events caused by
  2749. * the ring buffer filling up (only if RB_FL_OVERWRITE is off).
  2750. * @buffer: The ring buffer
  2751. * @cpu: The per CPU buffer to get the number of overruns from
  2752. */
  2753. unsigned long
  2754. ring_buffer_dropped_events_cpu(struct ring_buffer *buffer, int cpu)
  2755. {
  2756. struct ring_buffer_per_cpu *cpu_buffer;
  2757. unsigned long ret;
  2758. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2759. return 0;
  2760. cpu_buffer = buffer->buffers[cpu];
  2761. ret = local_read(&cpu_buffer->dropped_events);
  2762. return ret;
  2763. }
  2764. EXPORT_SYMBOL_GPL(ring_buffer_dropped_events_cpu);
  2765. /**
  2766. * ring_buffer_read_events_cpu - get the number of events successfully read
  2767. * @buffer: The ring buffer
  2768. * @cpu: The per CPU buffer to get the number of events read
  2769. */
  2770. unsigned long
  2771. ring_buffer_read_events_cpu(struct ring_buffer *buffer, int cpu)
  2772. {
  2773. struct ring_buffer_per_cpu *cpu_buffer;
  2774. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  2775. return 0;
  2776. cpu_buffer = buffer->buffers[cpu];
  2777. return cpu_buffer->read;
  2778. }
  2779. EXPORT_SYMBOL_GPL(ring_buffer_read_events_cpu);
  2780. /**
  2781. * ring_buffer_entries - get the number of entries in a buffer
  2782. * @buffer: The ring buffer
  2783. *
  2784. * Returns the total number of entries in the ring buffer
  2785. * (all CPU entries)
  2786. */
  2787. unsigned long ring_buffer_entries(struct ring_buffer *buffer)
  2788. {
  2789. struct ring_buffer_per_cpu *cpu_buffer;
  2790. unsigned long entries = 0;
  2791. int cpu;
  2792. /* if you care about this being correct, lock the buffer */
  2793. for_each_buffer_cpu(buffer, cpu) {
  2794. cpu_buffer = buffer->buffers[cpu];
  2795. entries += rb_num_of_entries(cpu_buffer);
  2796. }
  2797. return entries;
  2798. }
  2799. EXPORT_SYMBOL_GPL(ring_buffer_entries);
  2800. /**
  2801. * ring_buffer_overruns - get the number of overruns in buffer
  2802. * @buffer: The ring buffer
  2803. *
  2804. * Returns the total number of overruns in the ring buffer
  2805. * (all CPU entries)
  2806. */
  2807. unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
  2808. {
  2809. struct ring_buffer_per_cpu *cpu_buffer;
  2810. unsigned long overruns = 0;
  2811. int cpu;
  2812. /* if you care about this being correct, lock the buffer */
  2813. for_each_buffer_cpu(buffer, cpu) {
  2814. cpu_buffer = buffer->buffers[cpu];
  2815. overruns += local_read(&cpu_buffer->overrun);
  2816. }
  2817. return overruns;
  2818. }
  2819. EXPORT_SYMBOL_GPL(ring_buffer_overruns);
  2820. static void rb_iter_reset(struct ring_buffer_iter *iter)
  2821. {
  2822. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  2823. /* Iterator usage is expected to have record disabled */
  2824. if (list_empty(&cpu_buffer->reader_page->list)) {
  2825. iter->head_page = rb_set_head_page(cpu_buffer);
  2826. if (unlikely(!iter->head_page))
  2827. return;
  2828. iter->head = iter->head_page->read;
  2829. } else {
  2830. iter->head_page = cpu_buffer->reader_page;
  2831. iter->head = cpu_buffer->reader_page->read;
  2832. }
  2833. if (iter->head)
  2834. iter->read_stamp = cpu_buffer->read_stamp;
  2835. else
  2836. iter->read_stamp = iter->head_page->page->time_stamp;
  2837. iter->cache_reader_page = cpu_buffer->reader_page;
  2838. iter->cache_read = cpu_buffer->read;
  2839. }
  2840. /**
  2841. * ring_buffer_iter_reset - reset an iterator
  2842. * @iter: The iterator to reset
  2843. *
  2844. * Resets the iterator, so that it will start from the beginning
  2845. * again.
  2846. */
  2847. void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
  2848. {
  2849. struct ring_buffer_per_cpu *cpu_buffer;
  2850. unsigned long flags;
  2851. if (!iter)
  2852. return;
  2853. cpu_buffer = iter->cpu_buffer;
  2854. raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  2855. rb_iter_reset(iter);
  2856. raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  2857. }
  2858. EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);
  2859. /**
  2860. * ring_buffer_iter_empty - check if an iterator has no more to read
  2861. * @iter: The iterator to check
  2862. */
  2863. int ring_buffer_iter_empty(struct ring_buffer_iter *iter)
  2864. {
  2865. struct ring_buffer_per_cpu *cpu_buffer;
  2866. cpu_buffer = iter->cpu_buffer;
  2867. return iter->head_page == cpu_buffer->commit_page &&
  2868. iter->head == rb_commit_index(cpu_buffer);
  2869. }
  2870. EXPORT_SYMBOL_GPL(ring_buffer_iter_empty);
  2871. static void
  2872. rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
  2873. struct ring_buffer_event *event)
  2874. {
  2875. u64 delta;
  2876. switch (event->type_len) {
  2877. case RINGBUF_TYPE_PADDING:
  2878. return;
  2879. case RINGBUF_TYPE_TIME_EXTEND:
  2880. delta = event->array[0];
  2881. delta <<= TS_SHIFT;
  2882. delta += event->time_delta;
  2883. cpu_buffer->read_stamp += delta;
  2884. return;
  2885. case RINGBUF_TYPE_TIME_STAMP:
  2886. /* FIXME: not implemented */
  2887. return;
  2888. case RINGBUF_TYPE_DATA:
  2889. cpu_buffer->read_stamp += event->time_delta;
  2890. return;
  2891. default:
  2892. BUG();
  2893. }
  2894. return;
  2895. }
  2896. static void
  2897. rb_update_iter_read_stamp(struct ring_buffer_iter *iter,
  2898. struct ring_buffer_event *event)
  2899. {
  2900. u64 delta;
  2901. switch (event->type_len) {
  2902. case RINGBUF_TYPE_PADDING:
  2903. return;
  2904. case RINGBUF_TYPE_TIME_EXTEND:
  2905. delta = event->array[0];
  2906. delta <<= TS_SHIFT;
  2907. delta += event->time_delta;
  2908. iter->read_stamp += delta;
  2909. return;
  2910. case RINGBUF_TYPE_TIME_STAMP:
  2911. /* FIXME: not implemented */
  2912. return;
  2913. case RINGBUF_TYPE_DATA:
  2914. iter->read_stamp += event->time_delta;
  2915. return;
  2916. default:
  2917. BUG();
  2918. }
  2919. return;
  2920. }
  2921. static struct buffer_page *
  2922. rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
  2923. {
  2924. struct buffer_page *reader = NULL;
  2925. unsigned long overwrite;
  2926. unsigned long flags;
  2927. int nr_loops = 0;
  2928. int ret;
  2929. local_irq_save(flags);
  2930. arch_spin_lock(&cpu_buffer->lock);
  2931. again:
  2932. /*
  2933. * This should normally only loop twice. But because the
  2934. * start of the reader inserts an empty page, it causes
  2935. * a case where we will loop three times. There should be no
  2936. * reason to loop four times (that I know of).
  2937. */
  2938. if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) {
  2939. reader = NULL;
  2940. goto out;
  2941. }
  2942. reader = cpu_buffer->reader_page;
  2943. /* If there's more to read, return this page */
  2944. if (cpu_buffer->reader_page->read < rb_page_size(reader))
  2945. goto out;
  2946. /* Never should we have an index greater than the size */
  2947. if (RB_WARN_ON(cpu_buffer,
  2948. cpu_buffer->reader_page->read > rb_page_size(reader)))
  2949. goto out;
  2950. /* check if we caught up to the tail */
  2951. reader = NULL;
  2952. if (cpu_buffer->commit_page == cpu_buffer->reader_page)
  2953. goto out;
  2954. /* Don't bother swapping if the ring buffer is empty */
  2955. if (rb_num_of_entries(cpu_buffer) == 0)
  2956. goto out;
  2957. /*
  2958. * Reset the reader page to size zero.
  2959. */
  2960. local_set(&cpu_buffer->reader_page->write, 0);
  2961. local_set(&cpu_buffer->reader_page->entries, 0);
  2962. local_set(&cpu_buffer->reader_page->page->commit, 0);
  2963. cpu_buffer->reader_page->real_end = 0;
  2964. spin:
  2965. /*
  2966. * Splice the empty reader page into the list around the head.
  2967. */
  2968. reader = rb_set_head_page(cpu_buffer);
  2969. if (!reader)
  2970. goto out;
  2971. cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);
  2972. cpu_buffer->reader_page->list.prev = reader->list.prev;
  2973. /*
  2974. * cpu_buffer->pages just needs to point to the buffer, it
  2975. * has no specific buffer page to point to. Lets move it out
  2976. * of our way so we don't accidentally swap it.
  2977. */
  2978. cpu_buffer->pages = reader->list.prev;
  2979. /* The reader page will be pointing to the new head */
  2980. rb_set_list_to_head(cpu_buffer, &cpu_buffer->reader_page->list);
  2981. /*
  2982. * We want to make sure we read the overruns after we set up our
  2983. * pointers to the next object. The writer side does a
  2984. * cmpxchg to cross pages which acts as the mb on the writer
  2985. * side. Note, the reader will constantly fail the swap
  2986. * while the writer is updating the pointers, so this
  2987. * guarantees that the overwrite recorded here is the one we
  2988. * want to compare with the last_overrun.
  2989. */
  2990. smp_mb();
  2991. overwrite = local_read(&(cpu_buffer->overrun));
  2992. /*
  2993. * Here's the tricky part.
  2994. *
  2995. * We need to move the pointer past the header page.
  2996. * But we can only do that if a writer is not currently
  2997. * moving it. The page before the header page has the
  2998. * flag bit '1' set if it is pointing to the page we want.
  2999. * but if the writer is in the process of moving it
  3000. * than it will be '2' or already moved '0'.
  3001. */
  3002. ret = rb_head_page_replace(reader, cpu_buffer->reader_page);
  3003. /*
  3004. * If we did not convert it, then we must try again.
  3005. */
  3006. if (!ret)
  3007. goto spin;
  3008. /*
  3009. * Yeah! We succeeded in replacing the page.
  3010. *
  3011. * Now make the new head point back to the reader page.
  3012. */
  3013. rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;
  3014. rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
  3015. /* Finally update the reader page to the new head */
  3016. cpu_buffer->reader_page = reader;
  3017. rb_reset_reader_page(cpu_buffer);
  3018. if (overwrite != cpu_buffer->last_overrun) {
  3019. cpu_buffer->lost_events = overwrite - cpu_buffer->last_overrun;
  3020. cpu_buffer->last_overrun = overwrite;
  3021. }
  3022. goto again;
  3023. out:
  3024. arch_spin_unlock(&cpu_buffer->lock);
  3025. local_irq_restore(flags);
  3026. return reader;
  3027. }
  3028. static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
  3029. {
  3030. struct ring_buffer_event *event;
  3031. struct buffer_page *reader;
  3032. unsigned length;
  3033. reader = rb_get_reader_page(cpu_buffer);
  3034. /* This function should not be called when buffer is empty */
  3035. if (RB_WARN_ON(cpu_buffer, !reader))
  3036. return;
  3037. event = rb_reader_event(cpu_buffer);
  3038. if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
  3039. cpu_buffer->read++;
  3040. rb_update_read_stamp(cpu_buffer, event);
  3041. length = rb_event_length(event);
  3042. cpu_buffer->reader_page->read += length;
  3043. }
  3044. static void rb_advance_iter(struct ring_buffer_iter *iter)
  3045. {
  3046. struct ring_buffer_per_cpu *cpu_buffer;
  3047. struct ring_buffer_event *event;
  3048. unsigned length;
  3049. cpu_buffer = iter->cpu_buffer;
  3050. /*
  3051. * Check if we are at the end of the buffer.
  3052. */
  3053. if (iter->head >= rb_page_size(iter->head_page)) {
  3054. /* discarded commits can make the page empty */
  3055. if (iter->head_page == cpu_buffer->commit_page)
  3056. return;
  3057. rb_inc_iter(iter);
  3058. return;
  3059. }
  3060. event = rb_iter_head_event(iter);
  3061. length = rb_event_length(event);
  3062. /*
  3063. * This should not be called to advance the header if we are
  3064. * at the tail of the buffer.
  3065. */
  3066. if (RB_WARN_ON(cpu_buffer,
  3067. (iter->head_page == cpu_buffer->commit_page) &&
  3068. (iter->head + length > rb_commit_index(cpu_buffer))))
  3069. return;
  3070. rb_update_iter_read_stamp(iter, event);
  3071. iter->head += length;
  3072. /* check for end of page padding */
  3073. if ((iter->head >= rb_page_size(iter->head_page)) &&
  3074. (iter->head_page != cpu_buffer->commit_page))
  3075. rb_inc_iter(iter);
  3076. }
  3077. static int rb_lost_events(struct ring_buffer_per_cpu *cpu_buffer)
  3078. {
  3079. return cpu_buffer->lost_events;
  3080. }
  3081. static struct ring_buffer_event *
  3082. rb_buffer_peek(struct ring_buffer_per_cpu *cpu_buffer, u64 *ts,
  3083. unsigned long *lost_events)
  3084. {
  3085. struct ring_buffer_event *event;
  3086. struct buffer_page *reader;
  3087. int nr_loops = 0;
  3088. again:
  3089. /*
  3090. * We repeat when a time extend is encountered.
  3091. * Since the time extend is always attached to a data event,
  3092. * we should never loop more than once.
  3093. * (We never hit the following condition more than twice).
  3094. */
  3095. if (RB_WARN_ON(cpu_buffer, ++nr_loops > 2))
  3096. return NULL;
  3097. reader = rb_get_reader_page(cpu_buffer);
  3098. if (!reader)
  3099. return NULL;
  3100. event = rb_reader_event(cpu_buffer);
  3101. switch (event->type_len) {
  3102. case RINGBUF_TYPE_PADDING:
  3103. if (rb_null_event(event))
  3104. RB_WARN_ON(cpu_buffer, 1);
  3105. /*
  3106. * Because the writer could be discarding every
  3107. * event it creates (which would probably be bad)
  3108. * if we were to go back to "again" then we may never
  3109. * catch up, and will trigger the warn on, or lock
  3110. * the box. Return the padding, and we will release
  3111. * the current locks, and try again.
  3112. */
  3113. return event;
  3114. case RINGBUF_TYPE_TIME_EXTEND:
  3115. /* Internal data, OK to advance */
  3116. rb_advance_reader(cpu_buffer);
  3117. goto again;
  3118. case RINGBUF_TYPE_TIME_STAMP:
  3119. /* FIXME: not implemented */
  3120. rb_advance_reader(cpu_buffer);
  3121. goto again;
  3122. case RINGBUF_TYPE_DATA:
  3123. if (ts) {
  3124. *ts = cpu_buffer->read_stamp + event->time_delta;
  3125. ring_buffer_normalize_time_stamp(cpu_buffer->buffer,
  3126. cpu_buffer->cpu, ts);
  3127. }
  3128. if (lost_events)
  3129. *lost_events = rb_lost_events(cpu_buffer);
  3130. return event;
  3131. default:
  3132. BUG();
  3133. }
  3134. return NULL;
  3135. }
  3136. EXPORT_SYMBOL_GPL(ring_buffer_peek);
  3137. static struct ring_buffer_event *
  3138. rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
  3139. {
  3140. struct ring_buffer *buffer;
  3141. struct ring_buffer_per_cpu *cpu_buffer;
  3142. struct ring_buffer_event *event;
  3143. int nr_loops = 0;
  3144. cpu_buffer = iter->cpu_buffer;
  3145. buffer = cpu_buffer->buffer;
  3146. /*
  3147. * Check if someone performed a consuming read to
  3148. * the buffer. A consuming read invalidates the iterator
  3149. * and we need to reset the iterator in this case.
  3150. */
  3151. if (unlikely(iter->cache_read != cpu_buffer->read ||
  3152. iter->cache_reader_page != cpu_buffer->reader_page))
  3153. rb_iter_reset(iter);
  3154. again:
  3155. if (ring_buffer_iter_empty(iter))
  3156. return NULL;
  3157. /*
  3158. * We repeat when a time extend is encountered.
  3159. * Since the time extend is always attached to a data event,
  3160. * we should never loop more than once.
  3161. * (We never hit the following condition more than twice).
  3162. */
  3163. if (RB_WARN_ON(cpu_buffer, ++nr_loops > 2))
  3164. return NULL;
  3165. if (rb_per_cpu_empty(cpu_buffer))
  3166. return NULL;
  3167. if (iter->head >= local_read(&iter->head_page->page->commit)) {
  3168. rb_inc_iter(iter);
  3169. goto again;
  3170. }
  3171. event = rb_iter_head_event(iter);
  3172. switch (event->type_len) {
  3173. case RINGBUF_TYPE_PADDING:
  3174. if (rb_null_event(event)) {
  3175. rb_inc_iter(iter);
  3176. goto again;
  3177. }
  3178. rb_advance_iter(iter);
  3179. return event;
  3180. case RINGBUF_TYPE_TIME_EXTEND:
  3181. /* Internal data, OK to advance */
  3182. rb_advance_iter(iter);
  3183. goto again;
  3184. case RINGBUF_TYPE_TIME_STAMP:
  3185. /* FIXME: not implemented */
  3186. rb_advance_iter(iter);
  3187. goto again;
  3188. case RINGBUF_TYPE_DATA:
  3189. if (ts) {
  3190. *ts = iter->read_stamp + event->time_delta;
  3191. ring_buffer_normalize_time_stamp(buffer,
  3192. cpu_buffer->cpu, ts);
  3193. }
  3194. return event;
  3195. default:
  3196. BUG();
  3197. }
  3198. return NULL;
  3199. }
  3200. EXPORT_SYMBOL_GPL(ring_buffer_iter_peek);
  3201. static inline int rb_ok_to_lock(void)
  3202. {
  3203. /*
  3204. * If an NMI die dumps out the content of the ring buffer
  3205. * do not grab locks. We also permanently disable the ring
  3206. * buffer too. A one time deal is all you get from reading
  3207. * the ring buffer from an NMI.
  3208. */
  3209. if (likely(!in_nmi()))
  3210. return 1;
  3211. tracing_off_permanent();
  3212. return 0;
  3213. }
  3214. /**
  3215. * ring_buffer_peek - peek at the next event to be read
  3216. * @buffer: The ring buffer to read
  3217. * @cpu: The cpu to peak at
  3218. * @ts: The timestamp counter of this event.
  3219. * @lost_events: a variable to store if events were lost (may be NULL)
  3220. *
  3221. * This will return the event that will be read next, but does
  3222. * not consume the data.
  3223. */
  3224. struct ring_buffer_event *
  3225. ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts,
  3226. unsigned long *lost_events)
  3227. {
  3228. struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
  3229. struct ring_buffer_event *event;
  3230. unsigned long flags;
  3231. int dolock;
  3232. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  3233. return NULL;
  3234. dolock = rb_ok_to_lock();
  3235. again:
  3236. local_irq_save(flags);
  3237. if (dolock)
  3238. raw_spin_lock(&cpu_buffer->reader_lock);
  3239. event = rb_buffer_peek(cpu_buffer, ts, lost_events);
  3240. if (event && event->type_len == RINGBUF_TYPE_PADDING)
  3241. rb_advance_reader(cpu_buffer);
  3242. if (dolock)
  3243. raw_spin_unlock(&cpu_buffer->reader_lock);
  3244. local_irq_restore(flags);
  3245. if (event && event->type_len == RINGBUF_TYPE_PADDING)
  3246. goto again;
  3247. return event;
  3248. }
  3249. /**
  3250. * ring_buffer_iter_peek - peek at the next event to be read
  3251. * @iter: The ring buffer iterator
  3252. * @ts: The timestamp counter of this event.
  3253. *
  3254. * This will return the event that will be read next, but does
  3255. * not increment the iterator.
  3256. */
  3257. struct ring_buffer_event *
  3258. ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
  3259. {
  3260. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  3261. struct ring_buffer_event *event;
  3262. unsigned long flags;
  3263. again:
  3264. raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  3265. event = rb_iter_peek(iter, ts);
  3266. raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  3267. if (event && event->type_len == RINGBUF_TYPE_PADDING)
  3268. goto again;
  3269. return event;
  3270. }
  3271. /**
  3272. * ring_buffer_consume - return an event and consume it
  3273. * @buffer: The ring buffer to get the next event from
  3274. * @cpu: the cpu to read the buffer from
  3275. * @ts: a variable to store the timestamp (may be NULL)
  3276. * @lost_events: a variable to store if events were lost (may be NULL)
  3277. *
  3278. * Returns the next event in the ring buffer, and that event is consumed.
  3279. * Meaning, that sequential reads will keep returning a different event,
  3280. * and eventually empty the ring buffer if the producer is slower.
  3281. */
  3282. struct ring_buffer_event *
  3283. ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts,
  3284. unsigned long *lost_events)
  3285. {
  3286. struct ring_buffer_per_cpu *cpu_buffer;
  3287. struct ring_buffer_event *event = NULL;
  3288. unsigned long flags;
  3289. int dolock;
  3290. dolock = rb_ok_to_lock();
  3291. again:
  3292. /* might be called in atomic */
  3293. preempt_disable();
  3294. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  3295. goto out;
  3296. cpu_buffer = buffer->buffers[cpu];
  3297. local_irq_save(flags);
  3298. if (dolock)
  3299. raw_spin_lock(&cpu_buffer->reader_lock);
  3300. event = rb_buffer_peek(cpu_buffer, ts, lost_events);
  3301. if (event) {
  3302. cpu_buffer->lost_events = 0;
  3303. rb_advance_reader(cpu_buffer);
  3304. }
  3305. if (dolock)
  3306. raw_spin_unlock(&cpu_buffer->reader_lock);
  3307. local_irq_restore(flags);
  3308. out:
  3309. preempt_enable();
  3310. if (event && event->type_len == RINGBUF_TYPE_PADDING)
  3311. goto again;
  3312. return event;
  3313. }
  3314. EXPORT_SYMBOL_GPL(ring_buffer_consume);
  3315. /**
  3316. * ring_buffer_read_prepare - Prepare for a non consuming read of the buffer
  3317. * @buffer: The ring buffer to read from
  3318. * @cpu: The cpu buffer to iterate over
  3319. *
  3320. * This performs the initial preparations necessary to iterate
  3321. * through the buffer. Memory is allocated, buffer recording
  3322. * is disabled, and the iterator pointer is returned to the caller.
  3323. *
  3324. * Disabling buffer recordng prevents the reading from being
  3325. * corrupted. This is not a consuming read, so a producer is not
  3326. * expected.
  3327. *
  3328. * After a sequence of ring_buffer_read_prepare calls, the user is
  3329. * expected to make at least one call to ring_buffer_prepare_sync.
  3330. * Afterwards, ring_buffer_read_start is invoked to get things going
  3331. * for real.
  3332. *
  3333. * This overall must be paired with ring_buffer_finish.
  3334. */
  3335. struct ring_buffer_iter *
  3336. ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu)
  3337. {
  3338. struct ring_buffer_per_cpu *cpu_buffer;
  3339. struct ring_buffer_iter *iter;
  3340. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  3341. return NULL;
  3342. iter = kmalloc(sizeof(*iter), GFP_KERNEL);
  3343. if (!iter)
  3344. return NULL;
  3345. cpu_buffer = buffer->buffers[cpu];
  3346. iter->cpu_buffer = cpu_buffer;
  3347. atomic_inc(&buffer->resize_disabled);
  3348. atomic_inc(&cpu_buffer->record_disabled);
  3349. return iter;
  3350. }
  3351. EXPORT_SYMBOL_GPL(ring_buffer_read_prepare);
  3352. /**
  3353. * ring_buffer_read_prepare_sync - Synchronize a set of prepare calls
  3354. *
  3355. * All previously invoked ring_buffer_read_prepare calls to prepare
  3356. * iterators will be synchronized. Afterwards, read_buffer_read_start
  3357. * calls on those iterators are allowed.
  3358. */
  3359. void
  3360. ring_buffer_read_prepare_sync(void)
  3361. {
  3362. synchronize_sched();
  3363. }
  3364. EXPORT_SYMBOL_GPL(ring_buffer_read_prepare_sync);
  3365. /**
  3366. * ring_buffer_read_start - start a non consuming read of the buffer
  3367. * @iter: The iterator returned by ring_buffer_read_prepare
  3368. *
  3369. * This finalizes the startup of an iteration through the buffer.
  3370. * The iterator comes from a call to ring_buffer_read_prepare and
  3371. * an intervening ring_buffer_read_prepare_sync must have been
  3372. * performed.
  3373. *
  3374. * Must be paired with ring_buffer_finish.
  3375. */
  3376. void
  3377. ring_buffer_read_start(struct ring_buffer_iter *iter)
  3378. {
  3379. struct ring_buffer_per_cpu *cpu_buffer;
  3380. unsigned long flags;
  3381. if (!iter)
  3382. return;
  3383. cpu_buffer = iter->cpu_buffer;
  3384. raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  3385. arch_spin_lock(&cpu_buffer->lock);
  3386. rb_iter_reset(iter);
  3387. arch_spin_unlock(&cpu_buffer->lock);
  3388. raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  3389. }
  3390. EXPORT_SYMBOL_GPL(ring_buffer_read_start);
  3391. /**
  3392. * ring_buffer_finish - finish reading the iterator of the buffer
  3393. * @iter: The iterator retrieved by ring_buffer_start
  3394. *
  3395. * This re-enables the recording to the buffer, and frees the
  3396. * iterator.
  3397. */
  3398. void
  3399. ring_buffer_read_finish(struct ring_buffer_iter *iter)
  3400. {
  3401. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  3402. unsigned long flags;
  3403. /*
  3404. * Ring buffer is disabled from recording, here's a good place
  3405. * to check the integrity of the ring buffer.
  3406. * Must prevent readers from trying to read, as the check
  3407. * clears the HEAD page and readers require it.
  3408. */
  3409. raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  3410. rb_check_pages(cpu_buffer);
  3411. raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  3412. atomic_dec(&cpu_buffer->record_disabled);
  3413. atomic_dec(&cpu_buffer->buffer->resize_disabled);
  3414. kfree(iter);
  3415. }
  3416. EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
  3417. /**
  3418. * ring_buffer_read - read the next item in the ring buffer by the iterator
  3419. * @iter: The ring buffer iterator
  3420. * @ts: The time stamp of the event read.
  3421. *
  3422. * This reads the next event in the ring buffer and increments the iterator.
  3423. */
  3424. struct ring_buffer_event *
  3425. ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
  3426. {
  3427. struct ring_buffer_event *event;
  3428. struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
  3429. unsigned long flags;
  3430. raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  3431. again:
  3432. event = rb_iter_peek(iter, ts);
  3433. if (!event)
  3434. goto out;
  3435. if (event->type_len == RINGBUF_TYPE_PADDING)
  3436. goto again;
  3437. rb_advance_iter(iter);
  3438. out:
  3439. raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  3440. return event;
  3441. }
  3442. EXPORT_SYMBOL_GPL(ring_buffer_read);
  3443. /**
  3444. * ring_buffer_size - return the size of the ring buffer (in bytes)
  3445. * @buffer: The ring buffer.
  3446. */
  3447. unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu)
  3448. {
  3449. /*
  3450. * Earlier, this method returned
  3451. * BUF_PAGE_SIZE * buffer->nr_pages
  3452. * Since the nr_pages field is now removed, we have converted this to
  3453. * return the per cpu buffer value.
  3454. */
  3455. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  3456. return 0;
  3457. return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;
  3458. }
  3459. EXPORT_SYMBOL_GPL(ring_buffer_size);
  3460. static void
  3461. rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
  3462. {
  3463. rb_head_page_deactivate(cpu_buffer);
  3464. cpu_buffer->head_page
  3465. = list_entry(cpu_buffer->pages, struct buffer_page, list);
  3466. local_set(&cpu_buffer->head_page->write, 0);
  3467. local_set(&cpu_buffer->head_page->entries, 0);
  3468. local_set(&cpu_buffer->head_page->page->commit, 0);
  3469. cpu_buffer->head_page->read = 0;
  3470. cpu_buffer->tail_page = cpu_buffer->head_page;
  3471. cpu_buffer->commit_page = cpu_buffer->head_page;
  3472. INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
  3473. INIT_LIST_HEAD(&cpu_buffer->new_pages);
  3474. local_set(&cpu_buffer->reader_page->write, 0);
  3475. local_set(&cpu_buffer->reader_page->entries, 0);
  3476. local_set(&cpu_buffer->reader_page->page->commit, 0);
  3477. cpu_buffer->reader_page->read = 0;
  3478. local_set(&cpu_buffer->entries_bytes, 0);
  3479. local_set(&cpu_buffer->overrun, 0);
  3480. local_set(&cpu_buffer->commit_overrun, 0);
  3481. local_set(&cpu_buffer->dropped_events, 0);
  3482. local_set(&cpu_buffer->entries, 0);
  3483. local_set(&cpu_buffer->committing, 0);
  3484. local_set(&cpu_buffer->commits, 0);
  3485. cpu_buffer->read = 0;
  3486. cpu_buffer->read_bytes = 0;
  3487. cpu_buffer->write_stamp = 0;
  3488. cpu_buffer->read_stamp = 0;
  3489. cpu_buffer->lost_events = 0;
  3490. cpu_buffer->last_overrun = 0;
  3491. rb_head_page_activate(cpu_buffer);
  3492. }
  3493. /**
  3494. * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
  3495. * @buffer: The ring buffer to reset a per cpu buffer of
  3496. * @cpu: The CPU buffer to be reset
  3497. */
  3498. void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
  3499. {
  3500. struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
  3501. unsigned long flags;
  3502. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  3503. return;
  3504. atomic_inc(&buffer->resize_disabled);
  3505. atomic_inc(&cpu_buffer->record_disabled);
  3506. /* Make sure all commits have finished */
  3507. synchronize_sched();
  3508. raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  3509. if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing)))
  3510. goto out;
  3511. arch_spin_lock(&cpu_buffer->lock);
  3512. rb_reset_cpu(cpu_buffer);
  3513. arch_spin_unlock(&cpu_buffer->lock);
  3514. out:
  3515. raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  3516. atomic_dec(&cpu_buffer->record_disabled);
  3517. atomic_dec(&buffer->resize_disabled);
  3518. }
  3519. EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
  3520. /**
  3521. * ring_buffer_reset - reset a ring buffer
  3522. * @buffer: The ring buffer to reset all cpu buffers
  3523. */
  3524. void ring_buffer_reset(struct ring_buffer *buffer)
  3525. {
  3526. int cpu;
  3527. for_each_buffer_cpu(buffer, cpu)
  3528. ring_buffer_reset_cpu(buffer, cpu);
  3529. }
  3530. EXPORT_SYMBOL_GPL(ring_buffer_reset);
  3531. /**
  3532. * rind_buffer_empty - is the ring buffer empty?
  3533. * @buffer: The ring buffer to test
  3534. */
  3535. int ring_buffer_empty(struct ring_buffer *buffer)
  3536. {
  3537. struct ring_buffer_per_cpu *cpu_buffer;
  3538. unsigned long flags;
  3539. int dolock;
  3540. int cpu;
  3541. int ret;
  3542. dolock = rb_ok_to_lock();
  3543. /* yes this is racy, but if you don't like the race, lock the buffer */
  3544. for_each_buffer_cpu(buffer, cpu) {
  3545. cpu_buffer = buffer->buffers[cpu];
  3546. local_irq_save(flags);
  3547. if (dolock)
  3548. raw_spin_lock(&cpu_buffer->reader_lock);
  3549. ret = rb_per_cpu_empty(cpu_buffer);
  3550. if (dolock)
  3551. raw_spin_unlock(&cpu_buffer->reader_lock);
  3552. local_irq_restore(flags);
  3553. if (!ret)
  3554. return 0;
  3555. }
  3556. return 1;
  3557. }
  3558. EXPORT_SYMBOL_GPL(ring_buffer_empty);
  3559. /**
  3560. * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
  3561. * @buffer: The ring buffer
  3562. * @cpu: The CPU buffer to test
  3563. */
  3564. int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
  3565. {
  3566. struct ring_buffer_per_cpu *cpu_buffer;
  3567. unsigned long flags;
  3568. int dolock;
  3569. int ret;
  3570. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  3571. return 1;
  3572. dolock = rb_ok_to_lock();
  3573. cpu_buffer = buffer->buffers[cpu];
  3574. local_irq_save(flags);
  3575. if (dolock)
  3576. raw_spin_lock(&cpu_buffer->reader_lock);
  3577. ret = rb_per_cpu_empty(cpu_buffer);
  3578. if (dolock)
  3579. raw_spin_unlock(&cpu_buffer->reader_lock);
  3580. local_irq_restore(flags);
  3581. return ret;
  3582. }
  3583. EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
  3584. #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
  3585. /**
  3586. * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
  3587. * @buffer_a: One buffer to swap with
  3588. * @buffer_b: The other buffer to swap with
  3589. *
  3590. * This function is useful for tracers that want to take a "snapshot"
  3591. * of a CPU buffer and has another back up buffer lying around.
  3592. * it is expected that the tracer handles the cpu buffer not being
  3593. * used at the moment.
  3594. */
  3595. int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
  3596. struct ring_buffer *buffer_b, int cpu)
  3597. {
  3598. struct ring_buffer_per_cpu *cpu_buffer_a;
  3599. struct ring_buffer_per_cpu *cpu_buffer_b;
  3600. int ret = -EINVAL;
  3601. if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||
  3602. !cpumask_test_cpu(cpu, buffer_b->cpumask))
  3603. goto out;
  3604. cpu_buffer_a = buffer_a->buffers[cpu];
  3605. cpu_buffer_b = buffer_b->buffers[cpu];
  3606. /* At least make sure the two buffers are somewhat the same */
  3607. if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)
  3608. goto out;
  3609. ret = -EAGAIN;
  3610. if (ring_buffer_flags != RB_BUFFERS_ON)
  3611. goto out;
  3612. if (atomic_read(&buffer_a->record_disabled))
  3613. goto out;
  3614. if (atomic_read(&buffer_b->record_disabled))
  3615. goto out;
  3616. if (atomic_read(&cpu_buffer_a->record_disabled))
  3617. goto out;
  3618. if (atomic_read(&cpu_buffer_b->record_disabled))
  3619. goto out;
  3620. /*
  3621. * We can't do a synchronize_sched here because this
  3622. * function can be called in atomic context.
  3623. * Normally this will be called from the same CPU as cpu.
  3624. * If not it's up to the caller to protect this.
  3625. */
  3626. atomic_inc(&cpu_buffer_a->record_disabled);
  3627. atomic_inc(&cpu_buffer_b->record_disabled);
  3628. ret = -EBUSY;
  3629. if (local_read(&cpu_buffer_a->committing))
  3630. goto out_dec;
  3631. if (local_read(&cpu_buffer_b->committing))
  3632. goto out_dec;
  3633. buffer_a->buffers[cpu] = cpu_buffer_b;
  3634. buffer_b->buffers[cpu] = cpu_buffer_a;
  3635. cpu_buffer_b->buffer = buffer_a;
  3636. cpu_buffer_a->buffer = buffer_b;
  3637. ret = 0;
  3638. out_dec:
  3639. atomic_dec(&cpu_buffer_a->record_disabled);
  3640. atomic_dec(&cpu_buffer_b->record_disabled);
  3641. out:
  3642. return ret;
  3643. }
  3644. EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
  3645. #endif /* CONFIG_RING_BUFFER_ALLOW_SWAP */
  3646. /**
  3647. * ring_buffer_alloc_read_page - allocate a page to read from buffer
  3648. * @buffer: the buffer to allocate for.
  3649. *
  3650. * This function is used in conjunction with ring_buffer_read_page.
  3651. * When reading a full page from the ring buffer, these functions
  3652. * can be used to speed up the process. The calling function should
  3653. * allocate a few pages first with this function. Then when it
  3654. * needs to get pages from the ring buffer, it passes the result
  3655. * of this function into ring_buffer_read_page, which will swap
  3656. * the page that was allocated, with the read page of the buffer.
  3657. *
  3658. * Returns:
  3659. * The page allocated, or NULL on error.
  3660. */
  3661. void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu)
  3662. {
  3663. struct buffer_data_page *bpage;
  3664. struct page *page;
  3665. page = alloc_pages_node(cpu_to_node(cpu),
  3666. GFP_KERNEL | __GFP_NORETRY, 0);
  3667. if (!page)
  3668. return NULL;
  3669. bpage = page_address(page);
  3670. rb_init_page(bpage);
  3671. return bpage;
  3672. }
  3673. EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page);
  3674. /**
  3675. * ring_buffer_free_read_page - free an allocated read page
  3676. * @buffer: the buffer the page was allocate for
  3677. * @data: the page to free
  3678. *
  3679. * Free a page allocated from ring_buffer_alloc_read_page.
  3680. */
  3681. void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
  3682. {
  3683. free_page((unsigned long)data);
  3684. }
  3685. EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);
  3686. /**
  3687. * ring_buffer_read_page - extract a page from the ring buffer
  3688. * @buffer: buffer to extract from
  3689. * @data_page: the page to use allocated from ring_buffer_alloc_read_page
  3690. * @len: amount to extract
  3691. * @cpu: the cpu of the buffer to extract
  3692. * @full: should the extraction only happen when the page is full.
  3693. *
  3694. * This function will pull out a page from the ring buffer and consume it.
  3695. * @data_page must be the address of the variable that was returned
  3696. * from ring_buffer_alloc_read_page. This is because the page might be used
  3697. * to swap with a page in the ring buffer.
  3698. *
  3699. * for example:
  3700. * rpage = ring_buffer_alloc_read_page(buffer);
  3701. * if (!rpage)
  3702. * return error;
  3703. * ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);
  3704. * if (ret >= 0)
  3705. * process_page(rpage, ret);
  3706. *
  3707. * When @full is set, the function will not return true unless
  3708. * the writer is off the reader page.
  3709. *
  3710. * Note: it is up to the calling functions to handle sleeps and wakeups.
  3711. * The ring buffer can be used anywhere in the kernel and can not
  3712. * blindly call wake_up. The layer that uses the ring buffer must be
  3713. * responsible for that.
  3714. *
  3715. * Returns:
  3716. * >=0 if data has been transferred, returns the offset of consumed data.
  3717. * <0 if no data has been transferred.
  3718. */
  3719. int ring_buffer_read_page(struct ring_buffer *buffer,
  3720. void **data_page, size_t len, int cpu, int full)
  3721. {
  3722. struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
  3723. struct ring_buffer_event *event;
  3724. struct buffer_data_page *bpage;
  3725. struct buffer_page *reader;
  3726. unsigned long missed_events;
  3727. unsigned long flags;
  3728. unsigned int commit;
  3729. unsigned int read;
  3730. u64 save_timestamp;
  3731. int ret = -1;
  3732. if (!cpumask_test_cpu(cpu, buffer->cpumask))
  3733. goto out;
  3734. /*
  3735. * If len is not big enough to hold the page header, then
  3736. * we can not copy anything.
  3737. */
  3738. if (len <= BUF_PAGE_HDR_SIZE)
  3739. goto out;
  3740. len -= BUF_PAGE_HDR_SIZE;
  3741. if (!data_page)
  3742. goto out;
  3743. bpage = *data_page;
  3744. if (!bpage)
  3745. goto out;
  3746. raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
  3747. reader = rb_get_reader_page(cpu_buffer);
  3748. if (!reader)
  3749. goto out_unlock;
  3750. event = rb_reader_event(cpu_buffer);
  3751. read = reader->read;
  3752. commit = rb_page_commit(reader);
  3753. /* Check if any events were dropped */
  3754. missed_events = cpu_buffer->lost_events;
  3755. /*
  3756. * If this page has been partially read or
  3757. * if len is not big enough to read the rest of the page or
  3758. * a writer is still on the page, then
  3759. * we must copy the data from the page to the buffer.
  3760. * Otherwise, we can simply swap the page with the one passed in.
  3761. */
  3762. if (read || (len < (commit - read)) ||
  3763. cpu_buffer->reader_page == cpu_buffer->commit_page) {
  3764. struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
  3765. unsigned int rpos = read;
  3766. unsigned int pos = 0;
  3767. unsigned int size;
  3768. if (full)
  3769. goto out_unlock;
  3770. if (len > (commit - read))
  3771. len = (commit - read);
  3772. /* Always keep the time extend and data together */
  3773. size = rb_event_ts_length(event);
  3774. if (len < size)
  3775. goto out_unlock;
  3776. /* save the current timestamp, since the user will need it */
  3777. save_timestamp = cpu_buffer->read_stamp;
  3778. /* Need to copy one event at a time */
  3779. do {
  3780. /* We need the size of one event, because
  3781. * rb_advance_reader only advances by one event,
  3782. * whereas rb_event_ts_length may include the size of
  3783. * one or two events.
  3784. * We have already ensured there's enough space if this
  3785. * is a time extend. */
  3786. size = rb_event_length(event);
  3787. memcpy(bpage->data + pos, rpage->data + rpos, size);
  3788. len -= size;
  3789. rb_advance_reader(cpu_buffer);
  3790. rpos = reader->read;
  3791. pos += size;
  3792. if (rpos >= commit)
  3793. break;
  3794. event = rb_reader_event(cpu_buffer);
  3795. /* Always keep the time extend and data together */
  3796. size = rb_event_ts_length(event);
  3797. } while (len >= size);
  3798. /* update bpage */
  3799. local_set(&bpage->commit, pos);
  3800. bpage->time_stamp = save_timestamp;
  3801. /* we copied everything to the beginning */
  3802. read = 0;
  3803. } else {
  3804. /* update the entry counter */
  3805. cpu_buffer->read += rb_page_entries(reader);
  3806. cpu_buffer->read_bytes += BUF_PAGE_SIZE;
  3807. /* swap the pages */
  3808. rb_init_page(bpage);
  3809. bpage = reader->page;
  3810. reader->page = *data_page;
  3811. local_set(&reader->write, 0);
  3812. local_set(&reader->entries, 0);
  3813. reader->read = 0;
  3814. *data_page = bpage;
  3815. /*
  3816. * Use the real_end for the data size,
  3817. * This gives us a chance to store the lost events
  3818. * on the page.
  3819. */
  3820. if (reader->real_end)
  3821. local_set(&bpage->commit, reader->real_end);
  3822. }
  3823. ret = read;
  3824. cpu_buffer->lost_events = 0;
  3825. commit = local_read(&bpage->commit);
  3826. /*
  3827. * Set a flag in the commit field if we lost events
  3828. */
  3829. if (missed_events) {
  3830. /* If there is room at the end of the page to save the
  3831. * missed events, then record it there.
  3832. */
  3833. if (BUF_PAGE_SIZE - commit >= sizeof(missed_events)) {
  3834. memcpy(&bpage->data[commit], &missed_events,
  3835. sizeof(missed_events));
  3836. local_add(RB_MISSED_STORED, &bpage->commit);
  3837. commit += sizeof(missed_events);
  3838. }
  3839. local_add(RB_MISSED_EVENTS, &bpage->commit);
  3840. }
  3841. /*
  3842. * This page may be off to user land. Zero it out here.
  3843. */
  3844. if (commit < BUF_PAGE_SIZE)
  3845. memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit);
  3846. out_unlock:
  3847. raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
  3848. out:
  3849. return ret;
  3850. }
  3851. EXPORT_SYMBOL_GPL(ring_buffer_read_page);
  3852. #ifdef CONFIG_HOTPLUG_CPU
  3853. static int rb_cpu_notify(struct notifier_block *self,
  3854. unsigned long action, void *hcpu)
  3855. {
  3856. struct ring_buffer *buffer =
  3857. container_of(self, struct ring_buffer, cpu_notify);
  3858. long cpu = (long)hcpu;
  3859. int cpu_i, nr_pages_same;
  3860. unsigned int nr_pages;
  3861. switch (action) {
  3862. case CPU_UP_PREPARE:
  3863. case CPU_UP_PREPARE_FROZEN:
  3864. if (cpumask_test_cpu(cpu, buffer->cpumask))
  3865. return NOTIFY_OK;
  3866. nr_pages = 0;
  3867. nr_pages_same = 1;
  3868. /* check if all cpu sizes are same */
  3869. for_each_buffer_cpu(buffer, cpu_i) {
  3870. /* fill in the size from first enabled cpu */
  3871. if (nr_pages == 0)
  3872. nr_pages = buffer->buffers[cpu_i]->nr_pages;
  3873. if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {
  3874. nr_pages_same = 0;
  3875. break;
  3876. }
  3877. }
  3878. /* allocate minimum pages, user can later expand it */
  3879. if (!nr_pages_same)
  3880. nr_pages = 2;
  3881. buffer->buffers[cpu] =
  3882. rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
  3883. if (!buffer->buffers[cpu]) {
  3884. WARN(1, "failed to allocate ring buffer on CPU %ld\n",
  3885. cpu);
  3886. return NOTIFY_OK;
  3887. }
  3888. smp_wmb();
  3889. cpumask_set_cpu(cpu, buffer->cpumask);
  3890. break;
  3891. case CPU_DOWN_PREPARE:
  3892. case CPU_DOWN_PREPARE_FROZEN:
  3893. /*
  3894. * Do nothing.
  3895. * If we were to free the buffer, then the user would
  3896. * lose any trace that was in the buffer.
  3897. */
  3898. break;
  3899. default:
  3900. break;
  3901. }
  3902. return NOTIFY_OK;
  3903. }
  3904. #endif