core.c 183 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923
  1. /*
  2. * Performance events core code:
  3. *
  4. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  6. * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  7. * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  8. *
  9. * For licensing details see kernel-base/COPYING
  10. */
  11. #include <linux/fs.h>
  12. #include <linux/mm.h>
  13. #include <linux/cpu.h>
  14. #include <linux/smp.h>
  15. #include <linux/idr.h>
  16. #include <linux/file.h>
  17. #include <linux/poll.h>
  18. #include <linux/slab.h>
  19. #include <linux/hash.h>
  20. #include <linux/tick.h>
  21. #include <linux/sysfs.h>
  22. #include <linux/dcache.h>
  23. #include <linux/percpu.h>
  24. #include <linux/ptrace.h>
  25. #include <linux/reboot.h>
  26. #include <linux/vmstat.h>
  27. #include <linux/device.h>
  28. #include <linux/export.h>
  29. #include <linux/vmalloc.h>
  30. #include <linux/hardirq.h>
  31. #include <linux/rculist.h>
  32. #include <linux/uaccess.h>
  33. #include <linux/syscalls.h>
  34. #include <linux/anon_inodes.h>
  35. #include <linux/kernel_stat.h>
  36. #include <linux/perf_event.h>
  37. #include <linux/ftrace_event.h>
  38. #include <linux/hw_breakpoint.h>
  39. #include <linux/mm_types.h>
  40. #include <linux/cgroup.h>
  41. #include "internal.h"
  42. #include <asm/irq_regs.h>
  43. struct remote_function_call {
  44. struct task_struct *p;
  45. int (*func)(void *info);
  46. void *info;
  47. int ret;
  48. };
  49. static void remote_function(void *data)
  50. {
  51. struct remote_function_call *tfc = data;
  52. struct task_struct *p = tfc->p;
  53. if (p) {
  54. tfc->ret = -EAGAIN;
  55. if (task_cpu(p) != smp_processor_id() || !task_curr(p))
  56. return;
  57. }
  58. tfc->ret = tfc->func(tfc->info);
  59. }
  60. /**
  61. * task_function_call - call a function on the cpu on which a task runs
  62. * @p: the task to evaluate
  63. * @func: the function to be called
  64. * @info: the function call argument
  65. *
  66. * Calls the function @func when the task is currently running. This might
  67. * be on the current CPU, which just calls the function directly
  68. *
  69. * returns: @func return value, or
  70. * -ESRCH - when the process isn't running
  71. * -EAGAIN - when the process moved away
  72. */
  73. static int
  74. task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
  75. {
  76. struct remote_function_call data = {
  77. .p = p,
  78. .func = func,
  79. .info = info,
  80. .ret = -ESRCH, /* No such (running) process */
  81. };
  82. if (task_curr(p))
  83. smp_call_function_single(task_cpu(p), remote_function, &data, 1);
  84. return data.ret;
  85. }
  86. /**
  87. * cpu_function_call - call a function on the cpu
  88. * @func: the function to be called
  89. * @info: the function call argument
  90. *
  91. * Calls the function @func on the remote cpu.
  92. *
  93. * returns: @func return value or -ENXIO when the cpu is offline
  94. */
  95. static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
  96. {
  97. struct remote_function_call data = {
  98. .p = NULL,
  99. .func = func,
  100. .info = info,
  101. .ret = -ENXIO, /* No such CPU */
  102. };
  103. smp_call_function_single(cpu, remote_function, &data, 1);
  104. return data.ret;
  105. }
  106. #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
  107. PERF_FLAG_FD_OUTPUT |\
  108. PERF_FLAG_PID_CGROUP)
  109. /*
  110. * branch priv levels that need permission checks
  111. */
  112. #define PERF_SAMPLE_BRANCH_PERM_PLM \
  113. (PERF_SAMPLE_BRANCH_KERNEL |\
  114. PERF_SAMPLE_BRANCH_HV)
  115. enum event_type_t {
  116. EVENT_FLEXIBLE = 0x1,
  117. EVENT_PINNED = 0x2,
  118. EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
  119. };
  120. /*
  121. * perf_sched_events : >0 events exist
  122. * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
  123. */
  124. struct static_key_deferred perf_sched_events __read_mostly;
  125. static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
  126. static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
  127. static DEFINE_PER_CPU(atomic_t, perf_freq_events);
  128. static atomic_t nr_mmap_events __read_mostly;
  129. static atomic_t nr_comm_events __read_mostly;
  130. static atomic_t nr_task_events __read_mostly;
  131. static LIST_HEAD(pmus);
  132. static DEFINE_MUTEX(pmus_lock);
  133. static struct srcu_struct pmus_srcu;
  134. /*
  135. * perf event paranoia level:
  136. * -1 - not paranoid at all
  137. * 0 - disallow raw tracepoint access for unpriv
  138. * 1 - disallow cpu events for unpriv
  139. * 2 - disallow kernel profiling for unpriv
  140. */
  141. int sysctl_perf_event_paranoid __read_mostly = 1;
  142. /* Minimum for 512 kiB + 1 user control page */
  143. int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
  144. /*
  145. * max perf event sample rate
  146. */
  147. #define DEFAULT_MAX_SAMPLE_RATE 100000
  148. #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
  149. #define DEFAULT_CPU_TIME_MAX_PERCENT 25
  150. int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
  151. static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
  152. static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS;
  153. static atomic_t perf_sample_allowed_ns __read_mostly =
  154. ATOMIC_INIT( DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100);
  155. void update_perf_cpu_limits(void)
  156. {
  157. u64 tmp = perf_sample_period_ns;
  158. tmp *= sysctl_perf_cpu_time_max_percent;
  159. do_div(tmp, 100);
  160. atomic_set(&perf_sample_allowed_ns, tmp);
  161. }
  162. static int perf_rotate_context(struct perf_cpu_context *cpuctx);
  163. int perf_proc_update_handler(struct ctl_table *table, int write,
  164. void __user *buffer, size_t *lenp,
  165. loff_t *ppos)
  166. {
  167. int ret = proc_dointvec(table, write, buffer, lenp, ppos);
  168. if (ret || !write)
  169. return ret;
  170. max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
  171. perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
  172. update_perf_cpu_limits();
  173. return 0;
  174. }
  175. int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
  176. int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
  177. void __user *buffer, size_t *lenp,
  178. loff_t *ppos)
  179. {
  180. int ret = proc_dointvec(table, write, buffer, lenp, ppos);
  181. if (ret || !write)
  182. return ret;
  183. update_perf_cpu_limits();
  184. return 0;
  185. }
  186. /*
  187. * perf samples are done in some very critical code paths (NMIs).
  188. * If they take too much CPU time, the system can lock up and not
  189. * get any real work done. This will drop the sample rate when
  190. * we detect that events are taking too long.
  191. */
  192. #define NR_ACCUMULATED_SAMPLES 128
  193. DEFINE_PER_CPU(u64, running_sample_length);
  194. void perf_sample_event_took(u64 sample_len_ns)
  195. {
  196. u64 avg_local_sample_len;
  197. u64 local_samples_len;
  198. if (atomic_read(&perf_sample_allowed_ns) == 0)
  199. return;
  200. /* decay the counter by 1 average sample */
  201. local_samples_len = __get_cpu_var(running_sample_length);
  202. local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES;
  203. local_samples_len += sample_len_ns;
  204. __get_cpu_var(running_sample_length) = local_samples_len;
  205. /*
  206. * note: this will be biased artifically low until we have
  207. * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
  208. * from having to maintain a count.
  209. */
  210. avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
  211. if (avg_local_sample_len <= atomic_read(&perf_sample_allowed_ns))
  212. return;
  213. if (max_samples_per_tick <= 1)
  214. return;
  215. max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2);
  216. sysctl_perf_event_sample_rate = max_samples_per_tick * HZ;
  217. perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
  218. printk_ratelimited(KERN_WARNING
  219. "perf samples too long (%lld > %d), lowering "
  220. "kernel.perf_event_max_sample_rate to %d\n",
  221. avg_local_sample_len,
  222. atomic_read(&perf_sample_allowed_ns),
  223. sysctl_perf_event_sample_rate);
  224. update_perf_cpu_limits();
  225. }
  226. static atomic64_t perf_event_id;
  227. static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
  228. enum event_type_t event_type);
  229. static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
  230. enum event_type_t event_type,
  231. struct task_struct *task);
  232. static void update_context_time(struct perf_event_context *ctx);
  233. static u64 perf_event_time(struct perf_event *event);
  234. void __weak perf_event_print_debug(void) { }
  235. extern __weak const char *perf_pmu_name(void)
  236. {
  237. return "pmu";
  238. }
  239. static inline u64 perf_clock(void)
  240. {
  241. return local_clock();
  242. }
  243. static inline struct perf_cpu_context *
  244. __get_cpu_context(struct perf_event_context *ctx)
  245. {
  246. return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
  247. }
  248. static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
  249. struct perf_event_context *ctx)
  250. {
  251. raw_spin_lock(&cpuctx->ctx.lock);
  252. if (ctx)
  253. raw_spin_lock(&ctx->lock);
  254. }
  255. static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
  256. struct perf_event_context *ctx)
  257. {
  258. if (ctx)
  259. raw_spin_unlock(&ctx->lock);
  260. raw_spin_unlock(&cpuctx->ctx.lock);
  261. }
  262. #ifdef CONFIG_CGROUP_PERF
  263. /*
  264. * perf_cgroup_info keeps track of time_enabled for a cgroup.
  265. * This is a per-cpu dynamically allocated data structure.
  266. */
  267. struct perf_cgroup_info {
  268. u64 time;
  269. u64 timestamp;
  270. };
  271. struct perf_cgroup {
  272. struct cgroup_subsys_state css;
  273. struct perf_cgroup_info __percpu *info;
  274. };
  275. /*
  276. * Must ensure cgroup is pinned (css_get) before calling
  277. * this function. In other words, we cannot call this function
  278. * if there is no cgroup event for the current CPU context.
  279. */
  280. static inline struct perf_cgroup *
  281. perf_cgroup_from_task(struct task_struct *task)
  282. {
  283. return container_of(task_subsys_state(task, perf_subsys_id),
  284. struct perf_cgroup, css);
  285. }
  286. static inline bool
  287. perf_cgroup_match(struct perf_event *event)
  288. {
  289. struct perf_event_context *ctx = event->ctx;
  290. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  291. /* @event doesn't care about cgroup */
  292. if (!event->cgrp)
  293. return true;
  294. /* wants specific cgroup scope but @cpuctx isn't associated with any */
  295. if (!cpuctx->cgrp)
  296. return false;
  297. /*
  298. * Cgroup scoping is recursive. An event enabled for a cgroup is
  299. * also enabled for all its descendant cgroups. If @cpuctx's
  300. * cgroup is a descendant of @event's (the test covers identity
  301. * case), it's a match.
  302. */
  303. return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
  304. event->cgrp->css.cgroup);
  305. }
  306. static inline bool perf_tryget_cgroup(struct perf_event *event)
  307. {
  308. return css_tryget(&event->cgrp->css);
  309. }
  310. static inline void perf_put_cgroup(struct perf_event *event)
  311. {
  312. css_put(&event->cgrp->css);
  313. }
  314. static inline void perf_detach_cgroup(struct perf_event *event)
  315. {
  316. perf_put_cgroup(event);
  317. event->cgrp = NULL;
  318. }
  319. static inline int is_cgroup_event(struct perf_event *event)
  320. {
  321. return event->cgrp != NULL;
  322. }
  323. static inline u64 perf_cgroup_event_time(struct perf_event *event)
  324. {
  325. struct perf_cgroup_info *t;
  326. t = per_cpu_ptr(event->cgrp->info, event->cpu);
  327. return t->time;
  328. }
  329. static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
  330. {
  331. struct perf_cgroup_info *info;
  332. u64 now;
  333. now = perf_clock();
  334. info = this_cpu_ptr(cgrp->info);
  335. info->time += now - info->timestamp;
  336. info->timestamp = now;
  337. }
  338. static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
  339. {
  340. struct perf_cgroup *cgrp_out = cpuctx->cgrp;
  341. if (cgrp_out)
  342. __update_cgrp_time(cgrp_out);
  343. }
  344. static inline void update_cgrp_time_from_event(struct perf_event *event)
  345. {
  346. struct perf_cgroup *cgrp;
  347. /*
  348. * ensure we access cgroup data only when needed and
  349. * when we know the cgroup is pinned (css_get)
  350. */
  351. if (!is_cgroup_event(event))
  352. return;
  353. cgrp = perf_cgroup_from_task(current);
  354. /*
  355. * Do not update time when cgroup is not active
  356. */
  357. if (cgrp == event->cgrp)
  358. __update_cgrp_time(event->cgrp);
  359. }
  360. static inline void
  361. perf_cgroup_set_timestamp(struct task_struct *task,
  362. struct perf_event_context *ctx)
  363. {
  364. struct perf_cgroup *cgrp;
  365. struct perf_cgroup_info *info;
  366. /*
  367. * ctx->lock held by caller
  368. * ensure we do not access cgroup data
  369. * unless we have the cgroup pinned (css_get)
  370. */
  371. if (!task || !ctx->nr_cgroups)
  372. return;
  373. cgrp = perf_cgroup_from_task(task);
  374. info = this_cpu_ptr(cgrp->info);
  375. info->timestamp = ctx->timestamp;
  376. }
  377. #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */
  378. #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */
  379. /*
  380. * reschedule events based on the cgroup constraint of task.
  381. *
  382. * mode SWOUT : schedule out everything
  383. * mode SWIN : schedule in based on cgroup for next
  384. */
  385. void perf_cgroup_switch(struct task_struct *task, int mode)
  386. {
  387. struct perf_cpu_context *cpuctx;
  388. struct pmu *pmu;
  389. unsigned long flags;
  390. /*
  391. * disable interrupts to avoid geting nr_cgroup
  392. * changes via __perf_event_disable(). Also
  393. * avoids preemption.
  394. */
  395. local_irq_save(flags);
  396. /*
  397. * we reschedule only in the presence of cgroup
  398. * constrained events.
  399. */
  400. rcu_read_lock();
  401. list_for_each_entry_rcu(pmu, &pmus, entry) {
  402. cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  403. if (cpuctx->unique_pmu != pmu)
  404. continue; /* ensure we process each cpuctx once */
  405. /*
  406. * perf_cgroup_events says at least one
  407. * context on this CPU has cgroup events.
  408. *
  409. * ctx->nr_cgroups reports the number of cgroup
  410. * events for a context.
  411. */
  412. if (cpuctx->ctx.nr_cgroups > 0) {
  413. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  414. perf_pmu_disable(cpuctx->ctx.pmu);
  415. if (mode & PERF_CGROUP_SWOUT) {
  416. cpu_ctx_sched_out(cpuctx, EVENT_ALL);
  417. /*
  418. * must not be done before ctxswout due
  419. * to event_filter_match() in event_sched_out()
  420. */
  421. cpuctx->cgrp = NULL;
  422. }
  423. if (mode & PERF_CGROUP_SWIN) {
  424. WARN_ON_ONCE(cpuctx->cgrp);
  425. /*
  426. * set cgrp before ctxsw in to allow
  427. * event_filter_match() to not have to pass
  428. * task around
  429. */
  430. cpuctx->cgrp = perf_cgroup_from_task(task);
  431. cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
  432. }
  433. perf_pmu_enable(cpuctx->ctx.pmu);
  434. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  435. }
  436. }
  437. rcu_read_unlock();
  438. local_irq_restore(flags);
  439. }
  440. static inline void perf_cgroup_sched_out(struct task_struct *task,
  441. struct task_struct *next)
  442. {
  443. struct perf_cgroup *cgrp1;
  444. struct perf_cgroup *cgrp2 = NULL;
  445. /*
  446. * we come here when we know perf_cgroup_events > 0
  447. */
  448. cgrp1 = perf_cgroup_from_task(task);
  449. /*
  450. * next is NULL when called from perf_event_enable_on_exec()
  451. * that will systematically cause a cgroup_switch()
  452. */
  453. if (next)
  454. cgrp2 = perf_cgroup_from_task(next);
  455. /*
  456. * only schedule out current cgroup events if we know
  457. * that we are switching to a different cgroup. Otherwise,
  458. * do no touch the cgroup events.
  459. */
  460. if (cgrp1 != cgrp2)
  461. perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
  462. }
  463. static inline void perf_cgroup_sched_in(struct task_struct *prev,
  464. struct task_struct *task)
  465. {
  466. struct perf_cgroup *cgrp1;
  467. struct perf_cgroup *cgrp2 = NULL;
  468. /*
  469. * we come here when we know perf_cgroup_events > 0
  470. */
  471. cgrp1 = perf_cgroup_from_task(task);
  472. /* prev can never be NULL */
  473. cgrp2 = perf_cgroup_from_task(prev);
  474. /*
  475. * only need to schedule in cgroup events if we are changing
  476. * cgroup during ctxsw. Cgroup events were not scheduled
  477. * out of ctxsw out if that was not the case.
  478. */
  479. if (cgrp1 != cgrp2)
  480. perf_cgroup_switch(task, PERF_CGROUP_SWIN);
  481. }
  482. static inline int perf_cgroup_connect(int fd, struct perf_event *event,
  483. struct perf_event_attr *attr,
  484. struct perf_event *group_leader)
  485. {
  486. struct perf_cgroup *cgrp;
  487. struct cgroup_subsys_state *css;
  488. struct fd f = fdget(fd);
  489. int ret = 0;
  490. if (!f.file)
  491. return -EBADF;
  492. css = cgroup_css_from_dir(f.file, perf_subsys_id);
  493. if (IS_ERR(css)) {
  494. ret = PTR_ERR(css);
  495. goto out;
  496. }
  497. cgrp = container_of(css, struct perf_cgroup, css);
  498. event->cgrp = cgrp;
  499. /* must be done before we fput() the file */
  500. if (!perf_tryget_cgroup(event)) {
  501. event->cgrp = NULL;
  502. ret = -ENOENT;
  503. goto out;
  504. }
  505. /*
  506. * all events in a group must monitor
  507. * the same cgroup because a task belongs
  508. * to only one perf cgroup at a time
  509. */
  510. if (group_leader && group_leader->cgrp != cgrp) {
  511. perf_detach_cgroup(event);
  512. ret = -EINVAL;
  513. }
  514. out:
  515. fdput(f);
  516. return ret;
  517. }
  518. static inline void
  519. perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
  520. {
  521. struct perf_cgroup_info *t;
  522. t = per_cpu_ptr(event->cgrp->info, event->cpu);
  523. event->shadow_ctx_time = now - t->timestamp;
  524. }
  525. static inline void
  526. perf_cgroup_defer_enabled(struct perf_event *event)
  527. {
  528. /*
  529. * when the current task's perf cgroup does not match
  530. * the event's, we need to remember to call the
  531. * perf_mark_enable() function the first time a task with
  532. * a matching perf cgroup is scheduled in.
  533. */
  534. if (is_cgroup_event(event) && !perf_cgroup_match(event))
  535. event->cgrp_defer_enabled = 1;
  536. }
  537. static inline void
  538. perf_cgroup_mark_enabled(struct perf_event *event,
  539. struct perf_event_context *ctx)
  540. {
  541. struct perf_event *sub;
  542. u64 tstamp = perf_event_time(event);
  543. if (!event->cgrp_defer_enabled)
  544. return;
  545. event->cgrp_defer_enabled = 0;
  546. event->tstamp_enabled = tstamp - event->total_time_enabled;
  547. list_for_each_entry(sub, &event->sibling_list, group_entry) {
  548. if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
  549. sub->tstamp_enabled = tstamp - sub->total_time_enabled;
  550. sub->cgrp_defer_enabled = 0;
  551. }
  552. }
  553. }
  554. #else /* !CONFIG_CGROUP_PERF */
  555. static inline bool
  556. perf_cgroup_match(struct perf_event *event)
  557. {
  558. return true;
  559. }
  560. static inline void perf_detach_cgroup(struct perf_event *event)
  561. {}
  562. static inline int is_cgroup_event(struct perf_event *event)
  563. {
  564. return 0;
  565. }
  566. static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
  567. {
  568. return 0;
  569. }
  570. static inline void update_cgrp_time_from_event(struct perf_event *event)
  571. {
  572. }
  573. static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
  574. {
  575. }
  576. static inline void perf_cgroup_sched_out(struct task_struct *task,
  577. struct task_struct *next)
  578. {
  579. }
  580. static inline void perf_cgroup_sched_in(struct task_struct *prev,
  581. struct task_struct *task)
  582. {
  583. }
  584. static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
  585. struct perf_event_attr *attr,
  586. struct perf_event *group_leader)
  587. {
  588. return -EINVAL;
  589. }
  590. static inline void
  591. perf_cgroup_set_timestamp(struct task_struct *task,
  592. struct perf_event_context *ctx)
  593. {
  594. }
  595. void
  596. perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
  597. {
  598. }
  599. static inline void
  600. perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
  601. {
  602. }
  603. static inline u64 perf_cgroup_event_time(struct perf_event *event)
  604. {
  605. return 0;
  606. }
  607. static inline void
  608. perf_cgroup_defer_enabled(struct perf_event *event)
  609. {
  610. }
  611. static inline void
  612. perf_cgroup_mark_enabled(struct perf_event *event,
  613. struct perf_event_context *ctx)
  614. {
  615. }
  616. #endif
  617. /*
  618. * set default to be dependent on timer tick just
  619. * like original code
  620. */
  621. #define PERF_CPU_HRTIMER (1000 / HZ)
  622. /*
  623. * function must be called with interrupts disbled
  624. */
  625. static enum hrtimer_restart perf_cpu_hrtimer_handler(struct hrtimer *hr)
  626. {
  627. struct perf_cpu_context *cpuctx;
  628. enum hrtimer_restart ret = HRTIMER_NORESTART;
  629. int rotations = 0;
  630. WARN_ON(!irqs_disabled());
  631. cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
  632. rotations = perf_rotate_context(cpuctx);
  633. /*
  634. * arm timer if needed
  635. */
  636. if (rotations) {
  637. hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
  638. ret = HRTIMER_RESTART;
  639. }
  640. return ret;
  641. }
  642. /* CPU is going down */
  643. void perf_cpu_hrtimer_cancel(int cpu)
  644. {
  645. struct perf_cpu_context *cpuctx;
  646. struct pmu *pmu;
  647. unsigned long flags;
  648. if (WARN_ON(cpu != smp_processor_id()))
  649. return;
  650. local_irq_save(flags);
  651. rcu_read_lock();
  652. list_for_each_entry_rcu(pmu, &pmus, entry) {
  653. cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  654. if (pmu->task_ctx_nr == perf_sw_context)
  655. continue;
  656. hrtimer_cancel(&cpuctx->hrtimer);
  657. }
  658. rcu_read_unlock();
  659. local_irq_restore(flags);
  660. }
  661. static void __perf_cpu_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
  662. {
  663. struct hrtimer *hr = &cpuctx->hrtimer;
  664. struct pmu *pmu = cpuctx->ctx.pmu;
  665. int timer;
  666. /* no multiplexing needed for SW PMU */
  667. if (pmu->task_ctx_nr == perf_sw_context)
  668. return;
  669. /*
  670. * check default is sane, if not set then force to
  671. * default interval (1/tick)
  672. */
  673. timer = pmu->hrtimer_interval_ms;
  674. if (timer < 1)
  675. timer = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
  676. cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
  677. hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
  678. hr->function = perf_cpu_hrtimer_handler;
  679. }
  680. static void perf_cpu_hrtimer_restart(struct perf_cpu_context *cpuctx)
  681. {
  682. struct hrtimer *hr = &cpuctx->hrtimer;
  683. struct pmu *pmu = cpuctx->ctx.pmu;
  684. /* not for SW PMU */
  685. if (pmu->task_ctx_nr == perf_sw_context)
  686. return;
  687. if (hrtimer_active(hr))
  688. return;
  689. if (!hrtimer_callback_running(hr))
  690. __hrtimer_start_range_ns(hr, cpuctx->hrtimer_interval,
  691. 0, HRTIMER_MODE_REL_PINNED, 0);
  692. }
  693. void perf_pmu_disable(struct pmu *pmu)
  694. {
  695. int *count = this_cpu_ptr(pmu->pmu_disable_count);
  696. if (!(*count)++)
  697. pmu->pmu_disable(pmu);
  698. }
  699. void perf_pmu_enable(struct pmu *pmu)
  700. {
  701. int *count = this_cpu_ptr(pmu->pmu_disable_count);
  702. if (!--(*count))
  703. pmu->pmu_enable(pmu);
  704. }
  705. static DEFINE_PER_CPU(struct list_head, rotation_list);
  706. /*
  707. * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
  708. * because they're strictly cpu affine and rotate_start is called with IRQs
  709. * disabled, while rotate_context is called from IRQ context.
  710. */
  711. static void perf_pmu_rotate_start(struct pmu *pmu)
  712. {
  713. struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  714. struct list_head *head = &__get_cpu_var(rotation_list);
  715. WARN_ON(!irqs_disabled());
  716. if (list_empty(&cpuctx->rotation_list))
  717. list_add(&cpuctx->rotation_list, head);
  718. }
  719. static void get_ctx(struct perf_event_context *ctx)
  720. {
  721. WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
  722. }
  723. static void put_ctx(struct perf_event_context *ctx)
  724. {
  725. if (atomic_dec_and_test(&ctx->refcount)) {
  726. if (ctx->parent_ctx)
  727. put_ctx(ctx->parent_ctx);
  728. if (ctx->task)
  729. put_task_struct(ctx->task);
  730. kfree_rcu(ctx, rcu_head);
  731. }
  732. }
  733. static void unclone_ctx(struct perf_event_context *ctx)
  734. {
  735. if (ctx->parent_ctx) {
  736. put_ctx(ctx->parent_ctx);
  737. ctx->parent_ctx = NULL;
  738. }
  739. }
  740. static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
  741. {
  742. /*
  743. * only top level events have the pid namespace they were created in
  744. */
  745. if (event->parent)
  746. event = event->parent;
  747. return task_tgid_nr_ns(p, event->ns);
  748. }
  749. static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
  750. {
  751. /*
  752. * only top level events have the pid namespace they were created in
  753. */
  754. if (event->parent)
  755. event = event->parent;
  756. return task_pid_nr_ns(p, event->ns);
  757. }
  758. /*
  759. * If we inherit events we want to return the parent event id
  760. * to userspace.
  761. */
  762. static u64 primary_event_id(struct perf_event *event)
  763. {
  764. u64 id = event->id;
  765. if (event->parent)
  766. id = event->parent->id;
  767. return id;
  768. }
  769. /*
  770. * Get the perf_event_context for a task and lock it.
  771. * This has to cope with with the fact that until it is locked,
  772. * the context could get moved to another task.
  773. */
  774. static struct perf_event_context *
  775. perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
  776. {
  777. struct perf_event_context *ctx;
  778. retry:
  779. /*
  780. * One of the few rules of preemptible RCU is that one cannot do
  781. * rcu_read_unlock() while holding a scheduler (or nested) lock when
  782. * part of the read side critical section was preemptible -- see
  783. * rcu_read_unlock_special().
  784. *
  785. * Since ctx->lock nests under rq->lock we must ensure the entire read
  786. * side critical section is non-preemptible.
  787. */
  788. preempt_disable();
  789. rcu_read_lock();
  790. ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
  791. if (ctx) {
  792. /*
  793. * If this context is a clone of another, it might
  794. * get swapped for another underneath us by
  795. * perf_event_task_sched_out, though the
  796. * rcu_read_lock() protects us from any context
  797. * getting freed. Lock the context and check if it
  798. * got swapped before we could get the lock, and retry
  799. * if so. If we locked the right context, then it
  800. * can't get swapped on us any more.
  801. */
  802. raw_spin_lock_irqsave(&ctx->lock, *flags);
  803. if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
  804. raw_spin_unlock_irqrestore(&ctx->lock, *flags);
  805. rcu_read_unlock();
  806. preempt_enable();
  807. goto retry;
  808. }
  809. if (!atomic_inc_not_zero(&ctx->refcount)) {
  810. raw_spin_unlock_irqrestore(&ctx->lock, *flags);
  811. ctx = NULL;
  812. }
  813. }
  814. rcu_read_unlock();
  815. preempt_enable();
  816. return ctx;
  817. }
  818. /*
  819. * Get the context for a task and increment its pin_count so it
  820. * can't get swapped to another task. This also increments its
  821. * reference count so that the context can't get freed.
  822. */
  823. static struct perf_event_context *
  824. perf_pin_task_context(struct task_struct *task, int ctxn)
  825. {
  826. struct perf_event_context *ctx;
  827. unsigned long flags;
  828. ctx = perf_lock_task_context(task, ctxn, &flags);
  829. if (ctx) {
  830. ++ctx->pin_count;
  831. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  832. }
  833. return ctx;
  834. }
  835. static void perf_unpin_context(struct perf_event_context *ctx)
  836. {
  837. unsigned long flags;
  838. raw_spin_lock_irqsave(&ctx->lock, flags);
  839. --ctx->pin_count;
  840. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  841. }
  842. /*
  843. * Update the record of the current time in a context.
  844. */
  845. static void update_context_time(struct perf_event_context *ctx)
  846. {
  847. u64 now = perf_clock();
  848. ctx->time += now - ctx->timestamp;
  849. ctx->timestamp = now;
  850. }
  851. static u64 perf_event_time(struct perf_event *event)
  852. {
  853. struct perf_event_context *ctx = event->ctx;
  854. if (is_cgroup_event(event))
  855. return perf_cgroup_event_time(event);
  856. return ctx ? ctx->time : 0;
  857. }
  858. /*
  859. * Update the total_time_enabled and total_time_running fields for a event.
  860. * The caller of this function needs to hold the ctx->lock.
  861. */
  862. static void update_event_times(struct perf_event *event)
  863. {
  864. struct perf_event_context *ctx = event->ctx;
  865. u64 run_end;
  866. if (event->state < PERF_EVENT_STATE_INACTIVE ||
  867. event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
  868. return;
  869. /*
  870. * in cgroup mode, time_enabled represents
  871. * the time the event was enabled AND active
  872. * tasks were in the monitored cgroup. This is
  873. * independent of the activity of the context as
  874. * there may be a mix of cgroup and non-cgroup events.
  875. *
  876. * That is why we treat cgroup events differently
  877. * here.
  878. */
  879. if (is_cgroup_event(event))
  880. run_end = perf_cgroup_event_time(event);
  881. else if (ctx->is_active)
  882. run_end = ctx->time;
  883. else
  884. run_end = event->tstamp_stopped;
  885. event->total_time_enabled = run_end - event->tstamp_enabled;
  886. if (event->state == PERF_EVENT_STATE_INACTIVE)
  887. run_end = event->tstamp_stopped;
  888. else
  889. run_end = perf_event_time(event);
  890. event->total_time_running = run_end - event->tstamp_running;
  891. }
  892. /*
  893. * Update total_time_enabled and total_time_running for all events in a group.
  894. */
  895. static void update_group_times(struct perf_event *leader)
  896. {
  897. struct perf_event *event;
  898. update_event_times(leader);
  899. list_for_each_entry(event, &leader->sibling_list, group_entry)
  900. update_event_times(event);
  901. }
  902. static struct list_head *
  903. ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
  904. {
  905. if (event->attr.pinned)
  906. return &ctx->pinned_groups;
  907. else
  908. return &ctx->flexible_groups;
  909. }
  910. /*
  911. * Add a event from the lists for its context.
  912. * Must be called with ctx->mutex and ctx->lock held.
  913. */
  914. static void
  915. list_add_event(struct perf_event *event, struct perf_event_context *ctx)
  916. {
  917. WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
  918. event->attach_state |= PERF_ATTACH_CONTEXT;
  919. /*
  920. * If we're a stand alone event or group leader, we go to the context
  921. * list, group events are kept attached to the group so that
  922. * perf_group_detach can, at all times, locate all siblings.
  923. */
  924. if (event->group_leader == event) {
  925. struct list_head *list;
  926. if (is_software_event(event))
  927. event->group_flags |= PERF_GROUP_SOFTWARE;
  928. list = ctx_group_list(event, ctx);
  929. list_add_tail(&event->group_entry, list);
  930. }
  931. if (is_cgroup_event(event))
  932. ctx->nr_cgroups++;
  933. if (has_branch_stack(event))
  934. ctx->nr_branch_stack++;
  935. list_add_rcu(&event->event_entry, &ctx->event_list);
  936. if (!ctx->nr_events)
  937. perf_pmu_rotate_start(ctx->pmu);
  938. ctx->nr_events++;
  939. if (event->attr.inherit_stat)
  940. ctx->nr_stat++;
  941. }
  942. /*
  943. * Initialize event state based on the perf_event_attr::disabled.
  944. */
  945. static inline void perf_event__state_init(struct perf_event *event)
  946. {
  947. event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
  948. PERF_EVENT_STATE_INACTIVE;
  949. }
  950. /*
  951. * Called at perf_event creation and when events are attached/detached from a
  952. * group.
  953. */
  954. static void perf_event__read_size(struct perf_event *event)
  955. {
  956. int entry = sizeof(u64); /* value */
  957. int size = 0;
  958. int nr = 1;
  959. if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  960. size += sizeof(u64);
  961. if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  962. size += sizeof(u64);
  963. if (event->attr.read_format & PERF_FORMAT_ID)
  964. entry += sizeof(u64);
  965. if (event->attr.read_format & PERF_FORMAT_GROUP) {
  966. nr += event->group_leader->nr_siblings;
  967. size += sizeof(u64);
  968. }
  969. size += entry * nr;
  970. event->read_size = size;
  971. }
  972. static void perf_event__header_size(struct perf_event *event)
  973. {
  974. struct perf_sample_data *data;
  975. u64 sample_type = event->attr.sample_type;
  976. u16 size = 0;
  977. perf_event__read_size(event);
  978. if (sample_type & PERF_SAMPLE_IP)
  979. size += sizeof(data->ip);
  980. if (sample_type & PERF_SAMPLE_ADDR)
  981. size += sizeof(data->addr);
  982. if (sample_type & PERF_SAMPLE_PERIOD)
  983. size += sizeof(data->period);
  984. if (sample_type & PERF_SAMPLE_WEIGHT)
  985. size += sizeof(data->weight);
  986. if (sample_type & PERF_SAMPLE_READ)
  987. size += event->read_size;
  988. if (sample_type & PERF_SAMPLE_DATA_SRC)
  989. size += sizeof(data->data_src.val);
  990. event->header_size = size;
  991. }
  992. static void perf_event__id_header_size(struct perf_event *event)
  993. {
  994. struct perf_sample_data *data;
  995. u64 sample_type = event->attr.sample_type;
  996. u16 size = 0;
  997. if (sample_type & PERF_SAMPLE_TID)
  998. size += sizeof(data->tid_entry);
  999. if (sample_type & PERF_SAMPLE_TIME)
  1000. size += sizeof(data->time);
  1001. if (sample_type & PERF_SAMPLE_ID)
  1002. size += sizeof(data->id);
  1003. if (sample_type & PERF_SAMPLE_STREAM_ID)
  1004. size += sizeof(data->stream_id);
  1005. if (sample_type & PERF_SAMPLE_CPU)
  1006. size += sizeof(data->cpu_entry);
  1007. event->id_header_size = size;
  1008. }
  1009. static void perf_group_attach(struct perf_event *event)
  1010. {
  1011. struct perf_event *group_leader = event->group_leader, *pos;
  1012. /*
  1013. * We can have double attach due to group movement in perf_event_open.
  1014. */
  1015. if (event->attach_state & PERF_ATTACH_GROUP)
  1016. return;
  1017. event->attach_state |= PERF_ATTACH_GROUP;
  1018. if (group_leader == event)
  1019. return;
  1020. if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
  1021. !is_software_event(event))
  1022. group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
  1023. list_add_tail(&event->group_entry, &group_leader->sibling_list);
  1024. group_leader->nr_siblings++;
  1025. perf_event__header_size(group_leader);
  1026. list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
  1027. perf_event__header_size(pos);
  1028. }
  1029. /*
  1030. * Remove a event from the lists for its context.
  1031. * Must be called with ctx->mutex and ctx->lock held.
  1032. */
  1033. static void
  1034. list_del_event(struct perf_event *event, struct perf_event_context *ctx)
  1035. {
  1036. struct perf_cpu_context *cpuctx;
  1037. /*
  1038. * We can have double detach due to exit/hot-unplug + close.
  1039. */
  1040. if (!(event->attach_state & PERF_ATTACH_CONTEXT))
  1041. return;
  1042. event->attach_state &= ~PERF_ATTACH_CONTEXT;
  1043. if (is_cgroup_event(event)) {
  1044. ctx->nr_cgroups--;
  1045. cpuctx = __get_cpu_context(ctx);
  1046. /*
  1047. * if there are no more cgroup events
  1048. * then cler cgrp to avoid stale pointer
  1049. * in update_cgrp_time_from_cpuctx()
  1050. */
  1051. if (!ctx->nr_cgroups)
  1052. cpuctx->cgrp = NULL;
  1053. }
  1054. if (has_branch_stack(event))
  1055. ctx->nr_branch_stack--;
  1056. ctx->nr_events--;
  1057. if (event->attr.inherit_stat)
  1058. ctx->nr_stat--;
  1059. list_del_rcu(&event->event_entry);
  1060. if (event->group_leader == event)
  1061. list_del_init(&event->group_entry);
  1062. update_group_times(event);
  1063. /*
  1064. * If event was in error state, then keep it
  1065. * that way, otherwise bogus counts will be
  1066. * returned on read(). The only way to get out
  1067. * of error state is by explicit re-enabling
  1068. * of the event
  1069. */
  1070. if (event->state > PERF_EVENT_STATE_OFF)
  1071. event->state = PERF_EVENT_STATE_OFF;
  1072. }
  1073. static void perf_group_detach(struct perf_event *event)
  1074. {
  1075. struct perf_event *sibling, *tmp;
  1076. struct list_head *list = NULL;
  1077. /*
  1078. * We can have double detach due to exit/hot-unplug + close.
  1079. */
  1080. if (!(event->attach_state & PERF_ATTACH_GROUP))
  1081. return;
  1082. event->attach_state &= ~PERF_ATTACH_GROUP;
  1083. /*
  1084. * If this is a sibling, remove it from its group.
  1085. */
  1086. if (event->group_leader != event) {
  1087. list_del_init(&event->group_entry);
  1088. event->group_leader->nr_siblings--;
  1089. goto out;
  1090. }
  1091. if (!list_empty(&event->group_entry))
  1092. list = &event->group_entry;
  1093. /*
  1094. * If this was a group event with sibling events then
  1095. * upgrade the siblings to singleton events by adding them
  1096. * to whatever list we are on.
  1097. */
  1098. list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
  1099. if (list)
  1100. list_move_tail(&sibling->group_entry, list);
  1101. sibling->group_leader = sibling;
  1102. /* Inherit group flags from the previous leader */
  1103. sibling->group_flags = event->group_flags;
  1104. }
  1105. out:
  1106. perf_event__header_size(event->group_leader);
  1107. list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
  1108. perf_event__header_size(tmp);
  1109. }
  1110. static inline int
  1111. event_filter_match(struct perf_event *event)
  1112. {
  1113. return (event->cpu == -1 || event->cpu == smp_processor_id())
  1114. && perf_cgroup_match(event);
  1115. }
  1116. static void
  1117. event_sched_out(struct perf_event *event,
  1118. struct perf_cpu_context *cpuctx,
  1119. struct perf_event_context *ctx)
  1120. {
  1121. u64 tstamp = perf_event_time(event);
  1122. u64 delta;
  1123. /*
  1124. * An event which could not be activated because of
  1125. * filter mismatch still needs to have its timings
  1126. * maintained, otherwise bogus information is return
  1127. * via read() for time_enabled, time_running:
  1128. */
  1129. if (event->state == PERF_EVENT_STATE_INACTIVE
  1130. && !event_filter_match(event)) {
  1131. delta = tstamp - event->tstamp_stopped;
  1132. event->tstamp_running += delta;
  1133. event->tstamp_stopped = tstamp;
  1134. }
  1135. if (event->state != PERF_EVENT_STATE_ACTIVE)
  1136. return;
  1137. event->state = PERF_EVENT_STATE_INACTIVE;
  1138. if (event->pending_disable) {
  1139. event->pending_disable = 0;
  1140. event->state = PERF_EVENT_STATE_OFF;
  1141. }
  1142. event->tstamp_stopped = tstamp;
  1143. event->pmu->del(event, 0);
  1144. event->oncpu = -1;
  1145. if (!is_software_event(event))
  1146. cpuctx->active_oncpu--;
  1147. ctx->nr_active--;
  1148. if (event->attr.freq && event->attr.sample_freq)
  1149. ctx->nr_freq--;
  1150. if (event->attr.exclusive || !cpuctx->active_oncpu)
  1151. cpuctx->exclusive = 0;
  1152. }
  1153. static void
  1154. group_sched_out(struct perf_event *group_event,
  1155. struct perf_cpu_context *cpuctx,
  1156. struct perf_event_context *ctx)
  1157. {
  1158. struct perf_event *event;
  1159. int state = group_event->state;
  1160. event_sched_out(group_event, cpuctx, ctx);
  1161. /*
  1162. * Schedule out siblings (if any):
  1163. */
  1164. list_for_each_entry(event, &group_event->sibling_list, group_entry)
  1165. event_sched_out(event, cpuctx, ctx);
  1166. if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
  1167. cpuctx->exclusive = 0;
  1168. }
  1169. /*
  1170. * Cross CPU call to remove a performance event
  1171. *
  1172. * We disable the event on the hardware level first. After that we
  1173. * remove it from the context list.
  1174. */
  1175. static int __perf_remove_from_context(void *info)
  1176. {
  1177. struct perf_event *event = info;
  1178. struct perf_event_context *ctx = event->ctx;
  1179. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1180. raw_spin_lock(&ctx->lock);
  1181. event_sched_out(event, cpuctx, ctx);
  1182. list_del_event(event, ctx);
  1183. if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
  1184. ctx->is_active = 0;
  1185. cpuctx->task_ctx = NULL;
  1186. }
  1187. raw_spin_unlock(&ctx->lock);
  1188. return 0;
  1189. }
  1190. /*
  1191. * Remove the event from a task's (or a CPU's) list of events.
  1192. *
  1193. * CPU events are removed with a smp call. For task events we only
  1194. * call when the task is on a CPU.
  1195. *
  1196. * If event->ctx is a cloned context, callers must make sure that
  1197. * every task struct that event->ctx->task could possibly point to
  1198. * remains valid. This is OK when called from perf_release since
  1199. * that only calls us on the top-level context, which can't be a clone.
  1200. * When called from perf_event_exit_task, it's OK because the
  1201. * context has been detached from its task.
  1202. */
  1203. static void perf_remove_from_context(struct perf_event *event)
  1204. {
  1205. struct perf_event_context *ctx = event->ctx;
  1206. struct task_struct *task = ctx->task;
  1207. lockdep_assert_held(&ctx->mutex);
  1208. if (!task) {
  1209. /*
  1210. * Per cpu events are removed via an smp call and
  1211. * the removal is always successful.
  1212. */
  1213. cpu_function_call(event->cpu, __perf_remove_from_context, event);
  1214. return;
  1215. }
  1216. retry:
  1217. if (!task_function_call(task, __perf_remove_from_context, event))
  1218. return;
  1219. raw_spin_lock_irq(&ctx->lock);
  1220. /*
  1221. * If we failed to find a running task, but find the context active now
  1222. * that we've acquired the ctx->lock, retry.
  1223. */
  1224. if (ctx->is_active) {
  1225. raw_spin_unlock_irq(&ctx->lock);
  1226. goto retry;
  1227. }
  1228. /*
  1229. * Since the task isn't running, its safe to remove the event, us
  1230. * holding the ctx->lock ensures the task won't get scheduled in.
  1231. */
  1232. list_del_event(event, ctx);
  1233. raw_spin_unlock_irq(&ctx->lock);
  1234. }
  1235. /*
  1236. * Cross CPU call to disable a performance event
  1237. */
  1238. int __perf_event_disable(void *info)
  1239. {
  1240. struct perf_event *event = info;
  1241. struct perf_event_context *ctx = event->ctx;
  1242. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1243. /*
  1244. * If this is a per-task event, need to check whether this
  1245. * event's task is the current task on this cpu.
  1246. *
  1247. * Can trigger due to concurrent perf_event_context_sched_out()
  1248. * flipping contexts around.
  1249. */
  1250. if (ctx->task && cpuctx->task_ctx != ctx)
  1251. return -EINVAL;
  1252. raw_spin_lock(&ctx->lock);
  1253. /*
  1254. * If the event is on, turn it off.
  1255. * If it is in error state, leave it in error state.
  1256. */
  1257. if (event->state >= PERF_EVENT_STATE_INACTIVE) {
  1258. update_context_time(ctx);
  1259. update_cgrp_time_from_event(event);
  1260. update_group_times(event);
  1261. if (event == event->group_leader)
  1262. group_sched_out(event, cpuctx, ctx);
  1263. else
  1264. event_sched_out(event, cpuctx, ctx);
  1265. event->state = PERF_EVENT_STATE_OFF;
  1266. }
  1267. raw_spin_unlock(&ctx->lock);
  1268. return 0;
  1269. }
  1270. /*
  1271. * Disable a event.
  1272. *
  1273. * If event->ctx is a cloned context, callers must make sure that
  1274. * every task struct that event->ctx->task could possibly point to
  1275. * remains valid. This condition is satisifed when called through
  1276. * perf_event_for_each_child or perf_event_for_each because they
  1277. * hold the top-level event's child_mutex, so any descendant that
  1278. * goes to exit will block in sync_child_event.
  1279. * When called from perf_pending_event it's OK because event->ctx
  1280. * is the current context on this CPU and preemption is disabled,
  1281. * hence we can't get into perf_event_task_sched_out for this context.
  1282. */
  1283. void perf_event_disable(struct perf_event *event)
  1284. {
  1285. struct perf_event_context *ctx = event->ctx;
  1286. struct task_struct *task = ctx->task;
  1287. if (!task) {
  1288. /*
  1289. * Disable the event on the cpu that it's on
  1290. */
  1291. cpu_function_call(event->cpu, __perf_event_disable, event);
  1292. return;
  1293. }
  1294. retry:
  1295. if (!task_function_call(task, __perf_event_disable, event))
  1296. return;
  1297. raw_spin_lock_irq(&ctx->lock);
  1298. /*
  1299. * If the event is still active, we need to retry the cross-call.
  1300. */
  1301. if (event->state == PERF_EVENT_STATE_ACTIVE) {
  1302. raw_spin_unlock_irq(&ctx->lock);
  1303. /*
  1304. * Reload the task pointer, it might have been changed by
  1305. * a concurrent perf_event_context_sched_out().
  1306. */
  1307. task = ctx->task;
  1308. goto retry;
  1309. }
  1310. /*
  1311. * Since we have the lock this context can't be scheduled
  1312. * in, so we can change the state safely.
  1313. */
  1314. if (event->state == PERF_EVENT_STATE_INACTIVE) {
  1315. update_group_times(event);
  1316. event->state = PERF_EVENT_STATE_OFF;
  1317. }
  1318. raw_spin_unlock_irq(&ctx->lock);
  1319. }
  1320. EXPORT_SYMBOL_GPL(perf_event_disable);
  1321. static void perf_set_shadow_time(struct perf_event *event,
  1322. struct perf_event_context *ctx,
  1323. u64 tstamp)
  1324. {
  1325. /*
  1326. * use the correct time source for the time snapshot
  1327. *
  1328. * We could get by without this by leveraging the
  1329. * fact that to get to this function, the caller
  1330. * has most likely already called update_context_time()
  1331. * and update_cgrp_time_xx() and thus both timestamp
  1332. * are identical (or very close). Given that tstamp is,
  1333. * already adjusted for cgroup, we could say that:
  1334. * tstamp - ctx->timestamp
  1335. * is equivalent to
  1336. * tstamp - cgrp->timestamp.
  1337. *
  1338. * Then, in perf_output_read(), the calculation would
  1339. * work with no changes because:
  1340. * - event is guaranteed scheduled in
  1341. * - no scheduled out in between
  1342. * - thus the timestamp would be the same
  1343. *
  1344. * But this is a bit hairy.
  1345. *
  1346. * So instead, we have an explicit cgroup call to remain
  1347. * within the time time source all along. We believe it
  1348. * is cleaner and simpler to understand.
  1349. */
  1350. if (is_cgroup_event(event))
  1351. perf_cgroup_set_shadow_time(event, tstamp);
  1352. else
  1353. event->shadow_ctx_time = tstamp - ctx->timestamp;
  1354. }
  1355. #define MAX_INTERRUPTS (~0ULL)
  1356. static void perf_log_throttle(struct perf_event *event, int enable);
  1357. static int
  1358. event_sched_in(struct perf_event *event,
  1359. struct perf_cpu_context *cpuctx,
  1360. struct perf_event_context *ctx)
  1361. {
  1362. u64 tstamp = perf_event_time(event);
  1363. if (event->state <= PERF_EVENT_STATE_OFF)
  1364. return 0;
  1365. event->state = PERF_EVENT_STATE_ACTIVE;
  1366. event->oncpu = smp_processor_id();
  1367. /*
  1368. * Unthrottle events, since we scheduled we might have missed several
  1369. * ticks already, also for a heavily scheduling task there is little
  1370. * guarantee it'll get a tick in a timely manner.
  1371. */
  1372. if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
  1373. perf_log_throttle(event, 1);
  1374. event->hw.interrupts = 0;
  1375. }
  1376. /*
  1377. * The new state must be visible before we turn it on in the hardware:
  1378. */
  1379. smp_wmb();
  1380. if (event->pmu->add(event, PERF_EF_START)) {
  1381. event->state = PERF_EVENT_STATE_INACTIVE;
  1382. event->oncpu = -1;
  1383. return -EAGAIN;
  1384. }
  1385. event->tstamp_running += tstamp - event->tstamp_stopped;
  1386. perf_set_shadow_time(event, ctx, tstamp);
  1387. if (!is_software_event(event))
  1388. cpuctx->active_oncpu++;
  1389. ctx->nr_active++;
  1390. if (event->attr.freq && event->attr.sample_freq)
  1391. ctx->nr_freq++;
  1392. if (event->attr.exclusive)
  1393. cpuctx->exclusive = 1;
  1394. return 0;
  1395. }
  1396. static int
  1397. group_sched_in(struct perf_event *group_event,
  1398. struct perf_cpu_context *cpuctx,
  1399. struct perf_event_context *ctx)
  1400. {
  1401. struct perf_event *event, *partial_group = NULL;
  1402. struct pmu *pmu = group_event->pmu;
  1403. u64 now = ctx->time;
  1404. bool simulate = false;
  1405. if (group_event->state == PERF_EVENT_STATE_OFF)
  1406. return 0;
  1407. pmu->start_txn(pmu);
  1408. if (event_sched_in(group_event, cpuctx, ctx)) {
  1409. pmu->cancel_txn(pmu);
  1410. perf_cpu_hrtimer_restart(cpuctx);
  1411. return -EAGAIN;
  1412. }
  1413. /*
  1414. * Schedule in siblings as one group (if any):
  1415. */
  1416. list_for_each_entry(event, &group_event->sibling_list, group_entry) {
  1417. if (event_sched_in(event, cpuctx, ctx)) {
  1418. partial_group = event;
  1419. goto group_error;
  1420. }
  1421. }
  1422. if (!pmu->commit_txn(pmu))
  1423. return 0;
  1424. group_error:
  1425. /*
  1426. * Groups can be scheduled in as one unit only, so undo any
  1427. * partial group before returning:
  1428. * The events up to the failed event are scheduled out normally,
  1429. * tstamp_stopped will be updated.
  1430. *
  1431. * The failed events and the remaining siblings need to have
  1432. * their timings updated as if they had gone thru event_sched_in()
  1433. * and event_sched_out(). This is required to get consistent timings
  1434. * across the group. This also takes care of the case where the group
  1435. * could never be scheduled by ensuring tstamp_stopped is set to mark
  1436. * the time the event was actually stopped, such that time delta
  1437. * calculation in update_event_times() is correct.
  1438. */
  1439. list_for_each_entry(event, &group_event->sibling_list, group_entry) {
  1440. if (event == partial_group)
  1441. simulate = true;
  1442. if (simulate) {
  1443. event->tstamp_running += now - event->tstamp_stopped;
  1444. event->tstamp_stopped = now;
  1445. } else {
  1446. event_sched_out(event, cpuctx, ctx);
  1447. }
  1448. }
  1449. event_sched_out(group_event, cpuctx, ctx);
  1450. pmu->cancel_txn(pmu);
  1451. perf_cpu_hrtimer_restart(cpuctx);
  1452. return -EAGAIN;
  1453. }
  1454. /*
  1455. * Work out whether we can put this event group on the CPU now.
  1456. */
  1457. static int group_can_go_on(struct perf_event *event,
  1458. struct perf_cpu_context *cpuctx,
  1459. int can_add_hw)
  1460. {
  1461. /*
  1462. * Groups consisting entirely of software events can always go on.
  1463. */
  1464. if (event->group_flags & PERF_GROUP_SOFTWARE)
  1465. return 1;
  1466. /*
  1467. * If an exclusive group is already on, no other hardware
  1468. * events can go on.
  1469. */
  1470. if (cpuctx->exclusive)
  1471. return 0;
  1472. /*
  1473. * If this group is exclusive and there are already
  1474. * events on the CPU, it can't go on.
  1475. */
  1476. if (event->attr.exclusive && cpuctx->active_oncpu)
  1477. return 0;
  1478. /*
  1479. * Otherwise, try to add it if all previous groups were able
  1480. * to go on.
  1481. */
  1482. return can_add_hw;
  1483. }
  1484. static void add_event_to_ctx(struct perf_event *event,
  1485. struct perf_event_context *ctx)
  1486. {
  1487. u64 tstamp = perf_event_time(event);
  1488. list_add_event(event, ctx);
  1489. perf_group_attach(event);
  1490. event->tstamp_enabled = tstamp;
  1491. event->tstamp_running = tstamp;
  1492. event->tstamp_stopped = tstamp;
  1493. }
  1494. static void task_ctx_sched_out(struct perf_event_context *ctx);
  1495. static void
  1496. ctx_sched_in(struct perf_event_context *ctx,
  1497. struct perf_cpu_context *cpuctx,
  1498. enum event_type_t event_type,
  1499. struct task_struct *task);
  1500. static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
  1501. struct perf_event_context *ctx,
  1502. struct task_struct *task)
  1503. {
  1504. cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
  1505. if (ctx)
  1506. ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
  1507. cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
  1508. if (ctx)
  1509. ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
  1510. }
  1511. /*
  1512. * Cross CPU call to install and enable a performance event
  1513. *
  1514. * Must be called with ctx->mutex held
  1515. */
  1516. static int __perf_install_in_context(void *info)
  1517. {
  1518. struct perf_event *event = info;
  1519. struct perf_event_context *ctx = event->ctx;
  1520. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1521. struct perf_event_context *task_ctx = cpuctx->task_ctx;
  1522. struct task_struct *task = current;
  1523. perf_ctx_lock(cpuctx, task_ctx);
  1524. perf_pmu_disable(cpuctx->ctx.pmu);
  1525. /*
  1526. * If there was an active task_ctx schedule it out.
  1527. */
  1528. if (task_ctx)
  1529. task_ctx_sched_out(task_ctx);
  1530. /*
  1531. * If the context we're installing events in is not the
  1532. * active task_ctx, flip them.
  1533. */
  1534. if (ctx->task && task_ctx != ctx) {
  1535. if (task_ctx)
  1536. raw_spin_unlock(&task_ctx->lock);
  1537. raw_spin_lock(&ctx->lock);
  1538. task_ctx = ctx;
  1539. }
  1540. if (task_ctx) {
  1541. cpuctx->task_ctx = task_ctx;
  1542. task = task_ctx->task;
  1543. }
  1544. cpu_ctx_sched_out(cpuctx, EVENT_ALL);
  1545. update_context_time(ctx);
  1546. /*
  1547. * update cgrp time only if current cgrp
  1548. * matches event->cgrp. Must be done before
  1549. * calling add_event_to_ctx()
  1550. */
  1551. update_cgrp_time_from_event(event);
  1552. add_event_to_ctx(event, ctx);
  1553. /*
  1554. * Schedule everything back in
  1555. */
  1556. perf_event_sched_in(cpuctx, task_ctx, task);
  1557. perf_pmu_enable(cpuctx->ctx.pmu);
  1558. perf_ctx_unlock(cpuctx, task_ctx);
  1559. if (atomic_read(&__get_cpu_var(perf_freq_events)))
  1560. tick_nohz_full_kick();
  1561. return 0;
  1562. }
  1563. /*
  1564. * Attach a performance event to a context
  1565. *
  1566. * First we add the event to the list with the hardware enable bit
  1567. * in event->hw_config cleared.
  1568. *
  1569. * If the event is attached to a task which is on a CPU we use a smp
  1570. * call to enable it in the task context. The task might have been
  1571. * scheduled away, but we check this in the smp call again.
  1572. */
  1573. static void
  1574. perf_install_in_context(struct perf_event_context *ctx,
  1575. struct perf_event *event,
  1576. int cpu)
  1577. {
  1578. struct task_struct *task = ctx->task;
  1579. lockdep_assert_held(&ctx->mutex);
  1580. event->ctx = ctx;
  1581. if (event->cpu != -1)
  1582. event->cpu = cpu;
  1583. if (!task) {
  1584. /*
  1585. * Per cpu events are installed via an smp call and
  1586. * the install is always successful.
  1587. */
  1588. cpu_function_call(cpu, __perf_install_in_context, event);
  1589. return;
  1590. }
  1591. retry:
  1592. if (!task_function_call(task, __perf_install_in_context, event))
  1593. return;
  1594. raw_spin_lock_irq(&ctx->lock);
  1595. /*
  1596. * If we failed to find a running task, but find the context active now
  1597. * that we've acquired the ctx->lock, retry.
  1598. */
  1599. if (ctx->is_active) {
  1600. raw_spin_unlock_irq(&ctx->lock);
  1601. goto retry;
  1602. }
  1603. /*
  1604. * Since the task isn't running, its safe to add the event, us holding
  1605. * the ctx->lock ensures the task won't get scheduled in.
  1606. */
  1607. add_event_to_ctx(event, ctx);
  1608. raw_spin_unlock_irq(&ctx->lock);
  1609. }
  1610. /*
  1611. * Put a event into inactive state and update time fields.
  1612. * Enabling the leader of a group effectively enables all
  1613. * the group members that aren't explicitly disabled, so we
  1614. * have to update their ->tstamp_enabled also.
  1615. * Note: this works for group members as well as group leaders
  1616. * since the non-leader members' sibling_lists will be empty.
  1617. */
  1618. static void __perf_event_mark_enabled(struct perf_event *event)
  1619. {
  1620. struct perf_event *sub;
  1621. u64 tstamp = perf_event_time(event);
  1622. event->state = PERF_EVENT_STATE_INACTIVE;
  1623. event->tstamp_enabled = tstamp - event->total_time_enabled;
  1624. list_for_each_entry(sub, &event->sibling_list, group_entry) {
  1625. if (sub->state >= PERF_EVENT_STATE_INACTIVE)
  1626. sub->tstamp_enabled = tstamp - sub->total_time_enabled;
  1627. }
  1628. }
  1629. /*
  1630. * Cross CPU call to enable a performance event
  1631. */
  1632. static int __perf_event_enable(void *info)
  1633. {
  1634. struct perf_event *event = info;
  1635. struct perf_event_context *ctx = event->ctx;
  1636. struct perf_event *leader = event->group_leader;
  1637. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1638. int err;
  1639. /*
  1640. * There's a time window between 'ctx->is_active' check
  1641. * in perf_event_enable function and this place having:
  1642. * - IRQs on
  1643. * - ctx->lock unlocked
  1644. *
  1645. * where the task could be killed and 'ctx' deactivated
  1646. * by perf_event_exit_task.
  1647. */
  1648. if (!ctx->is_active)
  1649. return -EINVAL;
  1650. raw_spin_lock(&ctx->lock);
  1651. update_context_time(ctx);
  1652. if (event->state >= PERF_EVENT_STATE_INACTIVE)
  1653. goto unlock;
  1654. /*
  1655. * set current task's cgroup time reference point
  1656. */
  1657. perf_cgroup_set_timestamp(current, ctx);
  1658. __perf_event_mark_enabled(event);
  1659. if (!event_filter_match(event)) {
  1660. if (is_cgroup_event(event))
  1661. perf_cgroup_defer_enabled(event);
  1662. goto unlock;
  1663. }
  1664. /*
  1665. * If the event is in a group and isn't the group leader,
  1666. * then don't put it on unless the group is on.
  1667. */
  1668. if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
  1669. goto unlock;
  1670. if (!group_can_go_on(event, cpuctx, 1)) {
  1671. err = -EEXIST;
  1672. } else {
  1673. if (event == leader)
  1674. err = group_sched_in(event, cpuctx, ctx);
  1675. else
  1676. err = event_sched_in(event, cpuctx, ctx);
  1677. }
  1678. if (err) {
  1679. /*
  1680. * If this event can't go on and it's part of a
  1681. * group, then the whole group has to come off.
  1682. */
  1683. if (leader != event) {
  1684. group_sched_out(leader, cpuctx, ctx);
  1685. perf_cpu_hrtimer_restart(cpuctx);
  1686. }
  1687. if (leader->attr.pinned) {
  1688. update_group_times(leader);
  1689. leader->state = PERF_EVENT_STATE_ERROR;
  1690. }
  1691. }
  1692. unlock:
  1693. raw_spin_unlock(&ctx->lock);
  1694. return 0;
  1695. }
  1696. /*
  1697. * Enable a event.
  1698. *
  1699. * If event->ctx is a cloned context, callers must make sure that
  1700. * every task struct that event->ctx->task could possibly point to
  1701. * remains valid. This condition is satisfied when called through
  1702. * perf_event_for_each_child or perf_event_for_each as described
  1703. * for perf_event_disable.
  1704. */
  1705. void perf_event_enable(struct perf_event *event)
  1706. {
  1707. struct perf_event_context *ctx = event->ctx;
  1708. struct task_struct *task = ctx->task;
  1709. if (!task) {
  1710. /*
  1711. * Enable the event on the cpu that it's on
  1712. */
  1713. cpu_function_call(event->cpu, __perf_event_enable, event);
  1714. return;
  1715. }
  1716. raw_spin_lock_irq(&ctx->lock);
  1717. if (event->state >= PERF_EVENT_STATE_INACTIVE)
  1718. goto out;
  1719. /*
  1720. * If the event is in error state, clear that first.
  1721. * That way, if we see the event in error state below, we
  1722. * know that it has gone back into error state, as distinct
  1723. * from the task having been scheduled away before the
  1724. * cross-call arrived.
  1725. */
  1726. if (event->state == PERF_EVENT_STATE_ERROR)
  1727. event->state = PERF_EVENT_STATE_OFF;
  1728. retry:
  1729. if (!ctx->is_active) {
  1730. __perf_event_mark_enabled(event);
  1731. goto out;
  1732. }
  1733. raw_spin_unlock_irq(&ctx->lock);
  1734. if (!task_function_call(task, __perf_event_enable, event))
  1735. return;
  1736. raw_spin_lock_irq(&ctx->lock);
  1737. /*
  1738. * If the context is active and the event is still off,
  1739. * we need to retry the cross-call.
  1740. */
  1741. if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
  1742. /*
  1743. * task could have been flipped by a concurrent
  1744. * perf_event_context_sched_out()
  1745. */
  1746. task = ctx->task;
  1747. goto retry;
  1748. }
  1749. out:
  1750. raw_spin_unlock_irq(&ctx->lock);
  1751. }
  1752. EXPORT_SYMBOL_GPL(perf_event_enable);
  1753. int perf_event_refresh(struct perf_event *event, int refresh)
  1754. {
  1755. /*
  1756. * not supported on inherited events
  1757. */
  1758. if (event->attr.inherit || !is_sampling_event(event))
  1759. return -EINVAL;
  1760. atomic_add(refresh, &event->event_limit);
  1761. perf_event_enable(event);
  1762. return 0;
  1763. }
  1764. EXPORT_SYMBOL_GPL(perf_event_refresh);
  1765. static void ctx_sched_out(struct perf_event_context *ctx,
  1766. struct perf_cpu_context *cpuctx,
  1767. enum event_type_t event_type)
  1768. {
  1769. struct perf_event *event;
  1770. int is_active = ctx->is_active;
  1771. ctx->is_active &= ~event_type;
  1772. if (likely(!ctx->nr_events))
  1773. return;
  1774. update_context_time(ctx);
  1775. update_cgrp_time_from_cpuctx(cpuctx);
  1776. if (!ctx->nr_active)
  1777. return;
  1778. perf_pmu_disable(ctx->pmu);
  1779. if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
  1780. list_for_each_entry(event, &ctx->pinned_groups, group_entry)
  1781. group_sched_out(event, cpuctx, ctx);
  1782. }
  1783. if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
  1784. list_for_each_entry(event, &ctx->flexible_groups, group_entry)
  1785. group_sched_out(event, cpuctx, ctx);
  1786. }
  1787. perf_pmu_enable(ctx->pmu);
  1788. }
  1789. /*
  1790. * Test whether two contexts are equivalent, i.e. whether they
  1791. * have both been cloned from the same version of the same context
  1792. * and they both have the same number of enabled events.
  1793. * If the number of enabled events is the same, then the set
  1794. * of enabled events should be the same, because these are both
  1795. * inherited contexts, therefore we can't access individual events
  1796. * in them directly with an fd; we can only enable/disable all
  1797. * events via prctl, or enable/disable all events in a family
  1798. * via ioctl, which will have the same effect on both contexts.
  1799. */
  1800. static int context_equiv(struct perf_event_context *ctx1,
  1801. struct perf_event_context *ctx2)
  1802. {
  1803. return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
  1804. && ctx1->parent_gen == ctx2->parent_gen
  1805. && !ctx1->pin_count && !ctx2->pin_count;
  1806. }
  1807. static void __perf_event_sync_stat(struct perf_event *event,
  1808. struct perf_event *next_event)
  1809. {
  1810. u64 value;
  1811. if (!event->attr.inherit_stat)
  1812. return;
  1813. /*
  1814. * Update the event value, we cannot use perf_event_read()
  1815. * because we're in the middle of a context switch and have IRQs
  1816. * disabled, which upsets smp_call_function_single(), however
  1817. * we know the event must be on the current CPU, therefore we
  1818. * don't need to use it.
  1819. */
  1820. switch (event->state) {
  1821. case PERF_EVENT_STATE_ACTIVE:
  1822. event->pmu->read(event);
  1823. /* fall-through */
  1824. case PERF_EVENT_STATE_INACTIVE:
  1825. update_event_times(event);
  1826. break;
  1827. default:
  1828. break;
  1829. }
  1830. /*
  1831. * In order to keep per-task stats reliable we need to flip the event
  1832. * values when we flip the contexts.
  1833. */
  1834. value = local64_read(&next_event->count);
  1835. value = local64_xchg(&event->count, value);
  1836. local64_set(&next_event->count, value);
  1837. swap(event->total_time_enabled, next_event->total_time_enabled);
  1838. swap(event->total_time_running, next_event->total_time_running);
  1839. /*
  1840. * Since we swizzled the values, update the user visible data too.
  1841. */
  1842. perf_event_update_userpage(event);
  1843. perf_event_update_userpage(next_event);
  1844. }
  1845. #define list_next_entry(pos, member) \
  1846. list_entry(pos->member.next, typeof(*pos), member)
  1847. static void perf_event_sync_stat(struct perf_event_context *ctx,
  1848. struct perf_event_context *next_ctx)
  1849. {
  1850. struct perf_event *event, *next_event;
  1851. if (!ctx->nr_stat)
  1852. return;
  1853. update_context_time(ctx);
  1854. event = list_first_entry(&ctx->event_list,
  1855. struct perf_event, event_entry);
  1856. next_event = list_first_entry(&next_ctx->event_list,
  1857. struct perf_event, event_entry);
  1858. while (&event->event_entry != &ctx->event_list &&
  1859. &next_event->event_entry != &next_ctx->event_list) {
  1860. __perf_event_sync_stat(event, next_event);
  1861. event = list_next_entry(event, event_entry);
  1862. next_event = list_next_entry(next_event, event_entry);
  1863. }
  1864. }
  1865. static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
  1866. struct task_struct *next)
  1867. {
  1868. struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
  1869. struct perf_event_context *next_ctx;
  1870. struct perf_event_context *parent;
  1871. struct perf_cpu_context *cpuctx;
  1872. int do_switch = 1;
  1873. if (likely(!ctx))
  1874. return;
  1875. cpuctx = __get_cpu_context(ctx);
  1876. if (!cpuctx->task_ctx)
  1877. return;
  1878. rcu_read_lock();
  1879. parent = rcu_dereference(ctx->parent_ctx);
  1880. next_ctx = next->perf_event_ctxp[ctxn];
  1881. if (parent && next_ctx &&
  1882. rcu_dereference(next_ctx->parent_ctx) == parent) {
  1883. /*
  1884. * Looks like the two contexts are clones, so we might be
  1885. * able to optimize the context switch. We lock both
  1886. * contexts and check that they are clones under the
  1887. * lock (including re-checking that neither has been
  1888. * uncloned in the meantime). It doesn't matter which
  1889. * order we take the locks because no other cpu could
  1890. * be trying to lock both of these tasks.
  1891. */
  1892. raw_spin_lock(&ctx->lock);
  1893. raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
  1894. if (context_equiv(ctx, next_ctx)) {
  1895. /*
  1896. * XXX do we need a memory barrier of sorts
  1897. * wrt to rcu_dereference() of perf_event_ctxp
  1898. */
  1899. task->perf_event_ctxp[ctxn] = next_ctx;
  1900. next->perf_event_ctxp[ctxn] = ctx;
  1901. ctx->task = next;
  1902. next_ctx->task = task;
  1903. do_switch = 0;
  1904. perf_event_sync_stat(ctx, next_ctx);
  1905. }
  1906. raw_spin_unlock(&next_ctx->lock);
  1907. raw_spin_unlock(&ctx->lock);
  1908. }
  1909. rcu_read_unlock();
  1910. if (do_switch) {
  1911. raw_spin_lock(&ctx->lock);
  1912. ctx_sched_out(ctx, cpuctx, EVENT_ALL);
  1913. cpuctx->task_ctx = NULL;
  1914. raw_spin_unlock(&ctx->lock);
  1915. }
  1916. }
  1917. #define for_each_task_context_nr(ctxn) \
  1918. for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
  1919. /*
  1920. * Called from scheduler to remove the events of the current task,
  1921. * with interrupts disabled.
  1922. *
  1923. * We stop each event and update the event value in event->count.
  1924. *
  1925. * This does not protect us against NMI, but disable()
  1926. * sets the disabled bit in the control field of event _before_
  1927. * accessing the event control register. If a NMI hits, then it will
  1928. * not restart the event.
  1929. */
  1930. void __perf_event_task_sched_out(struct task_struct *task,
  1931. struct task_struct *next)
  1932. {
  1933. int ctxn;
  1934. for_each_task_context_nr(ctxn)
  1935. perf_event_context_sched_out(task, ctxn, next);
  1936. /*
  1937. * if cgroup events exist on this CPU, then we need
  1938. * to check if we have to switch out PMU state.
  1939. * cgroup event are system-wide mode only
  1940. */
  1941. if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
  1942. perf_cgroup_sched_out(task, next);
  1943. }
  1944. static void task_ctx_sched_out(struct perf_event_context *ctx)
  1945. {
  1946. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1947. if (!cpuctx->task_ctx)
  1948. return;
  1949. if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
  1950. return;
  1951. ctx_sched_out(ctx, cpuctx, EVENT_ALL);
  1952. cpuctx->task_ctx = NULL;
  1953. }
  1954. /*
  1955. * Called with IRQs disabled
  1956. */
  1957. static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
  1958. enum event_type_t event_type)
  1959. {
  1960. ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
  1961. }
  1962. static void
  1963. ctx_pinned_sched_in(struct perf_event_context *ctx,
  1964. struct perf_cpu_context *cpuctx)
  1965. {
  1966. struct perf_event *event;
  1967. list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
  1968. if (event->state <= PERF_EVENT_STATE_OFF)
  1969. continue;
  1970. if (!event_filter_match(event))
  1971. continue;
  1972. /* may need to reset tstamp_enabled */
  1973. if (is_cgroup_event(event))
  1974. perf_cgroup_mark_enabled(event, ctx);
  1975. if (group_can_go_on(event, cpuctx, 1))
  1976. group_sched_in(event, cpuctx, ctx);
  1977. /*
  1978. * If this pinned group hasn't been scheduled,
  1979. * put it in error state.
  1980. */
  1981. if (event->state == PERF_EVENT_STATE_INACTIVE) {
  1982. update_group_times(event);
  1983. event->state = PERF_EVENT_STATE_ERROR;
  1984. }
  1985. }
  1986. }
  1987. static void
  1988. ctx_flexible_sched_in(struct perf_event_context *ctx,
  1989. struct perf_cpu_context *cpuctx)
  1990. {
  1991. struct perf_event *event;
  1992. int can_add_hw = 1;
  1993. list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
  1994. /* Ignore events in OFF or ERROR state */
  1995. if (event->state <= PERF_EVENT_STATE_OFF)
  1996. continue;
  1997. /*
  1998. * Listen to the 'cpu' scheduling filter constraint
  1999. * of events:
  2000. */
  2001. if (!event_filter_match(event))
  2002. continue;
  2003. /* may need to reset tstamp_enabled */
  2004. if (is_cgroup_event(event))
  2005. perf_cgroup_mark_enabled(event, ctx);
  2006. if (group_can_go_on(event, cpuctx, can_add_hw)) {
  2007. if (group_sched_in(event, cpuctx, ctx))
  2008. can_add_hw = 0;
  2009. }
  2010. }
  2011. }
  2012. static void
  2013. ctx_sched_in(struct perf_event_context *ctx,
  2014. struct perf_cpu_context *cpuctx,
  2015. enum event_type_t event_type,
  2016. struct task_struct *task)
  2017. {
  2018. u64 now;
  2019. int is_active = ctx->is_active;
  2020. ctx->is_active |= event_type;
  2021. if (likely(!ctx->nr_events))
  2022. return;
  2023. now = perf_clock();
  2024. ctx->timestamp = now;
  2025. perf_cgroup_set_timestamp(task, ctx);
  2026. /*
  2027. * First go through the list and put on any pinned groups
  2028. * in order to give them the best chance of going on.
  2029. */
  2030. if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
  2031. ctx_pinned_sched_in(ctx, cpuctx);
  2032. /* Then walk through the lower prio flexible groups */
  2033. if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
  2034. ctx_flexible_sched_in(ctx, cpuctx);
  2035. }
  2036. static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
  2037. enum event_type_t event_type,
  2038. struct task_struct *task)
  2039. {
  2040. struct perf_event_context *ctx = &cpuctx->ctx;
  2041. ctx_sched_in(ctx, cpuctx, event_type, task);
  2042. }
  2043. static void perf_event_context_sched_in(struct perf_event_context *ctx,
  2044. struct task_struct *task)
  2045. {
  2046. struct perf_cpu_context *cpuctx;
  2047. cpuctx = __get_cpu_context(ctx);
  2048. if (cpuctx->task_ctx == ctx)
  2049. return;
  2050. perf_ctx_lock(cpuctx, ctx);
  2051. perf_pmu_disable(ctx->pmu);
  2052. /*
  2053. * We want to keep the following priority order:
  2054. * cpu pinned (that don't need to move), task pinned,
  2055. * cpu flexible, task flexible.
  2056. */
  2057. cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
  2058. if (ctx->nr_events)
  2059. cpuctx->task_ctx = ctx;
  2060. perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
  2061. perf_pmu_enable(ctx->pmu);
  2062. perf_ctx_unlock(cpuctx, ctx);
  2063. /*
  2064. * Since these rotations are per-cpu, we need to ensure the
  2065. * cpu-context we got scheduled on is actually rotating.
  2066. */
  2067. perf_pmu_rotate_start(ctx->pmu);
  2068. }
  2069. /*
  2070. * When sampling the branck stack in system-wide, it may be necessary
  2071. * to flush the stack on context switch. This happens when the branch
  2072. * stack does not tag its entries with the pid of the current task.
  2073. * Otherwise it becomes impossible to associate a branch entry with a
  2074. * task. This ambiguity is more likely to appear when the branch stack
  2075. * supports priv level filtering and the user sets it to monitor only
  2076. * at the user level (which could be a useful measurement in system-wide
  2077. * mode). In that case, the risk is high of having a branch stack with
  2078. * branch from multiple tasks. Flushing may mean dropping the existing
  2079. * entries or stashing them somewhere in the PMU specific code layer.
  2080. *
  2081. * This function provides the context switch callback to the lower code
  2082. * layer. It is invoked ONLY when there is at least one system-wide context
  2083. * with at least one active event using taken branch sampling.
  2084. */
  2085. static void perf_branch_stack_sched_in(struct task_struct *prev,
  2086. struct task_struct *task)
  2087. {
  2088. struct perf_cpu_context *cpuctx;
  2089. struct pmu *pmu;
  2090. unsigned long flags;
  2091. /* no need to flush branch stack if not changing task */
  2092. if (prev == task)
  2093. return;
  2094. local_irq_save(flags);
  2095. rcu_read_lock();
  2096. list_for_each_entry_rcu(pmu, &pmus, entry) {
  2097. cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  2098. /*
  2099. * check if the context has at least one
  2100. * event using PERF_SAMPLE_BRANCH_STACK
  2101. */
  2102. if (cpuctx->ctx.nr_branch_stack > 0
  2103. && pmu->flush_branch_stack) {
  2104. pmu = cpuctx->ctx.pmu;
  2105. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  2106. perf_pmu_disable(pmu);
  2107. pmu->flush_branch_stack();
  2108. perf_pmu_enable(pmu);
  2109. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  2110. }
  2111. }
  2112. rcu_read_unlock();
  2113. local_irq_restore(flags);
  2114. }
  2115. /*
  2116. * Called from scheduler to add the events of the current task
  2117. * with interrupts disabled.
  2118. *
  2119. * We restore the event value and then enable it.
  2120. *
  2121. * This does not protect us against NMI, but enable()
  2122. * sets the enabled bit in the control field of event _before_
  2123. * accessing the event control register. If a NMI hits, then it will
  2124. * keep the event running.
  2125. */
  2126. void __perf_event_task_sched_in(struct task_struct *prev,
  2127. struct task_struct *task)
  2128. {
  2129. struct perf_event_context *ctx;
  2130. int ctxn;
  2131. for_each_task_context_nr(ctxn) {
  2132. ctx = task->perf_event_ctxp[ctxn];
  2133. if (likely(!ctx))
  2134. continue;
  2135. perf_event_context_sched_in(ctx, task);
  2136. }
  2137. /*
  2138. * if cgroup events exist on this CPU, then we need
  2139. * to check if we have to switch in PMU state.
  2140. * cgroup event are system-wide mode only
  2141. */
  2142. if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
  2143. perf_cgroup_sched_in(prev, task);
  2144. /* check for system-wide branch_stack events */
  2145. if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
  2146. perf_branch_stack_sched_in(prev, task);
  2147. }
  2148. static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
  2149. {
  2150. u64 frequency = event->attr.sample_freq;
  2151. u64 sec = NSEC_PER_SEC;
  2152. u64 divisor, dividend;
  2153. int count_fls, nsec_fls, frequency_fls, sec_fls;
  2154. count_fls = fls64(count);
  2155. nsec_fls = fls64(nsec);
  2156. frequency_fls = fls64(frequency);
  2157. sec_fls = 30;
  2158. /*
  2159. * We got @count in @nsec, with a target of sample_freq HZ
  2160. * the target period becomes:
  2161. *
  2162. * @count * 10^9
  2163. * period = -------------------
  2164. * @nsec * sample_freq
  2165. *
  2166. */
  2167. /*
  2168. * Reduce accuracy by one bit such that @a and @b converge
  2169. * to a similar magnitude.
  2170. */
  2171. #define REDUCE_FLS(a, b) \
  2172. do { \
  2173. if (a##_fls > b##_fls) { \
  2174. a >>= 1; \
  2175. a##_fls--; \
  2176. } else { \
  2177. b >>= 1; \
  2178. b##_fls--; \
  2179. } \
  2180. } while (0)
  2181. /*
  2182. * Reduce accuracy until either term fits in a u64, then proceed with
  2183. * the other, so that finally we can do a u64/u64 division.
  2184. */
  2185. while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
  2186. REDUCE_FLS(nsec, frequency);
  2187. REDUCE_FLS(sec, count);
  2188. }
  2189. if (count_fls + sec_fls > 64) {
  2190. divisor = nsec * frequency;
  2191. while (count_fls + sec_fls > 64) {
  2192. REDUCE_FLS(count, sec);
  2193. divisor >>= 1;
  2194. }
  2195. dividend = count * sec;
  2196. } else {
  2197. dividend = count * sec;
  2198. while (nsec_fls + frequency_fls > 64) {
  2199. REDUCE_FLS(nsec, frequency);
  2200. dividend >>= 1;
  2201. }
  2202. divisor = nsec * frequency;
  2203. }
  2204. if (!divisor)
  2205. return dividend;
  2206. return div64_u64(dividend, divisor);
  2207. }
  2208. static DEFINE_PER_CPU(int, perf_throttled_count);
  2209. static DEFINE_PER_CPU(u64, perf_throttled_seq);
  2210. static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
  2211. {
  2212. struct hw_perf_event *hwc = &event->hw;
  2213. s64 period, sample_period;
  2214. s64 delta;
  2215. period = perf_calculate_period(event, nsec, count);
  2216. delta = (s64)(period - hwc->sample_period);
  2217. delta = (delta + 7) / 8; /* low pass filter */
  2218. sample_period = hwc->sample_period + delta;
  2219. if (!sample_period)
  2220. sample_period = 1;
  2221. hwc->sample_period = sample_period;
  2222. if (local64_read(&hwc->period_left) > 8*sample_period) {
  2223. if (disable)
  2224. event->pmu->stop(event, PERF_EF_UPDATE);
  2225. local64_set(&hwc->period_left, 0);
  2226. if (disable)
  2227. event->pmu->start(event, PERF_EF_RELOAD);
  2228. }
  2229. }
  2230. /*
  2231. * combine freq adjustment with unthrottling to avoid two passes over the
  2232. * events. At the same time, make sure, having freq events does not change
  2233. * the rate of unthrottling as that would introduce bias.
  2234. */
  2235. static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
  2236. int needs_unthr)
  2237. {
  2238. struct perf_event *event;
  2239. struct hw_perf_event *hwc;
  2240. u64 now, period = TICK_NSEC;
  2241. s64 delta;
  2242. /*
  2243. * only need to iterate over all events iff:
  2244. * - context have events in frequency mode (needs freq adjust)
  2245. * - there are events to unthrottle on this cpu
  2246. */
  2247. if (!(ctx->nr_freq || needs_unthr))
  2248. return;
  2249. raw_spin_lock(&ctx->lock);
  2250. perf_pmu_disable(ctx->pmu);
  2251. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  2252. if (event->state != PERF_EVENT_STATE_ACTIVE)
  2253. continue;
  2254. if (!event_filter_match(event))
  2255. continue;
  2256. hwc = &event->hw;
  2257. if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
  2258. hwc->interrupts = 0;
  2259. perf_log_throttle(event, 1);
  2260. event->pmu->start(event, 0);
  2261. }
  2262. if (!event->attr.freq || !event->attr.sample_freq)
  2263. continue;
  2264. /*
  2265. * stop the event and update event->count
  2266. */
  2267. event->pmu->stop(event, PERF_EF_UPDATE);
  2268. now = local64_read(&event->count);
  2269. delta = now - hwc->freq_count_stamp;
  2270. hwc->freq_count_stamp = now;
  2271. /*
  2272. * restart the event
  2273. * reload only if value has changed
  2274. * we have stopped the event so tell that
  2275. * to perf_adjust_period() to avoid stopping it
  2276. * twice.
  2277. */
  2278. if (delta > 0)
  2279. perf_adjust_period(event, period, delta, false);
  2280. event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
  2281. }
  2282. perf_pmu_enable(ctx->pmu);
  2283. raw_spin_unlock(&ctx->lock);
  2284. }
  2285. /*
  2286. * Round-robin a context's events:
  2287. */
  2288. static void rotate_ctx(struct perf_event_context *ctx)
  2289. {
  2290. /*
  2291. * Rotate the first entry last of non-pinned groups. Rotation might be
  2292. * disabled by the inheritance code.
  2293. */
  2294. if (!ctx->rotate_disable)
  2295. list_rotate_left(&ctx->flexible_groups);
  2296. }
  2297. /*
  2298. * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
  2299. * because they're strictly cpu affine and rotate_start is called with IRQs
  2300. * disabled, while rotate_context is called from IRQ context.
  2301. */
  2302. static int perf_rotate_context(struct perf_cpu_context *cpuctx)
  2303. {
  2304. struct perf_event_context *ctx = NULL;
  2305. int rotate = 0, remove = 1;
  2306. if (cpuctx->ctx.nr_events) {
  2307. remove = 0;
  2308. if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
  2309. rotate = 1;
  2310. }
  2311. ctx = cpuctx->task_ctx;
  2312. if (ctx && ctx->nr_events) {
  2313. remove = 0;
  2314. if (ctx->nr_events != ctx->nr_active)
  2315. rotate = 1;
  2316. }
  2317. if (!rotate)
  2318. goto done;
  2319. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  2320. perf_pmu_disable(cpuctx->ctx.pmu);
  2321. cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
  2322. if (ctx)
  2323. ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
  2324. rotate_ctx(&cpuctx->ctx);
  2325. if (ctx)
  2326. rotate_ctx(ctx);
  2327. perf_event_sched_in(cpuctx, ctx, current);
  2328. perf_pmu_enable(cpuctx->ctx.pmu);
  2329. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  2330. done:
  2331. if (remove)
  2332. list_del_init(&cpuctx->rotation_list);
  2333. return rotate;
  2334. }
  2335. #ifdef CONFIG_NO_HZ_FULL
  2336. bool perf_event_can_stop_tick(void)
  2337. {
  2338. if (atomic_read(&__get_cpu_var(perf_freq_events)) ||
  2339. __this_cpu_read(perf_throttled_count))
  2340. return false;
  2341. else
  2342. return true;
  2343. }
  2344. #endif
  2345. void perf_event_task_tick(void)
  2346. {
  2347. struct list_head *head = &__get_cpu_var(rotation_list);
  2348. struct perf_cpu_context *cpuctx, *tmp;
  2349. struct perf_event_context *ctx;
  2350. int throttled;
  2351. WARN_ON(!irqs_disabled());
  2352. __this_cpu_inc(perf_throttled_seq);
  2353. throttled = __this_cpu_xchg(perf_throttled_count, 0);
  2354. list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
  2355. ctx = &cpuctx->ctx;
  2356. perf_adjust_freq_unthr_context(ctx, throttled);
  2357. ctx = cpuctx->task_ctx;
  2358. if (ctx)
  2359. perf_adjust_freq_unthr_context(ctx, throttled);
  2360. }
  2361. }
  2362. static int event_enable_on_exec(struct perf_event *event,
  2363. struct perf_event_context *ctx)
  2364. {
  2365. if (!event->attr.enable_on_exec)
  2366. return 0;
  2367. event->attr.enable_on_exec = 0;
  2368. if (event->state >= PERF_EVENT_STATE_INACTIVE)
  2369. return 0;
  2370. __perf_event_mark_enabled(event);
  2371. return 1;
  2372. }
  2373. /*
  2374. * Enable all of a task's events that have been marked enable-on-exec.
  2375. * This expects task == current.
  2376. */
  2377. static void perf_event_enable_on_exec(struct perf_event_context *ctx)
  2378. {
  2379. struct perf_event *event;
  2380. unsigned long flags;
  2381. int enabled = 0;
  2382. int ret;
  2383. local_irq_save(flags);
  2384. if (!ctx || !ctx->nr_events)
  2385. goto out;
  2386. /*
  2387. * We must ctxsw out cgroup events to avoid conflict
  2388. * when invoking perf_task_event_sched_in() later on
  2389. * in this function. Otherwise we end up trying to
  2390. * ctxswin cgroup events which are already scheduled
  2391. * in.
  2392. */
  2393. perf_cgroup_sched_out(current, NULL);
  2394. raw_spin_lock(&ctx->lock);
  2395. task_ctx_sched_out(ctx);
  2396. list_for_each_entry(event, &ctx->event_list, event_entry) {
  2397. ret = event_enable_on_exec(event, ctx);
  2398. if (ret)
  2399. enabled = 1;
  2400. }
  2401. /*
  2402. * Unclone this context if we enabled any event.
  2403. */
  2404. if (enabled)
  2405. unclone_ctx(ctx);
  2406. raw_spin_unlock(&ctx->lock);
  2407. /*
  2408. * Also calls ctxswin for cgroup events, if any:
  2409. */
  2410. perf_event_context_sched_in(ctx, ctx->task);
  2411. out:
  2412. local_irq_restore(flags);
  2413. }
  2414. /*
  2415. * Cross CPU call to read the hardware event
  2416. */
  2417. static void __perf_event_read(void *info)
  2418. {
  2419. struct perf_event *event = info;
  2420. struct perf_event_context *ctx = event->ctx;
  2421. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  2422. /*
  2423. * If this is a task context, we need to check whether it is
  2424. * the current task context of this cpu. If not it has been
  2425. * scheduled out before the smp call arrived. In that case
  2426. * event->count would have been updated to a recent sample
  2427. * when the event was scheduled out.
  2428. */
  2429. if (ctx->task && cpuctx->task_ctx != ctx)
  2430. return;
  2431. raw_spin_lock(&ctx->lock);
  2432. if (ctx->is_active) {
  2433. update_context_time(ctx);
  2434. update_cgrp_time_from_event(event);
  2435. }
  2436. update_event_times(event);
  2437. if (event->state == PERF_EVENT_STATE_ACTIVE)
  2438. event->pmu->read(event);
  2439. raw_spin_unlock(&ctx->lock);
  2440. }
  2441. static inline u64 perf_event_count(struct perf_event *event)
  2442. {
  2443. return local64_read(&event->count) + atomic64_read(&event->child_count);
  2444. }
  2445. static u64 perf_event_read(struct perf_event *event)
  2446. {
  2447. /*
  2448. * If event is enabled and currently active on a CPU, update the
  2449. * value in the event structure:
  2450. */
  2451. if (event->state == PERF_EVENT_STATE_ACTIVE) {
  2452. smp_call_function_single(event->oncpu,
  2453. __perf_event_read, event, 1);
  2454. } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
  2455. struct perf_event_context *ctx = event->ctx;
  2456. unsigned long flags;
  2457. raw_spin_lock_irqsave(&ctx->lock, flags);
  2458. /*
  2459. * may read while context is not active
  2460. * (e.g., thread is blocked), in that case
  2461. * we cannot update context time
  2462. */
  2463. if (ctx->is_active) {
  2464. update_context_time(ctx);
  2465. update_cgrp_time_from_event(event);
  2466. }
  2467. update_event_times(event);
  2468. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  2469. }
  2470. return perf_event_count(event);
  2471. }
  2472. /*
  2473. * Initialize the perf_event context in a task_struct:
  2474. */
  2475. static void __perf_event_init_context(struct perf_event_context *ctx)
  2476. {
  2477. raw_spin_lock_init(&ctx->lock);
  2478. mutex_init(&ctx->mutex);
  2479. INIT_LIST_HEAD(&ctx->pinned_groups);
  2480. INIT_LIST_HEAD(&ctx->flexible_groups);
  2481. INIT_LIST_HEAD(&ctx->event_list);
  2482. atomic_set(&ctx->refcount, 1);
  2483. }
  2484. static struct perf_event_context *
  2485. alloc_perf_context(struct pmu *pmu, struct task_struct *task)
  2486. {
  2487. struct perf_event_context *ctx;
  2488. ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
  2489. if (!ctx)
  2490. return NULL;
  2491. __perf_event_init_context(ctx);
  2492. if (task) {
  2493. ctx->task = task;
  2494. get_task_struct(task);
  2495. }
  2496. ctx->pmu = pmu;
  2497. return ctx;
  2498. }
  2499. static struct task_struct *
  2500. find_lively_task_by_vpid(pid_t vpid)
  2501. {
  2502. struct task_struct *task;
  2503. int err;
  2504. rcu_read_lock();
  2505. if (!vpid)
  2506. task = current;
  2507. else
  2508. task = find_task_by_vpid(vpid);
  2509. if (task)
  2510. get_task_struct(task);
  2511. rcu_read_unlock();
  2512. if (!task)
  2513. return ERR_PTR(-ESRCH);
  2514. /* Reuse ptrace permission checks for now. */
  2515. err = -EACCES;
  2516. if (!ptrace_may_access(task, PTRACE_MODE_READ))
  2517. goto errout;
  2518. return task;
  2519. errout:
  2520. put_task_struct(task);
  2521. return ERR_PTR(err);
  2522. }
  2523. /*
  2524. * Returns a matching context with refcount and pincount.
  2525. */
  2526. static struct perf_event_context *
  2527. find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
  2528. {
  2529. struct perf_event_context *ctx;
  2530. struct perf_cpu_context *cpuctx;
  2531. unsigned long flags;
  2532. int ctxn, err;
  2533. if (!task) {
  2534. /* Must be root to operate on a CPU event: */
  2535. if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
  2536. return ERR_PTR(-EACCES);
  2537. /*
  2538. * We could be clever and allow to attach a event to an
  2539. * offline CPU and activate it when the CPU comes up, but
  2540. * that's for later.
  2541. */
  2542. if (!cpu_online(cpu))
  2543. return ERR_PTR(-ENODEV);
  2544. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  2545. ctx = &cpuctx->ctx;
  2546. get_ctx(ctx);
  2547. ++ctx->pin_count;
  2548. return ctx;
  2549. }
  2550. err = -EINVAL;
  2551. ctxn = pmu->task_ctx_nr;
  2552. if (ctxn < 0)
  2553. goto errout;
  2554. retry:
  2555. ctx = perf_lock_task_context(task, ctxn, &flags);
  2556. if (ctx) {
  2557. unclone_ctx(ctx);
  2558. ++ctx->pin_count;
  2559. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  2560. } else {
  2561. ctx = alloc_perf_context(pmu, task);
  2562. err = -ENOMEM;
  2563. if (!ctx)
  2564. goto errout;
  2565. err = 0;
  2566. mutex_lock(&task->perf_event_mutex);
  2567. /*
  2568. * If it has already passed perf_event_exit_task().
  2569. * we must see PF_EXITING, it takes this mutex too.
  2570. */
  2571. if (task->flags & PF_EXITING)
  2572. err = -ESRCH;
  2573. else if (task->perf_event_ctxp[ctxn])
  2574. err = -EAGAIN;
  2575. else {
  2576. get_ctx(ctx);
  2577. ++ctx->pin_count;
  2578. rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
  2579. }
  2580. mutex_unlock(&task->perf_event_mutex);
  2581. if (unlikely(err)) {
  2582. put_ctx(ctx);
  2583. if (err == -EAGAIN)
  2584. goto retry;
  2585. goto errout;
  2586. }
  2587. }
  2588. return ctx;
  2589. errout:
  2590. return ERR_PTR(err);
  2591. }
  2592. static void perf_event_free_filter(struct perf_event *event);
  2593. static void free_event_rcu(struct rcu_head *head)
  2594. {
  2595. struct perf_event *event;
  2596. event = container_of(head, struct perf_event, rcu_head);
  2597. if (event->ns)
  2598. put_pid_ns(event->ns);
  2599. perf_event_free_filter(event);
  2600. kfree(event);
  2601. }
  2602. static void ring_buffer_put(struct ring_buffer *rb);
  2603. static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb);
  2604. static void unaccount_event_cpu(struct perf_event *event, int cpu)
  2605. {
  2606. if (event->parent)
  2607. return;
  2608. if (has_branch_stack(event)) {
  2609. if (!(event->attach_state & PERF_ATTACH_TASK))
  2610. atomic_dec(&per_cpu(perf_branch_stack_events, cpu));
  2611. }
  2612. if (is_cgroup_event(event))
  2613. atomic_dec(&per_cpu(perf_cgroup_events, cpu));
  2614. if (event->attr.freq)
  2615. atomic_dec(&per_cpu(perf_freq_events, cpu));
  2616. }
  2617. static void unaccount_event(struct perf_event *event)
  2618. {
  2619. if (event->parent)
  2620. return;
  2621. if (event->attach_state & PERF_ATTACH_TASK)
  2622. static_key_slow_dec_deferred(&perf_sched_events);
  2623. if (event->attr.mmap || event->attr.mmap_data)
  2624. atomic_dec(&nr_mmap_events);
  2625. if (event->attr.comm)
  2626. atomic_dec(&nr_comm_events);
  2627. if (event->attr.task)
  2628. atomic_dec(&nr_task_events);
  2629. if (is_cgroup_event(event))
  2630. static_key_slow_dec_deferred(&perf_sched_events);
  2631. if (has_branch_stack(event))
  2632. static_key_slow_dec_deferred(&perf_sched_events);
  2633. unaccount_event_cpu(event, event->cpu);
  2634. }
  2635. static void __free_event(struct perf_event *event)
  2636. {
  2637. if (!event->parent) {
  2638. if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
  2639. put_callchain_buffers();
  2640. }
  2641. if (event->destroy)
  2642. event->destroy(event);
  2643. if (event->ctx)
  2644. put_ctx(event->ctx);
  2645. call_rcu(&event->rcu_head, free_event_rcu);
  2646. }
  2647. static void free_event(struct perf_event *event)
  2648. {
  2649. irq_work_sync(&event->pending);
  2650. unaccount_event(event);
  2651. if (event->rb) {
  2652. struct ring_buffer *rb;
  2653. /*
  2654. * Can happen when we close an event with re-directed output.
  2655. *
  2656. * Since we have a 0 refcount, perf_mmap_close() will skip
  2657. * over us; possibly making our ring_buffer_put() the last.
  2658. */
  2659. mutex_lock(&event->mmap_mutex);
  2660. rb = event->rb;
  2661. if (rb) {
  2662. rcu_assign_pointer(event->rb, NULL);
  2663. ring_buffer_detach(event, rb);
  2664. ring_buffer_put(rb); /* could be last */
  2665. }
  2666. mutex_unlock(&event->mmap_mutex);
  2667. }
  2668. if (is_cgroup_event(event))
  2669. perf_detach_cgroup(event);
  2670. __free_event(event);
  2671. }
  2672. int perf_event_release_kernel(struct perf_event *event)
  2673. {
  2674. struct perf_event_context *ctx = event->ctx;
  2675. WARN_ON_ONCE(ctx->parent_ctx);
  2676. /*
  2677. * There are two ways this annotation is useful:
  2678. *
  2679. * 1) there is a lock recursion from perf_event_exit_task
  2680. * see the comment there.
  2681. *
  2682. * 2) there is a lock-inversion with mmap_sem through
  2683. * perf_event_read_group(), which takes faults while
  2684. * holding ctx->mutex, however this is called after
  2685. * the last filedesc died, so there is no possibility
  2686. * to trigger the AB-BA case.
  2687. */
  2688. mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
  2689. raw_spin_lock_irq(&ctx->lock);
  2690. perf_group_detach(event);
  2691. raw_spin_unlock_irq(&ctx->lock);
  2692. perf_remove_from_context(event);
  2693. mutex_unlock(&ctx->mutex);
  2694. free_event(event);
  2695. return 0;
  2696. }
  2697. EXPORT_SYMBOL_GPL(perf_event_release_kernel);
  2698. /*
  2699. * Called when the last reference to the file is gone.
  2700. */
  2701. static void put_event(struct perf_event *event)
  2702. {
  2703. struct task_struct *owner;
  2704. if (!atomic_long_dec_and_test(&event->refcount))
  2705. return;
  2706. rcu_read_lock();
  2707. owner = ACCESS_ONCE(event->owner);
  2708. /*
  2709. * Matches the smp_wmb() in perf_event_exit_task(). If we observe
  2710. * !owner it means the list deletion is complete and we can indeed
  2711. * free this event, otherwise we need to serialize on
  2712. * owner->perf_event_mutex.
  2713. */
  2714. smp_read_barrier_depends();
  2715. if (owner) {
  2716. /*
  2717. * Since delayed_put_task_struct() also drops the last
  2718. * task reference we can safely take a new reference
  2719. * while holding the rcu_read_lock().
  2720. */
  2721. get_task_struct(owner);
  2722. }
  2723. rcu_read_unlock();
  2724. if (owner) {
  2725. mutex_lock(&owner->perf_event_mutex);
  2726. /*
  2727. * We have to re-check the event->owner field, if it is cleared
  2728. * we raced with perf_event_exit_task(), acquiring the mutex
  2729. * ensured they're done, and we can proceed with freeing the
  2730. * event.
  2731. */
  2732. if (event->owner)
  2733. list_del_init(&event->owner_entry);
  2734. mutex_unlock(&owner->perf_event_mutex);
  2735. put_task_struct(owner);
  2736. }
  2737. perf_event_release_kernel(event);
  2738. }
  2739. static int perf_release(struct inode *inode, struct file *file)
  2740. {
  2741. put_event(file->private_data);
  2742. return 0;
  2743. }
  2744. u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
  2745. {
  2746. struct perf_event *child;
  2747. u64 total = 0;
  2748. *enabled = 0;
  2749. *running = 0;
  2750. mutex_lock(&event->child_mutex);
  2751. total += perf_event_read(event);
  2752. *enabled += event->total_time_enabled +
  2753. atomic64_read(&event->child_total_time_enabled);
  2754. *running += event->total_time_running +
  2755. atomic64_read(&event->child_total_time_running);
  2756. list_for_each_entry(child, &event->child_list, child_list) {
  2757. total += perf_event_read(child);
  2758. *enabled += child->total_time_enabled;
  2759. *running += child->total_time_running;
  2760. }
  2761. mutex_unlock(&event->child_mutex);
  2762. return total;
  2763. }
  2764. EXPORT_SYMBOL_GPL(perf_event_read_value);
  2765. static int perf_event_read_group(struct perf_event *event,
  2766. u64 read_format, char __user *buf)
  2767. {
  2768. struct perf_event *leader = event->group_leader, *sub;
  2769. int n = 0, size = 0, ret = -EFAULT;
  2770. struct perf_event_context *ctx = leader->ctx;
  2771. u64 values[5];
  2772. u64 count, enabled, running;
  2773. mutex_lock(&ctx->mutex);
  2774. count = perf_event_read_value(leader, &enabled, &running);
  2775. values[n++] = 1 + leader->nr_siblings;
  2776. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  2777. values[n++] = enabled;
  2778. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  2779. values[n++] = running;
  2780. values[n++] = count;
  2781. if (read_format & PERF_FORMAT_ID)
  2782. values[n++] = primary_event_id(leader);
  2783. size = n * sizeof(u64);
  2784. if (copy_to_user(buf, values, size))
  2785. goto unlock;
  2786. ret = size;
  2787. list_for_each_entry(sub, &leader->sibling_list, group_entry) {
  2788. n = 0;
  2789. values[n++] = perf_event_read_value(sub, &enabled, &running);
  2790. if (read_format & PERF_FORMAT_ID)
  2791. values[n++] = primary_event_id(sub);
  2792. size = n * sizeof(u64);
  2793. if (copy_to_user(buf + ret, values, size)) {
  2794. ret = -EFAULT;
  2795. goto unlock;
  2796. }
  2797. ret += size;
  2798. }
  2799. unlock:
  2800. mutex_unlock(&ctx->mutex);
  2801. return ret;
  2802. }
  2803. static int perf_event_read_one(struct perf_event *event,
  2804. u64 read_format, char __user *buf)
  2805. {
  2806. u64 enabled, running;
  2807. u64 values[4];
  2808. int n = 0;
  2809. values[n++] = perf_event_read_value(event, &enabled, &running);
  2810. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  2811. values[n++] = enabled;
  2812. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  2813. values[n++] = running;
  2814. if (read_format & PERF_FORMAT_ID)
  2815. values[n++] = primary_event_id(event);
  2816. if (copy_to_user(buf, values, n * sizeof(u64)))
  2817. return -EFAULT;
  2818. return n * sizeof(u64);
  2819. }
  2820. /*
  2821. * Read the performance event - simple non blocking version for now
  2822. */
  2823. static ssize_t
  2824. perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
  2825. {
  2826. u64 read_format = event->attr.read_format;
  2827. int ret;
  2828. /*
  2829. * Return end-of-file for a read on a event that is in
  2830. * error state (i.e. because it was pinned but it couldn't be
  2831. * scheduled on to the CPU at some point).
  2832. */
  2833. if (event->state == PERF_EVENT_STATE_ERROR)
  2834. return 0;
  2835. if (count < event->read_size)
  2836. return -ENOSPC;
  2837. WARN_ON_ONCE(event->ctx->parent_ctx);
  2838. if (read_format & PERF_FORMAT_GROUP)
  2839. ret = perf_event_read_group(event, read_format, buf);
  2840. else
  2841. ret = perf_event_read_one(event, read_format, buf);
  2842. return ret;
  2843. }
  2844. static ssize_t
  2845. perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
  2846. {
  2847. struct perf_event *event = file->private_data;
  2848. return perf_read_hw(event, buf, count);
  2849. }
  2850. static unsigned int perf_poll(struct file *file, poll_table *wait)
  2851. {
  2852. struct perf_event *event = file->private_data;
  2853. struct ring_buffer *rb;
  2854. unsigned int events = POLL_HUP;
  2855. /*
  2856. * Pin the event->rb by taking event->mmap_mutex; otherwise
  2857. * perf_event_set_output() can swizzle our rb and make us miss wakeups.
  2858. */
  2859. mutex_lock(&event->mmap_mutex);
  2860. rb = event->rb;
  2861. if (rb)
  2862. events = atomic_xchg(&rb->poll, 0);
  2863. mutex_unlock(&event->mmap_mutex);
  2864. poll_wait(file, &event->waitq, wait);
  2865. return events;
  2866. }
  2867. static void perf_event_reset(struct perf_event *event)
  2868. {
  2869. (void)perf_event_read(event);
  2870. local64_set(&event->count, 0);
  2871. perf_event_update_userpage(event);
  2872. }
  2873. /*
  2874. * Holding the top-level event's child_mutex means that any
  2875. * descendant process that has inherited this event will block
  2876. * in sync_child_event if it goes to exit, thus satisfying the
  2877. * task existence requirements of perf_event_enable/disable.
  2878. */
  2879. static void perf_event_for_each_child(struct perf_event *event,
  2880. void (*func)(struct perf_event *))
  2881. {
  2882. struct perf_event *child;
  2883. WARN_ON_ONCE(event->ctx->parent_ctx);
  2884. mutex_lock(&event->child_mutex);
  2885. func(event);
  2886. list_for_each_entry(child, &event->child_list, child_list)
  2887. func(child);
  2888. mutex_unlock(&event->child_mutex);
  2889. }
  2890. static void perf_event_for_each(struct perf_event *event,
  2891. void (*func)(struct perf_event *))
  2892. {
  2893. struct perf_event_context *ctx = event->ctx;
  2894. struct perf_event *sibling;
  2895. WARN_ON_ONCE(ctx->parent_ctx);
  2896. mutex_lock(&ctx->mutex);
  2897. event = event->group_leader;
  2898. perf_event_for_each_child(event, func);
  2899. list_for_each_entry(sibling, &event->sibling_list, group_entry)
  2900. perf_event_for_each_child(sibling, func);
  2901. mutex_unlock(&ctx->mutex);
  2902. }
  2903. static int perf_event_period(struct perf_event *event, u64 __user *arg)
  2904. {
  2905. struct perf_event_context *ctx = event->ctx;
  2906. int ret = 0;
  2907. u64 value;
  2908. if (!is_sampling_event(event))
  2909. return -EINVAL;
  2910. if (copy_from_user(&value, arg, sizeof(value)))
  2911. return -EFAULT;
  2912. if (!value)
  2913. return -EINVAL;
  2914. raw_spin_lock_irq(&ctx->lock);
  2915. if (event->attr.freq) {
  2916. if (value > sysctl_perf_event_sample_rate) {
  2917. ret = -EINVAL;
  2918. goto unlock;
  2919. }
  2920. event->attr.sample_freq = value;
  2921. } else {
  2922. event->attr.sample_period = value;
  2923. event->hw.sample_period = value;
  2924. }
  2925. unlock:
  2926. raw_spin_unlock_irq(&ctx->lock);
  2927. return ret;
  2928. }
  2929. static const struct file_operations perf_fops;
  2930. static inline int perf_fget_light(int fd, struct fd *p)
  2931. {
  2932. struct fd f = fdget(fd);
  2933. if (!f.file)
  2934. return -EBADF;
  2935. if (f.file->f_op != &perf_fops) {
  2936. fdput(f);
  2937. return -EBADF;
  2938. }
  2939. *p = f;
  2940. return 0;
  2941. }
  2942. static int perf_event_set_output(struct perf_event *event,
  2943. struct perf_event *output_event);
  2944. static int perf_event_set_filter(struct perf_event *event, void __user *arg);
  2945. static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  2946. {
  2947. struct perf_event *event = file->private_data;
  2948. void (*func)(struct perf_event *);
  2949. u32 flags = arg;
  2950. switch (cmd) {
  2951. case PERF_EVENT_IOC_ENABLE:
  2952. func = perf_event_enable;
  2953. break;
  2954. case PERF_EVENT_IOC_DISABLE:
  2955. func = perf_event_disable;
  2956. break;
  2957. case PERF_EVENT_IOC_RESET:
  2958. func = perf_event_reset;
  2959. break;
  2960. case PERF_EVENT_IOC_REFRESH:
  2961. return perf_event_refresh(event, arg);
  2962. case PERF_EVENT_IOC_PERIOD:
  2963. return perf_event_period(event, (u64 __user *)arg);
  2964. case PERF_EVENT_IOC_ID:
  2965. {
  2966. u64 id = primary_event_id(event);
  2967. if (copy_to_user((void __user *)arg, &id, sizeof(id)))
  2968. return -EFAULT;
  2969. return 0;
  2970. }
  2971. case PERF_EVENT_IOC_SET_OUTPUT:
  2972. {
  2973. int ret;
  2974. if (arg != -1) {
  2975. struct perf_event *output_event;
  2976. struct fd output;
  2977. ret = perf_fget_light(arg, &output);
  2978. if (ret)
  2979. return ret;
  2980. output_event = output.file->private_data;
  2981. ret = perf_event_set_output(event, output_event);
  2982. fdput(output);
  2983. } else {
  2984. ret = perf_event_set_output(event, NULL);
  2985. }
  2986. return ret;
  2987. }
  2988. case PERF_EVENT_IOC_SET_FILTER:
  2989. return perf_event_set_filter(event, (void __user *)arg);
  2990. default:
  2991. return -ENOTTY;
  2992. }
  2993. if (flags & PERF_IOC_FLAG_GROUP)
  2994. perf_event_for_each(event, func);
  2995. else
  2996. perf_event_for_each_child(event, func);
  2997. return 0;
  2998. }
  2999. int perf_event_task_enable(void)
  3000. {
  3001. struct perf_event *event;
  3002. mutex_lock(&current->perf_event_mutex);
  3003. list_for_each_entry(event, &current->perf_event_list, owner_entry)
  3004. perf_event_for_each_child(event, perf_event_enable);
  3005. mutex_unlock(&current->perf_event_mutex);
  3006. return 0;
  3007. }
  3008. int perf_event_task_disable(void)
  3009. {
  3010. struct perf_event *event;
  3011. mutex_lock(&current->perf_event_mutex);
  3012. list_for_each_entry(event, &current->perf_event_list, owner_entry)
  3013. perf_event_for_each_child(event, perf_event_disable);
  3014. mutex_unlock(&current->perf_event_mutex);
  3015. return 0;
  3016. }
  3017. static int perf_event_index(struct perf_event *event)
  3018. {
  3019. if (event->hw.state & PERF_HES_STOPPED)
  3020. return 0;
  3021. if (event->state != PERF_EVENT_STATE_ACTIVE)
  3022. return 0;
  3023. return event->pmu->event_idx(event);
  3024. }
  3025. static void calc_timer_values(struct perf_event *event,
  3026. u64 *now,
  3027. u64 *enabled,
  3028. u64 *running)
  3029. {
  3030. u64 ctx_time;
  3031. *now = perf_clock();
  3032. ctx_time = event->shadow_ctx_time + *now;
  3033. *enabled = ctx_time - event->tstamp_enabled;
  3034. *running = ctx_time - event->tstamp_running;
  3035. }
  3036. void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
  3037. {
  3038. }
  3039. /*
  3040. * Callers need to ensure there can be no nesting of this function, otherwise
  3041. * the seqlock logic goes bad. We can not serialize this because the arch
  3042. * code calls this from NMI context.
  3043. */
  3044. void perf_event_update_userpage(struct perf_event *event)
  3045. {
  3046. struct perf_event_mmap_page *userpg;
  3047. struct ring_buffer *rb;
  3048. u64 enabled, running, now;
  3049. rcu_read_lock();
  3050. /*
  3051. * compute total_time_enabled, total_time_running
  3052. * based on snapshot values taken when the event
  3053. * was last scheduled in.
  3054. *
  3055. * we cannot simply called update_context_time()
  3056. * because of locking issue as we can be called in
  3057. * NMI context
  3058. */
  3059. calc_timer_values(event, &now, &enabled, &running);
  3060. rb = rcu_dereference(event->rb);
  3061. if (!rb)
  3062. goto unlock;
  3063. userpg = rb->user_page;
  3064. /*
  3065. * Disable preemption so as to not let the corresponding user-space
  3066. * spin too long if we get preempted.
  3067. */
  3068. preempt_disable();
  3069. ++userpg->lock;
  3070. barrier();
  3071. userpg->index = perf_event_index(event);
  3072. userpg->offset = perf_event_count(event);
  3073. if (userpg->index)
  3074. userpg->offset -= local64_read(&event->hw.prev_count);
  3075. userpg->time_enabled = enabled +
  3076. atomic64_read(&event->child_total_time_enabled);
  3077. userpg->time_running = running +
  3078. atomic64_read(&event->child_total_time_running);
  3079. arch_perf_update_userpage(userpg, now);
  3080. barrier();
  3081. ++userpg->lock;
  3082. preempt_enable();
  3083. unlock:
  3084. rcu_read_unlock();
  3085. }
  3086. static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  3087. {
  3088. struct perf_event *event = vma->vm_file->private_data;
  3089. struct ring_buffer *rb;
  3090. int ret = VM_FAULT_SIGBUS;
  3091. if (vmf->flags & FAULT_FLAG_MKWRITE) {
  3092. if (vmf->pgoff == 0)
  3093. ret = 0;
  3094. return ret;
  3095. }
  3096. rcu_read_lock();
  3097. rb = rcu_dereference(event->rb);
  3098. if (!rb)
  3099. goto unlock;
  3100. if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
  3101. goto unlock;
  3102. vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
  3103. if (!vmf->page)
  3104. goto unlock;
  3105. get_page(vmf->page);
  3106. vmf->page->mapping = vma->vm_file->f_mapping;
  3107. vmf->page->index = vmf->pgoff;
  3108. ret = 0;
  3109. unlock:
  3110. rcu_read_unlock();
  3111. return ret;
  3112. }
  3113. static void ring_buffer_attach(struct perf_event *event,
  3114. struct ring_buffer *rb)
  3115. {
  3116. unsigned long flags;
  3117. if (!list_empty(&event->rb_entry))
  3118. return;
  3119. spin_lock_irqsave(&rb->event_lock, flags);
  3120. if (list_empty(&event->rb_entry))
  3121. list_add(&event->rb_entry, &rb->event_list);
  3122. spin_unlock_irqrestore(&rb->event_lock, flags);
  3123. }
  3124. static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb)
  3125. {
  3126. unsigned long flags;
  3127. if (list_empty(&event->rb_entry))
  3128. return;
  3129. spin_lock_irqsave(&rb->event_lock, flags);
  3130. list_del_init(&event->rb_entry);
  3131. wake_up_all(&event->waitq);
  3132. spin_unlock_irqrestore(&rb->event_lock, flags);
  3133. }
  3134. static void ring_buffer_wakeup(struct perf_event *event)
  3135. {
  3136. struct ring_buffer *rb;
  3137. rcu_read_lock();
  3138. rb = rcu_dereference(event->rb);
  3139. if (rb) {
  3140. list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
  3141. wake_up_all(&event->waitq);
  3142. }
  3143. rcu_read_unlock();
  3144. }
  3145. static void rb_free_rcu(struct rcu_head *rcu_head)
  3146. {
  3147. struct ring_buffer *rb;
  3148. rb = container_of(rcu_head, struct ring_buffer, rcu_head);
  3149. rb_free(rb);
  3150. }
  3151. static struct ring_buffer *ring_buffer_get(struct perf_event *event)
  3152. {
  3153. struct ring_buffer *rb;
  3154. rcu_read_lock();
  3155. rb = rcu_dereference(event->rb);
  3156. if (rb) {
  3157. if (!atomic_inc_not_zero(&rb->refcount))
  3158. rb = NULL;
  3159. }
  3160. rcu_read_unlock();
  3161. return rb;
  3162. }
  3163. static void ring_buffer_put(struct ring_buffer *rb)
  3164. {
  3165. if (!atomic_dec_and_test(&rb->refcount))
  3166. return;
  3167. WARN_ON_ONCE(!list_empty(&rb->event_list));
  3168. call_rcu(&rb->rcu_head, rb_free_rcu);
  3169. }
  3170. static void perf_mmap_open(struct vm_area_struct *vma)
  3171. {
  3172. struct perf_event *event = vma->vm_file->private_data;
  3173. atomic_inc(&event->mmap_count);
  3174. atomic_inc(&event->rb->mmap_count);
  3175. }
  3176. /*
  3177. * A buffer can be mmap()ed multiple times; either directly through the same
  3178. * event, or through other events by use of perf_event_set_output().
  3179. *
  3180. * In order to undo the VM accounting done by perf_mmap() we need to destroy
  3181. * the buffer here, where we still have a VM context. This means we need
  3182. * to detach all events redirecting to us.
  3183. */
  3184. static void perf_mmap_close(struct vm_area_struct *vma)
  3185. {
  3186. struct perf_event *event = vma->vm_file->private_data;
  3187. struct ring_buffer *rb = event->rb;
  3188. struct user_struct *mmap_user = rb->mmap_user;
  3189. int mmap_locked = rb->mmap_locked;
  3190. unsigned long size = perf_data_size(rb);
  3191. atomic_dec(&rb->mmap_count);
  3192. if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
  3193. return;
  3194. /* Detach current event from the buffer. */
  3195. rcu_assign_pointer(event->rb, NULL);
  3196. ring_buffer_detach(event, rb);
  3197. mutex_unlock(&event->mmap_mutex);
  3198. /* If there's still other mmap()s of this buffer, we're done. */
  3199. if (atomic_read(&rb->mmap_count)) {
  3200. ring_buffer_put(rb); /* can't be last */
  3201. return;
  3202. }
  3203. /*
  3204. * No other mmap()s, detach from all other events that might redirect
  3205. * into the now unreachable buffer. Somewhat complicated by the
  3206. * fact that rb::event_lock otherwise nests inside mmap_mutex.
  3207. */
  3208. again:
  3209. rcu_read_lock();
  3210. list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
  3211. if (!atomic_long_inc_not_zero(&event->refcount)) {
  3212. /*
  3213. * This event is en-route to free_event() which will
  3214. * detach it and remove it from the list.
  3215. */
  3216. continue;
  3217. }
  3218. rcu_read_unlock();
  3219. mutex_lock(&event->mmap_mutex);
  3220. /*
  3221. * Check we didn't race with perf_event_set_output() which can
  3222. * swizzle the rb from under us while we were waiting to
  3223. * acquire mmap_mutex.
  3224. *
  3225. * If we find a different rb; ignore this event, a next
  3226. * iteration will no longer find it on the list. We have to
  3227. * still restart the iteration to make sure we're not now
  3228. * iterating the wrong list.
  3229. */
  3230. if (event->rb == rb) {
  3231. rcu_assign_pointer(event->rb, NULL);
  3232. ring_buffer_detach(event, rb);
  3233. ring_buffer_put(rb); /* can't be last, we still have one */
  3234. }
  3235. mutex_unlock(&event->mmap_mutex);
  3236. put_event(event);
  3237. /*
  3238. * Restart the iteration; either we're on the wrong list or
  3239. * destroyed its integrity by doing a deletion.
  3240. */
  3241. goto again;
  3242. }
  3243. rcu_read_unlock();
  3244. /*
  3245. * It could be there's still a few 0-ref events on the list; they'll
  3246. * get cleaned up by free_event() -- they'll also still have their
  3247. * ref on the rb and will free it whenever they are done with it.
  3248. *
  3249. * Aside from that, this buffer is 'fully' detached and unmapped,
  3250. * undo the VM accounting.
  3251. */
  3252. atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
  3253. vma->vm_mm->pinned_vm -= mmap_locked;
  3254. free_uid(mmap_user);
  3255. ring_buffer_put(rb); /* could be last */
  3256. }
  3257. static const struct vm_operations_struct perf_mmap_vmops = {
  3258. .open = perf_mmap_open,
  3259. .close = perf_mmap_close,
  3260. .fault = perf_mmap_fault,
  3261. .page_mkwrite = perf_mmap_fault,
  3262. };
  3263. static int perf_mmap(struct file *file, struct vm_area_struct *vma)
  3264. {
  3265. struct perf_event *event = file->private_data;
  3266. unsigned long user_locked, user_lock_limit;
  3267. struct user_struct *user = current_user();
  3268. unsigned long locked, lock_limit;
  3269. struct ring_buffer *rb;
  3270. unsigned long vma_size;
  3271. unsigned long nr_pages;
  3272. long user_extra, extra;
  3273. int ret = 0, flags = 0;
  3274. /*
  3275. * Don't allow mmap() of inherited per-task counters. This would
  3276. * create a performance issue due to all children writing to the
  3277. * same rb.
  3278. */
  3279. if (event->cpu == -1 && event->attr.inherit)
  3280. return -EINVAL;
  3281. if (!(vma->vm_flags & VM_SHARED))
  3282. return -EINVAL;
  3283. vma_size = vma->vm_end - vma->vm_start;
  3284. nr_pages = (vma_size / PAGE_SIZE) - 1;
  3285. /*
  3286. * If we have rb pages ensure they're a power-of-two number, so we
  3287. * can do bitmasks instead of modulo.
  3288. */
  3289. if (nr_pages != 0 && !is_power_of_2(nr_pages))
  3290. return -EINVAL;
  3291. if (vma_size != PAGE_SIZE * (1 + nr_pages))
  3292. return -EINVAL;
  3293. if (vma->vm_pgoff != 0)
  3294. return -EINVAL;
  3295. WARN_ON_ONCE(event->ctx->parent_ctx);
  3296. again:
  3297. mutex_lock(&event->mmap_mutex);
  3298. if (event->rb) {
  3299. if (event->rb->nr_pages != nr_pages) {
  3300. ret = -EINVAL;
  3301. goto unlock;
  3302. }
  3303. if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
  3304. /*
  3305. * Raced against perf_mmap_close() through
  3306. * perf_event_set_output(). Try again, hope for better
  3307. * luck.
  3308. */
  3309. mutex_unlock(&event->mmap_mutex);
  3310. goto again;
  3311. }
  3312. goto unlock;
  3313. }
  3314. user_extra = nr_pages + 1;
  3315. user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
  3316. /*
  3317. * Increase the limit linearly with more CPUs:
  3318. */
  3319. user_lock_limit *= num_online_cpus();
  3320. user_locked = atomic_long_read(&user->locked_vm) + user_extra;
  3321. extra = 0;
  3322. if (user_locked > user_lock_limit)
  3323. extra = user_locked - user_lock_limit;
  3324. lock_limit = rlimit(RLIMIT_MEMLOCK);
  3325. lock_limit >>= PAGE_SHIFT;
  3326. locked = vma->vm_mm->pinned_vm + extra;
  3327. if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
  3328. !capable(CAP_IPC_LOCK)) {
  3329. ret = -EPERM;
  3330. goto unlock;
  3331. }
  3332. WARN_ON(event->rb);
  3333. if (vma->vm_flags & VM_WRITE)
  3334. flags |= RING_BUFFER_WRITABLE;
  3335. rb = rb_alloc(nr_pages,
  3336. event->attr.watermark ? event->attr.wakeup_watermark : 0,
  3337. event->cpu, flags);
  3338. if (!rb) {
  3339. ret = -ENOMEM;
  3340. goto unlock;
  3341. }
  3342. atomic_set(&rb->mmap_count, 1);
  3343. rb->mmap_locked = extra;
  3344. rb->mmap_user = get_current_user();
  3345. atomic_long_add(user_extra, &user->locked_vm);
  3346. vma->vm_mm->pinned_vm += extra;
  3347. ring_buffer_attach(event, rb);
  3348. rcu_assign_pointer(event->rb, rb);
  3349. perf_event_update_userpage(event);
  3350. unlock:
  3351. if (!ret)
  3352. atomic_inc(&event->mmap_count);
  3353. mutex_unlock(&event->mmap_mutex);
  3354. /*
  3355. * Since pinned accounting is per vm we cannot allow fork() to copy our
  3356. * vma.
  3357. */
  3358. vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
  3359. vma->vm_ops = &perf_mmap_vmops;
  3360. return ret;
  3361. }
  3362. static int perf_fasync(int fd, struct file *filp, int on)
  3363. {
  3364. struct inode *inode = file_inode(filp);
  3365. struct perf_event *event = filp->private_data;
  3366. int retval;
  3367. mutex_lock(&inode->i_mutex);
  3368. retval = fasync_helper(fd, filp, on, &event->fasync);
  3369. mutex_unlock(&inode->i_mutex);
  3370. if (retval < 0)
  3371. return retval;
  3372. return 0;
  3373. }
  3374. static const struct file_operations perf_fops = {
  3375. .llseek = no_llseek,
  3376. .release = perf_release,
  3377. .read = perf_read,
  3378. .poll = perf_poll,
  3379. .unlocked_ioctl = perf_ioctl,
  3380. .compat_ioctl = perf_ioctl,
  3381. .mmap = perf_mmap,
  3382. .fasync = perf_fasync,
  3383. };
  3384. /*
  3385. * Perf event wakeup
  3386. *
  3387. * If there's data, ensure we set the poll() state and publish everything
  3388. * to user-space before waking everybody up.
  3389. */
  3390. void perf_event_wakeup(struct perf_event *event)
  3391. {
  3392. ring_buffer_wakeup(event);
  3393. if (event->pending_kill) {
  3394. kill_fasync(&event->fasync, SIGIO, event->pending_kill);
  3395. event->pending_kill = 0;
  3396. }
  3397. }
  3398. static void perf_pending_event(struct irq_work *entry)
  3399. {
  3400. struct perf_event *event = container_of(entry,
  3401. struct perf_event, pending);
  3402. if (event->pending_disable) {
  3403. event->pending_disable = 0;
  3404. __perf_event_disable(event);
  3405. }
  3406. if (event->pending_wakeup) {
  3407. event->pending_wakeup = 0;
  3408. perf_event_wakeup(event);
  3409. }
  3410. }
  3411. /*
  3412. * We assume there is only KVM supporting the callbacks.
  3413. * Later on, we might change it to a list if there is
  3414. * another virtualization implementation supporting the callbacks.
  3415. */
  3416. struct perf_guest_info_callbacks *perf_guest_cbs;
  3417. int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
  3418. {
  3419. perf_guest_cbs = cbs;
  3420. return 0;
  3421. }
  3422. EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
  3423. int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
  3424. {
  3425. perf_guest_cbs = NULL;
  3426. return 0;
  3427. }
  3428. EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
  3429. static void
  3430. perf_output_sample_regs(struct perf_output_handle *handle,
  3431. struct pt_regs *regs, u64 mask)
  3432. {
  3433. int bit;
  3434. for_each_set_bit(bit, (const unsigned long *) &mask,
  3435. sizeof(mask) * BITS_PER_BYTE) {
  3436. u64 val;
  3437. val = perf_reg_value(regs, bit);
  3438. perf_output_put(handle, val);
  3439. }
  3440. }
  3441. static void perf_sample_regs_user(struct perf_regs_user *regs_user,
  3442. struct pt_regs *regs)
  3443. {
  3444. if (!user_mode(regs)) {
  3445. if (current->mm)
  3446. regs = task_pt_regs(current);
  3447. else
  3448. regs = NULL;
  3449. }
  3450. if (regs) {
  3451. regs_user->regs = regs;
  3452. regs_user->abi = perf_reg_abi(current);
  3453. }
  3454. }
  3455. /*
  3456. * Get remaining task size from user stack pointer.
  3457. *
  3458. * It'd be better to take stack vma map and limit this more
  3459. * precisly, but there's no way to get it safely under interrupt,
  3460. * so using TASK_SIZE as limit.
  3461. */
  3462. static u64 perf_ustack_task_size(struct pt_regs *regs)
  3463. {
  3464. unsigned long addr = perf_user_stack_pointer(regs);
  3465. if (!addr || addr >= TASK_SIZE)
  3466. return 0;
  3467. return TASK_SIZE - addr;
  3468. }
  3469. static u16
  3470. perf_sample_ustack_size(u16 stack_size, u16 header_size,
  3471. struct pt_regs *regs)
  3472. {
  3473. u64 task_size;
  3474. /* No regs, no stack pointer, no dump. */
  3475. if (!regs)
  3476. return 0;
  3477. /*
  3478. * Check if we fit in with the requested stack size into the:
  3479. * - TASK_SIZE
  3480. * If we don't, we limit the size to the TASK_SIZE.
  3481. *
  3482. * - remaining sample size
  3483. * If we don't, we customize the stack size to
  3484. * fit in to the remaining sample size.
  3485. */
  3486. task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
  3487. stack_size = min(stack_size, (u16) task_size);
  3488. /* Current header size plus static size and dynamic size. */
  3489. header_size += 2 * sizeof(u64);
  3490. /* Do we fit in with the current stack dump size? */
  3491. if ((u16) (header_size + stack_size) < header_size) {
  3492. /*
  3493. * If we overflow the maximum size for the sample,
  3494. * we customize the stack dump size to fit in.
  3495. */
  3496. stack_size = USHRT_MAX - header_size - sizeof(u64);
  3497. stack_size = round_up(stack_size, sizeof(u64));
  3498. }
  3499. return stack_size;
  3500. }
  3501. static void
  3502. perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
  3503. struct pt_regs *regs)
  3504. {
  3505. /* Case of a kernel thread, nothing to dump */
  3506. if (!regs) {
  3507. u64 size = 0;
  3508. perf_output_put(handle, size);
  3509. } else {
  3510. unsigned long sp;
  3511. unsigned int rem;
  3512. u64 dyn_size;
  3513. /*
  3514. * We dump:
  3515. * static size
  3516. * - the size requested by user or the best one we can fit
  3517. * in to the sample max size
  3518. * data
  3519. * - user stack dump data
  3520. * dynamic size
  3521. * - the actual dumped size
  3522. */
  3523. /* Static size. */
  3524. perf_output_put(handle, dump_size);
  3525. /* Data. */
  3526. sp = perf_user_stack_pointer(regs);
  3527. rem = __output_copy_user(handle, (void *) sp, dump_size);
  3528. dyn_size = dump_size - rem;
  3529. perf_output_skip(handle, rem);
  3530. /* Dynamic size. */
  3531. perf_output_put(handle, dyn_size);
  3532. }
  3533. }
  3534. static void __perf_event_header__init_id(struct perf_event_header *header,
  3535. struct perf_sample_data *data,
  3536. struct perf_event *event)
  3537. {
  3538. u64 sample_type = event->attr.sample_type;
  3539. data->type = sample_type;
  3540. header->size += event->id_header_size;
  3541. if (sample_type & PERF_SAMPLE_TID) {
  3542. /* namespace issues */
  3543. data->tid_entry.pid = perf_event_pid(event, current);
  3544. data->tid_entry.tid = perf_event_tid(event, current);
  3545. }
  3546. if (sample_type & PERF_SAMPLE_TIME)
  3547. data->time = perf_clock();
  3548. if (sample_type & PERF_SAMPLE_ID)
  3549. data->id = primary_event_id(event);
  3550. if (sample_type & PERF_SAMPLE_STREAM_ID)
  3551. data->stream_id = event->id;
  3552. if (sample_type & PERF_SAMPLE_CPU) {
  3553. data->cpu_entry.cpu = raw_smp_processor_id();
  3554. data->cpu_entry.reserved = 0;
  3555. }
  3556. }
  3557. void perf_event_header__init_id(struct perf_event_header *header,
  3558. struct perf_sample_data *data,
  3559. struct perf_event *event)
  3560. {
  3561. if (event->attr.sample_id_all)
  3562. __perf_event_header__init_id(header, data, event);
  3563. }
  3564. static void __perf_event__output_id_sample(struct perf_output_handle *handle,
  3565. struct perf_sample_data *data)
  3566. {
  3567. u64 sample_type = data->type;
  3568. if (sample_type & PERF_SAMPLE_TID)
  3569. perf_output_put(handle, data->tid_entry);
  3570. if (sample_type & PERF_SAMPLE_TIME)
  3571. perf_output_put(handle, data->time);
  3572. if (sample_type & PERF_SAMPLE_ID)
  3573. perf_output_put(handle, data->id);
  3574. if (sample_type & PERF_SAMPLE_STREAM_ID)
  3575. perf_output_put(handle, data->stream_id);
  3576. if (sample_type & PERF_SAMPLE_CPU)
  3577. perf_output_put(handle, data->cpu_entry);
  3578. }
  3579. void perf_event__output_id_sample(struct perf_event *event,
  3580. struct perf_output_handle *handle,
  3581. struct perf_sample_data *sample)
  3582. {
  3583. if (event->attr.sample_id_all)
  3584. __perf_event__output_id_sample(handle, sample);
  3585. }
  3586. static void perf_output_read_one(struct perf_output_handle *handle,
  3587. struct perf_event *event,
  3588. u64 enabled, u64 running)
  3589. {
  3590. u64 read_format = event->attr.read_format;
  3591. u64 values[4];
  3592. int n = 0;
  3593. values[n++] = perf_event_count(event);
  3594. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
  3595. values[n++] = enabled +
  3596. atomic64_read(&event->child_total_time_enabled);
  3597. }
  3598. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
  3599. values[n++] = running +
  3600. atomic64_read(&event->child_total_time_running);
  3601. }
  3602. if (read_format & PERF_FORMAT_ID)
  3603. values[n++] = primary_event_id(event);
  3604. __output_copy(handle, values, n * sizeof(u64));
  3605. }
  3606. /*
  3607. * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
  3608. */
  3609. static void perf_output_read_group(struct perf_output_handle *handle,
  3610. struct perf_event *event,
  3611. u64 enabled, u64 running)
  3612. {
  3613. struct perf_event *leader = event->group_leader, *sub;
  3614. u64 read_format = event->attr.read_format;
  3615. u64 values[5];
  3616. int n = 0;
  3617. values[n++] = 1 + leader->nr_siblings;
  3618. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  3619. values[n++] = enabled;
  3620. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  3621. values[n++] = running;
  3622. if (leader != event)
  3623. leader->pmu->read(leader);
  3624. values[n++] = perf_event_count(leader);
  3625. if (read_format & PERF_FORMAT_ID)
  3626. values[n++] = primary_event_id(leader);
  3627. __output_copy(handle, values, n * sizeof(u64));
  3628. list_for_each_entry(sub, &leader->sibling_list, group_entry) {
  3629. n = 0;
  3630. if ((sub != event) &&
  3631. (sub->state == PERF_EVENT_STATE_ACTIVE))
  3632. sub->pmu->read(sub);
  3633. values[n++] = perf_event_count(sub);
  3634. if (read_format & PERF_FORMAT_ID)
  3635. values[n++] = primary_event_id(sub);
  3636. __output_copy(handle, values, n * sizeof(u64));
  3637. }
  3638. }
  3639. #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
  3640. PERF_FORMAT_TOTAL_TIME_RUNNING)
  3641. static void perf_output_read(struct perf_output_handle *handle,
  3642. struct perf_event *event)
  3643. {
  3644. u64 enabled = 0, running = 0, now;
  3645. u64 read_format = event->attr.read_format;
  3646. /*
  3647. * compute total_time_enabled, total_time_running
  3648. * based on snapshot values taken when the event
  3649. * was last scheduled in.
  3650. *
  3651. * we cannot simply called update_context_time()
  3652. * because of locking issue as we are called in
  3653. * NMI context
  3654. */
  3655. if (read_format & PERF_FORMAT_TOTAL_TIMES)
  3656. calc_timer_values(event, &now, &enabled, &running);
  3657. if (event->attr.read_format & PERF_FORMAT_GROUP)
  3658. perf_output_read_group(handle, event, enabled, running);
  3659. else
  3660. perf_output_read_one(handle, event, enabled, running);
  3661. }
  3662. void perf_output_sample(struct perf_output_handle *handle,
  3663. struct perf_event_header *header,
  3664. struct perf_sample_data *data,
  3665. struct perf_event *event)
  3666. {
  3667. u64 sample_type = data->type;
  3668. perf_output_put(handle, *header);
  3669. if (sample_type & PERF_SAMPLE_IP)
  3670. perf_output_put(handle, data->ip);
  3671. if (sample_type & PERF_SAMPLE_TID)
  3672. perf_output_put(handle, data->tid_entry);
  3673. if (sample_type & PERF_SAMPLE_TIME)
  3674. perf_output_put(handle, data->time);
  3675. if (sample_type & PERF_SAMPLE_ADDR)
  3676. perf_output_put(handle, data->addr);
  3677. if (sample_type & PERF_SAMPLE_ID)
  3678. perf_output_put(handle, data->id);
  3679. if (sample_type & PERF_SAMPLE_STREAM_ID)
  3680. perf_output_put(handle, data->stream_id);
  3681. if (sample_type & PERF_SAMPLE_CPU)
  3682. perf_output_put(handle, data->cpu_entry);
  3683. if (sample_type & PERF_SAMPLE_PERIOD)
  3684. perf_output_put(handle, data->period);
  3685. if (sample_type & PERF_SAMPLE_READ)
  3686. perf_output_read(handle, event);
  3687. if (sample_type & PERF_SAMPLE_CALLCHAIN) {
  3688. if (data->callchain) {
  3689. int size = 1;
  3690. if (data->callchain)
  3691. size += data->callchain->nr;
  3692. size *= sizeof(u64);
  3693. __output_copy(handle, data->callchain, size);
  3694. } else {
  3695. u64 nr = 0;
  3696. perf_output_put(handle, nr);
  3697. }
  3698. }
  3699. if (sample_type & PERF_SAMPLE_RAW) {
  3700. if (data->raw) {
  3701. perf_output_put(handle, data->raw->size);
  3702. __output_copy(handle, data->raw->data,
  3703. data->raw->size);
  3704. } else {
  3705. struct {
  3706. u32 size;
  3707. u32 data;
  3708. } raw = {
  3709. .size = sizeof(u32),
  3710. .data = 0,
  3711. };
  3712. perf_output_put(handle, raw);
  3713. }
  3714. }
  3715. if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
  3716. if (data->br_stack) {
  3717. size_t size;
  3718. size = data->br_stack->nr
  3719. * sizeof(struct perf_branch_entry);
  3720. perf_output_put(handle, data->br_stack->nr);
  3721. perf_output_copy(handle, data->br_stack->entries, size);
  3722. } else {
  3723. /*
  3724. * we always store at least the value of nr
  3725. */
  3726. u64 nr = 0;
  3727. perf_output_put(handle, nr);
  3728. }
  3729. }
  3730. if (sample_type & PERF_SAMPLE_REGS_USER) {
  3731. u64 abi = data->regs_user.abi;
  3732. /*
  3733. * If there are no regs to dump, notice it through
  3734. * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
  3735. */
  3736. perf_output_put(handle, abi);
  3737. if (abi) {
  3738. u64 mask = event->attr.sample_regs_user;
  3739. perf_output_sample_regs(handle,
  3740. data->regs_user.regs,
  3741. mask);
  3742. }
  3743. }
  3744. if (sample_type & PERF_SAMPLE_STACK_USER) {
  3745. perf_output_sample_ustack(handle,
  3746. data->stack_user_size,
  3747. data->regs_user.regs);
  3748. }
  3749. if (sample_type & PERF_SAMPLE_WEIGHT)
  3750. perf_output_put(handle, data->weight);
  3751. if (sample_type & PERF_SAMPLE_DATA_SRC)
  3752. perf_output_put(handle, data->data_src.val);
  3753. if (!event->attr.watermark) {
  3754. int wakeup_events = event->attr.wakeup_events;
  3755. if (wakeup_events) {
  3756. struct ring_buffer *rb = handle->rb;
  3757. int events = local_inc_return(&rb->events);
  3758. if (events >= wakeup_events) {
  3759. local_sub(wakeup_events, &rb->events);
  3760. local_inc(&rb->wakeup);
  3761. }
  3762. }
  3763. }
  3764. }
  3765. void perf_prepare_sample(struct perf_event_header *header,
  3766. struct perf_sample_data *data,
  3767. struct perf_event *event,
  3768. struct pt_regs *regs)
  3769. {
  3770. u64 sample_type = event->attr.sample_type;
  3771. header->type = PERF_RECORD_SAMPLE;
  3772. header->size = sizeof(*header) + event->header_size;
  3773. header->misc = 0;
  3774. header->misc |= perf_misc_flags(regs);
  3775. __perf_event_header__init_id(header, data, event);
  3776. if (sample_type & PERF_SAMPLE_IP)
  3777. data->ip = perf_instruction_pointer(regs);
  3778. if (sample_type & PERF_SAMPLE_CALLCHAIN) {
  3779. int size = 1;
  3780. data->callchain = perf_callchain(event, regs);
  3781. if (data->callchain)
  3782. size += data->callchain->nr;
  3783. header->size += size * sizeof(u64);
  3784. }
  3785. if (sample_type & PERF_SAMPLE_RAW) {
  3786. int size = sizeof(u32);
  3787. if (data->raw)
  3788. size += data->raw->size;
  3789. else
  3790. size += sizeof(u32);
  3791. WARN_ON_ONCE(size & (sizeof(u64)-1));
  3792. header->size += size;
  3793. }
  3794. if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
  3795. int size = sizeof(u64); /* nr */
  3796. if (data->br_stack) {
  3797. size += data->br_stack->nr
  3798. * sizeof(struct perf_branch_entry);
  3799. }
  3800. header->size += size;
  3801. }
  3802. if (sample_type & PERF_SAMPLE_REGS_USER) {
  3803. /* regs dump ABI info */
  3804. int size = sizeof(u64);
  3805. perf_sample_regs_user(&data->regs_user, regs);
  3806. if (data->regs_user.regs) {
  3807. u64 mask = event->attr.sample_regs_user;
  3808. size += hweight64(mask) * sizeof(u64);
  3809. }
  3810. header->size += size;
  3811. }
  3812. if (sample_type & PERF_SAMPLE_STACK_USER) {
  3813. /*
  3814. * Either we need PERF_SAMPLE_STACK_USER bit to be allways
  3815. * processed as the last one or have additional check added
  3816. * in case new sample type is added, because we could eat
  3817. * up the rest of the sample size.
  3818. */
  3819. struct perf_regs_user *uregs = &data->regs_user;
  3820. u16 stack_size = event->attr.sample_stack_user;
  3821. u16 size = sizeof(u64);
  3822. if (!uregs->abi)
  3823. perf_sample_regs_user(uregs, regs);
  3824. stack_size = perf_sample_ustack_size(stack_size, header->size,
  3825. uregs->regs);
  3826. /*
  3827. * If there is something to dump, add space for the dump
  3828. * itself and for the field that tells the dynamic size,
  3829. * which is how many have been actually dumped.
  3830. */
  3831. if (stack_size)
  3832. size += sizeof(u64) + stack_size;
  3833. data->stack_user_size = stack_size;
  3834. header->size += size;
  3835. }
  3836. }
  3837. static void perf_event_output(struct perf_event *event,
  3838. struct perf_sample_data *data,
  3839. struct pt_regs *regs)
  3840. {
  3841. struct perf_output_handle handle;
  3842. struct perf_event_header header;
  3843. /* protect the callchain buffers */
  3844. rcu_read_lock();
  3845. perf_prepare_sample(&header, data, event, regs);
  3846. if (perf_output_begin(&handle, event, header.size))
  3847. goto exit;
  3848. perf_output_sample(&handle, &header, data, event);
  3849. perf_output_end(&handle);
  3850. exit:
  3851. rcu_read_unlock();
  3852. }
  3853. /*
  3854. * read event_id
  3855. */
  3856. struct perf_read_event {
  3857. struct perf_event_header header;
  3858. u32 pid;
  3859. u32 tid;
  3860. };
  3861. static void
  3862. perf_event_read_event(struct perf_event *event,
  3863. struct task_struct *task)
  3864. {
  3865. struct perf_output_handle handle;
  3866. struct perf_sample_data sample;
  3867. struct perf_read_event read_event = {
  3868. .header = {
  3869. .type = PERF_RECORD_READ,
  3870. .misc = 0,
  3871. .size = sizeof(read_event) + event->read_size,
  3872. },
  3873. .pid = perf_event_pid(event, task),
  3874. .tid = perf_event_tid(event, task),
  3875. };
  3876. int ret;
  3877. perf_event_header__init_id(&read_event.header, &sample, event);
  3878. ret = perf_output_begin(&handle, event, read_event.header.size);
  3879. if (ret)
  3880. return;
  3881. perf_output_put(&handle, read_event);
  3882. perf_output_read(&handle, event);
  3883. perf_event__output_id_sample(event, &handle, &sample);
  3884. perf_output_end(&handle);
  3885. }
  3886. typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
  3887. static void
  3888. perf_event_aux_ctx(struct perf_event_context *ctx,
  3889. perf_event_aux_output_cb output,
  3890. void *data)
  3891. {
  3892. struct perf_event *event;
  3893. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  3894. if (event->state < PERF_EVENT_STATE_INACTIVE)
  3895. continue;
  3896. if (!event_filter_match(event))
  3897. continue;
  3898. output(event, data);
  3899. }
  3900. }
  3901. static void
  3902. perf_event_aux(perf_event_aux_output_cb output, void *data,
  3903. struct perf_event_context *task_ctx)
  3904. {
  3905. struct perf_cpu_context *cpuctx;
  3906. struct perf_event_context *ctx;
  3907. struct pmu *pmu;
  3908. int ctxn;
  3909. rcu_read_lock();
  3910. list_for_each_entry_rcu(pmu, &pmus, entry) {
  3911. cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
  3912. if (cpuctx->unique_pmu != pmu)
  3913. goto next;
  3914. perf_event_aux_ctx(&cpuctx->ctx, output, data);
  3915. if (task_ctx)
  3916. goto next;
  3917. ctxn = pmu->task_ctx_nr;
  3918. if (ctxn < 0)
  3919. goto next;
  3920. ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
  3921. if (ctx)
  3922. perf_event_aux_ctx(ctx, output, data);
  3923. next:
  3924. put_cpu_ptr(pmu->pmu_cpu_context);
  3925. }
  3926. if (task_ctx) {
  3927. preempt_disable();
  3928. perf_event_aux_ctx(task_ctx, output, data);
  3929. preempt_enable();
  3930. }
  3931. rcu_read_unlock();
  3932. }
  3933. /*
  3934. * task tracking -- fork/exit
  3935. *
  3936. * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
  3937. */
  3938. struct perf_task_event {
  3939. struct task_struct *task;
  3940. struct perf_event_context *task_ctx;
  3941. struct {
  3942. struct perf_event_header header;
  3943. u32 pid;
  3944. u32 ppid;
  3945. u32 tid;
  3946. u32 ptid;
  3947. u64 time;
  3948. } event_id;
  3949. };
  3950. static int perf_event_task_match(struct perf_event *event)
  3951. {
  3952. return event->attr.comm || event->attr.mmap ||
  3953. event->attr.mmap_data || event->attr.task;
  3954. }
  3955. static void perf_event_task_output(struct perf_event *event,
  3956. void *data)
  3957. {
  3958. struct perf_task_event *task_event = data;
  3959. struct perf_output_handle handle;
  3960. struct perf_sample_data sample;
  3961. struct task_struct *task = task_event->task;
  3962. int ret, size = task_event->event_id.header.size;
  3963. if (!perf_event_task_match(event))
  3964. return;
  3965. perf_event_header__init_id(&task_event->event_id.header, &sample, event);
  3966. ret = perf_output_begin(&handle, event,
  3967. task_event->event_id.header.size);
  3968. if (ret)
  3969. goto out;
  3970. task_event->event_id.pid = perf_event_pid(event, task);
  3971. task_event->event_id.ppid = perf_event_pid(event, current);
  3972. task_event->event_id.tid = perf_event_tid(event, task);
  3973. task_event->event_id.ptid = perf_event_tid(event, current);
  3974. perf_output_put(&handle, task_event->event_id);
  3975. perf_event__output_id_sample(event, &handle, &sample);
  3976. perf_output_end(&handle);
  3977. out:
  3978. task_event->event_id.header.size = size;
  3979. }
  3980. static void perf_event_task(struct task_struct *task,
  3981. struct perf_event_context *task_ctx,
  3982. int new)
  3983. {
  3984. struct perf_task_event task_event;
  3985. if (!atomic_read(&nr_comm_events) &&
  3986. !atomic_read(&nr_mmap_events) &&
  3987. !atomic_read(&nr_task_events))
  3988. return;
  3989. task_event = (struct perf_task_event){
  3990. .task = task,
  3991. .task_ctx = task_ctx,
  3992. .event_id = {
  3993. .header = {
  3994. .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
  3995. .misc = 0,
  3996. .size = sizeof(task_event.event_id),
  3997. },
  3998. /* .pid */
  3999. /* .ppid */
  4000. /* .tid */
  4001. /* .ptid */
  4002. .time = perf_clock(),
  4003. },
  4004. };
  4005. perf_event_aux(perf_event_task_output,
  4006. &task_event,
  4007. task_ctx);
  4008. }
  4009. void perf_event_fork(struct task_struct *task)
  4010. {
  4011. perf_event_task(task, NULL, 1);
  4012. }
  4013. /*
  4014. * comm tracking
  4015. */
  4016. struct perf_comm_event {
  4017. struct task_struct *task;
  4018. char *comm;
  4019. int comm_size;
  4020. struct {
  4021. struct perf_event_header header;
  4022. u32 pid;
  4023. u32 tid;
  4024. } event_id;
  4025. };
  4026. static int perf_event_comm_match(struct perf_event *event)
  4027. {
  4028. return event->attr.comm;
  4029. }
  4030. static void perf_event_comm_output(struct perf_event *event,
  4031. void *data)
  4032. {
  4033. struct perf_comm_event *comm_event = data;
  4034. struct perf_output_handle handle;
  4035. struct perf_sample_data sample;
  4036. int size = comm_event->event_id.header.size;
  4037. int ret;
  4038. if (!perf_event_comm_match(event))
  4039. return;
  4040. perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
  4041. ret = perf_output_begin(&handle, event,
  4042. comm_event->event_id.header.size);
  4043. if (ret)
  4044. goto out;
  4045. comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
  4046. comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
  4047. perf_output_put(&handle, comm_event->event_id);
  4048. __output_copy(&handle, comm_event->comm,
  4049. comm_event->comm_size);
  4050. perf_event__output_id_sample(event, &handle, &sample);
  4051. perf_output_end(&handle);
  4052. out:
  4053. comm_event->event_id.header.size = size;
  4054. }
  4055. static void perf_event_comm_event(struct perf_comm_event *comm_event)
  4056. {
  4057. char comm[TASK_COMM_LEN];
  4058. unsigned int size;
  4059. memset(comm, 0, sizeof(comm));
  4060. strlcpy(comm, comm_event->task->comm, sizeof(comm));
  4061. size = ALIGN(strlen(comm)+1, sizeof(u64));
  4062. comm_event->comm = comm;
  4063. comm_event->comm_size = size;
  4064. comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
  4065. perf_event_aux(perf_event_comm_output,
  4066. comm_event,
  4067. NULL);
  4068. }
  4069. void perf_event_comm(struct task_struct *task)
  4070. {
  4071. struct perf_comm_event comm_event;
  4072. struct perf_event_context *ctx;
  4073. int ctxn;
  4074. rcu_read_lock();
  4075. for_each_task_context_nr(ctxn) {
  4076. ctx = task->perf_event_ctxp[ctxn];
  4077. if (!ctx)
  4078. continue;
  4079. perf_event_enable_on_exec(ctx);
  4080. }
  4081. rcu_read_unlock();
  4082. if (!atomic_read(&nr_comm_events))
  4083. return;
  4084. comm_event = (struct perf_comm_event){
  4085. .task = task,
  4086. /* .comm */
  4087. /* .comm_size */
  4088. .event_id = {
  4089. .header = {
  4090. .type = PERF_RECORD_COMM,
  4091. .misc = 0,
  4092. /* .size */
  4093. },
  4094. /* .pid */
  4095. /* .tid */
  4096. },
  4097. };
  4098. perf_event_comm_event(&comm_event);
  4099. }
  4100. /*
  4101. * mmap tracking
  4102. */
  4103. struct perf_mmap_event {
  4104. struct vm_area_struct *vma;
  4105. const char *file_name;
  4106. int file_size;
  4107. struct {
  4108. struct perf_event_header header;
  4109. u32 pid;
  4110. u32 tid;
  4111. u64 start;
  4112. u64 len;
  4113. u64 pgoff;
  4114. } event_id;
  4115. };
  4116. static int perf_event_mmap_match(struct perf_event *event,
  4117. void *data)
  4118. {
  4119. struct perf_mmap_event *mmap_event = data;
  4120. struct vm_area_struct *vma = mmap_event->vma;
  4121. int executable = vma->vm_flags & VM_EXEC;
  4122. return (!executable && event->attr.mmap_data) ||
  4123. (executable && event->attr.mmap);
  4124. }
  4125. static void perf_event_mmap_output(struct perf_event *event,
  4126. void *data)
  4127. {
  4128. struct perf_mmap_event *mmap_event = data;
  4129. struct perf_output_handle handle;
  4130. struct perf_sample_data sample;
  4131. int size = mmap_event->event_id.header.size;
  4132. int ret;
  4133. if (!perf_event_mmap_match(event, data))
  4134. return;
  4135. perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
  4136. ret = perf_output_begin(&handle, event,
  4137. mmap_event->event_id.header.size);
  4138. if (ret)
  4139. goto out;
  4140. mmap_event->event_id.pid = perf_event_pid(event, current);
  4141. mmap_event->event_id.tid = perf_event_tid(event, current);
  4142. perf_output_put(&handle, mmap_event->event_id);
  4143. __output_copy(&handle, mmap_event->file_name,
  4144. mmap_event->file_size);
  4145. perf_event__output_id_sample(event, &handle, &sample);
  4146. perf_output_end(&handle);
  4147. out:
  4148. mmap_event->event_id.header.size = size;
  4149. }
  4150. static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
  4151. {
  4152. struct vm_area_struct *vma = mmap_event->vma;
  4153. struct file *file = vma->vm_file;
  4154. unsigned int size;
  4155. char tmp[16];
  4156. char *buf = NULL;
  4157. const char *name;
  4158. memset(tmp, 0, sizeof(tmp));
  4159. if (file) {
  4160. /*
  4161. * d_path works from the end of the rb backwards, so we
  4162. * need to add enough zero bytes after the string to handle
  4163. * the 64bit alignment we do later.
  4164. */
  4165. buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
  4166. if (!buf) {
  4167. name = strncpy(tmp, "//enomem", sizeof(tmp));
  4168. goto got_name;
  4169. }
  4170. name = d_path(&file->f_path, buf, PATH_MAX);
  4171. if (IS_ERR(name)) {
  4172. name = strncpy(tmp, "//toolong", sizeof(tmp));
  4173. goto got_name;
  4174. }
  4175. } else {
  4176. if (arch_vma_name(mmap_event->vma)) {
  4177. name = strncpy(tmp, arch_vma_name(mmap_event->vma),
  4178. sizeof(tmp) - 1);
  4179. tmp[sizeof(tmp) - 1] = '\0';
  4180. goto got_name;
  4181. }
  4182. if (!vma->vm_mm) {
  4183. name = strncpy(tmp, "[vdso]", sizeof(tmp));
  4184. goto got_name;
  4185. } else if (vma->vm_start <= vma->vm_mm->start_brk &&
  4186. vma->vm_end >= vma->vm_mm->brk) {
  4187. name = strncpy(tmp, "[heap]", sizeof(tmp));
  4188. goto got_name;
  4189. } else if (vma->vm_start <= vma->vm_mm->start_stack &&
  4190. vma->vm_end >= vma->vm_mm->start_stack) {
  4191. name = strncpy(tmp, "[stack]", sizeof(tmp));
  4192. goto got_name;
  4193. }
  4194. name = strncpy(tmp, "//anon", sizeof(tmp));
  4195. goto got_name;
  4196. }
  4197. got_name:
  4198. size = ALIGN(strlen(name)+1, sizeof(u64));
  4199. mmap_event->file_name = name;
  4200. mmap_event->file_size = size;
  4201. if (!(vma->vm_flags & VM_EXEC))
  4202. mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
  4203. mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
  4204. perf_event_aux(perf_event_mmap_output,
  4205. mmap_event,
  4206. NULL);
  4207. kfree(buf);
  4208. }
  4209. void perf_event_mmap(struct vm_area_struct *vma)
  4210. {
  4211. struct perf_mmap_event mmap_event;
  4212. if (!atomic_read(&nr_mmap_events))
  4213. return;
  4214. mmap_event = (struct perf_mmap_event){
  4215. .vma = vma,
  4216. /* .file_name */
  4217. /* .file_size */
  4218. .event_id = {
  4219. .header = {
  4220. .type = PERF_RECORD_MMAP,
  4221. .misc = PERF_RECORD_MISC_USER,
  4222. /* .size */
  4223. },
  4224. /* .pid */
  4225. /* .tid */
  4226. .start = vma->vm_start,
  4227. .len = vma->vm_end - vma->vm_start,
  4228. .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
  4229. },
  4230. };
  4231. perf_event_mmap_event(&mmap_event);
  4232. }
  4233. /*
  4234. * IRQ throttle logging
  4235. */
  4236. static void perf_log_throttle(struct perf_event *event, int enable)
  4237. {
  4238. struct perf_output_handle handle;
  4239. struct perf_sample_data sample;
  4240. int ret;
  4241. struct {
  4242. struct perf_event_header header;
  4243. u64 time;
  4244. u64 id;
  4245. u64 stream_id;
  4246. } throttle_event = {
  4247. .header = {
  4248. .type = PERF_RECORD_THROTTLE,
  4249. .misc = 0,
  4250. .size = sizeof(throttle_event),
  4251. },
  4252. .time = perf_clock(),
  4253. .id = primary_event_id(event),
  4254. .stream_id = event->id,
  4255. };
  4256. if (enable)
  4257. throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
  4258. perf_event_header__init_id(&throttle_event.header, &sample, event);
  4259. ret = perf_output_begin(&handle, event,
  4260. throttle_event.header.size);
  4261. if (ret)
  4262. return;
  4263. perf_output_put(&handle, throttle_event);
  4264. perf_event__output_id_sample(event, &handle, &sample);
  4265. perf_output_end(&handle);
  4266. }
  4267. /*
  4268. * Generic event overflow handling, sampling.
  4269. */
  4270. static int __perf_event_overflow(struct perf_event *event,
  4271. int throttle, struct perf_sample_data *data,
  4272. struct pt_regs *regs)
  4273. {
  4274. int events = atomic_read(&event->event_limit);
  4275. struct hw_perf_event *hwc = &event->hw;
  4276. u64 seq;
  4277. int ret = 0;
  4278. /*
  4279. * Non-sampling counters might still use the PMI to fold short
  4280. * hardware counters, ignore those.
  4281. */
  4282. if (unlikely(!is_sampling_event(event)))
  4283. return 0;
  4284. seq = __this_cpu_read(perf_throttled_seq);
  4285. if (seq != hwc->interrupts_seq) {
  4286. hwc->interrupts_seq = seq;
  4287. hwc->interrupts = 1;
  4288. } else {
  4289. hwc->interrupts++;
  4290. if (unlikely(throttle
  4291. && hwc->interrupts >= max_samples_per_tick)) {
  4292. __this_cpu_inc(perf_throttled_count);
  4293. hwc->interrupts = MAX_INTERRUPTS;
  4294. perf_log_throttle(event, 0);
  4295. tick_nohz_full_kick();
  4296. ret = 1;
  4297. }
  4298. }
  4299. if (event->attr.freq) {
  4300. u64 now = perf_clock();
  4301. s64 delta = now - hwc->freq_time_stamp;
  4302. hwc->freq_time_stamp = now;
  4303. if (delta > 0 && delta < 2*TICK_NSEC)
  4304. perf_adjust_period(event, delta, hwc->last_period, true);
  4305. }
  4306. /*
  4307. * XXX event_limit might not quite work as expected on inherited
  4308. * events
  4309. */
  4310. event->pending_kill = POLL_IN;
  4311. if (events && atomic_dec_and_test(&event->event_limit)) {
  4312. ret = 1;
  4313. event->pending_kill = POLL_HUP;
  4314. event->pending_disable = 1;
  4315. irq_work_queue(&event->pending);
  4316. }
  4317. if (event->overflow_handler)
  4318. event->overflow_handler(event, data, regs);
  4319. else
  4320. perf_event_output(event, data, regs);
  4321. if (event->fasync && event->pending_kill) {
  4322. event->pending_wakeup = 1;
  4323. irq_work_queue(&event->pending);
  4324. }
  4325. return ret;
  4326. }
  4327. int perf_event_overflow(struct perf_event *event,
  4328. struct perf_sample_data *data,
  4329. struct pt_regs *regs)
  4330. {
  4331. return __perf_event_overflow(event, 1, data, regs);
  4332. }
  4333. /*
  4334. * Generic software event infrastructure
  4335. */
  4336. struct swevent_htable {
  4337. struct swevent_hlist *swevent_hlist;
  4338. struct mutex hlist_mutex;
  4339. int hlist_refcount;
  4340. /* Recursion avoidance in each contexts */
  4341. int recursion[PERF_NR_CONTEXTS];
  4342. };
  4343. static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
  4344. /*
  4345. * We directly increment event->count and keep a second value in
  4346. * event->hw.period_left to count intervals. This period event
  4347. * is kept in the range [-sample_period, 0] so that we can use the
  4348. * sign as trigger.
  4349. */
  4350. u64 perf_swevent_set_period(struct perf_event *event)
  4351. {
  4352. struct hw_perf_event *hwc = &event->hw;
  4353. u64 period = hwc->last_period;
  4354. u64 nr, offset;
  4355. s64 old, val;
  4356. hwc->last_period = hwc->sample_period;
  4357. again:
  4358. old = val = local64_read(&hwc->period_left);
  4359. if (val < 0)
  4360. return 0;
  4361. nr = div64_u64(period + val, period);
  4362. offset = nr * period;
  4363. val -= offset;
  4364. if (local64_cmpxchg(&hwc->period_left, old, val) != old)
  4365. goto again;
  4366. return nr;
  4367. }
  4368. static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
  4369. struct perf_sample_data *data,
  4370. struct pt_regs *regs)
  4371. {
  4372. struct hw_perf_event *hwc = &event->hw;
  4373. int throttle = 0;
  4374. if (!overflow)
  4375. overflow = perf_swevent_set_period(event);
  4376. if (hwc->interrupts == MAX_INTERRUPTS)
  4377. return;
  4378. for (; overflow; overflow--) {
  4379. if (__perf_event_overflow(event, throttle,
  4380. data, regs)) {
  4381. /*
  4382. * We inhibit the overflow from happening when
  4383. * hwc->interrupts == MAX_INTERRUPTS.
  4384. */
  4385. break;
  4386. }
  4387. throttle = 1;
  4388. }
  4389. }
  4390. static void perf_swevent_event(struct perf_event *event, u64 nr,
  4391. struct perf_sample_data *data,
  4392. struct pt_regs *regs)
  4393. {
  4394. struct hw_perf_event *hwc = &event->hw;
  4395. local64_add(nr, &event->count);
  4396. if (!regs)
  4397. return;
  4398. if (!is_sampling_event(event))
  4399. return;
  4400. if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
  4401. data->period = nr;
  4402. return perf_swevent_overflow(event, 1, data, regs);
  4403. } else
  4404. data->period = event->hw.last_period;
  4405. if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
  4406. return perf_swevent_overflow(event, 1, data, regs);
  4407. if (local64_add_negative(nr, &hwc->period_left))
  4408. return;
  4409. perf_swevent_overflow(event, 0, data, regs);
  4410. }
  4411. static int perf_exclude_event(struct perf_event *event,
  4412. struct pt_regs *regs)
  4413. {
  4414. if (event->hw.state & PERF_HES_STOPPED)
  4415. return 1;
  4416. if (regs) {
  4417. if (event->attr.exclude_user && user_mode(regs))
  4418. return 1;
  4419. if (event->attr.exclude_kernel && !user_mode(regs))
  4420. return 1;
  4421. }
  4422. return 0;
  4423. }
  4424. static int perf_swevent_match(struct perf_event *event,
  4425. enum perf_type_id type,
  4426. u32 event_id,
  4427. struct perf_sample_data *data,
  4428. struct pt_regs *regs)
  4429. {
  4430. if (event->attr.type != type)
  4431. return 0;
  4432. if (event->attr.config != event_id)
  4433. return 0;
  4434. if (perf_exclude_event(event, regs))
  4435. return 0;
  4436. return 1;
  4437. }
  4438. static inline u64 swevent_hash(u64 type, u32 event_id)
  4439. {
  4440. u64 val = event_id | (type << 32);
  4441. return hash_64(val, SWEVENT_HLIST_BITS);
  4442. }
  4443. static inline struct hlist_head *
  4444. __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
  4445. {
  4446. u64 hash = swevent_hash(type, event_id);
  4447. return &hlist->heads[hash];
  4448. }
  4449. /* For the read side: events when they trigger */
  4450. static inline struct hlist_head *
  4451. find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
  4452. {
  4453. struct swevent_hlist *hlist;
  4454. hlist = rcu_dereference(swhash->swevent_hlist);
  4455. if (!hlist)
  4456. return NULL;
  4457. return __find_swevent_head(hlist, type, event_id);
  4458. }
  4459. /* For the event head insertion and removal in the hlist */
  4460. static inline struct hlist_head *
  4461. find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
  4462. {
  4463. struct swevent_hlist *hlist;
  4464. u32 event_id = event->attr.config;
  4465. u64 type = event->attr.type;
  4466. /*
  4467. * Event scheduling is always serialized against hlist allocation
  4468. * and release. Which makes the protected version suitable here.
  4469. * The context lock guarantees that.
  4470. */
  4471. hlist = rcu_dereference_protected(swhash->swevent_hlist,
  4472. lockdep_is_held(&event->ctx->lock));
  4473. if (!hlist)
  4474. return NULL;
  4475. return __find_swevent_head(hlist, type, event_id);
  4476. }
  4477. static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
  4478. u64 nr,
  4479. struct perf_sample_data *data,
  4480. struct pt_regs *regs)
  4481. {
  4482. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4483. struct perf_event *event;
  4484. struct hlist_head *head;
  4485. rcu_read_lock();
  4486. head = find_swevent_head_rcu(swhash, type, event_id);
  4487. if (!head)
  4488. goto end;
  4489. hlist_for_each_entry_rcu(event, head, hlist_entry) {
  4490. if (perf_swevent_match(event, type, event_id, data, regs))
  4491. perf_swevent_event(event, nr, data, regs);
  4492. }
  4493. end:
  4494. rcu_read_unlock();
  4495. }
  4496. int perf_swevent_get_recursion_context(void)
  4497. {
  4498. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4499. return get_recursion_context(swhash->recursion);
  4500. }
  4501. EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
  4502. inline void perf_swevent_put_recursion_context(int rctx)
  4503. {
  4504. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4505. put_recursion_context(swhash->recursion, rctx);
  4506. }
  4507. void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
  4508. {
  4509. struct perf_sample_data data;
  4510. int rctx;
  4511. preempt_disable_notrace();
  4512. rctx = perf_swevent_get_recursion_context();
  4513. if (rctx < 0)
  4514. return;
  4515. perf_sample_data_init(&data, addr, 0);
  4516. do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
  4517. perf_swevent_put_recursion_context(rctx);
  4518. preempt_enable_notrace();
  4519. }
  4520. static void perf_swevent_read(struct perf_event *event)
  4521. {
  4522. }
  4523. static int perf_swevent_add(struct perf_event *event, int flags)
  4524. {
  4525. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4526. struct hw_perf_event *hwc = &event->hw;
  4527. struct hlist_head *head;
  4528. if (is_sampling_event(event)) {
  4529. hwc->last_period = hwc->sample_period;
  4530. perf_swevent_set_period(event);
  4531. }
  4532. hwc->state = !(flags & PERF_EF_START);
  4533. head = find_swevent_head(swhash, event);
  4534. if (WARN_ON_ONCE(!head))
  4535. return -EINVAL;
  4536. hlist_add_head_rcu(&event->hlist_entry, head);
  4537. return 0;
  4538. }
  4539. static void perf_swevent_del(struct perf_event *event, int flags)
  4540. {
  4541. hlist_del_rcu(&event->hlist_entry);
  4542. }
  4543. static void perf_swevent_start(struct perf_event *event, int flags)
  4544. {
  4545. event->hw.state = 0;
  4546. }
  4547. static void perf_swevent_stop(struct perf_event *event, int flags)
  4548. {
  4549. event->hw.state = PERF_HES_STOPPED;
  4550. }
  4551. /* Deref the hlist from the update side */
  4552. static inline struct swevent_hlist *
  4553. swevent_hlist_deref(struct swevent_htable *swhash)
  4554. {
  4555. return rcu_dereference_protected(swhash->swevent_hlist,
  4556. lockdep_is_held(&swhash->hlist_mutex));
  4557. }
  4558. static void swevent_hlist_release(struct swevent_htable *swhash)
  4559. {
  4560. struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
  4561. if (!hlist)
  4562. return;
  4563. rcu_assign_pointer(swhash->swevent_hlist, NULL);
  4564. kfree_rcu(hlist, rcu_head);
  4565. }
  4566. static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
  4567. {
  4568. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  4569. mutex_lock(&swhash->hlist_mutex);
  4570. if (!--swhash->hlist_refcount)
  4571. swevent_hlist_release(swhash);
  4572. mutex_unlock(&swhash->hlist_mutex);
  4573. }
  4574. static void swevent_hlist_put(struct perf_event *event)
  4575. {
  4576. int cpu;
  4577. if (event->cpu != -1) {
  4578. swevent_hlist_put_cpu(event, event->cpu);
  4579. return;
  4580. }
  4581. for_each_possible_cpu(cpu)
  4582. swevent_hlist_put_cpu(event, cpu);
  4583. }
  4584. static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
  4585. {
  4586. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  4587. int err = 0;
  4588. mutex_lock(&swhash->hlist_mutex);
  4589. if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
  4590. struct swevent_hlist *hlist;
  4591. hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
  4592. if (!hlist) {
  4593. err = -ENOMEM;
  4594. goto exit;
  4595. }
  4596. rcu_assign_pointer(swhash->swevent_hlist, hlist);
  4597. }
  4598. swhash->hlist_refcount++;
  4599. exit:
  4600. mutex_unlock(&swhash->hlist_mutex);
  4601. return err;
  4602. }
  4603. static int swevent_hlist_get(struct perf_event *event)
  4604. {
  4605. int err;
  4606. int cpu, failed_cpu;
  4607. if (event->cpu != -1)
  4608. return swevent_hlist_get_cpu(event, event->cpu);
  4609. get_online_cpus();
  4610. for_each_possible_cpu(cpu) {
  4611. err = swevent_hlist_get_cpu(event, cpu);
  4612. if (err) {
  4613. failed_cpu = cpu;
  4614. goto fail;
  4615. }
  4616. }
  4617. put_online_cpus();
  4618. return 0;
  4619. fail:
  4620. for_each_possible_cpu(cpu) {
  4621. if (cpu == failed_cpu)
  4622. break;
  4623. swevent_hlist_put_cpu(event, cpu);
  4624. }
  4625. put_online_cpus();
  4626. return err;
  4627. }
  4628. struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
  4629. static void sw_perf_event_destroy(struct perf_event *event)
  4630. {
  4631. u64 event_id = event->attr.config;
  4632. WARN_ON(event->parent);
  4633. static_key_slow_dec(&perf_swevent_enabled[event_id]);
  4634. swevent_hlist_put(event);
  4635. }
  4636. static int perf_swevent_init(struct perf_event *event)
  4637. {
  4638. u64 event_id = event->attr.config;
  4639. if (event->attr.type != PERF_TYPE_SOFTWARE)
  4640. return -ENOENT;
  4641. /*
  4642. * no branch sampling for software events
  4643. */
  4644. if (has_branch_stack(event))
  4645. return -EOPNOTSUPP;
  4646. switch (event_id) {
  4647. case PERF_COUNT_SW_CPU_CLOCK:
  4648. case PERF_COUNT_SW_TASK_CLOCK:
  4649. return -ENOENT;
  4650. default:
  4651. break;
  4652. }
  4653. if (event_id >= PERF_COUNT_SW_MAX)
  4654. return -ENOENT;
  4655. if (!event->parent) {
  4656. int err;
  4657. err = swevent_hlist_get(event);
  4658. if (err)
  4659. return err;
  4660. static_key_slow_inc(&perf_swevent_enabled[event_id]);
  4661. event->destroy = sw_perf_event_destroy;
  4662. }
  4663. return 0;
  4664. }
  4665. static int perf_swevent_event_idx(struct perf_event *event)
  4666. {
  4667. return 0;
  4668. }
  4669. static struct pmu perf_swevent = {
  4670. .task_ctx_nr = perf_sw_context,
  4671. .event_init = perf_swevent_init,
  4672. .add = perf_swevent_add,
  4673. .del = perf_swevent_del,
  4674. .start = perf_swevent_start,
  4675. .stop = perf_swevent_stop,
  4676. .read = perf_swevent_read,
  4677. .event_idx = perf_swevent_event_idx,
  4678. };
  4679. #ifdef CONFIG_EVENT_TRACING
  4680. static int perf_tp_filter_match(struct perf_event *event,
  4681. struct perf_sample_data *data)
  4682. {
  4683. void *record = data->raw->data;
  4684. if (likely(!event->filter) || filter_match_preds(event->filter, record))
  4685. return 1;
  4686. return 0;
  4687. }
  4688. static int perf_tp_event_match(struct perf_event *event,
  4689. struct perf_sample_data *data,
  4690. struct pt_regs *regs)
  4691. {
  4692. if (event->hw.state & PERF_HES_STOPPED)
  4693. return 0;
  4694. /*
  4695. * All tracepoints are from kernel-space.
  4696. */
  4697. if (event->attr.exclude_kernel)
  4698. return 0;
  4699. if (!perf_tp_filter_match(event, data))
  4700. return 0;
  4701. return 1;
  4702. }
  4703. void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
  4704. struct pt_regs *regs, struct hlist_head *head, int rctx,
  4705. struct task_struct *task)
  4706. {
  4707. struct perf_sample_data data;
  4708. struct perf_event *event;
  4709. struct perf_raw_record raw = {
  4710. .size = entry_size,
  4711. .data = record,
  4712. };
  4713. perf_sample_data_init(&data, addr, 0);
  4714. data.raw = &raw;
  4715. hlist_for_each_entry_rcu(event, head, hlist_entry) {
  4716. if (perf_tp_event_match(event, &data, regs))
  4717. perf_swevent_event(event, count, &data, regs);
  4718. }
  4719. /*
  4720. * If we got specified a target task, also iterate its context and
  4721. * deliver this event there too.
  4722. */
  4723. if (task && task != current) {
  4724. struct perf_event_context *ctx;
  4725. struct trace_entry *entry = record;
  4726. rcu_read_lock();
  4727. ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
  4728. if (!ctx)
  4729. goto unlock;
  4730. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  4731. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  4732. continue;
  4733. if (event->attr.config != entry->type)
  4734. continue;
  4735. if (perf_tp_event_match(event, &data, regs))
  4736. perf_swevent_event(event, count, &data, regs);
  4737. }
  4738. unlock:
  4739. rcu_read_unlock();
  4740. }
  4741. perf_swevent_put_recursion_context(rctx);
  4742. }
  4743. EXPORT_SYMBOL_GPL(perf_tp_event);
  4744. static void tp_perf_event_destroy(struct perf_event *event)
  4745. {
  4746. perf_trace_destroy(event);
  4747. }
  4748. static int perf_tp_event_init(struct perf_event *event)
  4749. {
  4750. int err;
  4751. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  4752. return -ENOENT;
  4753. /*
  4754. * no branch sampling for tracepoint events
  4755. */
  4756. if (has_branch_stack(event))
  4757. return -EOPNOTSUPP;
  4758. err = perf_trace_init(event);
  4759. if (err)
  4760. return err;
  4761. event->destroy = tp_perf_event_destroy;
  4762. return 0;
  4763. }
  4764. static struct pmu perf_tracepoint = {
  4765. .task_ctx_nr = perf_sw_context,
  4766. .event_init = perf_tp_event_init,
  4767. .add = perf_trace_add,
  4768. .del = perf_trace_del,
  4769. .start = perf_swevent_start,
  4770. .stop = perf_swevent_stop,
  4771. .read = perf_swevent_read,
  4772. .event_idx = perf_swevent_event_idx,
  4773. };
  4774. static inline void perf_tp_register(void)
  4775. {
  4776. perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
  4777. }
  4778. static int perf_event_set_filter(struct perf_event *event, void __user *arg)
  4779. {
  4780. char *filter_str;
  4781. int ret;
  4782. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  4783. return -EINVAL;
  4784. filter_str = strndup_user(arg, PAGE_SIZE);
  4785. if (IS_ERR(filter_str))
  4786. return PTR_ERR(filter_str);
  4787. ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
  4788. kfree(filter_str);
  4789. return ret;
  4790. }
  4791. static void perf_event_free_filter(struct perf_event *event)
  4792. {
  4793. ftrace_profile_free_filter(event);
  4794. }
  4795. #else
  4796. static inline void perf_tp_register(void)
  4797. {
  4798. }
  4799. static int perf_event_set_filter(struct perf_event *event, void __user *arg)
  4800. {
  4801. return -ENOENT;
  4802. }
  4803. static void perf_event_free_filter(struct perf_event *event)
  4804. {
  4805. }
  4806. #endif /* CONFIG_EVENT_TRACING */
  4807. #ifdef CONFIG_HAVE_HW_BREAKPOINT
  4808. void perf_bp_event(struct perf_event *bp, void *data)
  4809. {
  4810. struct perf_sample_data sample;
  4811. struct pt_regs *regs = data;
  4812. perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
  4813. if (!bp->hw.state && !perf_exclude_event(bp, regs))
  4814. perf_swevent_event(bp, 1, &sample, regs);
  4815. }
  4816. #endif
  4817. /*
  4818. * hrtimer based swevent callback
  4819. */
  4820. static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
  4821. {
  4822. enum hrtimer_restart ret = HRTIMER_RESTART;
  4823. struct perf_sample_data data;
  4824. struct pt_regs *regs;
  4825. struct perf_event *event;
  4826. u64 period;
  4827. event = container_of(hrtimer, struct perf_event, hw.hrtimer);
  4828. if (event->state != PERF_EVENT_STATE_ACTIVE)
  4829. return HRTIMER_NORESTART;
  4830. event->pmu->read(event);
  4831. perf_sample_data_init(&data, 0, event->hw.last_period);
  4832. regs = get_irq_regs();
  4833. if (regs && !perf_exclude_event(event, regs)) {
  4834. if (!(event->attr.exclude_idle && is_idle_task(current)))
  4835. if (__perf_event_overflow(event, 1, &data, regs))
  4836. ret = HRTIMER_NORESTART;
  4837. }
  4838. period = max_t(u64, 10000, event->hw.sample_period);
  4839. hrtimer_forward_now(hrtimer, ns_to_ktime(period));
  4840. return ret;
  4841. }
  4842. static void perf_swevent_start_hrtimer(struct perf_event *event)
  4843. {
  4844. struct hw_perf_event *hwc = &event->hw;
  4845. s64 period;
  4846. if (!is_sampling_event(event))
  4847. return;
  4848. period = local64_read(&hwc->period_left);
  4849. if (period) {
  4850. if (period < 0)
  4851. period = 10000;
  4852. local64_set(&hwc->period_left, 0);
  4853. } else {
  4854. period = max_t(u64, 10000, hwc->sample_period);
  4855. }
  4856. __hrtimer_start_range_ns(&hwc->hrtimer,
  4857. ns_to_ktime(period), 0,
  4858. HRTIMER_MODE_REL_PINNED, 0);
  4859. }
  4860. static void perf_swevent_cancel_hrtimer(struct perf_event *event)
  4861. {
  4862. struct hw_perf_event *hwc = &event->hw;
  4863. if (is_sampling_event(event)) {
  4864. ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
  4865. local64_set(&hwc->period_left, ktime_to_ns(remaining));
  4866. hrtimer_cancel(&hwc->hrtimer);
  4867. }
  4868. }
  4869. static void perf_swevent_init_hrtimer(struct perf_event *event)
  4870. {
  4871. struct hw_perf_event *hwc = &event->hw;
  4872. if (!is_sampling_event(event))
  4873. return;
  4874. hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  4875. hwc->hrtimer.function = perf_swevent_hrtimer;
  4876. /*
  4877. * Since hrtimers have a fixed rate, we can do a static freq->period
  4878. * mapping and avoid the whole period adjust feedback stuff.
  4879. */
  4880. if (event->attr.freq) {
  4881. long freq = event->attr.sample_freq;
  4882. event->attr.sample_period = NSEC_PER_SEC / freq;
  4883. hwc->sample_period = event->attr.sample_period;
  4884. local64_set(&hwc->period_left, hwc->sample_period);
  4885. hwc->last_period = hwc->sample_period;
  4886. event->attr.freq = 0;
  4887. }
  4888. }
  4889. /*
  4890. * Software event: cpu wall time clock
  4891. */
  4892. static void cpu_clock_event_update(struct perf_event *event)
  4893. {
  4894. s64 prev;
  4895. u64 now;
  4896. now = local_clock();
  4897. prev = local64_xchg(&event->hw.prev_count, now);
  4898. local64_add(now - prev, &event->count);
  4899. }
  4900. static void cpu_clock_event_start(struct perf_event *event, int flags)
  4901. {
  4902. local64_set(&event->hw.prev_count, local_clock());
  4903. perf_swevent_start_hrtimer(event);
  4904. }
  4905. static void cpu_clock_event_stop(struct perf_event *event, int flags)
  4906. {
  4907. perf_swevent_cancel_hrtimer(event);
  4908. cpu_clock_event_update(event);
  4909. }
  4910. static int cpu_clock_event_add(struct perf_event *event, int flags)
  4911. {
  4912. if (flags & PERF_EF_START)
  4913. cpu_clock_event_start(event, flags);
  4914. return 0;
  4915. }
  4916. static void cpu_clock_event_del(struct perf_event *event, int flags)
  4917. {
  4918. cpu_clock_event_stop(event, flags);
  4919. }
  4920. static void cpu_clock_event_read(struct perf_event *event)
  4921. {
  4922. cpu_clock_event_update(event);
  4923. }
  4924. static int cpu_clock_event_init(struct perf_event *event)
  4925. {
  4926. if (event->attr.type != PERF_TYPE_SOFTWARE)
  4927. return -ENOENT;
  4928. if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
  4929. return -ENOENT;
  4930. /*
  4931. * no branch sampling for software events
  4932. */
  4933. if (has_branch_stack(event))
  4934. return -EOPNOTSUPP;
  4935. perf_swevent_init_hrtimer(event);
  4936. return 0;
  4937. }
  4938. static struct pmu perf_cpu_clock = {
  4939. .task_ctx_nr = perf_sw_context,
  4940. .event_init = cpu_clock_event_init,
  4941. .add = cpu_clock_event_add,
  4942. .del = cpu_clock_event_del,
  4943. .start = cpu_clock_event_start,
  4944. .stop = cpu_clock_event_stop,
  4945. .read = cpu_clock_event_read,
  4946. .event_idx = perf_swevent_event_idx,
  4947. };
  4948. /*
  4949. * Software event: task time clock
  4950. */
  4951. static void task_clock_event_update(struct perf_event *event, u64 now)
  4952. {
  4953. u64 prev;
  4954. s64 delta;
  4955. prev = local64_xchg(&event->hw.prev_count, now);
  4956. delta = now - prev;
  4957. local64_add(delta, &event->count);
  4958. }
  4959. static void task_clock_event_start(struct perf_event *event, int flags)
  4960. {
  4961. local64_set(&event->hw.prev_count, event->ctx->time);
  4962. perf_swevent_start_hrtimer(event);
  4963. }
  4964. static void task_clock_event_stop(struct perf_event *event, int flags)
  4965. {
  4966. perf_swevent_cancel_hrtimer(event);
  4967. task_clock_event_update(event, event->ctx->time);
  4968. }
  4969. static int task_clock_event_add(struct perf_event *event, int flags)
  4970. {
  4971. if (flags & PERF_EF_START)
  4972. task_clock_event_start(event, flags);
  4973. return 0;
  4974. }
  4975. static void task_clock_event_del(struct perf_event *event, int flags)
  4976. {
  4977. task_clock_event_stop(event, PERF_EF_UPDATE);
  4978. }
  4979. static void task_clock_event_read(struct perf_event *event)
  4980. {
  4981. u64 now = perf_clock();
  4982. u64 delta = now - event->ctx->timestamp;
  4983. u64 time = event->ctx->time + delta;
  4984. task_clock_event_update(event, time);
  4985. }
  4986. static int task_clock_event_init(struct perf_event *event)
  4987. {
  4988. if (event->attr.type != PERF_TYPE_SOFTWARE)
  4989. return -ENOENT;
  4990. if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
  4991. return -ENOENT;
  4992. /*
  4993. * no branch sampling for software events
  4994. */
  4995. if (has_branch_stack(event))
  4996. return -EOPNOTSUPP;
  4997. perf_swevent_init_hrtimer(event);
  4998. return 0;
  4999. }
  5000. static struct pmu perf_task_clock = {
  5001. .task_ctx_nr = perf_sw_context,
  5002. .event_init = task_clock_event_init,
  5003. .add = task_clock_event_add,
  5004. .del = task_clock_event_del,
  5005. .start = task_clock_event_start,
  5006. .stop = task_clock_event_stop,
  5007. .read = task_clock_event_read,
  5008. .event_idx = perf_swevent_event_idx,
  5009. };
  5010. static void perf_pmu_nop_void(struct pmu *pmu)
  5011. {
  5012. }
  5013. static int perf_pmu_nop_int(struct pmu *pmu)
  5014. {
  5015. return 0;
  5016. }
  5017. static void perf_pmu_start_txn(struct pmu *pmu)
  5018. {
  5019. perf_pmu_disable(pmu);
  5020. }
  5021. static int perf_pmu_commit_txn(struct pmu *pmu)
  5022. {
  5023. perf_pmu_enable(pmu);
  5024. return 0;
  5025. }
  5026. static void perf_pmu_cancel_txn(struct pmu *pmu)
  5027. {
  5028. perf_pmu_enable(pmu);
  5029. }
  5030. static int perf_event_idx_default(struct perf_event *event)
  5031. {
  5032. return event->hw.idx + 1;
  5033. }
  5034. /*
  5035. * Ensures all contexts with the same task_ctx_nr have the same
  5036. * pmu_cpu_context too.
  5037. */
  5038. static void *find_pmu_context(int ctxn)
  5039. {
  5040. struct pmu *pmu;
  5041. if (ctxn < 0)
  5042. return NULL;
  5043. list_for_each_entry(pmu, &pmus, entry) {
  5044. if (pmu->task_ctx_nr == ctxn)
  5045. return pmu->pmu_cpu_context;
  5046. }
  5047. return NULL;
  5048. }
  5049. static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
  5050. {
  5051. int cpu;
  5052. for_each_possible_cpu(cpu) {
  5053. struct perf_cpu_context *cpuctx;
  5054. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  5055. if (cpuctx->unique_pmu == old_pmu)
  5056. cpuctx->unique_pmu = pmu;
  5057. }
  5058. }
  5059. static void free_pmu_context(struct pmu *pmu)
  5060. {
  5061. struct pmu *i;
  5062. mutex_lock(&pmus_lock);
  5063. /*
  5064. * Like a real lame refcount.
  5065. */
  5066. list_for_each_entry(i, &pmus, entry) {
  5067. if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
  5068. update_pmu_context(i, pmu);
  5069. goto out;
  5070. }
  5071. }
  5072. free_percpu(pmu->pmu_cpu_context);
  5073. out:
  5074. mutex_unlock(&pmus_lock);
  5075. }
  5076. static struct idr pmu_idr;
  5077. static ssize_t
  5078. type_show(struct device *dev, struct device_attribute *attr, char *page)
  5079. {
  5080. struct pmu *pmu = dev_get_drvdata(dev);
  5081. return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
  5082. }
  5083. static ssize_t
  5084. perf_event_mux_interval_ms_show(struct device *dev,
  5085. struct device_attribute *attr,
  5086. char *page)
  5087. {
  5088. struct pmu *pmu = dev_get_drvdata(dev);
  5089. return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms);
  5090. }
  5091. static ssize_t
  5092. perf_event_mux_interval_ms_store(struct device *dev,
  5093. struct device_attribute *attr,
  5094. const char *buf, size_t count)
  5095. {
  5096. struct pmu *pmu = dev_get_drvdata(dev);
  5097. int timer, cpu, ret;
  5098. ret = kstrtoint(buf, 0, &timer);
  5099. if (ret)
  5100. return ret;
  5101. if (timer < 1)
  5102. return -EINVAL;
  5103. /* same value, noting to do */
  5104. if (timer == pmu->hrtimer_interval_ms)
  5105. return count;
  5106. pmu->hrtimer_interval_ms = timer;
  5107. /* update all cpuctx for this PMU */
  5108. for_each_possible_cpu(cpu) {
  5109. struct perf_cpu_context *cpuctx;
  5110. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  5111. cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
  5112. if (hrtimer_active(&cpuctx->hrtimer))
  5113. hrtimer_forward_now(&cpuctx->hrtimer, cpuctx->hrtimer_interval);
  5114. }
  5115. return count;
  5116. }
  5117. static struct device_attribute pmu_dev_attrs[] = {
  5118. __ATTR_RO(type),
  5119. __ATTR_RW(perf_event_mux_interval_ms),
  5120. __ATTR_NULL,
  5121. };
  5122. static int pmu_bus_running;
  5123. static struct bus_type pmu_bus = {
  5124. .name = "event_source",
  5125. .dev_attrs = pmu_dev_attrs,
  5126. };
  5127. static void pmu_dev_release(struct device *dev)
  5128. {
  5129. kfree(dev);
  5130. }
  5131. static int pmu_dev_alloc(struct pmu *pmu)
  5132. {
  5133. int ret = -ENOMEM;
  5134. pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
  5135. if (!pmu->dev)
  5136. goto out;
  5137. pmu->dev->groups = pmu->attr_groups;
  5138. device_initialize(pmu->dev);
  5139. ret = dev_set_name(pmu->dev, "%s", pmu->name);
  5140. if (ret)
  5141. goto free_dev;
  5142. dev_set_drvdata(pmu->dev, pmu);
  5143. pmu->dev->bus = &pmu_bus;
  5144. pmu->dev->release = pmu_dev_release;
  5145. ret = device_add(pmu->dev);
  5146. if (ret)
  5147. goto free_dev;
  5148. out:
  5149. return ret;
  5150. free_dev:
  5151. put_device(pmu->dev);
  5152. goto out;
  5153. }
  5154. static struct lock_class_key cpuctx_mutex;
  5155. static struct lock_class_key cpuctx_lock;
  5156. int perf_pmu_register(struct pmu *pmu, const char *name, int type)
  5157. {
  5158. int cpu, ret;
  5159. mutex_lock(&pmus_lock);
  5160. ret = -ENOMEM;
  5161. pmu->pmu_disable_count = alloc_percpu(int);
  5162. if (!pmu->pmu_disable_count)
  5163. goto unlock;
  5164. pmu->type = -1;
  5165. if (!name)
  5166. goto skip_type;
  5167. pmu->name = name;
  5168. if (type < 0) {
  5169. type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL);
  5170. if (type < 0) {
  5171. ret = type;
  5172. goto free_pdc;
  5173. }
  5174. }
  5175. pmu->type = type;
  5176. if (pmu_bus_running) {
  5177. ret = pmu_dev_alloc(pmu);
  5178. if (ret)
  5179. goto free_idr;
  5180. }
  5181. skip_type:
  5182. pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
  5183. if (pmu->pmu_cpu_context)
  5184. goto got_cpu_context;
  5185. ret = -ENOMEM;
  5186. pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
  5187. if (!pmu->pmu_cpu_context)
  5188. goto free_dev;
  5189. for_each_possible_cpu(cpu) {
  5190. struct perf_cpu_context *cpuctx;
  5191. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  5192. __perf_event_init_context(&cpuctx->ctx);
  5193. lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
  5194. lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
  5195. cpuctx->ctx.type = cpu_context;
  5196. cpuctx->ctx.pmu = pmu;
  5197. __perf_cpu_hrtimer_init(cpuctx, cpu);
  5198. INIT_LIST_HEAD(&cpuctx->rotation_list);
  5199. cpuctx->unique_pmu = pmu;
  5200. }
  5201. got_cpu_context:
  5202. if (!pmu->start_txn) {
  5203. if (pmu->pmu_enable) {
  5204. /*
  5205. * If we have pmu_enable/pmu_disable calls, install
  5206. * transaction stubs that use that to try and batch
  5207. * hardware accesses.
  5208. */
  5209. pmu->start_txn = perf_pmu_start_txn;
  5210. pmu->commit_txn = perf_pmu_commit_txn;
  5211. pmu->cancel_txn = perf_pmu_cancel_txn;
  5212. } else {
  5213. pmu->start_txn = perf_pmu_nop_void;
  5214. pmu->commit_txn = perf_pmu_nop_int;
  5215. pmu->cancel_txn = perf_pmu_nop_void;
  5216. }
  5217. }
  5218. if (!pmu->pmu_enable) {
  5219. pmu->pmu_enable = perf_pmu_nop_void;
  5220. pmu->pmu_disable = perf_pmu_nop_void;
  5221. }
  5222. if (!pmu->event_idx)
  5223. pmu->event_idx = perf_event_idx_default;
  5224. list_add_rcu(&pmu->entry, &pmus);
  5225. ret = 0;
  5226. unlock:
  5227. mutex_unlock(&pmus_lock);
  5228. return ret;
  5229. free_dev:
  5230. device_del(pmu->dev);
  5231. put_device(pmu->dev);
  5232. free_idr:
  5233. if (pmu->type >= PERF_TYPE_MAX)
  5234. idr_remove(&pmu_idr, pmu->type);
  5235. free_pdc:
  5236. free_percpu(pmu->pmu_disable_count);
  5237. goto unlock;
  5238. }
  5239. void perf_pmu_unregister(struct pmu *pmu)
  5240. {
  5241. mutex_lock(&pmus_lock);
  5242. list_del_rcu(&pmu->entry);
  5243. mutex_unlock(&pmus_lock);
  5244. /*
  5245. * We dereference the pmu list under both SRCU and regular RCU, so
  5246. * synchronize against both of those.
  5247. */
  5248. synchronize_srcu(&pmus_srcu);
  5249. synchronize_rcu();
  5250. free_percpu(pmu->pmu_disable_count);
  5251. if (pmu->type >= PERF_TYPE_MAX)
  5252. idr_remove(&pmu_idr, pmu->type);
  5253. device_del(pmu->dev);
  5254. put_device(pmu->dev);
  5255. free_pmu_context(pmu);
  5256. }
  5257. struct pmu *perf_init_event(struct perf_event *event)
  5258. {
  5259. struct pmu *pmu = NULL;
  5260. int idx;
  5261. int ret;
  5262. idx = srcu_read_lock(&pmus_srcu);
  5263. rcu_read_lock();
  5264. pmu = idr_find(&pmu_idr, event->attr.type);
  5265. rcu_read_unlock();
  5266. if (pmu) {
  5267. event->pmu = pmu;
  5268. ret = pmu->event_init(event);
  5269. if (ret)
  5270. pmu = ERR_PTR(ret);
  5271. goto unlock;
  5272. }
  5273. list_for_each_entry_rcu(pmu, &pmus, entry) {
  5274. event->pmu = pmu;
  5275. ret = pmu->event_init(event);
  5276. if (!ret)
  5277. goto unlock;
  5278. if (ret != -ENOENT) {
  5279. pmu = ERR_PTR(ret);
  5280. goto unlock;
  5281. }
  5282. }
  5283. pmu = ERR_PTR(-ENOENT);
  5284. unlock:
  5285. srcu_read_unlock(&pmus_srcu, idx);
  5286. return pmu;
  5287. }
  5288. static void account_event_cpu(struct perf_event *event, int cpu)
  5289. {
  5290. if (event->parent)
  5291. return;
  5292. if (has_branch_stack(event)) {
  5293. if (!(event->attach_state & PERF_ATTACH_TASK))
  5294. atomic_inc(&per_cpu(perf_branch_stack_events, cpu));
  5295. }
  5296. if (is_cgroup_event(event))
  5297. atomic_inc(&per_cpu(perf_cgroup_events, cpu));
  5298. if (event->attr.freq)
  5299. atomic_inc(&per_cpu(perf_freq_events, cpu));
  5300. }
  5301. static void account_event(struct perf_event *event)
  5302. {
  5303. if (event->parent)
  5304. return;
  5305. if (event->attach_state & PERF_ATTACH_TASK)
  5306. static_key_slow_inc(&perf_sched_events.key);
  5307. if (event->attr.mmap || event->attr.mmap_data)
  5308. atomic_inc(&nr_mmap_events);
  5309. if (event->attr.comm)
  5310. atomic_inc(&nr_comm_events);
  5311. if (event->attr.task)
  5312. atomic_inc(&nr_task_events);
  5313. if (has_branch_stack(event))
  5314. static_key_slow_inc(&perf_sched_events.key);
  5315. if (is_cgroup_event(event))
  5316. static_key_slow_inc(&perf_sched_events.key);
  5317. account_event_cpu(event, event->cpu);
  5318. }
  5319. /*
  5320. * Allocate and initialize a event structure
  5321. */
  5322. static struct perf_event *
  5323. perf_event_alloc(struct perf_event_attr *attr, int cpu,
  5324. struct task_struct *task,
  5325. struct perf_event *group_leader,
  5326. struct perf_event *parent_event,
  5327. perf_overflow_handler_t overflow_handler,
  5328. void *context)
  5329. {
  5330. struct pmu *pmu;
  5331. struct perf_event *event;
  5332. struct hw_perf_event *hwc;
  5333. long err = -EINVAL;
  5334. if ((unsigned)cpu >= nr_cpu_ids) {
  5335. if (!task || cpu != -1)
  5336. return ERR_PTR(-EINVAL);
  5337. }
  5338. event = kzalloc(sizeof(*event), GFP_KERNEL);
  5339. if (!event)
  5340. return ERR_PTR(-ENOMEM);
  5341. /*
  5342. * Single events are their own group leaders, with an
  5343. * empty sibling list:
  5344. */
  5345. if (!group_leader)
  5346. group_leader = event;
  5347. mutex_init(&event->child_mutex);
  5348. INIT_LIST_HEAD(&event->child_list);
  5349. INIT_LIST_HEAD(&event->group_entry);
  5350. INIT_LIST_HEAD(&event->event_entry);
  5351. INIT_LIST_HEAD(&event->sibling_list);
  5352. INIT_LIST_HEAD(&event->rb_entry);
  5353. init_waitqueue_head(&event->waitq);
  5354. init_irq_work(&event->pending, perf_pending_event);
  5355. mutex_init(&event->mmap_mutex);
  5356. atomic_long_set(&event->refcount, 1);
  5357. event->cpu = cpu;
  5358. event->attr = *attr;
  5359. event->group_leader = group_leader;
  5360. event->pmu = NULL;
  5361. event->oncpu = -1;
  5362. event->parent = parent_event;
  5363. event->ns = get_pid_ns(task_active_pid_ns(current));
  5364. event->id = atomic64_inc_return(&perf_event_id);
  5365. event->state = PERF_EVENT_STATE_INACTIVE;
  5366. if (task) {
  5367. event->attach_state = PERF_ATTACH_TASK;
  5368. if (attr->type == PERF_TYPE_TRACEPOINT)
  5369. event->hw.tp_target = task;
  5370. #ifdef CONFIG_HAVE_HW_BREAKPOINT
  5371. /*
  5372. * hw_breakpoint is a bit difficult here..
  5373. */
  5374. else if (attr->type == PERF_TYPE_BREAKPOINT)
  5375. event->hw.bp_target = task;
  5376. #endif
  5377. }
  5378. if (!overflow_handler && parent_event) {
  5379. overflow_handler = parent_event->overflow_handler;
  5380. context = parent_event->overflow_handler_context;
  5381. }
  5382. event->overflow_handler = overflow_handler;
  5383. event->overflow_handler_context = context;
  5384. perf_event__state_init(event);
  5385. pmu = NULL;
  5386. hwc = &event->hw;
  5387. hwc->sample_period = attr->sample_period;
  5388. if (attr->freq && attr->sample_freq)
  5389. hwc->sample_period = 1;
  5390. hwc->last_period = hwc->sample_period;
  5391. local64_set(&hwc->period_left, hwc->sample_period);
  5392. /*
  5393. * we currently do not support PERF_FORMAT_GROUP on inherited events
  5394. */
  5395. if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
  5396. goto err_ns;
  5397. pmu = perf_init_event(event);
  5398. if (!pmu)
  5399. goto err_ns;
  5400. else if (IS_ERR(pmu)) {
  5401. err = PTR_ERR(pmu);
  5402. goto err_ns;
  5403. }
  5404. if (!event->parent) {
  5405. if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
  5406. err = get_callchain_buffers();
  5407. if (err)
  5408. goto err_pmu;
  5409. }
  5410. }
  5411. return event;
  5412. err_pmu:
  5413. if (event->destroy)
  5414. event->destroy(event);
  5415. err_ns:
  5416. if (event->ns)
  5417. put_pid_ns(event->ns);
  5418. kfree(event);
  5419. return ERR_PTR(err);
  5420. }
  5421. static int perf_copy_attr(struct perf_event_attr __user *uattr,
  5422. struct perf_event_attr *attr)
  5423. {
  5424. u32 size;
  5425. int ret;
  5426. if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
  5427. return -EFAULT;
  5428. /*
  5429. * zero the full structure, so that a short copy will be nice.
  5430. */
  5431. memset(attr, 0, sizeof(*attr));
  5432. ret = get_user(size, &uattr->size);
  5433. if (ret)
  5434. return ret;
  5435. if (size > PAGE_SIZE) /* silly large */
  5436. goto err_size;
  5437. if (!size) /* abi compat */
  5438. size = PERF_ATTR_SIZE_VER0;
  5439. if (size < PERF_ATTR_SIZE_VER0)
  5440. goto err_size;
  5441. /*
  5442. * If we're handed a bigger struct than we know of,
  5443. * ensure all the unknown bits are 0 - i.e. new
  5444. * user-space does not rely on any kernel feature
  5445. * extensions we dont know about yet.
  5446. */
  5447. if (size > sizeof(*attr)) {
  5448. unsigned char __user *addr;
  5449. unsigned char __user *end;
  5450. unsigned char val;
  5451. addr = (void __user *)uattr + sizeof(*attr);
  5452. end = (void __user *)uattr + size;
  5453. for (; addr < end; addr++) {
  5454. ret = get_user(val, addr);
  5455. if (ret)
  5456. return ret;
  5457. if (val)
  5458. goto err_size;
  5459. }
  5460. size = sizeof(*attr);
  5461. }
  5462. ret = copy_from_user(attr, uattr, size);
  5463. if (ret)
  5464. return -EFAULT;
  5465. if (attr->__reserved_1)
  5466. return -EINVAL;
  5467. if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
  5468. return -EINVAL;
  5469. if (attr->read_format & ~(PERF_FORMAT_MAX-1))
  5470. return -EINVAL;
  5471. if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
  5472. u64 mask = attr->branch_sample_type;
  5473. /* only using defined bits */
  5474. if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
  5475. return -EINVAL;
  5476. /* at least one branch bit must be set */
  5477. if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
  5478. return -EINVAL;
  5479. /* propagate priv level, when not set for branch */
  5480. if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
  5481. /* exclude_kernel checked on syscall entry */
  5482. if (!attr->exclude_kernel)
  5483. mask |= PERF_SAMPLE_BRANCH_KERNEL;
  5484. if (!attr->exclude_user)
  5485. mask |= PERF_SAMPLE_BRANCH_USER;
  5486. if (!attr->exclude_hv)
  5487. mask |= PERF_SAMPLE_BRANCH_HV;
  5488. /*
  5489. * adjust user setting (for HW filter setup)
  5490. */
  5491. attr->branch_sample_type = mask;
  5492. }
  5493. /* privileged levels capture (kernel, hv): check permissions */
  5494. if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
  5495. && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
  5496. return -EACCES;
  5497. }
  5498. if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
  5499. ret = perf_reg_validate(attr->sample_regs_user);
  5500. if (ret)
  5501. return ret;
  5502. }
  5503. if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
  5504. if (!arch_perf_have_user_stack_dump())
  5505. return -ENOSYS;
  5506. /*
  5507. * We have __u32 type for the size, but so far
  5508. * we can only use __u16 as maximum due to the
  5509. * __u16 sample size limit.
  5510. */
  5511. if (attr->sample_stack_user >= USHRT_MAX)
  5512. ret = -EINVAL;
  5513. else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
  5514. ret = -EINVAL;
  5515. }
  5516. out:
  5517. return ret;
  5518. err_size:
  5519. put_user(sizeof(*attr), &uattr->size);
  5520. ret = -E2BIG;
  5521. goto out;
  5522. }
  5523. static int
  5524. perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
  5525. {
  5526. struct ring_buffer *rb = NULL, *old_rb = NULL;
  5527. int ret = -EINVAL;
  5528. if (!output_event)
  5529. goto set;
  5530. /* don't allow circular references */
  5531. if (event == output_event)
  5532. goto out;
  5533. /*
  5534. * Don't allow cross-cpu buffers
  5535. */
  5536. if (output_event->cpu != event->cpu)
  5537. goto out;
  5538. /*
  5539. * If its not a per-cpu rb, it must be the same task.
  5540. */
  5541. if (output_event->cpu == -1 && output_event->ctx != event->ctx)
  5542. goto out;
  5543. set:
  5544. mutex_lock(&event->mmap_mutex);
  5545. /* Can't redirect output if we've got an active mmap() */
  5546. if (atomic_read(&event->mmap_count))
  5547. goto unlock;
  5548. old_rb = event->rb;
  5549. if (output_event) {
  5550. /* get the rb we want to redirect to */
  5551. rb = ring_buffer_get(output_event);
  5552. if (!rb)
  5553. goto unlock;
  5554. }
  5555. if (old_rb)
  5556. ring_buffer_detach(event, old_rb);
  5557. if (rb)
  5558. ring_buffer_attach(event, rb);
  5559. rcu_assign_pointer(event->rb, rb);
  5560. if (old_rb) {
  5561. ring_buffer_put(old_rb);
  5562. /*
  5563. * Since we detached before setting the new rb, so that we
  5564. * could attach the new rb, we could have missed a wakeup.
  5565. * Provide it now.
  5566. */
  5567. wake_up_all(&event->waitq);
  5568. }
  5569. ret = 0;
  5570. unlock:
  5571. mutex_unlock(&event->mmap_mutex);
  5572. out:
  5573. return ret;
  5574. }
  5575. /**
  5576. * sys_perf_event_open - open a performance event, associate it to a task/cpu
  5577. *
  5578. * @attr_uptr: event_id type attributes for monitoring/sampling
  5579. * @pid: target pid
  5580. * @cpu: target cpu
  5581. * @group_fd: group leader event fd
  5582. */
  5583. SYSCALL_DEFINE5(perf_event_open,
  5584. struct perf_event_attr __user *, attr_uptr,
  5585. pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
  5586. {
  5587. struct perf_event *group_leader = NULL, *output_event = NULL;
  5588. struct perf_event *event, *sibling;
  5589. struct perf_event_attr attr;
  5590. struct perf_event_context *ctx;
  5591. struct file *event_file = NULL;
  5592. struct fd group = {NULL, 0};
  5593. struct task_struct *task = NULL;
  5594. struct pmu *pmu;
  5595. int event_fd;
  5596. int move_group = 0;
  5597. int err;
  5598. /* for future expandability... */
  5599. if (flags & ~PERF_FLAG_ALL)
  5600. return -EINVAL;
  5601. err = perf_copy_attr(attr_uptr, &attr);
  5602. if (err)
  5603. return err;
  5604. if (!attr.exclude_kernel) {
  5605. if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
  5606. return -EACCES;
  5607. }
  5608. if (attr.freq) {
  5609. if (attr.sample_freq > sysctl_perf_event_sample_rate)
  5610. return -EINVAL;
  5611. }
  5612. /*
  5613. * In cgroup mode, the pid argument is used to pass the fd
  5614. * opened to the cgroup directory in cgroupfs. The cpu argument
  5615. * designates the cpu on which to monitor threads from that
  5616. * cgroup.
  5617. */
  5618. if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
  5619. return -EINVAL;
  5620. event_fd = get_unused_fd();
  5621. if (event_fd < 0)
  5622. return event_fd;
  5623. if (group_fd != -1) {
  5624. err = perf_fget_light(group_fd, &group);
  5625. if (err)
  5626. goto err_fd;
  5627. group_leader = group.file->private_data;
  5628. if (flags & PERF_FLAG_FD_OUTPUT)
  5629. output_event = group_leader;
  5630. if (flags & PERF_FLAG_FD_NO_GROUP)
  5631. group_leader = NULL;
  5632. }
  5633. if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
  5634. task = find_lively_task_by_vpid(pid);
  5635. if (IS_ERR(task)) {
  5636. err = PTR_ERR(task);
  5637. goto err_group_fd;
  5638. }
  5639. }
  5640. get_online_cpus();
  5641. event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
  5642. NULL, NULL);
  5643. if (IS_ERR(event)) {
  5644. err = PTR_ERR(event);
  5645. goto err_task;
  5646. }
  5647. if (flags & PERF_FLAG_PID_CGROUP) {
  5648. err = perf_cgroup_connect(pid, event, &attr, group_leader);
  5649. if (err) {
  5650. __free_event(event);
  5651. goto err_task;
  5652. }
  5653. }
  5654. account_event(event);
  5655. /*
  5656. * Special case software events and allow them to be part of
  5657. * any hardware group.
  5658. */
  5659. pmu = event->pmu;
  5660. if (group_leader &&
  5661. (is_software_event(event) != is_software_event(group_leader))) {
  5662. if (is_software_event(event)) {
  5663. /*
  5664. * If event and group_leader are not both a software
  5665. * event, and event is, then group leader is not.
  5666. *
  5667. * Allow the addition of software events to !software
  5668. * groups, this is safe because software events never
  5669. * fail to schedule.
  5670. */
  5671. pmu = group_leader->pmu;
  5672. } else if (is_software_event(group_leader) &&
  5673. (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
  5674. /*
  5675. * In case the group is a pure software group, and we
  5676. * try to add a hardware event, move the whole group to
  5677. * the hardware context.
  5678. */
  5679. move_group = 1;
  5680. }
  5681. }
  5682. /*
  5683. * Get the target context (task or percpu):
  5684. */
  5685. ctx = find_get_context(pmu, task, event->cpu);
  5686. if (IS_ERR(ctx)) {
  5687. err = PTR_ERR(ctx);
  5688. goto err_alloc;
  5689. }
  5690. if (task) {
  5691. put_task_struct(task);
  5692. task = NULL;
  5693. }
  5694. /*
  5695. * Look up the group leader (we will attach this event to it):
  5696. */
  5697. if (group_leader) {
  5698. err = -EINVAL;
  5699. /*
  5700. * Do not allow a recursive hierarchy (this new sibling
  5701. * becoming part of another group-sibling):
  5702. */
  5703. if (group_leader->group_leader != group_leader)
  5704. goto err_context;
  5705. /*
  5706. * Do not allow to attach to a group in a different
  5707. * task or CPU context:
  5708. */
  5709. if (move_group) {
  5710. if (group_leader->ctx->type != ctx->type)
  5711. goto err_context;
  5712. } else {
  5713. if (group_leader->ctx != ctx)
  5714. goto err_context;
  5715. }
  5716. /*
  5717. * Only a group leader can be exclusive or pinned
  5718. */
  5719. if (attr.exclusive || attr.pinned)
  5720. goto err_context;
  5721. }
  5722. if (output_event) {
  5723. err = perf_event_set_output(event, output_event);
  5724. if (err)
  5725. goto err_context;
  5726. }
  5727. event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
  5728. if (IS_ERR(event_file)) {
  5729. err = PTR_ERR(event_file);
  5730. goto err_context;
  5731. }
  5732. if (move_group) {
  5733. struct perf_event_context *gctx = group_leader->ctx;
  5734. mutex_lock(&gctx->mutex);
  5735. perf_remove_from_context(group_leader);
  5736. /*
  5737. * Removing from the context ends up with disabled
  5738. * event. What we want here is event in the initial
  5739. * startup state, ready to be add into new context.
  5740. */
  5741. perf_event__state_init(group_leader);
  5742. list_for_each_entry(sibling, &group_leader->sibling_list,
  5743. group_entry) {
  5744. perf_remove_from_context(sibling);
  5745. perf_event__state_init(sibling);
  5746. put_ctx(gctx);
  5747. }
  5748. mutex_unlock(&gctx->mutex);
  5749. put_ctx(gctx);
  5750. }
  5751. WARN_ON_ONCE(ctx->parent_ctx);
  5752. mutex_lock(&ctx->mutex);
  5753. if (move_group) {
  5754. synchronize_rcu();
  5755. perf_install_in_context(ctx, group_leader, event->cpu);
  5756. get_ctx(ctx);
  5757. list_for_each_entry(sibling, &group_leader->sibling_list,
  5758. group_entry) {
  5759. perf_install_in_context(ctx, sibling, event->cpu);
  5760. get_ctx(ctx);
  5761. }
  5762. }
  5763. perf_install_in_context(ctx, event, event->cpu);
  5764. ++ctx->generation;
  5765. perf_unpin_context(ctx);
  5766. mutex_unlock(&ctx->mutex);
  5767. put_online_cpus();
  5768. event->owner = current;
  5769. mutex_lock(&current->perf_event_mutex);
  5770. list_add_tail(&event->owner_entry, &current->perf_event_list);
  5771. mutex_unlock(&current->perf_event_mutex);
  5772. /*
  5773. * Precalculate sample_data sizes
  5774. */
  5775. perf_event__header_size(event);
  5776. perf_event__id_header_size(event);
  5777. /*
  5778. * Drop the reference on the group_event after placing the
  5779. * new event on the sibling_list. This ensures destruction
  5780. * of the group leader will find the pointer to itself in
  5781. * perf_group_detach().
  5782. */
  5783. fdput(group);
  5784. fd_install(event_fd, event_file);
  5785. return event_fd;
  5786. err_context:
  5787. perf_unpin_context(ctx);
  5788. put_ctx(ctx);
  5789. err_alloc:
  5790. free_event(event);
  5791. err_task:
  5792. put_online_cpus();
  5793. if (task)
  5794. put_task_struct(task);
  5795. err_group_fd:
  5796. fdput(group);
  5797. err_fd:
  5798. put_unused_fd(event_fd);
  5799. return err;
  5800. }
  5801. /**
  5802. * perf_event_create_kernel_counter
  5803. *
  5804. * @attr: attributes of the counter to create
  5805. * @cpu: cpu in which the counter is bound
  5806. * @task: task to profile (NULL for percpu)
  5807. */
  5808. struct perf_event *
  5809. perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
  5810. struct task_struct *task,
  5811. perf_overflow_handler_t overflow_handler,
  5812. void *context)
  5813. {
  5814. struct perf_event_context *ctx;
  5815. struct perf_event *event;
  5816. int err;
  5817. /*
  5818. * Get the target context (task or percpu):
  5819. */
  5820. event = perf_event_alloc(attr, cpu, task, NULL, NULL,
  5821. overflow_handler, context);
  5822. if (IS_ERR(event)) {
  5823. err = PTR_ERR(event);
  5824. goto err;
  5825. }
  5826. account_event(event);
  5827. ctx = find_get_context(event->pmu, task, cpu);
  5828. if (IS_ERR(ctx)) {
  5829. err = PTR_ERR(ctx);
  5830. goto err_free;
  5831. }
  5832. WARN_ON_ONCE(ctx->parent_ctx);
  5833. mutex_lock(&ctx->mutex);
  5834. perf_install_in_context(ctx, event, cpu);
  5835. ++ctx->generation;
  5836. perf_unpin_context(ctx);
  5837. mutex_unlock(&ctx->mutex);
  5838. return event;
  5839. err_free:
  5840. free_event(event);
  5841. err:
  5842. return ERR_PTR(err);
  5843. }
  5844. EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
  5845. void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
  5846. {
  5847. struct perf_event_context *src_ctx;
  5848. struct perf_event_context *dst_ctx;
  5849. struct perf_event *event, *tmp;
  5850. LIST_HEAD(events);
  5851. src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx;
  5852. dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx;
  5853. mutex_lock(&src_ctx->mutex);
  5854. list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
  5855. event_entry) {
  5856. perf_remove_from_context(event);
  5857. unaccount_event_cpu(event, src_cpu);
  5858. put_ctx(src_ctx);
  5859. list_add(&event->event_entry, &events);
  5860. }
  5861. mutex_unlock(&src_ctx->mutex);
  5862. synchronize_rcu();
  5863. mutex_lock(&dst_ctx->mutex);
  5864. list_for_each_entry_safe(event, tmp, &events, event_entry) {
  5865. list_del(&event->event_entry);
  5866. if (event->state >= PERF_EVENT_STATE_OFF)
  5867. event->state = PERF_EVENT_STATE_INACTIVE;
  5868. account_event_cpu(event, dst_cpu);
  5869. perf_install_in_context(dst_ctx, event, dst_cpu);
  5870. get_ctx(dst_ctx);
  5871. }
  5872. mutex_unlock(&dst_ctx->mutex);
  5873. }
  5874. EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
  5875. static void sync_child_event(struct perf_event *child_event,
  5876. struct task_struct *child)
  5877. {
  5878. struct perf_event *parent_event = child_event->parent;
  5879. u64 child_val;
  5880. if (child_event->attr.inherit_stat)
  5881. perf_event_read_event(child_event, child);
  5882. child_val = perf_event_count(child_event);
  5883. /*
  5884. * Add back the child's count to the parent's count:
  5885. */
  5886. atomic64_add(child_val, &parent_event->child_count);
  5887. atomic64_add(child_event->total_time_enabled,
  5888. &parent_event->child_total_time_enabled);
  5889. atomic64_add(child_event->total_time_running,
  5890. &parent_event->child_total_time_running);
  5891. /*
  5892. * Remove this event from the parent's list
  5893. */
  5894. WARN_ON_ONCE(parent_event->ctx->parent_ctx);
  5895. mutex_lock(&parent_event->child_mutex);
  5896. list_del_init(&child_event->child_list);
  5897. mutex_unlock(&parent_event->child_mutex);
  5898. /*
  5899. * Release the parent event, if this was the last
  5900. * reference to it.
  5901. */
  5902. put_event(parent_event);
  5903. }
  5904. static void
  5905. __perf_event_exit_task(struct perf_event *child_event,
  5906. struct perf_event_context *child_ctx,
  5907. struct task_struct *child)
  5908. {
  5909. if (child_event->parent) {
  5910. raw_spin_lock_irq(&child_ctx->lock);
  5911. perf_group_detach(child_event);
  5912. raw_spin_unlock_irq(&child_ctx->lock);
  5913. }
  5914. perf_remove_from_context(child_event);
  5915. /*
  5916. * It can happen that the parent exits first, and has events
  5917. * that are still around due to the child reference. These
  5918. * events need to be zapped.
  5919. */
  5920. if (child_event->parent) {
  5921. sync_child_event(child_event, child);
  5922. free_event(child_event);
  5923. }
  5924. }
  5925. static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
  5926. {
  5927. struct perf_event *child_event, *tmp;
  5928. struct perf_event_context *child_ctx;
  5929. unsigned long flags;
  5930. if (likely(!child->perf_event_ctxp[ctxn])) {
  5931. perf_event_task(child, NULL, 0);
  5932. return;
  5933. }
  5934. local_irq_save(flags);
  5935. /*
  5936. * We can't reschedule here because interrupts are disabled,
  5937. * and either child is current or it is a task that can't be
  5938. * scheduled, so we are now safe from rescheduling changing
  5939. * our context.
  5940. */
  5941. child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
  5942. /*
  5943. * Take the context lock here so that if find_get_context is
  5944. * reading child->perf_event_ctxp, we wait until it has
  5945. * incremented the context's refcount before we do put_ctx below.
  5946. */
  5947. raw_spin_lock(&child_ctx->lock);
  5948. task_ctx_sched_out(child_ctx);
  5949. child->perf_event_ctxp[ctxn] = NULL;
  5950. /*
  5951. * If this context is a clone; unclone it so it can't get
  5952. * swapped to another process while we're removing all
  5953. * the events from it.
  5954. */
  5955. unclone_ctx(child_ctx);
  5956. update_context_time(child_ctx);
  5957. raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
  5958. /*
  5959. * Report the task dead after unscheduling the events so that we
  5960. * won't get any samples after PERF_RECORD_EXIT. We can however still
  5961. * get a few PERF_RECORD_READ events.
  5962. */
  5963. perf_event_task(child, child_ctx, 0);
  5964. /*
  5965. * We can recurse on the same lock type through:
  5966. *
  5967. * __perf_event_exit_task()
  5968. * sync_child_event()
  5969. * put_event()
  5970. * mutex_lock(&ctx->mutex)
  5971. *
  5972. * But since its the parent context it won't be the same instance.
  5973. */
  5974. mutex_lock(&child_ctx->mutex);
  5975. again:
  5976. list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
  5977. group_entry)
  5978. __perf_event_exit_task(child_event, child_ctx, child);
  5979. list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
  5980. group_entry)
  5981. __perf_event_exit_task(child_event, child_ctx, child);
  5982. /*
  5983. * If the last event was a group event, it will have appended all
  5984. * its siblings to the list, but we obtained 'tmp' before that which
  5985. * will still point to the list head terminating the iteration.
  5986. */
  5987. if (!list_empty(&child_ctx->pinned_groups) ||
  5988. !list_empty(&child_ctx->flexible_groups))
  5989. goto again;
  5990. mutex_unlock(&child_ctx->mutex);
  5991. put_ctx(child_ctx);
  5992. }
  5993. /*
  5994. * When a child task exits, feed back event values to parent events.
  5995. */
  5996. void perf_event_exit_task(struct task_struct *child)
  5997. {
  5998. struct perf_event *event, *tmp;
  5999. int ctxn;
  6000. mutex_lock(&child->perf_event_mutex);
  6001. list_for_each_entry_safe(event, tmp, &child->perf_event_list,
  6002. owner_entry) {
  6003. list_del_init(&event->owner_entry);
  6004. /*
  6005. * Ensure the list deletion is visible before we clear
  6006. * the owner, closes a race against perf_release() where
  6007. * we need to serialize on the owner->perf_event_mutex.
  6008. */
  6009. smp_wmb();
  6010. event->owner = NULL;
  6011. }
  6012. mutex_unlock(&child->perf_event_mutex);
  6013. for_each_task_context_nr(ctxn)
  6014. perf_event_exit_task_context(child, ctxn);
  6015. }
  6016. static void perf_free_event(struct perf_event *event,
  6017. struct perf_event_context *ctx)
  6018. {
  6019. struct perf_event *parent = event->parent;
  6020. if (WARN_ON_ONCE(!parent))
  6021. return;
  6022. mutex_lock(&parent->child_mutex);
  6023. list_del_init(&event->child_list);
  6024. mutex_unlock(&parent->child_mutex);
  6025. put_event(parent);
  6026. perf_group_detach(event);
  6027. list_del_event(event, ctx);
  6028. free_event(event);
  6029. }
  6030. /*
  6031. * free an unexposed, unused context as created by inheritance by
  6032. * perf_event_init_task below, used by fork() in case of fail.
  6033. */
  6034. void perf_event_free_task(struct task_struct *task)
  6035. {
  6036. struct perf_event_context *ctx;
  6037. struct perf_event *event, *tmp;
  6038. int ctxn;
  6039. for_each_task_context_nr(ctxn) {
  6040. ctx = task->perf_event_ctxp[ctxn];
  6041. if (!ctx)
  6042. continue;
  6043. mutex_lock(&ctx->mutex);
  6044. again:
  6045. list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
  6046. group_entry)
  6047. perf_free_event(event, ctx);
  6048. list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
  6049. group_entry)
  6050. perf_free_event(event, ctx);
  6051. if (!list_empty(&ctx->pinned_groups) ||
  6052. !list_empty(&ctx->flexible_groups))
  6053. goto again;
  6054. mutex_unlock(&ctx->mutex);
  6055. put_ctx(ctx);
  6056. }
  6057. }
  6058. void perf_event_delayed_put(struct task_struct *task)
  6059. {
  6060. int ctxn;
  6061. for_each_task_context_nr(ctxn)
  6062. WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
  6063. }
  6064. /*
  6065. * inherit a event from parent task to child task:
  6066. */
  6067. static struct perf_event *
  6068. inherit_event(struct perf_event *parent_event,
  6069. struct task_struct *parent,
  6070. struct perf_event_context *parent_ctx,
  6071. struct task_struct *child,
  6072. struct perf_event *group_leader,
  6073. struct perf_event_context *child_ctx)
  6074. {
  6075. struct perf_event *child_event;
  6076. unsigned long flags;
  6077. /*
  6078. * Instead of creating recursive hierarchies of events,
  6079. * we link inherited events back to the original parent,
  6080. * which has a filp for sure, which we use as the reference
  6081. * count:
  6082. */
  6083. if (parent_event->parent)
  6084. parent_event = parent_event->parent;
  6085. child_event = perf_event_alloc(&parent_event->attr,
  6086. parent_event->cpu,
  6087. child,
  6088. group_leader, parent_event,
  6089. NULL, NULL);
  6090. if (IS_ERR(child_event))
  6091. return child_event;
  6092. if (!atomic_long_inc_not_zero(&parent_event->refcount)) {
  6093. free_event(child_event);
  6094. return NULL;
  6095. }
  6096. get_ctx(child_ctx);
  6097. /*
  6098. * Make the child state follow the state of the parent event,
  6099. * not its attr.disabled bit. We hold the parent's mutex,
  6100. * so we won't race with perf_event_{en, dis}able_family.
  6101. */
  6102. if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
  6103. child_event->state = PERF_EVENT_STATE_INACTIVE;
  6104. else
  6105. child_event->state = PERF_EVENT_STATE_OFF;
  6106. if (parent_event->attr.freq) {
  6107. u64 sample_period = parent_event->hw.sample_period;
  6108. struct hw_perf_event *hwc = &child_event->hw;
  6109. hwc->sample_period = sample_period;
  6110. hwc->last_period = sample_period;
  6111. local64_set(&hwc->period_left, sample_period);
  6112. }
  6113. child_event->ctx = child_ctx;
  6114. child_event->overflow_handler = parent_event->overflow_handler;
  6115. child_event->overflow_handler_context
  6116. = parent_event->overflow_handler_context;
  6117. /*
  6118. * Precalculate sample_data sizes
  6119. */
  6120. perf_event__header_size(child_event);
  6121. perf_event__id_header_size(child_event);
  6122. /*
  6123. * Link it up in the child's context:
  6124. */
  6125. raw_spin_lock_irqsave(&child_ctx->lock, flags);
  6126. add_event_to_ctx(child_event, child_ctx);
  6127. raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
  6128. /*
  6129. * Link this into the parent event's child list
  6130. */
  6131. WARN_ON_ONCE(parent_event->ctx->parent_ctx);
  6132. mutex_lock(&parent_event->child_mutex);
  6133. list_add_tail(&child_event->child_list, &parent_event->child_list);
  6134. mutex_unlock(&parent_event->child_mutex);
  6135. return child_event;
  6136. }
  6137. static int inherit_group(struct perf_event *parent_event,
  6138. struct task_struct *parent,
  6139. struct perf_event_context *parent_ctx,
  6140. struct task_struct *child,
  6141. struct perf_event_context *child_ctx)
  6142. {
  6143. struct perf_event *leader;
  6144. struct perf_event *sub;
  6145. struct perf_event *child_ctr;
  6146. leader = inherit_event(parent_event, parent, parent_ctx,
  6147. child, NULL, child_ctx);
  6148. if (IS_ERR(leader))
  6149. return PTR_ERR(leader);
  6150. list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
  6151. child_ctr = inherit_event(sub, parent, parent_ctx,
  6152. child, leader, child_ctx);
  6153. if (IS_ERR(child_ctr))
  6154. return PTR_ERR(child_ctr);
  6155. }
  6156. return 0;
  6157. }
  6158. static int
  6159. inherit_task_group(struct perf_event *event, struct task_struct *parent,
  6160. struct perf_event_context *parent_ctx,
  6161. struct task_struct *child, int ctxn,
  6162. int *inherited_all)
  6163. {
  6164. int ret;
  6165. struct perf_event_context *child_ctx;
  6166. if (!event->attr.inherit) {
  6167. *inherited_all = 0;
  6168. return 0;
  6169. }
  6170. child_ctx = child->perf_event_ctxp[ctxn];
  6171. if (!child_ctx) {
  6172. /*
  6173. * This is executed from the parent task context, so
  6174. * inherit events that have been marked for cloning.
  6175. * First allocate and initialize a context for the
  6176. * child.
  6177. */
  6178. child_ctx = alloc_perf_context(parent_ctx->pmu, child);
  6179. if (!child_ctx)
  6180. return -ENOMEM;
  6181. child->perf_event_ctxp[ctxn] = child_ctx;
  6182. }
  6183. ret = inherit_group(event, parent, parent_ctx,
  6184. child, child_ctx);
  6185. if (ret)
  6186. *inherited_all = 0;
  6187. return ret;
  6188. }
  6189. /*
  6190. * Initialize the perf_event context in task_struct
  6191. */
  6192. int perf_event_init_context(struct task_struct *child, int ctxn)
  6193. {
  6194. struct perf_event_context *child_ctx, *parent_ctx;
  6195. struct perf_event_context *cloned_ctx;
  6196. struct perf_event *event;
  6197. struct task_struct *parent = current;
  6198. int inherited_all = 1;
  6199. unsigned long flags;
  6200. int ret = 0;
  6201. if (likely(!parent->perf_event_ctxp[ctxn]))
  6202. return 0;
  6203. /*
  6204. * If the parent's context is a clone, pin it so it won't get
  6205. * swapped under us.
  6206. */
  6207. parent_ctx = perf_pin_task_context(parent, ctxn);
  6208. /*
  6209. * No need to check if parent_ctx != NULL here; since we saw
  6210. * it non-NULL earlier, the only reason for it to become NULL
  6211. * is if we exit, and since we're currently in the middle of
  6212. * a fork we can't be exiting at the same time.
  6213. */
  6214. /*
  6215. * Lock the parent list. No need to lock the child - not PID
  6216. * hashed yet and not running, so nobody can access it.
  6217. */
  6218. mutex_lock(&parent_ctx->mutex);
  6219. /*
  6220. * We dont have to disable NMIs - we are only looking at
  6221. * the list, not manipulating it:
  6222. */
  6223. list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
  6224. ret = inherit_task_group(event, parent, parent_ctx,
  6225. child, ctxn, &inherited_all);
  6226. if (ret)
  6227. break;
  6228. }
  6229. /*
  6230. * We can't hold ctx->lock when iterating the ->flexible_group list due
  6231. * to allocations, but we need to prevent rotation because
  6232. * rotate_ctx() will change the list from interrupt context.
  6233. */
  6234. raw_spin_lock_irqsave(&parent_ctx->lock, flags);
  6235. parent_ctx->rotate_disable = 1;
  6236. raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
  6237. list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
  6238. ret = inherit_task_group(event, parent, parent_ctx,
  6239. child, ctxn, &inherited_all);
  6240. if (ret)
  6241. break;
  6242. }
  6243. raw_spin_lock_irqsave(&parent_ctx->lock, flags);
  6244. parent_ctx->rotate_disable = 0;
  6245. child_ctx = child->perf_event_ctxp[ctxn];
  6246. if (child_ctx && inherited_all) {
  6247. /*
  6248. * Mark the child context as a clone of the parent
  6249. * context, or of whatever the parent is a clone of.
  6250. *
  6251. * Note that if the parent is a clone, the holding of
  6252. * parent_ctx->lock avoids it from being uncloned.
  6253. */
  6254. cloned_ctx = parent_ctx->parent_ctx;
  6255. if (cloned_ctx) {
  6256. child_ctx->parent_ctx = cloned_ctx;
  6257. child_ctx->parent_gen = parent_ctx->parent_gen;
  6258. } else {
  6259. child_ctx->parent_ctx = parent_ctx;
  6260. child_ctx->parent_gen = parent_ctx->generation;
  6261. }
  6262. get_ctx(child_ctx->parent_ctx);
  6263. }
  6264. raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
  6265. mutex_unlock(&parent_ctx->mutex);
  6266. perf_unpin_context(parent_ctx);
  6267. put_ctx(parent_ctx);
  6268. return ret;
  6269. }
  6270. /*
  6271. * Initialize the perf_event context in task_struct
  6272. */
  6273. int perf_event_init_task(struct task_struct *child)
  6274. {
  6275. int ctxn, ret;
  6276. memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
  6277. mutex_init(&child->perf_event_mutex);
  6278. INIT_LIST_HEAD(&child->perf_event_list);
  6279. for_each_task_context_nr(ctxn) {
  6280. ret = perf_event_init_context(child, ctxn);
  6281. if (ret)
  6282. return ret;
  6283. }
  6284. return 0;
  6285. }
  6286. static void __init perf_event_init_all_cpus(void)
  6287. {
  6288. struct swevent_htable *swhash;
  6289. int cpu;
  6290. for_each_possible_cpu(cpu) {
  6291. swhash = &per_cpu(swevent_htable, cpu);
  6292. mutex_init(&swhash->hlist_mutex);
  6293. INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
  6294. }
  6295. }
  6296. static void perf_event_init_cpu(int cpu)
  6297. {
  6298. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  6299. mutex_lock(&swhash->hlist_mutex);
  6300. if (swhash->hlist_refcount > 0) {
  6301. struct swevent_hlist *hlist;
  6302. hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
  6303. WARN_ON(!hlist);
  6304. rcu_assign_pointer(swhash->swevent_hlist, hlist);
  6305. }
  6306. mutex_unlock(&swhash->hlist_mutex);
  6307. }
  6308. #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
  6309. static void perf_pmu_rotate_stop(struct pmu *pmu)
  6310. {
  6311. struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  6312. WARN_ON(!irqs_disabled());
  6313. list_del_init(&cpuctx->rotation_list);
  6314. }
  6315. static void __perf_event_exit_context(void *__info)
  6316. {
  6317. struct perf_event_context *ctx = __info;
  6318. struct perf_event *event, *tmp;
  6319. perf_pmu_rotate_stop(ctx->pmu);
  6320. list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
  6321. __perf_remove_from_context(event);
  6322. list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
  6323. __perf_remove_from_context(event);
  6324. }
  6325. static void perf_event_exit_cpu_context(int cpu)
  6326. {
  6327. struct perf_event_context *ctx;
  6328. struct pmu *pmu;
  6329. int idx;
  6330. idx = srcu_read_lock(&pmus_srcu);
  6331. list_for_each_entry_rcu(pmu, &pmus, entry) {
  6332. ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
  6333. mutex_lock(&ctx->mutex);
  6334. smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
  6335. mutex_unlock(&ctx->mutex);
  6336. }
  6337. srcu_read_unlock(&pmus_srcu, idx);
  6338. }
  6339. static void perf_event_exit_cpu(int cpu)
  6340. {
  6341. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  6342. mutex_lock(&swhash->hlist_mutex);
  6343. swevent_hlist_release(swhash);
  6344. mutex_unlock(&swhash->hlist_mutex);
  6345. perf_event_exit_cpu_context(cpu);
  6346. }
  6347. #else
  6348. static inline void perf_event_exit_cpu(int cpu) { }
  6349. #endif
  6350. static int
  6351. perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
  6352. {
  6353. int cpu;
  6354. for_each_online_cpu(cpu)
  6355. perf_event_exit_cpu(cpu);
  6356. return NOTIFY_OK;
  6357. }
  6358. /*
  6359. * Run the perf reboot notifier at the very last possible moment so that
  6360. * the generic watchdog code runs as long as possible.
  6361. */
  6362. static struct notifier_block perf_reboot_notifier = {
  6363. .notifier_call = perf_reboot,
  6364. .priority = INT_MIN,
  6365. };
  6366. static int
  6367. perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
  6368. {
  6369. unsigned int cpu = (long)hcpu;
  6370. switch (action & ~CPU_TASKS_FROZEN) {
  6371. case CPU_UP_PREPARE:
  6372. case CPU_DOWN_FAILED:
  6373. perf_event_init_cpu(cpu);
  6374. break;
  6375. case CPU_UP_CANCELED:
  6376. case CPU_DOWN_PREPARE:
  6377. perf_event_exit_cpu(cpu);
  6378. break;
  6379. default:
  6380. break;
  6381. }
  6382. return NOTIFY_OK;
  6383. }
  6384. void __init perf_event_init(void)
  6385. {
  6386. int ret;
  6387. idr_init(&pmu_idr);
  6388. perf_event_init_all_cpus();
  6389. init_srcu_struct(&pmus_srcu);
  6390. perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
  6391. perf_pmu_register(&perf_cpu_clock, NULL, -1);
  6392. perf_pmu_register(&perf_task_clock, NULL, -1);
  6393. perf_tp_register();
  6394. perf_cpu_notifier(perf_cpu_notify);
  6395. register_reboot_notifier(&perf_reboot_notifier);
  6396. ret = init_hw_breakpoint();
  6397. WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
  6398. /* do not patch jump label more than once per second */
  6399. jump_label_rate_limit(&perf_sched_events, HZ);
  6400. /*
  6401. * Build time assertion that we keep the data_head at the intended
  6402. * location. IOW, validation we got the __reserved[] size right.
  6403. */
  6404. BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
  6405. != 1024);
  6406. }
  6407. static int __init perf_event_sysfs_init(void)
  6408. {
  6409. struct pmu *pmu;
  6410. int ret;
  6411. mutex_lock(&pmus_lock);
  6412. ret = bus_register(&pmu_bus);
  6413. if (ret)
  6414. goto unlock;
  6415. list_for_each_entry(pmu, &pmus, entry) {
  6416. if (!pmu->name || pmu->type < 0)
  6417. continue;
  6418. ret = pmu_dev_alloc(pmu);
  6419. WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
  6420. }
  6421. pmu_bus_running = 1;
  6422. ret = 0;
  6423. unlock:
  6424. mutex_unlock(&pmus_lock);
  6425. return ret;
  6426. }
  6427. device_initcall(perf_event_sysfs_init);
  6428. #ifdef CONFIG_CGROUP_PERF
  6429. static struct cgroup_subsys_state *perf_cgroup_css_alloc(struct cgroup *cont)
  6430. {
  6431. struct perf_cgroup *jc;
  6432. jc = kzalloc(sizeof(*jc), GFP_KERNEL);
  6433. if (!jc)
  6434. return ERR_PTR(-ENOMEM);
  6435. jc->info = alloc_percpu(struct perf_cgroup_info);
  6436. if (!jc->info) {
  6437. kfree(jc);
  6438. return ERR_PTR(-ENOMEM);
  6439. }
  6440. return &jc->css;
  6441. }
  6442. static void perf_cgroup_css_free(struct cgroup *cont)
  6443. {
  6444. struct perf_cgroup *jc;
  6445. jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
  6446. struct perf_cgroup, css);
  6447. free_percpu(jc->info);
  6448. kfree(jc);
  6449. }
  6450. static int __perf_cgroup_move(void *info)
  6451. {
  6452. struct task_struct *task = info;
  6453. perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
  6454. return 0;
  6455. }
  6456. static void perf_cgroup_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
  6457. {
  6458. struct task_struct *task;
  6459. cgroup_taskset_for_each(task, cgrp, tset)
  6460. task_function_call(task, __perf_cgroup_move, task);
  6461. }
  6462. static void perf_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
  6463. struct task_struct *task)
  6464. {
  6465. /*
  6466. * cgroup_exit() is called in the copy_process() failure path.
  6467. * Ignore this case since the task hasn't ran yet, this avoids
  6468. * trying to poke a half freed task state from generic code.
  6469. */
  6470. if (!(task->flags & PF_EXITING))
  6471. return;
  6472. task_function_call(task, __perf_cgroup_move, task);
  6473. }
  6474. struct cgroup_subsys perf_subsys = {
  6475. .name = "perf_event",
  6476. .subsys_id = perf_subsys_id,
  6477. .css_alloc = perf_cgroup_css_alloc,
  6478. .css_free = perf_cgroup_css_free,
  6479. .exit = perf_cgroup_exit,
  6480. .attach = perf_cgroup_attach,
  6481. };
  6482. #endif /* CONFIG_CGROUP_PERF */