core.c 185 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998
  1. /*
  2. * Performance events core code:
  3. *
  4. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  6. * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  7. * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  8. *
  9. * For licensing details see kernel-base/COPYING
  10. */
  11. #include <linux/fs.h>
  12. #include <linux/mm.h>
  13. #include <linux/cpu.h>
  14. #include <linux/smp.h>
  15. #include <linux/idr.h>
  16. #include <linux/file.h>
  17. #include <linux/poll.h>
  18. #include <linux/slab.h>
  19. #include <linux/hash.h>
  20. #include <linux/tick.h>
  21. #include <linux/sysfs.h>
  22. #include <linux/dcache.h>
  23. #include <linux/percpu.h>
  24. #include <linux/ptrace.h>
  25. #include <linux/reboot.h>
  26. #include <linux/vmstat.h>
  27. #include <linux/device.h>
  28. #include <linux/export.h>
  29. #include <linux/vmalloc.h>
  30. #include <linux/hardirq.h>
  31. #include <linux/rculist.h>
  32. #include <linux/uaccess.h>
  33. #include <linux/syscalls.h>
  34. #include <linux/anon_inodes.h>
  35. #include <linux/kernel_stat.h>
  36. #include <linux/perf_event.h>
  37. #include <linux/ftrace_event.h>
  38. #include <linux/hw_breakpoint.h>
  39. #include <linux/mm_types.h>
  40. #include <linux/cgroup.h>
  41. #include "internal.h"
  42. #include <asm/irq_regs.h>
  43. struct remote_function_call {
  44. struct task_struct *p;
  45. int (*func)(void *info);
  46. void *info;
  47. int ret;
  48. };
  49. static void remote_function(void *data)
  50. {
  51. struct remote_function_call *tfc = data;
  52. struct task_struct *p = tfc->p;
  53. if (p) {
  54. tfc->ret = -EAGAIN;
  55. if (task_cpu(p) != smp_processor_id() || !task_curr(p))
  56. return;
  57. }
  58. tfc->ret = tfc->func(tfc->info);
  59. }
  60. /**
  61. * task_function_call - call a function on the cpu on which a task runs
  62. * @p: the task to evaluate
  63. * @func: the function to be called
  64. * @info: the function call argument
  65. *
  66. * Calls the function @func when the task is currently running. This might
  67. * be on the current CPU, which just calls the function directly
  68. *
  69. * returns: @func return value, or
  70. * -ESRCH - when the process isn't running
  71. * -EAGAIN - when the process moved away
  72. */
  73. static int
  74. task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
  75. {
  76. struct remote_function_call data = {
  77. .p = p,
  78. .func = func,
  79. .info = info,
  80. .ret = -ESRCH, /* No such (running) process */
  81. };
  82. if (task_curr(p))
  83. smp_call_function_single(task_cpu(p), remote_function, &data, 1);
  84. return data.ret;
  85. }
  86. /**
  87. * cpu_function_call - call a function on the cpu
  88. * @func: the function to be called
  89. * @info: the function call argument
  90. *
  91. * Calls the function @func on the remote cpu.
  92. *
  93. * returns: @func return value or -ENXIO when the cpu is offline
  94. */
  95. static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
  96. {
  97. struct remote_function_call data = {
  98. .p = NULL,
  99. .func = func,
  100. .info = info,
  101. .ret = -ENXIO, /* No such CPU */
  102. };
  103. smp_call_function_single(cpu, remote_function, &data, 1);
  104. return data.ret;
  105. }
  106. #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
  107. PERF_FLAG_FD_OUTPUT |\
  108. PERF_FLAG_PID_CGROUP)
  109. /*
  110. * branch priv levels that need permission checks
  111. */
  112. #define PERF_SAMPLE_BRANCH_PERM_PLM \
  113. (PERF_SAMPLE_BRANCH_KERNEL |\
  114. PERF_SAMPLE_BRANCH_HV)
  115. enum event_type_t {
  116. EVENT_FLEXIBLE = 0x1,
  117. EVENT_PINNED = 0x2,
  118. EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
  119. };
  120. /*
  121. * perf_sched_events : >0 events exist
  122. * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
  123. */
  124. struct static_key_deferred perf_sched_events __read_mostly;
  125. static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
  126. static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
  127. static atomic_t nr_mmap_events __read_mostly;
  128. static atomic_t nr_comm_events __read_mostly;
  129. static atomic_t nr_task_events __read_mostly;
  130. static atomic_t nr_freq_events __read_mostly;
  131. static LIST_HEAD(pmus);
  132. static DEFINE_MUTEX(pmus_lock);
  133. static struct srcu_struct pmus_srcu;
  134. /*
  135. * perf event paranoia level:
  136. * -1 - not paranoid at all
  137. * 0 - disallow raw tracepoint access for unpriv
  138. * 1 - disallow cpu events for unpriv
  139. * 2 - disallow kernel profiling for unpriv
  140. */
  141. int sysctl_perf_event_paranoid __read_mostly = 1;
  142. /* Minimum for 512 kiB + 1 user control page */
  143. int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
  144. /*
  145. * max perf event sample rate
  146. */
  147. #define DEFAULT_MAX_SAMPLE_RATE 100000
  148. #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
  149. #define DEFAULT_CPU_TIME_MAX_PERCENT 25
  150. int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
  151. static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
  152. static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS;
  153. static atomic_t perf_sample_allowed_ns __read_mostly =
  154. ATOMIC_INIT( DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100);
  155. void update_perf_cpu_limits(void)
  156. {
  157. u64 tmp = perf_sample_period_ns;
  158. tmp *= sysctl_perf_cpu_time_max_percent;
  159. do_div(tmp, 100);
  160. atomic_set(&perf_sample_allowed_ns, tmp);
  161. }
  162. static int perf_rotate_context(struct perf_cpu_context *cpuctx);
  163. int perf_proc_update_handler(struct ctl_table *table, int write,
  164. void __user *buffer, size_t *lenp,
  165. loff_t *ppos)
  166. {
  167. int ret = proc_dointvec(table, write, buffer, lenp, ppos);
  168. if (ret || !write)
  169. return ret;
  170. max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
  171. perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
  172. update_perf_cpu_limits();
  173. return 0;
  174. }
  175. int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
  176. int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
  177. void __user *buffer, size_t *lenp,
  178. loff_t *ppos)
  179. {
  180. int ret = proc_dointvec(table, write, buffer, lenp, ppos);
  181. if (ret || !write)
  182. return ret;
  183. update_perf_cpu_limits();
  184. return 0;
  185. }
  186. /*
  187. * perf samples are done in some very critical code paths (NMIs).
  188. * If they take too much CPU time, the system can lock up and not
  189. * get any real work done. This will drop the sample rate when
  190. * we detect that events are taking too long.
  191. */
  192. #define NR_ACCUMULATED_SAMPLES 128
  193. DEFINE_PER_CPU(u64, running_sample_length);
  194. void perf_sample_event_took(u64 sample_len_ns)
  195. {
  196. u64 avg_local_sample_len;
  197. u64 local_samples_len;
  198. if (atomic_read(&perf_sample_allowed_ns) == 0)
  199. return;
  200. /* decay the counter by 1 average sample */
  201. local_samples_len = __get_cpu_var(running_sample_length);
  202. local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES;
  203. local_samples_len += sample_len_ns;
  204. __get_cpu_var(running_sample_length) = local_samples_len;
  205. /*
  206. * note: this will be biased artifically low until we have
  207. * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
  208. * from having to maintain a count.
  209. */
  210. avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
  211. if (avg_local_sample_len <= atomic_read(&perf_sample_allowed_ns))
  212. return;
  213. if (max_samples_per_tick <= 1)
  214. return;
  215. max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2);
  216. sysctl_perf_event_sample_rate = max_samples_per_tick * HZ;
  217. perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
  218. printk_ratelimited(KERN_WARNING
  219. "perf samples too long (%lld > %d), lowering "
  220. "kernel.perf_event_max_sample_rate to %d\n",
  221. avg_local_sample_len,
  222. atomic_read(&perf_sample_allowed_ns),
  223. sysctl_perf_event_sample_rate);
  224. update_perf_cpu_limits();
  225. }
  226. static atomic64_t perf_event_id;
  227. static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
  228. enum event_type_t event_type);
  229. static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
  230. enum event_type_t event_type,
  231. struct task_struct *task);
  232. static void update_context_time(struct perf_event_context *ctx);
  233. static u64 perf_event_time(struct perf_event *event);
  234. void __weak perf_event_print_debug(void) { }
  235. extern __weak const char *perf_pmu_name(void)
  236. {
  237. return "pmu";
  238. }
  239. static inline u64 perf_clock(void)
  240. {
  241. return local_clock();
  242. }
  243. static inline struct perf_cpu_context *
  244. __get_cpu_context(struct perf_event_context *ctx)
  245. {
  246. return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
  247. }
  248. static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
  249. struct perf_event_context *ctx)
  250. {
  251. raw_spin_lock(&cpuctx->ctx.lock);
  252. if (ctx)
  253. raw_spin_lock(&ctx->lock);
  254. }
  255. static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
  256. struct perf_event_context *ctx)
  257. {
  258. if (ctx)
  259. raw_spin_unlock(&ctx->lock);
  260. raw_spin_unlock(&cpuctx->ctx.lock);
  261. }
  262. #ifdef CONFIG_CGROUP_PERF
  263. /*
  264. * perf_cgroup_info keeps track of time_enabled for a cgroup.
  265. * This is a per-cpu dynamically allocated data structure.
  266. */
  267. struct perf_cgroup_info {
  268. u64 time;
  269. u64 timestamp;
  270. };
  271. struct perf_cgroup {
  272. struct cgroup_subsys_state css;
  273. struct perf_cgroup_info __percpu *info;
  274. };
  275. /*
  276. * Must ensure cgroup is pinned (css_get) before calling
  277. * this function. In other words, we cannot call this function
  278. * if there is no cgroup event for the current CPU context.
  279. */
  280. static inline struct perf_cgroup *
  281. perf_cgroup_from_task(struct task_struct *task)
  282. {
  283. return container_of(task_css(task, perf_subsys_id),
  284. struct perf_cgroup, css);
  285. }
  286. static inline bool
  287. perf_cgroup_match(struct perf_event *event)
  288. {
  289. struct perf_event_context *ctx = event->ctx;
  290. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  291. /* @event doesn't care about cgroup */
  292. if (!event->cgrp)
  293. return true;
  294. /* wants specific cgroup scope but @cpuctx isn't associated with any */
  295. if (!cpuctx->cgrp)
  296. return false;
  297. /*
  298. * Cgroup scoping is recursive. An event enabled for a cgroup is
  299. * also enabled for all its descendant cgroups. If @cpuctx's
  300. * cgroup is a descendant of @event's (the test covers identity
  301. * case), it's a match.
  302. */
  303. return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
  304. event->cgrp->css.cgroup);
  305. }
  306. static inline bool perf_tryget_cgroup(struct perf_event *event)
  307. {
  308. return css_tryget(&event->cgrp->css);
  309. }
  310. static inline void perf_put_cgroup(struct perf_event *event)
  311. {
  312. css_put(&event->cgrp->css);
  313. }
  314. static inline void perf_detach_cgroup(struct perf_event *event)
  315. {
  316. perf_put_cgroup(event);
  317. event->cgrp = NULL;
  318. }
  319. static inline int is_cgroup_event(struct perf_event *event)
  320. {
  321. return event->cgrp != NULL;
  322. }
  323. static inline u64 perf_cgroup_event_time(struct perf_event *event)
  324. {
  325. struct perf_cgroup_info *t;
  326. t = per_cpu_ptr(event->cgrp->info, event->cpu);
  327. return t->time;
  328. }
  329. static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
  330. {
  331. struct perf_cgroup_info *info;
  332. u64 now;
  333. now = perf_clock();
  334. info = this_cpu_ptr(cgrp->info);
  335. info->time += now - info->timestamp;
  336. info->timestamp = now;
  337. }
  338. static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
  339. {
  340. struct perf_cgroup *cgrp_out = cpuctx->cgrp;
  341. if (cgrp_out)
  342. __update_cgrp_time(cgrp_out);
  343. }
  344. static inline void update_cgrp_time_from_event(struct perf_event *event)
  345. {
  346. struct perf_cgroup *cgrp;
  347. /*
  348. * ensure we access cgroup data only when needed and
  349. * when we know the cgroup is pinned (css_get)
  350. */
  351. if (!is_cgroup_event(event))
  352. return;
  353. cgrp = perf_cgroup_from_task(current);
  354. /*
  355. * Do not update time when cgroup is not active
  356. */
  357. if (cgrp == event->cgrp)
  358. __update_cgrp_time(event->cgrp);
  359. }
  360. static inline void
  361. perf_cgroup_set_timestamp(struct task_struct *task,
  362. struct perf_event_context *ctx)
  363. {
  364. struct perf_cgroup *cgrp;
  365. struct perf_cgroup_info *info;
  366. /*
  367. * ctx->lock held by caller
  368. * ensure we do not access cgroup data
  369. * unless we have the cgroup pinned (css_get)
  370. */
  371. if (!task || !ctx->nr_cgroups)
  372. return;
  373. cgrp = perf_cgroup_from_task(task);
  374. info = this_cpu_ptr(cgrp->info);
  375. info->timestamp = ctx->timestamp;
  376. }
  377. #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */
  378. #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */
  379. /*
  380. * reschedule events based on the cgroup constraint of task.
  381. *
  382. * mode SWOUT : schedule out everything
  383. * mode SWIN : schedule in based on cgroup for next
  384. */
  385. void perf_cgroup_switch(struct task_struct *task, int mode)
  386. {
  387. struct perf_cpu_context *cpuctx;
  388. struct pmu *pmu;
  389. unsigned long flags;
  390. /*
  391. * disable interrupts to avoid geting nr_cgroup
  392. * changes via __perf_event_disable(). Also
  393. * avoids preemption.
  394. */
  395. local_irq_save(flags);
  396. /*
  397. * we reschedule only in the presence of cgroup
  398. * constrained events.
  399. */
  400. rcu_read_lock();
  401. list_for_each_entry_rcu(pmu, &pmus, entry) {
  402. cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  403. if (cpuctx->unique_pmu != pmu)
  404. continue; /* ensure we process each cpuctx once */
  405. /*
  406. * perf_cgroup_events says at least one
  407. * context on this CPU has cgroup events.
  408. *
  409. * ctx->nr_cgroups reports the number of cgroup
  410. * events for a context.
  411. */
  412. if (cpuctx->ctx.nr_cgroups > 0) {
  413. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  414. perf_pmu_disable(cpuctx->ctx.pmu);
  415. if (mode & PERF_CGROUP_SWOUT) {
  416. cpu_ctx_sched_out(cpuctx, EVENT_ALL);
  417. /*
  418. * must not be done before ctxswout due
  419. * to event_filter_match() in event_sched_out()
  420. */
  421. cpuctx->cgrp = NULL;
  422. }
  423. if (mode & PERF_CGROUP_SWIN) {
  424. WARN_ON_ONCE(cpuctx->cgrp);
  425. /*
  426. * set cgrp before ctxsw in to allow
  427. * event_filter_match() to not have to pass
  428. * task around
  429. */
  430. cpuctx->cgrp = perf_cgroup_from_task(task);
  431. cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
  432. }
  433. perf_pmu_enable(cpuctx->ctx.pmu);
  434. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  435. }
  436. }
  437. rcu_read_unlock();
  438. local_irq_restore(flags);
  439. }
  440. static inline void perf_cgroup_sched_out(struct task_struct *task,
  441. struct task_struct *next)
  442. {
  443. struct perf_cgroup *cgrp1;
  444. struct perf_cgroup *cgrp2 = NULL;
  445. /*
  446. * we come here when we know perf_cgroup_events > 0
  447. */
  448. cgrp1 = perf_cgroup_from_task(task);
  449. /*
  450. * next is NULL when called from perf_event_enable_on_exec()
  451. * that will systematically cause a cgroup_switch()
  452. */
  453. if (next)
  454. cgrp2 = perf_cgroup_from_task(next);
  455. /*
  456. * only schedule out current cgroup events if we know
  457. * that we are switching to a different cgroup. Otherwise,
  458. * do no touch the cgroup events.
  459. */
  460. if (cgrp1 != cgrp2)
  461. perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
  462. }
  463. static inline void perf_cgroup_sched_in(struct task_struct *prev,
  464. struct task_struct *task)
  465. {
  466. struct perf_cgroup *cgrp1;
  467. struct perf_cgroup *cgrp2 = NULL;
  468. /*
  469. * we come here when we know perf_cgroup_events > 0
  470. */
  471. cgrp1 = perf_cgroup_from_task(task);
  472. /* prev can never be NULL */
  473. cgrp2 = perf_cgroup_from_task(prev);
  474. /*
  475. * only need to schedule in cgroup events if we are changing
  476. * cgroup during ctxsw. Cgroup events were not scheduled
  477. * out of ctxsw out if that was not the case.
  478. */
  479. if (cgrp1 != cgrp2)
  480. perf_cgroup_switch(task, PERF_CGROUP_SWIN);
  481. }
  482. static inline int perf_cgroup_connect(int fd, struct perf_event *event,
  483. struct perf_event_attr *attr,
  484. struct perf_event *group_leader)
  485. {
  486. struct perf_cgroup *cgrp;
  487. struct cgroup_subsys_state *css;
  488. struct fd f = fdget(fd);
  489. int ret = 0;
  490. if (!f.file)
  491. return -EBADF;
  492. rcu_read_lock();
  493. css = css_from_dir(f.file->f_dentry, &perf_subsys);
  494. if (IS_ERR(css)) {
  495. ret = PTR_ERR(css);
  496. goto out;
  497. }
  498. cgrp = container_of(css, struct perf_cgroup, css);
  499. event->cgrp = cgrp;
  500. /* must be done before we fput() the file */
  501. if (!perf_tryget_cgroup(event)) {
  502. event->cgrp = NULL;
  503. ret = -ENOENT;
  504. goto out;
  505. }
  506. /*
  507. * all events in a group must monitor
  508. * the same cgroup because a task belongs
  509. * to only one perf cgroup at a time
  510. */
  511. if (group_leader && group_leader->cgrp != cgrp) {
  512. perf_detach_cgroup(event);
  513. ret = -EINVAL;
  514. }
  515. out:
  516. rcu_read_unlock();
  517. fdput(f);
  518. return ret;
  519. }
  520. static inline void
  521. perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
  522. {
  523. struct perf_cgroup_info *t;
  524. t = per_cpu_ptr(event->cgrp->info, event->cpu);
  525. event->shadow_ctx_time = now - t->timestamp;
  526. }
  527. static inline void
  528. perf_cgroup_defer_enabled(struct perf_event *event)
  529. {
  530. /*
  531. * when the current task's perf cgroup does not match
  532. * the event's, we need to remember to call the
  533. * perf_mark_enable() function the first time a task with
  534. * a matching perf cgroup is scheduled in.
  535. */
  536. if (is_cgroup_event(event) && !perf_cgroup_match(event))
  537. event->cgrp_defer_enabled = 1;
  538. }
  539. static inline void
  540. perf_cgroup_mark_enabled(struct perf_event *event,
  541. struct perf_event_context *ctx)
  542. {
  543. struct perf_event *sub;
  544. u64 tstamp = perf_event_time(event);
  545. if (!event->cgrp_defer_enabled)
  546. return;
  547. event->cgrp_defer_enabled = 0;
  548. event->tstamp_enabled = tstamp - event->total_time_enabled;
  549. list_for_each_entry(sub, &event->sibling_list, group_entry) {
  550. if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
  551. sub->tstamp_enabled = tstamp - sub->total_time_enabled;
  552. sub->cgrp_defer_enabled = 0;
  553. }
  554. }
  555. }
  556. #else /* !CONFIG_CGROUP_PERF */
  557. static inline bool
  558. perf_cgroup_match(struct perf_event *event)
  559. {
  560. return true;
  561. }
  562. static inline void perf_detach_cgroup(struct perf_event *event)
  563. {}
  564. static inline int is_cgroup_event(struct perf_event *event)
  565. {
  566. return 0;
  567. }
  568. static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
  569. {
  570. return 0;
  571. }
  572. static inline void update_cgrp_time_from_event(struct perf_event *event)
  573. {
  574. }
  575. static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
  576. {
  577. }
  578. static inline void perf_cgroup_sched_out(struct task_struct *task,
  579. struct task_struct *next)
  580. {
  581. }
  582. static inline void perf_cgroup_sched_in(struct task_struct *prev,
  583. struct task_struct *task)
  584. {
  585. }
  586. static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
  587. struct perf_event_attr *attr,
  588. struct perf_event *group_leader)
  589. {
  590. return -EINVAL;
  591. }
  592. static inline void
  593. perf_cgroup_set_timestamp(struct task_struct *task,
  594. struct perf_event_context *ctx)
  595. {
  596. }
  597. void
  598. perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
  599. {
  600. }
  601. static inline void
  602. perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
  603. {
  604. }
  605. static inline u64 perf_cgroup_event_time(struct perf_event *event)
  606. {
  607. return 0;
  608. }
  609. static inline void
  610. perf_cgroup_defer_enabled(struct perf_event *event)
  611. {
  612. }
  613. static inline void
  614. perf_cgroup_mark_enabled(struct perf_event *event,
  615. struct perf_event_context *ctx)
  616. {
  617. }
  618. #endif
  619. /*
  620. * set default to be dependent on timer tick just
  621. * like original code
  622. */
  623. #define PERF_CPU_HRTIMER (1000 / HZ)
  624. /*
  625. * function must be called with interrupts disbled
  626. */
  627. static enum hrtimer_restart perf_cpu_hrtimer_handler(struct hrtimer *hr)
  628. {
  629. struct perf_cpu_context *cpuctx;
  630. enum hrtimer_restart ret = HRTIMER_NORESTART;
  631. int rotations = 0;
  632. WARN_ON(!irqs_disabled());
  633. cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
  634. rotations = perf_rotate_context(cpuctx);
  635. /*
  636. * arm timer if needed
  637. */
  638. if (rotations) {
  639. hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
  640. ret = HRTIMER_RESTART;
  641. }
  642. return ret;
  643. }
  644. /* CPU is going down */
  645. void perf_cpu_hrtimer_cancel(int cpu)
  646. {
  647. struct perf_cpu_context *cpuctx;
  648. struct pmu *pmu;
  649. unsigned long flags;
  650. if (WARN_ON(cpu != smp_processor_id()))
  651. return;
  652. local_irq_save(flags);
  653. rcu_read_lock();
  654. list_for_each_entry_rcu(pmu, &pmus, entry) {
  655. cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  656. if (pmu->task_ctx_nr == perf_sw_context)
  657. continue;
  658. hrtimer_cancel(&cpuctx->hrtimer);
  659. }
  660. rcu_read_unlock();
  661. local_irq_restore(flags);
  662. }
  663. static void __perf_cpu_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
  664. {
  665. struct hrtimer *hr = &cpuctx->hrtimer;
  666. struct pmu *pmu = cpuctx->ctx.pmu;
  667. int timer;
  668. /* no multiplexing needed for SW PMU */
  669. if (pmu->task_ctx_nr == perf_sw_context)
  670. return;
  671. /*
  672. * check default is sane, if not set then force to
  673. * default interval (1/tick)
  674. */
  675. timer = pmu->hrtimer_interval_ms;
  676. if (timer < 1)
  677. timer = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
  678. cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
  679. hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
  680. hr->function = perf_cpu_hrtimer_handler;
  681. }
  682. static void perf_cpu_hrtimer_restart(struct perf_cpu_context *cpuctx)
  683. {
  684. struct hrtimer *hr = &cpuctx->hrtimer;
  685. struct pmu *pmu = cpuctx->ctx.pmu;
  686. /* not for SW PMU */
  687. if (pmu->task_ctx_nr == perf_sw_context)
  688. return;
  689. if (hrtimer_active(hr))
  690. return;
  691. if (!hrtimer_callback_running(hr))
  692. __hrtimer_start_range_ns(hr, cpuctx->hrtimer_interval,
  693. 0, HRTIMER_MODE_REL_PINNED, 0);
  694. }
  695. void perf_pmu_disable(struct pmu *pmu)
  696. {
  697. int *count = this_cpu_ptr(pmu->pmu_disable_count);
  698. if (!(*count)++)
  699. pmu->pmu_disable(pmu);
  700. }
  701. void perf_pmu_enable(struct pmu *pmu)
  702. {
  703. int *count = this_cpu_ptr(pmu->pmu_disable_count);
  704. if (!--(*count))
  705. pmu->pmu_enable(pmu);
  706. }
  707. static DEFINE_PER_CPU(struct list_head, rotation_list);
  708. /*
  709. * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
  710. * because they're strictly cpu affine and rotate_start is called with IRQs
  711. * disabled, while rotate_context is called from IRQ context.
  712. */
  713. static void perf_pmu_rotate_start(struct pmu *pmu)
  714. {
  715. struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  716. struct list_head *head = &__get_cpu_var(rotation_list);
  717. WARN_ON(!irqs_disabled());
  718. if (list_empty(&cpuctx->rotation_list))
  719. list_add(&cpuctx->rotation_list, head);
  720. }
  721. static void get_ctx(struct perf_event_context *ctx)
  722. {
  723. WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
  724. }
  725. static void put_ctx(struct perf_event_context *ctx)
  726. {
  727. if (atomic_dec_and_test(&ctx->refcount)) {
  728. if (ctx->parent_ctx)
  729. put_ctx(ctx->parent_ctx);
  730. if (ctx->task)
  731. put_task_struct(ctx->task);
  732. kfree_rcu(ctx, rcu_head);
  733. }
  734. }
  735. static void unclone_ctx(struct perf_event_context *ctx)
  736. {
  737. if (ctx->parent_ctx) {
  738. put_ctx(ctx->parent_ctx);
  739. ctx->parent_ctx = NULL;
  740. }
  741. }
  742. static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
  743. {
  744. /*
  745. * only top level events have the pid namespace they were created in
  746. */
  747. if (event->parent)
  748. event = event->parent;
  749. return task_tgid_nr_ns(p, event->ns);
  750. }
  751. static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
  752. {
  753. /*
  754. * only top level events have the pid namespace they were created in
  755. */
  756. if (event->parent)
  757. event = event->parent;
  758. return task_pid_nr_ns(p, event->ns);
  759. }
  760. /*
  761. * If we inherit events we want to return the parent event id
  762. * to userspace.
  763. */
  764. static u64 primary_event_id(struct perf_event *event)
  765. {
  766. u64 id = event->id;
  767. if (event->parent)
  768. id = event->parent->id;
  769. return id;
  770. }
  771. /*
  772. * Get the perf_event_context for a task and lock it.
  773. * This has to cope with with the fact that until it is locked,
  774. * the context could get moved to another task.
  775. */
  776. static struct perf_event_context *
  777. perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
  778. {
  779. struct perf_event_context *ctx;
  780. retry:
  781. /*
  782. * One of the few rules of preemptible RCU is that one cannot do
  783. * rcu_read_unlock() while holding a scheduler (or nested) lock when
  784. * part of the read side critical section was preemptible -- see
  785. * rcu_read_unlock_special().
  786. *
  787. * Since ctx->lock nests under rq->lock we must ensure the entire read
  788. * side critical section is non-preemptible.
  789. */
  790. preempt_disable();
  791. rcu_read_lock();
  792. ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
  793. if (ctx) {
  794. /*
  795. * If this context is a clone of another, it might
  796. * get swapped for another underneath us by
  797. * perf_event_task_sched_out, though the
  798. * rcu_read_lock() protects us from any context
  799. * getting freed. Lock the context and check if it
  800. * got swapped before we could get the lock, and retry
  801. * if so. If we locked the right context, then it
  802. * can't get swapped on us any more.
  803. */
  804. raw_spin_lock_irqsave(&ctx->lock, *flags);
  805. if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
  806. raw_spin_unlock_irqrestore(&ctx->lock, *flags);
  807. rcu_read_unlock();
  808. preempt_enable();
  809. goto retry;
  810. }
  811. if (!atomic_inc_not_zero(&ctx->refcount)) {
  812. raw_spin_unlock_irqrestore(&ctx->lock, *flags);
  813. ctx = NULL;
  814. }
  815. }
  816. rcu_read_unlock();
  817. preempt_enable();
  818. return ctx;
  819. }
  820. /*
  821. * Get the context for a task and increment its pin_count so it
  822. * can't get swapped to another task. This also increments its
  823. * reference count so that the context can't get freed.
  824. */
  825. static struct perf_event_context *
  826. perf_pin_task_context(struct task_struct *task, int ctxn)
  827. {
  828. struct perf_event_context *ctx;
  829. unsigned long flags;
  830. ctx = perf_lock_task_context(task, ctxn, &flags);
  831. if (ctx) {
  832. ++ctx->pin_count;
  833. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  834. }
  835. return ctx;
  836. }
  837. static void perf_unpin_context(struct perf_event_context *ctx)
  838. {
  839. unsigned long flags;
  840. raw_spin_lock_irqsave(&ctx->lock, flags);
  841. --ctx->pin_count;
  842. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  843. }
  844. /*
  845. * Update the record of the current time in a context.
  846. */
  847. static void update_context_time(struct perf_event_context *ctx)
  848. {
  849. u64 now = perf_clock();
  850. ctx->time += now - ctx->timestamp;
  851. ctx->timestamp = now;
  852. }
  853. static u64 perf_event_time(struct perf_event *event)
  854. {
  855. struct perf_event_context *ctx = event->ctx;
  856. if (is_cgroup_event(event))
  857. return perf_cgroup_event_time(event);
  858. return ctx ? ctx->time : 0;
  859. }
  860. /*
  861. * Update the total_time_enabled and total_time_running fields for a event.
  862. * The caller of this function needs to hold the ctx->lock.
  863. */
  864. static void update_event_times(struct perf_event *event)
  865. {
  866. struct perf_event_context *ctx = event->ctx;
  867. u64 run_end;
  868. if (event->state < PERF_EVENT_STATE_INACTIVE ||
  869. event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
  870. return;
  871. /*
  872. * in cgroup mode, time_enabled represents
  873. * the time the event was enabled AND active
  874. * tasks were in the monitored cgroup. This is
  875. * independent of the activity of the context as
  876. * there may be a mix of cgroup and non-cgroup events.
  877. *
  878. * That is why we treat cgroup events differently
  879. * here.
  880. */
  881. if (is_cgroup_event(event))
  882. run_end = perf_cgroup_event_time(event);
  883. else if (ctx->is_active)
  884. run_end = ctx->time;
  885. else
  886. run_end = event->tstamp_stopped;
  887. event->total_time_enabled = run_end - event->tstamp_enabled;
  888. if (event->state == PERF_EVENT_STATE_INACTIVE)
  889. run_end = event->tstamp_stopped;
  890. else
  891. run_end = perf_event_time(event);
  892. event->total_time_running = run_end - event->tstamp_running;
  893. }
  894. /*
  895. * Update total_time_enabled and total_time_running for all events in a group.
  896. */
  897. static void update_group_times(struct perf_event *leader)
  898. {
  899. struct perf_event *event;
  900. update_event_times(leader);
  901. list_for_each_entry(event, &leader->sibling_list, group_entry)
  902. update_event_times(event);
  903. }
  904. static struct list_head *
  905. ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
  906. {
  907. if (event->attr.pinned)
  908. return &ctx->pinned_groups;
  909. else
  910. return &ctx->flexible_groups;
  911. }
  912. /*
  913. * Add a event from the lists for its context.
  914. * Must be called with ctx->mutex and ctx->lock held.
  915. */
  916. static void
  917. list_add_event(struct perf_event *event, struct perf_event_context *ctx)
  918. {
  919. WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
  920. event->attach_state |= PERF_ATTACH_CONTEXT;
  921. /*
  922. * If we're a stand alone event or group leader, we go to the context
  923. * list, group events are kept attached to the group so that
  924. * perf_group_detach can, at all times, locate all siblings.
  925. */
  926. if (event->group_leader == event) {
  927. struct list_head *list;
  928. if (is_software_event(event))
  929. event->group_flags |= PERF_GROUP_SOFTWARE;
  930. list = ctx_group_list(event, ctx);
  931. list_add_tail(&event->group_entry, list);
  932. }
  933. if (is_cgroup_event(event))
  934. ctx->nr_cgroups++;
  935. if (has_branch_stack(event))
  936. ctx->nr_branch_stack++;
  937. list_add_rcu(&event->event_entry, &ctx->event_list);
  938. if (!ctx->nr_events)
  939. perf_pmu_rotate_start(ctx->pmu);
  940. ctx->nr_events++;
  941. if (event->attr.inherit_stat)
  942. ctx->nr_stat++;
  943. }
  944. /*
  945. * Initialize event state based on the perf_event_attr::disabled.
  946. */
  947. static inline void perf_event__state_init(struct perf_event *event)
  948. {
  949. event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
  950. PERF_EVENT_STATE_INACTIVE;
  951. }
  952. /*
  953. * Called at perf_event creation and when events are attached/detached from a
  954. * group.
  955. */
  956. static void perf_event__read_size(struct perf_event *event)
  957. {
  958. int entry = sizeof(u64); /* value */
  959. int size = 0;
  960. int nr = 1;
  961. if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  962. size += sizeof(u64);
  963. if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  964. size += sizeof(u64);
  965. if (event->attr.read_format & PERF_FORMAT_ID)
  966. entry += sizeof(u64);
  967. if (event->attr.read_format & PERF_FORMAT_GROUP) {
  968. nr += event->group_leader->nr_siblings;
  969. size += sizeof(u64);
  970. }
  971. size += entry * nr;
  972. event->read_size = size;
  973. }
  974. static void perf_event__header_size(struct perf_event *event)
  975. {
  976. struct perf_sample_data *data;
  977. u64 sample_type = event->attr.sample_type;
  978. u16 size = 0;
  979. perf_event__read_size(event);
  980. if (sample_type & PERF_SAMPLE_IP)
  981. size += sizeof(data->ip);
  982. if (sample_type & PERF_SAMPLE_ADDR)
  983. size += sizeof(data->addr);
  984. if (sample_type & PERF_SAMPLE_PERIOD)
  985. size += sizeof(data->period);
  986. if (sample_type & PERF_SAMPLE_WEIGHT)
  987. size += sizeof(data->weight);
  988. if (sample_type & PERF_SAMPLE_READ)
  989. size += event->read_size;
  990. if (sample_type & PERF_SAMPLE_DATA_SRC)
  991. size += sizeof(data->data_src.val);
  992. event->header_size = size;
  993. }
  994. static void perf_event__id_header_size(struct perf_event *event)
  995. {
  996. struct perf_sample_data *data;
  997. u64 sample_type = event->attr.sample_type;
  998. u16 size = 0;
  999. if (sample_type & PERF_SAMPLE_TID)
  1000. size += sizeof(data->tid_entry);
  1001. if (sample_type & PERF_SAMPLE_TIME)
  1002. size += sizeof(data->time);
  1003. if (sample_type & PERF_SAMPLE_IDENTIFIER)
  1004. size += sizeof(data->id);
  1005. if (sample_type & PERF_SAMPLE_ID)
  1006. size += sizeof(data->id);
  1007. if (sample_type & PERF_SAMPLE_STREAM_ID)
  1008. size += sizeof(data->stream_id);
  1009. if (sample_type & PERF_SAMPLE_CPU)
  1010. size += sizeof(data->cpu_entry);
  1011. event->id_header_size = size;
  1012. }
  1013. static void perf_group_attach(struct perf_event *event)
  1014. {
  1015. struct perf_event *group_leader = event->group_leader, *pos;
  1016. /*
  1017. * We can have double attach due to group movement in perf_event_open.
  1018. */
  1019. if (event->attach_state & PERF_ATTACH_GROUP)
  1020. return;
  1021. event->attach_state |= PERF_ATTACH_GROUP;
  1022. if (group_leader == event)
  1023. return;
  1024. if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
  1025. !is_software_event(event))
  1026. group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
  1027. list_add_tail(&event->group_entry, &group_leader->sibling_list);
  1028. group_leader->nr_siblings++;
  1029. perf_event__header_size(group_leader);
  1030. list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
  1031. perf_event__header_size(pos);
  1032. }
  1033. /*
  1034. * Remove a event from the lists for its context.
  1035. * Must be called with ctx->mutex and ctx->lock held.
  1036. */
  1037. static void
  1038. list_del_event(struct perf_event *event, struct perf_event_context *ctx)
  1039. {
  1040. struct perf_cpu_context *cpuctx;
  1041. /*
  1042. * We can have double detach due to exit/hot-unplug + close.
  1043. */
  1044. if (!(event->attach_state & PERF_ATTACH_CONTEXT))
  1045. return;
  1046. event->attach_state &= ~PERF_ATTACH_CONTEXT;
  1047. if (is_cgroup_event(event)) {
  1048. ctx->nr_cgroups--;
  1049. cpuctx = __get_cpu_context(ctx);
  1050. /*
  1051. * if there are no more cgroup events
  1052. * then cler cgrp to avoid stale pointer
  1053. * in update_cgrp_time_from_cpuctx()
  1054. */
  1055. if (!ctx->nr_cgroups)
  1056. cpuctx->cgrp = NULL;
  1057. }
  1058. if (has_branch_stack(event))
  1059. ctx->nr_branch_stack--;
  1060. ctx->nr_events--;
  1061. if (event->attr.inherit_stat)
  1062. ctx->nr_stat--;
  1063. list_del_rcu(&event->event_entry);
  1064. if (event->group_leader == event)
  1065. list_del_init(&event->group_entry);
  1066. update_group_times(event);
  1067. /*
  1068. * If event was in error state, then keep it
  1069. * that way, otherwise bogus counts will be
  1070. * returned on read(). The only way to get out
  1071. * of error state is by explicit re-enabling
  1072. * of the event
  1073. */
  1074. if (event->state > PERF_EVENT_STATE_OFF)
  1075. event->state = PERF_EVENT_STATE_OFF;
  1076. }
  1077. static void perf_group_detach(struct perf_event *event)
  1078. {
  1079. struct perf_event *sibling, *tmp;
  1080. struct list_head *list = NULL;
  1081. /*
  1082. * We can have double detach due to exit/hot-unplug + close.
  1083. */
  1084. if (!(event->attach_state & PERF_ATTACH_GROUP))
  1085. return;
  1086. event->attach_state &= ~PERF_ATTACH_GROUP;
  1087. /*
  1088. * If this is a sibling, remove it from its group.
  1089. */
  1090. if (event->group_leader != event) {
  1091. list_del_init(&event->group_entry);
  1092. event->group_leader->nr_siblings--;
  1093. goto out;
  1094. }
  1095. if (!list_empty(&event->group_entry))
  1096. list = &event->group_entry;
  1097. /*
  1098. * If this was a group event with sibling events then
  1099. * upgrade the siblings to singleton events by adding them
  1100. * to whatever list we are on.
  1101. */
  1102. list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
  1103. if (list)
  1104. list_move_tail(&sibling->group_entry, list);
  1105. sibling->group_leader = sibling;
  1106. /* Inherit group flags from the previous leader */
  1107. sibling->group_flags = event->group_flags;
  1108. }
  1109. out:
  1110. perf_event__header_size(event->group_leader);
  1111. list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
  1112. perf_event__header_size(tmp);
  1113. }
  1114. static inline int
  1115. event_filter_match(struct perf_event *event)
  1116. {
  1117. return (event->cpu == -1 || event->cpu == smp_processor_id())
  1118. && perf_cgroup_match(event);
  1119. }
  1120. static void
  1121. event_sched_out(struct perf_event *event,
  1122. struct perf_cpu_context *cpuctx,
  1123. struct perf_event_context *ctx)
  1124. {
  1125. u64 tstamp = perf_event_time(event);
  1126. u64 delta;
  1127. /*
  1128. * An event which could not be activated because of
  1129. * filter mismatch still needs to have its timings
  1130. * maintained, otherwise bogus information is return
  1131. * via read() for time_enabled, time_running:
  1132. */
  1133. if (event->state == PERF_EVENT_STATE_INACTIVE
  1134. && !event_filter_match(event)) {
  1135. delta = tstamp - event->tstamp_stopped;
  1136. event->tstamp_running += delta;
  1137. event->tstamp_stopped = tstamp;
  1138. }
  1139. if (event->state != PERF_EVENT_STATE_ACTIVE)
  1140. return;
  1141. event->state = PERF_EVENT_STATE_INACTIVE;
  1142. if (event->pending_disable) {
  1143. event->pending_disable = 0;
  1144. event->state = PERF_EVENT_STATE_OFF;
  1145. }
  1146. event->tstamp_stopped = tstamp;
  1147. event->pmu->del(event, 0);
  1148. event->oncpu = -1;
  1149. if (!is_software_event(event))
  1150. cpuctx->active_oncpu--;
  1151. ctx->nr_active--;
  1152. if (event->attr.freq && event->attr.sample_freq)
  1153. ctx->nr_freq--;
  1154. if (event->attr.exclusive || !cpuctx->active_oncpu)
  1155. cpuctx->exclusive = 0;
  1156. }
  1157. static void
  1158. group_sched_out(struct perf_event *group_event,
  1159. struct perf_cpu_context *cpuctx,
  1160. struct perf_event_context *ctx)
  1161. {
  1162. struct perf_event *event;
  1163. int state = group_event->state;
  1164. event_sched_out(group_event, cpuctx, ctx);
  1165. /*
  1166. * Schedule out siblings (if any):
  1167. */
  1168. list_for_each_entry(event, &group_event->sibling_list, group_entry)
  1169. event_sched_out(event, cpuctx, ctx);
  1170. if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
  1171. cpuctx->exclusive = 0;
  1172. }
  1173. /*
  1174. * Cross CPU call to remove a performance event
  1175. *
  1176. * We disable the event on the hardware level first. After that we
  1177. * remove it from the context list.
  1178. */
  1179. static int __perf_remove_from_context(void *info)
  1180. {
  1181. struct perf_event *event = info;
  1182. struct perf_event_context *ctx = event->ctx;
  1183. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1184. raw_spin_lock(&ctx->lock);
  1185. event_sched_out(event, cpuctx, ctx);
  1186. list_del_event(event, ctx);
  1187. if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
  1188. ctx->is_active = 0;
  1189. cpuctx->task_ctx = NULL;
  1190. }
  1191. raw_spin_unlock(&ctx->lock);
  1192. return 0;
  1193. }
  1194. /*
  1195. * Remove the event from a task's (or a CPU's) list of events.
  1196. *
  1197. * CPU events are removed with a smp call. For task events we only
  1198. * call when the task is on a CPU.
  1199. *
  1200. * If event->ctx is a cloned context, callers must make sure that
  1201. * every task struct that event->ctx->task could possibly point to
  1202. * remains valid. This is OK when called from perf_release since
  1203. * that only calls us on the top-level context, which can't be a clone.
  1204. * When called from perf_event_exit_task, it's OK because the
  1205. * context has been detached from its task.
  1206. */
  1207. static void perf_remove_from_context(struct perf_event *event)
  1208. {
  1209. struct perf_event_context *ctx = event->ctx;
  1210. struct task_struct *task = ctx->task;
  1211. lockdep_assert_held(&ctx->mutex);
  1212. if (!task) {
  1213. /*
  1214. * Per cpu events are removed via an smp call and
  1215. * the removal is always successful.
  1216. */
  1217. cpu_function_call(event->cpu, __perf_remove_from_context, event);
  1218. return;
  1219. }
  1220. retry:
  1221. if (!task_function_call(task, __perf_remove_from_context, event))
  1222. return;
  1223. raw_spin_lock_irq(&ctx->lock);
  1224. /*
  1225. * If we failed to find a running task, but find the context active now
  1226. * that we've acquired the ctx->lock, retry.
  1227. */
  1228. if (ctx->is_active) {
  1229. raw_spin_unlock_irq(&ctx->lock);
  1230. goto retry;
  1231. }
  1232. /*
  1233. * Since the task isn't running, its safe to remove the event, us
  1234. * holding the ctx->lock ensures the task won't get scheduled in.
  1235. */
  1236. list_del_event(event, ctx);
  1237. raw_spin_unlock_irq(&ctx->lock);
  1238. }
  1239. /*
  1240. * Cross CPU call to disable a performance event
  1241. */
  1242. int __perf_event_disable(void *info)
  1243. {
  1244. struct perf_event *event = info;
  1245. struct perf_event_context *ctx = event->ctx;
  1246. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1247. /*
  1248. * If this is a per-task event, need to check whether this
  1249. * event's task is the current task on this cpu.
  1250. *
  1251. * Can trigger due to concurrent perf_event_context_sched_out()
  1252. * flipping contexts around.
  1253. */
  1254. if (ctx->task && cpuctx->task_ctx != ctx)
  1255. return -EINVAL;
  1256. raw_spin_lock(&ctx->lock);
  1257. /*
  1258. * If the event is on, turn it off.
  1259. * If it is in error state, leave it in error state.
  1260. */
  1261. if (event->state >= PERF_EVENT_STATE_INACTIVE) {
  1262. update_context_time(ctx);
  1263. update_cgrp_time_from_event(event);
  1264. update_group_times(event);
  1265. if (event == event->group_leader)
  1266. group_sched_out(event, cpuctx, ctx);
  1267. else
  1268. event_sched_out(event, cpuctx, ctx);
  1269. event->state = PERF_EVENT_STATE_OFF;
  1270. }
  1271. raw_spin_unlock(&ctx->lock);
  1272. return 0;
  1273. }
  1274. /*
  1275. * Disable a event.
  1276. *
  1277. * If event->ctx is a cloned context, callers must make sure that
  1278. * every task struct that event->ctx->task could possibly point to
  1279. * remains valid. This condition is satisifed when called through
  1280. * perf_event_for_each_child or perf_event_for_each because they
  1281. * hold the top-level event's child_mutex, so any descendant that
  1282. * goes to exit will block in sync_child_event.
  1283. * When called from perf_pending_event it's OK because event->ctx
  1284. * is the current context on this CPU and preemption is disabled,
  1285. * hence we can't get into perf_event_task_sched_out for this context.
  1286. */
  1287. void perf_event_disable(struct perf_event *event)
  1288. {
  1289. struct perf_event_context *ctx = event->ctx;
  1290. struct task_struct *task = ctx->task;
  1291. if (!task) {
  1292. /*
  1293. * Disable the event on the cpu that it's on
  1294. */
  1295. cpu_function_call(event->cpu, __perf_event_disable, event);
  1296. return;
  1297. }
  1298. retry:
  1299. if (!task_function_call(task, __perf_event_disable, event))
  1300. return;
  1301. raw_spin_lock_irq(&ctx->lock);
  1302. /*
  1303. * If the event is still active, we need to retry the cross-call.
  1304. */
  1305. if (event->state == PERF_EVENT_STATE_ACTIVE) {
  1306. raw_spin_unlock_irq(&ctx->lock);
  1307. /*
  1308. * Reload the task pointer, it might have been changed by
  1309. * a concurrent perf_event_context_sched_out().
  1310. */
  1311. task = ctx->task;
  1312. goto retry;
  1313. }
  1314. /*
  1315. * Since we have the lock this context can't be scheduled
  1316. * in, so we can change the state safely.
  1317. */
  1318. if (event->state == PERF_EVENT_STATE_INACTIVE) {
  1319. update_group_times(event);
  1320. event->state = PERF_EVENT_STATE_OFF;
  1321. }
  1322. raw_spin_unlock_irq(&ctx->lock);
  1323. }
  1324. EXPORT_SYMBOL_GPL(perf_event_disable);
  1325. static void perf_set_shadow_time(struct perf_event *event,
  1326. struct perf_event_context *ctx,
  1327. u64 tstamp)
  1328. {
  1329. /*
  1330. * use the correct time source for the time snapshot
  1331. *
  1332. * We could get by without this by leveraging the
  1333. * fact that to get to this function, the caller
  1334. * has most likely already called update_context_time()
  1335. * and update_cgrp_time_xx() and thus both timestamp
  1336. * are identical (or very close). Given that tstamp is,
  1337. * already adjusted for cgroup, we could say that:
  1338. * tstamp - ctx->timestamp
  1339. * is equivalent to
  1340. * tstamp - cgrp->timestamp.
  1341. *
  1342. * Then, in perf_output_read(), the calculation would
  1343. * work with no changes because:
  1344. * - event is guaranteed scheduled in
  1345. * - no scheduled out in between
  1346. * - thus the timestamp would be the same
  1347. *
  1348. * But this is a bit hairy.
  1349. *
  1350. * So instead, we have an explicit cgroup call to remain
  1351. * within the time time source all along. We believe it
  1352. * is cleaner and simpler to understand.
  1353. */
  1354. if (is_cgroup_event(event))
  1355. perf_cgroup_set_shadow_time(event, tstamp);
  1356. else
  1357. event->shadow_ctx_time = tstamp - ctx->timestamp;
  1358. }
  1359. #define MAX_INTERRUPTS (~0ULL)
  1360. static void perf_log_throttle(struct perf_event *event, int enable);
  1361. static int
  1362. event_sched_in(struct perf_event *event,
  1363. struct perf_cpu_context *cpuctx,
  1364. struct perf_event_context *ctx)
  1365. {
  1366. u64 tstamp = perf_event_time(event);
  1367. if (event->state <= PERF_EVENT_STATE_OFF)
  1368. return 0;
  1369. event->state = PERF_EVENT_STATE_ACTIVE;
  1370. event->oncpu = smp_processor_id();
  1371. /*
  1372. * Unthrottle events, since we scheduled we might have missed several
  1373. * ticks already, also for a heavily scheduling task there is little
  1374. * guarantee it'll get a tick in a timely manner.
  1375. */
  1376. if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
  1377. perf_log_throttle(event, 1);
  1378. event->hw.interrupts = 0;
  1379. }
  1380. /*
  1381. * The new state must be visible before we turn it on in the hardware:
  1382. */
  1383. smp_wmb();
  1384. if (event->pmu->add(event, PERF_EF_START)) {
  1385. event->state = PERF_EVENT_STATE_INACTIVE;
  1386. event->oncpu = -1;
  1387. return -EAGAIN;
  1388. }
  1389. event->tstamp_running += tstamp - event->tstamp_stopped;
  1390. perf_set_shadow_time(event, ctx, tstamp);
  1391. if (!is_software_event(event))
  1392. cpuctx->active_oncpu++;
  1393. ctx->nr_active++;
  1394. if (event->attr.freq && event->attr.sample_freq)
  1395. ctx->nr_freq++;
  1396. if (event->attr.exclusive)
  1397. cpuctx->exclusive = 1;
  1398. return 0;
  1399. }
  1400. static int
  1401. group_sched_in(struct perf_event *group_event,
  1402. struct perf_cpu_context *cpuctx,
  1403. struct perf_event_context *ctx)
  1404. {
  1405. struct perf_event *event, *partial_group = NULL;
  1406. struct pmu *pmu = group_event->pmu;
  1407. u64 now = ctx->time;
  1408. bool simulate = false;
  1409. if (group_event->state == PERF_EVENT_STATE_OFF)
  1410. return 0;
  1411. pmu->start_txn(pmu);
  1412. if (event_sched_in(group_event, cpuctx, ctx)) {
  1413. pmu->cancel_txn(pmu);
  1414. perf_cpu_hrtimer_restart(cpuctx);
  1415. return -EAGAIN;
  1416. }
  1417. /*
  1418. * Schedule in siblings as one group (if any):
  1419. */
  1420. list_for_each_entry(event, &group_event->sibling_list, group_entry) {
  1421. if (event_sched_in(event, cpuctx, ctx)) {
  1422. partial_group = event;
  1423. goto group_error;
  1424. }
  1425. }
  1426. if (!pmu->commit_txn(pmu))
  1427. return 0;
  1428. group_error:
  1429. /*
  1430. * Groups can be scheduled in as one unit only, so undo any
  1431. * partial group before returning:
  1432. * The events up to the failed event are scheduled out normally,
  1433. * tstamp_stopped will be updated.
  1434. *
  1435. * The failed events and the remaining siblings need to have
  1436. * their timings updated as if they had gone thru event_sched_in()
  1437. * and event_sched_out(). This is required to get consistent timings
  1438. * across the group. This also takes care of the case where the group
  1439. * could never be scheduled by ensuring tstamp_stopped is set to mark
  1440. * the time the event was actually stopped, such that time delta
  1441. * calculation in update_event_times() is correct.
  1442. */
  1443. list_for_each_entry(event, &group_event->sibling_list, group_entry) {
  1444. if (event == partial_group)
  1445. simulate = true;
  1446. if (simulate) {
  1447. event->tstamp_running += now - event->tstamp_stopped;
  1448. event->tstamp_stopped = now;
  1449. } else {
  1450. event_sched_out(event, cpuctx, ctx);
  1451. }
  1452. }
  1453. event_sched_out(group_event, cpuctx, ctx);
  1454. pmu->cancel_txn(pmu);
  1455. perf_cpu_hrtimer_restart(cpuctx);
  1456. return -EAGAIN;
  1457. }
  1458. /*
  1459. * Work out whether we can put this event group on the CPU now.
  1460. */
  1461. static int group_can_go_on(struct perf_event *event,
  1462. struct perf_cpu_context *cpuctx,
  1463. int can_add_hw)
  1464. {
  1465. /*
  1466. * Groups consisting entirely of software events can always go on.
  1467. */
  1468. if (event->group_flags & PERF_GROUP_SOFTWARE)
  1469. return 1;
  1470. /*
  1471. * If an exclusive group is already on, no other hardware
  1472. * events can go on.
  1473. */
  1474. if (cpuctx->exclusive)
  1475. return 0;
  1476. /*
  1477. * If this group is exclusive and there are already
  1478. * events on the CPU, it can't go on.
  1479. */
  1480. if (event->attr.exclusive && cpuctx->active_oncpu)
  1481. return 0;
  1482. /*
  1483. * Otherwise, try to add it if all previous groups were able
  1484. * to go on.
  1485. */
  1486. return can_add_hw;
  1487. }
  1488. static void add_event_to_ctx(struct perf_event *event,
  1489. struct perf_event_context *ctx)
  1490. {
  1491. u64 tstamp = perf_event_time(event);
  1492. list_add_event(event, ctx);
  1493. perf_group_attach(event);
  1494. event->tstamp_enabled = tstamp;
  1495. event->tstamp_running = tstamp;
  1496. event->tstamp_stopped = tstamp;
  1497. }
  1498. static void task_ctx_sched_out(struct perf_event_context *ctx);
  1499. static void
  1500. ctx_sched_in(struct perf_event_context *ctx,
  1501. struct perf_cpu_context *cpuctx,
  1502. enum event_type_t event_type,
  1503. struct task_struct *task);
  1504. static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
  1505. struct perf_event_context *ctx,
  1506. struct task_struct *task)
  1507. {
  1508. cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
  1509. if (ctx)
  1510. ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
  1511. cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
  1512. if (ctx)
  1513. ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
  1514. }
  1515. /*
  1516. * Cross CPU call to install and enable a performance event
  1517. *
  1518. * Must be called with ctx->mutex held
  1519. */
  1520. static int __perf_install_in_context(void *info)
  1521. {
  1522. struct perf_event *event = info;
  1523. struct perf_event_context *ctx = event->ctx;
  1524. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1525. struct perf_event_context *task_ctx = cpuctx->task_ctx;
  1526. struct task_struct *task = current;
  1527. perf_ctx_lock(cpuctx, task_ctx);
  1528. perf_pmu_disable(cpuctx->ctx.pmu);
  1529. /*
  1530. * If there was an active task_ctx schedule it out.
  1531. */
  1532. if (task_ctx)
  1533. task_ctx_sched_out(task_ctx);
  1534. /*
  1535. * If the context we're installing events in is not the
  1536. * active task_ctx, flip them.
  1537. */
  1538. if (ctx->task && task_ctx != ctx) {
  1539. if (task_ctx)
  1540. raw_spin_unlock(&task_ctx->lock);
  1541. raw_spin_lock(&ctx->lock);
  1542. task_ctx = ctx;
  1543. }
  1544. if (task_ctx) {
  1545. cpuctx->task_ctx = task_ctx;
  1546. task = task_ctx->task;
  1547. }
  1548. cpu_ctx_sched_out(cpuctx, EVENT_ALL);
  1549. update_context_time(ctx);
  1550. /*
  1551. * update cgrp time only if current cgrp
  1552. * matches event->cgrp. Must be done before
  1553. * calling add_event_to_ctx()
  1554. */
  1555. update_cgrp_time_from_event(event);
  1556. add_event_to_ctx(event, ctx);
  1557. /*
  1558. * Schedule everything back in
  1559. */
  1560. perf_event_sched_in(cpuctx, task_ctx, task);
  1561. perf_pmu_enable(cpuctx->ctx.pmu);
  1562. perf_ctx_unlock(cpuctx, task_ctx);
  1563. return 0;
  1564. }
  1565. /*
  1566. * Attach a performance event to a context
  1567. *
  1568. * First we add the event to the list with the hardware enable bit
  1569. * in event->hw_config cleared.
  1570. *
  1571. * If the event is attached to a task which is on a CPU we use a smp
  1572. * call to enable it in the task context. The task might have been
  1573. * scheduled away, but we check this in the smp call again.
  1574. */
  1575. static void
  1576. perf_install_in_context(struct perf_event_context *ctx,
  1577. struct perf_event *event,
  1578. int cpu)
  1579. {
  1580. struct task_struct *task = ctx->task;
  1581. lockdep_assert_held(&ctx->mutex);
  1582. event->ctx = ctx;
  1583. if (event->cpu != -1)
  1584. event->cpu = cpu;
  1585. if (!task) {
  1586. /*
  1587. * Per cpu events are installed via an smp call and
  1588. * the install is always successful.
  1589. */
  1590. cpu_function_call(cpu, __perf_install_in_context, event);
  1591. return;
  1592. }
  1593. retry:
  1594. if (!task_function_call(task, __perf_install_in_context, event))
  1595. return;
  1596. raw_spin_lock_irq(&ctx->lock);
  1597. /*
  1598. * If we failed to find a running task, but find the context active now
  1599. * that we've acquired the ctx->lock, retry.
  1600. */
  1601. if (ctx->is_active) {
  1602. raw_spin_unlock_irq(&ctx->lock);
  1603. goto retry;
  1604. }
  1605. /*
  1606. * Since the task isn't running, its safe to add the event, us holding
  1607. * the ctx->lock ensures the task won't get scheduled in.
  1608. */
  1609. add_event_to_ctx(event, ctx);
  1610. raw_spin_unlock_irq(&ctx->lock);
  1611. }
  1612. /*
  1613. * Put a event into inactive state and update time fields.
  1614. * Enabling the leader of a group effectively enables all
  1615. * the group members that aren't explicitly disabled, so we
  1616. * have to update their ->tstamp_enabled also.
  1617. * Note: this works for group members as well as group leaders
  1618. * since the non-leader members' sibling_lists will be empty.
  1619. */
  1620. static void __perf_event_mark_enabled(struct perf_event *event)
  1621. {
  1622. struct perf_event *sub;
  1623. u64 tstamp = perf_event_time(event);
  1624. event->state = PERF_EVENT_STATE_INACTIVE;
  1625. event->tstamp_enabled = tstamp - event->total_time_enabled;
  1626. list_for_each_entry(sub, &event->sibling_list, group_entry) {
  1627. if (sub->state >= PERF_EVENT_STATE_INACTIVE)
  1628. sub->tstamp_enabled = tstamp - sub->total_time_enabled;
  1629. }
  1630. }
  1631. /*
  1632. * Cross CPU call to enable a performance event
  1633. */
  1634. static int __perf_event_enable(void *info)
  1635. {
  1636. struct perf_event *event = info;
  1637. struct perf_event_context *ctx = event->ctx;
  1638. struct perf_event *leader = event->group_leader;
  1639. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1640. int err;
  1641. /*
  1642. * There's a time window between 'ctx->is_active' check
  1643. * in perf_event_enable function and this place having:
  1644. * - IRQs on
  1645. * - ctx->lock unlocked
  1646. *
  1647. * where the task could be killed and 'ctx' deactivated
  1648. * by perf_event_exit_task.
  1649. */
  1650. if (!ctx->is_active)
  1651. return -EINVAL;
  1652. raw_spin_lock(&ctx->lock);
  1653. update_context_time(ctx);
  1654. if (event->state >= PERF_EVENT_STATE_INACTIVE)
  1655. goto unlock;
  1656. /*
  1657. * set current task's cgroup time reference point
  1658. */
  1659. perf_cgroup_set_timestamp(current, ctx);
  1660. __perf_event_mark_enabled(event);
  1661. if (!event_filter_match(event)) {
  1662. if (is_cgroup_event(event))
  1663. perf_cgroup_defer_enabled(event);
  1664. goto unlock;
  1665. }
  1666. /*
  1667. * If the event is in a group and isn't the group leader,
  1668. * then don't put it on unless the group is on.
  1669. */
  1670. if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
  1671. goto unlock;
  1672. if (!group_can_go_on(event, cpuctx, 1)) {
  1673. err = -EEXIST;
  1674. } else {
  1675. if (event == leader)
  1676. err = group_sched_in(event, cpuctx, ctx);
  1677. else
  1678. err = event_sched_in(event, cpuctx, ctx);
  1679. }
  1680. if (err) {
  1681. /*
  1682. * If this event can't go on and it's part of a
  1683. * group, then the whole group has to come off.
  1684. */
  1685. if (leader != event) {
  1686. group_sched_out(leader, cpuctx, ctx);
  1687. perf_cpu_hrtimer_restart(cpuctx);
  1688. }
  1689. if (leader->attr.pinned) {
  1690. update_group_times(leader);
  1691. leader->state = PERF_EVENT_STATE_ERROR;
  1692. }
  1693. }
  1694. unlock:
  1695. raw_spin_unlock(&ctx->lock);
  1696. return 0;
  1697. }
  1698. /*
  1699. * Enable a event.
  1700. *
  1701. * If event->ctx is a cloned context, callers must make sure that
  1702. * every task struct that event->ctx->task could possibly point to
  1703. * remains valid. This condition is satisfied when called through
  1704. * perf_event_for_each_child or perf_event_for_each as described
  1705. * for perf_event_disable.
  1706. */
  1707. void perf_event_enable(struct perf_event *event)
  1708. {
  1709. struct perf_event_context *ctx = event->ctx;
  1710. struct task_struct *task = ctx->task;
  1711. if (!task) {
  1712. /*
  1713. * Enable the event on the cpu that it's on
  1714. */
  1715. cpu_function_call(event->cpu, __perf_event_enable, event);
  1716. return;
  1717. }
  1718. raw_spin_lock_irq(&ctx->lock);
  1719. if (event->state >= PERF_EVENT_STATE_INACTIVE)
  1720. goto out;
  1721. /*
  1722. * If the event is in error state, clear that first.
  1723. * That way, if we see the event in error state below, we
  1724. * know that it has gone back into error state, as distinct
  1725. * from the task having been scheduled away before the
  1726. * cross-call arrived.
  1727. */
  1728. if (event->state == PERF_EVENT_STATE_ERROR)
  1729. event->state = PERF_EVENT_STATE_OFF;
  1730. retry:
  1731. if (!ctx->is_active) {
  1732. __perf_event_mark_enabled(event);
  1733. goto out;
  1734. }
  1735. raw_spin_unlock_irq(&ctx->lock);
  1736. if (!task_function_call(task, __perf_event_enable, event))
  1737. return;
  1738. raw_spin_lock_irq(&ctx->lock);
  1739. /*
  1740. * If the context is active and the event is still off,
  1741. * we need to retry the cross-call.
  1742. */
  1743. if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
  1744. /*
  1745. * task could have been flipped by a concurrent
  1746. * perf_event_context_sched_out()
  1747. */
  1748. task = ctx->task;
  1749. goto retry;
  1750. }
  1751. out:
  1752. raw_spin_unlock_irq(&ctx->lock);
  1753. }
  1754. EXPORT_SYMBOL_GPL(perf_event_enable);
  1755. int perf_event_refresh(struct perf_event *event, int refresh)
  1756. {
  1757. /*
  1758. * not supported on inherited events
  1759. */
  1760. if (event->attr.inherit || !is_sampling_event(event))
  1761. return -EINVAL;
  1762. atomic_add(refresh, &event->event_limit);
  1763. perf_event_enable(event);
  1764. return 0;
  1765. }
  1766. EXPORT_SYMBOL_GPL(perf_event_refresh);
  1767. static void ctx_sched_out(struct perf_event_context *ctx,
  1768. struct perf_cpu_context *cpuctx,
  1769. enum event_type_t event_type)
  1770. {
  1771. struct perf_event *event;
  1772. int is_active = ctx->is_active;
  1773. ctx->is_active &= ~event_type;
  1774. if (likely(!ctx->nr_events))
  1775. return;
  1776. update_context_time(ctx);
  1777. update_cgrp_time_from_cpuctx(cpuctx);
  1778. if (!ctx->nr_active)
  1779. return;
  1780. perf_pmu_disable(ctx->pmu);
  1781. if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
  1782. list_for_each_entry(event, &ctx->pinned_groups, group_entry)
  1783. group_sched_out(event, cpuctx, ctx);
  1784. }
  1785. if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
  1786. list_for_each_entry(event, &ctx->flexible_groups, group_entry)
  1787. group_sched_out(event, cpuctx, ctx);
  1788. }
  1789. perf_pmu_enable(ctx->pmu);
  1790. }
  1791. /*
  1792. * Test whether two contexts are equivalent, i.e. whether they
  1793. * have both been cloned from the same version of the same context
  1794. * and they both have the same number of enabled events.
  1795. * If the number of enabled events is the same, then the set
  1796. * of enabled events should be the same, because these are both
  1797. * inherited contexts, therefore we can't access individual events
  1798. * in them directly with an fd; we can only enable/disable all
  1799. * events via prctl, or enable/disable all events in a family
  1800. * via ioctl, which will have the same effect on both contexts.
  1801. */
  1802. static int context_equiv(struct perf_event_context *ctx1,
  1803. struct perf_event_context *ctx2)
  1804. {
  1805. return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
  1806. && ctx1->parent_gen == ctx2->parent_gen
  1807. && !ctx1->pin_count && !ctx2->pin_count;
  1808. }
  1809. static void __perf_event_sync_stat(struct perf_event *event,
  1810. struct perf_event *next_event)
  1811. {
  1812. u64 value;
  1813. if (!event->attr.inherit_stat)
  1814. return;
  1815. /*
  1816. * Update the event value, we cannot use perf_event_read()
  1817. * because we're in the middle of a context switch and have IRQs
  1818. * disabled, which upsets smp_call_function_single(), however
  1819. * we know the event must be on the current CPU, therefore we
  1820. * don't need to use it.
  1821. */
  1822. switch (event->state) {
  1823. case PERF_EVENT_STATE_ACTIVE:
  1824. event->pmu->read(event);
  1825. /* fall-through */
  1826. case PERF_EVENT_STATE_INACTIVE:
  1827. update_event_times(event);
  1828. break;
  1829. default:
  1830. break;
  1831. }
  1832. /*
  1833. * In order to keep per-task stats reliable we need to flip the event
  1834. * values when we flip the contexts.
  1835. */
  1836. value = local64_read(&next_event->count);
  1837. value = local64_xchg(&event->count, value);
  1838. local64_set(&next_event->count, value);
  1839. swap(event->total_time_enabled, next_event->total_time_enabled);
  1840. swap(event->total_time_running, next_event->total_time_running);
  1841. /*
  1842. * Since we swizzled the values, update the user visible data too.
  1843. */
  1844. perf_event_update_userpage(event);
  1845. perf_event_update_userpage(next_event);
  1846. }
  1847. #define list_next_entry(pos, member) \
  1848. list_entry(pos->member.next, typeof(*pos), member)
  1849. static void perf_event_sync_stat(struct perf_event_context *ctx,
  1850. struct perf_event_context *next_ctx)
  1851. {
  1852. struct perf_event *event, *next_event;
  1853. if (!ctx->nr_stat)
  1854. return;
  1855. update_context_time(ctx);
  1856. event = list_first_entry(&ctx->event_list,
  1857. struct perf_event, event_entry);
  1858. next_event = list_first_entry(&next_ctx->event_list,
  1859. struct perf_event, event_entry);
  1860. while (&event->event_entry != &ctx->event_list &&
  1861. &next_event->event_entry != &next_ctx->event_list) {
  1862. __perf_event_sync_stat(event, next_event);
  1863. event = list_next_entry(event, event_entry);
  1864. next_event = list_next_entry(next_event, event_entry);
  1865. }
  1866. }
  1867. static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
  1868. struct task_struct *next)
  1869. {
  1870. struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
  1871. struct perf_event_context *next_ctx;
  1872. struct perf_event_context *parent;
  1873. struct perf_cpu_context *cpuctx;
  1874. int do_switch = 1;
  1875. if (likely(!ctx))
  1876. return;
  1877. cpuctx = __get_cpu_context(ctx);
  1878. if (!cpuctx->task_ctx)
  1879. return;
  1880. rcu_read_lock();
  1881. parent = rcu_dereference(ctx->parent_ctx);
  1882. next_ctx = next->perf_event_ctxp[ctxn];
  1883. if (parent && next_ctx &&
  1884. rcu_dereference(next_ctx->parent_ctx) == parent) {
  1885. /*
  1886. * Looks like the two contexts are clones, so we might be
  1887. * able to optimize the context switch. We lock both
  1888. * contexts and check that they are clones under the
  1889. * lock (including re-checking that neither has been
  1890. * uncloned in the meantime). It doesn't matter which
  1891. * order we take the locks because no other cpu could
  1892. * be trying to lock both of these tasks.
  1893. */
  1894. raw_spin_lock(&ctx->lock);
  1895. raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
  1896. if (context_equiv(ctx, next_ctx)) {
  1897. /*
  1898. * XXX do we need a memory barrier of sorts
  1899. * wrt to rcu_dereference() of perf_event_ctxp
  1900. */
  1901. task->perf_event_ctxp[ctxn] = next_ctx;
  1902. next->perf_event_ctxp[ctxn] = ctx;
  1903. ctx->task = next;
  1904. next_ctx->task = task;
  1905. do_switch = 0;
  1906. perf_event_sync_stat(ctx, next_ctx);
  1907. }
  1908. raw_spin_unlock(&next_ctx->lock);
  1909. raw_spin_unlock(&ctx->lock);
  1910. }
  1911. rcu_read_unlock();
  1912. if (do_switch) {
  1913. raw_spin_lock(&ctx->lock);
  1914. ctx_sched_out(ctx, cpuctx, EVENT_ALL);
  1915. cpuctx->task_ctx = NULL;
  1916. raw_spin_unlock(&ctx->lock);
  1917. }
  1918. }
  1919. #define for_each_task_context_nr(ctxn) \
  1920. for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
  1921. /*
  1922. * Called from scheduler to remove the events of the current task,
  1923. * with interrupts disabled.
  1924. *
  1925. * We stop each event and update the event value in event->count.
  1926. *
  1927. * This does not protect us against NMI, but disable()
  1928. * sets the disabled bit in the control field of event _before_
  1929. * accessing the event control register. If a NMI hits, then it will
  1930. * not restart the event.
  1931. */
  1932. void __perf_event_task_sched_out(struct task_struct *task,
  1933. struct task_struct *next)
  1934. {
  1935. int ctxn;
  1936. for_each_task_context_nr(ctxn)
  1937. perf_event_context_sched_out(task, ctxn, next);
  1938. /*
  1939. * if cgroup events exist on this CPU, then we need
  1940. * to check if we have to switch out PMU state.
  1941. * cgroup event are system-wide mode only
  1942. */
  1943. if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
  1944. perf_cgroup_sched_out(task, next);
  1945. }
  1946. static void task_ctx_sched_out(struct perf_event_context *ctx)
  1947. {
  1948. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1949. if (!cpuctx->task_ctx)
  1950. return;
  1951. if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
  1952. return;
  1953. ctx_sched_out(ctx, cpuctx, EVENT_ALL);
  1954. cpuctx->task_ctx = NULL;
  1955. }
  1956. /*
  1957. * Called with IRQs disabled
  1958. */
  1959. static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
  1960. enum event_type_t event_type)
  1961. {
  1962. ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
  1963. }
  1964. static void
  1965. ctx_pinned_sched_in(struct perf_event_context *ctx,
  1966. struct perf_cpu_context *cpuctx)
  1967. {
  1968. struct perf_event *event;
  1969. list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
  1970. if (event->state <= PERF_EVENT_STATE_OFF)
  1971. continue;
  1972. if (!event_filter_match(event))
  1973. continue;
  1974. /* may need to reset tstamp_enabled */
  1975. if (is_cgroup_event(event))
  1976. perf_cgroup_mark_enabled(event, ctx);
  1977. if (group_can_go_on(event, cpuctx, 1))
  1978. group_sched_in(event, cpuctx, ctx);
  1979. /*
  1980. * If this pinned group hasn't been scheduled,
  1981. * put it in error state.
  1982. */
  1983. if (event->state == PERF_EVENT_STATE_INACTIVE) {
  1984. update_group_times(event);
  1985. event->state = PERF_EVENT_STATE_ERROR;
  1986. }
  1987. }
  1988. }
  1989. static void
  1990. ctx_flexible_sched_in(struct perf_event_context *ctx,
  1991. struct perf_cpu_context *cpuctx)
  1992. {
  1993. struct perf_event *event;
  1994. int can_add_hw = 1;
  1995. list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
  1996. /* Ignore events in OFF or ERROR state */
  1997. if (event->state <= PERF_EVENT_STATE_OFF)
  1998. continue;
  1999. /*
  2000. * Listen to the 'cpu' scheduling filter constraint
  2001. * of events:
  2002. */
  2003. if (!event_filter_match(event))
  2004. continue;
  2005. /* may need to reset tstamp_enabled */
  2006. if (is_cgroup_event(event))
  2007. perf_cgroup_mark_enabled(event, ctx);
  2008. if (group_can_go_on(event, cpuctx, can_add_hw)) {
  2009. if (group_sched_in(event, cpuctx, ctx))
  2010. can_add_hw = 0;
  2011. }
  2012. }
  2013. }
  2014. static void
  2015. ctx_sched_in(struct perf_event_context *ctx,
  2016. struct perf_cpu_context *cpuctx,
  2017. enum event_type_t event_type,
  2018. struct task_struct *task)
  2019. {
  2020. u64 now;
  2021. int is_active = ctx->is_active;
  2022. ctx->is_active |= event_type;
  2023. if (likely(!ctx->nr_events))
  2024. return;
  2025. now = perf_clock();
  2026. ctx->timestamp = now;
  2027. perf_cgroup_set_timestamp(task, ctx);
  2028. /*
  2029. * First go through the list and put on any pinned groups
  2030. * in order to give them the best chance of going on.
  2031. */
  2032. if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
  2033. ctx_pinned_sched_in(ctx, cpuctx);
  2034. /* Then walk through the lower prio flexible groups */
  2035. if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
  2036. ctx_flexible_sched_in(ctx, cpuctx);
  2037. }
  2038. static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
  2039. enum event_type_t event_type,
  2040. struct task_struct *task)
  2041. {
  2042. struct perf_event_context *ctx = &cpuctx->ctx;
  2043. ctx_sched_in(ctx, cpuctx, event_type, task);
  2044. }
  2045. static void perf_event_context_sched_in(struct perf_event_context *ctx,
  2046. struct task_struct *task)
  2047. {
  2048. struct perf_cpu_context *cpuctx;
  2049. cpuctx = __get_cpu_context(ctx);
  2050. if (cpuctx->task_ctx == ctx)
  2051. return;
  2052. perf_ctx_lock(cpuctx, ctx);
  2053. perf_pmu_disable(ctx->pmu);
  2054. /*
  2055. * We want to keep the following priority order:
  2056. * cpu pinned (that don't need to move), task pinned,
  2057. * cpu flexible, task flexible.
  2058. */
  2059. cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
  2060. if (ctx->nr_events)
  2061. cpuctx->task_ctx = ctx;
  2062. perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
  2063. perf_pmu_enable(ctx->pmu);
  2064. perf_ctx_unlock(cpuctx, ctx);
  2065. /*
  2066. * Since these rotations are per-cpu, we need to ensure the
  2067. * cpu-context we got scheduled on is actually rotating.
  2068. */
  2069. perf_pmu_rotate_start(ctx->pmu);
  2070. }
  2071. /*
  2072. * When sampling the branck stack in system-wide, it may be necessary
  2073. * to flush the stack on context switch. This happens when the branch
  2074. * stack does not tag its entries with the pid of the current task.
  2075. * Otherwise it becomes impossible to associate a branch entry with a
  2076. * task. This ambiguity is more likely to appear when the branch stack
  2077. * supports priv level filtering and the user sets it to monitor only
  2078. * at the user level (which could be a useful measurement in system-wide
  2079. * mode). In that case, the risk is high of having a branch stack with
  2080. * branch from multiple tasks. Flushing may mean dropping the existing
  2081. * entries or stashing them somewhere in the PMU specific code layer.
  2082. *
  2083. * This function provides the context switch callback to the lower code
  2084. * layer. It is invoked ONLY when there is at least one system-wide context
  2085. * with at least one active event using taken branch sampling.
  2086. */
  2087. static void perf_branch_stack_sched_in(struct task_struct *prev,
  2088. struct task_struct *task)
  2089. {
  2090. struct perf_cpu_context *cpuctx;
  2091. struct pmu *pmu;
  2092. unsigned long flags;
  2093. /* no need to flush branch stack if not changing task */
  2094. if (prev == task)
  2095. return;
  2096. local_irq_save(flags);
  2097. rcu_read_lock();
  2098. list_for_each_entry_rcu(pmu, &pmus, entry) {
  2099. cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  2100. /*
  2101. * check if the context has at least one
  2102. * event using PERF_SAMPLE_BRANCH_STACK
  2103. */
  2104. if (cpuctx->ctx.nr_branch_stack > 0
  2105. && pmu->flush_branch_stack) {
  2106. pmu = cpuctx->ctx.pmu;
  2107. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  2108. perf_pmu_disable(pmu);
  2109. pmu->flush_branch_stack();
  2110. perf_pmu_enable(pmu);
  2111. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  2112. }
  2113. }
  2114. rcu_read_unlock();
  2115. local_irq_restore(flags);
  2116. }
  2117. /*
  2118. * Called from scheduler to add the events of the current task
  2119. * with interrupts disabled.
  2120. *
  2121. * We restore the event value and then enable it.
  2122. *
  2123. * This does not protect us against NMI, but enable()
  2124. * sets the enabled bit in the control field of event _before_
  2125. * accessing the event control register. If a NMI hits, then it will
  2126. * keep the event running.
  2127. */
  2128. void __perf_event_task_sched_in(struct task_struct *prev,
  2129. struct task_struct *task)
  2130. {
  2131. struct perf_event_context *ctx;
  2132. int ctxn;
  2133. for_each_task_context_nr(ctxn) {
  2134. ctx = task->perf_event_ctxp[ctxn];
  2135. if (likely(!ctx))
  2136. continue;
  2137. perf_event_context_sched_in(ctx, task);
  2138. }
  2139. /*
  2140. * if cgroup events exist on this CPU, then we need
  2141. * to check if we have to switch in PMU state.
  2142. * cgroup event are system-wide mode only
  2143. */
  2144. if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
  2145. perf_cgroup_sched_in(prev, task);
  2146. /* check for system-wide branch_stack events */
  2147. if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
  2148. perf_branch_stack_sched_in(prev, task);
  2149. }
  2150. static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
  2151. {
  2152. u64 frequency = event->attr.sample_freq;
  2153. u64 sec = NSEC_PER_SEC;
  2154. u64 divisor, dividend;
  2155. int count_fls, nsec_fls, frequency_fls, sec_fls;
  2156. count_fls = fls64(count);
  2157. nsec_fls = fls64(nsec);
  2158. frequency_fls = fls64(frequency);
  2159. sec_fls = 30;
  2160. /*
  2161. * We got @count in @nsec, with a target of sample_freq HZ
  2162. * the target period becomes:
  2163. *
  2164. * @count * 10^9
  2165. * period = -------------------
  2166. * @nsec * sample_freq
  2167. *
  2168. */
  2169. /*
  2170. * Reduce accuracy by one bit such that @a and @b converge
  2171. * to a similar magnitude.
  2172. */
  2173. #define REDUCE_FLS(a, b) \
  2174. do { \
  2175. if (a##_fls > b##_fls) { \
  2176. a >>= 1; \
  2177. a##_fls--; \
  2178. } else { \
  2179. b >>= 1; \
  2180. b##_fls--; \
  2181. } \
  2182. } while (0)
  2183. /*
  2184. * Reduce accuracy until either term fits in a u64, then proceed with
  2185. * the other, so that finally we can do a u64/u64 division.
  2186. */
  2187. while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
  2188. REDUCE_FLS(nsec, frequency);
  2189. REDUCE_FLS(sec, count);
  2190. }
  2191. if (count_fls + sec_fls > 64) {
  2192. divisor = nsec * frequency;
  2193. while (count_fls + sec_fls > 64) {
  2194. REDUCE_FLS(count, sec);
  2195. divisor >>= 1;
  2196. }
  2197. dividend = count * sec;
  2198. } else {
  2199. dividend = count * sec;
  2200. while (nsec_fls + frequency_fls > 64) {
  2201. REDUCE_FLS(nsec, frequency);
  2202. dividend >>= 1;
  2203. }
  2204. divisor = nsec * frequency;
  2205. }
  2206. if (!divisor)
  2207. return dividend;
  2208. return div64_u64(dividend, divisor);
  2209. }
  2210. static DEFINE_PER_CPU(int, perf_throttled_count);
  2211. static DEFINE_PER_CPU(u64, perf_throttled_seq);
  2212. static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
  2213. {
  2214. struct hw_perf_event *hwc = &event->hw;
  2215. s64 period, sample_period;
  2216. s64 delta;
  2217. period = perf_calculate_period(event, nsec, count);
  2218. delta = (s64)(period - hwc->sample_period);
  2219. delta = (delta + 7) / 8; /* low pass filter */
  2220. sample_period = hwc->sample_period + delta;
  2221. if (!sample_period)
  2222. sample_period = 1;
  2223. hwc->sample_period = sample_period;
  2224. if (local64_read(&hwc->period_left) > 8*sample_period) {
  2225. if (disable)
  2226. event->pmu->stop(event, PERF_EF_UPDATE);
  2227. local64_set(&hwc->period_left, 0);
  2228. if (disable)
  2229. event->pmu->start(event, PERF_EF_RELOAD);
  2230. }
  2231. }
  2232. /*
  2233. * combine freq adjustment with unthrottling to avoid two passes over the
  2234. * events. At the same time, make sure, having freq events does not change
  2235. * the rate of unthrottling as that would introduce bias.
  2236. */
  2237. static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
  2238. int needs_unthr)
  2239. {
  2240. struct perf_event *event;
  2241. struct hw_perf_event *hwc;
  2242. u64 now, period = TICK_NSEC;
  2243. s64 delta;
  2244. /*
  2245. * only need to iterate over all events iff:
  2246. * - context have events in frequency mode (needs freq adjust)
  2247. * - there are events to unthrottle on this cpu
  2248. */
  2249. if (!(ctx->nr_freq || needs_unthr))
  2250. return;
  2251. raw_spin_lock(&ctx->lock);
  2252. perf_pmu_disable(ctx->pmu);
  2253. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  2254. if (event->state != PERF_EVENT_STATE_ACTIVE)
  2255. continue;
  2256. if (!event_filter_match(event))
  2257. continue;
  2258. hwc = &event->hw;
  2259. if (hwc->interrupts == MAX_INTERRUPTS) {
  2260. hwc->interrupts = 0;
  2261. perf_log_throttle(event, 1);
  2262. event->pmu->start(event, 0);
  2263. }
  2264. if (!event->attr.freq || !event->attr.sample_freq)
  2265. continue;
  2266. /*
  2267. * stop the event and update event->count
  2268. */
  2269. event->pmu->stop(event, PERF_EF_UPDATE);
  2270. now = local64_read(&event->count);
  2271. delta = now - hwc->freq_count_stamp;
  2272. hwc->freq_count_stamp = now;
  2273. /*
  2274. * restart the event
  2275. * reload only if value has changed
  2276. * we have stopped the event so tell that
  2277. * to perf_adjust_period() to avoid stopping it
  2278. * twice.
  2279. */
  2280. if (delta > 0)
  2281. perf_adjust_period(event, period, delta, false);
  2282. event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
  2283. }
  2284. perf_pmu_enable(ctx->pmu);
  2285. raw_spin_unlock(&ctx->lock);
  2286. }
  2287. /*
  2288. * Round-robin a context's events:
  2289. */
  2290. static void rotate_ctx(struct perf_event_context *ctx)
  2291. {
  2292. /*
  2293. * Rotate the first entry last of non-pinned groups. Rotation might be
  2294. * disabled by the inheritance code.
  2295. */
  2296. if (!ctx->rotate_disable)
  2297. list_rotate_left(&ctx->flexible_groups);
  2298. }
  2299. /*
  2300. * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
  2301. * because they're strictly cpu affine and rotate_start is called with IRQs
  2302. * disabled, while rotate_context is called from IRQ context.
  2303. */
  2304. static int perf_rotate_context(struct perf_cpu_context *cpuctx)
  2305. {
  2306. struct perf_event_context *ctx = NULL;
  2307. int rotate = 0, remove = 1;
  2308. if (cpuctx->ctx.nr_events) {
  2309. remove = 0;
  2310. if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
  2311. rotate = 1;
  2312. }
  2313. ctx = cpuctx->task_ctx;
  2314. if (ctx && ctx->nr_events) {
  2315. remove = 0;
  2316. if (ctx->nr_events != ctx->nr_active)
  2317. rotate = 1;
  2318. }
  2319. if (!rotate)
  2320. goto done;
  2321. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  2322. perf_pmu_disable(cpuctx->ctx.pmu);
  2323. cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
  2324. if (ctx)
  2325. ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
  2326. rotate_ctx(&cpuctx->ctx);
  2327. if (ctx)
  2328. rotate_ctx(ctx);
  2329. perf_event_sched_in(cpuctx, ctx, current);
  2330. perf_pmu_enable(cpuctx->ctx.pmu);
  2331. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  2332. done:
  2333. if (remove)
  2334. list_del_init(&cpuctx->rotation_list);
  2335. return rotate;
  2336. }
  2337. #ifdef CONFIG_NO_HZ_FULL
  2338. bool perf_event_can_stop_tick(void)
  2339. {
  2340. if (atomic_read(&nr_freq_events) ||
  2341. __this_cpu_read(perf_throttled_count))
  2342. return false;
  2343. else
  2344. return true;
  2345. }
  2346. #endif
  2347. void perf_event_task_tick(void)
  2348. {
  2349. struct list_head *head = &__get_cpu_var(rotation_list);
  2350. struct perf_cpu_context *cpuctx, *tmp;
  2351. struct perf_event_context *ctx;
  2352. int throttled;
  2353. WARN_ON(!irqs_disabled());
  2354. __this_cpu_inc(perf_throttled_seq);
  2355. throttled = __this_cpu_xchg(perf_throttled_count, 0);
  2356. list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
  2357. ctx = &cpuctx->ctx;
  2358. perf_adjust_freq_unthr_context(ctx, throttled);
  2359. ctx = cpuctx->task_ctx;
  2360. if (ctx)
  2361. perf_adjust_freq_unthr_context(ctx, throttled);
  2362. }
  2363. }
  2364. static int event_enable_on_exec(struct perf_event *event,
  2365. struct perf_event_context *ctx)
  2366. {
  2367. if (!event->attr.enable_on_exec)
  2368. return 0;
  2369. event->attr.enable_on_exec = 0;
  2370. if (event->state >= PERF_EVENT_STATE_INACTIVE)
  2371. return 0;
  2372. __perf_event_mark_enabled(event);
  2373. return 1;
  2374. }
  2375. /*
  2376. * Enable all of a task's events that have been marked enable-on-exec.
  2377. * This expects task == current.
  2378. */
  2379. static void perf_event_enable_on_exec(struct perf_event_context *ctx)
  2380. {
  2381. struct perf_event *event;
  2382. unsigned long flags;
  2383. int enabled = 0;
  2384. int ret;
  2385. local_irq_save(flags);
  2386. if (!ctx || !ctx->nr_events)
  2387. goto out;
  2388. /*
  2389. * We must ctxsw out cgroup events to avoid conflict
  2390. * when invoking perf_task_event_sched_in() later on
  2391. * in this function. Otherwise we end up trying to
  2392. * ctxswin cgroup events which are already scheduled
  2393. * in.
  2394. */
  2395. perf_cgroup_sched_out(current, NULL);
  2396. raw_spin_lock(&ctx->lock);
  2397. task_ctx_sched_out(ctx);
  2398. list_for_each_entry(event, &ctx->event_list, event_entry) {
  2399. ret = event_enable_on_exec(event, ctx);
  2400. if (ret)
  2401. enabled = 1;
  2402. }
  2403. /*
  2404. * Unclone this context if we enabled any event.
  2405. */
  2406. if (enabled)
  2407. unclone_ctx(ctx);
  2408. raw_spin_unlock(&ctx->lock);
  2409. /*
  2410. * Also calls ctxswin for cgroup events, if any:
  2411. */
  2412. perf_event_context_sched_in(ctx, ctx->task);
  2413. out:
  2414. local_irq_restore(flags);
  2415. }
  2416. /*
  2417. * Cross CPU call to read the hardware event
  2418. */
  2419. static void __perf_event_read(void *info)
  2420. {
  2421. struct perf_event *event = info;
  2422. struct perf_event_context *ctx = event->ctx;
  2423. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  2424. /*
  2425. * If this is a task context, we need to check whether it is
  2426. * the current task context of this cpu. If not it has been
  2427. * scheduled out before the smp call arrived. In that case
  2428. * event->count would have been updated to a recent sample
  2429. * when the event was scheduled out.
  2430. */
  2431. if (ctx->task && cpuctx->task_ctx != ctx)
  2432. return;
  2433. raw_spin_lock(&ctx->lock);
  2434. if (ctx->is_active) {
  2435. update_context_time(ctx);
  2436. update_cgrp_time_from_event(event);
  2437. }
  2438. update_event_times(event);
  2439. if (event->state == PERF_EVENT_STATE_ACTIVE)
  2440. event->pmu->read(event);
  2441. raw_spin_unlock(&ctx->lock);
  2442. }
  2443. static inline u64 perf_event_count(struct perf_event *event)
  2444. {
  2445. return local64_read(&event->count) + atomic64_read(&event->child_count);
  2446. }
  2447. static u64 perf_event_read(struct perf_event *event)
  2448. {
  2449. /*
  2450. * If event is enabled and currently active on a CPU, update the
  2451. * value in the event structure:
  2452. */
  2453. if (event->state == PERF_EVENT_STATE_ACTIVE) {
  2454. smp_call_function_single(event->oncpu,
  2455. __perf_event_read, event, 1);
  2456. } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
  2457. struct perf_event_context *ctx = event->ctx;
  2458. unsigned long flags;
  2459. raw_spin_lock_irqsave(&ctx->lock, flags);
  2460. /*
  2461. * may read while context is not active
  2462. * (e.g., thread is blocked), in that case
  2463. * we cannot update context time
  2464. */
  2465. if (ctx->is_active) {
  2466. update_context_time(ctx);
  2467. update_cgrp_time_from_event(event);
  2468. }
  2469. update_event_times(event);
  2470. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  2471. }
  2472. return perf_event_count(event);
  2473. }
  2474. /*
  2475. * Initialize the perf_event context in a task_struct:
  2476. */
  2477. static void __perf_event_init_context(struct perf_event_context *ctx)
  2478. {
  2479. raw_spin_lock_init(&ctx->lock);
  2480. mutex_init(&ctx->mutex);
  2481. INIT_LIST_HEAD(&ctx->pinned_groups);
  2482. INIT_LIST_HEAD(&ctx->flexible_groups);
  2483. INIT_LIST_HEAD(&ctx->event_list);
  2484. atomic_set(&ctx->refcount, 1);
  2485. }
  2486. static struct perf_event_context *
  2487. alloc_perf_context(struct pmu *pmu, struct task_struct *task)
  2488. {
  2489. struct perf_event_context *ctx;
  2490. ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
  2491. if (!ctx)
  2492. return NULL;
  2493. __perf_event_init_context(ctx);
  2494. if (task) {
  2495. ctx->task = task;
  2496. get_task_struct(task);
  2497. }
  2498. ctx->pmu = pmu;
  2499. return ctx;
  2500. }
  2501. static struct task_struct *
  2502. find_lively_task_by_vpid(pid_t vpid)
  2503. {
  2504. struct task_struct *task;
  2505. int err;
  2506. rcu_read_lock();
  2507. if (!vpid)
  2508. task = current;
  2509. else
  2510. task = find_task_by_vpid(vpid);
  2511. if (task)
  2512. get_task_struct(task);
  2513. rcu_read_unlock();
  2514. if (!task)
  2515. return ERR_PTR(-ESRCH);
  2516. /* Reuse ptrace permission checks for now. */
  2517. err = -EACCES;
  2518. if (!ptrace_may_access(task, PTRACE_MODE_READ))
  2519. goto errout;
  2520. return task;
  2521. errout:
  2522. put_task_struct(task);
  2523. return ERR_PTR(err);
  2524. }
  2525. /*
  2526. * Returns a matching context with refcount and pincount.
  2527. */
  2528. static struct perf_event_context *
  2529. find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
  2530. {
  2531. struct perf_event_context *ctx;
  2532. struct perf_cpu_context *cpuctx;
  2533. unsigned long flags;
  2534. int ctxn, err;
  2535. if (!task) {
  2536. /* Must be root to operate on a CPU event: */
  2537. if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
  2538. return ERR_PTR(-EACCES);
  2539. /*
  2540. * We could be clever and allow to attach a event to an
  2541. * offline CPU and activate it when the CPU comes up, but
  2542. * that's for later.
  2543. */
  2544. if (!cpu_online(cpu))
  2545. return ERR_PTR(-ENODEV);
  2546. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  2547. ctx = &cpuctx->ctx;
  2548. get_ctx(ctx);
  2549. ++ctx->pin_count;
  2550. return ctx;
  2551. }
  2552. err = -EINVAL;
  2553. ctxn = pmu->task_ctx_nr;
  2554. if (ctxn < 0)
  2555. goto errout;
  2556. retry:
  2557. ctx = perf_lock_task_context(task, ctxn, &flags);
  2558. if (ctx) {
  2559. unclone_ctx(ctx);
  2560. ++ctx->pin_count;
  2561. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  2562. } else {
  2563. ctx = alloc_perf_context(pmu, task);
  2564. err = -ENOMEM;
  2565. if (!ctx)
  2566. goto errout;
  2567. err = 0;
  2568. mutex_lock(&task->perf_event_mutex);
  2569. /*
  2570. * If it has already passed perf_event_exit_task().
  2571. * we must see PF_EXITING, it takes this mutex too.
  2572. */
  2573. if (task->flags & PF_EXITING)
  2574. err = -ESRCH;
  2575. else if (task->perf_event_ctxp[ctxn])
  2576. err = -EAGAIN;
  2577. else {
  2578. get_ctx(ctx);
  2579. ++ctx->pin_count;
  2580. rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
  2581. }
  2582. mutex_unlock(&task->perf_event_mutex);
  2583. if (unlikely(err)) {
  2584. put_ctx(ctx);
  2585. if (err == -EAGAIN)
  2586. goto retry;
  2587. goto errout;
  2588. }
  2589. }
  2590. return ctx;
  2591. errout:
  2592. return ERR_PTR(err);
  2593. }
  2594. static void perf_event_free_filter(struct perf_event *event);
  2595. static void free_event_rcu(struct rcu_head *head)
  2596. {
  2597. struct perf_event *event;
  2598. event = container_of(head, struct perf_event, rcu_head);
  2599. if (event->ns)
  2600. put_pid_ns(event->ns);
  2601. perf_event_free_filter(event);
  2602. kfree(event);
  2603. }
  2604. static void ring_buffer_put(struct ring_buffer *rb);
  2605. static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb);
  2606. static void unaccount_event_cpu(struct perf_event *event, int cpu)
  2607. {
  2608. if (event->parent)
  2609. return;
  2610. if (has_branch_stack(event)) {
  2611. if (!(event->attach_state & PERF_ATTACH_TASK))
  2612. atomic_dec(&per_cpu(perf_branch_stack_events, cpu));
  2613. }
  2614. if (is_cgroup_event(event))
  2615. atomic_dec(&per_cpu(perf_cgroup_events, cpu));
  2616. }
  2617. static void unaccount_event(struct perf_event *event)
  2618. {
  2619. if (event->parent)
  2620. return;
  2621. if (event->attach_state & PERF_ATTACH_TASK)
  2622. static_key_slow_dec_deferred(&perf_sched_events);
  2623. if (event->attr.mmap || event->attr.mmap_data)
  2624. atomic_dec(&nr_mmap_events);
  2625. if (event->attr.comm)
  2626. atomic_dec(&nr_comm_events);
  2627. if (event->attr.task)
  2628. atomic_dec(&nr_task_events);
  2629. if (event->attr.freq)
  2630. atomic_dec(&nr_freq_events);
  2631. if (is_cgroup_event(event))
  2632. static_key_slow_dec_deferred(&perf_sched_events);
  2633. if (has_branch_stack(event))
  2634. static_key_slow_dec_deferred(&perf_sched_events);
  2635. unaccount_event_cpu(event, event->cpu);
  2636. }
  2637. static void __free_event(struct perf_event *event)
  2638. {
  2639. if (!event->parent) {
  2640. if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
  2641. put_callchain_buffers();
  2642. }
  2643. if (event->destroy)
  2644. event->destroy(event);
  2645. if (event->ctx)
  2646. put_ctx(event->ctx);
  2647. call_rcu(&event->rcu_head, free_event_rcu);
  2648. }
  2649. static void free_event(struct perf_event *event)
  2650. {
  2651. irq_work_sync(&event->pending);
  2652. unaccount_event(event);
  2653. if (event->rb) {
  2654. struct ring_buffer *rb;
  2655. /*
  2656. * Can happen when we close an event with re-directed output.
  2657. *
  2658. * Since we have a 0 refcount, perf_mmap_close() will skip
  2659. * over us; possibly making our ring_buffer_put() the last.
  2660. */
  2661. mutex_lock(&event->mmap_mutex);
  2662. rb = event->rb;
  2663. if (rb) {
  2664. rcu_assign_pointer(event->rb, NULL);
  2665. ring_buffer_detach(event, rb);
  2666. ring_buffer_put(rb); /* could be last */
  2667. }
  2668. mutex_unlock(&event->mmap_mutex);
  2669. }
  2670. if (is_cgroup_event(event))
  2671. perf_detach_cgroup(event);
  2672. __free_event(event);
  2673. }
  2674. int perf_event_release_kernel(struct perf_event *event)
  2675. {
  2676. struct perf_event_context *ctx = event->ctx;
  2677. WARN_ON_ONCE(ctx->parent_ctx);
  2678. /*
  2679. * There are two ways this annotation is useful:
  2680. *
  2681. * 1) there is a lock recursion from perf_event_exit_task
  2682. * see the comment there.
  2683. *
  2684. * 2) there is a lock-inversion with mmap_sem through
  2685. * perf_event_read_group(), which takes faults while
  2686. * holding ctx->mutex, however this is called after
  2687. * the last filedesc died, so there is no possibility
  2688. * to trigger the AB-BA case.
  2689. */
  2690. mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
  2691. raw_spin_lock_irq(&ctx->lock);
  2692. perf_group_detach(event);
  2693. raw_spin_unlock_irq(&ctx->lock);
  2694. perf_remove_from_context(event);
  2695. mutex_unlock(&ctx->mutex);
  2696. free_event(event);
  2697. return 0;
  2698. }
  2699. EXPORT_SYMBOL_GPL(perf_event_release_kernel);
  2700. /*
  2701. * Called when the last reference to the file is gone.
  2702. */
  2703. static void put_event(struct perf_event *event)
  2704. {
  2705. struct task_struct *owner;
  2706. if (!atomic_long_dec_and_test(&event->refcount))
  2707. return;
  2708. rcu_read_lock();
  2709. owner = ACCESS_ONCE(event->owner);
  2710. /*
  2711. * Matches the smp_wmb() in perf_event_exit_task(). If we observe
  2712. * !owner it means the list deletion is complete and we can indeed
  2713. * free this event, otherwise we need to serialize on
  2714. * owner->perf_event_mutex.
  2715. */
  2716. smp_read_barrier_depends();
  2717. if (owner) {
  2718. /*
  2719. * Since delayed_put_task_struct() also drops the last
  2720. * task reference we can safely take a new reference
  2721. * while holding the rcu_read_lock().
  2722. */
  2723. get_task_struct(owner);
  2724. }
  2725. rcu_read_unlock();
  2726. if (owner) {
  2727. mutex_lock(&owner->perf_event_mutex);
  2728. /*
  2729. * We have to re-check the event->owner field, if it is cleared
  2730. * we raced with perf_event_exit_task(), acquiring the mutex
  2731. * ensured they're done, and we can proceed with freeing the
  2732. * event.
  2733. */
  2734. if (event->owner)
  2735. list_del_init(&event->owner_entry);
  2736. mutex_unlock(&owner->perf_event_mutex);
  2737. put_task_struct(owner);
  2738. }
  2739. perf_event_release_kernel(event);
  2740. }
  2741. static int perf_release(struct inode *inode, struct file *file)
  2742. {
  2743. put_event(file->private_data);
  2744. return 0;
  2745. }
  2746. u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
  2747. {
  2748. struct perf_event *child;
  2749. u64 total = 0;
  2750. *enabled = 0;
  2751. *running = 0;
  2752. mutex_lock(&event->child_mutex);
  2753. total += perf_event_read(event);
  2754. *enabled += event->total_time_enabled +
  2755. atomic64_read(&event->child_total_time_enabled);
  2756. *running += event->total_time_running +
  2757. atomic64_read(&event->child_total_time_running);
  2758. list_for_each_entry(child, &event->child_list, child_list) {
  2759. total += perf_event_read(child);
  2760. *enabled += child->total_time_enabled;
  2761. *running += child->total_time_running;
  2762. }
  2763. mutex_unlock(&event->child_mutex);
  2764. return total;
  2765. }
  2766. EXPORT_SYMBOL_GPL(perf_event_read_value);
  2767. static int perf_event_read_group(struct perf_event *event,
  2768. u64 read_format, char __user *buf)
  2769. {
  2770. struct perf_event *leader = event->group_leader, *sub;
  2771. int n = 0, size = 0, ret = -EFAULT;
  2772. struct perf_event_context *ctx = leader->ctx;
  2773. u64 values[5];
  2774. u64 count, enabled, running;
  2775. mutex_lock(&ctx->mutex);
  2776. count = perf_event_read_value(leader, &enabled, &running);
  2777. values[n++] = 1 + leader->nr_siblings;
  2778. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  2779. values[n++] = enabled;
  2780. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  2781. values[n++] = running;
  2782. values[n++] = count;
  2783. if (read_format & PERF_FORMAT_ID)
  2784. values[n++] = primary_event_id(leader);
  2785. size = n * sizeof(u64);
  2786. if (copy_to_user(buf, values, size))
  2787. goto unlock;
  2788. ret = size;
  2789. list_for_each_entry(sub, &leader->sibling_list, group_entry) {
  2790. n = 0;
  2791. values[n++] = perf_event_read_value(sub, &enabled, &running);
  2792. if (read_format & PERF_FORMAT_ID)
  2793. values[n++] = primary_event_id(sub);
  2794. size = n * sizeof(u64);
  2795. if (copy_to_user(buf + ret, values, size)) {
  2796. ret = -EFAULT;
  2797. goto unlock;
  2798. }
  2799. ret += size;
  2800. }
  2801. unlock:
  2802. mutex_unlock(&ctx->mutex);
  2803. return ret;
  2804. }
  2805. static int perf_event_read_one(struct perf_event *event,
  2806. u64 read_format, char __user *buf)
  2807. {
  2808. u64 enabled, running;
  2809. u64 values[4];
  2810. int n = 0;
  2811. values[n++] = perf_event_read_value(event, &enabled, &running);
  2812. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  2813. values[n++] = enabled;
  2814. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  2815. values[n++] = running;
  2816. if (read_format & PERF_FORMAT_ID)
  2817. values[n++] = primary_event_id(event);
  2818. if (copy_to_user(buf, values, n * sizeof(u64)))
  2819. return -EFAULT;
  2820. return n * sizeof(u64);
  2821. }
  2822. /*
  2823. * Read the performance event - simple non blocking version for now
  2824. */
  2825. static ssize_t
  2826. perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
  2827. {
  2828. u64 read_format = event->attr.read_format;
  2829. int ret;
  2830. /*
  2831. * Return end-of-file for a read on a event that is in
  2832. * error state (i.e. because it was pinned but it couldn't be
  2833. * scheduled on to the CPU at some point).
  2834. */
  2835. if (event->state == PERF_EVENT_STATE_ERROR)
  2836. return 0;
  2837. if (count < event->read_size)
  2838. return -ENOSPC;
  2839. WARN_ON_ONCE(event->ctx->parent_ctx);
  2840. if (read_format & PERF_FORMAT_GROUP)
  2841. ret = perf_event_read_group(event, read_format, buf);
  2842. else
  2843. ret = perf_event_read_one(event, read_format, buf);
  2844. return ret;
  2845. }
  2846. static ssize_t
  2847. perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
  2848. {
  2849. struct perf_event *event = file->private_data;
  2850. return perf_read_hw(event, buf, count);
  2851. }
  2852. static unsigned int perf_poll(struct file *file, poll_table *wait)
  2853. {
  2854. struct perf_event *event = file->private_data;
  2855. struct ring_buffer *rb;
  2856. unsigned int events = POLL_HUP;
  2857. /*
  2858. * Pin the event->rb by taking event->mmap_mutex; otherwise
  2859. * perf_event_set_output() can swizzle our rb and make us miss wakeups.
  2860. */
  2861. mutex_lock(&event->mmap_mutex);
  2862. rb = event->rb;
  2863. if (rb)
  2864. events = atomic_xchg(&rb->poll, 0);
  2865. mutex_unlock(&event->mmap_mutex);
  2866. poll_wait(file, &event->waitq, wait);
  2867. return events;
  2868. }
  2869. static void perf_event_reset(struct perf_event *event)
  2870. {
  2871. (void)perf_event_read(event);
  2872. local64_set(&event->count, 0);
  2873. perf_event_update_userpage(event);
  2874. }
  2875. /*
  2876. * Holding the top-level event's child_mutex means that any
  2877. * descendant process that has inherited this event will block
  2878. * in sync_child_event if it goes to exit, thus satisfying the
  2879. * task existence requirements of perf_event_enable/disable.
  2880. */
  2881. static void perf_event_for_each_child(struct perf_event *event,
  2882. void (*func)(struct perf_event *))
  2883. {
  2884. struct perf_event *child;
  2885. WARN_ON_ONCE(event->ctx->parent_ctx);
  2886. mutex_lock(&event->child_mutex);
  2887. func(event);
  2888. list_for_each_entry(child, &event->child_list, child_list)
  2889. func(child);
  2890. mutex_unlock(&event->child_mutex);
  2891. }
  2892. static void perf_event_for_each(struct perf_event *event,
  2893. void (*func)(struct perf_event *))
  2894. {
  2895. struct perf_event_context *ctx = event->ctx;
  2896. struct perf_event *sibling;
  2897. WARN_ON_ONCE(ctx->parent_ctx);
  2898. mutex_lock(&ctx->mutex);
  2899. event = event->group_leader;
  2900. perf_event_for_each_child(event, func);
  2901. list_for_each_entry(sibling, &event->sibling_list, group_entry)
  2902. perf_event_for_each_child(sibling, func);
  2903. mutex_unlock(&ctx->mutex);
  2904. }
  2905. static int perf_event_period(struct perf_event *event, u64 __user *arg)
  2906. {
  2907. struct perf_event_context *ctx = event->ctx;
  2908. int ret = 0;
  2909. u64 value;
  2910. if (!is_sampling_event(event))
  2911. return -EINVAL;
  2912. if (copy_from_user(&value, arg, sizeof(value)))
  2913. return -EFAULT;
  2914. if (!value)
  2915. return -EINVAL;
  2916. raw_spin_lock_irq(&ctx->lock);
  2917. if (event->attr.freq) {
  2918. if (value > sysctl_perf_event_sample_rate) {
  2919. ret = -EINVAL;
  2920. goto unlock;
  2921. }
  2922. event->attr.sample_freq = value;
  2923. } else {
  2924. event->attr.sample_period = value;
  2925. event->hw.sample_period = value;
  2926. }
  2927. unlock:
  2928. raw_spin_unlock_irq(&ctx->lock);
  2929. return ret;
  2930. }
  2931. static const struct file_operations perf_fops;
  2932. static inline int perf_fget_light(int fd, struct fd *p)
  2933. {
  2934. struct fd f = fdget(fd);
  2935. if (!f.file)
  2936. return -EBADF;
  2937. if (f.file->f_op != &perf_fops) {
  2938. fdput(f);
  2939. return -EBADF;
  2940. }
  2941. *p = f;
  2942. return 0;
  2943. }
  2944. static int perf_event_set_output(struct perf_event *event,
  2945. struct perf_event *output_event);
  2946. static int perf_event_set_filter(struct perf_event *event, void __user *arg);
  2947. static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  2948. {
  2949. struct perf_event *event = file->private_data;
  2950. void (*func)(struct perf_event *);
  2951. u32 flags = arg;
  2952. switch (cmd) {
  2953. case PERF_EVENT_IOC_ENABLE:
  2954. func = perf_event_enable;
  2955. break;
  2956. case PERF_EVENT_IOC_DISABLE:
  2957. func = perf_event_disable;
  2958. break;
  2959. case PERF_EVENT_IOC_RESET:
  2960. func = perf_event_reset;
  2961. break;
  2962. case PERF_EVENT_IOC_REFRESH:
  2963. return perf_event_refresh(event, arg);
  2964. case PERF_EVENT_IOC_PERIOD:
  2965. return perf_event_period(event, (u64 __user *)arg);
  2966. case PERF_EVENT_IOC_ID:
  2967. {
  2968. u64 id = primary_event_id(event);
  2969. if (copy_to_user((void __user *)arg, &id, sizeof(id)))
  2970. return -EFAULT;
  2971. return 0;
  2972. }
  2973. case PERF_EVENT_IOC_SET_OUTPUT:
  2974. {
  2975. int ret;
  2976. if (arg != -1) {
  2977. struct perf_event *output_event;
  2978. struct fd output;
  2979. ret = perf_fget_light(arg, &output);
  2980. if (ret)
  2981. return ret;
  2982. output_event = output.file->private_data;
  2983. ret = perf_event_set_output(event, output_event);
  2984. fdput(output);
  2985. } else {
  2986. ret = perf_event_set_output(event, NULL);
  2987. }
  2988. return ret;
  2989. }
  2990. case PERF_EVENT_IOC_SET_FILTER:
  2991. return perf_event_set_filter(event, (void __user *)arg);
  2992. default:
  2993. return -ENOTTY;
  2994. }
  2995. if (flags & PERF_IOC_FLAG_GROUP)
  2996. perf_event_for_each(event, func);
  2997. else
  2998. perf_event_for_each_child(event, func);
  2999. return 0;
  3000. }
  3001. int perf_event_task_enable(void)
  3002. {
  3003. struct perf_event *event;
  3004. mutex_lock(&current->perf_event_mutex);
  3005. list_for_each_entry(event, &current->perf_event_list, owner_entry)
  3006. perf_event_for_each_child(event, perf_event_enable);
  3007. mutex_unlock(&current->perf_event_mutex);
  3008. return 0;
  3009. }
  3010. int perf_event_task_disable(void)
  3011. {
  3012. struct perf_event *event;
  3013. mutex_lock(&current->perf_event_mutex);
  3014. list_for_each_entry(event, &current->perf_event_list, owner_entry)
  3015. perf_event_for_each_child(event, perf_event_disable);
  3016. mutex_unlock(&current->perf_event_mutex);
  3017. return 0;
  3018. }
  3019. static int perf_event_index(struct perf_event *event)
  3020. {
  3021. if (event->hw.state & PERF_HES_STOPPED)
  3022. return 0;
  3023. if (event->state != PERF_EVENT_STATE_ACTIVE)
  3024. return 0;
  3025. return event->pmu->event_idx(event);
  3026. }
  3027. static void calc_timer_values(struct perf_event *event,
  3028. u64 *now,
  3029. u64 *enabled,
  3030. u64 *running)
  3031. {
  3032. u64 ctx_time;
  3033. *now = perf_clock();
  3034. ctx_time = event->shadow_ctx_time + *now;
  3035. *enabled = ctx_time - event->tstamp_enabled;
  3036. *running = ctx_time - event->tstamp_running;
  3037. }
  3038. static void perf_event_init_userpage(struct perf_event *event)
  3039. {
  3040. struct perf_event_mmap_page *userpg;
  3041. struct ring_buffer *rb;
  3042. rcu_read_lock();
  3043. rb = rcu_dereference(event->rb);
  3044. if (!rb)
  3045. goto unlock;
  3046. userpg = rb->user_page;
  3047. /* Allow new userspace to detect that bit 0 is deprecated */
  3048. userpg->cap_bit0_is_deprecated = 1;
  3049. userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
  3050. unlock:
  3051. rcu_read_unlock();
  3052. }
  3053. void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
  3054. {
  3055. }
  3056. /*
  3057. * Callers need to ensure there can be no nesting of this function, otherwise
  3058. * the seqlock logic goes bad. We can not serialize this because the arch
  3059. * code calls this from NMI context.
  3060. */
  3061. void perf_event_update_userpage(struct perf_event *event)
  3062. {
  3063. struct perf_event_mmap_page *userpg;
  3064. struct ring_buffer *rb;
  3065. u64 enabled, running, now;
  3066. rcu_read_lock();
  3067. rb = rcu_dereference(event->rb);
  3068. if (!rb)
  3069. goto unlock;
  3070. /*
  3071. * compute total_time_enabled, total_time_running
  3072. * based on snapshot values taken when the event
  3073. * was last scheduled in.
  3074. *
  3075. * we cannot simply called update_context_time()
  3076. * because of locking issue as we can be called in
  3077. * NMI context
  3078. */
  3079. calc_timer_values(event, &now, &enabled, &running);
  3080. userpg = rb->user_page;
  3081. /*
  3082. * Disable preemption so as to not let the corresponding user-space
  3083. * spin too long if we get preempted.
  3084. */
  3085. preempt_disable();
  3086. ++userpg->lock;
  3087. barrier();
  3088. userpg->index = perf_event_index(event);
  3089. userpg->offset = perf_event_count(event);
  3090. if (userpg->index)
  3091. userpg->offset -= local64_read(&event->hw.prev_count);
  3092. userpg->time_enabled = enabled +
  3093. atomic64_read(&event->child_total_time_enabled);
  3094. userpg->time_running = running +
  3095. atomic64_read(&event->child_total_time_running);
  3096. arch_perf_update_userpage(userpg, now);
  3097. barrier();
  3098. ++userpg->lock;
  3099. preempt_enable();
  3100. unlock:
  3101. rcu_read_unlock();
  3102. }
  3103. static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  3104. {
  3105. struct perf_event *event = vma->vm_file->private_data;
  3106. struct ring_buffer *rb;
  3107. int ret = VM_FAULT_SIGBUS;
  3108. if (vmf->flags & FAULT_FLAG_MKWRITE) {
  3109. if (vmf->pgoff == 0)
  3110. ret = 0;
  3111. return ret;
  3112. }
  3113. rcu_read_lock();
  3114. rb = rcu_dereference(event->rb);
  3115. if (!rb)
  3116. goto unlock;
  3117. if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
  3118. goto unlock;
  3119. vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
  3120. if (!vmf->page)
  3121. goto unlock;
  3122. get_page(vmf->page);
  3123. vmf->page->mapping = vma->vm_file->f_mapping;
  3124. vmf->page->index = vmf->pgoff;
  3125. ret = 0;
  3126. unlock:
  3127. rcu_read_unlock();
  3128. return ret;
  3129. }
  3130. static void ring_buffer_attach(struct perf_event *event,
  3131. struct ring_buffer *rb)
  3132. {
  3133. unsigned long flags;
  3134. if (!list_empty(&event->rb_entry))
  3135. return;
  3136. spin_lock_irqsave(&rb->event_lock, flags);
  3137. if (list_empty(&event->rb_entry))
  3138. list_add(&event->rb_entry, &rb->event_list);
  3139. spin_unlock_irqrestore(&rb->event_lock, flags);
  3140. }
  3141. static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb)
  3142. {
  3143. unsigned long flags;
  3144. if (list_empty(&event->rb_entry))
  3145. return;
  3146. spin_lock_irqsave(&rb->event_lock, flags);
  3147. list_del_init(&event->rb_entry);
  3148. wake_up_all(&event->waitq);
  3149. spin_unlock_irqrestore(&rb->event_lock, flags);
  3150. }
  3151. static void ring_buffer_wakeup(struct perf_event *event)
  3152. {
  3153. struct ring_buffer *rb;
  3154. rcu_read_lock();
  3155. rb = rcu_dereference(event->rb);
  3156. if (rb) {
  3157. list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
  3158. wake_up_all(&event->waitq);
  3159. }
  3160. rcu_read_unlock();
  3161. }
  3162. static void rb_free_rcu(struct rcu_head *rcu_head)
  3163. {
  3164. struct ring_buffer *rb;
  3165. rb = container_of(rcu_head, struct ring_buffer, rcu_head);
  3166. rb_free(rb);
  3167. }
  3168. static struct ring_buffer *ring_buffer_get(struct perf_event *event)
  3169. {
  3170. struct ring_buffer *rb;
  3171. rcu_read_lock();
  3172. rb = rcu_dereference(event->rb);
  3173. if (rb) {
  3174. if (!atomic_inc_not_zero(&rb->refcount))
  3175. rb = NULL;
  3176. }
  3177. rcu_read_unlock();
  3178. return rb;
  3179. }
  3180. static void ring_buffer_put(struct ring_buffer *rb)
  3181. {
  3182. if (!atomic_dec_and_test(&rb->refcount))
  3183. return;
  3184. WARN_ON_ONCE(!list_empty(&rb->event_list));
  3185. call_rcu(&rb->rcu_head, rb_free_rcu);
  3186. }
  3187. static void perf_mmap_open(struct vm_area_struct *vma)
  3188. {
  3189. struct perf_event *event = vma->vm_file->private_data;
  3190. atomic_inc(&event->mmap_count);
  3191. atomic_inc(&event->rb->mmap_count);
  3192. }
  3193. /*
  3194. * A buffer can be mmap()ed multiple times; either directly through the same
  3195. * event, or through other events by use of perf_event_set_output().
  3196. *
  3197. * In order to undo the VM accounting done by perf_mmap() we need to destroy
  3198. * the buffer here, where we still have a VM context. This means we need
  3199. * to detach all events redirecting to us.
  3200. */
  3201. static void perf_mmap_close(struct vm_area_struct *vma)
  3202. {
  3203. struct perf_event *event = vma->vm_file->private_data;
  3204. struct ring_buffer *rb = event->rb;
  3205. struct user_struct *mmap_user = rb->mmap_user;
  3206. int mmap_locked = rb->mmap_locked;
  3207. unsigned long size = perf_data_size(rb);
  3208. atomic_dec(&rb->mmap_count);
  3209. if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
  3210. return;
  3211. /* Detach current event from the buffer. */
  3212. rcu_assign_pointer(event->rb, NULL);
  3213. ring_buffer_detach(event, rb);
  3214. mutex_unlock(&event->mmap_mutex);
  3215. /* If there's still other mmap()s of this buffer, we're done. */
  3216. if (atomic_read(&rb->mmap_count)) {
  3217. ring_buffer_put(rb); /* can't be last */
  3218. return;
  3219. }
  3220. /*
  3221. * No other mmap()s, detach from all other events that might redirect
  3222. * into the now unreachable buffer. Somewhat complicated by the
  3223. * fact that rb::event_lock otherwise nests inside mmap_mutex.
  3224. */
  3225. again:
  3226. rcu_read_lock();
  3227. list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
  3228. if (!atomic_long_inc_not_zero(&event->refcount)) {
  3229. /*
  3230. * This event is en-route to free_event() which will
  3231. * detach it and remove it from the list.
  3232. */
  3233. continue;
  3234. }
  3235. rcu_read_unlock();
  3236. mutex_lock(&event->mmap_mutex);
  3237. /*
  3238. * Check we didn't race with perf_event_set_output() which can
  3239. * swizzle the rb from under us while we were waiting to
  3240. * acquire mmap_mutex.
  3241. *
  3242. * If we find a different rb; ignore this event, a next
  3243. * iteration will no longer find it on the list. We have to
  3244. * still restart the iteration to make sure we're not now
  3245. * iterating the wrong list.
  3246. */
  3247. if (event->rb == rb) {
  3248. rcu_assign_pointer(event->rb, NULL);
  3249. ring_buffer_detach(event, rb);
  3250. ring_buffer_put(rb); /* can't be last, we still have one */
  3251. }
  3252. mutex_unlock(&event->mmap_mutex);
  3253. put_event(event);
  3254. /*
  3255. * Restart the iteration; either we're on the wrong list or
  3256. * destroyed its integrity by doing a deletion.
  3257. */
  3258. goto again;
  3259. }
  3260. rcu_read_unlock();
  3261. /*
  3262. * It could be there's still a few 0-ref events on the list; they'll
  3263. * get cleaned up by free_event() -- they'll also still have their
  3264. * ref on the rb and will free it whenever they are done with it.
  3265. *
  3266. * Aside from that, this buffer is 'fully' detached and unmapped,
  3267. * undo the VM accounting.
  3268. */
  3269. atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
  3270. vma->vm_mm->pinned_vm -= mmap_locked;
  3271. free_uid(mmap_user);
  3272. ring_buffer_put(rb); /* could be last */
  3273. }
  3274. static const struct vm_operations_struct perf_mmap_vmops = {
  3275. .open = perf_mmap_open,
  3276. .close = perf_mmap_close,
  3277. .fault = perf_mmap_fault,
  3278. .page_mkwrite = perf_mmap_fault,
  3279. };
  3280. static int perf_mmap(struct file *file, struct vm_area_struct *vma)
  3281. {
  3282. struct perf_event *event = file->private_data;
  3283. unsigned long user_locked, user_lock_limit;
  3284. struct user_struct *user = current_user();
  3285. unsigned long locked, lock_limit;
  3286. struct ring_buffer *rb;
  3287. unsigned long vma_size;
  3288. unsigned long nr_pages;
  3289. long user_extra, extra;
  3290. int ret = 0, flags = 0;
  3291. /*
  3292. * Don't allow mmap() of inherited per-task counters. This would
  3293. * create a performance issue due to all children writing to the
  3294. * same rb.
  3295. */
  3296. if (event->cpu == -1 && event->attr.inherit)
  3297. return -EINVAL;
  3298. if (!(vma->vm_flags & VM_SHARED))
  3299. return -EINVAL;
  3300. vma_size = vma->vm_end - vma->vm_start;
  3301. nr_pages = (vma_size / PAGE_SIZE) - 1;
  3302. /*
  3303. * If we have rb pages ensure they're a power-of-two number, so we
  3304. * can do bitmasks instead of modulo.
  3305. */
  3306. if (nr_pages != 0 && !is_power_of_2(nr_pages))
  3307. return -EINVAL;
  3308. if (vma_size != PAGE_SIZE * (1 + nr_pages))
  3309. return -EINVAL;
  3310. if (vma->vm_pgoff != 0)
  3311. return -EINVAL;
  3312. WARN_ON_ONCE(event->ctx->parent_ctx);
  3313. again:
  3314. mutex_lock(&event->mmap_mutex);
  3315. if (event->rb) {
  3316. if (event->rb->nr_pages != nr_pages) {
  3317. ret = -EINVAL;
  3318. goto unlock;
  3319. }
  3320. if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
  3321. /*
  3322. * Raced against perf_mmap_close() through
  3323. * perf_event_set_output(). Try again, hope for better
  3324. * luck.
  3325. */
  3326. mutex_unlock(&event->mmap_mutex);
  3327. goto again;
  3328. }
  3329. goto unlock;
  3330. }
  3331. user_extra = nr_pages + 1;
  3332. user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
  3333. /*
  3334. * Increase the limit linearly with more CPUs:
  3335. */
  3336. user_lock_limit *= num_online_cpus();
  3337. user_locked = atomic_long_read(&user->locked_vm) + user_extra;
  3338. extra = 0;
  3339. if (user_locked > user_lock_limit)
  3340. extra = user_locked - user_lock_limit;
  3341. lock_limit = rlimit(RLIMIT_MEMLOCK);
  3342. lock_limit >>= PAGE_SHIFT;
  3343. locked = vma->vm_mm->pinned_vm + extra;
  3344. if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
  3345. !capable(CAP_IPC_LOCK)) {
  3346. ret = -EPERM;
  3347. goto unlock;
  3348. }
  3349. WARN_ON(event->rb);
  3350. if (vma->vm_flags & VM_WRITE)
  3351. flags |= RING_BUFFER_WRITABLE;
  3352. rb = rb_alloc(nr_pages,
  3353. event->attr.watermark ? event->attr.wakeup_watermark : 0,
  3354. event->cpu, flags);
  3355. if (!rb) {
  3356. ret = -ENOMEM;
  3357. goto unlock;
  3358. }
  3359. atomic_set(&rb->mmap_count, 1);
  3360. rb->mmap_locked = extra;
  3361. rb->mmap_user = get_current_user();
  3362. atomic_long_add(user_extra, &user->locked_vm);
  3363. vma->vm_mm->pinned_vm += extra;
  3364. ring_buffer_attach(event, rb);
  3365. rcu_assign_pointer(event->rb, rb);
  3366. perf_event_init_userpage(event);
  3367. perf_event_update_userpage(event);
  3368. unlock:
  3369. if (!ret)
  3370. atomic_inc(&event->mmap_count);
  3371. mutex_unlock(&event->mmap_mutex);
  3372. /*
  3373. * Since pinned accounting is per vm we cannot allow fork() to copy our
  3374. * vma.
  3375. */
  3376. vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
  3377. vma->vm_ops = &perf_mmap_vmops;
  3378. return ret;
  3379. }
  3380. static int perf_fasync(int fd, struct file *filp, int on)
  3381. {
  3382. struct inode *inode = file_inode(filp);
  3383. struct perf_event *event = filp->private_data;
  3384. int retval;
  3385. mutex_lock(&inode->i_mutex);
  3386. retval = fasync_helper(fd, filp, on, &event->fasync);
  3387. mutex_unlock(&inode->i_mutex);
  3388. if (retval < 0)
  3389. return retval;
  3390. return 0;
  3391. }
  3392. static const struct file_operations perf_fops = {
  3393. .llseek = no_llseek,
  3394. .release = perf_release,
  3395. .read = perf_read,
  3396. .poll = perf_poll,
  3397. .unlocked_ioctl = perf_ioctl,
  3398. .compat_ioctl = perf_ioctl,
  3399. .mmap = perf_mmap,
  3400. .fasync = perf_fasync,
  3401. };
  3402. /*
  3403. * Perf event wakeup
  3404. *
  3405. * If there's data, ensure we set the poll() state and publish everything
  3406. * to user-space before waking everybody up.
  3407. */
  3408. void perf_event_wakeup(struct perf_event *event)
  3409. {
  3410. ring_buffer_wakeup(event);
  3411. if (event->pending_kill) {
  3412. kill_fasync(&event->fasync, SIGIO, event->pending_kill);
  3413. event->pending_kill = 0;
  3414. }
  3415. }
  3416. static void perf_pending_event(struct irq_work *entry)
  3417. {
  3418. struct perf_event *event = container_of(entry,
  3419. struct perf_event, pending);
  3420. if (event->pending_disable) {
  3421. event->pending_disable = 0;
  3422. __perf_event_disable(event);
  3423. }
  3424. if (event->pending_wakeup) {
  3425. event->pending_wakeup = 0;
  3426. perf_event_wakeup(event);
  3427. }
  3428. }
  3429. /*
  3430. * We assume there is only KVM supporting the callbacks.
  3431. * Later on, we might change it to a list if there is
  3432. * another virtualization implementation supporting the callbacks.
  3433. */
  3434. struct perf_guest_info_callbacks *perf_guest_cbs;
  3435. int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
  3436. {
  3437. perf_guest_cbs = cbs;
  3438. return 0;
  3439. }
  3440. EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
  3441. int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
  3442. {
  3443. perf_guest_cbs = NULL;
  3444. return 0;
  3445. }
  3446. EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
  3447. static void
  3448. perf_output_sample_regs(struct perf_output_handle *handle,
  3449. struct pt_regs *regs, u64 mask)
  3450. {
  3451. int bit;
  3452. for_each_set_bit(bit, (const unsigned long *) &mask,
  3453. sizeof(mask) * BITS_PER_BYTE) {
  3454. u64 val;
  3455. val = perf_reg_value(regs, bit);
  3456. perf_output_put(handle, val);
  3457. }
  3458. }
  3459. static void perf_sample_regs_user(struct perf_regs_user *regs_user,
  3460. struct pt_regs *regs)
  3461. {
  3462. if (!user_mode(regs)) {
  3463. if (current->mm)
  3464. regs = task_pt_regs(current);
  3465. else
  3466. regs = NULL;
  3467. }
  3468. if (regs) {
  3469. regs_user->regs = regs;
  3470. regs_user->abi = perf_reg_abi(current);
  3471. }
  3472. }
  3473. /*
  3474. * Get remaining task size from user stack pointer.
  3475. *
  3476. * It'd be better to take stack vma map and limit this more
  3477. * precisly, but there's no way to get it safely under interrupt,
  3478. * so using TASK_SIZE as limit.
  3479. */
  3480. static u64 perf_ustack_task_size(struct pt_regs *regs)
  3481. {
  3482. unsigned long addr = perf_user_stack_pointer(regs);
  3483. if (!addr || addr >= TASK_SIZE)
  3484. return 0;
  3485. return TASK_SIZE - addr;
  3486. }
  3487. static u16
  3488. perf_sample_ustack_size(u16 stack_size, u16 header_size,
  3489. struct pt_regs *regs)
  3490. {
  3491. u64 task_size;
  3492. /* No regs, no stack pointer, no dump. */
  3493. if (!regs)
  3494. return 0;
  3495. /*
  3496. * Check if we fit in with the requested stack size into the:
  3497. * - TASK_SIZE
  3498. * If we don't, we limit the size to the TASK_SIZE.
  3499. *
  3500. * - remaining sample size
  3501. * If we don't, we customize the stack size to
  3502. * fit in to the remaining sample size.
  3503. */
  3504. task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
  3505. stack_size = min(stack_size, (u16) task_size);
  3506. /* Current header size plus static size and dynamic size. */
  3507. header_size += 2 * sizeof(u64);
  3508. /* Do we fit in with the current stack dump size? */
  3509. if ((u16) (header_size + stack_size) < header_size) {
  3510. /*
  3511. * If we overflow the maximum size for the sample,
  3512. * we customize the stack dump size to fit in.
  3513. */
  3514. stack_size = USHRT_MAX - header_size - sizeof(u64);
  3515. stack_size = round_up(stack_size, sizeof(u64));
  3516. }
  3517. return stack_size;
  3518. }
  3519. static void
  3520. perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
  3521. struct pt_regs *regs)
  3522. {
  3523. /* Case of a kernel thread, nothing to dump */
  3524. if (!regs) {
  3525. u64 size = 0;
  3526. perf_output_put(handle, size);
  3527. } else {
  3528. unsigned long sp;
  3529. unsigned int rem;
  3530. u64 dyn_size;
  3531. /*
  3532. * We dump:
  3533. * static size
  3534. * - the size requested by user or the best one we can fit
  3535. * in to the sample max size
  3536. * data
  3537. * - user stack dump data
  3538. * dynamic size
  3539. * - the actual dumped size
  3540. */
  3541. /* Static size. */
  3542. perf_output_put(handle, dump_size);
  3543. /* Data. */
  3544. sp = perf_user_stack_pointer(regs);
  3545. rem = __output_copy_user(handle, (void *) sp, dump_size);
  3546. dyn_size = dump_size - rem;
  3547. perf_output_skip(handle, rem);
  3548. /* Dynamic size. */
  3549. perf_output_put(handle, dyn_size);
  3550. }
  3551. }
  3552. static void __perf_event_header__init_id(struct perf_event_header *header,
  3553. struct perf_sample_data *data,
  3554. struct perf_event *event)
  3555. {
  3556. u64 sample_type = event->attr.sample_type;
  3557. data->type = sample_type;
  3558. header->size += event->id_header_size;
  3559. if (sample_type & PERF_SAMPLE_TID) {
  3560. /* namespace issues */
  3561. data->tid_entry.pid = perf_event_pid(event, current);
  3562. data->tid_entry.tid = perf_event_tid(event, current);
  3563. }
  3564. if (sample_type & PERF_SAMPLE_TIME)
  3565. data->time = perf_clock();
  3566. if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
  3567. data->id = primary_event_id(event);
  3568. if (sample_type & PERF_SAMPLE_STREAM_ID)
  3569. data->stream_id = event->id;
  3570. if (sample_type & PERF_SAMPLE_CPU) {
  3571. data->cpu_entry.cpu = raw_smp_processor_id();
  3572. data->cpu_entry.reserved = 0;
  3573. }
  3574. }
  3575. void perf_event_header__init_id(struct perf_event_header *header,
  3576. struct perf_sample_data *data,
  3577. struct perf_event *event)
  3578. {
  3579. if (event->attr.sample_id_all)
  3580. __perf_event_header__init_id(header, data, event);
  3581. }
  3582. static void __perf_event__output_id_sample(struct perf_output_handle *handle,
  3583. struct perf_sample_data *data)
  3584. {
  3585. u64 sample_type = data->type;
  3586. if (sample_type & PERF_SAMPLE_TID)
  3587. perf_output_put(handle, data->tid_entry);
  3588. if (sample_type & PERF_SAMPLE_TIME)
  3589. perf_output_put(handle, data->time);
  3590. if (sample_type & PERF_SAMPLE_ID)
  3591. perf_output_put(handle, data->id);
  3592. if (sample_type & PERF_SAMPLE_STREAM_ID)
  3593. perf_output_put(handle, data->stream_id);
  3594. if (sample_type & PERF_SAMPLE_CPU)
  3595. perf_output_put(handle, data->cpu_entry);
  3596. if (sample_type & PERF_SAMPLE_IDENTIFIER)
  3597. perf_output_put(handle, data->id);
  3598. }
  3599. void perf_event__output_id_sample(struct perf_event *event,
  3600. struct perf_output_handle *handle,
  3601. struct perf_sample_data *sample)
  3602. {
  3603. if (event->attr.sample_id_all)
  3604. __perf_event__output_id_sample(handle, sample);
  3605. }
  3606. static void perf_output_read_one(struct perf_output_handle *handle,
  3607. struct perf_event *event,
  3608. u64 enabled, u64 running)
  3609. {
  3610. u64 read_format = event->attr.read_format;
  3611. u64 values[4];
  3612. int n = 0;
  3613. values[n++] = perf_event_count(event);
  3614. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
  3615. values[n++] = enabled +
  3616. atomic64_read(&event->child_total_time_enabled);
  3617. }
  3618. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
  3619. values[n++] = running +
  3620. atomic64_read(&event->child_total_time_running);
  3621. }
  3622. if (read_format & PERF_FORMAT_ID)
  3623. values[n++] = primary_event_id(event);
  3624. __output_copy(handle, values, n * sizeof(u64));
  3625. }
  3626. /*
  3627. * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
  3628. */
  3629. static void perf_output_read_group(struct perf_output_handle *handle,
  3630. struct perf_event *event,
  3631. u64 enabled, u64 running)
  3632. {
  3633. struct perf_event *leader = event->group_leader, *sub;
  3634. u64 read_format = event->attr.read_format;
  3635. u64 values[5];
  3636. int n = 0;
  3637. values[n++] = 1 + leader->nr_siblings;
  3638. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  3639. values[n++] = enabled;
  3640. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  3641. values[n++] = running;
  3642. if (leader != event)
  3643. leader->pmu->read(leader);
  3644. values[n++] = perf_event_count(leader);
  3645. if (read_format & PERF_FORMAT_ID)
  3646. values[n++] = primary_event_id(leader);
  3647. __output_copy(handle, values, n * sizeof(u64));
  3648. list_for_each_entry(sub, &leader->sibling_list, group_entry) {
  3649. n = 0;
  3650. if ((sub != event) &&
  3651. (sub->state == PERF_EVENT_STATE_ACTIVE))
  3652. sub->pmu->read(sub);
  3653. values[n++] = perf_event_count(sub);
  3654. if (read_format & PERF_FORMAT_ID)
  3655. values[n++] = primary_event_id(sub);
  3656. __output_copy(handle, values, n * sizeof(u64));
  3657. }
  3658. }
  3659. #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
  3660. PERF_FORMAT_TOTAL_TIME_RUNNING)
  3661. static void perf_output_read(struct perf_output_handle *handle,
  3662. struct perf_event *event)
  3663. {
  3664. u64 enabled = 0, running = 0, now;
  3665. u64 read_format = event->attr.read_format;
  3666. /*
  3667. * compute total_time_enabled, total_time_running
  3668. * based on snapshot values taken when the event
  3669. * was last scheduled in.
  3670. *
  3671. * we cannot simply called update_context_time()
  3672. * because of locking issue as we are called in
  3673. * NMI context
  3674. */
  3675. if (read_format & PERF_FORMAT_TOTAL_TIMES)
  3676. calc_timer_values(event, &now, &enabled, &running);
  3677. if (event->attr.read_format & PERF_FORMAT_GROUP)
  3678. perf_output_read_group(handle, event, enabled, running);
  3679. else
  3680. perf_output_read_one(handle, event, enabled, running);
  3681. }
  3682. void perf_output_sample(struct perf_output_handle *handle,
  3683. struct perf_event_header *header,
  3684. struct perf_sample_data *data,
  3685. struct perf_event *event)
  3686. {
  3687. u64 sample_type = data->type;
  3688. perf_output_put(handle, *header);
  3689. if (sample_type & PERF_SAMPLE_IDENTIFIER)
  3690. perf_output_put(handle, data->id);
  3691. if (sample_type & PERF_SAMPLE_IP)
  3692. perf_output_put(handle, data->ip);
  3693. if (sample_type & PERF_SAMPLE_TID)
  3694. perf_output_put(handle, data->tid_entry);
  3695. if (sample_type & PERF_SAMPLE_TIME)
  3696. perf_output_put(handle, data->time);
  3697. if (sample_type & PERF_SAMPLE_ADDR)
  3698. perf_output_put(handle, data->addr);
  3699. if (sample_type & PERF_SAMPLE_ID)
  3700. perf_output_put(handle, data->id);
  3701. if (sample_type & PERF_SAMPLE_STREAM_ID)
  3702. perf_output_put(handle, data->stream_id);
  3703. if (sample_type & PERF_SAMPLE_CPU)
  3704. perf_output_put(handle, data->cpu_entry);
  3705. if (sample_type & PERF_SAMPLE_PERIOD)
  3706. perf_output_put(handle, data->period);
  3707. if (sample_type & PERF_SAMPLE_READ)
  3708. perf_output_read(handle, event);
  3709. if (sample_type & PERF_SAMPLE_CALLCHAIN) {
  3710. if (data->callchain) {
  3711. int size = 1;
  3712. if (data->callchain)
  3713. size += data->callchain->nr;
  3714. size *= sizeof(u64);
  3715. __output_copy(handle, data->callchain, size);
  3716. } else {
  3717. u64 nr = 0;
  3718. perf_output_put(handle, nr);
  3719. }
  3720. }
  3721. if (sample_type & PERF_SAMPLE_RAW) {
  3722. if (data->raw) {
  3723. perf_output_put(handle, data->raw->size);
  3724. __output_copy(handle, data->raw->data,
  3725. data->raw->size);
  3726. } else {
  3727. struct {
  3728. u32 size;
  3729. u32 data;
  3730. } raw = {
  3731. .size = sizeof(u32),
  3732. .data = 0,
  3733. };
  3734. perf_output_put(handle, raw);
  3735. }
  3736. }
  3737. if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
  3738. if (data->br_stack) {
  3739. size_t size;
  3740. size = data->br_stack->nr
  3741. * sizeof(struct perf_branch_entry);
  3742. perf_output_put(handle, data->br_stack->nr);
  3743. perf_output_copy(handle, data->br_stack->entries, size);
  3744. } else {
  3745. /*
  3746. * we always store at least the value of nr
  3747. */
  3748. u64 nr = 0;
  3749. perf_output_put(handle, nr);
  3750. }
  3751. }
  3752. if (sample_type & PERF_SAMPLE_REGS_USER) {
  3753. u64 abi = data->regs_user.abi;
  3754. /*
  3755. * If there are no regs to dump, notice it through
  3756. * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
  3757. */
  3758. perf_output_put(handle, abi);
  3759. if (abi) {
  3760. u64 mask = event->attr.sample_regs_user;
  3761. perf_output_sample_regs(handle,
  3762. data->regs_user.regs,
  3763. mask);
  3764. }
  3765. }
  3766. if (sample_type & PERF_SAMPLE_STACK_USER) {
  3767. perf_output_sample_ustack(handle,
  3768. data->stack_user_size,
  3769. data->regs_user.regs);
  3770. }
  3771. if (sample_type & PERF_SAMPLE_WEIGHT)
  3772. perf_output_put(handle, data->weight);
  3773. if (sample_type & PERF_SAMPLE_DATA_SRC)
  3774. perf_output_put(handle, data->data_src.val);
  3775. if (!event->attr.watermark) {
  3776. int wakeup_events = event->attr.wakeup_events;
  3777. if (wakeup_events) {
  3778. struct ring_buffer *rb = handle->rb;
  3779. int events = local_inc_return(&rb->events);
  3780. if (events >= wakeup_events) {
  3781. local_sub(wakeup_events, &rb->events);
  3782. local_inc(&rb->wakeup);
  3783. }
  3784. }
  3785. }
  3786. }
  3787. void perf_prepare_sample(struct perf_event_header *header,
  3788. struct perf_sample_data *data,
  3789. struct perf_event *event,
  3790. struct pt_regs *regs)
  3791. {
  3792. u64 sample_type = event->attr.sample_type;
  3793. header->type = PERF_RECORD_SAMPLE;
  3794. header->size = sizeof(*header) + event->header_size;
  3795. header->misc = 0;
  3796. header->misc |= perf_misc_flags(regs);
  3797. __perf_event_header__init_id(header, data, event);
  3798. if (sample_type & PERF_SAMPLE_IP)
  3799. data->ip = perf_instruction_pointer(regs);
  3800. if (sample_type & PERF_SAMPLE_CALLCHAIN) {
  3801. int size = 1;
  3802. data->callchain = perf_callchain(event, regs);
  3803. if (data->callchain)
  3804. size += data->callchain->nr;
  3805. header->size += size * sizeof(u64);
  3806. }
  3807. if (sample_type & PERF_SAMPLE_RAW) {
  3808. int size = sizeof(u32);
  3809. if (data->raw)
  3810. size += data->raw->size;
  3811. else
  3812. size += sizeof(u32);
  3813. WARN_ON_ONCE(size & (sizeof(u64)-1));
  3814. header->size += size;
  3815. }
  3816. if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
  3817. int size = sizeof(u64); /* nr */
  3818. if (data->br_stack) {
  3819. size += data->br_stack->nr
  3820. * sizeof(struct perf_branch_entry);
  3821. }
  3822. header->size += size;
  3823. }
  3824. if (sample_type & PERF_SAMPLE_REGS_USER) {
  3825. /* regs dump ABI info */
  3826. int size = sizeof(u64);
  3827. perf_sample_regs_user(&data->regs_user, regs);
  3828. if (data->regs_user.regs) {
  3829. u64 mask = event->attr.sample_regs_user;
  3830. size += hweight64(mask) * sizeof(u64);
  3831. }
  3832. header->size += size;
  3833. }
  3834. if (sample_type & PERF_SAMPLE_STACK_USER) {
  3835. /*
  3836. * Either we need PERF_SAMPLE_STACK_USER bit to be allways
  3837. * processed as the last one or have additional check added
  3838. * in case new sample type is added, because we could eat
  3839. * up the rest of the sample size.
  3840. */
  3841. struct perf_regs_user *uregs = &data->regs_user;
  3842. u16 stack_size = event->attr.sample_stack_user;
  3843. u16 size = sizeof(u64);
  3844. if (!uregs->abi)
  3845. perf_sample_regs_user(uregs, regs);
  3846. stack_size = perf_sample_ustack_size(stack_size, header->size,
  3847. uregs->regs);
  3848. /*
  3849. * If there is something to dump, add space for the dump
  3850. * itself and for the field that tells the dynamic size,
  3851. * which is how many have been actually dumped.
  3852. */
  3853. if (stack_size)
  3854. size += sizeof(u64) + stack_size;
  3855. data->stack_user_size = stack_size;
  3856. header->size += size;
  3857. }
  3858. }
  3859. static void perf_event_output(struct perf_event *event,
  3860. struct perf_sample_data *data,
  3861. struct pt_regs *regs)
  3862. {
  3863. struct perf_output_handle handle;
  3864. struct perf_event_header header;
  3865. /* protect the callchain buffers */
  3866. rcu_read_lock();
  3867. perf_prepare_sample(&header, data, event, regs);
  3868. if (perf_output_begin(&handle, event, header.size))
  3869. goto exit;
  3870. perf_output_sample(&handle, &header, data, event);
  3871. perf_output_end(&handle);
  3872. exit:
  3873. rcu_read_unlock();
  3874. }
  3875. /*
  3876. * read event_id
  3877. */
  3878. struct perf_read_event {
  3879. struct perf_event_header header;
  3880. u32 pid;
  3881. u32 tid;
  3882. };
  3883. static void
  3884. perf_event_read_event(struct perf_event *event,
  3885. struct task_struct *task)
  3886. {
  3887. struct perf_output_handle handle;
  3888. struct perf_sample_data sample;
  3889. struct perf_read_event read_event = {
  3890. .header = {
  3891. .type = PERF_RECORD_READ,
  3892. .misc = 0,
  3893. .size = sizeof(read_event) + event->read_size,
  3894. },
  3895. .pid = perf_event_pid(event, task),
  3896. .tid = perf_event_tid(event, task),
  3897. };
  3898. int ret;
  3899. perf_event_header__init_id(&read_event.header, &sample, event);
  3900. ret = perf_output_begin(&handle, event, read_event.header.size);
  3901. if (ret)
  3902. return;
  3903. perf_output_put(&handle, read_event);
  3904. perf_output_read(&handle, event);
  3905. perf_event__output_id_sample(event, &handle, &sample);
  3906. perf_output_end(&handle);
  3907. }
  3908. typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
  3909. static void
  3910. perf_event_aux_ctx(struct perf_event_context *ctx,
  3911. perf_event_aux_output_cb output,
  3912. void *data)
  3913. {
  3914. struct perf_event *event;
  3915. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  3916. if (event->state < PERF_EVENT_STATE_INACTIVE)
  3917. continue;
  3918. if (!event_filter_match(event))
  3919. continue;
  3920. output(event, data);
  3921. }
  3922. }
  3923. static void
  3924. perf_event_aux(perf_event_aux_output_cb output, void *data,
  3925. struct perf_event_context *task_ctx)
  3926. {
  3927. struct perf_cpu_context *cpuctx;
  3928. struct perf_event_context *ctx;
  3929. struct pmu *pmu;
  3930. int ctxn;
  3931. rcu_read_lock();
  3932. list_for_each_entry_rcu(pmu, &pmus, entry) {
  3933. cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
  3934. if (cpuctx->unique_pmu != pmu)
  3935. goto next;
  3936. perf_event_aux_ctx(&cpuctx->ctx, output, data);
  3937. if (task_ctx)
  3938. goto next;
  3939. ctxn = pmu->task_ctx_nr;
  3940. if (ctxn < 0)
  3941. goto next;
  3942. ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
  3943. if (ctx)
  3944. perf_event_aux_ctx(ctx, output, data);
  3945. next:
  3946. put_cpu_ptr(pmu->pmu_cpu_context);
  3947. }
  3948. if (task_ctx) {
  3949. preempt_disable();
  3950. perf_event_aux_ctx(task_ctx, output, data);
  3951. preempt_enable();
  3952. }
  3953. rcu_read_unlock();
  3954. }
  3955. /*
  3956. * task tracking -- fork/exit
  3957. *
  3958. * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
  3959. */
  3960. struct perf_task_event {
  3961. struct task_struct *task;
  3962. struct perf_event_context *task_ctx;
  3963. struct {
  3964. struct perf_event_header header;
  3965. u32 pid;
  3966. u32 ppid;
  3967. u32 tid;
  3968. u32 ptid;
  3969. u64 time;
  3970. } event_id;
  3971. };
  3972. static int perf_event_task_match(struct perf_event *event)
  3973. {
  3974. return event->attr.comm || event->attr.mmap ||
  3975. event->attr.mmap2 || event->attr.mmap_data ||
  3976. event->attr.task;
  3977. }
  3978. static void perf_event_task_output(struct perf_event *event,
  3979. void *data)
  3980. {
  3981. struct perf_task_event *task_event = data;
  3982. struct perf_output_handle handle;
  3983. struct perf_sample_data sample;
  3984. struct task_struct *task = task_event->task;
  3985. int ret, size = task_event->event_id.header.size;
  3986. if (!perf_event_task_match(event))
  3987. return;
  3988. perf_event_header__init_id(&task_event->event_id.header, &sample, event);
  3989. ret = perf_output_begin(&handle, event,
  3990. task_event->event_id.header.size);
  3991. if (ret)
  3992. goto out;
  3993. task_event->event_id.pid = perf_event_pid(event, task);
  3994. task_event->event_id.ppid = perf_event_pid(event, current);
  3995. task_event->event_id.tid = perf_event_tid(event, task);
  3996. task_event->event_id.ptid = perf_event_tid(event, current);
  3997. perf_output_put(&handle, task_event->event_id);
  3998. perf_event__output_id_sample(event, &handle, &sample);
  3999. perf_output_end(&handle);
  4000. out:
  4001. task_event->event_id.header.size = size;
  4002. }
  4003. static void perf_event_task(struct task_struct *task,
  4004. struct perf_event_context *task_ctx,
  4005. int new)
  4006. {
  4007. struct perf_task_event task_event;
  4008. if (!atomic_read(&nr_comm_events) &&
  4009. !atomic_read(&nr_mmap_events) &&
  4010. !atomic_read(&nr_task_events))
  4011. return;
  4012. task_event = (struct perf_task_event){
  4013. .task = task,
  4014. .task_ctx = task_ctx,
  4015. .event_id = {
  4016. .header = {
  4017. .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
  4018. .misc = 0,
  4019. .size = sizeof(task_event.event_id),
  4020. },
  4021. /* .pid */
  4022. /* .ppid */
  4023. /* .tid */
  4024. /* .ptid */
  4025. .time = perf_clock(),
  4026. },
  4027. };
  4028. perf_event_aux(perf_event_task_output,
  4029. &task_event,
  4030. task_ctx);
  4031. }
  4032. void perf_event_fork(struct task_struct *task)
  4033. {
  4034. perf_event_task(task, NULL, 1);
  4035. }
  4036. /*
  4037. * comm tracking
  4038. */
  4039. struct perf_comm_event {
  4040. struct task_struct *task;
  4041. char *comm;
  4042. int comm_size;
  4043. struct {
  4044. struct perf_event_header header;
  4045. u32 pid;
  4046. u32 tid;
  4047. } event_id;
  4048. };
  4049. static int perf_event_comm_match(struct perf_event *event)
  4050. {
  4051. return event->attr.comm;
  4052. }
  4053. static void perf_event_comm_output(struct perf_event *event,
  4054. void *data)
  4055. {
  4056. struct perf_comm_event *comm_event = data;
  4057. struct perf_output_handle handle;
  4058. struct perf_sample_data sample;
  4059. int size = comm_event->event_id.header.size;
  4060. int ret;
  4061. if (!perf_event_comm_match(event))
  4062. return;
  4063. perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
  4064. ret = perf_output_begin(&handle, event,
  4065. comm_event->event_id.header.size);
  4066. if (ret)
  4067. goto out;
  4068. comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
  4069. comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
  4070. perf_output_put(&handle, comm_event->event_id);
  4071. __output_copy(&handle, comm_event->comm,
  4072. comm_event->comm_size);
  4073. perf_event__output_id_sample(event, &handle, &sample);
  4074. perf_output_end(&handle);
  4075. out:
  4076. comm_event->event_id.header.size = size;
  4077. }
  4078. static void perf_event_comm_event(struct perf_comm_event *comm_event)
  4079. {
  4080. char comm[TASK_COMM_LEN];
  4081. unsigned int size;
  4082. memset(comm, 0, sizeof(comm));
  4083. strlcpy(comm, comm_event->task->comm, sizeof(comm));
  4084. size = ALIGN(strlen(comm)+1, sizeof(u64));
  4085. comm_event->comm = comm;
  4086. comm_event->comm_size = size;
  4087. comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
  4088. perf_event_aux(perf_event_comm_output,
  4089. comm_event,
  4090. NULL);
  4091. }
  4092. void perf_event_comm(struct task_struct *task)
  4093. {
  4094. struct perf_comm_event comm_event;
  4095. struct perf_event_context *ctx;
  4096. int ctxn;
  4097. rcu_read_lock();
  4098. for_each_task_context_nr(ctxn) {
  4099. ctx = task->perf_event_ctxp[ctxn];
  4100. if (!ctx)
  4101. continue;
  4102. perf_event_enable_on_exec(ctx);
  4103. }
  4104. rcu_read_unlock();
  4105. if (!atomic_read(&nr_comm_events))
  4106. return;
  4107. comm_event = (struct perf_comm_event){
  4108. .task = task,
  4109. /* .comm */
  4110. /* .comm_size */
  4111. .event_id = {
  4112. .header = {
  4113. .type = PERF_RECORD_COMM,
  4114. .misc = 0,
  4115. /* .size */
  4116. },
  4117. /* .pid */
  4118. /* .tid */
  4119. },
  4120. };
  4121. perf_event_comm_event(&comm_event);
  4122. }
  4123. /*
  4124. * mmap tracking
  4125. */
  4126. struct perf_mmap_event {
  4127. struct vm_area_struct *vma;
  4128. const char *file_name;
  4129. int file_size;
  4130. int maj, min;
  4131. u64 ino;
  4132. u64 ino_generation;
  4133. struct {
  4134. struct perf_event_header header;
  4135. u32 pid;
  4136. u32 tid;
  4137. u64 start;
  4138. u64 len;
  4139. u64 pgoff;
  4140. } event_id;
  4141. };
  4142. static int perf_event_mmap_match(struct perf_event *event,
  4143. void *data)
  4144. {
  4145. struct perf_mmap_event *mmap_event = data;
  4146. struct vm_area_struct *vma = mmap_event->vma;
  4147. int executable = vma->vm_flags & VM_EXEC;
  4148. return (!executable && event->attr.mmap_data) ||
  4149. (executable && (event->attr.mmap || event->attr.mmap2));
  4150. }
  4151. static void perf_event_mmap_output(struct perf_event *event,
  4152. void *data)
  4153. {
  4154. struct perf_mmap_event *mmap_event = data;
  4155. struct perf_output_handle handle;
  4156. struct perf_sample_data sample;
  4157. int size = mmap_event->event_id.header.size;
  4158. int ret;
  4159. if (!perf_event_mmap_match(event, data))
  4160. return;
  4161. if (event->attr.mmap2) {
  4162. mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
  4163. mmap_event->event_id.header.size += sizeof(mmap_event->maj);
  4164. mmap_event->event_id.header.size += sizeof(mmap_event->min);
  4165. mmap_event->event_id.header.size += sizeof(mmap_event->ino);
  4166. mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
  4167. }
  4168. perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
  4169. ret = perf_output_begin(&handle, event,
  4170. mmap_event->event_id.header.size);
  4171. if (ret)
  4172. goto out;
  4173. mmap_event->event_id.pid = perf_event_pid(event, current);
  4174. mmap_event->event_id.tid = perf_event_tid(event, current);
  4175. perf_output_put(&handle, mmap_event->event_id);
  4176. if (event->attr.mmap2) {
  4177. perf_output_put(&handle, mmap_event->maj);
  4178. perf_output_put(&handle, mmap_event->min);
  4179. perf_output_put(&handle, mmap_event->ino);
  4180. perf_output_put(&handle, mmap_event->ino_generation);
  4181. }
  4182. __output_copy(&handle, mmap_event->file_name,
  4183. mmap_event->file_size);
  4184. perf_event__output_id_sample(event, &handle, &sample);
  4185. perf_output_end(&handle);
  4186. out:
  4187. mmap_event->event_id.header.size = size;
  4188. }
  4189. static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
  4190. {
  4191. struct vm_area_struct *vma = mmap_event->vma;
  4192. struct file *file = vma->vm_file;
  4193. int maj = 0, min = 0;
  4194. u64 ino = 0, gen = 0;
  4195. unsigned int size;
  4196. char tmp[16];
  4197. char *buf = NULL;
  4198. const char *name;
  4199. memset(tmp, 0, sizeof(tmp));
  4200. if (file) {
  4201. struct inode *inode;
  4202. dev_t dev;
  4203. /*
  4204. * d_path works from the end of the rb backwards, so we
  4205. * need to add enough zero bytes after the string to handle
  4206. * the 64bit alignment we do later.
  4207. */
  4208. buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
  4209. if (!buf) {
  4210. name = strncpy(tmp, "//enomem", sizeof(tmp));
  4211. goto got_name;
  4212. }
  4213. name = d_path(&file->f_path, buf, PATH_MAX);
  4214. if (IS_ERR(name)) {
  4215. name = strncpy(tmp, "//toolong", sizeof(tmp));
  4216. goto got_name;
  4217. }
  4218. inode = file_inode(vma->vm_file);
  4219. dev = inode->i_sb->s_dev;
  4220. ino = inode->i_ino;
  4221. gen = inode->i_generation;
  4222. maj = MAJOR(dev);
  4223. min = MINOR(dev);
  4224. } else {
  4225. if (arch_vma_name(mmap_event->vma)) {
  4226. name = strncpy(tmp, arch_vma_name(mmap_event->vma),
  4227. sizeof(tmp) - 1);
  4228. tmp[sizeof(tmp) - 1] = '\0';
  4229. goto got_name;
  4230. }
  4231. if (!vma->vm_mm) {
  4232. name = strncpy(tmp, "[vdso]", sizeof(tmp));
  4233. goto got_name;
  4234. } else if (vma->vm_start <= vma->vm_mm->start_brk &&
  4235. vma->vm_end >= vma->vm_mm->brk) {
  4236. name = strncpy(tmp, "[heap]", sizeof(tmp));
  4237. goto got_name;
  4238. } else if (vma->vm_start <= vma->vm_mm->start_stack &&
  4239. vma->vm_end >= vma->vm_mm->start_stack) {
  4240. name = strncpy(tmp, "[stack]", sizeof(tmp));
  4241. goto got_name;
  4242. }
  4243. name = strncpy(tmp, "//anon", sizeof(tmp));
  4244. goto got_name;
  4245. }
  4246. got_name:
  4247. size = ALIGN(strlen(name)+1, sizeof(u64));
  4248. mmap_event->file_name = name;
  4249. mmap_event->file_size = size;
  4250. mmap_event->maj = maj;
  4251. mmap_event->min = min;
  4252. mmap_event->ino = ino;
  4253. mmap_event->ino_generation = gen;
  4254. if (!(vma->vm_flags & VM_EXEC))
  4255. mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
  4256. mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
  4257. perf_event_aux(perf_event_mmap_output,
  4258. mmap_event,
  4259. NULL);
  4260. kfree(buf);
  4261. }
  4262. void perf_event_mmap(struct vm_area_struct *vma)
  4263. {
  4264. struct perf_mmap_event mmap_event;
  4265. if (!atomic_read(&nr_mmap_events))
  4266. return;
  4267. mmap_event = (struct perf_mmap_event){
  4268. .vma = vma,
  4269. /* .file_name */
  4270. /* .file_size */
  4271. .event_id = {
  4272. .header = {
  4273. .type = PERF_RECORD_MMAP,
  4274. .misc = PERF_RECORD_MISC_USER,
  4275. /* .size */
  4276. },
  4277. /* .pid */
  4278. /* .tid */
  4279. .start = vma->vm_start,
  4280. .len = vma->vm_end - vma->vm_start,
  4281. .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
  4282. },
  4283. /* .maj (attr_mmap2 only) */
  4284. /* .min (attr_mmap2 only) */
  4285. /* .ino (attr_mmap2 only) */
  4286. /* .ino_generation (attr_mmap2 only) */
  4287. };
  4288. perf_event_mmap_event(&mmap_event);
  4289. }
  4290. /*
  4291. * IRQ throttle logging
  4292. */
  4293. static void perf_log_throttle(struct perf_event *event, int enable)
  4294. {
  4295. struct perf_output_handle handle;
  4296. struct perf_sample_data sample;
  4297. int ret;
  4298. struct {
  4299. struct perf_event_header header;
  4300. u64 time;
  4301. u64 id;
  4302. u64 stream_id;
  4303. } throttle_event = {
  4304. .header = {
  4305. .type = PERF_RECORD_THROTTLE,
  4306. .misc = 0,
  4307. .size = sizeof(throttle_event),
  4308. },
  4309. .time = perf_clock(),
  4310. .id = primary_event_id(event),
  4311. .stream_id = event->id,
  4312. };
  4313. if (enable)
  4314. throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
  4315. perf_event_header__init_id(&throttle_event.header, &sample, event);
  4316. ret = perf_output_begin(&handle, event,
  4317. throttle_event.header.size);
  4318. if (ret)
  4319. return;
  4320. perf_output_put(&handle, throttle_event);
  4321. perf_event__output_id_sample(event, &handle, &sample);
  4322. perf_output_end(&handle);
  4323. }
  4324. /*
  4325. * Generic event overflow handling, sampling.
  4326. */
  4327. static int __perf_event_overflow(struct perf_event *event,
  4328. int throttle, struct perf_sample_data *data,
  4329. struct pt_regs *regs)
  4330. {
  4331. int events = atomic_read(&event->event_limit);
  4332. struct hw_perf_event *hwc = &event->hw;
  4333. u64 seq;
  4334. int ret = 0;
  4335. /*
  4336. * Non-sampling counters might still use the PMI to fold short
  4337. * hardware counters, ignore those.
  4338. */
  4339. if (unlikely(!is_sampling_event(event)))
  4340. return 0;
  4341. seq = __this_cpu_read(perf_throttled_seq);
  4342. if (seq != hwc->interrupts_seq) {
  4343. hwc->interrupts_seq = seq;
  4344. hwc->interrupts = 1;
  4345. } else {
  4346. hwc->interrupts++;
  4347. if (unlikely(throttle
  4348. && hwc->interrupts >= max_samples_per_tick)) {
  4349. __this_cpu_inc(perf_throttled_count);
  4350. hwc->interrupts = MAX_INTERRUPTS;
  4351. perf_log_throttle(event, 0);
  4352. tick_nohz_full_kick();
  4353. ret = 1;
  4354. }
  4355. }
  4356. if (event->attr.freq) {
  4357. u64 now = perf_clock();
  4358. s64 delta = now - hwc->freq_time_stamp;
  4359. hwc->freq_time_stamp = now;
  4360. if (delta > 0 && delta < 2*TICK_NSEC)
  4361. perf_adjust_period(event, delta, hwc->last_period, true);
  4362. }
  4363. /*
  4364. * XXX event_limit might not quite work as expected on inherited
  4365. * events
  4366. */
  4367. event->pending_kill = POLL_IN;
  4368. if (events && atomic_dec_and_test(&event->event_limit)) {
  4369. ret = 1;
  4370. event->pending_kill = POLL_HUP;
  4371. event->pending_disable = 1;
  4372. irq_work_queue(&event->pending);
  4373. }
  4374. if (event->overflow_handler)
  4375. event->overflow_handler(event, data, regs);
  4376. else
  4377. perf_event_output(event, data, regs);
  4378. if (event->fasync && event->pending_kill) {
  4379. event->pending_wakeup = 1;
  4380. irq_work_queue(&event->pending);
  4381. }
  4382. return ret;
  4383. }
  4384. int perf_event_overflow(struct perf_event *event,
  4385. struct perf_sample_data *data,
  4386. struct pt_regs *regs)
  4387. {
  4388. return __perf_event_overflow(event, 1, data, regs);
  4389. }
  4390. /*
  4391. * Generic software event infrastructure
  4392. */
  4393. struct swevent_htable {
  4394. struct swevent_hlist *swevent_hlist;
  4395. struct mutex hlist_mutex;
  4396. int hlist_refcount;
  4397. /* Recursion avoidance in each contexts */
  4398. int recursion[PERF_NR_CONTEXTS];
  4399. };
  4400. static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
  4401. /*
  4402. * We directly increment event->count and keep a second value in
  4403. * event->hw.period_left to count intervals. This period event
  4404. * is kept in the range [-sample_period, 0] so that we can use the
  4405. * sign as trigger.
  4406. */
  4407. u64 perf_swevent_set_period(struct perf_event *event)
  4408. {
  4409. struct hw_perf_event *hwc = &event->hw;
  4410. u64 period = hwc->last_period;
  4411. u64 nr, offset;
  4412. s64 old, val;
  4413. hwc->last_period = hwc->sample_period;
  4414. again:
  4415. old = val = local64_read(&hwc->period_left);
  4416. if (val < 0)
  4417. return 0;
  4418. nr = div64_u64(period + val, period);
  4419. offset = nr * period;
  4420. val -= offset;
  4421. if (local64_cmpxchg(&hwc->period_left, old, val) != old)
  4422. goto again;
  4423. return nr;
  4424. }
  4425. static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
  4426. struct perf_sample_data *data,
  4427. struct pt_regs *regs)
  4428. {
  4429. struct hw_perf_event *hwc = &event->hw;
  4430. int throttle = 0;
  4431. if (!overflow)
  4432. overflow = perf_swevent_set_period(event);
  4433. if (hwc->interrupts == MAX_INTERRUPTS)
  4434. return;
  4435. for (; overflow; overflow--) {
  4436. if (__perf_event_overflow(event, throttle,
  4437. data, regs)) {
  4438. /*
  4439. * We inhibit the overflow from happening when
  4440. * hwc->interrupts == MAX_INTERRUPTS.
  4441. */
  4442. break;
  4443. }
  4444. throttle = 1;
  4445. }
  4446. }
  4447. static void perf_swevent_event(struct perf_event *event, u64 nr,
  4448. struct perf_sample_data *data,
  4449. struct pt_regs *regs)
  4450. {
  4451. struct hw_perf_event *hwc = &event->hw;
  4452. local64_add(nr, &event->count);
  4453. if (!regs)
  4454. return;
  4455. if (!is_sampling_event(event))
  4456. return;
  4457. if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
  4458. data->period = nr;
  4459. return perf_swevent_overflow(event, 1, data, regs);
  4460. } else
  4461. data->period = event->hw.last_period;
  4462. if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
  4463. return perf_swevent_overflow(event, 1, data, regs);
  4464. if (local64_add_negative(nr, &hwc->period_left))
  4465. return;
  4466. perf_swevent_overflow(event, 0, data, regs);
  4467. }
  4468. static int perf_exclude_event(struct perf_event *event,
  4469. struct pt_regs *regs)
  4470. {
  4471. if (event->hw.state & PERF_HES_STOPPED)
  4472. return 1;
  4473. if (regs) {
  4474. if (event->attr.exclude_user && user_mode(regs))
  4475. return 1;
  4476. if (event->attr.exclude_kernel && !user_mode(regs))
  4477. return 1;
  4478. }
  4479. return 0;
  4480. }
  4481. static int perf_swevent_match(struct perf_event *event,
  4482. enum perf_type_id type,
  4483. u32 event_id,
  4484. struct perf_sample_data *data,
  4485. struct pt_regs *regs)
  4486. {
  4487. if (event->attr.type != type)
  4488. return 0;
  4489. if (event->attr.config != event_id)
  4490. return 0;
  4491. if (perf_exclude_event(event, regs))
  4492. return 0;
  4493. return 1;
  4494. }
  4495. static inline u64 swevent_hash(u64 type, u32 event_id)
  4496. {
  4497. u64 val = event_id | (type << 32);
  4498. return hash_64(val, SWEVENT_HLIST_BITS);
  4499. }
  4500. static inline struct hlist_head *
  4501. __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
  4502. {
  4503. u64 hash = swevent_hash(type, event_id);
  4504. return &hlist->heads[hash];
  4505. }
  4506. /* For the read side: events when they trigger */
  4507. static inline struct hlist_head *
  4508. find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
  4509. {
  4510. struct swevent_hlist *hlist;
  4511. hlist = rcu_dereference(swhash->swevent_hlist);
  4512. if (!hlist)
  4513. return NULL;
  4514. return __find_swevent_head(hlist, type, event_id);
  4515. }
  4516. /* For the event head insertion and removal in the hlist */
  4517. static inline struct hlist_head *
  4518. find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
  4519. {
  4520. struct swevent_hlist *hlist;
  4521. u32 event_id = event->attr.config;
  4522. u64 type = event->attr.type;
  4523. /*
  4524. * Event scheduling is always serialized against hlist allocation
  4525. * and release. Which makes the protected version suitable here.
  4526. * The context lock guarantees that.
  4527. */
  4528. hlist = rcu_dereference_protected(swhash->swevent_hlist,
  4529. lockdep_is_held(&event->ctx->lock));
  4530. if (!hlist)
  4531. return NULL;
  4532. return __find_swevent_head(hlist, type, event_id);
  4533. }
  4534. static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
  4535. u64 nr,
  4536. struct perf_sample_data *data,
  4537. struct pt_regs *regs)
  4538. {
  4539. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4540. struct perf_event *event;
  4541. struct hlist_head *head;
  4542. rcu_read_lock();
  4543. head = find_swevent_head_rcu(swhash, type, event_id);
  4544. if (!head)
  4545. goto end;
  4546. hlist_for_each_entry_rcu(event, head, hlist_entry) {
  4547. if (perf_swevent_match(event, type, event_id, data, regs))
  4548. perf_swevent_event(event, nr, data, regs);
  4549. }
  4550. end:
  4551. rcu_read_unlock();
  4552. }
  4553. int perf_swevent_get_recursion_context(void)
  4554. {
  4555. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4556. return get_recursion_context(swhash->recursion);
  4557. }
  4558. EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
  4559. inline void perf_swevent_put_recursion_context(int rctx)
  4560. {
  4561. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4562. put_recursion_context(swhash->recursion, rctx);
  4563. }
  4564. void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
  4565. {
  4566. struct perf_sample_data data;
  4567. int rctx;
  4568. preempt_disable_notrace();
  4569. rctx = perf_swevent_get_recursion_context();
  4570. if (rctx < 0)
  4571. return;
  4572. perf_sample_data_init(&data, addr, 0);
  4573. do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
  4574. perf_swevent_put_recursion_context(rctx);
  4575. preempt_enable_notrace();
  4576. }
  4577. static void perf_swevent_read(struct perf_event *event)
  4578. {
  4579. }
  4580. static int perf_swevent_add(struct perf_event *event, int flags)
  4581. {
  4582. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4583. struct hw_perf_event *hwc = &event->hw;
  4584. struct hlist_head *head;
  4585. if (is_sampling_event(event)) {
  4586. hwc->last_period = hwc->sample_period;
  4587. perf_swevent_set_period(event);
  4588. }
  4589. hwc->state = !(flags & PERF_EF_START);
  4590. head = find_swevent_head(swhash, event);
  4591. if (WARN_ON_ONCE(!head))
  4592. return -EINVAL;
  4593. hlist_add_head_rcu(&event->hlist_entry, head);
  4594. return 0;
  4595. }
  4596. static void perf_swevent_del(struct perf_event *event, int flags)
  4597. {
  4598. hlist_del_rcu(&event->hlist_entry);
  4599. }
  4600. static void perf_swevent_start(struct perf_event *event, int flags)
  4601. {
  4602. event->hw.state = 0;
  4603. }
  4604. static void perf_swevent_stop(struct perf_event *event, int flags)
  4605. {
  4606. event->hw.state = PERF_HES_STOPPED;
  4607. }
  4608. /* Deref the hlist from the update side */
  4609. static inline struct swevent_hlist *
  4610. swevent_hlist_deref(struct swevent_htable *swhash)
  4611. {
  4612. return rcu_dereference_protected(swhash->swevent_hlist,
  4613. lockdep_is_held(&swhash->hlist_mutex));
  4614. }
  4615. static void swevent_hlist_release(struct swevent_htable *swhash)
  4616. {
  4617. struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
  4618. if (!hlist)
  4619. return;
  4620. rcu_assign_pointer(swhash->swevent_hlist, NULL);
  4621. kfree_rcu(hlist, rcu_head);
  4622. }
  4623. static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
  4624. {
  4625. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  4626. mutex_lock(&swhash->hlist_mutex);
  4627. if (!--swhash->hlist_refcount)
  4628. swevent_hlist_release(swhash);
  4629. mutex_unlock(&swhash->hlist_mutex);
  4630. }
  4631. static void swevent_hlist_put(struct perf_event *event)
  4632. {
  4633. int cpu;
  4634. if (event->cpu != -1) {
  4635. swevent_hlist_put_cpu(event, event->cpu);
  4636. return;
  4637. }
  4638. for_each_possible_cpu(cpu)
  4639. swevent_hlist_put_cpu(event, cpu);
  4640. }
  4641. static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
  4642. {
  4643. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  4644. int err = 0;
  4645. mutex_lock(&swhash->hlist_mutex);
  4646. if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
  4647. struct swevent_hlist *hlist;
  4648. hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
  4649. if (!hlist) {
  4650. err = -ENOMEM;
  4651. goto exit;
  4652. }
  4653. rcu_assign_pointer(swhash->swevent_hlist, hlist);
  4654. }
  4655. swhash->hlist_refcount++;
  4656. exit:
  4657. mutex_unlock(&swhash->hlist_mutex);
  4658. return err;
  4659. }
  4660. static int swevent_hlist_get(struct perf_event *event)
  4661. {
  4662. int err;
  4663. int cpu, failed_cpu;
  4664. if (event->cpu != -1)
  4665. return swevent_hlist_get_cpu(event, event->cpu);
  4666. get_online_cpus();
  4667. for_each_possible_cpu(cpu) {
  4668. err = swevent_hlist_get_cpu(event, cpu);
  4669. if (err) {
  4670. failed_cpu = cpu;
  4671. goto fail;
  4672. }
  4673. }
  4674. put_online_cpus();
  4675. return 0;
  4676. fail:
  4677. for_each_possible_cpu(cpu) {
  4678. if (cpu == failed_cpu)
  4679. break;
  4680. swevent_hlist_put_cpu(event, cpu);
  4681. }
  4682. put_online_cpus();
  4683. return err;
  4684. }
  4685. struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
  4686. static void sw_perf_event_destroy(struct perf_event *event)
  4687. {
  4688. u64 event_id = event->attr.config;
  4689. WARN_ON(event->parent);
  4690. static_key_slow_dec(&perf_swevent_enabled[event_id]);
  4691. swevent_hlist_put(event);
  4692. }
  4693. static int perf_swevent_init(struct perf_event *event)
  4694. {
  4695. u64 event_id = event->attr.config;
  4696. if (event->attr.type != PERF_TYPE_SOFTWARE)
  4697. return -ENOENT;
  4698. /*
  4699. * no branch sampling for software events
  4700. */
  4701. if (has_branch_stack(event))
  4702. return -EOPNOTSUPP;
  4703. switch (event_id) {
  4704. case PERF_COUNT_SW_CPU_CLOCK:
  4705. case PERF_COUNT_SW_TASK_CLOCK:
  4706. return -ENOENT;
  4707. default:
  4708. break;
  4709. }
  4710. if (event_id >= PERF_COUNT_SW_MAX)
  4711. return -ENOENT;
  4712. if (!event->parent) {
  4713. int err;
  4714. err = swevent_hlist_get(event);
  4715. if (err)
  4716. return err;
  4717. static_key_slow_inc(&perf_swevent_enabled[event_id]);
  4718. event->destroy = sw_perf_event_destroy;
  4719. }
  4720. return 0;
  4721. }
  4722. static int perf_swevent_event_idx(struct perf_event *event)
  4723. {
  4724. return 0;
  4725. }
  4726. static struct pmu perf_swevent = {
  4727. .task_ctx_nr = perf_sw_context,
  4728. .event_init = perf_swevent_init,
  4729. .add = perf_swevent_add,
  4730. .del = perf_swevent_del,
  4731. .start = perf_swevent_start,
  4732. .stop = perf_swevent_stop,
  4733. .read = perf_swevent_read,
  4734. .event_idx = perf_swevent_event_idx,
  4735. };
  4736. #ifdef CONFIG_EVENT_TRACING
  4737. static int perf_tp_filter_match(struct perf_event *event,
  4738. struct perf_sample_data *data)
  4739. {
  4740. void *record = data->raw->data;
  4741. if (likely(!event->filter) || filter_match_preds(event->filter, record))
  4742. return 1;
  4743. return 0;
  4744. }
  4745. static int perf_tp_event_match(struct perf_event *event,
  4746. struct perf_sample_data *data,
  4747. struct pt_regs *regs)
  4748. {
  4749. if (event->hw.state & PERF_HES_STOPPED)
  4750. return 0;
  4751. /*
  4752. * All tracepoints are from kernel-space.
  4753. */
  4754. if (event->attr.exclude_kernel)
  4755. return 0;
  4756. if (!perf_tp_filter_match(event, data))
  4757. return 0;
  4758. return 1;
  4759. }
  4760. void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
  4761. struct pt_regs *regs, struct hlist_head *head, int rctx,
  4762. struct task_struct *task)
  4763. {
  4764. struct perf_sample_data data;
  4765. struct perf_event *event;
  4766. struct perf_raw_record raw = {
  4767. .size = entry_size,
  4768. .data = record,
  4769. };
  4770. perf_sample_data_init(&data, addr, 0);
  4771. data.raw = &raw;
  4772. hlist_for_each_entry_rcu(event, head, hlist_entry) {
  4773. if (perf_tp_event_match(event, &data, regs))
  4774. perf_swevent_event(event, count, &data, regs);
  4775. }
  4776. /*
  4777. * If we got specified a target task, also iterate its context and
  4778. * deliver this event there too.
  4779. */
  4780. if (task && task != current) {
  4781. struct perf_event_context *ctx;
  4782. struct trace_entry *entry = record;
  4783. rcu_read_lock();
  4784. ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
  4785. if (!ctx)
  4786. goto unlock;
  4787. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  4788. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  4789. continue;
  4790. if (event->attr.config != entry->type)
  4791. continue;
  4792. if (perf_tp_event_match(event, &data, regs))
  4793. perf_swevent_event(event, count, &data, regs);
  4794. }
  4795. unlock:
  4796. rcu_read_unlock();
  4797. }
  4798. perf_swevent_put_recursion_context(rctx);
  4799. }
  4800. EXPORT_SYMBOL_GPL(perf_tp_event);
  4801. static void tp_perf_event_destroy(struct perf_event *event)
  4802. {
  4803. perf_trace_destroy(event);
  4804. }
  4805. static int perf_tp_event_init(struct perf_event *event)
  4806. {
  4807. int err;
  4808. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  4809. return -ENOENT;
  4810. /*
  4811. * no branch sampling for tracepoint events
  4812. */
  4813. if (has_branch_stack(event))
  4814. return -EOPNOTSUPP;
  4815. err = perf_trace_init(event);
  4816. if (err)
  4817. return err;
  4818. event->destroy = tp_perf_event_destroy;
  4819. return 0;
  4820. }
  4821. static struct pmu perf_tracepoint = {
  4822. .task_ctx_nr = perf_sw_context,
  4823. .event_init = perf_tp_event_init,
  4824. .add = perf_trace_add,
  4825. .del = perf_trace_del,
  4826. .start = perf_swevent_start,
  4827. .stop = perf_swevent_stop,
  4828. .read = perf_swevent_read,
  4829. .event_idx = perf_swevent_event_idx,
  4830. };
  4831. static inline void perf_tp_register(void)
  4832. {
  4833. perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
  4834. }
  4835. static int perf_event_set_filter(struct perf_event *event, void __user *arg)
  4836. {
  4837. char *filter_str;
  4838. int ret;
  4839. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  4840. return -EINVAL;
  4841. filter_str = strndup_user(arg, PAGE_SIZE);
  4842. if (IS_ERR(filter_str))
  4843. return PTR_ERR(filter_str);
  4844. ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
  4845. kfree(filter_str);
  4846. return ret;
  4847. }
  4848. static void perf_event_free_filter(struct perf_event *event)
  4849. {
  4850. ftrace_profile_free_filter(event);
  4851. }
  4852. #else
  4853. static inline void perf_tp_register(void)
  4854. {
  4855. }
  4856. static int perf_event_set_filter(struct perf_event *event, void __user *arg)
  4857. {
  4858. return -ENOENT;
  4859. }
  4860. static void perf_event_free_filter(struct perf_event *event)
  4861. {
  4862. }
  4863. #endif /* CONFIG_EVENT_TRACING */
  4864. #ifdef CONFIG_HAVE_HW_BREAKPOINT
  4865. void perf_bp_event(struct perf_event *bp, void *data)
  4866. {
  4867. struct perf_sample_data sample;
  4868. struct pt_regs *regs = data;
  4869. perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
  4870. if (!bp->hw.state && !perf_exclude_event(bp, regs))
  4871. perf_swevent_event(bp, 1, &sample, regs);
  4872. }
  4873. #endif
  4874. /*
  4875. * hrtimer based swevent callback
  4876. */
  4877. static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
  4878. {
  4879. enum hrtimer_restart ret = HRTIMER_RESTART;
  4880. struct perf_sample_data data;
  4881. struct pt_regs *regs;
  4882. struct perf_event *event;
  4883. u64 period;
  4884. event = container_of(hrtimer, struct perf_event, hw.hrtimer);
  4885. if (event->state != PERF_EVENT_STATE_ACTIVE)
  4886. return HRTIMER_NORESTART;
  4887. event->pmu->read(event);
  4888. perf_sample_data_init(&data, 0, event->hw.last_period);
  4889. regs = get_irq_regs();
  4890. if (regs && !perf_exclude_event(event, regs)) {
  4891. if (!(event->attr.exclude_idle && is_idle_task(current)))
  4892. if (__perf_event_overflow(event, 1, &data, regs))
  4893. ret = HRTIMER_NORESTART;
  4894. }
  4895. period = max_t(u64, 10000, event->hw.sample_period);
  4896. hrtimer_forward_now(hrtimer, ns_to_ktime(period));
  4897. return ret;
  4898. }
  4899. static void perf_swevent_start_hrtimer(struct perf_event *event)
  4900. {
  4901. struct hw_perf_event *hwc = &event->hw;
  4902. s64 period;
  4903. if (!is_sampling_event(event))
  4904. return;
  4905. period = local64_read(&hwc->period_left);
  4906. if (period) {
  4907. if (period < 0)
  4908. period = 10000;
  4909. local64_set(&hwc->period_left, 0);
  4910. } else {
  4911. period = max_t(u64, 10000, hwc->sample_period);
  4912. }
  4913. __hrtimer_start_range_ns(&hwc->hrtimer,
  4914. ns_to_ktime(period), 0,
  4915. HRTIMER_MODE_REL_PINNED, 0);
  4916. }
  4917. static void perf_swevent_cancel_hrtimer(struct perf_event *event)
  4918. {
  4919. struct hw_perf_event *hwc = &event->hw;
  4920. if (is_sampling_event(event)) {
  4921. ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
  4922. local64_set(&hwc->period_left, ktime_to_ns(remaining));
  4923. hrtimer_cancel(&hwc->hrtimer);
  4924. }
  4925. }
  4926. static void perf_swevent_init_hrtimer(struct perf_event *event)
  4927. {
  4928. struct hw_perf_event *hwc = &event->hw;
  4929. if (!is_sampling_event(event))
  4930. return;
  4931. hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  4932. hwc->hrtimer.function = perf_swevent_hrtimer;
  4933. /*
  4934. * Since hrtimers have a fixed rate, we can do a static freq->period
  4935. * mapping and avoid the whole period adjust feedback stuff.
  4936. */
  4937. if (event->attr.freq) {
  4938. long freq = event->attr.sample_freq;
  4939. event->attr.sample_period = NSEC_PER_SEC / freq;
  4940. hwc->sample_period = event->attr.sample_period;
  4941. local64_set(&hwc->period_left, hwc->sample_period);
  4942. hwc->last_period = hwc->sample_period;
  4943. event->attr.freq = 0;
  4944. }
  4945. }
  4946. /*
  4947. * Software event: cpu wall time clock
  4948. */
  4949. static void cpu_clock_event_update(struct perf_event *event)
  4950. {
  4951. s64 prev;
  4952. u64 now;
  4953. now = local_clock();
  4954. prev = local64_xchg(&event->hw.prev_count, now);
  4955. local64_add(now - prev, &event->count);
  4956. }
  4957. static void cpu_clock_event_start(struct perf_event *event, int flags)
  4958. {
  4959. local64_set(&event->hw.prev_count, local_clock());
  4960. perf_swevent_start_hrtimer(event);
  4961. }
  4962. static void cpu_clock_event_stop(struct perf_event *event, int flags)
  4963. {
  4964. perf_swevent_cancel_hrtimer(event);
  4965. cpu_clock_event_update(event);
  4966. }
  4967. static int cpu_clock_event_add(struct perf_event *event, int flags)
  4968. {
  4969. if (flags & PERF_EF_START)
  4970. cpu_clock_event_start(event, flags);
  4971. return 0;
  4972. }
  4973. static void cpu_clock_event_del(struct perf_event *event, int flags)
  4974. {
  4975. cpu_clock_event_stop(event, flags);
  4976. }
  4977. static void cpu_clock_event_read(struct perf_event *event)
  4978. {
  4979. cpu_clock_event_update(event);
  4980. }
  4981. static int cpu_clock_event_init(struct perf_event *event)
  4982. {
  4983. if (event->attr.type != PERF_TYPE_SOFTWARE)
  4984. return -ENOENT;
  4985. if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
  4986. return -ENOENT;
  4987. /*
  4988. * no branch sampling for software events
  4989. */
  4990. if (has_branch_stack(event))
  4991. return -EOPNOTSUPP;
  4992. perf_swevent_init_hrtimer(event);
  4993. return 0;
  4994. }
  4995. static struct pmu perf_cpu_clock = {
  4996. .task_ctx_nr = perf_sw_context,
  4997. .event_init = cpu_clock_event_init,
  4998. .add = cpu_clock_event_add,
  4999. .del = cpu_clock_event_del,
  5000. .start = cpu_clock_event_start,
  5001. .stop = cpu_clock_event_stop,
  5002. .read = cpu_clock_event_read,
  5003. .event_idx = perf_swevent_event_idx,
  5004. };
  5005. /*
  5006. * Software event: task time clock
  5007. */
  5008. static void task_clock_event_update(struct perf_event *event, u64 now)
  5009. {
  5010. u64 prev;
  5011. s64 delta;
  5012. prev = local64_xchg(&event->hw.prev_count, now);
  5013. delta = now - prev;
  5014. local64_add(delta, &event->count);
  5015. }
  5016. static void task_clock_event_start(struct perf_event *event, int flags)
  5017. {
  5018. local64_set(&event->hw.prev_count, event->ctx->time);
  5019. perf_swevent_start_hrtimer(event);
  5020. }
  5021. static void task_clock_event_stop(struct perf_event *event, int flags)
  5022. {
  5023. perf_swevent_cancel_hrtimer(event);
  5024. task_clock_event_update(event, event->ctx->time);
  5025. }
  5026. static int task_clock_event_add(struct perf_event *event, int flags)
  5027. {
  5028. if (flags & PERF_EF_START)
  5029. task_clock_event_start(event, flags);
  5030. return 0;
  5031. }
  5032. static void task_clock_event_del(struct perf_event *event, int flags)
  5033. {
  5034. task_clock_event_stop(event, PERF_EF_UPDATE);
  5035. }
  5036. static void task_clock_event_read(struct perf_event *event)
  5037. {
  5038. u64 now = perf_clock();
  5039. u64 delta = now - event->ctx->timestamp;
  5040. u64 time = event->ctx->time + delta;
  5041. task_clock_event_update(event, time);
  5042. }
  5043. static int task_clock_event_init(struct perf_event *event)
  5044. {
  5045. if (event->attr.type != PERF_TYPE_SOFTWARE)
  5046. return -ENOENT;
  5047. if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
  5048. return -ENOENT;
  5049. /*
  5050. * no branch sampling for software events
  5051. */
  5052. if (has_branch_stack(event))
  5053. return -EOPNOTSUPP;
  5054. perf_swevent_init_hrtimer(event);
  5055. return 0;
  5056. }
  5057. static struct pmu perf_task_clock = {
  5058. .task_ctx_nr = perf_sw_context,
  5059. .event_init = task_clock_event_init,
  5060. .add = task_clock_event_add,
  5061. .del = task_clock_event_del,
  5062. .start = task_clock_event_start,
  5063. .stop = task_clock_event_stop,
  5064. .read = task_clock_event_read,
  5065. .event_idx = perf_swevent_event_idx,
  5066. };
  5067. static void perf_pmu_nop_void(struct pmu *pmu)
  5068. {
  5069. }
  5070. static int perf_pmu_nop_int(struct pmu *pmu)
  5071. {
  5072. return 0;
  5073. }
  5074. static void perf_pmu_start_txn(struct pmu *pmu)
  5075. {
  5076. perf_pmu_disable(pmu);
  5077. }
  5078. static int perf_pmu_commit_txn(struct pmu *pmu)
  5079. {
  5080. perf_pmu_enable(pmu);
  5081. return 0;
  5082. }
  5083. static void perf_pmu_cancel_txn(struct pmu *pmu)
  5084. {
  5085. perf_pmu_enable(pmu);
  5086. }
  5087. static int perf_event_idx_default(struct perf_event *event)
  5088. {
  5089. return event->hw.idx + 1;
  5090. }
  5091. /*
  5092. * Ensures all contexts with the same task_ctx_nr have the same
  5093. * pmu_cpu_context too.
  5094. */
  5095. static void *find_pmu_context(int ctxn)
  5096. {
  5097. struct pmu *pmu;
  5098. if (ctxn < 0)
  5099. return NULL;
  5100. list_for_each_entry(pmu, &pmus, entry) {
  5101. if (pmu->task_ctx_nr == ctxn)
  5102. return pmu->pmu_cpu_context;
  5103. }
  5104. return NULL;
  5105. }
  5106. static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
  5107. {
  5108. int cpu;
  5109. for_each_possible_cpu(cpu) {
  5110. struct perf_cpu_context *cpuctx;
  5111. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  5112. if (cpuctx->unique_pmu == old_pmu)
  5113. cpuctx->unique_pmu = pmu;
  5114. }
  5115. }
  5116. static void free_pmu_context(struct pmu *pmu)
  5117. {
  5118. struct pmu *i;
  5119. mutex_lock(&pmus_lock);
  5120. /*
  5121. * Like a real lame refcount.
  5122. */
  5123. list_for_each_entry(i, &pmus, entry) {
  5124. if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
  5125. update_pmu_context(i, pmu);
  5126. goto out;
  5127. }
  5128. }
  5129. free_percpu(pmu->pmu_cpu_context);
  5130. out:
  5131. mutex_unlock(&pmus_lock);
  5132. }
  5133. static struct idr pmu_idr;
  5134. static ssize_t
  5135. type_show(struct device *dev, struct device_attribute *attr, char *page)
  5136. {
  5137. struct pmu *pmu = dev_get_drvdata(dev);
  5138. return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
  5139. }
  5140. static ssize_t
  5141. perf_event_mux_interval_ms_show(struct device *dev,
  5142. struct device_attribute *attr,
  5143. char *page)
  5144. {
  5145. struct pmu *pmu = dev_get_drvdata(dev);
  5146. return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms);
  5147. }
  5148. static ssize_t
  5149. perf_event_mux_interval_ms_store(struct device *dev,
  5150. struct device_attribute *attr,
  5151. const char *buf, size_t count)
  5152. {
  5153. struct pmu *pmu = dev_get_drvdata(dev);
  5154. int timer, cpu, ret;
  5155. ret = kstrtoint(buf, 0, &timer);
  5156. if (ret)
  5157. return ret;
  5158. if (timer < 1)
  5159. return -EINVAL;
  5160. /* same value, noting to do */
  5161. if (timer == pmu->hrtimer_interval_ms)
  5162. return count;
  5163. pmu->hrtimer_interval_ms = timer;
  5164. /* update all cpuctx for this PMU */
  5165. for_each_possible_cpu(cpu) {
  5166. struct perf_cpu_context *cpuctx;
  5167. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  5168. cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
  5169. if (hrtimer_active(&cpuctx->hrtimer))
  5170. hrtimer_forward_now(&cpuctx->hrtimer, cpuctx->hrtimer_interval);
  5171. }
  5172. return count;
  5173. }
  5174. static struct device_attribute pmu_dev_attrs[] = {
  5175. __ATTR_RO(type),
  5176. __ATTR_RW(perf_event_mux_interval_ms),
  5177. __ATTR_NULL,
  5178. };
  5179. static int pmu_bus_running;
  5180. static struct bus_type pmu_bus = {
  5181. .name = "event_source",
  5182. .dev_attrs = pmu_dev_attrs,
  5183. };
  5184. static void pmu_dev_release(struct device *dev)
  5185. {
  5186. kfree(dev);
  5187. }
  5188. static int pmu_dev_alloc(struct pmu *pmu)
  5189. {
  5190. int ret = -ENOMEM;
  5191. pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
  5192. if (!pmu->dev)
  5193. goto out;
  5194. pmu->dev->groups = pmu->attr_groups;
  5195. device_initialize(pmu->dev);
  5196. ret = dev_set_name(pmu->dev, "%s", pmu->name);
  5197. if (ret)
  5198. goto free_dev;
  5199. dev_set_drvdata(pmu->dev, pmu);
  5200. pmu->dev->bus = &pmu_bus;
  5201. pmu->dev->release = pmu_dev_release;
  5202. ret = device_add(pmu->dev);
  5203. if (ret)
  5204. goto free_dev;
  5205. out:
  5206. return ret;
  5207. free_dev:
  5208. put_device(pmu->dev);
  5209. goto out;
  5210. }
  5211. static struct lock_class_key cpuctx_mutex;
  5212. static struct lock_class_key cpuctx_lock;
  5213. int perf_pmu_register(struct pmu *pmu, const char *name, int type)
  5214. {
  5215. int cpu, ret;
  5216. mutex_lock(&pmus_lock);
  5217. ret = -ENOMEM;
  5218. pmu->pmu_disable_count = alloc_percpu(int);
  5219. if (!pmu->pmu_disable_count)
  5220. goto unlock;
  5221. pmu->type = -1;
  5222. if (!name)
  5223. goto skip_type;
  5224. pmu->name = name;
  5225. if (type < 0) {
  5226. type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL);
  5227. if (type < 0) {
  5228. ret = type;
  5229. goto free_pdc;
  5230. }
  5231. }
  5232. pmu->type = type;
  5233. if (pmu_bus_running) {
  5234. ret = pmu_dev_alloc(pmu);
  5235. if (ret)
  5236. goto free_idr;
  5237. }
  5238. skip_type:
  5239. pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
  5240. if (pmu->pmu_cpu_context)
  5241. goto got_cpu_context;
  5242. ret = -ENOMEM;
  5243. pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
  5244. if (!pmu->pmu_cpu_context)
  5245. goto free_dev;
  5246. for_each_possible_cpu(cpu) {
  5247. struct perf_cpu_context *cpuctx;
  5248. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  5249. __perf_event_init_context(&cpuctx->ctx);
  5250. lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
  5251. lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
  5252. cpuctx->ctx.type = cpu_context;
  5253. cpuctx->ctx.pmu = pmu;
  5254. __perf_cpu_hrtimer_init(cpuctx, cpu);
  5255. INIT_LIST_HEAD(&cpuctx->rotation_list);
  5256. cpuctx->unique_pmu = pmu;
  5257. }
  5258. got_cpu_context:
  5259. if (!pmu->start_txn) {
  5260. if (pmu->pmu_enable) {
  5261. /*
  5262. * If we have pmu_enable/pmu_disable calls, install
  5263. * transaction stubs that use that to try and batch
  5264. * hardware accesses.
  5265. */
  5266. pmu->start_txn = perf_pmu_start_txn;
  5267. pmu->commit_txn = perf_pmu_commit_txn;
  5268. pmu->cancel_txn = perf_pmu_cancel_txn;
  5269. } else {
  5270. pmu->start_txn = perf_pmu_nop_void;
  5271. pmu->commit_txn = perf_pmu_nop_int;
  5272. pmu->cancel_txn = perf_pmu_nop_void;
  5273. }
  5274. }
  5275. if (!pmu->pmu_enable) {
  5276. pmu->pmu_enable = perf_pmu_nop_void;
  5277. pmu->pmu_disable = perf_pmu_nop_void;
  5278. }
  5279. if (!pmu->event_idx)
  5280. pmu->event_idx = perf_event_idx_default;
  5281. list_add_rcu(&pmu->entry, &pmus);
  5282. ret = 0;
  5283. unlock:
  5284. mutex_unlock(&pmus_lock);
  5285. return ret;
  5286. free_dev:
  5287. device_del(pmu->dev);
  5288. put_device(pmu->dev);
  5289. free_idr:
  5290. if (pmu->type >= PERF_TYPE_MAX)
  5291. idr_remove(&pmu_idr, pmu->type);
  5292. free_pdc:
  5293. free_percpu(pmu->pmu_disable_count);
  5294. goto unlock;
  5295. }
  5296. void perf_pmu_unregister(struct pmu *pmu)
  5297. {
  5298. mutex_lock(&pmus_lock);
  5299. list_del_rcu(&pmu->entry);
  5300. mutex_unlock(&pmus_lock);
  5301. /*
  5302. * We dereference the pmu list under both SRCU and regular RCU, so
  5303. * synchronize against both of those.
  5304. */
  5305. synchronize_srcu(&pmus_srcu);
  5306. synchronize_rcu();
  5307. free_percpu(pmu->pmu_disable_count);
  5308. if (pmu->type >= PERF_TYPE_MAX)
  5309. idr_remove(&pmu_idr, pmu->type);
  5310. device_del(pmu->dev);
  5311. put_device(pmu->dev);
  5312. free_pmu_context(pmu);
  5313. }
  5314. struct pmu *perf_init_event(struct perf_event *event)
  5315. {
  5316. struct pmu *pmu = NULL;
  5317. int idx;
  5318. int ret;
  5319. idx = srcu_read_lock(&pmus_srcu);
  5320. rcu_read_lock();
  5321. pmu = idr_find(&pmu_idr, event->attr.type);
  5322. rcu_read_unlock();
  5323. if (pmu) {
  5324. event->pmu = pmu;
  5325. ret = pmu->event_init(event);
  5326. if (ret)
  5327. pmu = ERR_PTR(ret);
  5328. goto unlock;
  5329. }
  5330. list_for_each_entry_rcu(pmu, &pmus, entry) {
  5331. event->pmu = pmu;
  5332. ret = pmu->event_init(event);
  5333. if (!ret)
  5334. goto unlock;
  5335. if (ret != -ENOENT) {
  5336. pmu = ERR_PTR(ret);
  5337. goto unlock;
  5338. }
  5339. }
  5340. pmu = ERR_PTR(-ENOENT);
  5341. unlock:
  5342. srcu_read_unlock(&pmus_srcu, idx);
  5343. return pmu;
  5344. }
  5345. static void account_event_cpu(struct perf_event *event, int cpu)
  5346. {
  5347. if (event->parent)
  5348. return;
  5349. if (has_branch_stack(event)) {
  5350. if (!(event->attach_state & PERF_ATTACH_TASK))
  5351. atomic_inc(&per_cpu(perf_branch_stack_events, cpu));
  5352. }
  5353. if (is_cgroup_event(event))
  5354. atomic_inc(&per_cpu(perf_cgroup_events, cpu));
  5355. }
  5356. static void account_event(struct perf_event *event)
  5357. {
  5358. if (event->parent)
  5359. return;
  5360. if (event->attach_state & PERF_ATTACH_TASK)
  5361. static_key_slow_inc(&perf_sched_events.key);
  5362. if (event->attr.mmap || event->attr.mmap_data)
  5363. atomic_inc(&nr_mmap_events);
  5364. if (event->attr.comm)
  5365. atomic_inc(&nr_comm_events);
  5366. if (event->attr.task)
  5367. atomic_inc(&nr_task_events);
  5368. if (event->attr.freq) {
  5369. if (atomic_inc_return(&nr_freq_events) == 1)
  5370. tick_nohz_full_kick_all();
  5371. }
  5372. if (has_branch_stack(event))
  5373. static_key_slow_inc(&perf_sched_events.key);
  5374. if (is_cgroup_event(event))
  5375. static_key_slow_inc(&perf_sched_events.key);
  5376. account_event_cpu(event, event->cpu);
  5377. }
  5378. /*
  5379. * Allocate and initialize a event structure
  5380. */
  5381. static struct perf_event *
  5382. perf_event_alloc(struct perf_event_attr *attr, int cpu,
  5383. struct task_struct *task,
  5384. struct perf_event *group_leader,
  5385. struct perf_event *parent_event,
  5386. perf_overflow_handler_t overflow_handler,
  5387. void *context)
  5388. {
  5389. struct pmu *pmu;
  5390. struct perf_event *event;
  5391. struct hw_perf_event *hwc;
  5392. long err = -EINVAL;
  5393. if ((unsigned)cpu >= nr_cpu_ids) {
  5394. if (!task || cpu != -1)
  5395. return ERR_PTR(-EINVAL);
  5396. }
  5397. event = kzalloc(sizeof(*event), GFP_KERNEL);
  5398. if (!event)
  5399. return ERR_PTR(-ENOMEM);
  5400. /*
  5401. * Single events are their own group leaders, with an
  5402. * empty sibling list:
  5403. */
  5404. if (!group_leader)
  5405. group_leader = event;
  5406. mutex_init(&event->child_mutex);
  5407. INIT_LIST_HEAD(&event->child_list);
  5408. INIT_LIST_HEAD(&event->group_entry);
  5409. INIT_LIST_HEAD(&event->event_entry);
  5410. INIT_LIST_HEAD(&event->sibling_list);
  5411. INIT_LIST_HEAD(&event->rb_entry);
  5412. init_waitqueue_head(&event->waitq);
  5413. init_irq_work(&event->pending, perf_pending_event);
  5414. mutex_init(&event->mmap_mutex);
  5415. atomic_long_set(&event->refcount, 1);
  5416. event->cpu = cpu;
  5417. event->attr = *attr;
  5418. event->group_leader = group_leader;
  5419. event->pmu = NULL;
  5420. event->oncpu = -1;
  5421. event->parent = parent_event;
  5422. event->ns = get_pid_ns(task_active_pid_ns(current));
  5423. event->id = atomic64_inc_return(&perf_event_id);
  5424. event->state = PERF_EVENT_STATE_INACTIVE;
  5425. if (task) {
  5426. event->attach_state = PERF_ATTACH_TASK;
  5427. if (attr->type == PERF_TYPE_TRACEPOINT)
  5428. event->hw.tp_target = task;
  5429. #ifdef CONFIG_HAVE_HW_BREAKPOINT
  5430. /*
  5431. * hw_breakpoint is a bit difficult here..
  5432. */
  5433. else if (attr->type == PERF_TYPE_BREAKPOINT)
  5434. event->hw.bp_target = task;
  5435. #endif
  5436. }
  5437. if (!overflow_handler && parent_event) {
  5438. overflow_handler = parent_event->overflow_handler;
  5439. context = parent_event->overflow_handler_context;
  5440. }
  5441. event->overflow_handler = overflow_handler;
  5442. event->overflow_handler_context = context;
  5443. perf_event__state_init(event);
  5444. pmu = NULL;
  5445. hwc = &event->hw;
  5446. hwc->sample_period = attr->sample_period;
  5447. if (attr->freq && attr->sample_freq)
  5448. hwc->sample_period = 1;
  5449. hwc->last_period = hwc->sample_period;
  5450. local64_set(&hwc->period_left, hwc->sample_period);
  5451. /*
  5452. * we currently do not support PERF_FORMAT_GROUP on inherited events
  5453. */
  5454. if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
  5455. goto err_ns;
  5456. pmu = perf_init_event(event);
  5457. if (!pmu)
  5458. goto err_ns;
  5459. else if (IS_ERR(pmu)) {
  5460. err = PTR_ERR(pmu);
  5461. goto err_ns;
  5462. }
  5463. if (!event->parent) {
  5464. if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
  5465. err = get_callchain_buffers();
  5466. if (err)
  5467. goto err_pmu;
  5468. }
  5469. }
  5470. return event;
  5471. err_pmu:
  5472. if (event->destroy)
  5473. event->destroy(event);
  5474. err_ns:
  5475. if (event->ns)
  5476. put_pid_ns(event->ns);
  5477. kfree(event);
  5478. return ERR_PTR(err);
  5479. }
  5480. static int perf_copy_attr(struct perf_event_attr __user *uattr,
  5481. struct perf_event_attr *attr)
  5482. {
  5483. u32 size;
  5484. int ret;
  5485. if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
  5486. return -EFAULT;
  5487. /*
  5488. * zero the full structure, so that a short copy will be nice.
  5489. */
  5490. memset(attr, 0, sizeof(*attr));
  5491. ret = get_user(size, &uattr->size);
  5492. if (ret)
  5493. return ret;
  5494. if (size > PAGE_SIZE) /* silly large */
  5495. goto err_size;
  5496. if (!size) /* abi compat */
  5497. size = PERF_ATTR_SIZE_VER0;
  5498. if (size < PERF_ATTR_SIZE_VER0)
  5499. goto err_size;
  5500. /*
  5501. * If we're handed a bigger struct than we know of,
  5502. * ensure all the unknown bits are 0 - i.e. new
  5503. * user-space does not rely on any kernel feature
  5504. * extensions we dont know about yet.
  5505. */
  5506. if (size > sizeof(*attr)) {
  5507. unsigned char __user *addr;
  5508. unsigned char __user *end;
  5509. unsigned char val;
  5510. addr = (void __user *)uattr + sizeof(*attr);
  5511. end = (void __user *)uattr + size;
  5512. for (; addr < end; addr++) {
  5513. ret = get_user(val, addr);
  5514. if (ret)
  5515. return ret;
  5516. if (val)
  5517. goto err_size;
  5518. }
  5519. size = sizeof(*attr);
  5520. }
  5521. ret = copy_from_user(attr, uattr, size);
  5522. if (ret)
  5523. return -EFAULT;
  5524. /* disabled for now */
  5525. if (attr->mmap2)
  5526. return -EINVAL;
  5527. if (attr->__reserved_1)
  5528. return -EINVAL;
  5529. if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
  5530. return -EINVAL;
  5531. if (attr->read_format & ~(PERF_FORMAT_MAX-1))
  5532. return -EINVAL;
  5533. if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
  5534. u64 mask = attr->branch_sample_type;
  5535. /* only using defined bits */
  5536. if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
  5537. return -EINVAL;
  5538. /* at least one branch bit must be set */
  5539. if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
  5540. return -EINVAL;
  5541. /* propagate priv level, when not set for branch */
  5542. if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
  5543. /* exclude_kernel checked on syscall entry */
  5544. if (!attr->exclude_kernel)
  5545. mask |= PERF_SAMPLE_BRANCH_KERNEL;
  5546. if (!attr->exclude_user)
  5547. mask |= PERF_SAMPLE_BRANCH_USER;
  5548. if (!attr->exclude_hv)
  5549. mask |= PERF_SAMPLE_BRANCH_HV;
  5550. /*
  5551. * adjust user setting (for HW filter setup)
  5552. */
  5553. attr->branch_sample_type = mask;
  5554. }
  5555. /* privileged levels capture (kernel, hv): check permissions */
  5556. if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
  5557. && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
  5558. return -EACCES;
  5559. }
  5560. if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
  5561. ret = perf_reg_validate(attr->sample_regs_user);
  5562. if (ret)
  5563. return ret;
  5564. }
  5565. if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
  5566. if (!arch_perf_have_user_stack_dump())
  5567. return -ENOSYS;
  5568. /*
  5569. * We have __u32 type for the size, but so far
  5570. * we can only use __u16 as maximum due to the
  5571. * __u16 sample size limit.
  5572. */
  5573. if (attr->sample_stack_user >= USHRT_MAX)
  5574. ret = -EINVAL;
  5575. else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
  5576. ret = -EINVAL;
  5577. }
  5578. out:
  5579. return ret;
  5580. err_size:
  5581. put_user(sizeof(*attr), &uattr->size);
  5582. ret = -E2BIG;
  5583. goto out;
  5584. }
  5585. static int
  5586. perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
  5587. {
  5588. struct ring_buffer *rb = NULL, *old_rb = NULL;
  5589. int ret = -EINVAL;
  5590. if (!output_event)
  5591. goto set;
  5592. /* don't allow circular references */
  5593. if (event == output_event)
  5594. goto out;
  5595. /*
  5596. * Don't allow cross-cpu buffers
  5597. */
  5598. if (output_event->cpu != event->cpu)
  5599. goto out;
  5600. /*
  5601. * If its not a per-cpu rb, it must be the same task.
  5602. */
  5603. if (output_event->cpu == -1 && output_event->ctx != event->ctx)
  5604. goto out;
  5605. set:
  5606. mutex_lock(&event->mmap_mutex);
  5607. /* Can't redirect output if we've got an active mmap() */
  5608. if (atomic_read(&event->mmap_count))
  5609. goto unlock;
  5610. old_rb = event->rb;
  5611. if (output_event) {
  5612. /* get the rb we want to redirect to */
  5613. rb = ring_buffer_get(output_event);
  5614. if (!rb)
  5615. goto unlock;
  5616. }
  5617. if (old_rb)
  5618. ring_buffer_detach(event, old_rb);
  5619. if (rb)
  5620. ring_buffer_attach(event, rb);
  5621. rcu_assign_pointer(event->rb, rb);
  5622. if (old_rb) {
  5623. ring_buffer_put(old_rb);
  5624. /*
  5625. * Since we detached before setting the new rb, so that we
  5626. * could attach the new rb, we could have missed a wakeup.
  5627. * Provide it now.
  5628. */
  5629. wake_up_all(&event->waitq);
  5630. }
  5631. ret = 0;
  5632. unlock:
  5633. mutex_unlock(&event->mmap_mutex);
  5634. out:
  5635. return ret;
  5636. }
  5637. /**
  5638. * sys_perf_event_open - open a performance event, associate it to a task/cpu
  5639. *
  5640. * @attr_uptr: event_id type attributes for monitoring/sampling
  5641. * @pid: target pid
  5642. * @cpu: target cpu
  5643. * @group_fd: group leader event fd
  5644. */
  5645. SYSCALL_DEFINE5(perf_event_open,
  5646. struct perf_event_attr __user *, attr_uptr,
  5647. pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
  5648. {
  5649. struct perf_event *group_leader = NULL, *output_event = NULL;
  5650. struct perf_event *event, *sibling;
  5651. struct perf_event_attr attr;
  5652. struct perf_event_context *ctx;
  5653. struct file *event_file = NULL;
  5654. struct fd group = {NULL, 0};
  5655. struct task_struct *task = NULL;
  5656. struct pmu *pmu;
  5657. int event_fd;
  5658. int move_group = 0;
  5659. int err;
  5660. /* for future expandability... */
  5661. if (flags & ~PERF_FLAG_ALL)
  5662. return -EINVAL;
  5663. err = perf_copy_attr(attr_uptr, &attr);
  5664. if (err)
  5665. return err;
  5666. if (!attr.exclude_kernel) {
  5667. if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
  5668. return -EACCES;
  5669. }
  5670. if (attr.freq) {
  5671. if (attr.sample_freq > sysctl_perf_event_sample_rate)
  5672. return -EINVAL;
  5673. }
  5674. /*
  5675. * In cgroup mode, the pid argument is used to pass the fd
  5676. * opened to the cgroup directory in cgroupfs. The cpu argument
  5677. * designates the cpu on which to monitor threads from that
  5678. * cgroup.
  5679. */
  5680. if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
  5681. return -EINVAL;
  5682. event_fd = get_unused_fd();
  5683. if (event_fd < 0)
  5684. return event_fd;
  5685. if (group_fd != -1) {
  5686. err = perf_fget_light(group_fd, &group);
  5687. if (err)
  5688. goto err_fd;
  5689. group_leader = group.file->private_data;
  5690. if (flags & PERF_FLAG_FD_OUTPUT)
  5691. output_event = group_leader;
  5692. if (flags & PERF_FLAG_FD_NO_GROUP)
  5693. group_leader = NULL;
  5694. }
  5695. if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
  5696. task = find_lively_task_by_vpid(pid);
  5697. if (IS_ERR(task)) {
  5698. err = PTR_ERR(task);
  5699. goto err_group_fd;
  5700. }
  5701. }
  5702. get_online_cpus();
  5703. event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
  5704. NULL, NULL);
  5705. if (IS_ERR(event)) {
  5706. err = PTR_ERR(event);
  5707. goto err_task;
  5708. }
  5709. if (flags & PERF_FLAG_PID_CGROUP) {
  5710. err = perf_cgroup_connect(pid, event, &attr, group_leader);
  5711. if (err) {
  5712. __free_event(event);
  5713. goto err_task;
  5714. }
  5715. }
  5716. account_event(event);
  5717. /*
  5718. * Special case software events and allow them to be part of
  5719. * any hardware group.
  5720. */
  5721. pmu = event->pmu;
  5722. if (group_leader &&
  5723. (is_software_event(event) != is_software_event(group_leader))) {
  5724. if (is_software_event(event)) {
  5725. /*
  5726. * If event and group_leader are not both a software
  5727. * event, and event is, then group leader is not.
  5728. *
  5729. * Allow the addition of software events to !software
  5730. * groups, this is safe because software events never
  5731. * fail to schedule.
  5732. */
  5733. pmu = group_leader->pmu;
  5734. } else if (is_software_event(group_leader) &&
  5735. (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
  5736. /*
  5737. * In case the group is a pure software group, and we
  5738. * try to add a hardware event, move the whole group to
  5739. * the hardware context.
  5740. */
  5741. move_group = 1;
  5742. }
  5743. }
  5744. /*
  5745. * Get the target context (task or percpu):
  5746. */
  5747. ctx = find_get_context(pmu, task, event->cpu);
  5748. if (IS_ERR(ctx)) {
  5749. err = PTR_ERR(ctx);
  5750. goto err_alloc;
  5751. }
  5752. if (task) {
  5753. put_task_struct(task);
  5754. task = NULL;
  5755. }
  5756. /*
  5757. * Look up the group leader (we will attach this event to it):
  5758. */
  5759. if (group_leader) {
  5760. err = -EINVAL;
  5761. /*
  5762. * Do not allow a recursive hierarchy (this new sibling
  5763. * becoming part of another group-sibling):
  5764. */
  5765. if (group_leader->group_leader != group_leader)
  5766. goto err_context;
  5767. /*
  5768. * Do not allow to attach to a group in a different
  5769. * task or CPU context:
  5770. */
  5771. if (move_group) {
  5772. if (group_leader->ctx->type != ctx->type)
  5773. goto err_context;
  5774. } else {
  5775. if (group_leader->ctx != ctx)
  5776. goto err_context;
  5777. }
  5778. /*
  5779. * Only a group leader can be exclusive or pinned
  5780. */
  5781. if (attr.exclusive || attr.pinned)
  5782. goto err_context;
  5783. }
  5784. if (output_event) {
  5785. err = perf_event_set_output(event, output_event);
  5786. if (err)
  5787. goto err_context;
  5788. }
  5789. event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
  5790. if (IS_ERR(event_file)) {
  5791. err = PTR_ERR(event_file);
  5792. goto err_context;
  5793. }
  5794. if (move_group) {
  5795. struct perf_event_context *gctx = group_leader->ctx;
  5796. mutex_lock(&gctx->mutex);
  5797. perf_remove_from_context(group_leader);
  5798. /*
  5799. * Removing from the context ends up with disabled
  5800. * event. What we want here is event in the initial
  5801. * startup state, ready to be add into new context.
  5802. */
  5803. perf_event__state_init(group_leader);
  5804. list_for_each_entry(sibling, &group_leader->sibling_list,
  5805. group_entry) {
  5806. perf_remove_from_context(sibling);
  5807. perf_event__state_init(sibling);
  5808. put_ctx(gctx);
  5809. }
  5810. mutex_unlock(&gctx->mutex);
  5811. put_ctx(gctx);
  5812. }
  5813. WARN_ON_ONCE(ctx->parent_ctx);
  5814. mutex_lock(&ctx->mutex);
  5815. if (move_group) {
  5816. synchronize_rcu();
  5817. perf_install_in_context(ctx, group_leader, event->cpu);
  5818. get_ctx(ctx);
  5819. list_for_each_entry(sibling, &group_leader->sibling_list,
  5820. group_entry) {
  5821. perf_install_in_context(ctx, sibling, event->cpu);
  5822. get_ctx(ctx);
  5823. }
  5824. }
  5825. perf_install_in_context(ctx, event, event->cpu);
  5826. ++ctx->generation;
  5827. perf_unpin_context(ctx);
  5828. mutex_unlock(&ctx->mutex);
  5829. put_online_cpus();
  5830. event->owner = current;
  5831. mutex_lock(&current->perf_event_mutex);
  5832. list_add_tail(&event->owner_entry, &current->perf_event_list);
  5833. mutex_unlock(&current->perf_event_mutex);
  5834. /*
  5835. * Precalculate sample_data sizes
  5836. */
  5837. perf_event__header_size(event);
  5838. perf_event__id_header_size(event);
  5839. /*
  5840. * Drop the reference on the group_event after placing the
  5841. * new event on the sibling_list. This ensures destruction
  5842. * of the group leader will find the pointer to itself in
  5843. * perf_group_detach().
  5844. */
  5845. fdput(group);
  5846. fd_install(event_fd, event_file);
  5847. return event_fd;
  5848. err_context:
  5849. perf_unpin_context(ctx);
  5850. put_ctx(ctx);
  5851. err_alloc:
  5852. free_event(event);
  5853. err_task:
  5854. put_online_cpus();
  5855. if (task)
  5856. put_task_struct(task);
  5857. err_group_fd:
  5858. fdput(group);
  5859. err_fd:
  5860. put_unused_fd(event_fd);
  5861. return err;
  5862. }
  5863. /**
  5864. * perf_event_create_kernel_counter
  5865. *
  5866. * @attr: attributes of the counter to create
  5867. * @cpu: cpu in which the counter is bound
  5868. * @task: task to profile (NULL for percpu)
  5869. */
  5870. struct perf_event *
  5871. perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
  5872. struct task_struct *task,
  5873. perf_overflow_handler_t overflow_handler,
  5874. void *context)
  5875. {
  5876. struct perf_event_context *ctx;
  5877. struct perf_event *event;
  5878. int err;
  5879. /*
  5880. * Get the target context (task or percpu):
  5881. */
  5882. event = perf_event_alloc(attr, cpu, task, NULL, NULL,
  5883. overflow_handler, context);
  5884. if (IS_ERR(event)) {
  5885. err = PTR_ERR(event);
  5886. goto err;
  5887. }
  5888. account_event(event);
  5889. ctx = find_get_context(event->pmu, task, cpu);
  5890. if (IS_ERR(ctx)) {
  5891. err = PTR_ERR(ctx);
  5892. goto err_free;
  5893. }
  5894. WARN_ON_ONCE(ctx->parent_ctx);
  5895. mutex_lock(&ctx->mutex);
  5896. perf_install_in_context(ctx, event, cpu);
  5897. ++ctx->generation;
  5898. perf_unpin_context(ctx);
  5899. mutex_unlock(&ctx->mutex);
  5900. return event;
  5901. err_free:
  5902. free_event(event);
  5903. err:
  5904. return ERR_PTR(err);
  5905. }
  5906. EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
  5907. void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
  5908. {
  5909. struct perf_event_context *src_ctx;
  5910. struct perf_event_context *dst_ctx;
  5911. struct perf_event *event, *tmp;
  5912. LIST_HEAD(events);
  5913. src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx;
  5914. dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx;
  5915. mutex_lock(&src_ctx->mutex);
  5916. list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
  5917. event_entry) {
  5918. perf_remove_from_context(event);
  5919. unaccount_event_cpu(event, src_cpu);
  5920. put_ctx(src_ctx);
  5921. list_add(&event->migrate_entry, &events);
  5922. }
  5923. mutex_unlock(&src_ctx->mutex);
  5924. synchronize_rcu();
  5925. mutex_lock(&dst_ctx->mutex);
  5926. list_for_each_entry_safe(event, tmp, &events, migrate_entry) {
  5927. list_del(&event->migrate_entry);
  5928. if (event->state >= PERF_EVENT_STATE_OFF)
  5929. event->state = PERF_EVENT_STATE_INACTIVE;
  5930. account_event_cpu(event, dst_cpu);
  5931. perf_install_in_context(dst_ctx, event, dst_cpu);
  5932. get_ctx(dst_ctx);
  5933. }
  5934. mutex_unlock(&dst_ctx->mutex);
  5935. }
  5936. EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
  5937. static void sync_child_event(struct perf_event *child_event,
  5938. struct task_struct *child)
  5939. {
  5940. struct perf_event *parent_event = child_event->parent;
  5941. u64 child_val;
  5942. if (child_event->attr.inherit_stat)
  5943. perf_event_read_event(child_event, child);
  5944. child_val = perf_event_count(child_event);
  5945. /*
  5946. * Add back the child's count to the parent's count:
  5947. */
  5948. atomic64_add(child_val, &parent_event->child_count);
  5949. atomic64_add(child_event->total_time_enabled,
  5950. &parent_event->child_total_time_enabled);
  5951. atomic64_add(child_event->total_time_running,
  5952. &parent_event->child_total_time_running);
  5953. /*
  5954. * Remove this event from the parent's list
  5955. */
  5956. WARN_ON_ONCE(parent_event->ctx->parent_ctx);
  5957. mutex_lock(&parent_event->child_mutex);
  5958. list_del_init(&child_event->child_list);
  5959. mutex_unlock(&parent_event->child_mutex);
  5960. /*
  5961. * Release the parent event, if this was the last
  5962. * reference to it.
  5963. */
  5964. put_event(parent_event);
  5965. }
  5966. static void
  5967. __perf_event_exit_task(struct perf_event *child_event,
  5968. struct perf_event_context *child_ctx,
  5969. struct task_struct *child)
  5970. {
  5971. if (child_event->parent) {
  5972. raw_spin_lock_irq(&child_ctx->lock);
  5973. perf_group_detach(child_event);
  5974. raw_spin_unlock_irq(&child_ctx->lock);
  5975. }
  5976. perf_remove_from_context(child_event);
  5977. /*
  5978. * It can happen that the parent exits first, and has events
  5979. * that are still around due to the child reference. These
  5980. * events need to be zapped.
  5981. */
  5982. if (child_event->parent) {
  5983. sync_child_event(child_event, child);
  5984. free_event(child_event);
  5985. }
  5986. }
  5987. static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
  5988. {
  5989. struct perf_event *child_event, *tmp;
  5990. struct perf_event_context *child_ctx;
  5991. unsigned long flags;
  5992. if (likely(!child->perf_event_ctxp[ctxn])) {
  5993. perf_event_task(child, NULL, 0);
  5994. return;
  5995. }
  5996. local_irq_save(flags);
  5997. /*
  5998. * We can't reschedule here because interrupts are disabled,
  5999. * and either child is current or it is a task that can't be
  6000. * scheduled, so we are now safe from rescheduling changing
  6001. * our context.
  6002. */
  6003. child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
  6004. /*
  6005. * Take the context lock here so that if find_get_context is
  6006. * reading child->perf_event_ctxp, we wait until it has
  6007. * incremented the context's refcount before we do put_ctx below.
  6008. */
  6009. raw_spin_lock(&child_ctx->lock);
  6010. task_ctx_sched_out(child_ctx);
  6011. child->perf_event_ctxp[ctxn] = NULL;
  6012. /*
  6013. * If this context is a clone; unclone it so it can't get
  6014. * swapped to another process while we're removing all
  6015. * the events from it.
  6016. */
  6017. unclone_ctx(child_ctx);
  6018. update_context_time(child_ctx);
  6019. raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
  6020. /*
  6021. * Report the task dead after unscheduling the events so that we
  6022. * won't get any samples after PERF_RECORD_EXIT. We can however still
  6023. * get a few PERF_RECORD_READ events.
  6024. */
  6025. perf_event_task(child, child_ctx, 0);
  6026. /*
  6027. * We can recurse on the same lock type through:
  6028. *
  6029. * __perf_event_exit_task()
  6030. * sync_child_event()
  6031. * put_event()
  6032. * mutex_lock(&ctx->mutex)
  6033. *
  6034. * But since its the parent context it won't be the same instance.
  6035. */
  6036. mutex_lock(&child_ctx->mutex);
  6037. again:
  6038. list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
  6039. group_entry)
  6040. __perf_event_exit_task(child_event, child_ctx, child);
  6041. list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
  6042. group_entry)
  6043. __perf_event_exit_task(child_event, child_ctx, child);
  6044. /*
  6045. * If the last event was a group event, it will have appended all
  6046. * its siblings to the list, but we obtained 'tmp' before that which
  6047. * will still point to the list head terminating the iteration.
  6048. */
  6049. if (!list_empty(&child_ctx->pinned_groups) ||
  6050. !list_empty(&child_ctx->flexible_groups))
  6051. goto again;
  6052. mutex_unlock(&child_ctx->mutex);
  6053. put_ctx(child_ctx);
  6054. }
  6055. /*
  6056. * When a child task exits, feed back event values to parent events.
  6057. */
  6058. void perf_event_exit_task(struct task_struct *child)
  6059. {
  6060. struct perf_event *event, *tmp;
  6061. int ctxn;
  6062. mutex_lock(&child->perf_event_mutex);
  6063. list_for_each_entry_safe(event, tmp, &child->perf_event_list,
  6064. owner_entry) {
  6065. list_del_init(&event->owner_entry);
  6066. /*
  6067. * Ensure the list deletion is visible before we clear
  6068. * the owner, closes a race against perf_release() where
  6069. * we need to serialize on the owner->perf_event_mutex.
  6070. */
  6071. smp_wmb();
  6072. event->owner = NULL;
  6073. }
  6074. mutex_unlock(&child->perf_event_mutex);
  6075. for_each_task_context_nr(ctxn)
  6076. perf_event_exit_task_context(child, ctxn);
  6077. }
  6078. static void perf_free_event(struct perf_event *event,
  6079. struct perf_event_context *ctx)
  6080. {
  6081. struct perf_event *parent = event->parent;
  6082. if (WARN_ON_ONCE(!parent))
  6083. return;
  6084. mutex_lock(&parent->child_mutex);
  6085. list_del_init(&event->child_list);
  6086. mutex_unlock(&parent->child_mutex);
  6087. put_event(parent);
  6088. perf_group_detach(event);
  6089. list_del_event(event, ctx);
  6090. free_event(event);
  6091. }
  6092. /*
  6093. * free an unexposed, unused context as created by inheritance by
  6094. * perf_event_init_task below, used by fork() in case of fail.
  6095. */
  6096. void perf_event_free_task(struct task_struct *task)
  6097. {
  6098. struct perf_event_context *ctx;
  6099. struct perf_event *event, *tmp;
  6100. int ctxn;
  6101. for_each_task_context_nr(ctxn) {
  6102. ctx = task->perf_event_ctxp[ctxn];
  6103. if (!ctx)
  6104. continue;
  6105. mutex_lock(&ctx->mutex);
  6106. again:
  6107. list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
  6108. group_entry)
  6109. perf_free_event(event, ctx);
  6110. list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
  6111. group_entry)
  6112. perf_free_event(event, ctx);
  6113. if (!list_empty(&ctx->pinned_groups) ||
  6114. !list_empty(&ctx->flexible_groups))
  6115. goto again;
  6116. mutex_unlock(&ctx->mutex);
  6117. put_ctx(ctx);
  6118. }
  6119. }
  6120. void perf_event_delayed_put(struct task_struct *task)
  6121. {
  6122. int ctxn;
  6123. for_each_task_context_nr(ctxn)
  6124. WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
  6125. }
  6126. /*
  6127. * inherit a event from parent task to child task:
  6128. */
  6129. static struct perf_event *
  6130. inherit_event(struct perf_event *parent_event,
  6131. struct task_struct *parent,
  6132. struct perf_event_context *parent_ctx,
  6133. struct task_struct *child,
  6134. struct perf_event *group_leader,
  6135. struct perf_event_context *child_ctx)
  6136. {
  6137. struct perf_event *child_event;
  6138. unsigned long flags;
  6139. /*
  6140. * Instead of creating recursive hierarchies of events,
  6141. * we link inherited events back to the original parent,
  6142. * which has a filp for sure, which we use as the reference
  6143. * count:
  6144. */
  6145. if (parent_event->parent)
  6146. parent_event = parent_event->parent;
  6147. child_event = perf_event_alloc(&parent_event->attr,
  6148. parent_event->cpu,
  6149. child,
  6150. group_leader, parent_event,
  6151. NULL, NULL);
  6152. if (IS_ERR(child_event))
  6153. return child_event;
  6154. if (!atomic_long_inc_not_zero(&parent_event->refcount)) {
  6155. free_event(child_event);
  6156. return NULL;
  6157. }
  6158. get_ctx(child_ctx);
  6159. /*
  6160. * Make the child state follow the state of the parent event,
  6161. * not its attr.disabled bit. We hold the parent's mutex,
  6162. * so we won't race with perf_event_{en, dis}able_family.
  6163. */
  6164. if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
  6165. child_event->state = PERF_EVENT_STATE_INACTIVE;
  6166. else
  6167. child_event->state = PERF_EVENT_STATE_OFF;
  6168. if (parent_event->attr.freq) {
  6169. u64 sample_period = parent_event->hw.sample_period;
  6170. struct hw_perf_event *hwc = &child_event->hw;
  6171. hwc->sample_period = sample_period;
  6172. hwc->last_period = sample_period;
  6173. local64_set(&hwc->period_left, sample_period);
  6174. }
  6175. child_event->ctx = child_ctx;
  6176. child_event->overflow_handler = parent_event->overflow_handler;
  6177. child_event->overflow_handler_context
  6178. = parent_event->overflow_handler_context;
  6179. /*
  6180. * Precalculate sample_data sizes
  6181. */
  6182. perf_event__header_size(child_event);
  6183. perf_event__id_header_size(child_event);
  6184. /*
  6185. * Link it up in the child's context:
  6186. */
  6187. raw_spin_lock_irqsave(&child_ctx->lock, flags);
  6188. add_event_to_ctx(child_event, child_ctx);
  6189. raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
  6190. /*
  6191. * Link this into the parent event's child list
  6192. */
  6193. WARN_ON_ONCE(parent_event->ctx->parent_ctx);
  6194. mutex_lock(&parent_event->child_mutex);
  6195. list_add_tail(&child_event->child_list, &parent_event->child_list);
  6196. mutex_unlock(&parent_event->child_mutex);
  6197. return child_event;
  6198. }
  6199. static int inherit_group(struct perf_event *parent_event,
  6200. struct task_struct *parent,
  6201. struct perf_event_context *parent_ctx,
  6202. struct task_struct *child,
  6203. struct perf_event_context *child_ctx)
  6204. {
  6205. struct perf_event *leader;
  6206. struct perf_event *sub;
  6207. struct perf_event *child_ctr;
  6208. leader = inherit_event(parent_event, parent, parent_ctx,
  6209. child, NULL, child_ctx);
  6210. if (IS_ERR(leader))
  6211. return PTR_ERR(leader);
  6212. list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
  6213. child_ctr = inherit_event(sub, parent, parent_ctx,
  6214. child, leader, child_ctx);
  6215. if (IS_ERR(child_ctr))
  6216. return PTR_ERR(child_ctr);
  6217. }
  6218. return 0;
  6219. }
  6220. static int
  6221. inherit_task_group(struct perf_event *event, struct task_struct *parent,
  6222. struct perf_event_context *parent_ctx,
  6223. struct task_struct *child, int ctxn,
  6224. int *inherited_all)
  6225. {
  6226. int ret;
  6227. struct perf_event_context *child_ctx;
  6228. if (!event->attr.inherit) {
  6229. *inherited_all = 0;
  6230. return 0;
  6231. }
  6232. child_ctx = child->perf_event_ctxp[ctxn];
  6233. if (!child_ctx) {
  6234. /*
  6235. * This is executed from the parent task context, so
  6236. * inherit events that have been marked for cloning.
  6237. * First allocate and initialize a context for the
  6238. * child.
  6239. */
  6240. child_ctx = alloc_perf_context(parent_ctx->pmu, child);
  6241. if (!child_ctx)
  6242. return -ENOMEM;
  6243. child->perf_event_ctxp[ctxn] = child_ctx;
  6244. }
  6245. ret = inherit_group(event, parent, parent_ctx,
  6246. child, child_ctx);
  6247. if (ret)
  6248. *inherited_all = 0;
  6249. return ret;
  6250. }
  6251. /*
  6252. * Initialize the perf_event context in task_struct
  6253. */
  6254. int perf_event_init_context(struct task_struct *child, int ctxn)
  6255. {
  6256. struct perf_event_context *child_ctx, *parent_ctx;
  6257. struct perf_event_context *cloned_ctx;
  6258. struct perf_event *event;
  6259. struct task_struct *parent = current;
  6260. int inherited_all = 1;
  6261. unsigned long flags;
  6262. int ret = 0;
  6263. if (likely(!parent->perf_event_ctxp[ctxn]))
  6264. return 0;
  6265. /*
  6266. * If the parent's context is a clone, pin it so it won't get
  6267. * swapped under us.
  6268. */
  6269. parent_ctx = perf_pin_task_context(parent, ctxn);
  6270. /*
  6271. * No need to check if parent_ctx != NULL here; since we saw
  6272. * it non-NULL earlier, the only reason for it to become NULL
  6273. * is if we exit, and since we're currently in the middle of
  6274. * a fork we can't be exiting at the same time.
  6275. */
  6276. /*
  6277. * Lock the parent list. No need to lock the child - not PID
  6278. * hashed yet and not running, so nobody can access it.
  6279. */
  6280. mutex_lock(&parent_ctx->mutex);
  6281. /*
  6282. * We dont have to disable NMIs - we are only looking at
  6283. * the list, not manipulating it:
  6284. */
  6285. list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
  6286. ret = inherit_task_group(event, parent, parent_ctx,
  6287. child, ctxn, &inherited_all);
  6288. if (ret)
  6289. break;
  6290. }
  6291. /*
  6292. * We can't hold ctx->lock when iterating the ->flexible_group list due
  6293. * to allocations, but we need to prevent rotation because
  6294. * rotate_ctx() will change the list from interrupt context.
  6295. */
  6296. raw_spin_lock_irqsave(&parent_ctx->lock, flags);
  6297. parent_ctx->rotate_disable = 1;
  6298. raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
  6299. list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
  6300. ret = inherit_task_group(event, parent, parent_ctx,
  6301. child, ctxn, &inherited_all);
  6302. if (ret)
  6303. break;
  6304. }
  6305. raw_spin_lock_irqsave(&parent_ctx->lock, flags);
  6306. parent_ctx->rotate_disable = 0;
  6307. child_ctx = child->perf_event_ctxp[ctxn];
  6308. if (child_ctx && inherited_all) {
  6309. /*
  6310. * Mark the child context as a clone of the parent
  6311. * context, or of whatever the parent is a clone of.
  6312. *
  6313. * Note that if the parent is a clone, the holding of
  6314. * parent_ctx->lock avoids it from being uncloned.
  6315. */
  6316. cloned_ctx = parent_ctx->parent_ctx;
  6317. if (cloned_ctx) {
  6318. child_ctx->parent_ctx = cloned_ctx;
  6319. child_ctx->parent_gen = parent_ctx->parent_gen;
  6320. } else {
  6321. child_ctx->parent_ctx = parent_ctx;
  6322. child_ctx->parent_gen = parent_ctx->generation;
  6323. }
  6324. get_ctx(child_ctx->parent_ctx);
  6325. }
  6326. raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
  6327. mutex_unlock(&parent_ctx->mutex);
  6328. perf_unpin_context(parent_ctx);
  6329. put_ctx(parent_ctx);
  6330. return ret;
  6331. }
  6332. /*
  6333. * Initialize the perf_event context in task_struct
  6334. */
  6335. int perf_event_init_task(struct task_struct *child)
  6336. {
  6337. int ctxn, ret;
  6338. memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
  6339. mutex_init(&child->perf_event_mutex);
  6340. INIT_LIST_HEAD(&child->perf_event_list);
  6341. for_each_task_context_nr(ctxn) {
  6342. ret = perf_event_init_context(child, ctxn);
  6343. if (ret)
  6344. return ret;
  6345. }
  6346. return 0;
  6347. }
  6348. static void __init perf_event_init_all_cpus(void)
  6349. {
  6350. struct swevent_htable *swhash;
  6351. int cpu;
  6352. for_each_possible_cpu(cpu) {
  6353. swhash = &per_cpu(swevent_htable, cpu);
  6354. mutex_init(&swhash->hlist_mutex);
  6355. INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
  6356. }
  6357. }
  6358. static void perf_event_init_cpu(int cpu)
  6359. {
  6360. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  6361. mutex_lock(&swhash->hlist_mutex);
  6362. if (swhash->hlist_refcount > 0) {
  6363. struct swevent_hlist *hlist;
  6364. hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
  6365. WARN_ON(!hlist);
  6366. rcu_assign_pointer(swhash->swevent_hlist, hlist);
  6367. }
  6368. mutex_unlock(&swhash->hlist_mutex);
  6369. }
  6370. #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
  6371. static void perf_pmu_rotate_stop(struct pmu *pmu)
  6372. {
  6373. struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  6374. WARN_ON(!irqs_disabled());
  6375. list_del_init(&cpuctx->rotation_list);
  6376. }
  6377. static void __perf_event_exit_context(void *__info)
  6378. {
  6379. struct perf_event_context *ctx = __info;
  6380. struct perf_event *event, *tmp;
  6381. perf_pmu_rotate_stop(ctx->pmu);
  6382. list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
  6383. __perf_remove_from_context(event);
  6384. list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
  6385. __perf_remove_from_context(event);
  6386. }
  6387. static void perf_event_exit_cpu_context(int cpu)
  6388. {
  6389. struct perf_event_context *ctx;
  6390. struct pmu *pmu;
  6391. int idx;
  6392. idx = srcu_read_lock(&pmus_srcu);
  6393. list_for_each_entry_rcu(pmu, &pmus, entry) {
  6394. ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
  6395. mutex_lock(&ctx->mutex);
  6396. smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
  6397. mutex_unlock(&ctx->mutex);
  6398. }
  6399. srcu_read_unlock(&pmus_srcu, idx);
  6400. }
  6401. static void perf_event_exit_cpu(int cpu)
  6402. {
  6403. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  6404. mutex_lock(&swhash->hlist_mutex);
  6405. swevent_hlist_release(swhash);
  6406. mutex_unlock(&swhash->hlist_mutex);
  6407. perf_event_exit_cpu_context(cpu);
  6408. }
  6409. #else
  6410. static inline void perf_event_exit_cpu(int cpu) { }
  6411. #endif
  6412. static int
  6413. perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
  6414. {
  6415. int cpu;
  6416. for_each_online_cpu(cpu)
  6417. perf_event_exit_cpu(cpu);
  6418. return NOTIFY_OK;
  6419. }
  6420. /*
  6421. * Run the perf reboot notifier at the very last possible moment so that
  6422. * the generic watchdog code runs as long as possible.
  6423. */
  6424. static struct notifier_block perf_reboot_notifier = {
  6425. .notifier_call = perf_reboot,
  6426. .priority = INT_MIN,
  6427. };
  6428. static int
  6429. perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
  6430. {
  6431. unsigned int cpu = (long)hcpu;
  6432. switch (action & ~CPU_TASKS_FROZEN) {
  6433. case CPU_UP_PREPARE:
  6434. case CPU_DOWN_FAILED:
  6435. perf_event_init_cpu(cpu);
  6436. break;
  6437. case CPU_UP_CANCELED:
  6438. case CPU_DOWN_PREPARE:
  6439. perf_event_exit_cpu(cpu);
  6440. break;
  6441. default:
  6442. break;
  6443. }
  6444. return NOTIFY_OK;
  6445. }
  6446. void __init perf_event_init(void)
  6447. {
  6448. int ret;
  6449. idr_init(&pmu_idr);
  6450. perf_event_init_all_cpus();
  6451. init_srcu_struct(&pmus_srcu);
  6452. perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
  6453. perf_pmu_register(&perf_cpu_clock, NULL, -1);
  6454. perf_pmu_register(&perf_task_clock, NULL, -1);
  6455. perf_tp_register();
  6456. perf_cpu_notifier(perf_cpu_notify);
  6457. register_reboot_notifier(&perf_reboot_notifier);
  6458. ret = init_hw_breakpoint();
  6459. WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
  6460. /* do not patch jump label more than once per second */
  6461. jump_label_rate_limit(&perf_sched_events, HZ);
  6462. /*
  6463. * Build time assertion that we keep the data_head at the intended
  6464. * location. IOW, validation we got the __reserved[] size right.
  6465. */
  6466. BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
  6467. != 1024);
  6468. }
  6469. static int __init perf_event_sysfs_init(void)
  6470. {
  6471. struct pmu *pmu;
  6472. int ret;
  6473. mutex_lock(&pmus_lock);
  6474. ret = bus_register(&pmu_bus);
  6475. if (ret)
  6476. goto unlock;
  6477. list_for_each_entry(pmu, &pmus, entry) {
  6478. if (!pmu->name || pmu->type < 0)
  6479. continue;
  6480. ret = pmu_dev_alloc(pmu);
  6481. WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
  6482. }
  6483. pmu_bus_running = 1;
  6484. ret = 0;
  6485. unlock:
  6486. mutex_unlock(&pmus_lock);
  6487. return ret;
  6488. }
  6489. device_initcall(perf_event_sysfs_init);
  6490. #ifdef CONFIG_CGROUP_PERF
  6491. static struct cgroup_subsys_state *
  6492. perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
  6493. {
  6494. struct perf_cgroup *jc;
  6495. jc = kzalloc(sizeof(*jc), GFP_KERNEL);
  6496. if (!jc)
  6497. return ERR_PTR(-ENOMEM);
  6498. jc->info = alloc_percpu(struct perf_cgroup_info);
  6499. if (!jc->info) {
  6500. kfree(jc);
  6501. return ERR_PTR(-ENOMEM);
  6502. }
  6503. return &jc->css;
  6504. }
  6505. static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
  6506. {
  6507. struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
  6508. free_percpu(jc->info);
  6509. kfree(jc);
  6510. }
  6511. static int __perf_cgroup_move(void *info)
  6512. {
  6513. struct task_struct *task = info;
  6514. perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
  6515. return 0;
  6516. }
  6517. static void perf_cgroup_attach(struct cgroup_subsys_state *css,
  6518. struct cgroup_taskset *tset)
  6519. {
  6520. struct task_struct *task;
  6521. cgroup_taskset_for_each(task, css, tset)
  6522. task_function_call(task, __perf_cgroup_move, task);
  6523. }
  6524. static void perf_cgroup_exit(struct cgroup_subsys_state *css,
  6525. struct cgroup_subsys_state *old_css,
  6526. struct task_struct *task)
  6527. {
  6528. /*
  6529. * cgroup_exit() is called in the copy_process() failure path.
  6530. * Ignore this case since the task hasn't ran yet, this avoids
  6531. * trying to poke a half freed task state from generic code.
  6532. */
  6533. if (!(task->flags & PF_EXITING))
  6534. return;
  6535. task_function_call(task, __perf_cgroup_move, task);
  6536. }
  6537. struct cgroup_subsys perf_subsys = {
  6538. .name = "perf_event",
  6539. .subsys_id = perf_subsys_id,
  6540. .css_alloc = perf_cgroup_css_alloc,
  6541. .css_free = perf_cgroup_css_free,
  6542. .exit = perf_cgroup_exit,
  6543. .attach = perf_cgroup_attach,
  6544. };
  6545. #endif /* CONFIG_CGROUP_PERF */