core.c 185 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972
  1. /*
  2. * Performance events core code:
  3. *
  4. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  6. * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  7. * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  8. *
  9. * For licensing details see kernel-base/COPYING
  10. */
  11. #include <linux/fs.h>
  12. #include <linux/mm.h>
  13. #include <linux/cpu.h>
  14. #include <linux/smp.h>
  15. #include <linux/idr.h>
  16. #include <linux/file.h>
  17. #include <linux/poll.h>
  18. #include <linux/slab.h>
  19. #include <linux/hash.h>
  20. #include <linux/tick.h>
  21. #include <linux/sysfs.h>
  22. #include <linux/dcache.h>
  23. #include <linux/percpu.h>
  24. #include <linux/ptrace.h>
  25. #include <linux/reboot.h>
  26. #include <linux/vmstat.h>
  27. #include <linux/device.h>
  28. #include <linux/export.h>
  29. #include <linux/vmalloc.h>
  30. #include <linux/hardirq.h>
  31. #include <linux/rculist.h>
  32. #include <linux/uaccess.h>
  33. #include <linux/syscalls.h>
  34. #include <linux/anon_inodes.h>
  35. #include <linux/kernel_stat.h>
  36. #include <linux/perf_event.h>
  37. #include <linux/ftrace_event.h>
  38. #include <linux/hw_breakpoint.h>
  39. #include <linux/mm_types.h>
  40. #include <linux/cgroup.h>
  41. #include "internal.h"
  42. #include <asm/irq_regs.h>
  43. struct remote_function_call {
  44. struct task_struct *p;
  45. int (*func)(void *info);
  46. void *info;
  47. int ret;
  48. };
  49. static void remote_function(void *data)
  50. {
  51. struct remote_function_call *tfc = data;
  52. struct task_struct *p = tfc->p;
  53. if (p) {
  54. tfc->ret = -EAGAIN;
  55. if (task_cpu(p) != smp_processor_id() || !task_curr(p))
  56. return;
  57. }
  58. tfc->ret = tfc->func(tfc->info);
  59. }
  60. /**
  61. * task_function_call - call a function on the cpu on which a task runs
  62. * @p: the task to evaluate
  63. * @func: the function to be called
  64. * @info: the function call argument
  65. *
  66. * Calls the function @func when the task is currently running. This might
  67. * be on the current CPU, which just calls the function directly
  68. *
  69. * returns: @func return value, or
  70. * -ESRCH - when the process isn't running
  71. * -EAGAIN - when the process moved away
  72. */
  73. static int
  74. task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
  75. {
  76. struct remote_function_call data = {
  77. .p = p,
  78. .func = func,
  79. .info = info,
  80. .ret = -ESRCH, /* No such (running) process */
  81. };
  82. if (task_curr(p))
  83. smp_call_function_single(task_cpu(p), remote_function, &data, 1);
  84. return data.ret;
  85. }
  86. /**
  87. * cpu_function_call - call a function on the cpu
  88. * @func: the function to be called
  89. * @info: the function call argument
  90. *
  91. * Calls the function @func on the remote cpu.
  92. *
  93. * returns: @func return value or -ENXIO when the cpu is offline
  94. */
  95. static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
  96. {
  97. struct remote_function_call data = {
  98. .p = NULL,
  99. .func = func,
  100. .info = info,
  101. .ret = -ENXIO, /* No such CPU */
  102. };
  103. smp_call_function_single(cpu, remote_function, &data, 1);
  104. return data.ret;
  105. }
  106. #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
  107. PERF_FLAG_FD_OUTPUT |\
  108. PERF_FLAG_PID_CGROUP)
  109. /*
  110. * branch priv levels that need permission checks
  111. */
  112. #define PERF_SAMPLE_BRANCH_PERM_PLM \
  113. (PERF_SAMPLE_BRANCH_KERNEL |\
  114. PERF_SAMPLE_BRANCH_HV)
  115. enum event_type_t {
  116. EVENT_FLEXIBLE = 0x1,
  117. EVENT_PINNED = 0x2,
  118. EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
  119. };
  120. /*
  121. * perf_sched_events : >0 events exist
  122. * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
  123. */
  124. struct static_key_deferred perf_sched_events __read_mostly;
  125. static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
  126. static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
  127. static atomic_t nr_mmap_events __read_mostly;
  128. static atomic_t nr_comm_events __read_mostly;
  129. static atomic_t nr_task_events __read_mostly;
  130. static atomic_t nr_freq_events __read_mostly;
  131. static LIST_HEAD(pmus);
  132. static DEFINE_MUTEX(pmus_lock);
  133. static struct srcu_struct pmus_srcu;
  134. /*
  135. * perf event paranoia level:
  136. * -1 - not paranoid at all
  137. * 0 - disallow raw tracepoint access for unpriv
  138. * 1 - disallow cpu events for unpriv
  139. * 2 - disallow kernel profiling for unpriv
  140. */
  141. int sysctl_perf_event_paranoid __read_mostly = 1;
  142. /* Minimum for 512 kiB + 1 user control page */
  143. int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
  144. /*
  145. * max perf event sample rate
  146. */
  147. #define DEFAULT_MAX_SAMPLE_RATE 100000
  148. #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
  149. #define DEFAULT_CPU_TIME_MAX_PERCENT 25
  150. int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
  151. static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
  152. static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS;
  153. static atomic_t perf_sample_allowed_ns __read_mostly =
  154. ATOMIC_INIT( DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100);
  155. void update_perf_cpu_limits(void)
  156. {
  157. u64 tmp = perf_sample_period_ns;
  158. tmp *= sysctl_perf_cpu_time_max_percent;
  159. do_div(tmp, 100);
  160. atomic_set(&perf_sample_allowed_ns, tmp);
  161. }
  162. static int perf_rotate_context(struct perf_cpu_context *cpuctx);
  163. int perf_proc_update_handler(struct ctl_table *table, int write,
  164. void __user *buffer, size_t *lenp,
  165. loff_t *ppos)
  166. {
  167. int ret = proc_dointvec(table, write, buffer, lenp, ppos);
  168. if (ret || !write)
  169. return ret;
  170. max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
  171. perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
  172. update_perf_cpu_limits();
  173. return 0;
  174. }
  175. int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
  176. int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
  177. void __user *buffer, size_t *lenp,
  178. loff_t *ppos)
  179. {
  180. int ret = proc_dointvec(table, write, buffer, lenp, ppos);
  181. if (ret || !write)
  182. return ret;
  183. update_perf_cpu_limits();
  184. return 0;
  185. }
  186. /*
  187. * perf samples are done in some very critical code paths (NMIs).
  188. * If they take too much CPU time, the system can lock up and not
  189. * get any real work done. This will drop the sample rate when
  190. * we detect that events are taking too long.
  191. */
  192. #define NR_ACCUMULATED_SAMPLES 128
  193. DEFINE_PER_CPU(u64, running_sample_length);
  194. void perf_sample_event_took(u64 sample_len_ns)
  195. {
  196. u64 avg_local_sample_len;
  197. u64 local_samples_len;
  198. if (atomic_read(&perf_sample_allowed_ns) == 0)
  199. return;
  200. /* decay the counter by 1 average sample */
  201. local_samples_len = __get_cpu_var(running_sample_length);
  202. local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES;
  203. local_samples_len += sample_len_ns;
  204. __get_cpu_var(running_sample_length) = local_samples_len;
  205. /*
  206. * note: this will be biased artifically low until we have
  207. * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
  208. * from having to maintain a count.
  209. */
  210. avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
  211. if (avg_local_sample_len <= atomic_read(&perf_sample_allowed_ns))
  212. return;
  213. if (max_samples_per_tick <= 1)
  214. return;
  215. max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2);
  216. sysctl_perf_event_sample_rate = max_samples_per_tick * HZ;
  217. perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
  218. printk_ratelimited(KERN_WARNING
  219. "perf samples too long (%lld > %d), lowering "
  220. "kernel.perf_event_max_sample_rate to %d\n",
  221. avg_local_sample_len,
  222. atomic_read(&perf_sample_allowed_ns),
  223. sysctl_perf_event_sample_rate);
  224. update_perf_cpu_limits();
  225. }
  226. static atomic64_t perf_event_id;
  227. static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
  228. enum event_type_t event_type);
  229. static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
  230. enum event_type_t event_type,
  231. struct task_struct *task);
  232. static void update_context_time(struct perf_event_context *ctx);
  233. static u64 perf_event_time(struct perf_event *event);
  234. void __weak perf_event_print_debug(void) { }
  235. extern __weak const char *perf_pmu_name(void)
  236. {
  237. return "pmu";
  238. }
  239. static inline u64 perf_clock(void)
  240. {
  241. return local_clock();
  242. }
  243. static inline struct perf_cpu_context *
  244. __get_cpu_context(struct perf_event_context *ctx)
  245. {
  246. return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
  247. }
  248. static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
  249. struct perf_event_context *ctx)
  250. {
  251. raw_spin_lock(&cpuctx->ctx.lock);
  252. if (ctx)
  253. raw_spin_lock(&ctx->lock);
  254. }
  255. static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
  256. struct perf_event_context *ctx)
  257. {
  258. if (ctx)
  259. raw_spin_unlock(&ctx->lock);
  260. raw_spin_unlock(&cpuctx->ctx.lock);
  261. }
  262. #ifdef CONFIG_CGROUP_PERF
  263. /*
  264. * perf_cgroup_info keeps track of time_enabled for a cgroup.
  265. * This is a per-cpu dynamically allocated data structure.
  266. */
  267. struct perf_cgroup_info {
  268. u64 time;
  269. u64 timestamp;
  270. };
  271. struct perf_cgroup {
  272. struct cgroup_subsys_state css;
  273. struct perf_cgroup_info __percpu *info;
  274. };
  275. /*
  276. * Must ensure cgroup is pinned (css_get) before calling
  277. * this function. In other words, we cannot call this function
  278. * if there is no cgroup event for the current CPU context.
  279. */
  280. static inline struct perf_cgroup *
  281. perf_cgroup_from_task(struct task_struct *task)
  282. {
  283. return container_of(task_css(task, perf_subsys_id),
  284. struct perf_cgroup, css);
  285. }
  286. static inline bool
  287. perf_cgroup_match(struct perf_event *event)
  288. {
  289. struct perf_event_context *ctx = event->ctx;
  290. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  291. /* @event doesn't care about cgroup */
  292. if (!event->cgrp)
  293. return true;
  294. /* wants specific cgroup scope but @cpuctx isn't associated with any */
  295. if (!cpuctx->cgrp)
  296. return false;
  297. /*
  298. * Cgroup scoping is recursive. An event enabled for a cgroup is
  299. * also enabled for all its descendant cgroups. If @cpuctx's
  300. * cgroup is a descendant of @event's (the test covers identity
  301. * case), it's a match.
  302. */
  303. return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
  304. event->cgrp->css.cgroup);
  305. }
  306. static inline bool perf_tryget_cgroup(struct perf_event *event)
  307. {
  308. return css_tryget(&event->cgrp->css);
  309. }
  310. static inline void perf_put_cgroup(struct perf_event *event)
  311. {
  312. css_put(&event->cgrp->css);
  313. }
  314. static inline void perf_detach_cgroup(struct perf_event *event)
  315. {
  316. perf_put_cgroup(event);
  317. event->cgrp = NULL;
  318. }
  319. static inline int is_cgroup_event(struct perf_event *event)
  320. {
  321. return event->cgrp != NULL;
  322. }
  323. static inline u64 perf_cgroup_event_time(struct perf_event *event)
  324. {
  325. struct perf_cgroup_info *t;
  326. t = per_cpu_ptr(event->cgrp->info, event->cpu);
  327. return t->time;
  328. }
  329. static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
  330. {
  331. struct perf_cgroup_info *info;
  332. u64 now;
  333. now = perf_clock();
  334. info = this_cpu_ptr(cgrp->info);
  335. info->time += now - info->timestamp;
  336. info->timestamp = now;
  337. }
  338. static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
  339. {
  340. struct perf_cgroup *cgrp_out = cpuctx->cgrp;
  341. if (cgrp_out)
  342. __update_cgrp_time(cgrp_out);
  343. }
  344. static inline void update_cgrp_time_from_event(struct perf_event *event)
  345. {
  346. struct perf_cgroup *cgrp;
  347. /*
  348. * ensure we access cgroup data only when needed and
  349. * when we know the cgroup is pinned (css_get)
  350. */
  351. if (!is_cgroup_event(event))
  352. return;
  353. cgrp = perf_cgroup_from_task(current);
  354. /*
  355. * Do not update time when cgroup is not active
  356. */
  357. if (cgrp == event->cgrp)
  358. __update_cgrp_time(event->cgrp);
  359. }
  360. static inline void
  361. perf_cgroup_set_timestamp(struct task_struct *task,
  362. struct perf_event_context *ctx)
  363. {
  364. struct perf_cgroup *cgrp;
  365. struct perf_cgroup_info *info;
  366. /*
  367. * ctx->lock held by caller
  368. * ensure we do not access cgroup data
  369. * unless we have the cgroup pinned (css_get)
  370. */
  371. if (!task || !ctx->nr_cgroups)
  372. return;
  373. cgrp = perf_cgroup_from_task(task);
  374. info = this_cpu_ptr(cgrp->info);
  375. info->timestamp = ctx->timestamp;
  376. }
  377. #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */
  378. #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */
  379. /*
  380. * reschedule events based on the cgroup constraint of task.
  381. *
  382. * mode SWOUT : schedule out everything
  383. * mode SWIN : schedule in based on cgroup for next
  384. */
  385. void perf_cgroup_switch(struct task_struct *task, int mode)
  386. {
  387. struct perf_cpu_context *cpuctx;
  388. struct pmu *pmu;
  389. unsigned long flags;
  390. /*
  391. * disable interrupts to avoid geting nr_cgroup
  392. * changes via __perf_event_disable(). Also
  393. * avoids preemption.
  394. */
  395. local_irq_save(flags);
  396. /*
  397. * we reschedule only in the presence of cgroup
  398. * constrained events.
  399. */
  400. rcu_read_lock();
  401. list_for_each_entry_rcu(pmu, &pmus, entry) {
  402. cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  403. if (cpuctx->unique_pmu != pmu)
  404. continue; /* ensure we process each cpuctx once */
  405. /*
  406. * perf_cgroup_events says at least one
  407. * context on this CPU has cgroup events.
  408. *
  409. * ctx->nr_cgroups reports the number of cgroup
  410. * events for a context.
  411. */
  412. if (cpuctx->ctx.nr_cgroups > 0) {
  413. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  414. perf_pmu_disable(cpuctx->ctx.pmu);
  415. if (mode & PERF_CGROUP_SWOUT) {
  416. cpu_ctx_sched_out(cpuctx, EVENT_ALL);
  417. /*
  418. * must not be done before ctxswout due
  419. * to event_filter_match() in event_sched_out()
  420. */
  421. cpuctx->cgrp = NULL;
  422. }
  423. if (mode & PERF_CGROUP_SWIN) {
  424. WARN_ON_ONCE(cpuctx->cgrp);
  425. /*
  426. * set cgrp before ctxsw in to allow
  427. * event_filter_match() to not have to pass
  428. * task around
  429. */
  430. cpuctx->cgrp = perf_cgroup_from_task(task);
  431. cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
  432. }
  433. perf_pmu_enable(cpuctx->ctx.pmu);
  434. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  435. }
  436. }
  437. rcu_read_unlock();
  438. local_irq_restore(flags);
  439. }
  440. static inline void perf_cgroup_sched_out(struct task_struct *task,
  441. struct task_struct *next)
  442. {
  443. struct perf_cgroup *cgrp1;
  444. struct perf_cgroup *cgrp2 = NULL;
  445. /*
  446. * we come here when we know perf_cgroup_events > 0
  447. */
  448. cgrp1 = perf_cgroup_from_task(task);
  449. /*
  450. * next is NULL when called from perf_event_enable_on_exec()
  451. * that will systematically cause a cgroup_switch()
  452. */
  453. if (next)
  454. cgrp2 = perf_cgroup_from_task(next);
  455. /*
  456. * only schedule out current cgroup events if we know
  457. * that we are switching to a different cgroup. Otherwise,
  458. * do no touch the cgroup events.
  459. */
  460. if (cgrp1 != cgrp2)
  461. perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
  462. }
  463. static inline void perf_cgroup_sched_in(struct task_struct *prev,
  464. struct task_struct *task)
  465. {
  466. struct perf_cgroup *cgrp1;
  467. struct perf_cgroup *cgrp2 = NULL;
  468. /*
  469. * we come here when we know perf_cgroup_events > 0
  470. */
  471. cgrp1 = perf_cgroup_from_task(task);
  472. /* prev can never be NULL */
  473. cgrp2 = perf_cgroup_from_task(prev);
  474. /*
  475. * only need to schedule in cgroup events if we are changing
  476. * cgroup during ctxsw. Cgroup events were not scheduled
  477. * out of ctxsw out if that was not the case.
  478. */
  479. if (cgrp1 != cgrp2)
  480. perf_cgroup_switch(task, PERF_CGROUP_SWIN);
  481. }
  482. static inline int perf_cgroup_connect(int fd, struct perf_event *event,
  483. struct perf_event_attr *attr,
  484. struct perf_event *group_leader)
  485. {
  486. struct perf_cgroup *cgrp;
  487. struct cgroup_subsys_state *css;
  488. struct fd f = fdget(fd);
  489. int ret = 0;
  490. if (!f.file)
  491. return -EBADF;
  492. rcu_read_lock();
  493. css = css_from_dir(f.file->f_dentry, &perf_subsys);
  494. if (IS_ERR(css)) {
  495. ret = PTR_ERR(css);
  496. goto out;
  497. }
  498. cgrp = container_of(css, struct perf_cgroup, css);
  499. event->cgrp = cgrp;
  500. /* must be done before we fput() the file */
  501. if (!perf_tryget_cgroup(event)) {
  502. event->cgrp = NULL;
  503. ret = -ENOENT;
  504. goto out;
  505. }
  506. /*
  507. * all events in a group must monitor
  508. * the same cgroup because a task belongs
  509. * to only one perf cgroup at a time
  510. */
  511. if (group_leader && group_leader->cgrp != cgrp) {
  512. perf_detach_cgroup(event);
  513. ret = -EINVAL;
  514. }
  515. out:
  516. rcu_read_unlock();
  517. fdput(f);
  518. return ret;
  519. }
  520. static inline void
  521. perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
  522. {
  523. struct perf_cgroup_info *t;
  524. t = per_cpu_ptr(event->cgrp->info, event->cpu);
  525. event->shadow_ctx_time = now - t->timestamp;
  526. }
  527. static inline void
  528. perf_cgroup_defer_enabled(struct perf_event *event)
  529. {
  530. /*
  531. * when the current task's perf cgroup does not match
  532. * the event's, we need to remember to call the
  533. * perf_mark_enable() function the first time a task with
  534. * a matching perf cgroup is scheduled in.
  535. */
  536. if (is_cgroup_event(event) && !perf_cgroup_match(event))
  537. event->cgrp_defer_enabled = 1;
  538. }
  539. static inline void
  540. perf_cgroup_mark_enabled(struct perf_event *event,
  541. struct perf_event_context *ctx)
  542. {
  543. struct perf_event *sub;
  544. u64 tstamp = perf_event_time(event);
  545. if (!event->cgrp_defer_enabled)
  546. return;
  547. event->cgrp_defer_enabled = 0;
  548. event->tstamp_enabled = tstamp - event->total_time_enabled;
  549. list_for_each_entry(sub, &event->sibling_list, group_entry) {
  550. if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
  551. sub->tstamp_enabled = tstamp - sub->total_time_enabled;
  552. sub->cgrp_defer_enabled = 0;
  553. }
  554. }
  555. }
  556. #else /* !CONFIG_CGROUP_PERF */
  557. static inline bool
  558. perf_cgroup_match(struct perf_event *event)
  559. {
  560. return true;
  561. }
  562. static inline void perf_detach_cgroup(struct perf_event *event)
  563. {}
  564. static inline int is_cgroup_event(struct perf_event *event)
  565. {
  566. return 0;
  567. }
  568. static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
  569. {
  570. return 0;
  571. }
  572. static inline void update_cgrp_time_from_event(struct perf_event *event)
  573. {
  574. }
  575. static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
  576. {
  577. }
  578. static inline void perf_cgroup_sched_out(struct task_struct *task,
  579. struct task_struct *next)
  580. {
  581. }
  582. static inline void perf_cgroup_sched_in(struct task_struct *prev,
  583. struct task_struct *task)
  584. {
  585. }
  586. static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
  587. struct perf_event_attr *attr,
  588. struct perf_event *group_leader)
  589. {
  590. return -EINVAL;
  591. }
  592. static inline void
  593. perf_cgroup_set_timestamp(struct task_struct *task,
  594. struct perf_event_context *ctx)
  595. {
  596. }
  597. void
  598. perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
  599. {
  600. }
  601. static inline void
  602. perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
  603. {
  604. }
  605. static inline u64 perf_cgroup_event_time(struct perf_event *event)
  606. {
  607. return 0;
  608. }
  609. static inline void
  610. perf_cgroup_defer_enabled(struct perf_event *event)
  611. {
  612. }
  613. static inline void
  614. perf_cgroup_mark_enabled(struct perf_event *event,
  615. struct perf_event_context *ctx)
  616. {
  617. }
  618. #endif
  619. /*
  620. * set default to be dependent on timer tick just
  621. * like original code
  622. */
  623. #define PERF_CPU_HRTIMER (1000 / HZ)
  624. /*
  625. * function must be called with interrupts disbled
  626. */
  627. static enum hrtimer_restart perf_cpu_hrtimer_handler(struct hrtimer *hr)
  628. {
  629. struct perf_cpu_context *cpuctx;
  630. enum hrtimer_restart ret = HRTIMER_NORESTART;
  631. int rotations = 0;
  632. WARN_ON(!irqs_disabled());
  633. cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
  634. rotations = perf_rotate_context(cpuctx);
  635. /*
  636. * arm timer if needed
  637. */
  638. if (rotations) {
  639. hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
  640. ret = HRTIMER_RESTART;
  641. }
  642. return ret;
  643. }
  644. /* CPU is going down */
  645. void perf_cpu_hrtimer_cancel(int cpu)
  646. {
  647. struct perf_cpu_context *cpuctx;
  648. struct pmu *pmu;
  649. unsigned long flags;
  650. if (WARN_ON(cpu != smp_processor_id()))
  651. return;
  652. local_irq_save(flags);
  653. rcu_read_lock();
  654. list_for_each_entry_rcu(pmu, &pmus, entry) {
  655. cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  656. if (pmu->task_ctx_nr == perf_sw_context)
  657. continue;
  658. hrtimer_cancel(&cpuctx->hrtimer);
  659. }
  660. rcu_read_unlock();
  661. local_irq_restore(flags);
  662. }
  663. static void __perf_cpu_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
  664. {
  665. struct hrtimer *hr = &cpuctx->hrtimer;
  666. struct pmu *pmu = cpuctx->ctx.pmu;
  667. int timer;
  668. /* no multiplexing needed for SW PMU */
  669. if (pmu->task_ctx_nr == perf_sw_context)
  670. return;
  671. /*
  672. * check default is sane, if not set then force to
  673. * default interval (1/tick)
  674. */
  675. timer = pmu->hrtimer_interval_ms;
  676. if (timer < 1)
  677. timer = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
  678. cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
  679. hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
  680. hr->function = perf_cpu_hrtimer_handler;
  681. }
  682. static void perf_cpu_hrtimer_restart(struct perf_cpu_context *cpuctx)
  683. {
  684. struct hrtimer *hr = &cpuctx->hrtimer;
  685. struct pmu *pmu = cpuctx->ctx.pmu;
  686. /* not for SW PMU */
  687. if (pmu->task_ctx_nr == perf_sw_context)
  688. return;
  689. if (hrtimer_active(hr))
  690. return;
  691. if (!hrtimer_callback_running(hr))
  692. __hrtimer_start_range_ns(hr, cpuctx->hrtimer_interval,
  693. 0, HRTIMER_MODE_REL_PINNED, 0);
  694. }
  695. void perf_pmu_disable(struct pmu *pmu)
  696. {
  697. int *count = this_cpu_ptr(pmu->pmu_disable_count);
  698. if (!(*count)++)
  699. pmu->pmu_disable(pmu);
  700. }
  701. void perf_pmu_enable(struct pmu *pmu)
  702. {
  703. int *count = this_cpu_ptr(pmu->pmu_disable_count);
  704. if (!--(*count))
  705. pmu->pmu_enable(pmu);
  706. }
  707. static DEFINE_PER_CPU(struct list_head, rotation_list);
  708. /*
  709. * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
  710. * because they're strictly cpu affine and rotate_start is called with IRQs
  711. * disabled, while rotate_context is called from IRQ context.
  712. */
  713. static void perf_pmu_rotate_start(struct pmu *pmu)
  714. {
  715. struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  716. struct list_head *head = &__get_cpu_var(rotation_list);
  717. WARN_ON(!irqs_disabled());
  718. if (list_empty(&cpuctx->rotation_list))
  719. list_add(&cpuctx->rotation_list, head);
  720. }
  721. static void get_ctx(struct perf_event_context *ctx)
  722. {
  723. WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
  724. }
  725. static void put_ctx(struct perf_event_context *ctx)
  726. {
  727. if (atomic_dec_and_test(&ctx->refcount)) {
  728. if (ctx->parent_ctx)
  729. put_ctx(ctx->parent_ctx);
  730. if (ctx->task)
  731. put_task_struct(ctx->task);
  732. kfree_rcu(ctx, rcu_head);
  733. }
  734. }
  735. static void unclone_ctx(struct perf_event_context *ctx)
  736. {
  737. if (ctx->parent_ctx) {
  738. put_ctx(ctx->parent_ctx);
  739. ctx->parent_ctx = NULL;
  740. }
  741. }
  742. static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
  743. {
  744. /*
  745. * only top level events have the pid namespace they were created in
  746. */
  747. if (event->parent)
  748. event = event->parent;
  749. return task_tgid_nr_ns(p, event->ns);
  750. }
  751. static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
  752. {
  753. /*
  754. * only top level events have the pid namespace they were created in
  755. */
  756. if (event->parent)
  757. event = event->parent;
  758. return task_pid_nr_ns(p, event->ns);
  759. }
  760. /*
  761. * If we inherit events we want to return the parent event id
  762. * to userspace.
  763. */
  764. static u64 primary_event_id(struct perf_event *event)
  765. {
  766. u64 id = event->id;
  767. if (event->parent)
  768. id = event->parent->id;
  769. return id;
  770. }
  771. /*
  772. * Get the perf_event_context for a task and lock it.
  773. * This has to cope with with the fact that until it is locked,
  774. * the context could get moved to another task.
  775. */
  776. static struct perf_event_context *
  777. perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
  778. {
  779. struct perf_event_context *ctx;
  780. retry:
  781. /*
  782. * One of the few rules of preemptible RCU is that one cannot do
  783. * rcu_read_unlock() while holding a scheduler (or nested) lock when
  784. * part of the read side critical section was preemptible -- see
  785. * rcu_read_unlock_special().
  786. *
  787. * Since ctx->lock nests under rq->lock we must ensure the entire read
  788. * side critical section is non-preemptible.
  789. */
  790. preempt_disable();
  791. rcu_read_lock();
  792. ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
  793. if (ctx) {
  794. /*
  795. * If this context is a clone of another, it might
  796. * get swapped for another underneath us by
  797. * perf_event_task_sched_out, though the
  798. * rcu_read_lock() protects us from any context
  799. * getting freed. Lock the context and check if it
  800. * got swapped before we could get the lock, and retry
  801. * if so. If we locked the right context, then it
  802. * can't get swapped on us any more.
  803. */
  804. raw_spin_lock_irqsave(&ctx->lock, *flags);
  805. if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
  806. raw_spin_unlock_irqrestore(&ctx->lock, *flags);
  807. rcu_read_unlock();
  808. preempt_enable();
  809. goto retry;
  810. }
  811. if (!atomic_inc_not_zero(&ctx->refcount)) {
  812. raw_spin_unlock_irqrestore(&ctx->lock, *flags);
  813. ctx = NULL;
  814. }
  815. }
  816. rcu_read_unlock();
  817. preempt_enable();
  818. return ctx;
  819. }
  820. /*
  821. * Get the context for a task and increment its pin_count so it
  822. * can't get swapped to another task. This also increments its
  823. * reference count so that the context can't get freed.
  824. */
  825. static struct perf_event_context *
  826. perf_pin_task_context(struct task_struct *task, int ctxn)
  827. {
  828. struct perf_event_context *ctx;
  829. unsigned long flags;
  830. ctx = perf_lock_task_context(task, ctxn, &flags);
  831. if (ctx) {
  832. ++ctx->pin_count;
  833. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  834. }
  835. return ctx;
  836. }
  837. static void perf_unpin_context(struct perf_event_context *ctx)
  838. {
  839. unsigned long flags;
  840. raw_spin_lock_irqsave(&ctx->lock, flags);
  841. --ctx->pin_count;
  842. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  843. }
  844. /*
  845. * Update the record of the current time in a context.
  846. */
  847. static void update_context_time(struct perf_event_context *ctx)
  848. {
  849. u64 now = perf_clock();
  850. ctx->time += now - ctx->timestamp;
  851. ctx->timestamp = now;
  852. }
  853. static u64 perf_event_time(struct perf_event *event)
  854. {
  855. struct perf_event_context *ctx = event->ctx;
  856. if (is_cgroup_event(event))
  857. return perf_cgroup_event_time(event);
  858. return ctx ? ctx->time : 0;
  859. }
  860. /*
  861. * Update the total_time_enabled and total_time_running fields for a event.
  862. * The caller of this function needs to hold the ctx->lock.
  863. */
  864. static void update_event_times(struct perf_event *event)
  865. {
  866. struct perf_event_context *ctx = event->ctx;
  867. u64 run_end;
  868. if (event->state < PERF_EVENT_STATE_INACTIVE ||
  869. event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
  870. return;
  871. /*
  872. * in cgroup mode, time_enabled represents
  873. * the time the event was enabled AND active
  874. * tasks were in the monitored cgroup. This is
  875. * independent of the activity of the context as
  876. * there may be a mix of cgroup and non-cgroup events.
  877. *
  878. * That is why we treat cgroup events differently
  879. * here.
  880. */
  881. if (is_cgroup_event(event))
  882. run_end = perf_cgroup_event_time(event);
  883. else if (ctx->is_active)
  884. run_end = ctx->time;
  885. else
  886. run_end = event->tstamp_stopped;
  887. event->total_time_enabled = run_end - event->tstamp_enabled;
  888. if (event->state == PERF_EVENT_STATE_INACTIVE)
  889. run_end = event->tstamp_stopped;
  890. else
  891. run_end = perf_event_time(event);
  892. event->total_time_running = run_end - event->tstamp_running;
  893. }
  894. /*
  895. * Update total_time_enabled and total_time_running for all events in a group.
  896. */
  897. static void update_group_times(struct perf_event *leader)
  898. {
  899. struct perf_event *event;
  900. update_event_times(leader);
  901. list_for_each_entry(event, &leader->sibling_list, group_entry)
  902. update_event_times(event);
  903. }
  904. static struct list_head *
  905. ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
  906. {
  907. if (event->attr.pinned)
  908. return &ctx->pinned_groups;
  909. else
  910. return &ctx->flexible_groups;
  911. }
  912. /*
  913. * Add a event from the lists for its context.
  914. * Must be called with ctx->mutex and ctx->lock held.
  915. */
  916. static void
  917. list_add_event(struct perf_event *event, struct perf_event_context *ctx)
  918. {
  919. WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
  920. event->attach_state |= PERF_ATTACH_CONTEXT;
  921. /*
  922. * If we're a stand alone event or group leader, we go to the context
  923. * list, group events are kept attached to the group so that
  924. * perf_group_detach can, at all times, locate all siblings.
  925. */
  926. if (event->group_leader == event) {
  927. struct list_head *list;
  928. if (is_software_event(event))
  929. event->group_flags |= PERF_GROUP_SOFTWARE;
  930. list = ctx_group_list(event, ctx);
  931. list_add_tail(&event->group_entry, list);
  932. }
  933. if (is_cgroup_event(event))
  934. ctx->nr_cgroups++;
  935. if (has_branch_stack(event))
  936. ctx->nr_branch_stack++;
  937. list_add_rcu(&event->event_entry, &ctx->event_list);
  938. if (!ctx->nr_events)
  939. perf_pmu_rotate_start(ctx->pmu);
  940. ctx->nr_events++;
  941. if (event->attr.inherit_stat)
  942. ctx->nr_stat++;
  943. }
  944. /*
  945. * Initialize event state based on the perf_event_attr::disabled.
  946. */
  947. static inline void perf_event__state_init(struct perf_event *event)
  948. {
  949. event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
  950. PERF_EVENT_STATE_INACTIVE;
  951. }
  952. /*
  953. * Called at perf_event creation and when events are attached/detached from a
  954. * group.
  955. */
  956. static void perf_event__read_size(struct perf_event *event)
  957. {
  958. int entry = sizeof(u64); /* value */
  959. int size = 0;
  960. int nr = 1;
  961. if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  962. size += sizeof(u64);
  963. if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  964. size += sizeof(u64);
  965. if (event->attr.read_format & PERF_FORMAT_ID)
  966. entry += sizeof(u64);
  967. if (event->attr.read_format & PERF_FORMAT_GROUP) {
  968. nr += event->group_leader->nr_siblings;
  969. size += sizeof(u64);
  970. }
  971. size += entry * nr;
  972. event->read_size = size;
  973. }
  974. static void perf_event__header_size(struct perf_event *event)
  975. {
  976. struct perf_sample_data *data;
  977. u64 sample_type = event->attr.sample_type;
  978. u16 size = 0;
  979. perf_event__read_size(event);
  980. if (sample_type & PERF_SAMPLE_IP)
  981. size += sizeof(data->ip);
  982. if (sample_type & PERF_SAMPLE_ADDR)
  983. size += sizeof(data->addr);
  984. if (sample_type & PERF_SAMPLE_PERIOD)
  985. size += sizeof(data->period);
  986. if (sample_type & PERF_SAMPLE_WEIGHT)
  987. size += sizeof(data->weight);
  988. if (sample_type & PERF_SAMPLE_READ)
  989. size += event->read_size;
  990. if (sample_type & PERF_SAMPLE_DATA_SRC)
  991. size += sizeof(data->data_src.val);
  992. event->header_size = size;
  993. }
  994. static void perf_event__id_header_size(struct perf_event *event)
  995. {
  996. struct perf_sample_data *data;
  997. u64 sample_type = event->attr.sample_type;
  998. u16 size = 0;
  999. if (sample_type & PERF_SAMPLE_TID)
  1000. size += sizeof(data->tid_entry);
  1001. if (sample_type & PERF_SAMPLE_TIME)
  1002. size += sizeof(data->time);
  1003. if (sample_type & PERF_SAMPLE_IDENTIFIER)
  1004. size += sizeof(data->id);
  1005. if (sample_type & PERF_SAMPLE_ID)
  1006. size += sizeof(data->id);
  1007. if (sample_type & PERF_SAMPLE_STREAM_ID)
  1008. size += sizeof(data->stream_id);
  1009. if (sample_type & PERF_SAMPLE_CPU)
  1010. size += sizeof(data->cpu_entry);
  1011. event->id_header_size = size;
  1012. }
  1013. static void perf_group_attach(struct perf_event *event)
  1014. {
  1015. struct perf_event *group_leader = event->group_leader, *pos;
  1016. /*
  1017. * We can have double attach due to group movement in perf_event_open.
  1018. */
  1019. if (event->attach_state & PERF_ATTACH_GROUP)
  1020. return;
  1021. event->attach_state |= PERF_ATTACH_GROUP;
  1022. if (group_leader == event)
  1023. return;
  1024. if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
  1025. !is_software_event(event))
  1026. group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
  1027. list_add_tail(&event->group_entry, &group_leader->sibling_list);
  1028. group_leader->nr_siblings++;
  1029. perf_event__header_size(group_leader);
  1030. list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
  1031. perf_event__header_size(pos);
  1032. }
  1033. /*
  1034. * Remove a event from the lists for its context.
  1035. * Must be called with ctx->mutex and ctx->lock held.
  1036. */
  1037. static void
  1038. list_del_event(struct perf_event *event, struct perf_event_context *ctx)
  1039. {
  1040. struct perf_cpu_context *cpuctx;
  1041. /*
  1042. * We can have double detach due to exit/hot-unplug + close.
  1043. */
  1044. if (!(event->attach_state & PERF_ATTACH_CONTEXT))
  1045. return;
  1046. event->attach_state &= ~PERF_ATTACH_CONTEXT;
  1047. if (is_cgroup_event(event)) {
  1048. ctx->nr_cgroups--;
  1049. cpuctx = __get_cpu_context(ctx);
  1050. /*
  1051. * if there are no more cgroup events
  1052. * then cler cgrp to avoid stale pointer
  1053. * in update_cgrp_time_from_cpuctx()
  1054. */
  1055. if (!ctx->nr_cgroups)
  1056. cpuctx->cgrp = NULL;
  1057. }
  1058. if (has_branch_stack(event))
  1059. ctx->nr_branch_stack--;
  1060. ctx->nr_events--;
  1061. if (event->attr.inherit_stat)
  1062. ctx->nr_stat--;
  1063. list_del_rcu(&event->event_entry);
  1064. if (event->group_leader == event)
  1065. list_del_init(&event->group_entry);
  1066. update_group_times(event);
  1067. /*
  1068. * If event was in error state, then keep it
  1069. * that way, otherwise bogus counts will be
  1070. * returned on read(). The only way to get out
  1071. * of error state is by explicit re-enabling
  1072. * of the event
  1073. */
  1074. if (event->state > PERF_EVENT_STATE_OFF)
  1075. event->state = PERF_EVENT_STATE_OFF;
  1076. }
  1077. static void perf_group_detach(struct perf_event *event)
  1078. {
  1079. struct perf_event *sibling, *tmp;
  1080. struct list_head *list = NULL;
  1081. /*
  1082. * We can have double detach due to exit/hot-unplug + close.
  1083. */
  1084. if (!(event->attach_state & PERF_ATTACH_GROUP))
  1085. return;
  1086. event->attach_state &= ~PERF_ATTACH_GROUP;
  1087. /*
  1088. * If this is a sibling, remove it from its group.
  1089. */
  1090. if (event->group_leader != event) {
  1091. list_del_init(&event->group_entry);
  1092. event->group_leader->nr_siblings--;
  1093. goto out;
  1094. }
  1095. if (!list_empty(&event->group_entry))
  1096. list = &event->group_entry;
  1097. /*
  1098. * If this was a group event with sibling events then
  1099. * upgrade the siblings to singleton events by adding them
  1100. * to whatever list we are on.
  1101. */
  1102. list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
  1103. if (list)
  1104. list_move_tail(&sibling->group_entry, list);
  1105. sibling->group_leader = sibling;
  1106. /* Inherit group flags from the previous leader */
  1107. sibling->group_flags = event->group_flags;
  1108. }
  1109. out:
  1110. perf_event__header_size(event->group_leader);
  1111. list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
  1112. perf_event__header_size(tmp);
  1113. }
  1114. static inline int
  1115. event_filter_match(struct perf_event *event)
  1116. {
  1117. return (event->cpu == -1 || event->cpu == smp_processor_id())
  1118. && perf_cgroup_match(event);
  1119. }
  1120. static void
  1121. event_sched_out(struct perf_event *event,
  1122. struct perf_cpu_context *cpuctx,
  1123. struct perf_event_context *ctx)
  1124. {
  1125. u64 tstamp = perf_event_time(event);
  1126. u64 delta;
  1127. /*
  1128. * An event which could not be activated because of
  1129. * filter mismatch still needs to have its timings
  1130. * maintained, otherwise bogus information is return
  1131. * via read() for time_enabled, time_running:
  1132. */
  1133. if (event->state == PERF_EVENT_STATE_INACTIVE
  1134. && !event_filter_match(event)) {
  1135. delta = tstamp - event->tstamp_stopped;
  1136. event->tstamp_running += delta;
  1137. event->tstamp_stopped = tstamp;
  1138. }
  1139. if (event->state != PERF_EVENT_STATE_ACTIVE)
  1140. return;
  1141. event->state = PERF_EVENT_STATE_INACTIVE;
  1142. if (event->pending_disable) {
  1143. event->pending_disable = 0;
  1144. event->state = PERF_EVENT_STATE_OFF;
  1145. }
  1146. event->tstamp_stopped = tstamp;
  1147. event->pmu->del(event, 0);
  1148. event->oncpu = -1;
  1149. if (!is_software_event(event))
  1150. cpuctx->active_oncpu--;
  1151. ctx->nr_active--;
  1152. if (event->attr.freq && event->attr.sample_freq)
  1153. ctx->nr_freq--;
  1154. if (event->attr.exclusive || !cpuctx->active_oncpu)
  1155. cpuctx->exclusive = 0;
  1156. }
  1157. static void
  1158. group_sched_out(struct perf_event *group_event,
  1159. struct perf_cpu_context *cpuctx,
  1160. struct perf_event_context *ctx)
  1161. {
  1162. struct perf_event *event;
  1163. int state = group_event->state;
  1164. event_sched_out(group_event, cpuctx, ctx);
  1165. /*
  1166. * Schedule out siblings (if any):
  1167. */
  1168. list_for_each_entry(event, &group_event->sibling_list, group_entry)
  1169. event_sched_out(event, cpuctx, ctx);
  1170. if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
  1171. cpuctx->exclusive = 0;
  1172. }
  1173. /*
  1174. * Cross CPU call to remove a performance event
  1175. *
  1176. * We disable the event on the hardware level first. After that we
  1177. * remove it from the context list.
  1178. */
  1179. static int __perf_remove_from_context(void *info)
  1180. {
  1181. struct perf_event *event = info;
  1182. struct perf_event_context *ctx = event->ctx;
  1183. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1184. raw_spin_lock(&ctx->lock);
  1185. event_sched_out(event, cpuctx, ctx);
  1186. list_del_event(event, ctx);
  1187. if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
  1188. ctx->is_active = 0;
  1189. cpuctx->task_ctx = NULL;
  1190. }
  1191. raw_spin_unlock(&ctx->lock);
  1192. return 0;
  1193. }
  1194. /*
  1195. * Remove the event from a task's (or a CPU's) list of events.
  1196. *
  1197. * CPU events are removed with a smp call. For task events we only
  1198. * call when the task is on a CPU.
  1199. *
  1200. * If event->ctx is a cloned context, callers must make sure that
  1201. * every task struct that event->ctx->task could possibly point to
  1202. * remains valid. This is OK when called from perf_release since
  1203. * that only calls us on the top-level context, which can't be a clone.
  1204. * When called from perf_event_exit_task, it's OK because the
  1205. * context has been detached from its task.
  1206. */
  1207. static void perf_remove_from_context(struct perf_event *event)
  1208. {
  1209. struct perf_event_context *ctx = event->ctx;
  1210. struct task_struct *task = ctx->task;
  1211. lockdep_assert_held(&ctx->mutex);
  1212. if (!task) {
  1213. /*
  1214. * Per cpu events are removed via an smp call and
  1215. * the removal is always successful.
  1216. */
  1217. cpu_function_call(event->cpu, __perf_remove_from_context, event);
  1218. return;
  1219. }
  1220. retry:
  1221. if (!task_function_call(task, __perf_remove_from_context, event))
  1222. return;
  1223. raw_spin_lock_irq(&ctx->lock);
  1224. /*
  1225. * If we failed to find a running task, but find the context active now
  1226. * that we've acquired the ctx->lock, retry.
  1227. */
  1228. if (ctx->is_active) {
  1229. raw_spin_unlock_irq(&ctx->lock);
  1230. goto retry;
  1231. }
  1232. /*
  1233. * Since the task isn't running, its safe to remove the event, us
  1234. * holding the ctx->lock ensures the task won't get scheduled in.
  1235. */
  1236. list_del_event(event, ctx);
  1237. raw_spin_unlock_irq(&ctx->lock);
  1238. }
  1239. /*
  1240. * Cross CPU call to disable a performance event
  1241. */
  1242. int __perf_event_disable(void *info)
  1243. {
  1244. struct perf_event *event = info;
  1245. struct perf_event_context *ctx = event->ctx;
  1246. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1247. /*
  1248. * If this is a per-task event, need to check whether this
  1249. * event's task is the current task on this cpu.
  1250. *
  1251. * Can trigger due to concurrent perf_event_context_sched_out()
  1252. * flipping contexts around.
  1253. */
  1254. if (ctx->task && cpuctx->task_ctx != ctx)
  1255. return -EINVAL;
  1256. raw_spin_lock(&ctx->lock);
  1257. /*
  1258. * If the event is on, turn it off.
  1259. * If it is in error state, leave it in error state.
  1260. */
  1261. if (event->state >= PERF_EVENT_STATE_INACTIVE) {
  1262. update_context_time(ctx);
  1263. update_cgrp_time_from_event(event);
  1264. update_group_times(event);
  1265. if (event == event->group_leader)
  1266. group_sched_out(event, cpuctx, ctx);
  1267. else
  1268. event_sched_out(event, cpuctx, ctx);
  1269. event->state = PERF_EVENT_STATE_OFF;
  1270. }
  1271. raw_spin_unlock(&ctx->lock);
  1272. return 0;
  1273. }
  1274. /*
  1275. * Disable a event.
  1276. *
  1277. * If event->ctx is a cloned context, callers must make sure that
  1278. * every task struct that event->ctx->task could possibly point to
  1279. * remains valid. This condition is satisifed when called through
  1280. * perf_event_for_each_child or perf_event_for_each because they
  1281. * hold the top-level event's child_mutex, so any descendant that
  1282. * goes to exit will block in sync_child_event.
  1283. * When called from perf_pending_event it's OK because event->ctx
  1284. * is the current context on this CPU and preemption is disabled,
  1285. * hence we can't get into perf_event_task_sched_out for this context.
  1286. */
  1287. void perf_event_disable(struct perf_event *event)
  1288. {
  1289. struct perf_event_context *ctx = event->ctx;
  1290. struct task_struct *task = ctx->task;
  1291. if (!task) {
  1292. /*
  1293. * Disable the event on the cpu that it's on
  1294. */
  1295. cpu_function_call(event->cpu, __perf_event_disable, event);
  1296. return;
  1297. }
  1298. retry:
  1299. if (!task_function_call(task, __perf_event_disable, event))
  1300. return;
  1301. raw_spin_lock_irq(&ctx->lock);
  1302. /*
  1303. * If the event is still active, we need to retry the cross-call.
  1304. */
  1305. if (event->state == PERF_EVENT_STATE_ACTIVE) {
  1306. raw_spin_unlock_irq(&ctx->lock);
  1307. /*
  1308. * Reload the task pointer, it might have been changed by
  1309. * a concurrent perf_event_context_sched_out().
  1310. */
  1311. task = ctx->task;
  1312. goto retry;
  1313. }
  1314. /*
  1315. * Since we have the lock this context can't be scheduled
  1316. * in, so we can change the state safely.
  1317. */
  1318. if (event->state == PERF_EVENT_STATE_INACTIVE) {
  1319. update_group_times(event);
  1320. event->state = PERF_EVENT_STATE_OFF;
  1321. }
  1322. raw_spin_unlock_irq(&ctx->lock);
  1323. }
  1324. EXPORT_SYMBOL_GPL(perf_event_disable);
  1325. static void perf_set_shadow_time(struct perf_event *event,
  1326. struct perf_event_context *ctx,
  1327. u64 tstamp)
  1328. {
  1329. /*
  1330. * use the correct time source for the time snapshot
  1331. *
  1332. * We could get by without this by leveraging the
  1333. * fact that to get to this function, the caller
  1334. * has most likely already called update_context_time()
  1335. * and update_cgrp_time_xx() and thus both timestamp
  1336. * are identical (or very close). Given that tstamp is,
  1337. * already adjusted for cgroup, we could say that:
  1338. * tstamp - ctx->timestamp
  1339. * is equivalent to
  1340. * tstamp - cgrp->timestamp.
  1341. *
  1342. * Then, in perf_output_read(), the calculation would
  1343. * work with no changes because:
  1344. * - event is guaranteed scheduled in
  1345. * - no scheduled out in between
  1346. * - thus the timestamp would be the same
  1347. *
  1348. * But this is a bit hairy.
  1349. *
  1350. * So instead, we have an explicit cgroup call to remain
  1351. * within the time time source all along. We believe it
  1352. * is cleaner and simpler to understand.
  1353. */
  1354. if (is_cgroup_event(event))
  1355. perf_cgroup_set_shadow_time(event, tstamp);
  1356. else
  1357. event->shadow_ctx_time = tstamp - ctx->timestamp;
  1358. }
  1359. #define MAX_INTERRUPTS (~0ULL)
  1360. static void perf_log_throttle(struct perf_event *event, int enable);
  1361. static int
  1362. event_sched_in(struct perf_event *event,
  1363. struct perf_cpu_context *cpuctx,
  1364. struct perf_event_context *ctx)
  1365. {
  1366. u64 tstamp = perf_event_time(event);
  1367. if (event->state <= PERF_EVENT_STATE_OFF)
  1368. return 0;
  1369. event->state = PERF_EVENT_STATE_ACTIVE;
  1370. event->oncpu = smp_processor_id();
  1371. /*
  1372. * Unthrottle events, since we scheduled we might have missed several
  1373. * ticks already, also for a heavily scheduling task there is little
  1374. * guarantee it'll get a tick in a timely manner.
  1375. */
  1376. if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
  1377. perf_log_throttle(event, 1);
  1378. event->hw.interrupts = 0;
  1379. }
  1380. /*
  1381. * The new state must be visible before we turn it on in the hardware:
  1382. */
  1383. smp_wmb();
  1384. if (event->pmu->add(event, PERF_EF_START)) {
  1385. event->state = PERF_EVENT_STATE_INACTIVE;
  1386. event->oncpu = -1;
  1387. return -EAGAIN;
  1388. }
  1389. event->tstamp_running += tstamp - event->tstamp_stopped;
  1390. perf_set_shadow_time(event, ctx, tstamp);
  1391. if (!is_software_event(event))
  1392. cpuctx->active_oncpu++;
  1393. ctx->nr_active++;
  1394. if (event->attr.freq && event->attr.sample_freq)
  1395. ctx->nr_freq++;
  1396. if (event->attr.exclusive)
  1397. cpuctx->exclusive = 1;
  1398. return 0;
  1399. }
  1400. static int
  1401. group_sched_in(struct perf_event *group_event,
  1402. struct perf_cpu_context *cpuctx,
  1403. struct perf_event_context *ctx)
  1404. {
  1405. struct perf_event *event, *partial_group = NULL;
  1406. struct pmu *pmu = group_event->pmu;
  1407. u64 now = ctx->time;
  1408. bool simulate = false;
  1409. if (group_event->state == PERF_EVENT_STATE_OFF)
  1410. return 0;
  1411. pmu->start_txn(pmu);
  1412. if (event_sched_in(group_event, cpuctx, ctx)) {
  1413. pmu->cancel_txn(pmu);
  1414. perf_cpu_hrtimer_restart(cpuctx);
  1415. return -EAGAIN;
  1416. }
  1417. /*
  1418. * Schedule in siblings as one group (if any):
  1419. */
  1420. list_for_each_entry(event, &group_event->sibling_list, group_entry) {
  1421. if (event_sched_in(event, cpuctx, ctx)) {
  1422. partial_group = event;
  1423. goto group_error;
  1424. }
  1425. }
  1426. if (!pmu->commit_txn(pmu))
  1427. return 0;
  1428. group_error:
  1429. /*
  1430. * Groups can be scheduled in as one unit only, so undo any
  1431. * partial group before returning:
  1432. * The events up to the failed event are scheduled out normally,
  1433. * tstamp_stopped will be updated.
  1434. *
  1435. * The failed events and the remaining siblings need to have
  1436. * their timings updated as if they had gone thru event_sched_in()
  1437. * and event_sched_out(). This is required to get consistent timings
  1438. * across the group. This also takes care of the case where the group
  1439. * could never be scheduled by ensuring tstamp_stopped is set to mark
  1440. * the time the event was actually stopped, such that time delta
  1441. * calculation in update_event_times() is correct.
  1442. */
  1443. list_for_each_entry(event, &group_event->sibling_list, group_entry) {
  1444. if (event == partial_group)
  1445. simulate = true;
  1446. if (simulate) {
  1447. event->tstamp_running += now - event->tstamp_stopped;
  1448. event->tstamp_stopped = now;
  1449. } else {
  1450. event_sched_out(event, cpuctx, ctx);
  1451. }
  1452. }
  1453. event_sched_out(group_event, cpuctx, ctx);
  1454. pmu->cancel_txn(pmu);
  1455. perf_cpu_hrtimer_restart(cpuctx);
  1456. return -EAGAIN;
  1457. }
  1458. /*
  1459. * Work out whether we can put this event group on the CPU now.
  1460. */
  1461. static int group_can_go_on(struct perf_event *event,
  1462. struct perf_cpu_context *cpuctx,
  1463. int can_add_hw)
  1464. {
  1465. /*
  1466. * Groups consisting entirely of software events can always go on.
  1467. */
  1468. if (event->group_flags & PERF_GROUP_SOFTWARE)
  1469. return 1;
  1470. /*
  1471. * If an exclusive group is already on, no other hardware
  1472. * events can go on.
  1473. */
  1474. if (cpuctx->exclusive)
  1475. return 0;
  1476. /*
  1477. * If this group is exclusive and there are already
  1478. * events on the CPU, it can't go on.
  1479. */
  1480. if (event->attr.exclusive && cpuctx->active_oncpu)
  1481. return 0;
  1482. /*
  1483. * Otherwise, try to add it if all previous groups were able
  1484. * to go on.
  1485. */
  1486. return can_add_hw;
  1487. }
  1488. static void add_event_to_ctx(struct perf_event *event,
  1489. struct perf_event_context *ctx)
  1490. {
  1491. u64 tstamp = perf_event_time(event);
  1492. list_add_event(event, ctx);
  1493. perf_group_attach(event);
  1494. event->tstamp_enabled = tstamp;
  1495. event->tstamp_running = tstamp;
  1496. event->tstamp_stopped = tstamp;
  1497. }
  1498. static void task_ctx_sched_out(struct perf_event_context *ctx);
  1499. static void
  1500. ctx_sched_in(struct perf_event_context *ctx,
  1501. struct perf_cpu_context *cpuctx,
  1502. enum event_type_t event_type,
  1503. struct task_struct *task);
  1504. static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
  1505. struct perf_event_context *ctx,
  1506. struct task_struct *task)
  1507. {
  1508. cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
  1509. if (ctx)
  1510. ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
  1511. cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
  1512. if (ctx)
  1513. ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
  1514. }
  1515. /*
  1516. * Cross CPU call to install and enable a performance event
  1517. *
  1518. * Must be called with ctx->mutex held
  1519. */
  1520. static int __perf_install_in_context(void *info)
  1521. {
  1522. struct perf_event *event = info;
  1523. struct perf_event_context *ctx = event->ctx;
  1524. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1525. struct perf_event_context *task_ctx = cpuctx->task_ctx;
  1526. struct task_struct *task = current;
  1527. perf_ctx_lock(cpuctx, task_ctx);
  1528. perf_pmu_disable(cpuctx->ctx.pmu);
  1529. /*
  1530. * If there was an active task_ctx schedule it out.
  1531. */
  1532. if (task_ctx)
  1533. task_ctx_sched_out(task_ctx);
  1534. /*
  1535. * If the context we're installing events in is not the
  1536. * active task_ctx, flip them.
  1537. */
  1538. if (ctx->task && task_ctx != ctx) {
  1539. if (task_ctx)
  1540. raw_spin_unlock(&task_ctx->lock);
  1541. raw_spin_lock(&ctx->lock);
  1542. task_ctx = ctx;
  1543. }
  1544. if (task_ctx) {
  1545. cpuctx->task_ctx = task_ctx;
  1546. task = task_ctx->task;
  1547. }
  1548. cpu_ctx_sched_out(cpuctx, EVENT_ALL);
  1549. update_context_time(ctx);
  1550. /*
  1551. * update cgrp time only if current cgrp
  1552. * matches event->cgrp. Must be done before
  1553. * calling add_event_to_ctx()
  1554. */
  1555. update_cgrp_time_from_event(event);
  1556. add_event_to_ctx(event, ctx);
  1557. /*
  1558. * Schedule everything back in
  1559. */
  1560. perf_event_sched_in(cpuctx, task_ctx, task);
  1561. perf_pmu_enable(cpuctx->ctx.pmu);
  1562. perf_ctx_unlock(cpuctx, task_ctx);
  1563. return 0;
  1564. }
  1565. /*
  1566. * Attach a performance event to a context
  1567. *
  1568. * First we add the event to the list with the hardware enable bit
  1569. * in event->hw_config cleared.
  1570. *
  1571. * If the event is attached to a task which is on a CPU we use a smp
  1572. * call to enable it in the task context. The task might have been
  1573. * scheduled away, but we check this in the smp call again.
  1574. */
  1575. static void
  1576. perf_install_in_context(struct perf_event_context *ctx,
  1577. struct perf_event *event,
  1578. int cpu)
  1579. {
  1580. struct task_struct *task = ctx->task;
  1581. lockdep_assert_held(&ctx->mutex);
  1582. event->ctx = ctx;
  1583. if (event->cpu != -1)
  1584. event->cpu = cpu;
  1585. if (!task) {
  1586. /*
  1587. * Per cpu events are installed via an smp call and
  1588. * the install is always successful.
  1589. */
  1590. cpu_function_call(cpu, __perf_install_in_context, event);
  1591. return;
  1592. }
  1593. retry:
  1594. if (!task_function_call(task, __perf_install_in_context, event))
  1595. return;
  1596. raw_spin_lock_irq(&ctx->lock);
  1597. /*
  1598. * If we failed to find a running task, but find the context active now
  1599. * that we've acquired the ctx->lock, retry.
  1600. */
  1601. if (ctx->is_active) {
  1602. raw_spin_unlock_irq(&ctx->lock);
  1603. goto retry;
  1604. }
  1605. /*
  1606. * Since the task isn't running, its safe to add the event, us holding
  1607. * the ctx->lock ensures the task won't get scheduled in.
  1608. */
  1609. add_event_to_ctx(event, ctx);
  1610. raw_spin_unlock_irq(&ctx->lock);
  1611. }
  1612. /*
  1613. * Put a event into inactive state and update time fields.
  1614. * Enabling the leader of a group effectively enables all
  1615. * the group members that aren't explicitly disabled, so we
  1616. * have to update their ->tstamp_enabled also.
  1617. * Note: this works for group members as well as group leaders
  1618. * since the non-leader members' sibling_lists will be empty.
  1619. */
  1620. static void __perf_event_mark_enabled(struct perf_event *event)
  1621. {
  1622. struct perf_event *sub;
  1623. u64 tstamp = perf_event_time(event);
  1624. event->state = PERF_EVENT_STATE_INACTIVE;
  1625. event->tstamp_enabled = tstamp - event->total_time_enabled;
  1626. list_for_each_entry(sub, &event->sibling_list, group_entry) {
  1627. if (sub->state >= PERF_EVENT_STATE_INACTIVE)
  1628. sub->tstamp_enabled = tstamp - sub->total_time_enabled;
  1629. }
  1630. }
  1631. /*
  1632. * Cross CPU call to enable a performance event
  1633. */
  1634. static int __perf_event_enable(void *info)
  1635. {
  1636. struct perf_event *event = info;
  1637. struct perf_event_context *ctx = event->ctx;
  1638. struct perf_event *leader = event->group_leader;
  1639. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1640. int err;
  1641. /*
  1642. * There's a time window between 'ctx->is_active' check
  1643. * in perf_event_enable function and this place having:
  1644. * - IRQs on
  1645. * - ctx->lock unlocked
  1646. *
  1647. * where the task could be killed and 'ctx' deactivated
  1648. * by perf_event_exit_task.
  1649. */
  1650. if (!ctx->is_active)
  1651. return -EINVAL;
  1652. raw_spin_lock(&ctx->lock);
  1653. update_context_time(ctx);
  1654. if (event->state >= PERF_EVENT_STATE_INACTIVE)
  1655. goto unlock;
  1656. /*
  1657. * set current task's cgroup time reference point
  1658. */
  1659. perf_cgroup_set_timestamp(current, ctx);
  1660. __perf_event_mark_enabled(event);
  1661. if (!event_filter_match(event)) {
  1662. if (is_cgroup_event(event))
  1663. perf_cgroup_defer_enabled(event);
  1664. goto unlock;
  1665. }
  1666. /*
  1667. * If the event is in a group and isn't the group leader,
  1668. * then don't put it on unless the group is on.
  1669. */
  1670. if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
  1671. goto unlock;
  1672. if (!group_can_go_on(event, cpuctx, 1)) {
  1673. err = -EEXIST;
  1674. } else {
  1675. if (event == leader)
  1676. err = group_sched_in(event, cpuctx, ctx);
  1677. else
  1678. err = event_sched_in(event, cpuctx, ctx);
  1679. }
  1680. if (err) {
  1681. /*
  1682. * If this event can't go on and it's part of a
  1683. * group, then the whole group has to come off.
  1684. */
  1685. if (leader != event) {
  1686. group_sched_out(leader, cpuctx, ctx);
  1687. perf_cpu_hrtimer_restart(cpuctx);
  1688. }
  1689. if (leader->attr.pinned) {
  1690. update_group_times(leader);
  1691. leader->state = PERF_EVENT_STATE_ERROR;
  1692. }
  1693. }
  1694. unlock:
  1695. raw_spin_unlock(&ctx->lock);
  1696. return 0;
  1697. }
  1698. /*
  1699. * Enable a event.
  1700. *
  1701. * If event->ctx is a cloned context, callers must make sure that
  1702. * every task struct that event->ctx->task could possibly point to
  1703. * remains valid. This condition is satisfied when called through
  1704. * perf_event_for_each_child or perf_event_for_each as described
  1705. * for perf_event_disable.
  1706. */
  1707. void perf_event_enable(struct perf_event *event)
  1708. {
  1709. struct perf_event_context *ctx = event->ctx;
  1710. struct task_struct *task = ctx->task;
  1711. if (!task) {
  1712. /*
  1713. * Enable the event on the cpu that it's on
  1714. */
  1715. cpu_function_call(event->cpu, __perf_event_enable, event);
  1716. return;
  1717. }
  1718. raw_spin_lock_irq(&ctx->lock);
  1719. if (event->state >= PERF_EVENT_STATE_INACTIVE)
  1720. goto out;
  1721. /*
  1722. * If the event is in error state, clear that first.
  1723. * That way, if we see the event in error state below, we
  1724. * know that it has gone back into error state, as distinct
  1725. * from the task having been scheduled away before the
  1726. * cross-call arrived.
  1727. */
  1728. if (event->state == PERF_EVENT_STATE_ERROR)
  1729. event->state = PERF_EVENT_STATE_OFF;
  1730. retry:
  1731. if (!ctx->is_active) {
  1732. __perf_event_mark_enabled(event);
  1733. goto out;
  1734. }
  1735. raw_spin_unlock_irq(&ctx->lock);
  1736. if (!task_function_call(task, __perf_event_enable, event))
  1737. return;
  1738. raw_spin_lock_irq(&ctx->lock);
  1739. /*
  1740. * If the context is active and the event is still off,
  1741. * we need to retry the cross-call.
  1742. */
  1743. if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
  1744. /*
  1745. * task could have been flipped by a concurrent
  1746. * perf_event_context_sched_out()
  1747. */
  1748. task = ctx->task;
  1749. goto retry;
  1750. }
  1751. out:
  1752. raw_spin_unlock_irq(&ctx->lock);
  1753. }
  1754. EXPORT_SYMBOL_GPL(perf_event_enable);
  1755. int perf_event_refresh(struct perf_event *event, int refresh)
  1756. {
  1757. /*
  1758. * not supported on inherited events
  1759. */
  1760. if (event->attr.inherit || !is_sampling_event(event))
  1761. return -EINVAL;
  1762. atomic_add(refresh, &event->event_limit);
  1763. perf_event_enable(event);
  1764. return 0;
  1765. }
  1766. EXPORT_SYMBOL_GPL(perf_event_refresh);
  1767. static void ctx_sched_out(struct perf_event_context *ctx,
  1768. struct perf_cpu_context *cpuctx,
  1769. enum event_type_t event_type)
  1770. {
  1771. struct perf_event *event;
  1772. int is_active = ctx->is_active;
  1773. ctx->is_active &= ~event_type;
  1774. if (likely(!ctx->nr_events))
  1775. return;
  1776. update_context_time(ctx);
  1777. update_cgrp_time_from_cpuctx(cpuctx);
  1778. if (!ctx->nr_active)
  1779. return;
  1780. perf_pmu_disable(ctx->pmu);
  1781. if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
  1782. list_for_each_entry(event, &ctx->pinned_groups, group_entry)
  1783. group_sched_out(event, cpuctx, ctx);
  1784. }
  1785. if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
  1786. list_for_each_entry(event, &ctx->flexible_groups, group_entry)
  1787. group_sched_out(event, cpuctx, ctx);
  1788. }
  1789. perf_pmu_enable(ctx->pmu);
  1790. }
  1791. /*
  1792. * Test whether two contexts are equivalent, i.e. whether they
  1793. * have both been cloned from the same version of the same context
  1794. * and they both have the same number of enabled events.
  1795. * If the number of enabled events is the same, then the set
  1796. * of enabled events should be the same, because these are both
  1797. * inherited contexts, therefore we can't access individual events
  1798. * in them directly with an fd; we can only enable/disable all
  1799. * events via prctl, or enable/disable all events in a family
  1800. * via ioctl, which will have the same effect on both contexts.
  1801. */
  1802. static int context_equiv(struct perf_event_context *ctx1,
  1803. struct perf_event_context *ctx2)
  1804. {
  1805. return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
  1806. && ctx1->parent_gen == ctx2->parent_gen
  1807. && !ctx1->pin_count && !ctx2->pin_count;
  1808. }
  1809. static void __perf_event_sync_stat(struct perf_event *event,
  1810. struct perf_event *next_event)
  1811. {
  1812. u64 value;
  1813. if (!event->attr.inherit_stat)
  1814. return;
  1815. /*
  1816. * Update the event value, we cannot use perf_event_read()
  1817. * because we're in the middle of a context switch and have IRQs
  1818. * disabled, which upsets smp_call_function_single(), however
  1819. * we know the event must be on the current CPU, therefore we
  1820. * don't need to use it.
  1821. */
  1822. switch (event->state) {
  1823. case PERF_EVENT_STATE_ACTIVE:
  1824. event->pmu->read(event);
  1825. /* fall-through */
  1826. case PERF_EVENT_STATE_INACTIVE:
  1827. update_event_times(event);
  1828. break;
  1829. default:
  1830. break;
  1831. }
  1832. /*
  1833. * In order to keep per-task stats reliable we need to flip the event
  1834. * values when we flip the contexts.
  1835. */
  1836. value = local64_read(&next_event->count);
  1837. value = local64_xchg(&event->count, value);
  1838. local64_set(&next_event->count, value);
  1839. swap(event->total_time_enabled, next_event->total_time_enabled);
  1840. swap(event->total_time_running, next_event->total_time_running);
  1841. /*
  1842. * Since we swizzled the values, update the user visible data too.
  1843. */
  1844. perf_event_update_userpage(event);
  1845. perf_event_update_userpage(next_event);
  1846. }
  1847. #define list_next_entry(pos, member) \
  1848. list_entry(pos->member.next, typeof(*pos), member)
  1849. static void perf_event_sync_stat(struct perf_event_context *ctx,
  1850. struct perf_event_context *next_ctx)
  1851. {
  1852. struct perf_event *event, *next_event;
  1853. if (!ctx->nr_stat)
  1854. return;
  1855. update_context_time(ctx);
  1856. event = list_first_entry(&ctx->event_list,
  1857. struct perf_event, event_entry);
  1858. next_event = list_first_entry(&next_ctx->event_list,
  1859. struct perf_event, event_entry);
  1860. while (&event->event_entry != &ctx->event_list &&
  1861. &next_event->event_entry != &next_ctx->event_list) {
  1862. __perf_event_sync_stat(event, next_event);
  1863. event = list_next_entry(event, event_entry);
  1864. next_event = list_next_entry(next_event, event_entry);
  1865. }
  1866. }
  1867. static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
  1868. struct task_struct *next)
  1869. {
  1870. struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
  1871. struct perf_event_context *next_ctx;
  1872. struct perf_event_context *parent;
  1873. struct perf_cpu_context *cpuctx;
  1874. int do_switch = 1;
  1875. if (likely(!ctx))
  1876. return;
  1877. cpuctx = __get_cpu_context(ctx);
  1878. if (!cpuctx->task_ctx)
  1879. return;
  1880. rcu_read_lock();
  1881. parent = rcu_dereference(ctx->parent_ctx);
  1882. next_ctx = next->perf_event_ctxp[ctxn];
  1883. if (parent && next_ctx &&
  1884. rcu_dereference(next_ctx->parent_ctx) == parent) {
  1885. /*
  1886. * Looks like the two contexts are clones, so we might be
  1887. * able to optimize the context switch. We lock both
  1888. * contexts and check that they are clones under the
  1889. * lock (including re-checking that neither has been
  1890. * uncloned in the meantime). It doesn't matter which
  1891. * order we take the locks because no other cpu could
  1892. * be trying to lock both of these tasks.
  1893. */
  1894. raw_spin_lock(&ctx->lock);
  1895. raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
  1896. if (context_equiv(ctx, next_ctx)) {
  1897. /*
  1898. * XXX do we need a memory barrier of sorts
  1899. * wrt to rcu_dereference() of perf_event_ctxp
  1900. */
  1901. task->perf_event_ctxp[ctxn] = next_ctx;
  1902. next->perf_event_ctxp[ctxn] = ctx;
  1903. ctx->task = next;
  1904. next_ctx->task = task;
  1905. do_switch = 0;
  1906. perf_event_sync_stat(ctx, next_ctx);
  1907. }
  1908. raw_spin_unlock(&next_ctx->lock);
  1909. raw_spin_unlock(&ctx->lock);
  1910. }
  1911. rcu_read_unlock();
  1912. if (do_switch) {
  1913. raw_spin_lock(&ctx->lock);
  1914. ctx_sched_out(ctx, cpuctx, EVENT_ALL);
  1915. cpuctx->task_ctx = NULL;
  1916. raw_spin_unlock(&ctx->lock);
  1917. }
  1918. }
  1919. #define for_each_task_context_nr(ctxn) \
  1920. for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
  1921. /*
  1922. * Called from scheduler to remove the events of the current task,
  1923. * with interrupts disabled.
  1924. *
  1925. * We stop each event and update the event value in event->count.
  1926. *
  1927. * This does not protect us against NMI, but disable()
  1928. * sets the disabled bit in the control field of event _before_
  1929. * accessing the event control register. If a NMI hits, then it will
  1930. * not restart the event.
  1931. */
  1932. void __perf_event_task_sched_out(struct task_struct *task,
  1933. struct task_struct *next)
  1934. {
  1935. int ctxn;
  1936. for_each_task_context_nr(ctxn)
  1937. perf_event_context_sched_out(task, ctxn, next);
  1938. /*
  1939. * if cgroup events exist on this CPU, then we need
  1940. * to check if we have to switch out PMU state.
  1941. * cgroup event are system-wide mode only
  1942. */
  1943. if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
  1944. perf_cgroup_sched_out(task, next);
  1945. }
  1946. static void task_ctx_sched_out(struct perf_event_context *ctx)
  1947. {
  1948. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1949. if (!cpuctx->task_ctx)
  1950. return;
  1951. if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
  1952. return;
  1953. ctx_sched_out(ctx, cpuctx, EVENT_ALL);
  1954. cpuctx->task_ctx = NULL;
  1955. }
  1956. /*
  1957. * Called with IRQs disabled
  1958. */
  1959. static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
  1960. enum event_type_t event_type)
  1961. {
  1962. ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
  1963. }
  1964. static void
  1965. ctx_pinned_sched_in(struct perf_event_context *ctx,
  1966. struct perf_cpu_context *cpuctx)
  1967. {
  1968. struct perf_event *event;
  1969. list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
  1970. if (event->state <= PERF_EVENT_STATE_OFF)
  1971. continue;
  1972. if (!event_filter_match(event))
  1973. continue;
  1974. /* may need to reset tstamp_enabled */
  1975. if (is_cgroup_event(event))
  1976. perf_cgroup_mark_enabled(event, ctx);
  1977. if (group_can_go_on(event, cpuctx, 1))
  1978. group_sched_in(event, cpuctx, ctx);
  1979. /*
  1980. * If this pinned group hasn't been scheduled,
  1981. * put it in error state.
  1982. */
  1983. if (event->state == PERF_EVENT_STATE_INACTIVE) {
  1984. update_group_times(event);
  1985. event->state = PERF_EVENT_STATE_ERROR;
  1986. }
  1987. }
  1988. }
  1989. static void
  1990. ctx_flexible_sched_in(struct perf_event_context *ctx,
  1991. struct perf_cpu_context *cpuctx)
  1992. {
  1993. struct perf_event *event;
  1994. int can_add_hw = 1;
  1995. list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
  1996. /* Ignore events in OFF or ERROR state */
  1997. if (event->state <= PERF_EVENT_STATE_OFF)
  1998. continue;
  1999. /*
  2000. * Listen to the 'cpu' scheduling filter constraint
  2001. * of events:
  2002. */
  2003. if (!event_filter_match(event))
  2004. continue;
  2005. /* may need to reset tstamp_enabled */
  2006. if (is_cgroup_event(event))
  2007. perf_cgroup_mark_enabled(event, ctx);
  2008. if (group_can_go_on(event, cpuctx, can_add_hw)) {
  2009. if (group_sched_in(event, cpuctx, ctx))
  2010. can_add_hw = 0;
  2011. }
  2012. }
  2013. }
  2014. static void
  2015. ctx_sched_in(struct perf_event_context *ctx,
  2016. struct perf_cpu_context *cpuctx,
  2017. enum event_type_t event_type,
  2018. struct task_struct *task)
  2019. {
  2020. u64 now;
  2021. int is_active = ctx->is_active;
  2022. ctx->is_active |= event_type;
  2023. if (likely(!ctx->nr_events))
  2024. return;
  2025. now = perf_clock();
  2026. ctx->timestamp = now;
  2027. perf_cgroup_set_timestamp(task, ctx);
  2028. /*
  2029. * First go through the list and put on any pinned groups
  2030. * in order to give them the best chance of going on.
  2031. */
  2032. if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
  2033. ctx_pinned_sched_in(ctx, cpuctx);
  2034. /* Then walk through the lower prio flexible groups */
  2035. if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
  2036. ctx_flexible_sched_in(ctx, cpuctx);
  2037. }
  2038. static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
  2039. enum event_type_t event_type,
  2040. struct task_struct *task)
  2041. {
  2042. struct perf_event_context *ctx = &cpuctx->ctx;
  2043. ctx_sched_in(ctx, cpuctx, event_type, task);
  2044. }
  2045. static void perf_event_context_sched_in(struct perf_event_context *ctx,
  2046. struct task_struct *task)
  2047. {
  2048. struct perf_cpu_context *cpuctx;
  2049. cpuctx = __get_cpu_context(ctx);
  2050. if (cpuctx->task_ctx == ctx)
  2051. return;
  2052. perf_ctx_lock(cpuctx, ctx);
  2053. perf_pmu_disable(ctx->pmu);
  2054. /*
  2055. * We want to keep the following priority order:
  2056. * cpu pinned (that don't need to move), task pinned,
  2057. * cpu flexible, task flexible.
  2058. */
  2059. cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
  2060. if (ctx->nr_events)
  2061. cpuctx->task_ctx = ctx;
  2062. perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
  2063. perf_pmu_enable(ctx->pmu);
  2064. perf_ctx_unlock(cpuctx, ctx);
  2065. /*
  2066. * Since these rotations are per-cpu, we need to ensure the
  2067. * cpu-context we got scheduled on is actually rotating.
  2068. */
  2069. perf_pmu_rotate_start(ctx->pmu);
  2070. }
  2071. /*
  2072. * When sampling the branck stack in system-wide, it may be necessary
  2073. * to flush the stack on context switch. This happens when the branch
  2074. * stack does not tag its entries with the pid of the current task.
  2075. * Otherwise it becomes impossible to associate a branch entry with a
  2076. * task. This ambiguity is more likely to appear when the branch stack
  2077. * supports priv level filtering and the user sets it to monitor only
  2078. * at the user level (which could be a useful measurement in system-wide
  2079. * mode). In that case, the risk is high of having a branch stack with
  2080. * branch from multiple tasks. Flushing may mean dropping the existing
  2081. * entries or stashing them somewhere in the PMU specific code layer.
  2082. *
  2083. * This function provides the context switch callback to the lower code
  2084. * layer. It is invoked ONLY when there is at least one system-wide context
  2085. * with at least one active event using taken branch sampling.
  2086. */
  2087. static void perf_branch_stack_sched_in(struct task_struct *prev,
  2088. struct task_struct *task)
  2089. {
  2090. struct perf_cpu_context *cpuctx;
  2091. struct pmu *pmu;
  2092. unsigned long flags;
  2093. /* no need to flush branch stack if not changing task */
  2094. if (prev == task)
  2095. return;
  2096. local_irq_save(flags);
  2097. rcu_read_lock();
  2098. list_for_each_entry_rcu(pmu, &pmus, entry) {
  2099. cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  2100. /*
  2101. * check if the context has at least one
  2102. * event using PERF_SAMPLE_BRANCH_STACK
  2103. */
  2104. if (cpuctx->ctx.nr_branch_stack > 0
  2105. && pmu->flush_branch_stack) {
  2106. pmu = cpuctx->ctx.pmu;
  2107. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  2108. perf_pmu_disable(pmu);
  2109. pmu->flush_branch_stack();
  2110. perf_pmu_enable(pmu);
  2111. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  2112. }
  2113. }
  2114. rcu_read_unlock();
  2115. local_irq_restore(flags);
  2116. }
  2117. /*
  2118. * Called from scheduler to add the events of the current task
  2119. * with interrupts disabled.
  2120. *
  2121. * We restore the event value and then enable it.
  2122. *
  2123. * This does not protect us against NMI, but enable()
  2124. * sets the enabled bit in the control field of event _before_
  2125. * accessing the event control register. If a NMI hits, then it will
  2126. * keep the event running.
  2127. */
  2128. void __perf_event_task_sched_in(struct task_struct *prev,
  2129. struct task_struct *task)
  2130. {
  2131. struct perf_event_context *ctx;
  2132. int ctxn;
  2133. for_each_task_context_nr(ctxn) {
  2134. ctx = task->perf_event_ctxp[ctxn];
  2135. if (likely(!ctx))
  2136. continue;
  2137. perf_event_context_sched_in(ctx, task);
  2138. }
  2139. /*
  2140. * if cgroup events exist on this CPU, then we need
  2141. * to check if we have to switch in PMU state.
  2142. * cgroup event are system-wide mode only
  2143. */
  2144. if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
  2145. perf_cgroup_sched_in(prev, task);
  2146. /* check for system-wide branch_stack events */
  2147. if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
  2148. perf_branch_stack_sched_in(prev, task);
  2149. }
  2150. static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
  2151. {
  2152. u64 frequency = event->attr.sample_freq;
  2153. u64 sec = NSEC_PER_SEC;
  2154. u64 divisor, dividend;
  2155. int count_fls, nsec_fls, frequency_fls, sec_fls;
  2156. count_fls = fls64(count);
  2157. nsec_fls = fls64(nsec);
  2158. frequency_fls = fls64(frequency);
  2159. sec_fls = 30;
  2160. /*
  2161. * We got @count in @nsec, with a target of sample_freq HZ
  2162. * the target period becomes:
  2163. *
  2164. * @count * 10^9
  2165. * period = -------------------
  2166. * @nsec * sample_freq
  2167. *
  2168. */
  2169. /*
  2170. * Reduce accuracy by one bit such that @a and @b converge
  2171. * to a similar magnitude.
  2172. */
  2173. #define REDUCE_FLS(a, b) \
  2174. do { \
  2175. if (a##_fls > b##_fls) { \
  2176. a >>= 1; \
  2177. a##_fls--; \
  2178. } else { \
  2179. b >>= 1; \
  2180. b##_fls--; \
  2181. } \
  2182. } while (0)
  2183. /*
  2184. * Reduce accuracy until either term fits in a u64, then proceed with
  2185. * the other, so that finally we can do a u64/u64 division.
  2186. */
  2187. while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
  2188. REDUCE_FLS(nsec, frequency);
  2189. REDUCE_FLS(sec, count);
  2190. }
  2191. if (count_fls + sec_fls > 64) {
  2192. divisor = nsec * frequency;
  2193. while (count_fls + sec_fls > 64) {
  2194. REDUCE_FLS(count, sec);
  2195. divisor >>= 1;
  2196. }
  2197. dividend = count * sec;
  2198. } else {
  2199. dividend = count * sec;
  2200. while (nsec_fls + frequency_fls > 64) {
  2201. REDUCE_FLS(nsec, frequency);
  2202. dividend >>= 1;
  2203. }
  2204. divisor = nsec * frequency;
  2205. }
  2206. if (!divisor)
  2207. return dividend;
  2208. return div64_u64(dividend, divisor);
  2209. }
  2210. static DEFINE_PER_CPU(int, perf_throttled_count);
  2211. static DEFINE_PER_CPU(u64, perf_throttled_seq);
  2212. static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
  2213. {
  2214. struct hw_perf_event *hwc = &event->hw;
  2215. s64 period, sample_period;
  2216. s64 delta;
  2217. period = perf_calculate_period(event, nsec, count);
  2218. delta = (s64)(period - hwc->sample_period);
  2219. delta = (delta + 7) / 8; /* low pass filter */
  2220. sample_period = hwc->sample_period + delta;
  2221. if (!sample_period)
  2222. sample_period = 1;
  2223. hwc->sample_period = sample_period;
  2224. if (local64_read(&hwc->period_left) > 8*sample_period) {
  2225. if (disable)
  2226. event->pmu->stop(event, PERF_EF_UPDATE);
  2227. local64_set(&hwc->period_left, 0);
  2228. if (disable)
  2229. event->pmu->start(event, PERF_EF_RELOAD);
  2230. }
  2231. }
  2232. /*
  2233. * combine freq adjustment with unthrottling to avoid two passes over the
  2234. * events. At the same time, make sure, having freq events does not change
  2235. * the rate of unthrottling as that would introduce bias.
  2236. */
  2237. static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
  2238. int needs_unthr)
  2239. {
  2240. struct perf_event *event;
  2241. struct hw_perf_event *hwc;
  2242. u64 now, period = TICK_NSEC;
  2243. s64 delta;
  2244. /*
  2245. * only need to iterate over all events iff:
  2246. * - context have events in frequency mode (needs freq adjust)
  2247. * - there are events to unthrottle on this cpu
  2248. */
  2249. if (!(ctx->nr_freq || needs_unthr))
  2250. return;
  2251. raw_spin_lock(&ctx->lock);
  2252. perf_pmu_disable(ctx->pmu);
  2253. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  2254. if (event->state != PERF_EVENT_STATE_ACTIVE)
  2255. continue;
  2256. if (!event_filter_match(event))
  2257. continue;
  2258. hwc = &event->hw;
  2259. if (hwc->interrupts == MAX_INTERRUPTS) {
  2260. hwc->interrupts = 0;
  2261. perf_log_throttle(event, 1);
  2262. event->pmu->start(event, 0);
  2263. }
  2264. if (!event->attr.freq || !event->attr.sample_freq)
  2265. continue;
  2266. /*
  2267. * stop the event and update event->count
  2268. */
  2269. event->pmu->stop(event, PERF_EF_UPDATE);
  2270. now = local64_read(&event->count);
  2271. delta = now - hwc->freq_count_stamp;
  2272. hwc->freq_count_stamp = now;
  2273. /*
  2274. * restart the event
  2275. * reload only if value has changed
  2276. * we have stopped the event so tell that
  2277. * to perf_adjust_period() to avoid stopping it
  2278. * twice.
  2279. */
  2280. if (delta > 0)
  2281. perf_adjust_period(event, period, delta, false);
  2282. event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
  2283. }
  2284. perf_pmu_enable(ctx->pmu);
  2285. raw_spin_unlock(&ctx->lock);
  2286. }
  2287. /*
  2288. * Round-robin a context's events:
  2289. */
  2290. static void rotate_ctx(struct perf_event_context *ctx)
  2291. {
  2292. /*
  2293. * Rotate the first entry last of non-pinned groups. Rotation might be
  2294. * disabled by the inheritance code.
  2295. */
  2296. if (!ctx->rotate_disable)
  2297. list_rotate_left(&ctx->flexible_groups);
  2298. }
  2299. /*
  2300. * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
  2301. * because they're strictly cpu affine and rotate_start is called with IRQs
  2302. * disabled, while rotate_context is called from IRQ context.
  2303. */
  2304. static int perf_rotate_context(struct perf_cpu_context *cpuctx)
  2305. {
  2306. struct perf_event_context *ctx = NULL;
  2307. int rotate = 0, remove = 1;
  2308. if (cpuctx->ctx.nr_events) {
  2309. remove = 0;
  2310. if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
  2311. rotate = 1;
  2312. }
  2313. ctx = cpuctx->task_ctx;
  2314. if (ctx && ctx->nr_events) {
  2315. remove = 0;
  2316. if (ctx->nr_events != ctx->nr_active)
  2317. rotate = 1;
  2318. }
  2319. if (!rotate)
  2320. goto done;
  2321. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  2322. perf_pmu_disable(cpuctx->ctx.pmu);
  2323. cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
  2324. if (ctx)
  2325. ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
  2326. rotate_ctx(&cpuctx->ctx);
  2327. if (ctx)
  2328. rotate_ctx(ctx);
  2329. perf_event_sched_in(cpuctx, ctx, current);
  2330. perf_pmu_enable(cpuctx->ctx.pmu);
  2331. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  2332. done:
  2333. if (remove)
  2334. list_del_init(&cpuctx->rotation_list);
  2335. return rotate;
  2336. }
  2337. #ifdef CONFIG_NO_HZ_FULL
  2338. bool perf_event_can_stop_tick(void)
  2339. {
  2340. if (atomic_read(&nr_freq_events) ||
  2341. __this_cpu_read(perf_throttled_count))
  2342. return false;
  2343. else
  2344. return true;
  2345. }
  2346. #endif
  2347. void perf_event_task_tick(void)
  2348. {
  2349. struct list_head *head = &__get_cpu_var(rotation_list);
  2350. struct perf_cpu_context *cpuctx, *tmp;
  2351. struct perf_event_context *ctx;
  2352. int throttled;
  2353. WARN_ON(!irqs_disabled());
  2354. __this_cpu_inc(perf_throttled_seq);
  2355. throttled = __this_cpu_xchg(perf_throttled_count, 0);
  2356. list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
  2357. ctx = &cpuctx->ctx;
  2358. perf_adjust_freq_unthr_context(ctx, throttled);
  2359. ctx = cpuctx->task_ctx;
  2360. if (ctx)
  2361. perf_adjust_freq_unthr_context(ctx, throttled);
  2362. }
  2363. }
  2364. static int event_enable_on_exec(struct perf_event *event,
  2365. struct perf_event_context *ctx)
  2366. {
  2367. if (!event->attr.enable_on_exec)
  2368. return 0;
  2369. event->attr.enable_on_exec = 0;
  2370. if (event->state >= PERF_EVENT_STATE_INACTIVE)
  2371. return 0;
  2372. __perf_event_mark_enabled(event);
  2373. return 1;
  2374. }
  2375. /*
  2376. * Enable all of a task's events that have been marked enable-on-exec.
  2377. * This expects task == current.
  2378. */
  2379. static void perf_event_enable_on_exec(struct perf_event_context *ctx)
  2380. {
  2381. struct perf_event *event;
  2382. unsigned long flags;
  2383. int enabled = 0;
  2384. int ret;
  2385. local_irq_save(flags);
  2386. if (!ctx || !ctx->nr_events)
  2387. goto out;
  2388. /*
  2389. * We must ctxsw out cgroup events to avoid conflict
  2390. * when invoking perf_task_event_sched_in() later on
  2391. * in this function. Otherwise we end up trying to
  2392. * ctxswin cgroup events which are already scheduled
  2393. * in.
  2394. */
  2395. perf_cgroup_sched_out(current, NULL);
  2396. raw_spin_lock(&ctx->lock);
  2397. task_ctx_sched_out(ctx);
  2398. list_for_each_entry(event, &ctx->event_list, event_entry) {
  2399. ret = event_enable_on_exec(event, ctx);
  2400. if (ret)
  2401. enabled = 1;
  2402. }
  2403. /*
  2404. * Unclone this context if we enabled any event.
  2405. */
  2406. if (enabled)
  2407. unclone_ctx(ctx);
  2408. raw_spin_unlock(&ctx->lock);
  2409. /*
  2410. * Also calls ctxswin for cgroup events, if any:
  2411. */
  2412. perf_event_context_sched_in(ctx, ctx->task);
  2413. out:
  2414. local_irq_restore(flags);
  2415. }
  2416. /*
  2417. * Cross CPU call to read the hardware event
  2418. */
  2419. static void __perf_event_read(void *info)
  2420. {
  2421. struct perf_event *event = info;
  2422. struct perf_event_context *ctx = event->ctx;
  2423. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  2424. /*
  2425. * If this is a task context, we need to check whether it is
  2426. * the current task context of this cpu. If not it has been
  2427. * scheduled out before the smp call arrived. In that case
  2428. * event->count would have been updated to a recent sample
  2429. * when the event was scheduled out.
  2430. */
  2431. if (ctx->task && cpuctx->task_ctx != ctx)
  2432. return;
  2433. raw_spin_lock(&ctx->lock);
  2434. if (ctx->is_active) {
  2435. update_context_time(ctx);
  2436. update_cgrp_time_from_event(event);
  2437. }
  2438. update_event_times(event);
  2439. if (event->state == PERF_EVENT_STATE_ACTIVE)
  2440. event->pmu->read(event);
  2441. raw_spin_unlock(&ctx->lock);
  2442. }
  2443. static inline u64 perf_event_count(struct perf_event *event)
  2444. {
  2445. return local64_read(&event->count) + atomic64_read(&event->child_count);
  2446. }
  2447. static u64 perf_event_read(struct perf_event *event)
  2448. {
  2449. /*
  2450. * If event is enabled and currently active on a CPU, update the
  2451. * value in the event structure:
  2452. */
  2453. if (event->state == PERF_EVENT_STATE_ACTIVE) {
  2454. smp_call_function_single(event->oncpu,
  2455. __perf_event_read, event, 1);
  2456. } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
  2457. struct perf_event_context *ctx = event->ctx;
  2458. unsigned long flags;
  2459. raw_spin_lock_irqsave(&ctx->lock, flags);
  2460. /*
  2461. * may read while context is not active
  2462. * (e.g., thread is blocked), in that case
  2463. * we cannot update context time
  2464. */
  2465. if (ctx->is_active) {
  2466. update_context_time(ctx);
  2467. update_cgrp_time_from_event(event);
  2468. }
  2469. update_event_times(event);
  2470. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  2471. }
  2472. return perf_event_count(event);
  2473. }
  2474. /*
  2475. * Initialize the perf_event context in a task_struct:
  2476. */
  2477. static void __perf_event_init_context(struct perf_event_context *ctx)
  2478. {
  2479. raw_spin_lock_init(&ctx->lock);
  2480. mutex_init(&ctx->mutex);
  2481. INIT_LIST_HEAD(&ctx->pinned_groups);
  2482. INIT_LIST_HEAD(&ctx->flexible_groups);
  2483. INIT_LIST_HEAD(&ctx->event_list);
  2484. atomic_set(&ctx->refcount, 1);
  2485. }
  2486. static struct perf_event_context *
  2487. alloc_perf_context(struct pmu *pmu, struct task_struct *task)
  2488. {
  2489. struct perf_event_context *ctx;
  2490. ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
  2491. if (!ctx)
  2492. return NULL;
  2493. __perf_event_init_context(ctx);
  2494. if (task) {
  2495. ctx->task = task;
  2496. get_task_struct(task);
  2497. }
  2498. ctx->pmu = pmu;
  2499. return ctx;
  2500. }
  2501. static struct task_struct *
  2502. find_lively_task_by_vpid(pid_t vpid)
  2503. {
  2504. struct task_struct *task;
  2505. int err;
  2506. rcu_read_lock();
  2507. if (!vpid)
  2508. task = current;
  2509. else
  2510. task = find_task_by_vpid(vpid);
  2511. if (task)
  2512. get_task_struct(task);
  2513. rcu_read_unlock();
  2514. if (!task)
  2515. return ERR_PTR(-ESRCH);
  2516. /* Reuse ptrace permission checks for now. */
  2517. err = -EACCES;
  2518. if (!ptrace_may_access(task, PTRACE_MODE_READ))
  2519. goto errout;
  2520. return task;
  2521. errout:
  2522. put_task_struct(task);
  2523. return ERR_PTR(err);
  2524. }
  2525. /*
  2526. * Returns a matching context with refcount and pincount.
  2527. */
  2528. static struct perf_event_context *
  2529. find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
  2530. {
  2531. struct perf_event_context *ctx;
  2532. struct perf_cpu_context *cpuctx;
  2533. unsigned long flags;
  2534. int ctxn, err;
  2535. if (!task) {
  2536. /* Must be root to operate on a CPU event: */
  2537. if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
  2538. return ERR_PTR(-EACCES);
  2539. /*
  2540. * We could be clever and allow to attach a event to an
  2541. * offline CPU and activate it when the CPU comes up, but
  2542. * that's for later.
  2543. */
  2544. if (!cpu_online(cpu))
  2545. return ERR_PTR(-ENODEV);
  2546. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  2547. ctx = &cpuctx->ctx;
  2548. get_ctx(ctx);
  2549. ++ctx->pin_count;
  2550. return ctx;
  2551. }
  2552. err = -EINVAL;
  2553. ctxn = pmu->task_ctx_nr;
  2554. if (ctxn < 0)
  2555. goto errout;
  2556. retry:
  2557. ctx = perf_lock_task_context(task, ctxn, &flags);
  2558. if (ctx) {
  2559. unclone_ctx(ctx);
  2560. ++ctx->pin_count;
  2561. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  2562. } else {
  2563. ctx = alloc_perf_context(pmu, task);
  2564. err = -ENOMEM;
  2565. if (!ctx)
  2566. goto errout;
  2567. err = 0;
  2568. mutex_lock(&task->perf_event_mutex);
  2569. /*
  2570. * If it has already passed perf_event_exit_task().
  2571. * we must see PF_EXITING, it takes this mutex too.
  2572. */
  2573. if (task->flags & PF_EXITING)
  2574. err = -ESRCH;
  2575. else if (task->perf_event_ctxp[ctxn])
  2576. err = -EAGAIN;
  2577. else {
  2578. get_ctx(ctx);
  2579. ++ctx->pin_count;
  2580. rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
  2581. }
  2582. mutex_unlock(&task->perf_event_mutex);
  2583. if (unlikely(err)) {
  2584. put_ctx(ctx);
  2585. if (err == -EAGAIN)
  2586. goto retry;
  2587. goto errout;
  2588. }
  2589. }
  2590. return ctx;
  2591. errout:
  2592. return ERR_PTR(err);
  2593. }
  2594. static void perf_event_free_filter(struct perf_event *event);
  2595. static void free_event_rcu(struct rcu_head *head)
  2596. {
  2597. struct perf_event *event;
  2598. event = container_of(head, struct perf_event, rcu_head);
  2599. if (event->ns)
  2600. put_pid_ns(event->ns);
  2601. perf_event_free_filter(event);
  2602. kfree(event);
  2603. }
  2604. static void ring_buffer_put(struct ring_buffer *rb);
  2605. static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb);
  2606. static void unaccount_event_cpu(struct perf_event *event, int cpu)
  2607. {
  2608. if (event->parent)
  2609. return;
  2610. if (has_branch_stack(event)) {
  2611. if (!(event->attach_state & PERF_ATTACH_TASK))
  2612. atomic_dec(&per_cpu(perf_branch_stack_events, cpu));
  2613. }
  2614. if (is_cgroup_event(event))
  2615. atomic_dec(&per_cpu(perf_cgroup_events, cpu));
  2616. }
  2617. static void unaccount_event(struct perf_event *event)
  2618. {
  2619. if (event->parent)
  2620. return;
  2621. if (event->attach_state & PERF_ATTACH_TASK)
  2622. static_key_slow_dec_deferred(&perf_sched_events);
  2623. if (event->attr.mmap || event->attr.mmap_data)
  2624. atomic_dec(&nr_mmap_events);
  2625. if (event->attr.comm)
  2626. atomic_dec(&nr_comm_events);
  2627. if (event->attr.task)
  2628. atomic_dec(&nr_task_events);
  2629. if (event->attr.freq)
  2630. atomic_dec(&nr_freq_events);
  2631. if (is_cgroup_event(event))
  2632. static_key_slow_dec_deferred(&perf_sched_events);
  2633. if (has_branch_stack(event))
  2634. static_key_slow_dec_deferred(&perf_sched_events);
  2635. unaccount_event_cpu(event, event->cpu);
  2636. }
  2637. static void __free_event(struct perf_event *event)
  2638. {
  2639. if (!event->parent) {
  2640. if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
  2641. put_callchain_buffers();
  2642. }
  2643. if (event->destroy)
  2644. event->destroy(event);
  2645. if (event->ctx)
  2646. put_ctx(event->ctx);
  2647. call_rcu(&event->rcu_head, free_event_rcu);
  2648. }
  2649. static void free_event(struct perf_event *event)
  2650. {
  2651. irq_work_sync(&event->pending);
  2652. unaccount_event(event);
  2653. if (event->rb) {
  2654. struct ring_buffer *rb;
  2655. /*
  2656. * Can happen when we close an event with re-directed output.
  2657. *
  2658. * Since we have a 0 refcount, perf_mmap_close() will skip
  2659. * over us; possibly making our ring_buffer_put() the last.
  2660. */
  2661. mutex_lock(&event->mmap_mutex);
  2662. rb = event->rb;
  2663. if (rb) {
  2664. rcu_assign_pointer(event->rb, NULL);
  2665. ring_buffer_detach(event, rb);
  2666. ring_buffer_put(rb); /* could be last */
  2667. }
  2668. mutex_unlock(&event->mmap_mutex);
  2669. }
  2670. if (is_cgroup_event(event))
  2671. perf_detach_cgroup(event);
  2672. __free_event(event);
  2673. }
  2674. int perf_event_release_kernel(struct perf_event *event)
  2675. {
  2676. struct perf_event_context *ctx = event->ctx;
  2677. WARN_ON_ONCE(ctx->parent_ctx);
  2678. /*
  2679. * There are two ways this annotation is useful:
  2680. *
  2681. * 1) there is a lock recursion from perf_event_exit_task
  2682. * see the comment there.
  2683. *
  2684. * 2) there is a lock-inversion with mmap_sem through
  2685. * perf_event_read_group(), which takes faults while
  2686. * holding ctx->mutex, however this is called after
  2687. * the last filedesc died, so there is no possibility
  2688. * to trigger the AB-BA case.
  2689. */
  2690. mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
  2691. raw_spin_lock_irq(&ctx->lock);
  2692. perf_group_detach(event);
  2693. raw_spin_unlock_irq(&ctx->lock);
  2694. perf_remove_from_context(event);
  2695. mutex_unlock(&ctx->mutex);
  2696. free_event(event);
  2697. return 0;
  2698. }
  2699. EXPORT_SYMBOL_GPL(perf_event_release_kernel);
  2700. /*
  2701. * Called when the last reference to the file is gone.
  2702. */
  2703. static void put_event(struct perf_event *event)
  2704. {
  2705. struct task_struct *owner;
  2706. if (!atomic_long_dec_and_test(&event->refcount))
  2707. return;
  2708. rcu_read_lock();
  2709. owner = ACCESS_ONCE(event->owner);
  2710. /*
  2711. * Matches the smp_wmb() in perf_event_exit_task(). If we observe
  2712. * !owner it means the list deletion is complete and we can indeed
  2713. * free this event, otherwise we need to serialize on
  2714. * owner->perf_event_mutex.
  2715. */
  2716. smp_read_barrier_depends();
  2717. if (owner) {
  2718. /*
  2719. * Since delayed_put_task_struct() also drops the last
  2720. * task reference we can safely take a new reference
  2721. * while holding the rcu_read_lock().
  2722. */
  2723. get_task_struct(owner);
  2724. }
  2725. rcu_read_unlock();
  2726. if (owner) {
  2727. mutex_lock(&owner->perf_event_mutex);
  2728. /*
  2729. * We have to re-check the event->owner field, if it is cleared
  2730. * we raced with perf_event_exit_task(), acquiring the mutex
  2731. * ensured they're done, and we can proceed with freeing the
  2732. * event.
  2733. */
  2734. if (event->owner)
  2735. list_del_init(&event->owner_entry);
  2736. mutex_unlock(&owner->perf_event_mutex);
  2737. put_task_struct(owner);
  2738. }
  2739. perf_event_release_kernel(event);
  2740. }
  2741. static int perf_release(struct inode *inode, struct file *file)
  2742. {
  2743. put_event(file->private_data);
  2744. return 0;
  2745. }
  2746. u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
  2747. {
  2748. struct perf_event *child;
  2749. u64 total = 0;
  2750. *enabled = 0;
  2751. *running = 0;
  2752. mutex_lock(&event->child_mutex);
  2753. total += perf_event_read(event);
  2754. *enabled += event->total_time_enabled +
  2755. atomic64_read(&event->child_total_time_enabled);
  2756. *running += event->total_time_running +
  2757. atomic64_read(&event->child_total_time_running);
  2758. list_for_each_entry(child, &event->child_list, child_list) {
  2759. total += perf_event_read(child);
  2760. *enabled += child->total_time_enabled;
  2761. *running += child->total_time_running;
  2762. }
  2763. mutex_unlock(&event->child_mutex);
  2764. return total;
  2765. }
  2766. EXPORT_SYMBOL_GPL(perf_event_read_value);
  2767. static int perf_event_read_group(struct perf_event *event,
  2768. u64 read_format, char __user *buf)
  2769. {
  2770. struct perf_event *leader = event->group_leader, *sub;
  2771. int n = 0, size = 0, ret = -EFAULT;
  2772. struct perf_event_context *ctx = leader->ctx;
  2773. u64 values[5];
  2774. u64 count, enabled, running;
  2775. mutex_lock(&ctx->mutex);
  2776. count = perf_event_read_value(leader, &enabled, &running);
  2777. values[n++] = 1 + leader->nr_siblings;
  2778. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  2779. values[n++] = enabled;
  2780. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  2781. values[n++] = running;
  2782. values[n++] = count;
  2783. if (read_format & PERF_FORMAT_ID)
  2784. values[n++] = primary_event_id(leader);
  2785. size = n * sizeof(u64);
  2786. if (copy_to_user(buf, values, size))
  2787. goto unlock;
  2788. ret = size;
  2789. list_for_each_entry(sub, &leader->sibling_list, group_entry) {
  2790. n = 0;
  2791. values[n++] = perf_event_read_value(sub, &enabled, &running);
  2792. if (read_format & PERF_FORMAT_ID)
  2793. values[n++] = primary_event_id(sub);
  2794. size = n * sizeof(u64);
  2795. if (copy_to_user(buf + ret, values, size)) {
  2796. ret = -EFAULT;
  2797. goto unlock;
  2798. }
  2799. ret += size;
  2800. }
  2801. unlock:
  2802. mutex_unlock(&ctx->mutex);
  2803. return ret;
  2804. }
  2805. static int perf_event_read_one(struct perf_event *event,
  2806. u64 read_format, char __user *buf)
  2807. {
  2808. u64 enabled, running;
  2809. u64 values[4];
  2810. int n = 0;
  2811. values[n++] = perf_event_read_value(event, &enabled, &running);
  2812. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  2813. values[n++] = enabled;
  2814. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  2815. values[n++] = running;
  2816. if (read_format & PERF_FORMAT_ID)
  2817. values[n++] = primary_event_id(event);
  2818. if (copy_to_user(buf, values, n * sizeof(u64)))
  2819. return -EFAULT;
  2820. return n * sizeof(u64);
  2821. }
  2822. /*
  2823. * Read the performance event - simple non blocking version for now
  2824. */
  2825. static ssize_t
  2826. perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
  2827. {
  2828. u64 read_format = event->attr.read_format;
  2829. int ret;
  2830. /*
  2831. * Return end-of-file for a read on a event that is in
  2832. * error state (i.e. because it was pinned but it couldn't be
  2833. * scheduled on to the CPU at some point).
  2834. */
  2835. if (event->state == PERF_EVENT_STATE_ERROR)
  2836. return 0;
  2837. if (count < event->read_size)
  2838. return -ENOSPC;
  2839. WARN_ON_ONCE(event->ctx->parent_ctx);
  2840. if (read_format & PERF_FORMAT_GROUP)
  2841. ret = perf_event_read_group(event, read_format, buf);
  2842. else
  2843. ret = perf_event_read_one(event, read_format, buf);
  2844. return ret;
  2845. }
  2846. static ssize_t
  2847. perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
  2848. {
  2849. struct perf_event *event = file->private_data;
  2850. return perf_read_hw(event, buf, count);
  2851. }
  2852. static unsigned int perf_poll(struct file *file, poll_table *wait)
  2853. {
  2854. struct perf_event *event = file->private_data;
  2855. struct ring_buffer *rb;
  2856. unsigned int events = POLL_HUP;
  2857. /*
  2858. * Pin the event->rb by taking event->mmap_mutex; otherwise
  2859. * perf_event_set_output() can swizzle our rb and make us miss wakeups.
  2860. */
  2861. mutex_lock(&event->mmap_mutex);
  2862. rb = event->rb;
  2863. if (rb)
  2864. events = atomic_xchg(&rb->poll, 0);
  2865. mutex_unlock(&event->mmap_mutex);
  2866. poll_wait(file, &event->waitq, wait);
  2867. return events;
  2868. }
  2869. static void perf_event_reset(struct perf_event *event)
  2870. {
  2871. (void)perf_event_read(event);
  2872. local64_set(&event->count, 0);
  2873. perf_event_update_userpage(event);
  2874. }
  2875. /*
  2876. * Holding the top-level event's child_mutex means that any
  2877. * descendant process that has inherited this event will block
  2878. * in sync_child_event if it goes to exit, thus satisfying the
  2879. * task existence requirements of perf_event_enable/disable.
  2880. */
  2881. static void perf_event_for_each_child(struct perf_event *event,
  2882. void (*func)(struct perf_event *))
  2883. {
  2884. struct perf_event *child;
  2885. WARN_ON_ONCE(event->ctx->parent_ctx);
  2886. mutex_lock(&event->child_mutex);
  2887. func(event);
  2888. list_for_each_entry(child, &event->child_list, child_list)
  2889. func(child);
  2890. mutex_unlock(&event->child_mutex);
  2891. }
  2892. static void perf_event_for_each(struct perf_event *event,
  2893. void (*func)(struct perf_event *))
  2894. {
  2895. struct perf_event_context *ctx = event->ctx;
  2896. struct perf_event *sibling;
  2897. WARN_ON_ONCE(ctx->parent_ctx);
  2898. mutex_lock(&ctx->mutex);
  2899. event = event->group_leader;
  2900. perf_event_for_each_child(event, func);
  2901. list_for_each_entry(sibling, &event->sibling_list, group_entry)
  2902. perf_event_for_each_child(sibling, func);
  2903. mutex_unlock(&ctx->mutex);
  2904. }
  2905. static int perf_event_period(struct perf_event *event, u64 __user *arg)
  2906. {
  2907. struct perf_event_context *ctx = event->ctx;
  2908. int ret = 0;
  2909. u64 value;
  2910. if (!is_sampling_event(event))
  2911. return -EINVAL;
  2912. if (copy_from_user(&value, arg, sizeof(value)))
  2913. return -EFAULT;
  2914. if (!value)
  2915. return -EINVAL;
  2916. raw_spin_lock_irq(&ctx->lock);
  2917. if (event->attr.freq) {
  2918. if (value > sysctl_perf_event_sample_rate) {
  2919. ret = -EINVAL;
  2920. goto unlock;
  2921. }
  2922. event->attr.sample_freq = value;
  2923. } else {
  2924. event->attr.sample_period = value;
  2925. event->hw.sample_period = value;
  2926. }
  2927. unlock:
  2928. raw_spin_unlock_irq(&ctx->lock);
  2929. return ret;
  2930. }
  2931. static const struct file_operations perf_fops;
  2932. static inline int perf_fget_light(int fd, struct fd *p)
  2933. {
  2934. struct fd f = fdget(fd);
  2935. if (!f.file)
  2936. return -EBADF;
  2937. if (f.file->f_op != &perf_fops) {
  2938. fdput(f);
  2939. return -EBADF;
  2940. }
  2941. *p = f;
  2942. return 0;
  2943. }
  2944. static int perf_event_set_output(struct perf_event *event,
  2945. struct perf_event *output_event);
  2946. static int perf_event_set_filter(struct perf_event *event, void __user *arg);
  2947. static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  2948. {
  2949. struct perf_event *event = file->private_data;
  2950. void (*func)(struct perf_event *);
  2951. u32 flags = arg;
  2952. switch (cmd) {
  2953. case PERF_EVENT_IOC_ENABLE:
  2954. func = perf_event_enable;
  2955. break;
  2956. case PERF_EVENT_IOC_DISABLE:
  2957. func = perf_event_disable;
  2958. break;
  2959. case PERF_EVENT_IOC_RESET:
  2960. func = perf_event_reset;
  2961. break;
  2962. case PERF_EVENT_IOC_REFRESH:
  2963. return perf_event_refresh(event, arg);
  2964. case PERF_EVENT_IOC_PERIOD:
  2965. return perf_event_period(event, (u64 __user *)arg);
  2966. case PERF_EVENT_IOC_ID:
  2967. {
  2968. u64 id = primary_event_id(event);
  2969. if (copy_to_user((void __user *)arg, &id, sizeof(id)))
  2970. return -EFAULT;
  2971. return 0;
  2972. }
  2973. case PERF_EVENT_IOC_SET_OUTPUT:
  2974. {
  2975. int ret;
  2976. if (arg != -1) {
  2977. struct perf_event *output_event;
  2978. struct fd output;
  2979. ret = perf_fget_light(arg, &output);
  2980. if (ret)
  2981. return ret;
  2982. output_event = output.file->private_data;
  2983. ret = perf_event_set_output(event, output_event);
  2984. fdput(output);
  2985. } else {
  2986. ret = perf_event_set_output(event, NULL);
  2987. }
  2988. return ret;
  2989. }
  2990. case PERF_EVENT_IOC_SET_FILTER:
  2991. return perf_event_set_filter(event, (void __user *)arg);
  2992. default:
  2993. return -ENOTTY;
  2994. }
  2995. if (flags & PERF_IOC_FLAG_GROUP)
  2996. perf_event_for_each(event, func);
  2997. else
  2998. perf_event_for_each_child(event, func);
  2999. return 0;
  3000. }
  3001. int perf_event_task_enable(void)
  3002. {
  3003. struct perf_event *event;
  3004. mutex_lock(&current->perf_event_mutex);
  3005. list_for_each_entry(event, &current->perf_event_list, owner_entry)
  3006. perf_event_for_each_child(event, perf_event_enable);
  3007. mutex_unlock(&current->perf_event_mutex);
  3008. return 0;
  3009. }
  3010. int perf_event_task_disable(void)
  3011. {
  3012. struct perf_event *event;
  3013. mutex_lock(&current->perf_event_mutex);
  3014. list_for_each_entry(event, &current->perf_event_list, owner_entry)
  3015. perf_event_for_each_child(event, perf_event_disable);
  3016. mutex_unlock(&current->perf_event_mutex);
  3017. return 0;
  3018. }
  3019. static int perf_event_index(struct perf_event *event)
  3020. {
  3021. if (event->hw.state & PERF_HES_STOPPED)
  3022. return 0;
  3023. if (event->state != PERF_EVENT_STATE_ACTIVE)
  3024. return 0;
  3025. return event->pmu->event_idx(event);
  3026. }
  3027. static void calc_timer_values(struct perf_event *event,
  3028. u64 *now,
  3029. u64 *enabled,
  3030. u64 *running)
  3031. {
  3032. u64 ctx_time;
  3033. *now = perf_clock();
  3034. ctx_time = event->shadow_ctx_time + *now;
  3035. *enabled = ctx_time - event->tstamp_enabled;
  3036. *running = ctx_time - event->tstamp_running;
  3037. }
  3038. void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
  3039. {
  3040. }
  3041. /*
  3042. * Callers need to ensure there can be no nesting of this function, otherwise
  3043. * the seqlock logic goes bad. We can not serialize this because the arch
  3044. * code calls this from NMI context.
  3045. */
  3046. void perf_event_update_userpage(struct perf_event *event)
  3047. {
  3048. struct perf_event_mmap_page *userpg;
  3049. struct ring_buffer *rb;
  3050. u64 enabled, running, now;
  3051. rcu_read_lock();
  3052. rb = rcu_dereference(event->rb);
  3053. if (!rb)
  3054. goto unlock;
  3055. /*
  3056. * compute total_time_enabled, total_time_running
  3057. * based on snapshot values taken when the event
  3058. * was last scheduled in.
  3059. *
  3060. * we cannot simply called update_context_time()
  3061. * because of locking issue as we can be called in
  3062. * NMI context
  3063. */
  3064. calc_timer_values(event, &now, &enabled, &running);
  3065. userpg = rb->user_page;
  3066. /*
  3067. * Disable preemption so as to not let the corresponding user-space
  3068. * spin too long if we get preempted.
  3069. */
  3070. preempt_disable();
  3071. ++userpg->lock;
  3072. barrier();
  3073. userpg->index = perf_event_index(event);
  3074. userpg->offset = perf_event_count(event);
  3075. if (userpg->index)
  3076. userpg->offset -= local64_read(&event->hw.prev_count);
  3077. userpg->time_enabled = enabled +
  3078. atomic64_read(&event->child_total_time_enabled);
  3079. userpg->time_running = running +
  3080. atomic64_read(&event->child_total_time_running);
  3081. arch_perf_update_userpage(userpg, now);
  3082. barrier();
  3083. ++userpg->lock;
  3084. preempt_enable();
  3085. unlock:
  3086. rcu_read_unlock();
  3087. }
  3088. static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  3089. {
  3090. struct perf_event *event = vma->vm_file->private_data;
  3091. struct ring_buffer *rb;
  3092. int ret = VM_FAULT_SIGBUS;
  3093. if (vmf->flags & FAULT_FLAG_MKWRITE) {
  3094. if (vmf->pgoff == 0)
  3095. ret = 0;
  3096. return ret;
  3097. }
  3098. rcu_read_lock();
  3099. rb = rcu_dereference(event->rb);
  3100. if (!rb)
  3101. goto unlock;
  3102. if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
  3103. goto unlock;
  3104. vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
  3105. if (!vmf->page)
  3106. goto unlock;
  3107. get_page(vmf->page);
  3108. vmf->page->mapping = vma->vm_file->f_mapping;
  3109. vmf->page->index = vmf->pgoff;
  3110. ret = 0;
  3111. unlock:
  3112. rcu_read_unlock();
  3113. return ret;
  3114. }
  3115. static void ring_buffer_attach(struct perf_event *event,
  3116. struct ring_buffer *rb)
  3117. {
  3118. unsigned long flags;
  3119. if (!list_empty(&event->rb_entry))
  3120. return;
  3121. spin_lock_irqsave(&rb->event_lock, flags);
  3122. if (list_empty(&event->rb_entry))
  3123. list_add(&event->rb_entry, &rb->event_list);
  3124. spin_unlock_irqrestore(&rb->event_lock, flags);
  3125. }
  3126. static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb)
  3127. {
  3128. unsigned long flags;
  3129. if (list_empty(&event->rb_entry))
  3130. return;
  3131. spin_lock_irqsave(&rb->event_lock, flags);
  3132. list_del_init(&event->rb_entry);
  3133. wake_up_all(&event->waitq);
  3134. spin_unlock_irqrestore(&rb->event_lock, flags);
  3135. }
  3136. static void ring_buffer_wakeup(struct perf_event *event)
  3137. {
  3138. struct ring_buffer *rb;
  3139. rcu_read_lock();
  3140. rb = rcu_dereference(event->rb);
  3141. if (rb) {
  3142. list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
  3143. wake_up_all(&event->waitq);
  3144. }
  3145. rcu_read_unlock();
  3146. }
  3147. static void rb_free_rcu(struct rcu_head *rcu_head)
  3148. {
  3149. struct ring_buffer *rb;
  3150. rb = container_of(rcu_head, struct ring_buffer, rcu_head);
  3151. rb_free(rb);
  3152. }
  3153. static struct ring_buffer *ring_buffer_get(struct perf_event *event)
  3154. {
  3155. struct ring_buffer *rb;
  3156. rcu_read_lock();
  3157. rb = rcu_dereference(event->rb);
  3158. if (rb) {
  3159. if (!atomic_inc_not_zero(&rb->refcount))
  3160. rb = NULL;
  3161. }
  3162. rcu_read_unlock();
  3163. return rb;
  3164. }
  3165. static void ring_buffer_put(struct ring_buffer *rb)
  3166. {
  3167. if (!atomic_dec_and_test(&rb->refcount))
  3168. return;
  3169. WARN_ON_ONCE(!list_empty(&rb->event_list));
  3170. call_rcu(&rb->rcu_head, rb_free_rcu);
  3171. }
  3172. static void perf_mmap_open(struct vm_area_struct *vma)
  3173. {
  3174. struct perf_event *event = vma->vm_file->private_data;
  3175. atomic_inc(&event->mmap_count);
  3176. atomic_inc(&event->rb->mmap_count);
  3177. }
  3178. /*
  3179. * A buffer can be mmap()ed multiple times; either directly through the same
  3180. * event, or through other events by use of perf_event_set_output().
  3181. *
  3182. * In order to undo the VM accounting done by perf_mmap() we need to destroy
  3183. * the buffer here, where we still have a VM context. This means we need
  3184. * to detach all events redirecting to us.
  3185. */
  3186. static void perf_mmap_close(struct vm_area_struct *vma)
  3187. {
  3188. struct perf_event *event = vma->vm_file->private_data;
  3189. struct ring_buffer *rb = event->rb;
  3190. struct user_struct *mmap_user = rb->mmap_user;
  3191. int mmap_locked = rb->mmap_locked;
  3192. unsigned long size = perf_data_size(rb);
  3193. atomic_dec(&rb->mmap_count);
  3194. if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
  3195. return;
  3196. /* Detach current event from the buffer. */
  3197. rcu_assign_pointer(event->rb, NULL);
  3198. ring_buffer_detach(event, rb);
  3199. mutex_unlock(&event->mmap_mutex);
  3200. /* If there's still other mmap()s of this buffer, we're done. */
  3201. if (atomic_read(&rb->mmap_count)) {
  3202. ring_buffer_put(rb); /* can't be last */
  3203. return;
  3204. }
  3205. /*
  3206. * No other mmap()s, detach from all other events that might redirect
  3207. * into the now unreachable buffer. Somewhat complicated by the
  3208. * fact that rb::event_lock otherwise nests inside mmap_mutex.
  3209. */
  3210. again:
  3211. rcu_read_lock();
  3212. list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
  3213. if (!atomic_long_inc_not_zero(&event->refcount)) {
  3214. /*
  3215. * This event is en-route to free_event() which will
  3216. * detach it and remove it from the list.
  3217. */
  3218. continue;
  3219. }
  3220. rcu_read_unlock();
  3221. mutex_lock(&event->mmap_mutex);
  3222. /*
  3223. * Check we didn't race with perf_event_set_output() which can
  3224. * swizzle the rb from under us while we were waiting to
  3225. * acquire mmap_mutex.
  3226. *
  3227. * If we find a different rb; ignore this event, a next
  3228. * iteration will no longer find it on the list. We have to
  3229. * still restart the iteration to make sure we're not now
  3230. * iterating the wrong list.
  3231. */
  3232. if (event->rb == rb) {
  3233. rcu_assign_pointer(event->rb, NULL);
  3234. ring_buffer_detach(event, rb);
  3235. ring_buffer_put(rb); /* can't be last, we still have one */
  3236. }
  3237. mutex_unlock(&event->mmap_mutex);
  3238. put_event(event);
  3239. /*
  3240. * Restart the iteration; either we're on the wrong list or
  3241. * destroyed its integrity by doing a deletion.
  3242. */
  3243. goto again;
  3244. }
  3245. rcu_read_unlock();
  3246. /*
  3247. * It could be there's still a few 0-ref events on the list; they'll
  3248. * get cleaned up by free_event() -- they'll also still have their
  3249. * ref on the rb and will free it whenever they are done with it.
  3250. *
  3251. * Aside from that, this buffer is 'fully' detached and unmapped,
  3252. * undo the VM accounting.
  3253. */
  3254. atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
  3255. vma->vm_mm->pinned_vm -= mmap_locked;
  3256. free_uid(mmap_user);
  3257. ring_buffer_put(rb); /* could be last */
  3258. }
  3259. static const struct vm_operations_struct perf_mmap_vmops = {
  3260. .open = perf_mmap_open,
  3261. .close = perf_mmap_close,
  3262. .fault = perf_mmap_fault,
  3263. .page_mkwrite = perf_mmap_fault,
  3264. };
  3265. static int perf_mmap(struct file *file, struct vm_area_struct *vma)
  3266. {
  3267. struct perf_event *event = file->private_data;
  3268. unsigned long user_locked, user_lock_limit;
  3269. struct user_struct *user = current_user();
  3270. unsigned long locked, lock_limit;
  3271. struct ring_buffer *rb;
  3272. unsigned long vma_size;
  3273. unsigned long nr_pages;
  3274. long user_extra, extra;
  3275. int ret = 0, flags = 0;
  3276. /*
  3277. * Don't allow mmap() of inherited per-task counters. This would
  3278. * create a performance issue due to all children writing to the
  3279. * same rb.
  3280. */
  3281. if (event->cpu == -1 && event->attr.inherit)
  3282. return -EINVAL;
  3283. if (!(vma->vm_flags & VM_SHARED))
  3284. return -EINVAL;
  3285. vma_size = vma->vm_end - vma->vm_start;
  3286. nr_pages = (vma_size / PAGE_SIZE) - 1;
  3287. /*
  3288. * If we have rb pages ensure they're a power-of-two number, so we
  3289. * can do bitmasks instead of modulo.
  3290. */
  3291. if (nr_pages != 0 && !is_power_of_2(nr_pages))
  3292. return -EINVAL;
  3293. if (vma_size != PAGE_SIZE * (1 + nr_pages))
  3294. return -EINVAL;
  3295. if (vma->vm_pgoff != 0)
  3296. return -EINVAL;
  3297. WARN_ON_ONCE(event->ctx->parent_ctx);
  3298. again:
  3299. mutex_lock(&event->mmap_mutex);
  3300. if (event->rb) {
  3301. if (event->rb->nr_pages != nr_pages) {
  3302. ret = -EINVAL;
  3303. goto unlock;
  3304. }
  3305. if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
  3306. /*
  3307. * Raced against perf_mmap_close() through
  3308. * perf_event_set_output(). Try again, hope for better
  3309. * luck.
  3310. */
  3311. mutex_unlock(&event->mmap_mutex);
  3312. goto again;
  3313. }
  3314. goto unlock;
  3315. }
  3316. user_extra = nr_pages + 1;
  3317. user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
  3318. /*
  3319. * Increase the limit linearly with more CPUs:
  3320. */
  3321. user_lock_limit *= num_online_cpus();
  3322. user_locked = atomic_long_read(&user->locked_vm) + user_extra;
  3323. extra = 0;
  3324. if (user_locked > user_lock_limit)
  3325. extra = user_locked - user_lock_limit;
  3326. lock_limit = rlimit(RLIMIT_MEMLOCK);
  3327. lock_limit >>= PAGE_SHIFT;
  3328. locked = vma->vm_mm->pinned_vm + extra;
  3329. if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
  3330. !capable(CAP_IPC_LOCK)) {
  3331. ret = -EPERM;
  3332. goto unlock;
  3333. }
  3334. WARN_ON(event->rb);
  3335. if (vma->vm_flags & VM_WRITE)
  3336. flags |= RING_BUFFER_WRITABLE;
  3337. rb = rb_alloc(nr_pages,
  3338. event->attr.watermark ? event->attr.wakeup_watermark : 0,
  3339. event->cpu, flags);
  3340. if (!rb) {
  3341. ret = -ENOMEM;
  3342. goto unlock;
  3343. }
  3344. atomic_set(&rb->mmap_count, 1);
  3345. rb->mmap_locked = extra;
  3346. rb->mmap_user = get_current_user();
  3347. atomic_long_add(user_extra, &user->locked_vm);
  3348. vma->vm_mm->pinned_vm += extra;
  3349. ring_buffer_attach(event, rb);
  3350. rcu_assign_pointer(event->rb, rb);
  3351. perf_event_update_userpage(event);
  3352. unlock:
  3353. if (!ret)
  3354. atomic_inc(&event->mmap_count);
  3355. mutex_unlock(&event->mmap_mutex);
  3356. /*
  3357. * Since pinned accounting is per vm we cannot allow fork() to copy our
  3358. * vma.
  3359. */
  3360. vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
  3361. vma->vm_ops = &perf_mmap_vmops;
  3362. return ret;
  3363. }
  3364. static int perf_fasync(int fd, struct file *filp, int on)
  3365. {
  3366. struct inode *inode = file_inode(filp);
  3367. struct perf_event *event = filp->private_data;
  3368. int retval;
  3369. mutex_lock(&inode->i_mutex);
  3370. retval = fasync_helper(fd, filp, on, &event->fasync);
  3371. mutex_unlock(&inode->i_mutex);
  3372. if (retval < 0)
  3373. return retval;
  3374. return 0;
  3375. }
  3376. static const struct file_operations perf_fops = {
  3377. .llseek = no_llseek,
  3378. .release = perf_release,
  3379. .read = perf_read,
  3380. .poll = perf_poll,
  3381. .unlocked_ioctl = perf_ioctl,
  3382. .compat_ioctl = perf_ioctl,
  3383. .mmap = perf_mmap,
  3384. .fasync = perf_fasync,
  3385. };
  3386. /*
  3387. * Perf event wakeup
  3388. *
  3389. * If there's data, ensure we set the poll() state and publish everything
  3390. * to user-space before waking everybody up.
  3391. */
  3392. void perf_event_wakeup(struct perf_event *event)
  3393. {
  3394. ring_buffer_wakeup(event);
  3395. if (event->pending_kill) {
  3396. kill_fasync(&event->fasync, SIGIO, event->pending_kill);
  3397. event->pending_kill = 0;
  3398. }
  3399. }
  3400. static void perf_pending_event(struct irq_work *entry)
  3401. {
  3402. struct perf_event *event = container_of(entry,
  3403. struct perf_event, pending);
  3404. if (event->pending_disable) {
  3405. event->pending_disable = 0;
  3406. __perf_event_disable(event);
  3407. }
  3408. if (event->pending_wakeup) {
  3409. event->pending_wakeup = 0;
  3410. perf_event_wakeup(event);
  3411. }
  3412. }
  3413. /*
  3414. * We assume there is only KVM supporting the callbacks.
  3415. * Later on, we might change it to a list if there is
  3416. * another virtualization implementation supporting the callbacks.
  3417. */
  3418. struct perf_guest_info_callbacks *perf_guest_cbs;
  3419. int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
  3420. {
  3421. perf_guest_cbs = cbs;
  3422. return 0;
  3423. }
  3424. EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
  3425. int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
  3426. {
  3427. perf_guest_cbs = NULL;
  3428. return 0;
  3429. }
  3430. EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
  3431. static void
  3432. perf_output_sample_regs(struct perf_output_handle *handle,
  3433. struct pt_regs *regs, u64 mask)
  3434. {
  3435. int bit;
  3436. for_each_set_bit(bit, (const unsigned long *) &mask,
  3437. sizeof(mask) * BITS_PER_BYTE) {
  3438. u64 val;
  3439. val = perf_reg_value(regs, bit);
  3440. perf_output_put(handle, val);
  3441. }
  3442. }
  3443. static void perf_sample_regs_user(struct perf_regs_user *regs_user,
  3444. struct pt_regs *regs)
  3445. {
  3446. if (!user_mode(regs)) {
  3447. if (current->mm)
  3448. regs = task_pt_regs(current);
  3449. else
  3450. regs = NULL;
  3451. }
  3452. if (regs) {
  3453. regs_user->regs = regs;
  3454. regs_user->abi = perf_reg_abi(current);
  3455. }
  3456. }
  3457. /*
  3458. * Get remaining task size from user stack pointer.
  3459. *
  3460. * It'd be better to take stack vma map and limit this more
  3461. * precisly, but there's no way to get it safely under interrupt,
  3462. * so using TASK_SIZE as limit.
  3463. */
  3464. static u64 perf_ustack_task_size(struct pt_regs *regs)
  3465. {
  3466. unsigned long addr = perf_user_stack_pointer(regs);
  3467. if (!addr || addr >= TASK_SIZE)
  3468. return 0;
  3469. return TASK_SIZE - addr;
  3470. }
  3471. static u16
  3472. perf_sample_ustack_size(u16 stack_size, u16 header_size,
  3473. struct pt_regs *regs)
  3474. {
  3475. u64 task_size;
  3476. /* No regs, no stack pointer, no dump. */
  3477. if (!regs)
  3478. return 0;
  3479. /*
  3480. * Check if we fit in with the requested stack size into the:
  3481. * - TASK_SIZE
  3482. * If we don't, we limit the size to the TASK_SIZE.
  3483. *
  3484. * - remaining sample size
  3485. * If we don't, we customize the stack size to
  3486. * fit in to the remaining sample size.
  3487. */
  3488. task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
  3489. stack_size = min(stack_size, (u16) task_size);
  3490. /* Current header size plus static size and dynamic size. */
  3491. header_size += 2 * sizeof(u64);
  3492. /* Do we fit in with the current stack dump size? */
  3493. if ((u16) (header_size + stack_size) < header_size) {
  3494. /*
  3495. * If we overflow the maximum size for the sample,
  3496. * we customize the stack dump size to fit in.
  3497. */
  3498. stack_size = USHRT_MAX - header_size - sizeof(u64);
  3499. stack_size = round_up(stack_size, sizeof(u64));
  3500. }
  3501. return stack_size;
  3502. }
  3503. static void
  3504. perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
  3505. struct pt_regs *regs)
  3506. {
  3507. /* Case of a kernel thread, nothing to dump */
  3508. if (!regs) {
  3509. u64 size = 0;
  3510. perf_output_put(handle, size);
  3511. } else {
  3512. unsigned long sp;
  3513. unsigned int rem;
  3514. u64 dyn_size;
  3515. /*
  3516. * We dump:
  3517. * static size
  3518. * - the size requested by user or the best one we can fit
  3519. * in to the sample max size
  3520. * data
  3521. * - user stack dump data
  3522. * dynamic size
  3523. * - the actual dumped size
  3524. */
  3525. /* Static size. */
  3526. perf_output_put(handle, dump_size);
  3527. /* Data. */
  3528. sp = perf_user_stack_pointer(regs);
  3529. rem = __output_copy_user(handle, (void *) sp, dump_size);
  3530. dyn_size = dump_size - rem;
  3531. perf_output_skip(handle, rem);
  3532. /* Dynamic size. */
  3533. perf_output_put(handle, dyn_size);
  3534. }
  3535. }
  3536. static void __perf_event_header__init_id(struct perf_event_header *header,
  3537. struct perf_sample_data *data,
  3538. struct perf_event *event)
  3539. {
  3540. u64 sample_type = event->attr.sample_type;
  3541. data->type = sample_type;
  3542. header->size += event->id_header_size;
  3543. if (sample_type & PERF_SAMPLE_TID) {
  3544. /* namespace issues */
  3545. data->tid_entry.pid = perf_event_pid(event, current);
  3546. data->tid_entry.tid = perf_event_tid(event, current);
  3547. }
  3548. if (sample_type & PERF_SAMPLE_TIME)
  3549. data->time = perf_clock();
  3550. if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
  3551. data->id = primary_event_id(event);
  3552. if (sample_type & PERF_SAMPLE_STREAM_ID)
  3553. data->stream_id = event->id;
  3554. if (sample_type & PERF_SAMPLE_CPU) {
  3555. data->cpu_entry.cpu = raw_smp_processor_id();
  3556. data->cpu_entry.reserved = 0;
  3557. }
  3558. }
  3559. void perf_event_header__init_id(struct perf_event_header *header,
  3560. struct perf_sample_data *data,
  3561. struct perf_event *event)
  3562. {
  3563. if (event->attr.sample_id_all)
  3564. __perf_event_header__init_id(header, data, event);
  3565. }
  3566. static void __perf_event__output_id_sample(struct perf_output_handle *handle,
  3567. struct perf_sample_data *data)
  3568. {
  3569. u64 sample_type = data->type;
  3570. if (sample_type & PERF_SAMPLE_TID)
  3571. perf_output_put(handle, data->tid_entry);
  3572. if (sample_type & PERF_SAMPLE_TIME)
  3573. perf_output_put(handle, data->time);
  3574. if (sample_type & PERF_SAMPLE_ID)
  3575. perf_output_put(handle, data->id);
  3576. if (sample_type & PERF_SAMPLE_STREAM_ID)
  3577. perf_output_put(handle, data->stream_id);
  3578. if (sample_type & PERF_SAMPLE_CPU)
  3579. perf_output_put(handle, data->cpu_entry);
  3580. if (sample_type & PERF_SAMPLE_IDENTIFIER)
  3581. perf_output_put(handle, data->id);
  3582. }
  3583. void perf_event__output_id_sample(struct perf_event *event,
  3584. struct perf_output_handle *handle,
  3585. struct perf_sample_data *sample)
  3586. {
  3587. if (event->attr.sample_id_all)
  3588. __perf_event__output_id_sample(handle, sample);
  3589. }
  3590. static void perf_output_read_one(struct perf_output_handle *handle,
  3591. struct perf_event *event,
  3592. u64 enabled, u64 running)
  3593. {
  3594. u64 read_format = event->attr.read_format;
  3595. u64 values[4];
  3596. int n = 0;
  3597. values[n++] = perf_event_count(event);
  3598. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
  3599. values[n++] = enabled +
  3600. atomic64_read(&event->child_total_time_enabled);
  3601. }
  3602. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
  3603. values[n++] = running +
  3604. atomic64_read(&event->child_total_time_running);
  3605. }
  3606. if (read_format & PERF_FORMAT_ID)
  3607. values[n++] = primary_event_id(event);
  3608. __output_copy(handle, values, n * sizeof(u64));
  3609. }
  3610. /*
  3611. * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
  3612. */
  3613. static void perf_output_read_group(struct perf_output_handle *handle,
  3614. struct perf_event *event,
  3615. u64 enabled, u64 running)
  3616. {
  3617. struct perf_event *leader = event->group_leader, *sub;
  3618. u64 read_format = event->attr.read_format;
  3619. u64 values[5];
  3620. int n = 0;
  3621. values[n++] = 1 + leader->nr_siblings;
  3622. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  3623. values[n++] = enabled;
  3624. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  3625. values[n++] = running;
  3626. if (leader != event)
  3627. leader->pmu->read(leader);
  3628. values[n++] = perf_event_count(leader);
  3629. if (read_format & PERF_FORMAT_ID)
  3630. values[n++] = primary_event_id(leader);
  3631. __output_copy(handle, values, n * sizeof(u64));
  3632. list_for_each_entry(sub, &leader->sibling_list, group_entry) {
  3633. n = 0;
  3634. if ((sub != event) &&
  3635. (sub->state == PERF_EVENT_STATE_ACTIVE))
  3636. sub->pmu->read(sub);
  3637. values[n++] = perf_event_count(sub);
  3638. if (read_format & PERF_FORMAT_ID)
  3639. values[n++] = primary_event_id(sub);
  3640. __output_copy(handle, values, n * sizeof(u64));
  3641. }
  3642. }
  3643. #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
  3644. PERF_FORMAT_TOTAL_TIME_RUNNING)
  3645. static void perf_output_read(struct perf_output_handle *handle,
  3646. struct perf_event *event)
  3647. {
  3648. u64 enabled = 0, running = 0, now;
  3649. u64 read_format = event->attr.read_format;
  3650. /*
  3651. * compute total_time_enabled, total_time_running
  3652. * based on snapshot values taken when the event
  3653. * was last scheduled in.
  3654. *
  3655. * we cannot simply called update_context_time()
  3656. * because of locking issue as we are called in
  3657. * NMI context
  3658. */
  3659. if (read_format & PERF_FORMAT_TOTAL_TIMES)
  3660. calc_timer_values(event, &now, &enabled, &running);
  3661. if (event->attr.read_format & PERF_FORMAT_GROUP)
  3662. perf_output_read_group(handle, event, enabled, running);
  3663. else
  3664. perf_output_read_one(handle, event, enabled, running);
  3665. }
  3666. void perf_output_sample(struct perf_output_handle *handle,
  3667. struct perf_event_header *header,
  3668. struct perf_sample_data *data,
  3669. struct perf_event *event)
  3670. {
  3671. u64 sample_type = data->type;
  3672. perf_output_put(handle, *header);
  3673. if (sample_type & PERF_SAMPLE_IDENTIFIER)
  3674. perf_output_put(handle, data->id);
  3675. if (sample_type & PERF_SAMPLE_IP)
  3676. perf_output_put(handle, data->ip);
  3677. if (sample_type & PERF_SAMPLE_TID)
  3678. perf_output_put(handle, data->tid_entry);
  3679. if (sample_type & PERF_SAMPLE_TIME)
  3680. perf_output_put(handle, data->time);
  3681. if (sample_type & PERF_SAMPLE_ADDR)
  3682. perf_output_put(handle, data->addr);
  3683. if (sample_type & PERF_SAMPLE_ID)
  3684. perf_output_put(handle, data->id);
  3685. if (sample_type & PERF_SAMPLE_STREAM_ID)
  3686. perf_output_put(handle, data->stream_id);
  3687. if (sample_type & PERF_SAMPLE_CPU)
  3688. perf_output_put(handle, data->cpu_entry);
  3689. if (sample_type & PERF_SAMPLE_PERIOD)
  3690. perf_output_put(handle, data->period);
  3691. if (sample_type & PERF_SAMPLE_READ)
  3692. perf_output_read(handle, event);
  3693. if (sample_type & PERF_SAMPLE_CALLCHAIN) {
  3694. if (data->callchain) {
  3695. int size = 1;
  3696. if (data->callchain)
  3697. size += data->callchain->nr;
  3698. size *= sizeof(u64);
  3699. __output_copy(handle, data->callchain, size);
  3700. } else {
  3701. u64 nr = 0;
  3702. perf_output_put(handle, nr);
  3703. }
  3704. }
  3705. if (sample_type & PERF_SAMPLE_RAW) {
  3706. if (data->raw) {
  3707. perf_output_put(handle, data->raw->size);
  3708. __output_copy(handle, data->raw->data,
  3709. data->raw->size);
  3710. } else {
  3711. struct {
  3712. u32 size;
  3713. u32 data;
  3714. } raw = {
  3715. .size = sizeof(u32),
  3716. .data = 0,
  3717. };
  3718. perf_output_put(handle, raw);
  3719. }
  3720. }
  3721. if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
  3722. if (data->br_stack) {
  3723. size_t size;
  3724. size = data->br_stack->nr
  3725. * sizeof(struct perf_branch_entry);
  3726. perf_output_put(handle, data->br_stack->nr);
  3727. perf_output_copy(handle, data->br_stack->entries, size);
  3728. } else {
  3729. /*
  3730. * we always store at least the value of nr
  3731. */
  3732. u64 nr = 0;
  3733. perf_output_put(handle, nr);
  3734. }
  3735. }
  3736. if (sample_type & PERF_SAMPLE_REGS_USER) {
  3737. u64 abi = data->regs_user.abi;
  3738. /*
  3739. * If there are no regs to dump, notice it through
  3740. * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
  3741. */
  3742. perf_output_put(handle, abi);
  3743. if (abi) {
  3744. u64 mask = event->attr.sample_regs_user;
  3745. perf_output_sample_regs(handle,
  3746. data->regs_user.regs,
  3747. mask);
  3748. }
  3749. }
  3750. if (sample_type & PERF_SAMPLE_STACK_USER) {
  3751. perf_output_sample_ustack(handle,
  3752. data->stack_user_size,
  3753. data->regs_user.regs);
  3754. }
  3755. if (sample_type & PERF_SAMPLE_WEIGHT)
  3756. perf_output_put(handle, data->weight);
  3757. if (sample_type & PERF_SAMPLE_DATA_SRC)
  3758. perf_output_put(handle, data->data_src.val);
  3759. if (!event->attr.watermark) {
  3760. int wakeup_events = event->attr.wakeup_events;
  3761. if (wakeup_events) {
  3762. struct ring_buffer *rb = handle->rb;
  3763. int events = local_inc_return(&rb->events);
  3764. if (events >= wakeup_events) {
  3765. local_sub(wakeup_events, &rb->events);
  3766. local_inc(&rb->wakeup);
  3767. }
  3768. }
  3769. }
  3770. }
  3771. void perf_prepare_sample(struct perf_event_header *header,
  3772. struct perf_sample_data *data,
  3773. struct perf_event *event,
  3774. struct pt_regs *regs)
  3775. {
  3776. u64 sample_type = event->attr.sample_type;
  3777. header->type = PERF_RECORD_SAMPLE;
  3778. header->size = sizeof(*header) + event->header_size;
  3779. header->misc = 0;
  3780. header->misc |= perf_misc_flags(regs);
  3781. __perf_event_header__init_id(header, data, event);
  3782. if (sample_type & PERF_SAMPLE_IP)
  3783. data->ip = perf_instruction_pointer(regs);
  3784. if (sample_type & PERF_SAMPLE_CALLCHAIN) {
  3785. int size = 1;
  3786. data->callchain = perf_callchain(event, regs);
  3787. if (data->callchain)
  3788. size += data->callchain->nr;
  3789. header->size += size * sizeof(u64);
  3790. }
  3791. if (sample_type & PERF_SAMPLE_RAW) {
  3792. int size = sizeof(u32);
  3793. if (data->raw)
  3794. size += data->raw->size;
  3795. else
  3796. size += sizeof(u32);
  3797. WARN_ON_ONCE(size & (sizeof(u64)-1));
  3798. header->size += size;
  3799. }
  3800. if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
  3801. int size = sizeof(u64); /* nr */
  3802. if (data->br_stack) {
  3803. size += data->br_stack->nr
  3804. * sizeof(struct perf_branch_entry);
  3805. }
  3806. header->size += size;
  3807. }
  3808. if (sample_type & PERF_SAMPLE_REGS_USER) {
  3809. /* regs dump ABI info */
  3810. int size = sizeof(u64);
  3811. perf_sample_regs_user(&data->regs_user, regs);
  3812. if (data->regs_user.regs) {
  3813. u64 mask = event->attr.sample_regs_user;
  3814. size += hweight64(mask) * sizeof(u64);
  3815. }
  3816. header->size += size;
  3817. }
  3818. if (sample_type & PERF_SAMPLE_STACK_USER) {
  3819. /*
  3820. * Either we need PERF_SAMPLE_STACK_USER bit to be allways
  3821. * processed as the last one or have additional check added
  3822. * in case new sample type is added, because we could eat
  3823. * up the rest of the sample size.
  3824. */
  3825. struct perf_regs_user *uregs = &data->regs_user;
  3826. u16 stack_size = event->attr.sample_stack_user;
  3827. u16 size = sizeof(u64);
  3828. if (!uregs->abi)
  3829. perf_sample_regs_user(uregs, regs);
  3830. stack_size = perf_sample_ustack_size(stack_size, header->size,
  3831. uregs->regs);
  3832. /*
  3833. * If there is something to dump, add space for the dump
  3834. * itself and for the field that tells the dynamic size,
  3835. * which is how many have been actually dumped.
  3836. */
  3837. if (stack_size)
  3838. size += sizeof(u64) + stack_size;
  3839. data->stack_user_size = stack_size;
  3840. header->size += size;
  3841. }
  3842. }
  3843. static void perf_event_output(struct perf_event *event,
  3844. struct perf_sample_data *data,
  3845. struct pt_regs *regs)
  3846. {
  3847. struct perf_output_handle handle;
  3848. struct perf_event_header header;
  3849. /* protect the callchain buffers */
  3850. rcu_read_lock();
  3851. perf_prepare_sample(&header, data, event, regs);
  3852. if (perf_output_begin(&handle, event, header.size))
  3853. goto exit;
  3854. perf_output_sample(&handle, &header, data, event);
  3855. perf_output_end(&handle);
  3856. exit:
  3857. rcu_read_unlock();
  3858. }
  3859. /*
  3860. * read event_id
  3861. */
  3862. struct perf_read_event {
  3863. struct perf_event_header header;
  3864. u32 pid;
  3865. u32 tid;
  3866. };
  3867. static void
  3868. perf_event_read_event(struct perf_event *event,
  3869. struct task_struct *task)
  3870. {
  3871. struct perf_output_handle handle;
  3872. struct perf_sample_data sample;
  3873. struct perf_read_event read_event = {
  3874. .header = {
  3875. .type = PERF_RECORD_READ,
  3876. .misc = 0,
  3877. .size = sizeof(read_event) + event->read_size,
  3878. },
  3879. .pid = perf_event_pid(event, task),
  3880. .tid = perf_event_tid(event, task),
  3881. };
  3882. int ret;
  3883. perf_event_header__init_id(&read_event.header, &sample, event);
  3884. ret = perf_output_begin(&handle, event, read_event.header.size);
  3885. if (ret)
  3886. return;
  3887. perf_output_put(&handle, read_event);
  3888. perf_output_read(&handle, event);
  3889. perf_event__output_id_sample(event, &handle, &sample);
  3890. perf_output_end(&handle);
  3891. }
  3892. typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
  3893. static void
  3894. perf_event_aux_ctx(struct perf_event_context *ctx,
  3895. perf_event_aux_output_cb output,
  3896. void *data)
  3897. {
  3898. struct perf_event *event;
  3899. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  3900. if (event->state < PERF_EVENT_STATE_INACTIVE)
  3901. continue;
  3902. if (!event_filter_match(event))
  3903. continue;
  3904. output(event, data);
  3905. }
  3906. }
  3907. static void
  3908. perf_event_aux(perf_event_aux_output_cb output, void *data,
  3909. struct perf_event_context *task_ctx)
  3910. {
  3911. struct perf_cpu_context *cpuctx;
  3912. struct perf_event_context *ctx;
  3913. struct pmu *pmu;
  3914. int ctxn;
  3915. rcu_read_lock();
  3916. list_for_each_entry_rcu(pmu, &pmus, entry) {
  3917. cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
  3918. if (cpuctx->unique_pmu != pmu)
  3919. goto next;
  3920. perf_event_aux_ctx(&cpuctx->ctx, output, data);
  3921. if (task_ctx)
  3922. goto next;
  3923. ctxn = pmu->task_ctx_nr;
  3924. if (ctxn < 0)
  3925. goto next;
  3926. ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
  3927. if (ctx)
  3928. perf_event_aux_ctx(ctx, output, data);
  3929. next:
  3930. put_cpu_ptr(pmu->pmu_cpu_context);
  3931. }
  3932. if (task_ctx) {
  3933. preempt_disable();
  3934. perf_event_aux_ctx(task_ctx, output, data);
  3935. preempt_enable();
  3936. }
  3937. rcu_read_unlock();
  3938. }
  3939. /*
  3940. * task tracking -- fork/exit
  3941. *
  3942. * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
  3943. */
  3944. struct perf_task_event {
  3945. struct task_struct *task;
  3946. struct perf_event_context *task_ctx;
  3947. struct {
  3948. struct perf_event_header header;
  3949. u32 pid;
  3950. u32 ppid;
  3951. u32 tid;
  3952. u32 ptid;
  3953. u64 time;
  3954. } event_id;
  3955. };
  3956. static int perf_event_task_match(struct perf_event *event)
  3957. {
  3958. return event->attr.comm || event->attr.mmap ||
  3959. event->attr.mmap2 || event->attr.mmap_data ||
  3960. event->attr.task;
  3961. }
  3962. static void perf_event_task_output(struct perf_event *event,
  3963. void *data)
  3964. {
  3965. struct perf_task_event *task_event = data;
  3966. struct perf_output_handle handle;
  3967. struct perf_sample_data sample;
  3968. struct task_struct *task = task_event->task;
  3969. int ret, size = task_event->event_id.header.size;
  3970. if (!perf_event_task_match(event))
  3971. return;
  3972. perf_event_header__init_id(&task_event->event_id.header, &sample, event);
  3973. ret = perf_output_begin(&handle, event,
  3974. task_event->event_id.header.size);
  3975. if (ret)
  3976. goto out;
  3977. task_event->event_id.pid = perf_event_pid(event, task);
  3978. task_event->event_id.ppid = perf_event_pid(event, current);
  3979. task_event->event_id.tid = perf_event_tid(event, task);
  3980. task_event->event_id.ptid = perf_event_tid(event, current);
  3981. perf_output_put(&handle, task_event->event_id);
  3982. perf_event__output_id_sample(event, &handle, &sample);
  3983. perf_output_end(&handle);
  3984. out:
  3985. task_event->event_id.header.size = size;
  3986. }
  3987. static void perf_event_task(struct task_struct *task,
  3988. struct perf_event_context *task_ctx,
  3989. int new)
  3990. {
  3991. struct perf_task_event task_event;
  3992. if (!atomic_read(&nr_comm_events) &&
  3993. !atomic_read(&nr_mmap_events) &&
  3994. !atomic_read(&nr_task_events))
  3995. return;
  3996. task_event = (struct perf_task_event){
  3997. .task = task,
  3998. .task_ctx = task_ctx,
  3999. .event_id = {
  4000. .header = {
  4001. .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
  4002. .misc = 0,
  4003. .size = sizeof(task_event.event_id),
  4004. },
  4005. /* .pid */
  4006. /* .ppid */
  4007. /* .tid */
  4008. /* .ptid */
  4009. .time = perf_clock(),
  4010. },
  4011. };
  4012. perf_event_aux(perf_event_task_output,
  4013. &task_event,
  4014. task_ctx);
  4015. }
  4016. void perf_event_fork(struct task_struct *task)
  4017. {
  4018. perf_event_task(task, NULL, 1);
  4019. }
  4020. /*
  4021. * comm tracking
  4022. */
  4023. struct perf_comm_event {
  4024. struct task_struct *task;
  4025. char *comm;
  4026. int comm_size;
  4027. struct {
  4028. struct perf_event_header header;
  4029. u32 pid;
  4030. u32 tid;
  4031. } event_id;
  4032. };
  4033. static int perf_event_comm_match(struct perf_event *event)
  4034. {
  4035. return event->attr.comm;
  4036. }
  4037. static void perf_event_comm_output(struct perf_event *event,
  4038. void *data)
  4039. {
  4040. struct perf_comm_event *comm_event = data;
  4041. struct perf_output_handle handle;
  4042. struct perf_sample_data sample;
  4043. int size = comm_event->event_id.header.size;
  4044. int ret;
  4045. if (!perf_event_comm_match(event))
  4046. return;
  4047. perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
  4048. ret = perf_output_begin(&handle, event,
  4049. comm_event->event_id.header.size);
  4050. if (ret)
  4051. goto out;
  4052. comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
  4053. comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
  4054. perf_output_put(&handle, comm_event->event_id);
  4055. __output_copy(&handle, comm_event->comm,
  4056. comm_event->comm_size);
  4057. perf_event__output_id_sample(event, &handle, &sample);
  4058. perf_output_end(&handle);
  4059. out:
  4060. comm_event->event_id.header.size = size;
  4061. }
  4062. static void perf_event_comm_event(struct perf_comm_event *comm_event)
  4063. {
  4064. char comm[TASK_COMM_LEN];
  4065. unsigned int size;
  4066. memset(comm, 0, sizeof(comm));
  4067. strlcpy(comm, comm_event->task->comm, sizeof(comm));
  4068. size = ALIGN(strlen(comm)+1, sizeof(u64));
  4069. comm_event->comm = comm;
  4070. comm_event->comm_size = size;
  4071. comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
  4072. perf_event_aux(perf_event_comm_output,
  4073. comm_event,
  4074. NULL);
  4075. }
  4076. void perf_event_comm(struct task_struct *task)
  4077. {
  4078. struct perf_comm_event comm_event;
  4079. struct perf_event_context *ctx;
  4080. int ctxn;
  4081. rcu_read_lock();
  4082. for_each_task_context_nr(ctxn) {
  4083. ctx = task->perf_event_ctxp[ctxn];
  4084. if (!ctx)
  4085. continue;
  4086. perf_event_enable_on_exec(ctx);
  4087. }
  4088. rcu_read_unlock();
  4089. if (!atomic_read(&nr_comm_events))
  4090. return;
  4091. comm_event = (struct perf_comm_event){
  4092. .task = task,
  4093. /* .comm */
  4094. /* .comm_size */
  4095. .event_id = {
  4096. .header = {
  4097. .type = PERF_RECORD_COMM,
  4098. .misc = 0,
  4099. /* .size */
  4100. },
  4101. /* .pid */
  4102. /* .tid */
  4103. },
  4104. };
  4105. perf_event_comm_event(&comm_event);
  4106. }
  4107. /*
  4108. * mmap tracking
  4109. */
  4110. struct perf_mmap_event {
  4111. struct vm_area_struct *vma;
  4112. const char *file_name;
  4113. int file_size;
  4114. int maj, min;
  4115. u64 ino;
  4116. u64 ino_generation;
  4117. struct {
  4118. struct perf_event_header header;
  4119. u32 pid;
  4120. u32 tid;
  4121. u64 start;
  4122. u64 len;
  4123. u64 pgoff;
  4124. } event_id;
  4125. };
  4126. static int perf_event_mmap_match(struct perf_event *event,
  4127. void *data)
  4128. {
  4129. struct perf_mmap_event *mmap_event = data;
  4130. struct vm_area_struct *vma = mmap_event->vma;
  4131. int executable = vma->vm_flags & VM_EXEC;
  4132. return (!executable && event->attr.mmap_data) ||
  4133. (executable && (event->attr.mmap || event->attr.mmap2));
  4134. }
  4135. static void perf_event_mmap_output(struct perf_event *event,
  4136. void *data)
  4137. {
  4138. struct perf_mmap_event *mmap_event = data;
  4139. struct perf_output_handle handle;
  4140. struct perf_sample_data sample;
  4141. int size = mmap_event->event_id.header.size;
  4142. int ret;
  4143. if (!perf_event_mmap_match(event, data))
  4144. return;
  4145. if (event->attr.mmap2) {
  4146. mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
  4147. mmap_event->event_id.header.size += sizeof(mmap_event->maj);
  4148. mmap_event->event_id.header.size += sizeof(mmap_event->min);
  4149. mmap_event->event_id.header.size += sizeof(mmap_event->ino);
  4150. }
  4151. perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
  4152. ret = perf_output_begin(&handle, event,
  4153. mmap_event->event_id.header.size);
  4154. if (ret)
  4155. goto out;
  4156. mmap_event->event_id.pid = perf_event_pid(event, current);
  4157. mmap_event->event_id.tid = perf_event_tid(event, current);
  4158. perf_output_put(&handle, mmap_event->event_id);
  4159. if (event->attr.mmap2) {
  4160. perf_output_put(&handle, mmap_event->maj);
  4161. perf_output_put(&handle, mmap_event->min);
  4162. perf_output_put(&handle, mmap_event->ino);
  4163. perf_output_put(&handle, mmap_event->ino_generation);
  4164. }
  4165. __output_copy(&handle, mmap_event->file_name,
  4166. mmap_event->file_size);
  4167. perf_event__output_id_sample(event, &handle, &sample);
  4168. perf_output_end(&handle);
  4169. out:
  4170. mmap_event->event_id.header.size = size;
  4171. }
  4172. static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
  4173. {
  4174. struct vm_area_struct *vma = mmap_event->vma;
  4175. struct file *file = vma->vm_file;
  4176. int maj = 0, min = 0;
  4177. u64 ino = 0, gen = 0;
  4178. unsigned int size;
  4179. char tmp[16];
  4180. char *buf = NULL;
  4181. const char *name;
  4182. memset(tmp, 0, sizeof(tmp));
  4183. if (file) {
  4184. struct inode *inode;
  4185. dev_t dev;
  4186. /*
  4187. * d_path works from the end of the rb backwards, so we
  4188. * need to add enough zero bytes after the string to handle
  4189. * the 64bit alignment we do later.
  4190. */
  4191. buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
  4192. if (!buf) {
  4193. name = strncpy(tmp, "//enomem", sizeof(tmp));
  4194. goto got_name;
  4195. }
  4196. name = d_path(&file->f_path, buf, PATH_MAX);
  4197. if (IS_ERR(name)) {
  4198. name = strncpy(tmp, "//toolong", sizeof(tmp));
  4199. goto got_name;
  4200. }
  4201. inode = file_inode(vma->vm_file);
  4202. dev = inode->i_sb->s_dev;
  4203. ino = inode->i_ino;
  4204. gen = inode->i_generation;
  4205. maj = MAJOR(dev);
  4206. min = MINOR(dev);
  4207. } else {
  4208. if (arch_vma_name(mmap_event->vma)) {
  4209. name = strncpy(tmp, arch_vma_name(mmap_event->vma),
  4210. sizeof(tmp) - 1);
  4211. tmp[sizeof(tmp) - 1] = '\0';
  4212. goto got_name;
  4213. }
  4214. if (!vma->vm_mm) {
  4215. name = strncpy(tmp, "[vdso]", sizeof(tmp));
  4216. goto got_name;
  4217. } else if (vma->vm_start <= vma->vm_mm->start_brk &&
  4218. vma->vm_end >= vma->vm_mm->brk) {
  4219. name = strncpy(tmp, "[heap]", sizeof(tmp));
  4220. goto got_name;
  4221. } else if (vma->vm_start <= vma->vm_mm->start_stack &&
  4222. vma->vm_end >= vma->vm_mm->start_stack) {
  4223. name = strncpy(tmp, "[stack]", sizeof(tmp));
  4224. goto got_name;
  4225. }
  4226. name = strncpy(tmp, "//anon", sizeof(tmp));
  4227. goto got_name;
  4228. }
  4229. got_name:
  4230. size = ALIGN(strlen(name)+1, sizeof(u64));
  4231. mmap_event->file_name = name;
  4232. mmap_event->file_size = size;
  4233. mmap_event->maj = maj;
  4234. mmap_event->min = min;
  4235. mmap_event->ino = ino;
  4236. mmap_event->ino_generation = gen;
  4237. if (!(vma->vm_flags & VM_EXEC))
  4238. mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
  4239. mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
  4240. perf_event_aux(perf_event_mmap_output,
  4241. mmap_event,
  4242. NULL);
  4243. kfree(buf);
  4244. }
  4245. void perf_event_mmap(struct vm_area_struct *vma)
  4246. {
  4247. struct perf_mmap_event mmap_event;
  4248. if (!atomic_read(&nr_mmap_events))
  4249. return;
  4250. mmap_event = (struct perf_mmap_event){
  4251. .vma = vma,
  4252. /* .file_name */
  4253. /* .file_size */
  4254. .event_id = {
  4255. .header = {
  4256. .type = PERF_RECORD_MMAP,
  4257. .misc = PERF_RECORD_MISC_USER,
  4258. /* .size */
  4259. },
  4260. /* .pid */
  4261. /* .tid */
  4262. .start = vma->vm_start,
  4263. .len = vma->vm_end - vma->vm_start,
  4264. .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
  4265. },
  4266. /* .maj (attr_mmap2 only) */
  4267. /* .min (attr_mmap2 only) */
  4268. /* .ino (attr_mmap2 only) */
  4269. /* .ino_generation (attr_mmap2 only) */
  4270. };
  4271. perf_event_mmap_event(&mmap_event);
  4272. }
  4273. /*
  4274. * IRQ throttle logging
  4275. */
  4276. static void perf_log_throttle(struct perf_event *event, int enable)
  4277. {
  4278. struct perf_output_handle handle;
  4279. struct perf_sample_data sample;
  4280. int ret;
  4281. struct {
  4282. struct perf_event_header header;
  4283. u64 time;
  4284. u64 id;
  4285. u64 stream_id;
  4286. } throttle_event = {
  4287. .header = {
  4288. .type = PERF_RECORD_THROTTLE,
  4289. .misc = 0,
  4290. .size = sizeof(throttle_event),
  4291. },
  4292. .time = perf_clock(),
  4293. .id = primary_event_id(event),
  4294. .stream_id = event->id,
  4295. };
  4296. if (enable)
  4297. throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
  4298. perf_event_header__init_id(&throttle_event.header, &sample, event);
  4299. ret = perf_output_begin(&handle, event,
  4300. throttle_event.header.size);
  4301. if (ret)
  4302. return;
  4303. perf_output_put(&handle, throttle_event);
  4304. perf_event__output_id_sample(event, &handle, &sample);
  4305. perf_output_end(&handle);
  4306. }
  4307. /*
  4308. * Generic event overflow handling, sampling.
  4309. */
  4310. static int __perf_event_overflow(struct perf_event *event,
  4311. int throttle, struct perf_sample_data *data,
  4312. struct pt_regs *regs)
  4313. {
  4314. int events = atomic_read(&event->event_limit);
  4315. struct hw_perf_event *hwc = &event->hw;
  4316. u64 seq;
  4317. int ret = 0;
  4318. /*
  4319. * Non-sampling counters might still use the PMI to fold short
  4320. * hardware counters, ignore those.
  4321. */
  4322. if (unlikely(!is_sampling_event(event)))
  4323. return 0;
  4324. seq = __this_cpu_read(perf_throttled_seq);
  4325. if (seq != hwc->interrupts_seq) {
  4326. hwc->interrupts_seq = seq;
  4327. hwc->interrupts = 1;
  4328. } else {
  4329. hwc->interrupts++;
  4330. if (unlikely(throttle
  4331. && hwc->interrupts >= max_samples_per_tick)) {
  4332. __this_cpu_inc(perf_throttled_count);
  4333. hwc->interrupts = MAX_INTERRUPTS;
  4334. perf_log_throttle(event, 0);
  4335. tick_nohz_full_kick();
  4336. ret = 1;
  4337. }
  4338. }
  4339. if (event->attr.freq) {
  4340. u64 now = perf_clock();
  4341. s64 delta = now - hwc->freq_time_stamp;
  4342. hwc->freq_time_stamp = now;
  4343. if (delta > 0 && delta < 2*TICK_NSEC)
  4344. perf_adjust_period(event, delta, hwc->last_period, true);
  4345. }
  4346. /*
  4347. * XXX event_limit might not quite work as expected on inherited
  4348. * events
  4349. */
  4350. event->pending_kill = POLL_IN;
  4351. if (events && atomic_dec_and_test(&event->event_limit)) {
  4352. ret = 1;
  4353. event->pending_kill = POLL_HUP;
  4354. event->pending_disable = 1;
  4355. irq_work_queue(&event->pending);
  4356. }
  4357. if (event->overflow_handler)
  4358. event->overflow_handler(event, data, regs);
  4359. else
  4360. perf_event_output(event, data, regs);
  4361. if (event->fasync && event->pending_kill) {
  4362. event->pending_wakeup = 1;
  4363. irq_work_queue(&event->pending);
  4364. }
  4365. return ret;
  4366. }
  4367. int perf_event_overflow(struct perf_event *event,
  4368. struct perf_sample_data *data,
  4369. struct pt_regs *regs)
  4370. {
  4371. return __perf_event_overflow(event, 1, data, regs);
  4372. }
  4373. /*
  4374. * Generic software event infrastructure
  4375. */
  4376. struct swevent_htable {
  4377. struct swevent_hlist *swevent_hlist;
  4378. struct mutex hlist_mutex;
  4379. int hlist_refcount;
  4380. /* Recursion avoidance in each contexts */
  4381. int recursion[PERF_NR_CONTEXTS];
  4382. };
  4383. static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
  4384. /*
  4385. * We directly increment event->count and keep a second value in
  4386. * event->hw.period_left to count intervals. This period event
  4387. * is kept in the range [-sample_period, 0] so that we can use the
  4388. * sign as trigger.
  4389. */
  4390. u64 perf_swevent_set_period(struct perf_event *event)
  4391. {
  4392. struct hw_perf_event *hwc = &event->hw;
  4393. u64 period = hwc->last_period;
  4394. u64 nr, offset;
  4395. s64 old, val;
  4396. hwc->last_period = hwc->sample_period;
  4397. again:
  4398. old = val = local64_read(&hwc->period_left);
  4399. if (val < 0)
  4400. return 0;
  4401. nr = div64_u64(period + val, period);
  4402. offset = nr * period;
  4403. val -= offset;
  4404. if (local64_cmpxchg(&hwc->period_left, old, val) != old)
  4405. goto again;
  4406. return nr;
  4407. }
  4408. static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
  4409. struct perf_sample_data *data,
  4410. struct pt_regs *regs)
  4411. {
  4412. struct hw_perf_event *hwc = &event->hw;
  4413. int throttle = 0;
  4414. if (!overflow)
  4415. overflow = perf_swevent_set_period(event);
  4416. if (hwc->interrupts == MAX_INTERRUPTS)
  4417. return;
  4418. for (; overflow; overflow--) {
  4419. if (__perf_event_overflow(event, throttle,
  4420. data, regs)) {
  4421. /*
  4422. * We inhibit the overflow from happening when
  4423. * hwc->interrupts == MAX_INTERRUPTS.
  4424. */
  4425. break;
  4426. }
  4427. throttle = 1;
  4428. }
  4429. }
  4430. static void perf_swevent_event(struct perf_event *event, u64 nr,
  4431. struct perf_sample_data *data,
  4432. struct pt_regs *regs)
  4433. {
  4434. struct hw_perf_event *hwc = &event->hw;
  4435. local64_add(nr, &event->count);
  4436. if (!regs)
  4437. return;
  4438. if (!is_sampling_event(event))
  4439. return;
  4440. if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
  4441. data->period = nr;
  4442. return perf_swevent_overflow(event, 1, data, regs);
  4443. } else
  4444. data->period = event->hw.last_period;
  4445. if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
  4446. return perf_swevent_overflow(event, 1, data, regs);
  4447. if (local64_add_negative(nr, &hwc->period_left))
  4448. return;
  4449. perf_swevent_overflow(event, 0, data, regs);
  4450. }
  4451. static int perf_exclude_event(struct perf_event *event,
  4452. struct pt_regs *regs)
  4453. {
  4454. if (event->hw.state & PERF_HES_STOPPED)
  4455. return 1;
  4456. if (regs) {
  4457. if (event->attr.exclude_user && user_mode(regs))
  4458. return 1;
  4459. if (event->attr.exclude_kernel && !user_mode(regs))
  4460. return 1;
  4461. }
  4462. return 0;
  4463. }
  4464. static int perf_swevent_match(struct perf_event *event,
  4465. enum perf_type_id type,
  4466. u32 event_id,
  4467. struct perf_sample_data *data,
  4468. struct pt_regs *regs)
  4469. {
  4470. if (event->attr.type != type)
  4471. return 0;
  4472. if (event->attr.config != event_id)
  4473. return 0;
  4474. if (perf_exclude_event(event, regs))
  4475. return 0;
  4476. return 1;
  4477. }
  4478. static inline u64 swevent_hash(u64 type, u32 event_id)
  4479. {
  4480. u64 val = event_id | (type << 32);
  4481. return hash_64(val, SWEVENT_HLIST_BITS);
  4482. }
  4483. static inline struct hlist_head *
  4484. __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
  4485. {
  4486. u64 hash = swevent_hash(type, event_id);
  4487. return &hlist->heads[hash];
  4488. }
  4489. /* For the read side: events when they trigger */
  4490. static inline struct hlist_head *
  4491. find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
  4492. {
  4493. struct swevent_hlist *hlist;
  4494. hlist = rcu_dereference(swhash->swevent_hlist);
  4495. if (!hlist)
  4496. return NULL;
  4497. return __find_swevent_head(hlist, type, event_id);
  4498. }
  4499. /* For the event head insertion and removal in the hlist */
  4500. static inline struct hlist_head *
  4501. find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
  4502. {
  4503. struct swevent_hlist *hlist;
  4504. u32 event_id = event->attr.config;
  4505. u64 type = event->attr.type;
  4506. /*
  4507. * Event scheduling is always serialized against hlist allocation
  4508. * and release. Which makes the protected version suitable here.
  4509. * The context lock guarantees that.
  4510. */
  4511. hlist = rcu_dereference_protected(swhash->swevent_hlist,
  4512. lockdep_is_held(&event->ctx->lock));
  4513. if (!hlist)
  4514. return NULL;
  4515. return __find_swevent_head(hlist, type, event_id);
  4516. }
  4517. static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
  4518. u64 nr,
  4519. struct perf_sample_data *data,
  4520. struct pt_regs *regs)
  4521. {
  4522. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4523. struct perf_event *event;
  4524. struct hlist_head *head;
  4525. rcu_read_lock();
  4526. head = find_swevent_head_rcu(swhash, type, event_id);
  4527. if (!head)
  4528. goto end;
  4529. hlist_for_each_entry_rcu(event, head, hlist_entry) {
  4530. if (perf_swevent_match(event, type, event_id, data, regs))
  4531. perf_swevent_event(event, nr, data, regs);
  4532. }
  4533. end:
  4534. rcu_read_unlock();
  4535. }
  4536. int perf_swevent_get_recursion_context(void)
  4537. {
  4538. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4539. return get_recursion_context(swhash->recursion);
  4540. }
  4541. EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
  4542. inline void perf_swevent_put_recursion_context(int rctx)
  4543. {
  4544. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4545. put_recursion_context(swhash->recursion, rctx);
  4546. }
  4547. void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
  4548. {
  4549. struct perf_sample_data data;
  4550. int rctx;
  4551. preempt_disable_notrace();
  4552. rctx = perf_swevent_get_recursion_context();
  4553. if (rctx < 0)
  4554. return;
  4555. perf_sample_data_init(&data, addr, 0);
  4556. do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
  4557. perf_swevent_put_recursion_context(rctx);
  4558. preempt_enable_notrace();
  4559. }
  4560. static void perf_swevent_read(struct perf_event *event)
  4561. {
  4562. }
  4563. static int perf_swevent_add(struct perf_event *event, int flags)
  4564. {
  4565. struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
  4566. struct hw_perf_event *hwc = &event->hw;
  4567. struct hlist_head *head;
  4568. if (is_sampling_event(event)) {
  4569. hwc->last_period = hwc->sample_period;
  4570. perf_swevent_set_period(event);
  4571. }
  4572. hwc->state = !(flags & PERF_EF_START);
  4573. head = find_swevent_head(swhash, event);
  4574. if (WARN_ON_ONCE(!head))
  4575. return -EINVAL;
  4576. hlist_add_head_rcu(&event->hlist_entry, head);
  4577. return 0;
  4578. }
  4579. static void perf_swevent_del(struct perf_event *event, int flags)
  4580. {
  4581. hlist_del_rcu(&event->hlist_entry);
  4582. }
  4583. static void perf_swevent_start(struct perf_event *event, int flags)
  4584. {
  4585. event->hw.state = 0;
  4586. }
  4587. static void perf_swevent_stop(struct perf_event *event, int flags)
  4588. {
  4589. event->hw.state = PERF_HES_STOPPED;
  4590. }
  4591. /* Deref the hlist from the update side */
  4592. static inline struct swevent_hlist *
  4593. swevent_hlist_deref(struct swevent_htable *swhash)
  4594. {
  4595. return rcu_dereference_protected(swhash->swevent_hlist,
  4596. lockdep_is_held(&swhash->hlist_mutex));
  4597. }
  4598. static void swevent_hlist_release(struct swevent_htable *swhash)
  4599. {
  4600. struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
  4601. if (!hlist)
  4602. return;
  4603. rcu_assign_pointer(swhash->swevent_hlist, NULL);
  4604. kfree_rcu(hlist, rcu_head);
  4605. }
  4606. static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
  4607. {
  4608. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  4609. mutex_lock(&swhash->hlist_mutex);
  4610. if (!--swhash->hlist_refcount)
  4611. swevent_hlist_release(swhash);
  4612. mutex_unlock(&swhash->hlist_mutex);
  4613. }
  4614. static void swevent_hlist_put(struct perf_event *event)
  4615. {
  4616. int cpu;
  4617. if (event->cpu != -1) {
  4618. swevent_hlist_put_cpu(event, event->cpu);
  4619. return;
  4620. }
  4621. for_each_possible_cpu(cpu)
  4622. swevent_hlist_put_cpu(event, cpu);
  4623. }
  4624. static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
  4625. {
  4626. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  4627. int err = 0;
  4628. mutex_lock(&swhash->hlist_mutex);
  4629. if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
  4630. struct swevent_hlist *hlist;
  4631. hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
  4632. if (!hlist) {
  4633. err = -ENOMEM;
  4634. goto exit;
  4635. }
  4636. rcu_assign_pointer(swhash->swevent_hlist, hlist);
  4637. }
  4638. swhash->hlist_refcount++;
  4639. exit:
  4640. mutex_unlock(&swhash->hlist_mutex);
  4641. return err;
  4642. }
  4643. static int swevent_hlist_get(struct perf_event *event)
  4644. {
  4645. int err;
  4646. int cpu, failed_cpu;
  4647. if (event->cpu != -1)
  4648. return swevent_hlist_get_cpu(event, event->cpu);
  4649. get_online_cpus();
  4650. for_each_possible_cpu(cpu) {
  4651. err = swevent_hlist_get_cpu(event, cpu);
  4652. if (err) {
  4653. failed_cpu = cpu;
  4654. goto fail;
  4655. }
  4656. }
  4657. put_online_cpus();
  4658. return 0;
  4659. fail:
  4660. for_each_possible_cpu(cpu) {
  4661. if (cpu == failed_cpu)
  4662. break;
  4663. swevent_hlist_put_cpu(event, cpu);
  4664. }
  4665. put_online_cpus();
  4666. return err;
  4667. }
  4668. struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
  4669. static void sw_perf_event_destroy(struct perf_event *event)
  4670. {
  4671. u64 event_id = event->attr.config;
  4672. WARN_ON(event->parent);
  4673. static_key_slow_dec(&perf_swevent_enabled[event_id]);
  4674. swevent_hlist_put(event);
  4675. }
  4676. static int perf_swevent_init(struct perf_event *event)
  4677. {
  4678. u64 event_id = event->attr.config;
  4679. if (event->attr.type != PERF_TYPE_SOFTWARE)
  4680. return -ENOENT;
  4681. /*
  4682. * no branch sampling for software events
  4683. */
  4684. if (has_branch_stack(event))
  4685. return -EOPNOTSUPP;
  4686. switch (event_id) {
  4687. case PERF_COUNT_SW_CPU_CLOCK:
  4688. case PERF_COUNT_SW_TASK_CLOCK:
  4689. return -ENOENT;
  4690. default:
  4691. break;
  4692. }
  4693. if (event_id >= PERF_COUNT_SW_MAX)
  4694. return -ENOENT;
  4695. if (!event->parent) {
  4696. int err;
  4697. err = swevent_hlist_get(event);
  4698. if (err)
  4699. return err;
  4700. static_key_slow_inc(&perf_swevent_enabled[event_id]);
  4701. event->destroy = sw_perf_event_destroy;
  4702. }
  4703. return 0;
  4704. }
  4705. static int perf_swevent_event_idx(struct perf_event *event)
  4706. {
  4707. return 0;
  4708. }
  4709. static struct pmu perf_swevent = {
  4710. .task_ctx_nr = perf_sw_context,
  4711. .event_init = perf_swevent_init,
  4712. .add = perf_swevent_add,
  4713. .del = perf_swevent_del,
  4714. .start = perf_swevent_start,
  4715. .stop = perf_swevent_stop,
  4716. .read = perf_swevent_read,
  4717. .event_idx = perf_swevent_event_idx,
  4718. };
  4719. #ifdef CONFIG_EVENT_TRACING
  4720. static int perf_tp_filter_match(struct perf_event *event,
  4721. struct perf_sample_data *data)
  4722. {
  4723. void *record = data->raw->data;
  4724. if (likely(!event->filter) || filter_match_preds(event->filter, record))
  4725. return 1;
  4726. return 0;
  4727. }
  4728. static int perf_tp_event_match(struct perf_event *event,
  4729. struct perf_sample_data *data,
  4730. struct pt_regs *regs)
  4731. {
  4732. if (event->hw.state & PERF_HES_STOPPED)
  4733. return 0;
  4734. /*
  4735. * All tracepoints are from kernel-space.
  4736. */
  4737. if (event->attr.exclude_kernel)
  4738. return 0;
  4739. if (!perf_tp_filter_match(event, data))
  4740. return 0;
  4741. return 1;
  4742. }
  4743. void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
  4744. struct pt_regs *regs, struct hlist_head *head, int rctx,
  4745. struct task_struct *task)
  4746. {
  4747. struct perf_sample_data data;
  4748. struct perf_event *event;
  4749. struct perf_raw_record raw = {
  4750. .size = entry_size,
  4751. .data = record,
  4752. };
  4753. perf_sample_data_init(&data, addr, 0);
  4754. data.raw = &raw;
  4755. hlist_for_each_entry_rcu(event, head, hlist_entry) {
  4756. if (perf_tp_event_match(event, &data, regs))
  4757. perf_swevent_event(event, count, &data, regs);
  4758. }
  4759. /*
  4760. * If we got specified a target task, also iterate its context and
  4761. * deliver this event there too.
  4762. */
  4763. if (task && task != current) {
  4764. struct perf_event_context *ctx;
  4765. struct trace_entry *entry = record;
  4766. rcu_read_lock();
  4767. ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
  4768. if (!ctx)
  4769. goto unlock;
  4770. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  4771. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  4772. continue;
  4773. if (event->attr.config != entry->type)
  4774. continue;
  4775. if (perf_tp_event_match(event, &data, regs))
  4776. perf_swevent_event(event, count, &data, regs);
  4777. }
  4778. unlock:
  4779. rcu_read_unlock();
  4780. }
  4781. perf_swevent_put_recursion_context(rctx);
  4782. }
  4783. EXPORT_SYMBOL_GPL(perf_tp_event);
  4784. static void tp_perf_event_destroy(struct perf_event *event)
  4785. {
  4786. perf_trace_destroy(event);
  4787. }
  4788. static int perf_tp_event_init(struct perf_event *event)
  4789. {
  4790. int err;
  4791. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  4792. return -ENOENT;
  4793. /*
  4794. * no branch sampling for tracepoint events
  4795. */
  4796. if (has_branch_stack(event))
  4797. return -EOPNOTSUPP;
  4798. err = perf_trace_init(event);
  4799. if (err)
  4800. return err;
  4801. event->destroy = tp_perf_event_destroy;
  4802. return 0;
  4803. }
  4804. static struct pmu perf_tracepoint = {
  4805. .task_ctx_nr = perf_sw_context,
  4806. .event_init = perf_tp_event_init,
  4807. .add = perf_trace_add,
  4808. .del = perf_trace_del,
  4809. .start = perf_swevent_start,
  4810. .stop = perf_swevent_stop,
  4811. .read = perf_swevent_read,
  4812. .event_idx = perf_swevent_event_idx,
  4813. };
  4814. static inline void perf_tp_register(void)
  4815. {
  4816. perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
  4817. }
  4818. static int perf_event_set_filter(struct perf_event *event, void __user *arg)
  4819. {
  4820. char *filter_str;
  4821. int ret;
  4822. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  4823. return -EINVAL;
  4824. filter_str = strndup_user(arg, PAGE_SIZE);
  4825. if (IS_ERR(filter_str))
  4826. return PTR_ERR(filter_str);
  4827. ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
  4828. kfree(filter_str);
  4829. return ret;
  4830. }
  4831. static void perf_event_free_filter(struct perf_event *event)
  4832. {
  4833. ftrace_profile_free_filter(event);
  4834. }
  4835. #else
  4836. static inline void perf_tp_register(void)
  4837. {
  4838. }
  4839. static int perf_event_set_filter(struct perf_event *event, void __user *arg)
  4840. {
  4841. return -ENOENT;
  4842. }
  4843. static void perf_event_free_filter(struct perf_event *event)
  4844. {
  4845. }
  4846. #endif /* CONFIG_EVENT_TRACING */
  4847. #ifdef CONFIG_HAVE_HW_BREAKPOINT
  4848. void perf_bp_event(struct perf_event *bp, void *data)
  4849. {
  4850. struct perf_sample_data sample;
  4851. struct pt_regs *regs = data;
  4852. perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
  4853. if (!bp->hw.state && !perf_exclude_event(bp, regs))
  4854. perf_swevent_event(bp, 1, &sample, regs);
  4855. }
  4856. #endif
  4857. /*
  4858. * hrtimer based swevent callback
  4859. */
  4860. static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
  4861. {
  4862. enum hrtimer_restart ret = HRTIMER_RESTART;
  4863. struct perf_sample_data data;
  4864. struct pt_regs *regs;
  4865. struct perf_event *event;
  4866. u64 period;
  4867. event = container_of(hrtimer, struct perf_event, hw.hrtimer);
  4868. if (event->state != PERF_EVENT_STATE_ACTIVE)
  4869. return HRTIMER_NORESTART;
  4870. event->pmu->read(event);
  4871. perf_sample_data_init(&data, 0, event->hw.last_period);
  4872. regs = get_irq_regs();
  4873. if (regs && !perf_exclude_event(event, regs)) {
  4874. if (!(event->attr.exclude_idle && is_idle_task(current)))
  4875. if (__perf_event_overflow(event, 1, &data, regs))
  4876. ret = HRTIMER_NORESTART;
  4877. }
  4878. period = max_t(u64, 10000, event->hw.sample_period);
  4879. hrtimer_forward_now(hrtimer, ns_to_ktime(period));
  4880. return ret;
  4881. }
  4882. static void perf_swevent_start_hrtimer(struct perf_event *event)
  4883. {
  4884. struct hw_perf_event *hwc = &event->hw;
  4885. s64 period;
  4886. if (!is_sampling_event(event))
  4887. return;
  4888. period = local64_read(&hwc->period_left);
  4889. if (period) {
  4890. if (period < 0)
  4891. period = 10000;
  4892. local64_set(&hwc->period_left, 0);
  4893. } else {
  4894. period = max_t(u64, 10000, hwc->sample_period);
  4895. }
  4896. __hrtimer_start_range_ns(&hwc->hrtimer,
  4897. ns_to_ktime(period), 0,
  4898. HRTIMER_MODE_REL_PINNED, 0);
  4899. }
  4900. static void perf_swevent_cancel_hrtimer(struct perf_event *event)
  4901. {
  4902. struct hw_perf_event *hwc = &event->hw;
  4903. if (is_sampling_event(event)) {
  4904. ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
  4905. local64_set(&hwc->period_left, ktime_to_ns(remaining));
  4906. hrtimer_cancel(&hwc->hrtimer);
  4907. }
  4908. }
  4909. static void perf_swevent_init_hrtimer(struct perf_event *event)
  4910. {
  4911. struct hw_perf_event *hwc = &event->hw;
  4912. if (!is_sampling_event(event))
  4913. return;
  4914. hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  4915. hwc->hrtimer.function = perf_swevent_hrtimer;
  4916. /*
  4917. * Since hrtimers have a fixed rate, we can do a static freq->period
  4918. * mapping and avoid the whole period adjust feedback stuff.
  4919. */
  4920. if (event->attr.freq) {
  4921. long freq = event->attr.sample_freq;
  4922. event->attr.sample_period = NSEC_PER_SEC / freq;
  4923. hwc->sample_period = event->attr.sample_period;
  4924. local64_set(&hwc->period_left, hwc->sample_period);
  4925. hwc->last_period = hwc->sample_period;
  4926. event->attr.freq = 0;
  4927. }
  4928. }
  4929. /*
  4930. * Software event: cpu wall time clock
  4931. */
  4932. static void cpu_clock_event_update(struct perf_event *event)
  4933. {
  4934. s64 prev;
  4935. u64 now;
  4936. now = local_clock();
  4937. prev = local64_xchg(&event->hw.prev_count, now);
  4938. local64_add(now - prev, &event->count);
  4939. }
  4940. static void cpu_clock_event_start(struct perf_event *event, int flags)
  4941. {
  4942. local64_set(&event->hw.prev_count, local_clock());
  4943. perf_swevent_start_hrtimer(event);
  4944. }
  4945. static void cpu_clock_event_stop(struct perf_event *event, int flags)
  4946. {
  4947. perf_swevent_cancel_hrtimer(event);
  4948. cpu_clock_event_update(event);
  4949. }
  4950. static int cpu_clock_event_add(struct perf_event *event, int flags)
  4951. {
  4952. if (flags & PERF_EF_START)
  4953. cpu_clock_event_start(event, flags);
  4954. return 0;
  4955. }
  4956. static void cpu_clock_event_del(struct perf_event *event, int flags)
  4957. {
  4958. cpu_clock_event_stop(event, flags);
  4959. }
  4960. static void cpu_clock_event_read(struct perf_event *event)
  4961. {
  4962. cpu_clock_event_update(event);
  4963. }
  4964. static int cpu_clock_event_init(struct perf_event *event)
  4965. {
  4966. if (event->attr.type != PERF_TYPE_SOFTWARE)
  4967. return -ENOENT;
  4968. if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
  4969. return -ENOENT;
  4970. /*
  4971. * no branch sampling for software events
  4972. */
  4973. if (has_branch_stack(event))
  4974. return -EOPNOTSUPP;
  4975. perf_swevent_init_hrtimer(event);
  4976. return 0;
  4977. }
  4978. static struct pmu perf_cpu_clock = {
  4979. .task_ctx_nr = perf_sw_context,
  4980. .event_init = cpu_clock_event_init,
  4981. .add = cpu_clock_event_add,
  4982. .del = cpu_clock_event_del,
  4983. .start = cpu_clock_event_start,
  4984. .stop = cpu_clock_event_stop,
  4985. .read = cpu_clock_event_read,
  4986. .event_idx = perf_swevent_event_idx,
  4987. };
  4988. /*
  4989. * Software event: task time clock
  4990. */
  4991. static void task_clock_event_update(struct perf_event *event, u64 now)
  4992. {
  4993. u64 prev;
  4994. s64 delta;
  4995. prev = local64_xchg(&event->hw.prev_count, now);
  4996. delta = now - prev;
  4997. local64_add(delta, &event->count);
  4998. }
  4999. static void task_clock_event_start(struct perf_event *event, int flags)
  5000. {
  5001. local64_set(&event->hw.prev_count, event->ctx->time);
  5002. perf_swevent_start_hrtimer(event);
  5003. }
  5004. static void task_clock_event_stop(struct perf_event *event, int flags)
  5005. {
  5006. perf_swevent_cancel_hrtimer(event);
  5007. task_clock_event_update(event, event->ctx->time);
  5008. }
  5009. static int task_clock_event_add(struct perf_event *event, int flags)
  5010. {
  5011. if (flags & PERF_EF_START)
  5012. task_clock_event_start(event, flags);
  5013. return 0;
  5014. }
  5015. static void task_clock_event_del(struct perf_event *event, int flags)
  5016. {
  5017. task_clock_event_stop(event, PERF_EF_UPDATE);
  5018. }
  5019. static void task_clock_event_read(struct perf_event *event)
  5020. {
  5021. u64 now = perf_clock();
  5022. u64 delta = now - event->ctx->timestamp;
  5023. u64 time = event->ctx->time + delta;
  5024. task_clock_event_update(event, time);
  5025. }
  5026. static int task_clock_event_init(struct perf_event *event)
  5027. {
  5028. if (event->attr.type != PERF_TYPE_SOFTWARE)
  5029. return -ENOENT;
  5030. if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
  5031. return -ENOENT;
  5032. /*
  5033. * no branch sampling for software events
  5034. */
  5035. if (has_branch_stack(event))
  5036. return -EOPNOTSUPP;
  5037. perf_swevent_init_hrtimer(event);
  5038. return 0;
  5039. }
  5040. static struct pmu perf_task_clock = {
  5041. .task_ctx_nr = perf_sw_context,
  5042. .event_init = task_clock_event_init,
  5043. .add = task_clock_event_add,
  5044. .del = task_clock_event_del,
  5045. .start = task_clock_event_start,
  5046. .stop = task_clock_event_stop,
  5047. .read = task_clock_event_read,
  5048. .event_idx = perf_swevent_event_idx,
  5049. };
  5050. static void perf_pmu_nop_void(struct pmu *pmu)
  5051. {
  5052. }
  5053. static int perf_pmu_nop_int(struct pmu *pmu)
  5054. {
  5055. return 0;
  5056. }
  5057. static void perf_pmu_start_txn(struct pmu *pmu)
  5058. {
  5059. perf_pmu_disable(pmu);
  5060. }
  5061. static int perf_pmu_commit_txn(struct pmu *pmu)
  5062. {
  5063. perf_pmu_enable(pmu);
  5064. return 0;
  5065. }
  5066. static void perf_pmu_cancel_txn(struct pmu *pmu)
  5067. {
  5068. perf_pmu_enable(pmu);
  5069. }
  5070. static int perf_event_idx_default(struct perf_event *event)
  5071. {
  5072. return event->hw.idx + 1;
  5073. }
  5074. /*
  5075. * Ensures all contexts with the same task_ctx_nr have the same
  5076. * pmu_cpu_context too.
  5077. */
  5078. static void *find_pmu_context(int ctxn)
  5079. {
  5080. struct pmu *pmu;
  5081. if (ctxn < 0)
  5082. return NULL;
  5083. list_for_each_entry(pmu, &pmus, entry) {
  5084. if (pmu->task_ctx_nr == ctxn)
  5085. return pmu->pmu_cpu_context;
  5086. }
  5087. return NULL;
  5088. }
  5089. static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
  5090. {
  5091. int cpu;
  5092. for_each_possible_cpu(cpu) {
  5093. struct perf_cpu_context *cpuctx;
  5094. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  5095. if (cpuctx->unique_pmu == old_pmu)
  5096. cpuctx->unique_pmu = pmu;
  5097. }
  5098. }
  5099. static void free_pmu_context(struct pmu *pmu)
  5100. {
  5101. struct pmu *i;
  5102. mutex_lock(&pmus_lock);
  5103. /*
  5104. * Like a real lame refcount.
  5105. */
  5106. list_for_each_entry(i, &pmus, entry) {
  5107. if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
  5108. update_pmu_context(i, pmu);
  5109. goto out;
  5110. }
  5111. }
  5112. free_percpu(pmu->pmu_cpu_context);
  5113. out:
  5114. mutex_unlock(&pmus_lock);
  5115. }
  5116. static struct idr pmu_idr;
  5117. static ssize_t
  5118. type_show(struct device *dev, struct device_attribute *attr, char *page)
  5119. {
  5120. struct pmu *pmu = dev_get_drvdata(dev);
  5121. return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
  5122. }
  5123. static ssize_t
  5124. perf_event_mux_interval_ms_show(struct device *dev,
  5125. struct device_attribute *attr,
  5126. char *page)
  5127. {
  5128. struct pmu *pmu = dev_get_drvdata(dev);
  5129. return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms);
  5130. }
  5131. static ssize_t
  5132. perf_event_mux_interval_ms_store(struct device *dev,
  5133. struct device_attribute *attr,
  5134. const char *buf, size_t count)
  5135. {
  5136. struct pmu *pmu = dev_get_drvdata(dev);
  5137. int timer, cpu, ret;
  5138. ret = kstrtoint(buf, 0, &timer);
  5139. if (ret)
  5140. return ret;
  5141. if (timer < 1)
  5142. return -EINVAL;
  5143. /* same value, noting to do */
  5144. if (timer == pmu->hrtimer_interval_ms)
  5145. return count;
  5146. pmu->hrtimer_interval_ms = timer;
  5147. /* update all cpuctx for this PMU */
  5148. for_each_possible_cpu(cpu) {
  5149. struct perf_cpu_context *cpuctx;
  5150. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  5151. cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
  5152. if (hrtimer_active(&cpuctx->hrtimer))
  5153. hrtimer_forward_now(&cpuctx->hrtimer, cpuctx->hrtimer_interval);
  5154. }
  5155. return count;
  5156. }
  5157. static struct device_attribute pmu_dev_attrs[] = {
  5158. __ATTR_RO(type),
  5159. __ATTR_RW(perf_event_mux_interval_ms),
  5160. __ATTR_NULL,
  5161. };
  5162. static int pmu_bus_running;
  5163. static struct bus_type pmu_bus = {
  5164. .name = "event_source",
  5165. .dev_attrs = pmu_dev_attrs,
  5166. };
  5167. static void pmu_dev_release(struct device *dev)
  5168. {
  5169. kfree(dev);
  5170. }
  5171. static int pmu_dev_alloc(struct pmu *pmu)
  5172. {
  5173. int ret = -ENOMEM;
  5174. pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
  5175. if (!pmu->dev)
  5176. goto out;
  5177. pmu->dev->groups = pmu->attr_groups;
  5178. device_initialize(pmu->dev);
  5179. ret = dev_set_name(pmu->dev, "%s", pmu->name);
  5180. if (ret)
  5181. goto free_dev;
  5182. dev_set_drvdata(pmu->dev, pmu);
  5183. pmu->dev->bus = &pmu_bus;
  5184. pmu->dev->release = pmu_dev_release;
  5185. ret = device_add(pmu->dev);
  5186. if (ret)
  5187. goto free_dev;
  5188. out:
  5189. return ret;
  5190. free_dev:
  5191. put_device(pmu->dev);
  5192. goto out;
  5193. }
  5194. static struct lock_class_key cpuctx_mutex;
  5195. static struct lock_class_key cpuctx_lock;
  5196. int perf_pmu_register(struct pmu *pmu, const char *name, int type)
  5197. {
  5198. int cpu, ret;
  5199. mutex_lock(&pmus_lock);
  5200. ret = -ENOMEM;
  5201. pmu->pmu_disable_count = alloc_percpu(int);
  5202. if (!pmu->pmu_disable_count)
  5203. goto unlock;
  5204. pmu->type = -1;
  5205. if (!name)
  5206. goto skip_type;
  5207. pmu->name = name;
  5208. if (type < 0) {
  5209. type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL);
  5210. if (type < 0) {
  5211. ret = type;
  5212. goto free_pdc;
  5213. }
  5214. }
  5215. pmu->type = type;
  5216. if (pmu_bus_running) {
  5217. ret = pmu_dev_alloc(pmu);
  5218. if (ret)
  5219. goto free_idr;
  5220. }
  5221. skip_type:
  5222. pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
  5223. if (pmu->pmu_cpu_context)
  5224. goto got_cpu_context;
  5225. ret = -ENOMEM;
  5226. pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
  5227. if (!pmu->pmu_cpu_context)
  5228. goto free_dev;
  5229. for_each_possible_cpu(cpu) {
  5230. struct perf_cpu_context *cpuctx;
  5231. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  5232. __perf_event_init_context(&cpuctx->ctx);
  5233. lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
  5234. lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
  5235. cpuctx->ctx.type = cpu_context;
  5236. cpuctx->ctx.pmu = pmu;
  5237. __perf_cpu_hrtimer_init(cpuctx, cpu);
  5238. INIT_LIST_HEAD(&cpuctx->rotation_list);
  5239. cpuctx->unique_pmu = pmu;
  5240. }
  5241. got_cpu_context:
  5242. if (!pmu->start_txn) {
  5243. if (pmu->pmu_enable) {
  5244. /*
  5245. * If we have pmu_enable/pmu_disable calls, install
  5246. * transaction stubs that use that to try and batch
  5247. * hardware accesses.
  5248. */
  5249. pmu->start_txn = perf_pmu_start_txn;
  5250. pmu->commit_txn = perf_pmu_commit_txn;
  5251. pmu->cancel_txn = perf_pmu_cancel_txn;
  5252. } else {
  5253. pmu->start_txn = perf_pmu_nop_void;
  5254. pmu->commit_txn = perf_pmu_nop_int;
  5255. pmu->cancel_txn = perf_pmu_nop_void;
  5256. }
  5257. }
  5258. if (!pmu->pmu_enable) {
  5259. pmu->pmu_enable = perf_pmu_nop_void;
  5260. pmu->pmu_disable = perf_pmu_nop_void;
  5261. }
  5262. if (!pmu->event_idx)
  5263. pmu->event_idx = perf_event_idx_default;
  5264. list_add_rcu(&pmu->entry, &pmus);
  5265. ret = 0;
  5266. unlock:
  5267. mutex_unlock(&pmus_lock);
  5268. return ret;
  5269. free_dev:
  5270. device_del(pmu->dev);
  5271. put_device(pmu->dev);
  5272. free_idr:
  5273. if (pmu->type >= PERF_TYPE_MAX)
  5274. idr_remove(&pmu_idr, pmu->type);
  5275. free_pdc:
  5276. free_percpu(pmu->pmu_disable_count);
  5277. goto unlock;
  5278. }
  5279. void perf_pmu_unregister(struct pmu *pmu)
  5280. {
  5281. mutex_lock(&pmus_lock);
  5282. list_del_rcu(&pmu->entry);
  5283. mutex_unlock(&pmus_lock);
  5284. /*
  5285. * We dereference the pmu list under both SRCU and regular RCU, so
  5286. * synchronize against both of those.
  5287. */
  5288. synchronize_srcu(&pmus_srcu);
  5289. synchronize_rcu();
  5290. free_percpu(pmu->pmu_disable_count);
  5291. if (pmu->type >= PERF_TYPE_MAX)
  5292. idr_remove(&pmu_idr, pmu->type);
  5293. device_del(pmu->dev);
  5294. put_device(pmu->dev);
  5295. free_pmu_context(pmu);
  5296. }
  5297. struct pmu *perf_init_event(struct perf_event *event)
  5298. {
  5299. struct pmu *pmu = NULL;
  5300. int idx;
  5301. int ret;
  5302. idx = srcu_read_lock(&pmus_srcu);
  5303. rcu_read_lock();
  5304. pmu = idr_find(&pmu_idr, event->attr.type);
  5305. rcu_read_unlock();
  5306. if (pmu) {
  5307. event->pmu = pmu;
  5308. ret = pmu->event_init(event);
  5309. if (ret)
  5310. pmu = ERR_PTR(ret);
  5311. goto unlock;
  5312. }
  5313. list_for_each_entry_rcu(pmu, &pmus, entry) {
  5314. event->pmu = pmu;
  5315. ret = pmu->event_init(event);
  5316. if (!ret)
  5317. goto unlock;
  5318. if (ret != -ENOENT) {
  5319. pmu = ERR_PTR(ret);
  5320. goto unlock;
  5321. }
  5322. }
  5323. pmu = ERR_PTR(-ENOENT);
  5324. unlock:
  5325. srcu_read_unlock(&pmus_srcu, idx);
  5326. return pmu;
  5327. }
  5328. static void account_event_cpu(struct perf_event *event, int cpu)
  5329. {
  5330. if (event->parent)
  5331. return;
  5332. if (has_branch_stack(event)) {
  5333. if (!(event->attach_state & PERF_ATTACH_TASK))
  5334. atomic_inc(&per_cpu(perf_branch_stack_events, cpu));
  5335. }
  5336. if (is_cgroup_event(event))
  5337. atomic_inc(&per_cpu(perf_cgroup_events, cpu));
  5338. }
  5339. static void account_event(struct perf_event *event)
  5340. {
  5341. if (event->parent)
  5342. return;
  5343. if (event->attach_state & PERF_ATTACH_TASK)
  5344. static_key_slow_inc(&perf_sched_events.key);
  5345. if (event->attr.mmap || event->attr.mmap_data)
  5346. atomic_inc(&nr_mmap_events);
  5347. if (event->attr.comm)
  5348. atomic_inc(&nr_comm_events);
  5349. if (event->attr.task)
  5350. atomic_inc(&nr_task_events);
  5351. if (event->attr.freq) {
  5352. if (atomic_inc_return(&nr_freq_events) == 1)
  5353. tick_nohz_full_kick_all();
  5354. }
  5355. if (has_branch_stack(event))
  5356. static_key_slow_inc(&perf_sched_events.key);
  5357. if (is_cgroup_event(event))
  5358. static_key_slow_inc(&perf_sched_events.key);
  5359. account_event_cpu(event, event->cpu);
  5360. }
  5361. /*
  5362. * Allocate and initialize a event structure
  5363. */
  5364. static struct perf_event *
  5365. perf_event_alloc(struct perf_event_attr *attr, int cpu,
  5366. struct task_struct *task,
  5367. struct perf_event *group_leader,
  5368. struct perf_event *parent_event,
  5369. perf_overflow_handler_t overflow_handler,
  5370. void *context)
  5371. {
  5372. struct pmu *pmu;
  5373. struct perf_event *event;
  5374. struct hw_perf_event *hwc;
  5375. long err = -EINVAL;
  5376. if ((unsigned)cpu >= nr_cpu_ids) {
  5377. if (!task || cpu != -1)
  5378. return ERR_PTR(-EINVAL);
  5379. }
  5380. event = kzalloc(sizeof(*event), GFP_KERNEL);
  5381. if (!event)
  5382. return ERR_PTR(-ENOMEM);
  5383. /*
  5384. * Single events are their own group leaders, with an
  5385. * empty sibling list:
  5386. */
  5387. if (!group_leader)
  5388. group_leader = event;
  5389. mutex_init(&event->child_mutex);
  5390. INIT_LIST_HEAD(&event->child_list);
  5391. INIT_LIST_HEAD(&event->group_entry);
  5392. INIT_LIST_HEAD(&event->event_entry);
  5393. INIT_LIST_HEAD(&event->sibling_list);
  5394. INIT_LIST_HEAD(&event->rb_entry);
  5395. init_waitqueue_head(&event->waitq);
  5396. init_irq_work(&event->pending, perf_pending_event);
  5397. mutex_init(&event->mmap_mutex);
  5398. atomic_long_set(&event->refcount, 1);
  5399. event->cpu = cpu;
  5400. event->attr = *attr;
  5401. event->group_leader = group_leader;
  5402. event->pmu = NULL;
  5403. event->oncpu = -1;
  5404. event->parent = parent_event;
  5405. event->ns = get_pid_ns(task_active_pid_ns(current));
  5406. event->id = atomic64_inc_return(&perf_event_id);
  5407. event->state = PERF_EVENT_STATE_INACTIVE;
  5408. if (task) {
  5409. event->attach_state = PERF_ATTACH_TASK;
  5410. if (attr->type == PERF_TYPE_TRACEPOINT)
  5411. event->hw.tp_target = task;
  5412. #ifdef CONFIG_HAVE_HW_BREAKPOINT
  5413. /*
  5414. * hw_breakpoint is a bit difficult here..
  5415. */
  5416. else if (attr->type == PERF_TYPE_BREAKPOINT)
  5417. event->hw.bp_target = task;
  5418. #endif
  5419. }
  5420. if (!overflow_handler && parent_event) {
  5421. overflow_handler = parent_event->overflow_handler;
  5422. context = parent_event->overflow_handler_context;
  5423. }
  5424. event->overflow_handler = overflow_handler;
  5425. event->overflow_handler_context = context;
  5426. perf_event__state_init(event);
  5427. pmu = NULL;
  5428. hwc = &event->hw;
  5429. hwc->sample_period = attr->sample_period;
  5430. if (attr->freq && attr->sample_freq)
  5431. hwc->sample_period = 1;
  5432. hwc->last_period = hwc->sample_period;
  5433. local64_set(&hwc->period_left, hwc->sample_period);
  5434. /*
  5435. * we currently do not support PERF_FORMAT_GROUP on inherited events
  5436. */
  5437. if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
  5438. goto err_ns;
  5439. pmu = perf_init_event(event);
  5440. if (!pmu)
  5441. goto err_ns;
  5442. else if (IS_ERR(pmu)) {
  5443. err = PTR_ERR(pmu);
  5444. goto err_ns;
  5445. }
  5446. if (!event->parent) {
  5447. if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
  5448. err = get_callchain_buffers();
  5449. if (err)
  5450. goto err_pmu;
  5451. }
  5452. }
  5453. return event;
  5454. err_pmu:
  5455. if (event->destroy)
  5456. event->destroy(event);
  5457. err_ns:
  5458. if (event->ns)
  5459. put_pid_ns(event->ns);
  5460. kfree(event);
  5461. return ERR_PTR(err);
  5462. }
  5463. static int perf_copy_attr(struct perf_event_attr __user *uattr,
  5464. struct perf_event_attr *attr)
  5465. {
  5466. u32 size;
  5467. int ret;
  5468. if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
  5469. return -EFAULT;
  5470. /*
  5471. * zero the full structure, so that a short copy will be nice.
  5472. */
  5473. memset(attr, 0, sizeof(*attr));
  5474. ret = get_user(size, &uattr->size);
  5475. if (ret)
  5476. return ret;
  5477. if (size > PAGE_SIZE) /* silly large */
  5478. goto err_size;
  5479. if (!size) /* abi compat */
  5480. size = PERF_ATTR_SIZE_VER0;
  5481. if (size < PERF_ATTR_SIZE_VER0)
  5482. goto err_size;
  5483. /*
  5484. * If we're handed a bigger struct than we know of,
  5485. * ensure all the unknown bits are 0 - i.e. new
  5486. * user-space does not rely on any kernel feature
  5487. * extensions we dont know about yet.
  5488. */
  5489. if (size > sizeof(*attr)) {
  5490. unsigned char __user *addr;
  5491. unsigned char __user *end;
  5492. unsigned char val;
  5493. addr = (void __user *)uattr + sizeof(*attr);
  5494. end = (void __user *)uattr + size;
  5495. for (; addr < end; addr++) {
  5496. ret = get_user(val, addr);
  5497. if (ret)
  5498. return ret;
  5499. if (val)
  5500. goto err_size;
  5501. }
  5502. size = sizeof(*attr);
  5503. }
  5504. ret = copy_from_user(attr, uattr, size);
  5505. if (ret)
  5506. return -EFAULT;
  5507. if (attr->__reserved_1)
  5508. return -EINVAL;
  5509. if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
  5510. return -EINVAL;
  5511. if (attr->read_format & ~(PERF_FORMAT_MAX-1))
  5512. return -EINVAL;
  5513. if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
  5514. u64 mask = attr->branch_sample_type;
  5515. /* only using defined bits */
  5516. if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
  5517. return -EINVAL;
  5518. /* at least one branch bit must be set */
  5519. if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
  5520. return -EINVAL;
  5521. /* propagate priv level, when not set for branch */
  5522. if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
  5523. /* exclude_kernel checked on syscall entry */
  5524. if (!attr->exclude_kernel)
  5525. mask |= PERF_SAMPLE_BRANCH_KERNEL;
  5526. if (!attr->exclude_user)
  5527. mask |= PERF_SAMPLE_BRANCH_USER;
  5528. if (!attr->exclude_hv)
  5529. mask |= PERF_SAMPLE_BRANCH_HV;
  5530. /*
  5531. * adjust user setting (for HW filter setup)
  5532. */
  5533. attr->branch_sample_type = mask;
  5534. }
  5535. /* privileged levels capture (kernel, hv): check permissions */
  5536. if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
  5537. && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
  5538. return -EACCES;
  5539. }
  5540. if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
  5541. ret = perf_reg_validate(attr->sample_regs_user);
  5542. if (ret)
  5543. return ret;
  5544. }
  5545. if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
  5546. if (!arch_perf_have_user_stack_dump())
  5547. return -ENOSYS;
  5548. /*
  5549. * We have __u32 type for the size, but so far
  5550. * we can only use __u16 as maximum due to the
  5551. * __u16 sample size limit.
  5552. */
  5553. if (attr->sample_stack_user >= USHRT_MAX)
  5554. ret = -EINVAL;
  5555. else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
  5556. ret = -EINVAL;
  5557. }
  5558. out:
  5559. return ret;
  5560. err_size:
  5561. put_user(sizeof(*attr), &uattr->size);
  5562. ret = -E2BIG;
  5563. goto out;
  5564. }
  5565. static int
  5566. perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
  5567. {
  5568. struct ring_buffer *rb = NULL, *old_rb = NULL;
  5569. int ret = -EINVAL;
  5570. if (!output_event)
  5571. goto set;
  5572. /* don't allow circular references */
  5573. if (event == output_event)
  5574. goto out;
  5575. /*
  5576. * Don't allow cross-cpu buffers
  5577. */
  5578. if (output_event->cpu != event->cpu)
  5579. goto out;
  5580. /*
  5581. * If its not a per-cpu rb, it must be the same task.
  5582. */
  5583. if (output_event->cpu == -1 && output_event->ctx != event->ctx)
  5584. goto out;
  5585. set:
  5586. mutex_lock(&event->mmap_mutex);
  5587. /* Can't redirect output if we've got an active mmap() */
  5588. if (atomic_read(&event->mmap_count))
  5589. goto unlock;
  5590. old_rb = event->rb;
  5591. if (output_event) {
  5592. /* get the rb we want to redirect to */
  5593. rb = ring_buffer_get(output_event);
  5594. if (!rb)
  5595. goto unlock;
  5596. }
  5597. if (old_rb)
  5598. ring_buffer_detach(event, old_rb);
  5599. if (rb)
  5600. ring_buffer_attach(event, rb);
  5601. rcu_assign_pointer(event->rb, rb);
  5602. if (old_rb) {
  5603. ring_buffer_put(old_rb);
  5604. /*
  5605. * Since we detached before setting the new rb, so that we
  5606. * could attach the new rb, we could have missed a wakeup.
  5607. * Provide it now.
  5608. */
  5609. wake_up_all(&event->waitq);
  5610. }
  5611. ret = 0;
  5612. unlock:
  5613. mutex_unlock(&event->mmap_mutex);
  5614. out:
  5615. return ret;
  5616. }
  5617. /**
  5618. * sys_perf_event_open - open a performance event, associate it to a task/cpu
  5619. *
  5620. * @attr_uptr: event_id type attributes for monitoring/sampling
  5621. * @pid: target pid
  5622. * @cpu: target cpu
  5623. * @group_fd: group leader event fd
  5624. */
  5625. SYSCALL_DEFINE5(perf_event_open,
  5626. struct perf_event_attr __user *, attr_uptr,
  5627. pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
  5628. {
  5629. struct perf_event *group_leader = NULL, *output_event = NULL;
  5630. struct perf_event *event, *sibling;
  5631. struct perf_event_attr attr;
  5632. struct perf_event_context *ctx;
  5633. struct file *event_file = NULL;
  5634. struct fd group = {NULL, 0};
  5635. struct task_struct *task = NULL;
  5636. struct pmu *pmu;
  5637. int event_fd;
  5638. int move_group = 0;
  5639. int err;
  5640. /* for future expandability... */
  5641. if (flags & ~PERF_FLAG_ALL)
  5642. return -EINVAL;
  5643. err = perf_copy_attr(attr_uptr, &attr);
  5644. if (err)
  5645. return err;
  5646. if (!attr.exclude_kernel) {
  5647. if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
  5648. return -EACCES;
  5649. }
  5650. if (attr.freq) {
  5651. if (attr.sample_freq > sysctl_perf_event_sample_rate)
  5652. return -EINVAL;
  5653. }
  5654. /*
  5655. * In cgroup mode, the pid argument is used to pass the fd
  5656. * opened to the cgroup directory in cgroupfs. The cpu argument
  5657. * designates the cpu on which to monitor threads from that
  5658. * cgroup.
  5659. */
  5660. if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
  5661. return -EINVAL;
  5662. event_fd = get_unused_fd();
  5663. if (event_fd < 0)
  5664. return event_fd;
  5665. if (group_fd != -1) {
  5666. err = perf_fget_light(group_fd, &group);
  5667. if (err)
  5668. goto err_fd;
  5669. group_leader = group.file->private_data;
  5670. if (flags & PERF_FLAG_FD_OUTPUT)
  5671. output_event = group_leader;
  5672. if (flags & PERF_FLAG_FD_NO_GROUP)
  5673. group_leader = NULL;
  5674. }
  5675. if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
  5676. task = find_lively_task_by_vpid(pid);
  5677. if (IS_ERR(task)) {
  5678. err = PTR_ERR(task);
  5679. goto err_group_fd;
  5680. }
  5681. }
  5682. get_online_cpus();
  5683. event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
  5684. NULL, NULL);
  5685. if (IS_ERR(event)) {
  5686. err = PTR_ERR(event);
  5687. goto err_task;
  5688. }
  5689. if (flags & PERF_FLAG_PID_CGROUP) {
  5690. err = perf_cgroup_connect(pid, event, &attr, group_leader);
  5691. if (err) {
  5692. __free_event(event);
  5693. goto err_task;
  5694. }
  5695. }
  5696. account_event(event);
  5697. /*
  5698. * Special case software events and allow them to be part of
  5699. * any hardware group.
  5700. */
  5701. pmu = event->pmu;
  5702. if (group_leader &&
  5703. (is_software_event(event) != is_software_event(group_leader))) {
  5704. if (is_software_event(event)) {
  5705. /*
  5706. * If event and group_leader are not both a software
  5707. * event, and event is, then group leader is not.
  5708. *
  5709. * Allow the addition of software events to !software
  5710. * groups, this is safe because software events never
  5711. * fail to schedule.
  5712. */
  5713. pmu = group_leader->pmu;
  5714. } else if (is_software_event(group_leader) &&
  5715. (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
  5716. /*
  5717. * In case the group is a pure software group, and we
  5718. * try to add a hardware event, move the whole group to
  5719. * the hardware context.
  5720. */
  5721. move_group = 1;
  5722. }
  5723. }
  5724. /*
  5725. * Get the target context (task or percpu):
  5726. */
  5727. ctx = find_get_context(pmu, task, event->cpu);
  5728. if (IS_ERR(ctx)) {
  5729. err = PTR_ERR(ctx);
  5730. goto err_alloc;
  5731. }
  5732. if (task) {
  5733. put_task_struct(task);
  5734. task = NULL;
  5735. }
  5736. /*
  5737. * Look up the group leader (we will attach this event to it):
  5738. */
  5739. if (group_leader) {
  5740. err = -EINVAL;
  5741. /*
  5742. * Do not allow a recursive hierarchy (this new sibling
  5743. * becoming part of another group-sibling):
  5744. */
  5745. if (group_leader->group_leader != group_leader)
  5746. goto err_context;
  5747. /*
  5748. * Do not allow to attach to a group in a different
  5749. * task or CPU context:
  5750. */
  5751. if (move_group) {
  5752. if (group_leader->ctx->type != ctx->type)
  5753. goto err_context;
  5754. } else {
  5755. if (group_leader->ctx != ctx)
  5756. goto err_context;
  5757. }
  5758. /*
  5759. * Only a group leader can be exclusive or pinned
  5760. */
  5761. if (attr.exclusive || attr.pinned)
  5762. goto err_context;
  5763. }
  5764. if (output_event) {
  5765. err = perf_event_set_output(event, output_event);
  5766. if (err)
  5767. goto err_context;
  5768. }
  5769. event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
  5770. if (IS_ERR(event_file)) {
  5771. err = PTR_ERR(event_file);
  5772. goto err_context;
  5773. }
  5774. if (move_group) {
  5775. struct perf_event_context *gctx = group_leader->ctx;
  5776. mutex_lock(&gctx->mutex);
  5777. perf_remove_from_context(group_leader);
  5778. /*
  5779. * Removing from the context ends up with disabled
  5780. * event. What we want here is event in the initial
  5781. * startup state, ready to be add into new context.
  5782. */
  5783. perf_event__state_init(group_leader);
  5784. list_for_each_entry(sibling, &group_leader->sibling_list,
  5785. group_entry) {
  5786. perf_remove_from_context(sibling);
  5787. perf_event__state_init(sibling);
  5788. put_ctx(gctx);
  5789. }
  5790. mutex_unlock(&gctx->mutex);
  5791. put_ctx(gctx);
  5792. }
  5793. WARN_ON_ONCE(ctx->parent_ctx);
  5794. mutex_lock(&ctx->mutex);
  5795. if (move_group) {
  5796. synchronize_rcu();
  5797. perf_install_in_context(ctx, group_leader, event->cpu);
  5798. get_ctx(ctx);
  5799. list_for_each_entry(sibling, &group_leader->sibling_list,
  5800. group_entry) {
  5801. perf_install_in_context(ctx, sibling, event->cpu);
  5802. get_ctx(ctx);
  5803. }
  5804. }
  5805. perf_install_in_context(ctx, event, event->cpu);
  5806. ++ctx->generation;
  5807. perf_unpin_context(ctx);
  5808. mutex_unlock(&ctx->mutex);
  5809. put_online_cpus();
  5810. event->owner = current;
  5811. mutex_lock(&current->perf_event_mutex);
  5812. list_add_tail(&event->owner_entry, &current->perf_event_list);
  5813. mutex_unlock(&current->perf_event_mutex);
  5814. /*
  5815. * Precalculate sample_data sizes
  5816. */
  5817. perf_event__header_size(event);
  5818. perf_event__id_header_size(event);
  5819. /*
  5820. * Drop the reference on the group_event after placing the
  5821. * new event on the sibling_list. This ensures destruction
  5822. * of the group leader will find the pointer to itself in
  5823. * perf_group_detach().
  5824. */
  5825. fdput(group);
  5826. fd_install(event_fd, event_file);
  5827. return event_fd;
  5828. err_context:
  5829. perf_unpin_context(ctx);
  5830. put_ctx(ctx);
  5831. err_alloc:
  5832. free_event(event);
  5833. err_task:
  5834. put_online_cpus();
  5835. if (task)
  5836. put_task_struct(task);
  5837. err_group_fd:
  5838. fdput(group);
  5839. err_fd:
  5840. put_unused_fd(event_fd);
  5841. return err;
  5842. }
  5843. /**
  5844. * perf_event_create_kernel_counter
  5845. *
  5846. * @attr: attributes of the counter to create
  5847. * @cpu: cpu in which the counter is bound
  5848. * @task: task to profile (NULL for percpu)
  5849. */
  5850. struct perf_event *
  5851. perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
  5852. struct task_struct *task,
  5853. perf_overflow_handler_t overflow_handler,
  5854. void *context)
  5855. {
  5856. struct perf_event_context *ctx;
  5857. struct perf_event *event;
  5858. int err;
  5859. /*
  5860. * Get the target context (task or percpu):
  5861. */
  5862. event = perf_event_alloc(attr, cpu, task, NULL, NULL,
  5863. overflow_handler, context);
  5864. if (IS_ERR(event)) {
  5865. err = PTR_ERR(event);
  5866. goto err;
  5867. }
  5868. account_event(event);
  5869. ctx = find_get_context(event->pmu, task, cpu);
  5870. if (IS_ERR(ctx)) {
  5871. err = PTR_ERR(ctx);
  5872. goto err_free;
  5873. }
  5874. WARN_ON_ONCE(ctx->parent_ctx);
  5875. mutex_lock(&ctx->mutex);
  5876. perf_install_in_context(ctx, event, cpu);
  5877. ++ctx->generation;
  5878. perf_unpin_context(ctx);
  5879. mutex_unlock(&ctx->mutex);
  5880. return event;
  5881. err_free:
  5882. free_event(event);
  5883. err:
  5884. return ERR_PTR(err);
  5885. }
  5886. EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
  5887. void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
  5888. {
  5889. struct perf_event_context *src_ctx;
  5890. struct perf_event_context *dst_ctx;
  5891. struct perf_event *event, *tmp;
  5892. LIST_HEAD(events);
  5893. src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx;
  5894. dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx;
  5895. mutex_lock(&src_ctx->mutex);
  5896. list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
  5897. event_entry) {
  5898. perf_remove_from_context(event);
  5899. unaccount_event_cpu(event, src_cpu);
  5900. put_ctx(src_ctx);
  5901. list_add(&event->event_entry, &events);
  5902. }
  5903. mutex_unlock(&src_ctx->mutex);
  5904. synchronize_rcu();
  5905. mutex_lock(&dst_ctx->mutex);
  5906. list_for_each_entry_safe(event, tmp, &events, event_entry) {
  5907. list_del(&event->event_entry);
  5908. if (event->state >= PERF_EVENT_STATE_OFF)
  5909. event->state = PERF_EVENT_STATE_INACTIVE;
  5910. account_event_cpu(event, dst_cpu);
  5911. perf_install_in_context(dst_ctx, event, dst_cpu);
  5912. get_ctx(dst_ctx);
  5913. }
  5914. mutex_unlock(&dst_ctx->mutex);
  5915. }
  5916. EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
  5917. static void sync_child_event(struct perf_event *child_event,
  5918. struct task_struct *child)
  5919. {
  5920. struct perf_event *parent_event = child_event->parent;
  5921. u64 child_val;
  5922. if (child_event->attr.inherit_stat)
  5923. perf_event_read_event(child_event, child);
  5924. child_val = perf_event_count(child_event);
  5925. /*
  5926. * Add back the child's count to the parent's count:
  5927. */
  5928. atomic64_add(child_val, &parent_event->child_count);
  5929. atomic64_add(child_event->total_time_enabled,
  5930. &parent_event->child_total_time_enabled);
  5931. atomic64_add(child_event->total_time_running,
  5932. &parent_event->child_total_time_running);
  5933. /*
  5934. * Remove this event from the parent's list
  5935. */
  5936. WARN_ON_ONCE(parent_event->ctx->parent_ctx);
  5937. mutex_lock(&parent_event->child_mutex);
  5938. list_del_init(&child_event->child_list);
  5939. mutex_unlock(&parent_event->child_mutex);
  5940. /*
  5941. * Release the parent event, if this was the last
  5942. * reference to it.
  5943. */
  5944. put_event(parent_event);
  5945. }
  5946. static void
  5947. __perf_event_exit_task(struct perf_event *child_event,
  5948. struct perf_event_context *child_ctx,
  5949. struct task_struct *child)
  5950. {
  5951. if (child_event->parent) {
  5952. raw_spin_lock_irq(&child_ctx->lock);
  5953. perf_group_detach(child_event);
  5954. raw_spin_unlock_irq(&child_ctx->lock);
  5955. }
  5956. perf_remove_from_context(child_event);
  5957. /*
  5958. * It can happen that the parent exits first, and has events
  5959. * that are still around due to the child reference. These
  5960. * events need to be zapped.
  5961. */
  5962. if (child_event->parent) {
  5963. sync_child_event(child_event, child);
  5964. free_event(child_event);
  5965. }
  5966. }
  5967. static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
  5968. {
  5969. struct perf_event *child_event, *tmp;
  5970. struct perf_event_context *child_ctx;
  5971. unsigned long flags;
  5972. if (likely(!child->perf_event_ctxp[ctxn])) {
  5973. perf_event_task(child, NULL, 0);
  5974. return;
  5975. }
  5976. local_irq_save(flags);
  5977. /*
  5978. * We can't reschedule here because interrupts are disabled,
  5979. * and either child is current or it is a task that can't be
  5980. * scheduled, so we are now safe from rescheduling changing
  5981. * our context.
  5982. */
  5983. child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
  5984. /*
  5985. * Take the context lock here so that if find_get_context is
  5986. * reading child->perf_event_ctxp, we wait until it has
  5987. * incremented the context's refcount before we do put_ctx below.
  5988. */
  5989. raw_spin_lock(&child_ctx->lock);
  5990. task_ctx_sched_out(child_ctx);
  5991. child->perf_event_ctxp[ctxn] = NULL;
  5992. /*
  5993. * If this context is a clone; unclone it so it can't get
  5994. * swapped to another process while we're removing all
  5995. * the events from it.
  5996. */
  5997. unclone_ctx(child_ctx);
  5998. update_context_time(child_ctx);
  5999. raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
  6000. /*
  6001. * Report the task dead after unscheduling the events so that we
  6002. * won't get any samples after PERF_RECORD_EXIT. We can however still
  6003. * get a few PERF_RECORD_READ events.
  6004. */
  6005. perf_event_task(child, child_ctx, 0);
  6006. /*
  6007. * We can recurse on the same lock type through:
  6008. *
  6009. * __perf_event_exit_task()
  6010. * sync_child_event()
  6011. * put_event()
  6012. * mutex_lock(&ctx->mutex)
  6013. *
  6014. * But since its the parent context it won't be the same instance.
  6015. */
  6016. mutex_lock(&child_ctx->mutex);
  6017. again:
  6018. list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
  6019. group_entry)
  6020. __perf_event_exit_task(child_event, child_ctx, child);
  6021. list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
  6022. group_entry)
  6023. __perf_event_exit_task(child_event, child_ctx, child);
  6024. /*
  6025. * If the last event was a group event, it will have appended all
  6026. * its siblings to the list, but we obtained 'tmp' before that which
  6027. * will still point to the list head terminating the iteration.
  6028. */
  6029. if (!list_empty(&child_ctx->pinned_groups) ||
  6030. !list_empty(&child_ctx->flexible_groups))
  6031. goto again;
  6032. mutex_unlock(&child_ctx->mutex);
  6033. put_ctx(child_ctx);
  6034. }
  6035. /*
  6036. * When a child task exits, feed back event values to parent events.
  6037. */
  6038. void perf_event_exit_task(struct task_struct *child)
  6039. {
  6040. struct perf_event *event, *tmp;
  6041. int ctxn;
  6042. mutex_lock(&child->perf_event_mutex);
  6043. list_for_each_entry_safe(event, tmp, &child->perf_event_list,
  6044. owner_entry) {
  6045. list_del_init(&event->owner_entry);
  6046. /*
  6047. * Ensure the list deletion is visible before we clear
  6048. * the owner, closes a race against perf_release() where
  6049. * we need to serialize on the owner->perf_event_mutex.
  6050. */
  6051. smp_wmb();
  6052. event->owner = NULL;
  6053. }
  6054. mutex_unlock(&child->perf_event_mutex);
  6055. for_each_task_context_nr(ctxn)
  6056. perf_event_exit_task_context(child, ctxn);
  6057. }
  6058. static void perf_free_event(struct perf_event *event,
  6059. struct perf_event_context *ctx)
  6060. {
  6061. struct perf_event *parent = event->parent;
  6062. if (WARN_ON_ONCE(!parent))
  6063. return;
  6064. mutex_lock(&parent->child_mutex);
  6065. list_del_init(&event->child_list);
  6066. mutex_unlock(&parent->child_mutex);
  6067. put_event(parent);
  6068. perf_group_detach(event);
  6069. list_del_event(event, ctx);
  6070. free_event(event);
  6071. }
  6072. /*
  6073. * free an unexposed, unused context as created by inheritance by
  6074. * perf_event_init_task below, used by fork() in case of fail.
  6075. */
  6076. void perf_event_free_task(struct task_struct *task)
  6077. {
  6078. struct perf_event_context *ctx;
  6079. struct perf_event *event, *tmp;
  6080. int ctxn;
  6081. for_each_task_context_nr(ctxn) {
  6082. ctx = task->perf_event_ctxp[ctxn];
  6083. if (!ctx)
  6084. continue;
  6085. mutex_lock(&ctx->mutex);
  6086. again:
  6087. list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
  6088. group_entry)
  6089. perf_free_event(event, ctx);
  6090. list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
  6091. group_entry)
  6092. perf_free_event(event, ctx);
  6093. if (!list_empty(&ctx->pinned_groups) ||
  6094. !list_empty(&ctx->flexible_groups))
  6095. goto again;
  6096. mutex_unlock(&ctx->mutex);
  6097. put_ctx(ctx);
  6098. }
  6099. }
  6100. void perf_event_delayed_put(struct task_struct *task)
  6101. {
  6102. int ctxn;
  6103. for_each_task_context_nr(ctxn)
  6104. WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
  6105. }
  6106. /*
  6107. * inherit a event from parent task to child task:
  6108. */
  6109. static struct perf_event *
  6110. inherit_event(struct perf_event *parent_event,
  6111. struct task_struct *parent,
  6112. struct perf_event_context *parent_ctx,
  6113. struct task_struct *child,
  6114. struct perf_event *group_leader,
  6115. struct perf_event_context *child_ctx)
  6116. {
  6117. struct perf_event *child_event;
  6118. unsigned long flags;
  6119. /*
  6120. * Instead of creating recursive hierarchies of events,
  6121. * we link inherited events back to the original parent,
  6122. * which has a filp for sure, which we use as the reference
  6123. * count:
  6124. */
  6125. if (parent_event->parent)
  6126. parent_event = parent_event->parent;
  6127. child_event = perf_event_alloc(&parent_event->attr,
  6128. parent_event->cpu,
  6129. child,
  6130. group_leader, parent_event,
  6131. NULL, NULL);
  6132. if (IS_ERR(child_event))
  6133. return child_event;
  6134. if (!atomic_long_inc_not_zero(&parent_event->refcount)) {
  6135. free_event(child_event);
  6136. return NULL;
  6137. }
  6138. get_ctx(child_ctx);
  6139. /*
  6140. * Make the child state follow the state of the parent event,
  6141. * not its attr.disabled bit. We hold the parent's mutex,
  6142. * so we won't race with perf_event_{en, dis}able_family.
  6143. */
  6144. if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
  6145. child_event->state = PERF_EVENT_STATE_INACTIVE;
  6146. else
  6147. child_event->state = PERF_EVENT_STATE_OFF;
  6148. if (parent_event->attr.freq) {
  6149. u64 sample_period = parent_event->hw.sample_period;
  6150. struct hw_perf_event *hwc = &child_event->hw;
  6151. hwc->sample_period = sample_period;
  6152. hwc->last_period = sample_period;
  6153. local64_set(&hwc->period_left, sample_period);
  6154. }
  6155. child_event->ctx = child_ctx;
  6156. child_event->overflow_handler = parent_event->overflow_handler;
  6157. child_event->overflow_handler_context
  6158. = parent_event->overflow_handler_context;
  6159. /*
  6160. * Precalculate sample_data sizes
  6161. */
  6162. perf_event__header_size(child_event);
  6163. perf_event__id_header_size(child_event);
  6164. /*
  6165. * Link it up in the child's context:
  6166. */
  6167. raw_spin_lock_irqsave(&child_ctx->lock, flags);
  6168. add_event_to_ctx(child_event, child_ctx);
  6169. raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
  6170. /*
  6171. * Link this into the parent event's child list
  6172. */
  6173. WARN_ON_ONCE(parent_event->ctx->parent_ctx);
  6174. mutex_lock(&parent_event->child_mutex);
  6175. list_add_tail(&child_event->child_list, &parent_event->child_list);
  6176. mutex_unlock(&parent_event->child_mutex);
  6177. return child_event;
  6178. }
  6179. static int inherit_group(struct perf_event *parent_event,
  6180. struct task_struct *parent,
  6181. struct perf_event_context *parent_ctx,
  6182. struct task_struct *child,
  6183. struct perf_event_context *child_ctx)
  6184. {
  6185. struct perf_event *leader;
  6186. struct perf_event *sub;
  6187. struct perf_event *child_ctr;
  6188. leader = inherit_event(parent_event, parent, parent_ctx,
  6189. child, NULL, child_ctx);
  6190. if (IS_ERR(leader))
  6191. return PTR_ERR(leader);
  6192. list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
  6193. child_ctr = inherit_event(sub, parent, parent_ctx,
  6194. child, leader, child_ctx);
  6195. if (IS_ERR(child_ctr))
  6196. return PTR_ERR(child_ctr);
  6197. }
  6198. return 0;
  6199. }
  6200. static int
  6201. inherit_task_group(struct perf_event *event, struct task_struct *parent,
  6202. struct perf_event_context *parent_ctx,
  6203. struct task_struct *child, int ctxn,
  6204. int *inherited_all)
  6205. {
  6206. int ret;
  6207. struct perf_event_context *child_ctx;
  6208. if (!event->attr.inherit) {
  6209. *inherited_all = 0;
  6210. return 0;
  6211. }
  6212. child_ctx = child->perf_event_ctxp[ctxn];
  6213. if (!child_ctx) {
  6214. /*
  6215. * This is executed from the parent task context, so
  6216. * inherit events that have been marked for cloning.
  6217. * First allocate and initialize a context for the
  6218. * child.
  6219. */
  6220. child_ctx = alloc_perf_context(parent_ctx->pmu, child);
  6221. if (!child_ctx)
  6222. return -ENOMEM;
  6223. child->perf_event_ctxp[ctxn] = child_ctx;
  6224. }
  6225. ret = inherit_group(event, parent, parent_ctx,
  6226. child, child_ctx);
  6227. if (ret)
  6228. *inherited_all = 0;
  6229. return ret;
  6230. }
  6231. /*
  6232. * Initialize the perf_event context in task_struct
  6233. */
  6234. int perf_event_init_context(struct task_struct *child, int ctxn)
  6235. {
  6236. struct perf_event_context *child_ctx, *parent_ctx;
  6237. struct perf_event_context *cloned_ctx;
  6238. struct perf_event *event;
  6239. struct task_struct *parent = current;
  6240. int inherited_all = 1;
  6241. unsigned long flags;
  6242. int ret = 0;
  6243. if (likely(!parent->perf_event_ctxp[ctxn]))
  6244. return 0;
  6245. /*
  6246. * If the parent's context is a clone, pin it so it won't get
  6247. * swapped under us.
  6248. */
  6249. parent_ctx = perf_pin_task_context(parent, ctxn);
  6250. /*
  6251. * No need to check if parent_ctx != NULL here; since we saw
  6252. * it non-NULL earlier, the only reason for it to become NULL
  6253. * is if we exit, and since we're currently in the middle of
  6254. * a fork we can't be exiting at the same time.
  6255. */
  6256. /*
  6257. * Lock the parent list. No need to lock the child - not PID
  6258. * hashed yet and not running, so nobody can access it.
  6259. */
  6260. mutex_lock(&parent_ctx->mutex);
  6261. /*
  6262. * We dont have to disable NMIs - we are only looking at
  6263. * the list, not manipulating it:
  6264. */
  6265. list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
  6266. ret = inherit_task_group(event, parent, parent_ctx,
  6267. child, ctxn, &inherited_all);
  6268. if (ret)
  6269. break;
  6270. }
  6271. /*
  6272. * We can't hold ctx->lock when iterating the ->flexible_group list due
  6273. * to allocations, but we need to prevent rotation because
  6274. * rotate_ctx() will change the list from interrupt context.
  6275. */
  6276. raw_spin_lock_irqsave(&parent_ctx->lock, flags);
  6277. parent_ctx->rotate_disable = 1;
  6278. raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
  6279. list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
  6280. ret = inherit_task_group(event, parent, parent_ctx,
  6281. child, ctxn, &inherited_all);
  6282. if (ret)
  6283. break;
  6284. }
  6285. raw_spin_lock_irqsave(&parent_ctx->lock, flags);
  6286. parent_ctx->rotate_disable = 0;
  6287. child_ctx = child->perf_event_ctxp[ctxn];
  6288. if (child_ctx && inherited_all) {
  6289. /*
  6290. * Mark the child context as a clone of the parent
  6291. * context, or of whatever the parent is a clone of.
  6292. *
  6293. * Note that if the parent is a clone, the holding of
  6294. * parent_ctx->lock avoids it from being uncloned.
  6295. */
  6296. cloned_ctx = parent_ctx->parent_ctx;
  6297. if (cloned_ctx) {
  6298. child_ctx->parent_ctx = cloned_ctx;
  6299. child_ctx->parent_gen = parent_ctx->parent_gen;
  6300. } else {
  6301. child_ctx->parent_ctx = parent_ctx;
  6302. child_ctx->parent_gen = parent_ctx->generation;
  6303. }
  6304. get_ctx(child_ctx->parent_ctx);
  6305. }
  6306. raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
  6307. mutex_unlock(&parent_ctx->mutex);
  6308. perf_unpin_context(parent_ctx);
  6309. put_ctx(parent_ctx);
  6310. return ret;
  6311. }
  6312. /*
  6313. * Initialize the perf_event context in task_struct
  6314. */
  6315. int perf_event_init_task(struct task_struct *child)
  6316. {
  6317. int ctxn, ret;
  6318. memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
  6319. mutex_init(&child->perf_event_mutex);
  6320. INIT_LIST_HEAD(&child->perf_event_list);
  6321. for_each_task_context_nr(ctxn) {
  6322. ret = perf_event_init_context(child, ctxn);
  6323. if (ret)
  6324. return ret;
  6325. }
  6326. return 0;
  6327. }
  6328. static void __init perf_event_init_all_cpus(void)
  6329. {
  6330. struct swevent_htable *swhash;
  6331. int cpu;
  6332. for_each_possible_cpu(cpu) {
  6333. swhash = &per_cpu(swevent_htable, cpu);
  6334. mutex_init(&swhash->hlist_mutex);
  6335. INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
  6336. }
  6337. }
  6338. static void perf_event_init_cpu(int cpu)
  6339. {
  6340. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  6341. mutex_lock(&swhash->hlist_mutex);
  6342. if (swhash->hlist_refcount > 0) {
  6343. struct swevent_hlist *hlist;
  6344. hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
  6345. WARN_ON(!hlist);
  6346. rcu_assign_pointer(swhash->swevent_hlist, hlist);
  6347. }
  6348. mutex_unlock(&swhash->hlist_mutex);
  6349. }
  6350. #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
  6351. static void perf_pmu_rotate_stop(struct pmu *pmu)
  6352. {
  6353. struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  6354. WARN_ON(!irqs_disabled());
  6355. list_del_init(&cpuctx->rotation_list);
  6356. }
  6357. static void __perf_event_exit_context(void *__info)
  6358. {
  6359. struct perf_event_context *ctx = __info;
  6360. struct perf_event *event, *tmp;
  6361. perf_pmu_rotate_stop(ctx->pmu);
  6362. list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
  6363. __perf_remove_from_context(event);
  6364. list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
  6365. __perf_remove_from_context(event);
  6366. }
  6367. static void perf_event_exit_cpu_context(int cpu)
  6368. {
  6369. struct perf_event_context *ctx;
  6370. struct pmu *pmu;
  6371. int idx;
  6372. idx = srcu_read_lock(&pmus_srcu);
  6373. list_for_each_entry_rcu(pmu, &pmus, entry) {
  6374. ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
  6375. mutex_lock(&ctx->mutex);
  6376. smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
  6377. mutex_unlock(&ctx->mutex);
  6378. }
  6379. srcu_read_unlock(&pmus_srcu, idx);
  6380. }
  6381. static void perf_event_exit_cpu(int cpu)
  6382. {
  6383. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  6384. mutex_lock(&swhash->hlist_mutex);
  6385. swevent_hlist_release(swhash);
  6386. mutex_unlock(&swhash->hlist_mutex);
  6387. perf_event_exit_cpu_context(cpu);
  6388. }
  6389. #else
  6390. static inline void perf_event_exit_cpu(int cpu) { }
  6391. #endif
  6392. static int
  6393. perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
  6394. {
  6395. int cpu;
  6396. for_each_online_cpu(cpu)
  6397. perf_event_exit_cpu(cpu);
  6398. return NOTIFY_OK;
  6399. }
  6400. /*
  6401. * Run the perf reboot notifier at the very last possible moment so that
  6402. * the generic watchdog code runs as long as possible.
  6403. */
  6404. static struct notifier_block perf_reboot_notifier = {
  6405. .notifier_call = perf_reboot,
  6406. .priority = INT_MIN,
  6407. };
  6408. static int
  6409. perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
  6410. {
  6411. unsigned int cpu = (long)hcpu;
  6412. switch (action & ~CPU_TASKS_FROZEN) {
  6413. case CPU_UP_PREPARE:
  6414. case CPU_DOWN_FAILED:
  6415. perf_event_init_cpu(cpu);
  6416. break;
  6417. case CPU_UP_CANCELED:
  6418. case CPU_DOWN_PREPARE:
  6419. perf_event_exit_cpu(cpu);
  6420. break;
  6421. default:
  6422. break;
  6423. }
  6424. return NOTIFY_OK;
  6425. }
  6426. void __init perf_event_init(void)
  6427. {
  6428. int ret;
  6429. idr_init(&pmu_idr);
  6430. perf_event_init_all_cpus();
  6431. init_srcu_struct(&pmus_srcu);
  6432. perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
  6433. perf_pmu_register(&perf_cpu_clock, NULL, -1);
  6434. perf_pmu_register(&perf_task_clock, NULL, -1);
  6435. perf_tp_register();
  6436. perf_cpu_notifier(perf_cpu_notify);
  6437. register_reboot_notifier(&perf_reboot_notifier);
  6438. ret = init_hw_breakpoint();
  6439. WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
  6440. /* do not patch jump label more than once per second */
  6441. jump_label_rate_limit(&perf_sched_events, HZ);
  6442. /*
  6443. * Build time assertion that we keep the data_head at the intended
  6444. * location. IOW, validation we got the __reserved[] size right.
  6445. */
  6446. BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
  6447. != 1024);
  6448. }
  6449. static int __init perf_event_sysfs_init(void)
  6450. {
  6451. struct pmu *pmu;
  6452. int ret;
  6453. mutex_lock(&pmus_lock);
  6454. ret = bus_register(&pmu_bus);
  6455. if (ret)
  6456. goto unlock;
  6457. list_for_each_entry(pmu, &pmus, entry) {
  6458. if (!pmu->name || pmu->type < 0)
  6459. continue;
  6460. ret = pmu_dev_alloc(pmu);
  6461. WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
  6462. }
  6463. pmu_bus_running = 1;
  6464. ret = 0;
  6465. unlock:
  6466. mutex_unlock(&pmus_lock);
  6467. return ret;
  6468. }
  6469. device_initcall(perf_event_sysfs_init);
  6470. #ifdef CONFIG_CGROUP_PERF
  6471. static struct cgroup_subsys_state *
  6472. perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
  6473. {
  6474. struct perf_cgroup *jc;
  6475. jc = kzalloc(sizeof(*jc), GFP_KERNEL);
  6476. if (!jc)
  6477. return ERR_PTR(-ENOMEM);
  6478. jc->info = alloc_percpu(struct perf_cgroup_info);
  6479. if (!jc->info) {
  6480. kfree(jc);
  6481. return ERR_PTR(-ENOMEM);
  6482. }
  6483. return &jc->css;
  6484. }
  6485. static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
  6486. {
  6487. struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
  6488. free_percpu(jc->info);
  6489. kfree(jc);
  6490. }
  6491. static int __perf_cgroup_move(void *info)
  6492. {
  6493. struct task_struct *task = info;
  6494. perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
  6495. return 0;
  6496. }
  6497. static void perf_cgroup_attach(struct cgroup_subsys_state *css,
  6498. struct cgroup_taskset *tset)
  6499. {
  6500. struct task_struct *task;
  6501. cgroup_taskset_for_each(task, css, tset)
  6502. task_function_call(task, __perf_cgroup_move, task);
  6503. }
  6504. static void perf_cgroup_exit(struct cgroup_subsys_state *css,
  6505. struct cgroup_subsys_state *old_css,
  6506. struct task_struct *task)
  6507. {
  6508. /*
  6509. * cgroup_exit() is called in the copy_process() failure path.
  6510. * Ignore this case since the task hasn't ran yet, this avoids
  6511. * trying to poke a half freed task state from generic code.
  6512. */
  6513. if (!(task->flags & PF_EXITING))
  6514. return;
  6515. task_function_call(task, __perf_cgroup_move, task);
  6516. }
  6517. struct cgroup_subsys perf_subsys = {
  6518. .name = "perf_event",
  6519. .subsys_id = perf_subsys_id,
  6520. .css_alloc = perf_cgroup_css_alloc,
  6521. .css_free = perf_cgroup_css_free,
  6522. .exit = perf_cgroup_exit,
  6523. .attach = perf_cgroup_attach,
  6524. };
  6525. #endif /* CONFIG_CGROUP_PERF */