perfmon.c 169 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831
  1. /*
  2. * This file implements the perfmon-2 subsystem which is used
  3. * to program the IA-64 Performance Monitoring Unit (PMU).
  4. *
  5. * The initial version of perfmon.c was written by
  6. * Ganesh Venkitachalam, IBM Corp.
  7. *
  8. * Then it was modified for perfmon-1.x by Stephane Eranian and
  9. * David Mosberger, Hewlett Packard Co.
  10. *
  11. * Version Perfmon-2.x is a rewrite of perfmon-1.x
  12. * by Stephane Eranian, Hewlett Packard Co.
  13. *
  14. * Copyright (C) 1999-2005 Hewlett Packard Co
  15. * Stephane Eranian <eranian@hpl.hp.com>
  16. * David Mosberger-Tang <davidm@hpl.hp.com>
  17. *
  18. * More information about perfmon available at:
  19. * http://www.hpl.hp.com/research/linux/perfmon
  20. */
  21. #include <linux/module.h>
  22. #include <linux/kernel.h>
  23. #include <linux/sched.h>
  24. #include <linux/interrupt.h>
  25. #include <linux/proc_fs.h>
  26. #include <linux/seq_file.h>
  27. #include <linux/init.h>
  28. #include <linux/vmalloc.h>
  29. #include <linux/mm.h>
  30. #include <linux/sysctl.h>
  31. #include <linux/list.h>
  32. #include <linux/file.h>
  33. #include <linux/poll.h>
  34. #include <linux/vfs.h>
  35. #include <linux/smp.h>
  36. #include <linux/pagemap.h>
  37. #include <linux/mount.h>
  38. #include <linux/bitops.h>
  39. #include <linux/capability.h>
  40. #include <linux/rcupdate.h>
  41. #include <linux/completion.h>
  42. #include <asm/errno.h>
  43. #include <asm/intrinsics.h>
  44. #include <asm/page.h>
  45. #include <asm/perfmon.h>
  46. #include <asm/processor.h>
  47. #include <asm/signal.h>
  48. #include <asm/system.h>
  49. #include <asm/uaccess.h>
  50. #include <asm/delay.h>
  51. #ifdef CONFIG_PERFMON
  52. /*
  53. * perfmon context state
  54. */
  55. #define PFM_CTX_UNLOADED 1 /* context is not loaded onto any task */
  56. #define PFM_CTX_LOADED 2 /* context is loaded onto a task */
  57. #define PFM_CTX_MASKED 3 /* context is loaded but monitoring is masked due to overflow */
  58. #define PFM_CTX_ZOMBIE 4 /* owner of the context is closing it */
  59. #define PFM_INVALID_ACTIVATION (~0UL)
  60. #define PFM_NUM_PMC_REGS 64 /* PMC save area for ctxsw */
  61. #define PFM_NUM_PMD_REGS 64 /* PMD save area for ctxsw */
  62. /*
  63. * depth of message queue
  64. */
  65. #define PFM_MAX_MSGS 32
  66. #define PFM_CTXQ_EMPTY(g) ((g)->ctx_msgq_head == (g)->ctx_msgq_tail)
  67. /*
  68. * type of a PMU register (bitmask).
  69. * bitmask structure:
  70. * bit0 : register implemented
  71. * bit1 : end marker
  72. * bit2-3 : reserved
  73. * bit4 : pmc has pmc.pm
  74. * bit5 : pmc controls a counter (has pmc.oi), pmd is used as counter
  75. * bit6-7 : register type
  76. * bit8-31: reserved
  77. */
  78. #define PFM_REG_NOTIMPL 0x0 /* not implemented at all */
  79. #define PFM_REG_IMPL 0x1 /* register implemented */
  80. #define PFM_REG_END 0x2 /* end marker */
  81. #define PFM_REG_MONITOR (0x1<<4|PFM_REG_IMPL) /* a PMC with a pmc.pm field only */
  82. #define PFM_REG_COUNTING (0x2<<4|PFM_REG_MONITOR) /* a monitor + pmc.oi+ PMD used as a counter */
  83. #define PFM_REG_CONTROL (0x4<<4|PFM_REG_IMPL) /* PMU control register */
  84. #define PFM_REG_CONFIG (0x8<<4|PFM_REG_IMPL) /* configuration register */
  85. #define PFM_REG_BUFFER (0xc<<4|PFM_REG_IMPL) /* PMD used as buffer */
  86. #define PMC_IS_LAST(i) (pmu_conf->pmc_desc[i].type & PFM_REG_END)
  87. #define PMD_IS_LAST(i) (pmu_conf->pmd_desc[i].type & PFM_REG_END)
  88. #define PMC_OVFL_NOTIFY(ctx, i) ((ctx)->ctx_pmds[i].flags & PFM_REGFL_OVFL_NOTIFY)
  89. /* i assumed unsigned */
  90. #define PMC_IS_IMPL(i) (i< PMU_MAX_PMCS && (pmu_conf->pmc_desc[i].type & PFM_REG_IMPL))
  91. #define PMD_IS_IMPL(i) (i< PMU_MAX_PMDS && (pmu_conf->pmd_desc[i].type & PFM_REG_IMPL))
  92. /* XXX: these assume that register i is implemented */
  93. #define PMD_IS_COUNTING(i) ((pmu_conf->pmd_desc[i].type & PFM_REG_COUNTING) == PFM_REG_COUNTING)
  94. #define PMC_IS_COUNTING(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_COUNTING) == PFM_REG_COUNTING)
  95. #define PMC_IS_MONITOR(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_MONITOR) == PFM_REG_MONITOR)
  96. #define PMC_IS_CONTROL(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_CONTROL) == PFM_REG_CONTROL)
  97. #define PMC_DFL_VAL(i) pmu_conf->pmc_desc[i].default_value
  98. #define PMC_RSVD_MASK(i) pmu_conf->pmc_desc[i].reserved_mask
  99. #define PMD_PMD_DEP(i) pmu_conf->pmd_desc[i].dep_pmd[0]
  100. #define PMC_PMD_DEP(i) pmu_conf->pmc_desc[i].dep_pmd[0]
  101. #define PFM_NUM_IBRS IA64_NUM_DBG_REGS
  102. #define PFM_NUM_DBRS IA64_NUM_DBG_REGS
  103. #define CTX_OVFL_NOBLOCK(c) ((c)->ctx_fl_block == 0)
  104. #define CTX_HAS_SMPL(c) ((c)->ctx_fl_is_sampling)
  105. #define PFM_CTX_TASK(h) (h)->ctx_task
  106. #define PMU_PMC_OI 5 /* position of pmc.oi bit */
  107. /* XXX: does not support more than 64 PMDs */
  108. #define CTX_USED_PMD(ctx, mask) (ctx)->ctx_used_pmds[0] |= (mask)
  109. #define CTX_IS_USED_PMD(ctx, c) (((ctx)->ctx_used_pmds[0] & (1UL << (c))) != 0UL)
  110. #define CTX_USED_MONITOR(ctx, mask) (ctx)->ctx_used_monitors[0] |= (mask)
  111. #define CTX_USED_IBR(ctx,n) (ctx)->ctx_used_ibrs[(n)>>6] |= 1UL<< ((n) % 64)
  112. #define CTX_USED_DBR(ctx,n) (ctx)->ctx_used_dbrs[(n)>>6] |= 1UL<< ((n) % 64)
  113. #define CTX_USES_DBREGS(ctx) (((pfm_context_t *)(ctx))->ctx_fl_using_dbreg==1)
  114. #define PFM_CODE_RR 0 /* requesting code range restriction */
  115. #define PFM_DATA_RR 1 /* requestion data range restriction */
  116. #define PFM_CPUINFO_CLEAR(v) pfm_get_cpu_var(pfm_syst_info) &= ~(v)
  117. #define PFM_CPUINFO_SET(v) pfm_get_cpu_var(pfm_syst_info) |= (v)
  118. #define PFM_CPUINFO_GET() pfm_get_cpu_var(pfm_syst_info)
  119. #define RDEP(x) (1UL<<(x))
  120. /*
  121. * context protection macros
  122. * in SMP:
  123. * - we need to protect against CPU concurrency (spin_lock)
  124. * - we need to protect against PMU overflow interrupts (local_irq_disable)
  125. * in UP:
  126. * - we need to protect against PMU overflow interrupts (local_irq_disable)
  127. *
  128. * spin_lock_irqsave()/spin_unlock_irqrestore():
  129. * in SMP: local_irq_disable + spin_lock
  130. * in UP : local_irq_disable
  131. *
  132. * spin_lock()/spin_lock():
  133. * in UP : removed automatically
  134. * in SMP: protect against context accesses from other CPU. interrupts
  135. * are not masked. This is useful for the PMU interrupt handler
  136. * because we know we will not get PMU concurrency in that code.
  137. */
  138. #define PROTECT_CTX(c, f) \
  139. do { \
  140. DPRINT(("spinlock_irq_save ctx %p by [%d]\n", c, task_pid_nr(current))); \
  141. spin_lock_irqsave(&(c)->ctx_lock, f); \
  142. DPRINT(("spinlocked ctx %p by [%d]\n", c, task_pid_nr(current))); \
  143. } while(0)
  144. #define UNPROTECT_CTX(c, f) \
  145. do { \
  146. DPRINT(("spinlock_irq_restore ctx %p by [%d]\n", c, task_pid_nr(current))); \
  147. spin_unlock_irqrestore(&(c)->ctx_lock, f); \
  148. } while(0)
  149. #define PROTECT_CTX_NOPRINT(c, f) \
  150. do { \
  151. spin_lock_irqsave(&(c)->ctx_lock, f); \
  152. } while(0)
  153. #define UNPROTECT_CTX_NOPRINT(c, f) \
  154. do { \
  155. spin_unlock_irqrestore(&(c)->ctx_lock, f); \
  156. } while(0)
  157. #define PROTECT_CTX_NOIRQ(c) \
  158. do { \
  159. spin_lock(&(c)->ctx_lock); \
  160. } while(0)
  161. #define UNPROTECT_CTX_NOIRQ(c) \
  162. do { \
  163. spin_unlock(&(c)->ctx_lock); \
  164. } while(0)
  165. #ifdef CONFIG_SMP
  166. #define GET_ACTIVATION() pfm_get_cpu_var(pmu_activation_number)
  167. #define INC_ACTIVATION() pfm_get_cpu_var(pmu_activation_number)++
  168. #define SET_ACTIVATION(c) (c)->ctx_last_activation = GET_ACTIVATION()
  169. #else /* !CONFIG_SMP */
  170. #define SET_ACTIVATION(t) do {} while(0)
  171. #define GET_ACTIVATION(t) do {} while(0)
  172. #define INC_ACTIVATION(t) do {} while(0)
  173. #endif /* CONFIG_SMP */
  174. #define SET_PMU_OWNER(t, c) do { pfm_get_cpu_var(pmu_owner) = (t); pfm_get_cpu_var(pmu_ctx) = (c); } while(0)
  175. #define GET_PMU_OWNER() pfm_get_cpu_var(pmu_owner)
  176. #define GET_PMU_CTX() pfm_get_cpu_var(pmu_ctx)
  177. #define LOCK_PFS(g) spin_lock_irqsave(&pfm_sessions.pfs_lock, g)
  178. #define UNLOCK_PFS(g) spin_unlock_irqrestore(&pfm_sessions.pfs_lock, g)
  179. #define PFM_REG_RETFLAG_SET(flags, val) do { flags &= ~PFM_REG_RETFL_MASK; flags |= (val); } while(0)
  180. /*
  181. * cmp0 must be the value of pmc0
  182. */
  183. #define PMC0_HAS_OVFL(cmp0) (cmp0 & ~0x1UL)
  184. #define PFMFS_MAGIC 0xa0b4d889
  185. /*
  186. * debugging
  187. */
  188. #define PFM_DEBUGGING 1
  189. #ifdef PFM_DEBUGGING
  190. #define DPRINT(a) \
  191. do { \
  192. if (unlikely(pfm_sysctl.debug >0)) { printk("%s.%d: CPU%d [%d] ", __func__, __LINE__, smp_processor_id(), task_pid_nr(current)); printk a; } \
  193. } while (0)
  194. #define DPRINT_ovfl(a) \
  195. do { \
  196. if (unlikely(pfm_sysctl.debug > 0 && pfm_sysctl.debug_ovfl >0)) { printk("%s.%d: CPU%d [%d] ", __func__, __LINE__, smp_processor_id(), task_pid_nr(current)); printk a; } \
  197. } while (0)
  198. #endif
  199. /*
  200. * 64-bit software counter structure
  201. *
  202. * the next_reset_type is applied to the next call to pfm_reset_regs()
  203. */
  204. typedef struct {
  205. unsigned long val; /* virtual 64bit counter value */
  206. unsigned long lval; /* last reset value */
  207. unsigned long long_reset; /* reset value on sampling overflow */
  208. unsigned long short_reset; /* reset value on overflow */
  209. unsigned long reset_pmds[4]; /* which other pmds to reset when this counter overflows */
  210. unsigned long smpl_pmds[4]; /* which pmds are accessed when counter overflow */
  211. unsigned long seed; /* seed for random-number generator */
  212. unsigned long mask; /* mask for random-number generator */
  213. unsigned int flags; /* notify/do not notify */
  214. unsigned long eventid; /* overflow event identifier */
  215. } pfm_counter_t;
  216. /*
  217. * context flags
  218. */
  219. typedef struct {
  220. unsigned int block:1; /* when 1, task will blocked on user notifications */
  221. unsigned int system:1; /* do system wide monitoring */
  222. unsigned int using_dbreg:1; /* using range restrictions (debug registers) */
  223. unsigned int is_sampling:1; /* true if using a custom format */
  224. unsigned int excl_idle:1; /* exclude idle task in system wide session */
  225. unsigned int going_zombie:1; /* context is zombie (MASKED+blocking) */
  226. unsigned int trap_reason:2; /* reason for going into pfm_handle_work() */
  227. unsigned int no_msg:1; /* no message sent on overflow */
  228. unsigned int can_restart:1; /* allowed to issue a PFM_RESTART */
  229. unsigned int reserved:22;
  230. } pfm_context_flags_t;
  231. #define PFM_TRAP_REASON_NONE 0x0 /* default value */
  232. #define PFM_TRAP_REASON_BLOCK 0x1 /* we need to block on overflow */
  233. #define PFM_TRAP_REASON_RESET 0x2 /* we need to reset PMDs */
  234. /*
  235. * perfmon context: encapsulates all the state of a monitoring session
  236. */
  237. typedef struct pfm_context {
  238. spinlock_t ctx_lock; /* context protection */
  239. pfm_context_flags_t ctx_flags; /* bitmask of flags (block reason incl.) */
  240. unsigned int ctx_state; /* state: active/inactive (no bitfield) */
  241. struct task_struct *ctx_task; /* task to which context is attached */
  242. unsigned long ctx_ovfl_regs[4]; /* which registers overflowed (notification) */
  243. struct completion ctx_restart_done; /* use for blocking notification mode */
  244. unsigned long ctx_used_pmds[4]; /* bitmask of PMD used */
  245. unsigned long ctx_all_pmds[4]; /* bitmask of all accessible PMDs */
  246. unsigned long ctx_reload_pmds[4]; /* bitmask of force reload PMD on ctxsw in */
  247. unsigned long ctx_all_pmcs[4]; /* bitmask of all accessible PMCs */
  248. unsigned long ctx_reload_pmcs[4]; /* bitmask of force reload PMC on ctxsw in */
  249. unsigned long ctx_used_monitors[4]; /* bitmask of monitor PMC being used */
  250. unsigned long ctx_pmcs[PFM_NUM_PMC_REGS]; /* saved copies of PMC values */
  251. unsigned int ctx_used_ibrs[1]; /* bitmask of used IBR (speedup ctxsw in) */
  252. unsigned int ctx_used_dbrs[1]; /* bitmask of used DBR (speedup ctxsw in) */
  253. unsigned long ctx_dbrs[IA64_NUM_DBG_REGS]; /* DBR values (cache) when not loaded */
  254. unsigned long ctx_ibrs[IA64_NUM_DBG_REGS]; /* IBR values (cache) when not loaded */
  255. pfm_counter_t ctx_pmds[PFM_NUM_PMD_REGS]; /* software state for PMDS */
  256. unsigned long th_pmcs[PFM_NUM_PMC_REGS]; /* PMC thread save state */
  257. unsigned long th_pmds[PFM_NUM_PMD_REGS]; /* PMD thread save state */
  258. u64 ctx_saved_psr_up; /* only contains psr.up value */
  259. unsigned long ctx_last_activation; /* context last activation number for last_cpu */
  260. unsigned int ctx_last_cpu; /* CPU id of current or last CPU used (SMP only) */
  261. unsigned int ctx_cpu; /* cpu to which perfmon is applied (system wide) */
  262. int ctx_fd; /* file descriptor used my this context */
  263. pfm_ovfl_arg_t ctx_ovfl_arg; /* argument to custom buffer format handler */
  264. pfm_buffer_fmt_t *ctx_buf_fmt; /* buffer format callbacks */
  265. void *ctx_smpl_hdr; /* points to sampling buffer header kernel vaddr */
  266. unsigned long ctx_smpl_size; /* size of sampling buffer */
  267. void *ctx_smpl_vaddr; /* user level virtual address of smpl buffer */
  268. wait_queue_head_t ctx_msgq_wait;
  269. pfm_msg_t ctx_msgq[PFM_MAX_MSGS];
  270. int ctx_msgq_head;
  271. int ctx_msgq_tail;
  272. struct fasync_struct *ctx_async_queue;
  273. wait_queue_head_t ctx_zombieq; /* termination cleanup wait queue */
  274. } pfm_context_t;
  275. /*
  276. * magic number used to verify that structure is really
  277. * a perfmon context
  278. */
  279. #define PFM_IS_FILE(f) ((f)->f_op == &pfm_file_ops)
  280. #define PFM_GET_CTX(t) ((pfm_context_t *)(t)->thread.pfm_context)
  281. #ifdef CONFIG_SMP
  282. #define SET_LAST_CPU(ctx, v) (ctx)->ctx_last_cpu = (v)
  283. #define GET_LAST_CPU(ctx) (ctx)->ctx_last_cpu
  284. #else
  285. #define SET_LAST_CPU(ctx, v) do {} while(0)
  286. #define GET_LAST_CPU(ctx) do {} while(0)
  287. #endif
  288. #define ctx_fl_block ctx_flags.block
  289. #define ctx_fl_system ctx_flags.system
  290. #define ctx_fl_using_dbreg ctx_flags.using_dbreg
  291. #define ctx_fl_is_sampling ctx_flags.is_sampling
  292. #define ctx_fl_excl_idle ctx_flags.excl_idle
  293. #define ctx_fl_going_zombie ctx_flags.going_zombie
  294. #define ctx_fl_trap_reason ctx_flags.trap_reason
  295. #define ctx_fl_no_msg ctx_flags.no_msg
  296. #define ctx_fl_can_restart ctx_flags.can_restart
  297. #define PFM_SET_WORK_PENDING(t, v) do { (t)->thread.pfm_needs_checking = v; } while(0);
  298. #define PFM_GET_WORK_PENDING(t) (t)->thread.pfm_needs_checking
  299. /*
  300. * global information about all sessions
  301. * mostly used to synchronize between system wide and per-process
  302. */
  303. typedef struct {
  304. spinlock_t pfs_lock; /* lock the structure */
  305. unsigned int pfs_task_sessions; /* number of per task sessions */
  306. unsigned int pfs_sys_sessions; /* number of per system wide sessions */
  307. unsigned int pfs_sys_use_dbregs; /* incremented when a system wide session uses debug regs */
  308. unsigned int pfs_ptrace_use_dbregs; /* incremented when a process uses debug regs */
  309. struct task_struct *pfs_sys_session[NR_CPUS]; /* point to task owning a system-wide session */
  310. } pfm_session_t;
  311. /*
  312. * information about a PMC or PMD.
  313. * dep_pmd[]: a bitmask of dependent PMD registers
  314. * dep_pmc[]: a bitmask of dependent PMC registers
  315. */
  316. typedef int (*pfm_reg_check_t)(struct task_struct *task, pfm_context_t *ctx, unsigned int cnum, unsigned long *val, struct pt_regs *regs);
  317. typedef struct {
  318. unsigned int type;
  319. int pm_pos;
  320. unsigned long default_value; /* power-on default value */
  321. unsigned long reserved_mask; /* bitmask of reserved bits */
  322. pfm_reg_check_t read_check;
  323. pfm_reg_check_t write_check;
  324. unsigned long dep_pmd[4];
  325. unsigned long dep_pmc[4];
  326. } pfm_reg_desc_t;
  327. /* assume cnum is a valid monitor */
  328. #define PMC_PM(cnum, val) (((val) >> (pmu_conf->pmc_desc[cnum].pm_pos)) & 0x1)
  329. /*
  330. * This structure is initialized at boot time and contains
  331. * a description of the PMU main characteristics.
  332. *
  333. * If the probe function is defined, detection is based
  334. * on its return value:
  335. * - 0 means recognized PMU
  336. * - anything else means not supported
  337. * When the probe function is not defined, then the pmu_family field
  338. * is used and it must match the host CPU family such that:
  339. * - cpu->family & config->pmu_family != 0
  340. */
  341. typedef struct {
  342. unsigned long ovfl_val; /* overflow value for counters */
  343. pfm_reg_desc_t *pmc_desc; /* detailed PMC register dependencies descriptions */
  344. pfm_reg_desc_t *pmd_desc; /* detailed PMD register dependencies descriptions */
  345. unsigned int num_pmcs; /* number of PMCS: computed at init time */
  346. unsigned int num_pmds; /* number of PMDS: computed at init time */
  347. unsigned long impl_pmcs[4]; /* bitmask of implemented PMCS */
  348. unsigned long impl_pmds[4]; /* bitmask of implemented PMDS */
  349. char *pmu_name; /* PMU family name */
  350. unsigned int pmu_family; /* cpuid family pattern used to identify pmu */
  351. unsigned int flags; /* pmu specific flags */
  352. unsigned int num_ibrs; /* number of IBRS: computed at init time */
  353. unsigned int num_dbrs; /* number of DBRS: computed at init time */
  354. unsigned int num_counters; /* PMC/PMD counting pairs : computed at init time */
  355. int (*probe)(void); /* customized probe routine */
  356. unsigned int use_rr_dbregs:1; /* set if debug registers used for range restriction */
  357. } pmu_config_t;
  358. /*
  359. * PMU specific flags
  360. */
  361. #define PFM_PMU_IRQ_RESEND 1 /* PMU needs explicit IRQ resend */
  362. /*
  363. * debug register related type definitions
  364. */
  365. typedef struct {
  366. unsigned long ibr_mask:56;
  367. unsigned long ibr_plm:4;
  368. unsigned long ibr_ig:3;
  369. unsigned long ibr_x:1;
  370. } ibr_mask_reg_t;
  371. typedef struct {
  372. unsigned long dbr_mask:56;
  373. unsigned long dbr_plm:4;
  374. unsigned long dbr_ig:2;
  375. unsigned long dbr_w:1;
  376. unsigned long dbr_r:1;
  377. } dbr_mask_reg_t;
  378. typedef union {
  379. unsigned long val;
  380. ibr_mask_reg_t ibr;
  381. dbr_mask_reg_t dbr;
  382. } dbreg_t;
  383. /*
  384. * perfmon command descriptions
  385. */
  386. typedef struct {
  387. int (*cmd_func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
  388. char *cmd_name;
  389. int cmd_flags;
  390. unsigned int cmd_narg;
  391. size_t cmd_argsize;
  392. int (*cmd_getsize)(void *arg, size_t *sz);
  393. } pfm_cmd_desc_t;
  394. #define PFM_CMD_FD 0x01 /* command requires a file descriptor */
  395. #define PFM_CMD_ARG_READ 0x02 /* command must read argument(s) */
  396. #define PFM_CMD_ARG_RW 0x04 /* command must read/write argument(s) */
  397. #define PFM_CMD_STOP 0x08 /* command does not work on zombie context */
  398. #define PFM_CMD_NAME(cmd) pfm_cmd_tab[(cmd)].cmd_name
  399. #define PFM_CMD_READ_ARG(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_ARG_READ)
  400. #define PFM_CMD_RW_ARG(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_ARG_RW)
  401. #define PFM_CMD_USE_FD(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_FD)
  402. #define PFM_CMD_STOPPED(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_STOP)
  403. #define PFM_CMD_ARG_MANY -1 /* cannot be zero */
  404. typedef struct {
  405. unsigned long pfm_spurious_ovfl_intr_count; /* keep track of spurious ovfl interrupts */
  406. unsigned long pfm_replay_ovfl_intr_count; /* keep track of replayed ovfl interrupts */
  407. unsigned long pfm_ovfl_intr_count; /* keep track of ovfl interrupts */
  408. unsigned long pfm_ovfl_intr_cycles; /* cycles spent processing ovfl interrupts */
  409. unsigned long pfm_ovfl_intr_cycles_min; /* min cycles spent processing ovfl interrupts */
  410. unsigned long pfm_ovfl_intr_cycles_max; /* max cycles spent processing ovfl interrupts */
  411. unsigned long pfm_smpl_handler_calls;
  412. unsigned long pfm_smpl_handler_cycles;
  413. char pad[SMP_CACHE_BYTES] ____cacheline_aligned;
  414. } pfm_stats_t;
  415. /*
  416. * perfmon internal variables
  417. */
  418. static pfm_stats_t pfm_stats[NR_CPUS];
  419. static pfm_session_t pfm_sessions; /* global sessions information */
  420. static DEFINE_SPINLOCK(pfm_alt_install_check);
  421. static pfm_intr_handler_desc_t *pfm_alt_intr_handler;
  422. static struct proc_dir_entry *perfmon_dir;
  423. static pfm_uuid_t pfm_null_uuid = {0,};
  424. static spinlock_t pfm_buffer_fmt_lock;
  425. static LIST_HEAD(pfm_buffer_fmt_list);
  426. static pmu_config_t *pmu_conf;
  427. /* sysctl() controls */
  428. pfm_sysctl_t pfm_sysctl;
  429. EXPORT_SYMBOL(pfm_sysctl);
  430. static ctl_table pfm_ctl_table[]={
  431. {
  432. .ctl_name = CTL_UNNUMBERED,
  433. .procname = "debug",
  434. .data = &pfm_sysctl.debug,
  435. .maxlen = sizeof(int),
  436. .mode = 0666,
  437. .proc_handler = &proc_dointvec,
  438. },
  439. {
  440. .ctl_name = CTL_UNNUMBERED,
  441. .procname = "debug_ovfl",
  442. .data = &pfm_sysctl.debug_ovfl,
  443. .maxlen = sizeof(int),
  444. .mode = 0666,
  445. .proc_handler = &proc_dointvec,
  446. },
  447. {
  448. .ctl_name = CTL_UNNUMBERED,
  449. .procname = "fastctxsw",
  450. .data = &pfm_sysctl.fastctxsw,
  451. .maxlen = sizeof(int),
  452. .mode = 0600,
  453. .proc_handler = &proc_dointvec,
  454. },
  455. {
  456. .ctl_name = CTL_UNNUMBERED,
  457. .procname = "expert_mode",
  458. .data = &pfm_sysctl.expert_mode,
  459. .maxlen = sizeof(int),
  460. .mode = 0600,
  461. .proc_handler = &proc_dointvec,
  462. },
  463. {}
  464. };
  465. static ctl_table pfm_sysctl_dir[] = {
  466. {
  467. .ctl_name = CTL_UNNUMBERED,
  468. .procname = "perfmon",
  469. .mode = 0555,
  470. .child = pfm_ctl_table,
  471. },
  472. {}
  473. };
  474. static ctl_table pfm_sysctl_root[] = {
  475. {
  476. .ctl_name = CTL_KERN,
  477. .procname = "kernel",
  478. .mode = 0555,
  479. .child = pfm_sysctl_dir,
  480. },
  481. {}
  482. };
  483. static struct ctl_table_header *pfm_sysctl_header;
  484. static int pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
  485. #define pfm_get_cpu_var(v) __ia64_per_cpu_var(v)
  486. #define pfm_get_cpu_data(a,b) per_cpu(a, b)
  487. static inline void
  488. pfm_put_task(struct task_struct *task)
  489. {
  490. if (task != current) put_task_struct(task);
  491. }
  492. static inline void
  493. pfm_reserve_page(unsigned long a)
  494. {
  495. SetPageReserved(vmalloc_to_page((void *)a));
  496. }
  497. static inline void
  498. pfm_unreserve_page(unsigned long a)
  499. {
  500. ClearPageReserved(vmalloc_to_page((void*)a));
  501. }
  502. static inline unsigned long
  503. pfm_protect_ctx_ctxsw(pfm_context_t *x)
  504. {
  505. spin_lock(&(x)->ctx_lock);
  506. return 0UL;
  507. }
  508. static inline void
  509. pfm_unprotect_ctx_ctxsw(pfm_context_t *x, unsigned long f)
  510. {
  511. spin_unlock(&(x)->ctx_lock);
  512. }
  513. static inline unsigned int
  514. pfm_do_munmap(struct mm_struct *mm, unsigned long addr, size_t len, int acct)
  515. {
  516. return do_munmap(mm, addr, len);
  517. }
  518. static inline unsigned long
  519. pfm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, unsigned long exec)
  520. {
  521. return get_unmapped_area(file, addr, len, pgoff, flags);
  522. }
  523. static int
  524. pfmfs_get_sb(struct file_system_type *fs_type, int flags, const char *dev_name, void *data,
  525. struct vfsmount *mnt)
  526. {
  527. return get_sb_pseudo(fs_type, "pfm:", NULL, PFMFS_MAGIC, mnt);
  528. }
  529. static struct file_system_type pfm_fs_type = {
  530. .name = "pfmfs",
  531. .get_sb = pfmfs_get_sb,
  532. .kill_sb = kill_anon_super,
  533. };
  534. DEFINE_PER_CPU(unsigned long, pfm_syst_info);
  535. DEFINE_PER_CPU(struct task_struct *, pmu_owner);
  536. DEFINE_PER_CPU(pfm_context_t *, pmu_ctx);
  537. DEFINE_PER_CPU(unsigned long, pmu_activation_number);
  538. EXPORT_PER_CPU_SYMBOL_GPL(pfm_syst_info);
  539. /* forward declaration */
  540. static const struct file_operations pfm_file_ops;
  541. /*
  542. * forward declarations
  543. */
  544. #ifndef CONFIG_SMP
  545. static void pfm_lazy_save_regs (struct task_struct *ta);
  546. #endif
  547. void dump_pmu_state(const char *);
  548. static int pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
  549. #include "perfmon_itanium.h"
  550. #include "perfmon_mckinley.h"
  551. #include "perfmon_montecito.h"
  552. #include "perfmon_generic.h"
  553. static pmu_config_t *pmu_confs[]={
  554. &pmu_conf_mont,
  555. &pmu_conf_mck,
  556. &pmu_conf_ita,
  557. &pmu_conf_gen, /* must be last */
  558. NULL
  559. };
  560. static int pfm_end_notify_user(pfm_context_t *ctx);
  561. static inline void
  562. pfm_clear_psr_pp(void)
  563. {
  564. ia64_rsm(IA64_PSR_PP);
  565. ia64_srlz_i();
  566. }
  567. static inline void
  568. pfm_set_psr_pp(void)
  569. {
  570. ia64_ssm(IA64_PSR_PP);
  571. ia64_srlz_i();
  572. }
  573. static inline void
  574. pfm_clear_psr_up(void)
  575. {
  576. ia64_rsm(IA64_PSR_UP);
  577. ia64_srlz_i();
  578. }
  579. static inline void
  580. pfm_set_psr_up(void)
  581. {
  582. ia64_ssm(IA64_PSR_UP);
  583. ia64_srlz_i();
  584. }
  585. static inline unsigned long
  586. pfm_get_psr(void)
  587. {
  588. unsigned long tmp;
  589. tmp = ia64_getreg(_IA64_REG_PSR);
  590. ia64_srlz_i();
  591. return tmp;
  592. }
  593. static inline void
  594. pfm_set_psr_l(unsigned long val)
  595. {
  596. ia64_setreg(_IA64_REG_PSR_L, val);
  597. ia64_srlz_i();
  598. }
  599. static inline void
  600. pfm_freeze_pmu(void)
  601. {
  602. ia64_set_pmc(0,1UL);
  603. ia64_srlz_d();
  604. }
  605. static inline void
  606. pfm_unfreeze_pmu(void)
  607. {
  608. ia64_set_pmc(0,0UL);
  609. ia64_srlz_d();
  610. }
  611. static inline void
  612. pfm_restore_ibrs(unsigned long *ibrs, unsigned int nibrs)
  613. {
  614. int i;
  615. for (i=0; i < nibrs; i++) {
  616. ia64_set_ibr(i, ibrs[i]);
  617. ia64_dv_serialize_instruction();
  618. }
  619. ia64_srlz_i();
  620. }
  621. static inline void
  622. pfm_restore_dbrs(unsigned long *dbrs, unsigned int ndbrs)
  623. {
  624. int i;
  625. for (i=0; i < ndbrs; i++) {
  626. ia64_set_dbr(i, dbrs[i]);
  627. ia64_dv_serialize_data();
  628. }
  629. ia64_srlz_d();
  630. }
  631. /*
  632. * PMD[i] must be a counter. no check is made
  633. */
  634. static inline unsigned long
  635. pfm_read_soft_counter(pfm_context_t *ctx, int i)
  636. {
  637. return ctx->ctx_pmds[i].val + (ia64_get_pmd(i) & pmu_conf->ovfl_val);
  638. }
  639. /*
  640. * PMD[i] must be a counter. no check is made
  641. */
  642. static inline void
  643. pfm_write_soft_counter(pfm_context_t *ctx, int i, unsigned long val)
  644. {
  645. unsigned long ovfl_val = pmu_conf->ovfl_val;
  646. ctx->ctx_pmds[i].val = val & ~ovfl_val;
  647. /*
  648. * writing to unimplemented part is ignore, so we do not need to
  649. * mask off top part
  650. */
  651. ia64_set_pmd(i, val & ovfl_val);
  652. }
  653. static pfm_msg_t *
  654. pfm_get_new_msg(pfm_context_t *ctx)
  655. {
  656. int idx, next;
  657. next = (ctx->ctx_msgq_tail+1) % PFM_MAX_MSGS;
  658. DPRINT(("ctx_fd=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
  659. if (next == ctx->ctx_msgq_head) return NULL;
  660. idx = ctx->ctx_msgq_tail;
  661. ctx->ctx_msgq_tail = next;
  662. DPRINT(("ctx=%p head=%d tail=%d msg=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, idx));
  663. return ctx->ctx_msgq+idx;
  664. }
  665. static pfm_msg_t *
  666. pfm_get_next_msg(pfm_context_t *ctx)
  667. {
  668. pfm_msg_t *msg;
  669. DPRINT(("ctx=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
  670. if (PFM_CTXQ_EMPTY(ctx)) return NULL;
  671. /*
  672. * get oldest message
  673. */
  674. msg = ctx->ctx_msgq+ctx->ctx_msgq_head;
  675. /*
  676. * and move forward
  677. */
  678. ctx->ctx_msgq_head = (ctx->ctx_msgq_head+1) % PFM_MAX_MSGS;
  679. DPRINT(("ctx=%p head=%d tail=%d type=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, msg->pfm_gen_msg.msg_type));
  680. return msg;
  681. }
  682. static void
  683. pfm_reset_msgq(pfm_context_t *ctx)
  684. {
  685. ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0;
  686. DPRINT(("ctx=%p msgq reset\n", ctx));
  687. }
  688. static void *
  689. pfm_rvmalloc(unsigned long size)
  690. {
  691. void *mem;
  692. unsigned long addr;
  693. size = PAGE_ALIGN(size);
  694. mem = vmalloc(size);
  695. if (mem) {
  696. //printk("perfmon: CPU%d pfm_rvmalloc(%ld)=%p\n", smp_processor_id(), size, mem);
  697. memset(mem, 0, size);
  698. addr = (unsigned long)mem;
  699. while (size > 0) {
  700. pfm_reserve_page(addr);
  701. addr+=PAGE_SIZE;
  702. size-=PAGE_SIZE;
  703. }
  704. }
  705. return mem;
  706. }
  707. static void
  708. pfm_rvfree(void *mem, unsigned long size)
  709. {
  710. unsigned long addr;
  711. if (mem) {
  712. DPRINT(("freeing physical buffer @%p size=%lu\n", mem, size));
  713. addr = (unsigned long) mem;
  714. while ((long) size > 0) {
  715. pfm_unreserve_page(addr);
  716. addr+=PAGE_SIZE;
  717. size-=PAGE_SIZE;
  718. }
  719. vfree(mem);
  720. }
  721. return;
  722. }
  723. static pfm_context_t *
  724. pfm_context_alloc(int ctx_flags)
  725. {
  726. pfm_context_t *ctx;
  727. /*
  728. * allocate context descriptor
  729. * must be able to free with interrupts disabled
  730. */
  731. ctx = kzalloc(sizeof(pfm_context_t), GFP_KERNEL);
  732. if (ctx) {
  733. DPRINT(("alloc ctx @%p\n", ctx));
  734. /*
  735. * init context protection lock
  736. */
  737. spin_lock_init(&ctx->ctx_lock);
  738. /*
  739. * context is unloaded
  740. */
  741. ctx->ctx_state = PFM_CTX_UNLOADED;
  742. /*
  743. * initialization of context's flags
  744. */
  745. ctx->ctx_fl_block = (ctx_flags & PFM_FL_NOTIFY_BLOCK) ? 1 : 0;
  746. ctx->ctx_fl_system = (ctx_flags & PFM_FL_SYSTEM_WIDE) ? 1: 0;
  747. ctx->ctx_fl_no_msg = (ctx_flags & PFM_FL_OVFL_NO_MSG) ? 1: 0;
  748. /*
  749. * will move to set properties
  750. * ctx->ctx_fl_excl_idle = (ctx_flags & PFM_FL_EXCL_IDLE) ? 1: 0;
  751. */
  752. /*
  753. * init restart semaphore to locked
  754. */
  755. init_completion(&ctx->ctx_restart_done);
  756. /*
  757. * activation is used in SMP only
  758. */
  759. ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
  760. SET_LAST_CPU(ctx, -1);
  761. /*
  762. * initialize notification message queue
  763. */
  764. ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0;
  765. init_waitqueue_head(&ctx->ctx_msgq_wait);
  766. init_waitqueue_head(&ctx->ctx_zombieq);
  767. }
  768. return ctx;
  769. }
  770. static void
  771. pfm_context_free(pfm_context_t *ctx)
  772. {
  773. if (ctx) {
  774. DPRINT(("free ctx @%p\n", ctx));
  775. kfree(ctx);
  776. }
  777. }
  778. static void
  779. pfm_mask_monitoring(struct task_struct *task)
  780. {
  781. pfm_context_t *ctx = PFM_GET_CTX(task);
  782. unsigned long mask, val, ovfl_mask;
  783. int i;
  784. DPRINT_ovfl(("masking monitoring for [%d]\n", task_pid_nr(task)));
  785. ovfl_mask = pmu_conf->ovfl_val;
  786. /*
  787. * monitoring can only be masked as a result of a valid
  788. * counter overflow. In UP, it means that the PMU still
  789. * has an owner. Note that the owner can be different
  790. * from the current task. However the PMU state belongs
  791. * to the owner.
  792. * In SMP, a valid overflow only happens when task is
  793. * current. Therefore if we come here, we know that
  794. * the PMU state belongs to the current task, therefore
  795. * we can access the live registers.
  796. *
  797. * So in both cases, the live register contains the owner's
  798. * state. We can ONLY touch the PMU registers and NOT the PSR.
  799. *
  800. * As a consequence to this call, the ctx->th_pmds[] array
  801. * contains stale information which must be ignored
  802. * when context is reloaded AND monitoring is active (see
  803. * pfm_restart).
  804. */
  805. mask = ctx->ctx_used_pmds[0];
  806. for (i = 0; mask; i++, mask>>=1) {
  807. /* skip non used pmds */
  808. if ((mask & 0x1) == 0) continue;
  809. val = ia64_get_pmd(i);
  810. if (PMD_IS_COUNTING(i)) {
  811. /*
  812. * we rebuild the full 64 bit value of the counter
  813. */
  814. ctx->ctx_pmds[i].val += (val & ovfl_mask);
  815. } else {
  816. ctx->ctx_pmds[i].val = val;
  817. }
  818. DPRINT_ovfl(("pmd[%d]=0x%lx hw_pmd=0x%lx\n",
  819. i,
  820. ctx->ctx_pmds[i].val,
  821. val & ovfl_mask));
  822. }
  823. /*
  824. * mask monitoring by setting the privilege level to 0
  825. * we cannot use psr.pp/psr.up for this, it is controlled by
  826. * the user
  827. *
  828. * if task is current, modify actual registers, otherwise modify
  829. * thread save state, i.e., what will be restored in pfm_load_regs()
  830. */
  831. mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER;
  832. for(i= PMU_FIRST_COUNTER; mask; i++, mask>>=1) {
  833. if ((mask & 0x1) == 0UL) continue;
  834. ia64_set_pmc(i, ctx->th_pmcs[i] & ~0xfUL);
  835. ctx->th_pmcs[i] &= ~0xfUL;
  836. DPRINT_ovfl(("pmc[%d]=0x%lx\n", i, ctx->th_pmcs[i]));
  837. }
  838. /*
  839. * make all of this visible
  840. */
  841. ia64_srlz_d();
  842. }
  843. /*
  844. * must always be done with task == current
  845. *
  846. * context must be in MASKED state when calling
  847. */
  848. static void
  849. pfm_restore_monitoring(struct task_struct *task)
  850. {
  851. pfm_context_t *ctx = PFM_GET_CTX(task);
  852. unsigned long mask, ovfl_mask;
  853. unsigned long psr, val;
  854. int i, is_system;
  855. is_system = ctx->ctx_fl_system;
  856. ovfl_mask = pmu_conf->ovfl_val;
  857. if (task != current) {
  858. printk(KERN_ERR "perfmon.%d: invalid task[%d] current[%d]\n", __LINE__, task_pid_nr(task), task_pid_nr(current));
  859. return;
  860. }
  861. if (ctx->ctx_state != PFM_CTX_MASKED) {
  862. printk(KERN_ERR "perfmon.%d: task[%d] current[%d] invalid state=%d\n", __LINE__,
  863. task_pid_nr(task), task_pid_nr(current), ctx->ctx_state);
  864. return;
  865. }
  866. psr = pfm_get_psr();
  867. /*
  868. * monitoring is masked via the PMC.
  869. * As we restore their value, we do not want each counter to
  870. * restart right away. We stop monitoring using the PSR,
  871. * restore the PMC (and PMD) and then re-establish the psr
  872. * as it was. Note that there can be no pending overflow at
  873. * this point, because monitoring was MASKED.
  874. *
  875. * system-wide session are pinned and self-monitoring
  876. */
  877. if (is_system && (PFM_CPUINFO_GET() & PFM_CPUINFO_DCR_PP)) {
  878. /* disable dcr pp */
  879. ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) & ~IA64_DCR_PP);
  880. pfm_clear_psr_pp();
  881. } else {
  882. pfm_clear_psr_up();
  883. }
  884. /*
  885. * first, we restore the PMD
  886. */
  887. mask = ctx->ctx_used_pmds[0];
  888. for (i = 0; mask; i++, mask>>=1) {
  889. /* skip non used pmds */
  890. if ((mask & 0x1) == 0) continue;
  891. if (PMD_IS_COUNTING(i)) {
  892. /*
  893. * we split the 64bit value according to
  894. * counter width
  895. */
  896. val = ctx->ctx_pmds[i].val & ovfl_mask;
  897. ctx->ctx_pmds[i].val &= ~ovfl_mask;
  898. } else {
  899. val = ctx->ctx_pmds[i].val;
  900. }
  901. ia64_set_pmd(i, val);
  902. DPRINT(("pmd[%d]=0x%lx hw_pmd=0x%lx\n",
  903. i,
  904. ctx->ctx_pmds[i].val,
  905. val));
  906. }
  907. /*
  908. * restore the PMCs
  909. */
  910. mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER;
  911. for(i= PMU_FIRST_COUNTER; mask; i++, mask>>=1) {
  912. if ((mask & 0x1) == 0UL) continue;
  913. ctx->th_pmcs[i] = ctx->ctx_pmcs[i];
  914. ia64_set_pmc(i, ctx->th_pmcs[i]);
  915. DPRINT(("[%d] pmc[%d]=0x%lx\n",
  916. task_pid_nr(task), i, ctx->th_pmcs[i]));
  917. }
  918. ia64_srlz_d();
  919. /*
  920. * must restore DBR/IBR because could be modified while masked
  921. * XXX: need to optimize
  922. */
  923. if (ctx->ctx_fl_using_dbreg) {
  924. pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
  925. pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
  926. }
  927. /*
  928. * now restore PSR
  929. */
  930. if (is_system && (PFM_CPUINFO_GET() & PFM_CPUINFO_DCR_PP)) {
  931. /* enable dcr pp */
  932. ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) | IA64_DCR_PP);
  933. ia64_srlz_i();
  934. }
  935. pfm_set_psr_l(psr);
  936. }
  937. static inline void
  938. pfm_save_pmds(unsigned long *pmds, unsigned long mask)
  939. {
  940. int i;
  941. ia64_srlz_d();
  942. for (i=0; mask; i++, mask>>=1) {
  943. if (mask & 0x1) pmds[i] = ia64_get_pmd(i);
  944. }
  945. }
  946. /*
  947. * reload from thread state (used for ctxw only)
  948. */
  949. static inline void
  950. pfm_restore_pmds(unsigned long *pmds, unsigned long mask)
  951. {
  952. int i;
  953. unsigned long val, ovfl_val = pmu_conf->ovfl_val;
  954. for (i=0; mask; i++, mask>>=1) {
  955. if ((mask & 0x1) == 0) continue;
  956. val = PMD_IS_COUNTING(i) ? pmds[i] & ovfl_val : pmds[i];
  957. ia64_set_pmd(i, val);
  958. }
  959. ia64_srlz_d();
  960. }
  961. /*
  962. * propagate PMD from context to thread-state
  963. */
  964. static inline void
  965. pfm_copy_pmds(struct task_struct *task, pfm_context_t *ctx)
  966. {
  967. unsigned long ovfl_val = pmu_conf->ovfl_val;
  968. unsigned long mask = ctx->ctx_all_pmds[0];
  969. unsigned long val;
  970. int i;
  971. DPRINT(("mask=0x%lx\n", mask));
  972. for (i=0; mask; i++, mask>>=1) {
  973. val = ctx->ctx_pmds[i].val;
  974. /*
  975. * We break up the 64 bit value into 2 pieces
  976. * the lower bits go to the machine state in the
  977. * thread (will be reloaded on ctxsw in).
  978. * The upper part stays in the soft-counter.
  979. */
  980. if (PMD_IS_COUNTING(i)) {
  981. ctx->ctx_pmds[i].val = val & ~ovfl_val;
  982. val &= ovfl_val;
  983. }
  984. ctx->th_pmds[i] = val;
  985. DPRINT(("pmd[%d]=0x%lx soft_val=0x%lx\n",
  986. i,
  987. ctx->th_pmds[i],
  988. ctx->ctx_pmds[i].val));
  989. }
  990. }
  991. /*
  992. * propagate PMC from context to thread-state
  993. */
  994. static inline void
  995. pfm_copy_pmcs(struct task_struct *task, pfm_context_t *ctx)
  996. {
  997. unsigned long mask = ctx->ctx_all_pmcs[0];
  998. int i;
  999. DPRINT(("mask=0x%lx\n", mask));
  1000. for (i=0; mask; i++, mask>>=1) {
  1001. /* masking 0 with ovfl_val yields 0 */
  1002. ctx->th_pmcs[i] = ctx->ctx_pmcs[i];
  1003. DPRINT(("pmc[%d]=0x%lx\n", i, ctx->th_pmcs[i]));
  1004. }
  1005. }
  1006. static inline void
  1007. pfm_restore_pmcs(unsigned long *pmcs, unsigned long mask)
  1008. {
  1009. int i;
  1010. for (i=0; mask; i++, mask>>=1) {
  1011. if ((mask & 0x1) == 0) continue;
  1012. ia64_set_pmc(i, pmcs[i]);
  1013. }
  1014. ia64_srlz_d();
  1015. }
  1016. static inline int
  1017. pfm_uuid_cmp(pfm_uuid_t a, pfm_uuid_t b)
  1018. {
  1019. return memcmp(a, b, sizeof(pfm_uuid_t));
  1020. }
  1021. static inline int
  1022. pfm_buf_fmt_exit(pfm_buffer_fmt_t *fmt, struct task_struct *task, void *buf, struct pt_regs *regs)
  1023. {
  1024. int ret = 0;
  1025. if (fmt->fmt_exit) ret = (*fmt->fmt_exit)(task, buf, regs);
  1026. return ret;
  1027. }
  1028. static inline int
  1029. pfm_buf_fmt_getsize(pfm_buffer_fmt_t *fmt, struct task_struct *task, unsigned int flags, int cpu, void *arg, unsigned long *size)
  1030. {
  1031. int ret = 0;
  1032. if (fmt->fmt_getsize) ret = (*fmt->fmt_getsize)(task, flags, cpu, arg, size);
  1033. return ret;
  1034. }
  1035. static inline int
  1036. pfm_buf_fmt_validate(pfm_buffer_fmt_t *fmt, struct task_struct *task, unsigned int flags,
  1037. int cpu, void *arg)
  1038. {
  1039. int ret = 0;
  1040. if (fmt->fmt_validate) ret = (*fmt->fmt_validate)(task, flags, cpu, arg);
  1041. return ret;
  1042. }
  1043. static inline int
  1044. pfm_buf_fmt_init(pfm_buffer_fmt_t *fmt, struct task_struct *task, void *buf, unsigned int flags,
  1045. int cpu, void *arg)
  1046. {
  1047. int ret = 0;
  1048. if (fmt->fmt_init) ret = (*fmt->fmt_init)(task, buf, flags, cpu, arg);
  1049. return ret;
  1050. }
  1051. static inline int
  1052. pfm_buf_fmt_restart(pfm_buffer_fmt_t *fmt, struct task_struct *task, pfm_ovfl_ctrl_t *ctrl, void *buf, struct pt_regs *regs)
  1053. {
  1054. int ret = 0;
  1055. if (fmt->fmt_restart) ret = (*fmt->fmt_restart)(task, ctrl, buf, regs);
  1056. return ret;
  1057. }
  1058. static inline int
  1059. pfm_buf_fmt_restart_active(pfm_buffer_fmt_t *fmt, struct task_struct *task, pfm_ovfl_ctrl_t *ctrl, void *buf, struct pt_regs *regs)
  1060. {
  1061. int ret = 0;
  1062. if (fmt->fmt_restart_active) ret = (*fmt->fmt_restart_active)(task, ctrl, buf, regs);
  1063. return ret;
  1064. }
  1065. static pfm_buffer_fmt_t *
  1066. __pfm_find_buffer_fmt(pfm_uuid_t uuid)
  1067. {
  1068. struct list_head * pos;
  1069. pfm_buffer_fmt_t * entry;
  1070. list_for_each(pos, &pfm_buffer_fmt_list) {
  1071. entry = list_entry(pos, pfm_buffer_fmt_t, fmt_list);
  1072. if (pfm_uuid_cmp(uuid, entry->fmt_uuid) == 0)
  1073. return entry;
  1074. }
  1075. return NULL;
  1076. }
  1077. /*
  1078. * find a buffer format based on its uuid
  1079. */
  1080. static pfm_buffer_fmt_t *
  1081. pfm_find_buffer_fmt(pfm_uuid_t uuid)
  1082. {
  1083. pfm_buffer_fmt_t * fmt;
  1084. spin_lock(&pfm_buffer_fmt_lock);
  1085. fmt = __pfm_find_buffer_fmt(uuid);
  1086. spin_unlock(&pfm_buffer_fmt_lock);
  1087. return fmt;
  1088. }
  1089. int
  1090. pfm_register_buffer_fmt(pfm_buffer_fmt_t *fmt)
  1091. {
  1092. int ret = 0;
  1093. /* some sanity checks */
  1094. if (fmt == NULL || fmt->fmt_name == NULL) return -EINVAL;
  1095. /* we need at least a handler */
  1096. if (fmt->fmt_handler == NULL) return -EINVAL;
  1097. /*
  1098. * XXX: need check validity of fmt_arg_size
  1099. */
  1100. spin_lock(&pfm_buffer_fmt_lock);
  1101. if (__pfm_find_buffer_fmt(fmt->fmt_uuid)) {
  1102. printk(KERN_ERR "perfmon: duplicate sampling format: %s\n", fmt->fmt_name);
  1103. ret = -EBUSY;
  1104. goto out;
  1105. }
  1106. list_add(&fmt->fmt_list, &pfm_buffer_fmt_list);
  1107. printk(KERN_INFO "perfmon: added sampling format %s\n", fmt->fmt_name);
  1108. out:
  1109. spin_unlock(&pfm_buffer_fmt_lock);
  1110. return ret;
  1111. }
  1112. EXPORT_SYMBOL(pfm_register_buffer_fmt);
  1113. int
  1114. pfm_unregister_buffer_fmt(pfm_uuid_t uuid)
  1115. {
  1116. pfm_buffer_fmt_t *fmt;
  1117. int ret = 0;
  1118. spin_lock(&pfm_buffer_fmt_lock);
  1119. fmt = __pfm_find_buffer_fmt(uuid);
  1120. if (!fmt) {
  1121. printk(KERN_ERR "perfmon: cannot unregister format, not found\n");
  1122. ret = -EINVAL;
  1123. goto out;
  1124. }
  1125. list_del_init(&fmt->fmt_list);
  1126. printk(KERN_INFO "perfmon: removed sampling format: %s\n", fmt->fmt_name);
  1127. out:
  1128. spin_unlock(&pfm_buffer_fmt_lock);
  1129. return ret;
  1130. }
  1131. EXPORT_SYMBOL(pfm_unregister_buffer_fmt);
  1132. extern void update_pal_halt_status(int);
  1133. static int
  1134. pfm_reserve_session(struct task_struct *task, int is_syswide, unsigned int cpu)
  1135. {
  1136. unsigned long flags;
  1137. /*
  1138. * validity checks on cpu_mask have been done upstream
  1139. */
  1140. LOCK_PFS(flags);
  1141. DPRINT(("in sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
  1142. pfm_sessions.pfs_sys_sessions,
  1143. pfm_sessions.pfs_task_sessions,
  1144. pfm_sessions.pfs_sys_use_dbregs,
  1145. is_syswide,
  1146. cpu));
  1147. if (is_syswide) {
  1148. /*
  1149. * cannot mix system wide and per-task sessions
  1150. */
  1151. if (pfm_sessions.pfs_task_sessions > 0UL) {
  1152. DPRINT(("system wide not possible, %u conflicting task_sessions\n",
  1153. pfm_sessions.pfs_task_sessions));
  1154. goto abort;
  1155. }
  1156. if (pfm_sessions.pfs_sys_session[cpu]) goto error_conflict;
  1157. DPRINT(("reserving system wide session on CPU%u currently on CPU%u\n", cpu, smp_processor_id()));
  1158. pfm_sessions.pfs_sys_session[cpu] = task;
  1159. pfm_sessions.pfs_sys_sessions++ ;
  1160. } else {
  1161. if (pfm_sessions.pfs_sys_sessions) goto abort;
  1162. pfm_sessions.pfs_task_sessions++;
  1163. }
  1164. DPRINT(("out sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
  1165. pfm_sessions.pfs_sys_sessions,
  1166. pfm_sessions.pfs_task_sessions,
  1167. pfm_sessions.pfs_sys_use_dbregs,
  1168. is_syswide,
  1169. cpu));
  1170. /*
  1171. * disable default_idle() to go to PAL_HALT
  1172. */
  1173. update_pal_halt_status(0);
  1174. UNLOCK_PFS(flags);
  1175. return 0;
  1176. error_conflict:
  1177. DPRINT(("system wide not possible, conflicting session [%d] on CPU%d\n",
  1178. task_pid_nr(pfm_sessions.pfs_sys_session[cpu]),
  1179. cpu));
  1180. abort:
  1181. UNLOCK_PFS(flags);
  1182. return -EBUSY;
  1183. }
  1184. static int
  1185. pfm_unreserve_session(pfm_context_t *ctx, int is_syswide, unsigned int cpu)
  1186. {
  1187. unsigned long flags;
  1188. /*
  1189. * validity checks on cpu_mask have been done upstream
  1190. */
  1191. LOCK_PFS(flags);
  1192. DPRINT(("in sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
  1193. pfm_sessions.pfs_sys_sessions,
  1194. pfm_sessions.pfs_task_sessions,
  1195. pfm_sessions.pfs_sys_use_dbregs,
  1196. is_syswide,
  1197. cpu));
  1198. if (is_syswide) {
  1199. pfm_sessions.pfs_sys_session[cpu] = NULL;
  1200. /*
  1201. * would not work with perfmon+more than one bit in cpu_mask
  1202. */
  1203. if (ctx && ctx->ctx_fl_using_dbreg) {
  1204. if (pfm_sessions.pfs_sys_use_dbregs == 0) {
  1205. printk(KERN_ERR "perfmon: invalid release for ctx %p sys_use_dbregs=0\n", ctx);
  1206. } else {
  1207. pfm_sessions.pfs_sys_use_dbregs--;
  1208. }
  1209. }
  1210. pfm_sessions.pfs_sys_sessions--;
  1211. } else {
  1212. pfm_sessions.pfs_task_sessions--;
  1213. }
  1214. DPRINT(("out sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
  1215. pfm_sessions.pfs_sys_sessions,
  1216. pfm_sessions.pfs_task_sessions,
  1217. pfm_sessions.pfs_sys_use_dbregs,
  1218. is_syswide,
  1219. cpu));
  1220. /*
  1221. * if possible, enable default_idle() to go into PAL_HALT
  1222. */
  1223. if (pfm_sessions.pfs_task_sessions == 0 && pfm_sessions.pfs_sys_sessions == 0)
  1224. update_pal_halt_status(1);
  1225. UNLOCK_PFS(flags);
  1226. return 0;
  1227. }
  1228. /*
  1229. * removes virtual mapping of the sampling buffer.
  1230. * IMPORTANT: cannot be called with interrupts disable, e.g. inside
  1231. * a PROTECT_CTX() section.
  1232. */
  1233. static int
  1234. pfm_remove_smpl_mapping(struct task_struct *task, void *vaddr, unsigned long size)
  1235. {
  1236. int r;
  1237. /* sanity checks */
  1238. if (task->mm == NULL || size == 0UL || vaddr == NULL) {
  1239. printk(KERN_ERR "perfmon: pfm_remove_smpl_mapping [%d] invalid context mm=%p\n", task_pid_nr(task), task->mm);
  1240. return -EINVAL;
  1241. }
  1242. DPRINT(("smpl_vaddr=%p size=%lu\n", vaddr, size));
  1243. /*
  1244. * does the actual unmapping
  1245. */
  1246. down_write(&task->mm->mmap_sem);
  1247. DPRINT(("down_write done smpl_vaddr=%p size=%lu\n", vaddr, size));
  1248. r = pfm_do_munmap(task->mm, (unsigned long)vaddr, size, 0);
  1249. up_write(&task->mm->mmap_sem);
  1250. if (r !=0) {
  1251. printk(KERN_ERR "perfmon: [%d] unable to unmap sampling buffer @%p size=%lu\n", task_pid_nr(task), vaddr, size);
  1252. }
  1253. DPRINT(("do_unmap(%p, %lu)=%d\n", vaddr, size, r));
  1254. return 0;
  1255. }
  1256. /*
  1257. * free actual physical storage used by sampling buffer
  1258. */
  1259. #if 0
  1260. static int
  1261. pfm_free_smpl_buffer(pfm_context_t *ctx)
  1262. {
  1263. pfm_buffer_fmt_t *fmt;
  1264. if (ctx->ctx_smpl_hdr == NULL) goto invalid_free;
  1265. /*
  1266. * we won't use the buffer format anymore
  1267. */
  1268. fmt = ctx->ctx_buf_fmt;
  1269. DPRINT(("sampling buffer @%p size %lu vaddr=%p\n",
  1270. ctx->ctx_smpl_hdr,
  1271. ctx->ctx_smpl_size,
  1272. ctx->ctx_smpl_vaddr));
  1273. pfm_buf_fmt_exit(fmt, current, NULL, NULL);
  1274. /*
  1275. * free the buffer
  1276. */
  1277. pfm_rvfree(ctx->ctx_smpl_hdr, ctx->ctx_smpl_size);
  1278. ctx->ctx_smpl_hdr = NULL;
  1279. ctx->ctx_smpl_size = 0UL;
  1280. return 0;
  1281. invalid_free:
  1282. printk(KERN_ERR "perfmon: pfm_free_smpl_buffer [%d] no buffer\n", task_pid_nr(current));
  1283. return -EINVAL;
  1284. }
  1285. #endif
  1286. static inline void
  1287. pfm_exit_smpl_buffer(pfm_buffer_fmt_t *fmt)
  1288. {
  1289. if (fmt == NULL) return;
  1290. pfm_buf_fmt_exit(fmt, current, NULL, NULL);
  1291. }
  1292. /*
  1293. * pfmfs should _never_ be mounted by userland - too much of security hassle,
  1294. * no real gain from having the whole whorehouse mounted. So we don't need
  1295. * any operations on the root directory. However, we need a non-trivial
  1296. * d_name - pfm: will go nicely and kill the special-casing in procfs.
  1297. */
  1298. static struct vfsmount *pfmfs_mnt;
  1299. static int __init
  1300. init_pfm_fs(void)
  1301. {
  1302. int err = register_filesystem(&pfm_fs_type);
  1303. if (!err) {
  1304. pfmfs_mnt = kern_mount(&pfm_fs_type);
  1305. err = PTR_ERR(pfmfs_mnt);
  1306. if (IS_ERR(pfmfs_mnt))
  1307. unregister_filesystem(&pfm_fs_type);
  1308. else
  1309. err = 0;
  1310. }
  1311. return err;
  1312. }
  1313. static ssize_t
  1314. pfm_read(struct file *filp, char __user *buf, size_t size, loff_t *ppos)
  1315. {
  1316. pfm_context_t *ctx;
  1317. pfm_msg_t *msg;
  1318. ssize_t ret;
  1319. unsigned long flags;
  1320. DECLARE_WAITQUEUE(wait, current);
  1321. if (PFM_IS_FILE(filp) == 0) {
  1322. printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", task_pid_nr(current));
  1323. return -EINVAL;
  1324. }
  1325. ctx = (pfm_context_t *)filp->private_data;
  1326. if (ctx == NULL) {
  1327. printk(KERN_ERR "perfmon: pfm_read: NULL ctx [%d]\n", task_pid_nr(current));
  1328. return -EINVAL;
  1329. }
  1330. /*
  1331. * check even when there is no message
  1332. */
  1333. if (size < sizeof(pfm_msg_t)) {
  1334. DPRINT(("message is too small ctx=%p (>=%ld)\n", ctx, sizeof(pfm_msg_t)));
  1335. return -EINVAL;
  1336. }
  1337. PROTECT_CTX(ctx, flags);
  1338. /*
  1339. * put ourselves on the wait queue
  1340. */
  1341. add_wait_queue(&ctx->ctx_msgq_wait, &wait);
  1342. for(;;) {
  1343. /*
  1344. * check wait queue
  1345. */
  1346. set_current_state(TASK_INTERRUPTIBLE);
  1347. DPRINT(("head=%d tail=%d\n", ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
  1348. ret = 0;
  1349. if(PFM_CTXQ_EMPTY(ctx) == 0) break;
  1350. UNPROTECT_CTX(ctx, flags);
  1351. /*
  1352. * check non-blocking read
  1353. */
  1354. ret = -EAGAIN;
  1355. if(filp->f_flags & O_NONBLOCK) break;
  1356. /*
  1357. * check pending signals
  1358. */
  1359. if(signal_pending(current)) {
  1360. ret = -EINTR;
  1361. break;
  1362. }
  1363. /*
  1364. * no message, so wait
  1365. */
  1366. schedule();
  1367. PROTECT_CTX(ctx, flags);
  1368. }
  1369. DPRINT(("[%d] back to running ret=%ld\n", task_pid_nr(current), ret));
  1370. set_current_state(TASK_RUNNING);
  1371. remove_wait_queue(&ctx->ctx_msgq_wait, &wait);
  1372. if (ret < 0) goto abort;
  1373. ret = -EINVAL;
  1374. msg = pfm_get_next_msg(ctx);
  1375. if (msg == NULL) {
  1376. printk(KERN_ERR "perfmon: pfm_read no msg for ctx=%p [%d]\n", ctx, task_pid_nr(current));
  1377. goto abort_locked;
  1378. }
  1379. DPRINT(("fd=%d type=%d\n", msg->pfm_gen_msg.msg_ctx_fd, msg->pfm_gen_msg.msg_type));
  1380. ret = -EFAULT;
  1381. if(copy_to_user(buf, msg, sizeof(pfm_msg_t)) == 0) ret = sizeof(pfm_msg_t);
  1382. abort_locked:
  1383. UNPROTECT_CTX(ctx, flags);
  1384. abort:
  1385. return ret;
  1386. }
  1387. static ssize_t
  1388. pfm_write(struct file *file, const char __user *ubuf,
  1389. size_t size, loff_t *ppos)
  1390. {
  1391. DPRINT(("pfm_write called\n"));
  1392. return -EINVAL;
  1393. }
  1394. static unsigned int
  1395. pfm_poll(struct file *filp, poll_table * wait)
  1396. {
  1397. pfm_context_t *ctx;
  1398. unsigned long flags;
  1399. unsigned int mask = 0;
  1400. if (PFM_IS_FILE(filp) == 0) {
  1401. printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", task_pid_nr(current));
  1402. return 0;
  1403. }
  1404. ctx = (pfm_context_t *)filp->private_data;
  1405. if (ctx == NULL) {
  1406. printk(KERN_ERR "perfmon: pfm_poll: NULL ctx [%d]\n", task_pid_nr(current));
  1407. return 0;
  1408. }
  1409. DPRINT(("pfm_poll ctx_fd=%d before poll_wait\n", ctx->ctx_fd));
  1410. poll_wait(filp, &ctx->ctx_msgq_wait, wait);
  1411. PROTECT_CTX(ctx, flags);
  1412. if (PFM_CTXQ_EMPTY(ctx) == 0)
  1413. mask = POLLIN | POLLRDNORM;
  1414. UNPROTECT_CTX(ctx, flags);
  1415. DPRINT(("pfm_poll ctx_fd=%d mask=0x%x\n", ctx->ctx_fd, mask));
  1416. return mask;
  1417. }
  1418. static int
  1419. pfm_ioctl(struct inode *inode, struct file *file, unsigned int cmd, unsigned long arg)
  1420. {
  1421. DPRINT(("pfm_ioctl called\n"));
  1422. return -EINVAL;
  1423. }
  1424. /*
  1425. * interrupt cannot be masked when coming here
  1426. */
  1427. static inline int
  1428. pfm_do_fasync(int fd, struct file *filp, pfm_context_t *ctx, int on)
  1429. {
  1430. int ret;
  1431. ret = fasync_helper (fd, filp, on, &ctx->ctx_async_queue);
  1432. DPRINT(("pfm_fasync called by [%d] on ctx_fd=%d on=%d async_queue=%p ret=%d\n",
  1433. task_pid_nr(current),
  1434. fd,
  1435. on,
  1436. ctx->ctx_async_queue, ret));
  1437. return ret;
  1438. }
  1439. static int
  1440. pfm_fasync(int fd, struct file *filp, int on)
  1441. {
  1442. pfm_context_t *ctx;
  1443. int ret;
  1444. if (PFM_IS_FILE(filp) == 0) {
  1445. printk(KERN_ERR "perfmon: pfm_fasync bad magic [%d]\n", task_pid_nr(current));
  1446. return -EBADF;
  1447. }
  1448. ctx = (pfm_context_t *)filp->private_data;
  1449. if (ctx == NULL) {
  1450. printk(KERN_ERR "perfmon: pfm_fasync NULL ctx [%d]\n", task_pid_nr(current));
  1451. return -EBADF;
  1452. }
  1453. /*
  1454. * we cannot mask interrupts during this call because this may
  1455. * may go to sleep if memory is not readily avalaible.
  1456. *
  1457. * We are protected from the conetxt disappearing by the get_fd()/put_fd()
  1458. * done in caller. Serialization of this function is ensured by caller.
  1459. */
  1460. ret = pfm_do_fasync(fd, filp, ctx, on);
  1461. DPRINT(("pfm_fasync called on ctx_fd=%d on=%d async_queue=%p ret=%d\n",
  1462. fd,
  1463. on,
  1464. ctx->ctx_async_queue, ret));
  1465. return ret;
  1466. }
  1467. #ifdef CONFIG_SMP
  1468. /*
  1469. * this function is exclusively called from pfm_close().
  1470. * The context is not protected at that time, nor are interrupts
  1471. * on the remote CPU. That's necessary to avoid deadlocks.
  1472. */
  1473. static void
  1474. pfm_syswide_force_stop(void *info)
  1475. {
  1476. pfm_context_t *ctx = (pfm_context_t *)info;
  1477. struct pt_regs *regs = task_pt_regs(current);
  1478. struct task_struct *owner;
  1479. unsigned long flags;
  1480. int ret;
  1481. if (ctx->ctx_cpu != smp_processor_id()) {
  1482. printk(KERN_ERR "perfmon: pfm_syswide_force_stop for CPU%d but on CPU%d\n",
  1483. ctx->ctx_cpu,
  1484. smp_processor_id());
  1485. return;
  1486. }
  1487. owner = GET_PMU_OWNER();
  1488. if (owner != ctx->ctx_task) {
  1489. printk(KERN_ERR "perfmon: pfm_syswide_force_stop CPU%d unexpected owner [%d] instead of [%d]\n",
  1490. smp_processor_id(),
  1491. task_pid_nr(owner), task_pid_nr(ctx->ctx_task));
  1492. return;
  1493. }
  1494. if (GET_PMU_CTX() != ctx) {
  1495. printk(KERN_ERR "perfmon: pfm_syswide_force_stop CPU%d unexpected ctx %p instead of %p\n",
  1496. smp_processor_id(),
  1497. GET_PMU_CTX(), ctx);
  1498. return;
  1499. }
  1500. DPRINT(("on CPU%d forcing system wide stop for [%d]\n", smp_processor_id(), task_pid_nr(ctx->ctx_task)));
  1501. /*
  1502. * the context is already protected in pfm_close(), we simply
  1503. * need to mask interrupts to avoid a PMU interrupt race on
  1504. * this CPU
  1505. */
  1506. local_irq_save(flags);
  1507. ret = pfm_context_unload(ctx, NULL, 0, regs);
  1508. if (ret) {
  1509. DPRINT(("context_unload returned %d\n", ret));
  1510. }
  1511. /*
  1512. * unmask interrupts, PMU interrupts are now spurious here
  1513. */
  1514. local_irq_restore(flags);
  1515. }
  1516. static void
  1517. pfm_syswide_cleanup_other_cpu(pfm_context_t *ctx)
  1518. {
  1519. int ret;
  1520. DPRINT(("calling CPU%d for cleanup\n", ctx->ctx_cpu));
  1521. ret = smp_call_function_single(ctx->ctx_cpu, pfm_syswide_force_stop, ctx, 0, 1);
  1522. DPRINT(("called CPU%d for cleanup ret=%d\n", ctx->ctx_cpu, ret));
  1523. }
  1524. #endif /* CONFIG_SMP */
  1525. /*
  1526. * called for each close(). Partially free resources.
  1527. * When caller is self-monitoring, the context is unloaded.
  1528. */
  1529. static int
  1530. pfm_flush(struct file *filp, fl_owner_t id)
  1531. {
  1532. pfm_context_t *ctx;
  1533. struct task_struct *task;
  1534. struct pt_regs *regs;
  1535. unsigned long flags;
  1536. unsigned long smpl_buf_size = 0UL;
  1537. void *smpl_buf_vaddr = NULL;
  1538. int state, is_system;
  1539. if (PFM_IS_FILE(filp) == 0) {
  1540. DPRINT(("bad magic for\n"));
  1541. return -EBADF;
  1542. }
  1543. ctx = (pfm_context_t *)filp->private_data;
  1544. if (ctx == NULL) {
  1545. printk(KERN_ERR "perfmon: pfm_flush: NULL ctx [%d]\n", task_pid_nr(current));
  1546. return -EBADF;
  1547. }
  1548. /*
  1549. * remove our file from the async queue, if we use this mode.
  1550. * This can be done without the context being protected. We come
  1551. * here when the context has become unreachable by other tasks.
  1552. *
  1553. * We may still have active monitoring at this point and we may
  1554. * end up in pfm_overflow_handler(). However, fasync_helper()
  1555. * operates with interrupts disabled and it cleans up the
  1556. * queue. If the PMU handler is called prior to entering
  1557. * fasync_helper() then it will send a signal. If it is
  1558. * invoked after, it will find an empty queue and no
  1559. * signal will be sent. In both case, we are safe
  1560. */
  1561. if (filp->f_flags & FASYNC) {
  1562. DPRINT(("cleaning up async_queue=%p\n", ctx->ctx_async_queue));
  1563. pfm_do_fasync (-1, filp, ctx, 0);
  1564. }
  1565. PROTECT_CTX(ctx, flags);
  1566. state = ctx->ctx_state;
  1567. is_system = ctx->ctx_fl_system;
  1568. task = PFM_CTX_TASK(ctx);
  1569. regs = task_pt_regs(task);
  1570. DPRINT(("ctx_state=%d is_current=%d\n",
  1571. state,
  1572. task == current ? 1 : 0));
  1573. /*
  1574. * if state == UNLOADED, then task is NULL
  1575. */
  1576. /*
  1577. * we must stop and unload because we are losing access to the context.
  1578. */
  1579. if (task == current) {
  1580. #ifdef CONFIG_SMP
  1581. /*
  1582. * the task IS the owner but it migrated to another CPU: that's bad
  1583. * but we must handle this cleanly. Unfortunately, the kernel does
  1584. * not provide a mechanism to block migration (while the context is loaded).
  1585. *
  1586. * We need to release the resource on the ORIGINAL cpu.
  1587. */
  1588. if (is_system && ctx->ctx_cpu != smp_processor_id()) {
  1589. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  1590. /*
  1591. * keep context protected but unmask interrupt for IPI
  1592. */
  1593. local_irq_restore(flags);
  1594. pfm_syswide_cleanup_other_cpu(ctx);
  1595. /*
  1596. * restore interrupt masking
  1597. */
  1598. local_irq_save(flags);
  1599. /*
  1600. * context is unloaded at this point
  1601. */
  1602. } else
  1603. #endif /* CONFIG_SMP */
  1604. {
  1605. DPRINT(("forcing unload\n"));
  1606. /*
  1607. * stop and unload, returning with state UNLOADED
  1608. * and session unreserved.
  1609. */
  1610. pfm_context_unload(ctx, NULL, 0, regs);
  1611. DPRINT(("ctx_state=%d\n", ctx->ctx_state));
  1612. }
  1613. }
  1614. /*
  1615. * remove virtual mapping, if any, for the calling task.
  1616. * cannot reset ctx field until last user is calling close().
  1617. *
  1618. * ctx_smpl_vaddr must never be cleared because it is needed
  1619. * by every task with access to the context
  1620. *
  1621. * When called from do_exit(), the mm context is gone already, therefore
  1622. * mm is NULL, i.e., the VMA is already gone and we do not have to
  1623. * do anything here
  1624. */
  1625. if (ctx->ctx_smpl_vaddr && current->mm) {
  1626. smpl_buf_vaddr = ctx->ctx_smpl_vaddr;
  1627. smpl_buf_size = ctx->ctx_smpl_size;
  1628. }
  1629. UNPROTECT_CTX(ctx, flags);
  1630. /*
  1631. * if there was a mapping, then we systematically remove it
  1632. * at this point. Cannot be done inside critical section
  1633. * because some VM function reenables interrupts.
  1634. *
  1635. */
  1636. if (smpl_buf_vaddr) pfm_remove_smpl_mapping(current, smpl_buf_vaddr, smpl_buf_size);
  1637. return 0;
  1638. }
  1639. /*
  1640. * called either on explicit close() or from exit_files().
  1641. * Only the LAST user of the file gets to this point, i.e., it is
  1642. * called only ONCE.
  1643. *
  1644. * IMPORTANT: we get called ONLY when the refcnt on the file gets to zero
  1645. * (fput()),i.e, last task to access the file. Nobody else can access the
  1646. * file at this point.
  1647. *
  1648. * When called from exit_files(), the VMA has been freed because exit_mm()
  1649. * is executed before exit_files().
  1650. *
  1651. * When called from exit_files(), the current task is not yet ZOMBIE but we
  1652. * flush the PMU state to the context.
  1653. */
  1654. static int
  1655. pfm_close(struct inode *inode, struct file *filp)
  1656. {
  1657. pfm_context_t *ctx;
  1658. struct task_struct *task;
  1659. struct pt_regs *regs;
  1660. DECLARE_WAITQUEUE(wait, current);
  1661. unsigned long flags;
  1662. unsigned long smpl_buf_size = 0UL;
  1663. void *smpl_buf_addr = NULL;
  1664. int free_possible = 1;
  1665. int state, is_system;
  1666. DPRINT(("pfm_close called private=%p\n", filp->private_data));
  1667. if (PFM_IS_FILE(filp) == 0) {
  1668. DPRINT(("bad magic\n"));
  1669. return -EBADF;
  1670. }
  1671. ctx = (pfm_context_t *)filp->private_data;
  1672. if (ctx == NULL) {
  1673. printk(KERN_ERR "perfmon: pfm_close: NULL ctx [%d]\n", task_pid_nr(current));
  1674. return -EBADF;
  1675. }
  1676. PROTECT_CTX(ctx, flags);
  1677. state = ctx->ctx_state;
  1678. is_system = ctx->ctx_fl_system;
  1679. task = PFM_CTX_TASK(ctx);
  1680. regs = task_pt_regs(task);
  1681. DPRINT(("ctx_state=%d is_current=%d\n",
  1682. state,
  1683. task == current ? 1 : 0));
  1684. /*
  1685. * if task == current, then pfm_flush() unloaded the context
  1686. */
  1687. if (state == PFM_CTX_UNLOADED) goto doit;
  1688. /*
  1689. * context is loaded/masked and task != current, we need to
  1690. * either force an unload or go zombie
  1691. */
  1692. /*
  1693. * The task is currently blocked or will block after an overflow.
  1694. * we must force it to wakeup to get out of the
  1695. * MASKED state and transition to the unloaded state by itself.
  1696. *
  1697. * This situation is only possible for per-task mode
  1698. */
  1699. if (state == PFM_CTX_MASKED && CTX_OVFL_NOBLOCK(ctx) == 0) {
  1700. /*
  1701. * set a "partial" zombie state to be checked
  1702. * upon return from down() in pfm_handle_work().
  1703. *
  1704. * We cannot use the ZOMBIE state, because it is checked
  1705. * by pfm_load_regs() which is called upon wakeup from down().
  1706. * In such case, it would free the context and then we would
  1707. * return to pfm_handle_work() which would access the
  1708. * stale context. Instead, we set a flag invisible to pfm_load_regs()
  1709. * but visible to pfm_handle_work().
  1710. *
  1711. * For some window of time, we have a zombie context with
  1712. * ctx_state = MASKED and not ZOMBIE
  1713. */
  1714. ctx->ctx_fl_going_zombie = 1;
  1715. /*
  1716. * force task to wake up from MASKED state
  1717. */
  1718. complete(&ctx->ctx_restart_done);
  1719. DPRINT(("waking up ctx_state=%d\n", state));
  1720. /*
  1721. * put ourself to sleep waiting for the other
  1722. * task to report completion
  1723. *
  1724. * the context is protected by mutex, therefore there
  1725. * is no risk of being notified of completion before
  1726. * begin actually on the waitq.
  1727. */
  1728. set_current_state(TASK_INTERRUPTIBLE);
  1729. add_wait_queue(&ctx->ctx_zombieq, &wait);
  1730. UNPROTECT_CTX(ctx, flags);
  1731. /*
  1732. * XXX: check for signals :
  1733. * - ok for explicit close
  1734. * - not ok when coming from exit_files()
  1735. */
  1736. schedule();
  1737. PROTECT_CTX(ctx, flags);
  1738. remove_wait_queue(&ctx->ctx_zombieq, &wait);
  1739. set_current_state(TASK_RUNNING);
  1740. /*
  1741. * context is unloaded at this point
  1742. */
  1743. DPRINT(("after zombie wakeup ctx_state=%d for\n", state));
  1744. }
  1745. else if (task != current) {
  1746. #ifdef CONFIG_SMP
  1747. /*
  1748. * switch context to zombie state
  1749. */
  1750. ctx->ctx_state = PFM_CTX_ZOMBIE;
  1751. DPRINT(("zombie ctx for [%d]\n", task_pid_nr(task)));
  1752. /*
  1753. * cannot free the context on the spot. deferred until
  1754. * the task notices the ZOMBIE state
  1755. */
  1756. free_possible = 0;
  1757. #else
  1758. pfm_context_unload(ctx, NULL, 0, regs);
  1759. #endif
  1760. }
  1761. doit:
  1762. /* reload state, may have changed during opening of critical section */
  1763. state = ctx->ctx_state;
  1764. /*
  1765. * the context is still attached to a task (possibly current)
  1766. * we cannot destroy it right now
  1767. */
  1768. /*
  1769. * we must free the sampling buffer right here because
  1770. * we cannot rely on it being cleaned up later by the
  1771. * monitored task. It is not possible to free vmalloc'ed
  1772. * memory in pfm_load_regs(). Instead, we remove the buffer
  1773. * now. should there be subsequent PMU overflow originally
  1774. * meant for sampling, the will be converted to spurious
  1775. * and that's fine because the monitoring tools is gone anyway.
  1776. */
  1777. if (ctx->ctx_smpl_hdr) {
  1778. smpl_buf_addr = ctx->ctx_smpl_hdr;
  1779. smpl_buf_size = ctx->ctx_smpl_size;
  1780. /* no more sampling */
  1781. ctx->ctx_smpl_hdr = NULL;
  1782. ctx->ctx_fl_is_sampling = 0;
  1783. }
  1784. DPRINT(("ctx_state=%d free_possible=%d addr=%p size=%lu\n",
  1785. state,
  1786. free_possible,
  1787. smpl_buf_addr,
  1788. smpl_buf_size));
  1789. if (smpl_buf_addr) pfm_exit_smpl_buffer(ctx->ctx_buf_fmt);
  1790. /*
  1791. * UNLOADED that the session has already been unreserved.
  1792. */
  1793. if (state == PFM_CTX_ZOMBIE) {
  1794. pfm_unreserve_session(ctx, ctx->ctx_fl_system , ctx->ctx_cpu);
  1795. }
  1796. /*
  1797. * disconnect file descriptor from context must be done
  1798. * before we unlock.
  1799. */
  1800. filp->private_data = NULL;
  1801. /*
  1802. * if we free on the spot, the context is now completely unreachable
  1803. * from the callers side. The monitored task side is also cut, so we
  1804. * can freely cut.
  1805. *
  1806. * If we have a deferred free, only the caller side is disconnected.
  1807. */
  1808. UNPROTECT_CTX(ctx, flags);
  1809. /*
  1810. * All memory free operations (especially for vmalloc'ed memory)
  1811. * MUST be done with interrupts ENABLED.
  1812. */
  1813. if (smpl_buf_addr) pfm_rvfree(smpl_buf_addr, smpl_buf_size);
  1814. /*
  1815. * return the memory used by the context
  1816. */
  1817. if (free_possible) pfm_context_free(ctx);
  1818. return 0;
  1819. }
  1820. static int
  1821. pfm_no_open(struct inode *irrelevant, struct file *dontcare)
  1822. {
  1823. DPRINT(("pfm_no_open called\n"));
  1824. return -ENXIO;
  1825. }
  1826. static const struct file_operations pfm_file_ops = {
  1827. .llseek = no_llseek,
  1828. .read = pfm_read,
  1829. .write = pfm_write,
  1830. .poll = pfm_poll,
  1831. .ioctl = pfm_ioctl,
  1832. .open = pfm_no_open, /* special open code to disallow open via /proc */
  1833. .fasync = pfm_fasync,
  1834. .release = pfm_close,
  1835. .flush = pfm_flush
  1836. };
  1837. static int
  1838. pfmfs_delete_dentry(struct dentry *dentry)
  1839. {
  1840. return 1;
  1841. }
  1842. static struct dentry_operations pfmfs_dentry_operations = {
  1843. .d_delete = pfmfs_delete_dentry,
  1844. };
  1845. static struct file *
  1846. pfm_alloc_file(pfm_context_t *ctx)
  1847. {
  1848. struct file *file;
  1849. struct inode *inode;
  1850. struct dentry *dentry;
  1851. char name[32];
  1852. struct qstr this;
  1853. /*
  1854. * allocate a new inode
  1855. */
  1856. inode = new_inode(pfmfs_mnt->mnt_sb);
  1857. if (!inode)
  1858. return ERR_PTR(-ENOMEM);
  1859. DPRINT(("new inode ino=%ld @%p\n", inode->i_ino, inode));
  1860. inode->i_mode = S_IFCHR|S_IRUGO;
  1861. inode->i_uid = current->fsuid;
  1862. inode->i_gid = current->fsgid;
  1863. sprintf(name, "[%lu]", inode->i_ino);
  1864. this.name = name;
  1865. this.len = strlen(name);
  1866. this.hash = inode->i_ino;
  1867. /*
  1868. * allocate a new dcache entry
  1869. */
  1870. dentry = d_alloc(pfmfs_mnt->mnt_sb->s_root, &this);
  1871. if (!dentry) {
  1872. iput(inode);
  1873. return ERR_PTR(-ENOMEM);
  1874. }
  1875. dentry->d_op = &pfmfs_dentry_operations;
  1876. d_add(dentry, inode);
  1877. file = alloc_file(pfmfs_mnt, dentry, FMODE_READ, &pfm_file_ops);
  1878. if (!file) {
  1879. dput(dentry);
  1880. return ERR_PTR(-ENFILE);
  1881. }
  1882. file->f_flags = O_RDONLY;
  1883. file->private_data = ctx;
  1884. return file;
  1885. }
  1886. static int
  1887. pfm_remap_buffer(struct vm_area_struct *vma, unsigned long buf, unsigned long addr, unsigned long size)
  1888. {
  1889. DPRINT(("CPU%d buf=0x%lx addr=0x%lx size=%ld\n", smp_processor_id(), buf, addr, size));
  1890. while (size > 0) {
  1891. unsigned long pfn = ia64_tpa(buf) >> PAGE_SHIFT;
  1892. if (remap_pfn_range(vma, addr, pfn, PAGE_SIZE, PAGE_READONLY))
  1893. return -ENOMEM;
  1894. addr += PAGE_SIZE;
  1895. buf += PAGE_SIZE;
  1896. size -= PAGE_SIZE;
  1897. }
  1898. return 0;
  1899. }
  1900. /*
  1901. * allocate a sampling buffer and remaps it into the user address space of the task
  1902. */
  1903. static int
  1904. pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t *ctx, unsigned long rsize, void **user_vaddr)
  1905. {
  1906. struct mm_struct *mm = task->mm;
  1907. struct vm_area_struct *vma = NULL;
  1908. unsigned long size;
  1909. void *smpl_buf;
  1910. /*
  1911. * the fixed header + requested size and align to page boundary
  1912. */
  1913. size = PAGE_ALIGN(rsize);
  1914. DPRINT(("sampling buffer rsize=%lu size=%lu bytes\n", rsize, size));
  1915. /*
  1916. * check requested size to avoid Denial-of-service attacks
  1917. * XXX: may have to refine this test
  1918. * Check against address space limit.
  1919. *
  1920. * if ((mm->total_vm << PAGE_SHIFT) + len> task->rlim[RLIMIT_AS].rlim_cur)
  1921. * return -ENOMEM;
  1922. */
  1923. if (size > task->signal->rlim[RLIMIT_MEMLOCK].rlim_cur)
  1924. return -ENOMEM;
  1925. /*
  1926. * We do the easy to undo allocations first.
  1927. *
  1928. * pfm_rvmalloc(), clears the buffer, so there is no leak
  1929. */
  1930. smpl_buf = pfm_rvmalloc(size);
  1931. if (smpl_buf == NULL) {
  1932. DPRINT(("Can't allocate sampling buffer\n"));
  1933. return -ENOMEM;
  1934. }
  1935. DPRINT(("smpl_buf @%p\n", smpl_buf));
  1936. /* allocate vma */
  1937. vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
  1938. if (!vma) {
  1939. DPRINT(("Cannot allocate vma\n"));
  1940. goto error_kmem;
  1941. }
  1942. /*
  1943. * partially initialize the vma for the sampling buffer
  1944. */
  1945. vma->vm_mm = mm;
  1946. vma->vm_file = filp;
  1947. vma->vm_flags = VM_READ| VM_MAYREAD |VM_RESERVED;
  1948. vma->vm_page_prot = PAGE_READONLY; /* XXX may need to change */
  1949. /*
  1950. * Now we have everything we need and we can initialize
  1951. * and connect all the data structures
  1952. */
  1953. ctx->ctx_smpl_hdr = smpl_buf;
  1954. ctx->ctx_smpl_size = size; /* aligned size */
  1955. /*
  1956. * Let's do the difficult operations next.
  1957. *
  1958. * now we atomically find some area in the address space and
  1959. * remap the buffer in it.
  1960. */
  1961. down_write(&task->mm->mmap_sem);
  1962. /* find some free area in address space, must have mmap sem held */
  1963. vma->vm_start = pfm_get_unmapped_area(NULL, 0, size, 0, MAP_PRIVATE|MAP_ANONYMOUS, 0);
  1964. if (vma->vm_start == 0UL) {
  1965. DPRINT(("Cannot find unmapped area for size %ld\n", size));
  1966. up_write(&task->mm->mmap_sem);
  1967. goto error;
  1968. }
  1969. vma->vm_end = vma->vm_start + size;
  1970. vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
  1971. DPRINT(("aligned size=%ld, hdr=%p mapped @0x%lx\n", size, ctx->ctx_smpl_hdr, vma->vm_start));
  1972. /* can only be applied to current task, need to have the mm semaphore held when called */
  1973. if (pfm_remap_buffer(vma, (unsigned long)smpl_buf, vma->vm_start, size)) {
  1974. DPRINT(("Can't remap buffer\n"));
  1975. up_write(&task->mm->mmap_sem);
  1976. goto error;
  1977. }
  1978. get_file(filp);
  1979. /*
  1980. * now insert the vma in the vm list for the process, must be
  1981. * done with mmap lock held
  1982. */
  1983. insert_vm_struct(mm, vma);
  1984. mm->total_vm += size >> PAGE_SHIFT;
  1985. vm_stat_account(vma->vm_mm, vma->vm_flags, vma->vm_file,
  1986. vma_pages(vma));
  1987. up_write(&task->mm->mmap_sem);
  1988. /*
  1989. * keep track of user level virtual address
  1990. */
  1991. ctx->ctx_smpl_vaddr = (void *)vma->vm_start;
  1992. *(unsigned long *)user_vaddr = vma->vm_start;
  1993. return 0;
  1994. error:
  1995. kmem_cache_free(vm_area_cachep, vma);
  1996. error_kmem:
  1997. pfm_rvfree(smpl_buf, size);
  1998. return -ENOMEM;
  1999. }
  2000. /*
  2001. * XXX: do something better here
  2002. */
  2003. static int
  2004. pfm_bad_permissions(struct task_struct *task)
  2005. {
  2006. /* inspired by ptrace_attach() */
  2007. DPRINT(("cur: uid=%d gid=%d task: euid=%d suid=%d uid=%d egid=%d sgid=%d\n",
  2008. current->uid,
  2009. current->gid,
  2010. task->euid,
  2011. task->suid,
  2012. task->uid,
  2013. task->egid,
  2014. task->sgid));
  2015. return ((current->uid != task->euid)
  2016. || (current->uid != task->suid)
  2017. || (current->uid != task->uid)
  2018. || (current->gid != task->egid)
  2019. || (current->gid != task->sgid)
  2020. || (current->gid != task->gid)) && !capable(CAP_SYS_PTRACE);
  2021. }
  2022. static int
  2023. pfarg_is_sane(struct task_struct *task, pfarg_context_t *pfx)
  2024. {
  2025. int ctx_flags;
  2026. /* valid signal */
  2027. ctx_flags = pfx->ctx_flags;
  2028. if (ctx_flags & PFM_FL_SYSTEM_WIDE) {
  2029. /*
  2030. * cannot block in this mode
  2031. */
  2032. if (ctx_flags & PFM_FL_NOTIFY_BLOCK) {
  2033. DPRINT(("cannot use blocking mode when in system wide monitoring\n"));
  2034. return -EINVAL;
  2035. }
  2036. } else {
  2037. }
  2038. /* probably more to add here */
  2039. return 0;
  2040. }
  2041. static int
  2042. pfm_setup_buffer_fmt(struct task_struct *task, struct file *filp, pfm_context_t *ctx, unsigned int ctx_flags,
  2043. unsigned int cpu, pfarg_context_t *arg)
  2044. {
  2045. pfm_buffer_fmt_t *fmt = NULL;
  2046. unsigned long size = 0UL;
  2047. void *uaddr = NULL;
  2048. void *fmt_arg = NULL;
  2049. int ret = 0;
  2050. #define PFM_CTXARG_BUF_ARG(a) (pfm_buffer_fmt_t *)(a+1)
  2051. /* invoke and lock buffer format, if found */
  2052. fmt = pfm_find_buffer_fmt(arg->ctx_smpl_buf_id);
  2053. if (fmt == NULL) {
  2054. DPRINT(("[%d] cannot find buffer format\n", task_pid_nr(task)));
  2055. return -EINVAL;
  2056. }
  2057. /*
  2058. * buffer argument MUST be contiguous to pfarg_context_t
  2059. */
  2060. if (fmt->fmt_arg_size) fmt_arg = PFM_CTXARG_BUF_ARG(arg);
  2061. ret = pfm_buf_fmt_validate(fmt, task, ctx_flags, cpu, fmt_arg);
  2062. DPRINT(("[%d] after validate(0x%x,%d,%p)=%d\n", task_pid_nr(task), ctx_flags, cpu, fmt_arg, ret));
  2063. if (ret) goto error;
  2064. /* link buffer format and context */
  2065. ctx->ctx_buf_fmt = fmt;
  2066. ctx->ctx_fl_is_sampling = 1; /* assume record() is defined */
  2067. /*
  2068. * check if buffer format wants to use perfmon buffer allocation/mapping service
  2069. */
  2070. ret = pfm_buf_fmt_getsize(fmt, task, ctx_flags, cpu, fmt_arg, &size);
  2071. if (ret) goto error;
  2072. if (size) {
  2073. /*
  2074. * buffer is always remapped into the caller's address space
  2075. */
  2076. ret = pfm_smpl_buffer_alloc(current, filp, ctx, size, &uaddr);
  2077. if (ret) goto error;
  2078. /* keep track of user address of buffer */
  2079. arg->ctx_smpl_vaddr = uaddr;
  2080. }
  2081. ret = pfm_buf_fmt_init(fmt, task, ctx->ctx_smpl_hdr, ctx_flags, cpu, fmt_arg);
  2082. error:
  2083. return ret;
  2084. }
  2085. static void
  2086. pfm_reset_pmu_state(pfm_context_t *ctx)
  2087. {
  2088. int i;
  2089. /*
  2090. * install reset values for PMC.
  2091. */
  2092. for (i=1; PMC_IS_LAST(i) == 0; i++) {
  2093. if (PMC_IS_IMPL(i) == 0) continue;
  2094. ctx->ctx_pmcs[i] = PMC_DFL_VAL(i);
  2095. DPRINT(("pmc[%d]=0x%lx\n", i, ctx->ctx_pmcs[i]));
  2096. }
  2097. /*
  2098. * PMD registers are set to 0UL when the context in memset()
  2099. */
  2100. /*
  2101. * On context switched restore, we must restore ALL pmc and ALL pmd even
  2102. * when they are not actively used by the task. In UP, the incoming process
  2103. * may otherwise pick up left over PMC, PMD state from the previous process.
  2104. * As opposed to PMD, stale PMC can cause harm to the incoming
  2105. * process because they may change what is being measured.
  2106. * Therefore, we must systematically reinstall the entire
  2107. * PMC state. In SMP, the same thing is possible on the
  2108. * same CPU but also on between 2 CPUs.
  2109. *
  2110. * The problem with PMD is information leaking especially
  2111. * to user level when psr.sp=0
  2112. *
  2113. * There is unfortunately no easy way to avoid this problem
  2114. * on either UP or SMP. This definitively slows down the
  2115. * pfm_load_regs() function.
  2116. */
  2117. /*
  2118. * bitmask of all PMCs accessible to this context
  2119. *
  2120. * PMC0 is treated differently.
  2121. */
  2122. ctx->ctx_all_pmcs[0] = pmu_conf->impl_pmcs[0] & ~0x1;
  2123. /*
  2124. * bitmask of all PMDs that are accessible to this context
  2125. */
  2126. ctx->ctx_all_pmds[0] = pmu_conf->impl_pmds[0];
  2127. DPRINT(("<%d> all_pmcs=0x%lx all_pmds=0x%lx\n", ctx->ctx_fd, ctx->ctx_all_pmcs[0],ctx->ctx_all_pmds[0]));
  2128. /*
  2129. * useful in case of re-enable after disable
  2130. */
  2131. ctx->ctx_used_ibrs[0] = 0UL;
  2132. ctx->ctx_used_dbrs[0] = 0UL;
  2133. }
  2134. static int
  2135. pfm_ctx_getsize(void *arg, size_t *sz)
  2136. {
  2137. pfarg_context_t *req = (pfarg_context_t *)arg;
  2138. pfm_buffer_fmt_t *fmt;
  2139. *sz = 0;
  2140. if (!pfm_uuid_cmp(req->ctx_smpl_buf_id, pfm_null_uuid)) return 0;
  2141. fmt = pfm_find_buffer_fmt(req->ctx_smpl_buf_id);
  2142. if (fmt == NULL) {
  2143. DPRINT(("cannot find buffer format\n"));
  2144. return -EINVAL;
  2145. }
  2146. /* get just enough to copy in user parameters */
  2147. *sz = fmt->fmt_arg_size;
  2148. DPRINT(("arg_size=%lu\n", *sz));
  2149. return 0;
  2150. }
  2151. /*
  2152. * cannot attach if :
  2153. * - kernel task
  2154. * - task not owned by caller
  2155. * - task incompatible with context mode
  2156. */
  2157. static int
  2158. pfm_task_incompatible(pfm_context_t *ctx, struct task_struct *task)
  2159. {
  2160. /*
  2161. * no kernel task or task not owner by caller
  2162. */
  2163. if (task->mm == NULL) {
  2164. DPRINT(("task [%d] has not memory context (kernel thread)\n", task_pid_nr(task)));
  2165. return -EPERM;
  2166. }
  2167. if (pfm_bad_permissions(task)) {
  2168. DPRINT(("no permission to attach to [%d]\n", task_pid_nr(task)));
  2169. return -EPERM;
  2170. }
  2171. /*
  2172. * cannot block in self-monitoring mode
  2173. */
  2174. if (CTX_OVFL_NOBLOCK(ctx) == 0 && task == current) {
  2175. DPRINT(("cannot load a blocking context on self for [%d]\n", task_pid_nr(task)));
  2176. return -EINVAL;
  2177. }
  2178. if (task->exit_state == EXIT_ZOMBIE) {
  2179. DPRINT(("cannot attach to zombie task [%d]\n", task_pid_nr(task)));
  2180. return -EBUSY;
  2181. }
  2182. /*
  2183. * always ok for self
  2184. */
  2185. if (task == current) return 0;
  2186. if (!task_is_stopped_or_traced(task)) {
  2187. DPRINT(("cannot attach to non-stopped task [%d] state=%ld\n", task_pid_nr(task), task->state));
  2188. return -EBUSY;
  2189. }
  2190. /*
  2191. * make sure the task is off any CPU
  2192. */
  2193. wait_task_inactive(task);
  2194. /* more to come... */
  2195. return 0;
  2196. }
  2197. static int
  2198. pfm_get_task(pfm_context_t *ctx, pid_t pid, struct task_struct **task)
  2199. {
  2200. struct task_struct *p = current;
  2201. int ret;
  2202. /* XXX: need to add more checks here */
  2203. if (pid < 2) return -EPERM;
  2204. if (pid != task_pid_vnr(current)) {
  2205. read_lock(&tasklist_lock);
  2206. p = find_task_by_vpid(pid);
  2207. /* make sure task cannot go away while we operate on it */
  2208. if (p) get_task_struct(p);
  2209. read_unlock(&tasklist_lock);
  2210. if (p == NULL) return -ESRCH;
  2211. }
  2212. ret = pfm_task_incompatible(ctx, p);
  2213. if (ret == 0) {
  2214. *task = p;
  2215. } else if (p != current) {
  2216. pfm_put_task(p);
  2217. }
  2218. return ret;
  2219. }
  2220. static int
  2221. pfm_context_create(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  2222. {
  2223. pfarg_context_t *req = (pfarg_context_t *)arg;
  2224. struct file *filp;
  2225. struct path path;
  2226. int ctx_flags;
  2227. int fd;
  2228. int ret;
  2229. /* let's check the arguments first */
  2230. ret = pfarg_is_sane(current, req);
  2231. if (ret < 0)
  2232. return ret;
  2233. ctx_flags = req->ctx_flags;
  2234. ret = -ENOMEM;
  2235. fd = get_unused_fd();
  2236. if (fd < 0)
  2237. return fd;
  2238. ctx = pfm_context_alloc(ctx_flags);
  2239. if (!ctx)
  2240. goto error;
  2241. filp = pfm_alloc_file(ctx);
  2242. if (IS_ERR(filp)) {
  2243. ret = PTR_ERR(filp);
  2244. goto error_file;
  2245. }
  2246. req->ctx_fd = ctx->ctx_fd = fd;
  2247. /*
  2248. * does the user want to sample?
  2249. */
  2250. if (pfm_uuid_cmp(req->ctx_smpl_buf_id, pfm_null_uuid)) {
  2251. ret = pfm_setup_buffer_fmt(current, filp, ctx, ctx_flags, 0, req);
  2252. if (ret)
  2253. goto buffer_error;
  2254. }
  2255. DPRINT(("ctx=%p flags=0x%x system=%d notify_block=%d excl_idle=%d no_msg=%d ctx_fd=%d \n",
  2256. ctx,
  2257. ctx_flags,
  2258. ctx->ctx_fl_system,
  2259. ctx->ctx_fl_block,
  2260. ctx->ctx_fl_excl_idle,
  2261. ctx->ctx_fl_no_msg,
  2262. ctx->ctx_fd));
  2263. /*
  2264. * initialize soft PMU state
  2265. */
  2266. pfm_reset_pmu_state(ctx);
  2267. fd_install(fd, filp);
  2268. return 0;
  2269. buffer_error:
  2270. path = filp->f_path;
  2271. put_filp(filp);
  2272. path_put(&path);
  2273. if (ctx->ctx_buf_fmt) {
  2274. pfm_buf_fmt_exit(ctx->ctx_buf_fmt, current, NULL, regs);
  2275. }
  2276. error_file:
  2277. pfm_context_free(ctx);
  2278. error:
  2279. put_unused_fd(fd);
  2280. return ret;
  2281. }
  2282. static inline unsigned long
  2283. pfm_new_counter_value (pfm_counter_t *reg, int is_long_reset)
  2284. {
  2285. unsigned long val = is_long_reset ? reg->long_reset : reg->short_reset;
  2286. unsigned long new_seed, old_seed = reg->seed, mask = reg->mask;
  2287. extern unsigned long carta_random32 (unsigned long seed);
  2288. if (reg->flags & PFM_REGFL_RANDOM) {
  2289. new_seed = carta_random32(old_seed);
  2290. val -= (old_seed & mask); /* counter values are negative numbers! */
  2291. if ((mask >> 32) != 0)
  2292. /* construct a full 64-bit random value: */
  2293. new_seed |= carta_random32(old_seed >> 32) << 32;
  2294. reg->seed = new_seed;
  2295. }
  2296. reg->lval = val;
  2297. return val;
  2298. }
  2299. static void
  2300. pfm_reset_regs_masked(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset)
  2301. {
  2302. unsigned long mask = ovfl_regs[0];
  2303. unsigned long reset_others = 0UL;
  2304. unsigned long val;
  2305. int i;
  2306. /*
  2307. * now restore reset value on sampling overflowed counters
  2308. */
  2309. mask >>= PMU_FIRST_COUNTER;
  2310. for(i = PMU_FIRST_COUNTER; mask; i++, mask >>= 1) {
  2311. if ((mask & 0x1UL) == 0UL) continue;
  2312. ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset);
  2313. reset_others |= ctx->ctx_pmds[i].reset_pmds[0];
  2314. DPRINT_ovfl((" %s reset ctx_pmds[%d]=%lx\n", is_long_reset ? "long" : "short", i, val));
  2315. }
  2316. /*
  2317. * Now take care of resetting the other registers
  2318. */
  2319. for(i = 0; reset_others; i++, reset_others >>= 1) {
  2320. if ((reset_others & 0x1) == 0) continue;
  2321. ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset);
  2322. DPRINT_ovfl(("%s reset_others pmd[%d]=%lx\n",
  2323. is_long_reset ? "long" : "short", i, val));
  2324. }
  2325. }
  2326. static void
  2327. pfm_reset_regs(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset)
  2328. {
  2329. unsigned long mask = ovfl_regs[0];
  2330. unsigned long reset_others = 0UL;
  2331. unsigned long val;
  2332. int i;
  2333. DPRINT_ovfl(("ovfl_regs=0x%lx is_long_reset=%d\n", ovfl_regs[0], is_long_reset));
  2334. if (ctx->ctx_state == PFM_CTX_MASKED) {
  2335. pfm_reset_regs_masked(ctx, ovfl_regs, is_long_reset);
  2336. return;
  2337. }
  2338. /*
  2339. * now restore reset value on sampling overflowed counters
  2340. */
  2341. mask >>= PMU_FIRST_COUNTER;
  2342. for(i = PMU_FIRST_COUNTER; mask; i++, mask >>= 1) {
  2343. if ((mask & 0x1UL) == 0UL) continue;
  2344. val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset);
  2345. reset_others |= ctx->ctx_pmds[i].reset_pmds[0];
  2346. DPRINT_ovfl((" %s reset ctx_pmds[%d]=%lx\n", is_long_reset ? "long" : "short", i, val));
  2347. pfm_write_soft_counter(ctx, i, val);
  2348. }
  2349. /*
  2350. * Now take care of resetting the other registers
  2351. */
  2352. for(i = 0; reset_others; i++, reset_others >>= 1) {
  2353. if ((reset_others & 0x1) == 0) continue;
  2354. val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset);
  2355. if (PMD_IS_COUNTING(i)) {
  2356. pfm_write_soft_counter(ctx, i, val);
  2357. } else {
  2358. ia64_set_pmd(i, val);
  2359. }
  2360. DPRINT_ovfl(("%s reset_others pmd[%d]=%lx\n",
  2361. is_long_reset ? "long" : "short", i, val));
  2362. }
  2363. ia64_srlz_d();
  2364. }
  2365. static int
  2366. pfm_write_pmcs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  2367. {
  2368. struct task_struct *task;
  2369. pfarg_reg_t *req = (pfarg_reg_t *)arg;
  2370. unsigned long value, pmc_pm;
  2371. unsigned long smpl_pmds, reset_pmds, impl_pmds;
  2372. unsigned int cnum, reg_flags, flags, pmc_type;
  2373. int i, can_access_pmu = 0, is_loaded, is_system, expert_mode;
  2374. int is_monitor, is_counting, state;
  2375. int ret = -EINVAL;
  2376. pfm_reg_check_t wr_func;
  2377. #define PFM_CHECK_PMC_PM(x, y, z) ((x)->ctx_fl_system ^ PMC_PM(y, z))
  2378. state = ctx->ctx_state;
  2379. is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
  2380. is_system = ctx->ctx_fl_system;
  2381. task = ctx->ctx_task;
  2382. impl_pmds = pmu_conf->impl_pmds[0];
  2383. if (state == PFM_CTX_ZOMBIE) return -EINVAL;
  2384. if (is_loaded) {
  2385. /*
  2386. * In system wide and when the context is loaded, access can only happen
  2387. * when the caller is running on the CPU being monitored by the session.
  2388. * It does not have to be the owner (ctx_task) of the context per se.
  2389. */
  2390. if (is_system && ctx->ctx_cpu != smp_processor_id()) {
  2391. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  2392. return -EBUSY;
  2393. }
  2394. can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
  2395. }
  2396. expert_mode = pfm_sysctl.expert_mode;
  2397. for (i = 0; i < count; i++, req++) {
  2398. cnum = req->reg_num;
  2399. reg_flags = req->reg_flags;
  2400. value = req->reg_value;
  2401. smpl_pmds = req->reg_smpl_pmds[0];
  2402. reset_pmds = req->reg_reset_pmds[0];
  2403. flags = 0;
  2404. if (cnum >= PMU_MAX_PMCS) {
  2405. DPRINT(("pmc%u is invalid\n", cnum));
  2406. goto error;
  2407. }
  2408. pmc_type = pmu_conf->pmc_desc[cnum].type;
  2409. pmc_pm = (value >> pmu_conf->pmc_desc[cnum].pm_pos) & 0x1;
  2410. is_counting = (pmc_type & PFM_REG_COUNTING) == PFM_REG_COUNTING ? 1 : 0;
  2411. is_monitor = (pmc_type & PFM_REG_MONITOR) == PFM_REG_MONITOR ? 1 : 0;
  2412. /*
  2413. * we reject all non implemented PMC as well
  2414. * as attempts to modify PMC[0-3] which are used
  2415. * as status registers by the PMU
  2416. */
  2417. if ((pmc_type & PFM_REG_IMPL) == 0 || (pmc_type & PFM_REG_CONTROL) == PFM_REG_CONTROL) {
  2418. DPRINT(("pmc%u is unimplemented or no-access pmc_type=%x\n", cnum, pmc_type));
  2419. goto error;
  2420. }
  2421. wr_func = pmu_conf->pmc_desc[cnum].write_check;
  2422. /*
  2423. * If the PMC is a monitor, then if the value is not the default:
  2424. * - system-wide session: PMCx.pm=1 (privileged monitor)
  2425. * - per-task : PMCx.pm=0 (user monitor)
  2426. */
  2427. if (is_monitor && value != PMC_DFL_VAL(cnum) && is_system ^ pmc_pm) {
  2428. DPRINT(("pmc%u pmc_pm=%lu is_system=%d\n",
  2429. cnum,
  2430. pmc_pm,
  2431. is_system));
  2432. goto error;
  2433. }
  2434. if (is_counting) {
  2435. /*
  2436. * enforce generation of overflow interrupt. Necessary on all
  2437. * CPUs.
  2438. */
  2439. value |= 1 << PMU_PMC_OI;
  2440. if (reg_flags & PFM_REGFL_OVFL_NOTIFY) {
  2441. flags |= PFM_REGFL_OVFL_NOTIFY;
  2442. }
  2443. if (reg_flags & PFM_REGFL_RANDOM) flags |= PFM_REGFL_RANDOM;
  2444. /* verify validity of smpl_pmds */
  2445. if ((smpl_pmds & impl_pmds) != smpl_pmds) {
  2446. DPRINT(("invalid smpl_pmds 0x%lx for pmc%u\n", smpl_pmds, cnum));
  2447. goto error;
  2448. }
  2449. /* verify validity of reset_pmds */
  2450. if ((reset_pmds & impl_pmds) != reset_pmds) {
  2451. DPRINT(("invalid reset_pmds 0x%lx for pmc%u\n", reset_pmds, cnum));
  2452. goto error;
  2453. }
  2454. } else {
  2455. if (reg_flags & (PFM_REGFL_OVFL_NOTIFY|PFM_REGFL_RANDOM)) {
  2456. DPRINT(("cannot set ovfl_notify or random on pmc%u\n", cnum));
  2457. goto error;
  2458. }
  2459. /* eventid on non-counting monitors are ignored */
  2460. }
  2461. /*
  2462. * execute write checker, if any
  2463. */
  2464. if (likely(expert_mode == 0 && wr_func)) {
  2465. ret = (*wr_func)(task, ctx, cnum, &value, regs);
  2466. if (ret) goto error;
  2467. ret = -EINVAL;
  2468. }
  2469. /*
  2470. * no error on this register
  2471. */
  2472. PFM_REG_RETFLAG_SET(req->reg_flags, 0);
  2473. /*
  2474. * Now we commit the changes to the software state
  2475. */
  2476. /*
  2477. * update overflow information
  2478. */
  2479. if (is_counting) {
  2480. /*
  2481. * full flag update each time a register is programmed
  2482. */
  2483. ctx->ctx_pmds[cnum].flags = flags;
  2484. ctx->ctx_pmds[cnum].reset_pmds[0] = reset_pmds;
  2485. ctx->ctx_pmds[cnum].smpl_pmds[0] = smpl_pmds;
  2486. ctx->ctx_pmds[cnum].eventid = req->reg_smpl_eventid;
  2487. /*
  2488. * Mark all PMDS to be accessed as used.
  2489. *
  2490. * We do not keep track of PMC because we have to
  2491. * systematically restore ALL of them.
  2492. *
  2493. * We do not update the used_monitors mask, because
  2494. * if we have not programmed them, then will be in
  2495. * a quiescent state, therefore we will not need to
  2496. * mask/restore then when context is MASKED.
  2497. */
  2498. CTX_USED_PMD(ctx, reset_pmds);
  2499. CTX_USED_PMD(ctx, smpl_pmds);
  2500. /*
  2501. * make sure we do not try to reset on
  2502. * restart because we have established new values
  2503. */
  2504. if (state == PFM_CTX_MASKED) ctx->ctx_ovfl_regs[0] &= ~1UL << cnum;
  2505. }
  2506. /*
  2507. * Needed in case the user does not initialize the equivalent
  2508. * PMD. Clearing is done indirectly via pfm_reset_pmu_state() so there is no
  2509. * possible leak here.
  2510. */
  2511. CTX_USED_PMD(ctx, pmu_conf->pmc_desc[cnum].dep_pmd[0]);
  2512. /*
  2513. * keep track of the monitor PMC that we are using.
  2514. * we save the value of the pmc in ctx_pmcs[] and if
  2515. * the monitoring is not stopped for the context we also
  2516. * place it in the saved state area so that it will be
  2517. * picked up later by the context switch code.
  2518. *
  2519. * The value in ctx_pmcs[] can only be changed in pfm_write_pmcs().
  2520. *
  2521. * The value in th_pmcs[] may be modified on overflow, i.e., when
  2522. * monitoring needs to be stopped.
  2523. */
  2524. if (is_monitor) CTX_USED_MONITOR(ctx, 1UL << cnum);
  2525. /*
  2526. * update context state
  2527. */
  2528. ctx->ctx_pmcs[cnum] = value;
  2529. if (is_loaded) {
  2530. /*
  2531. * write thread state
  2532. */
  2533. if (is_system == 0) ctx->th_pmcs[cnum] = value;
  2534. /*
  2535. * write hardware register if we can
  2536. */
  2537. if (can_access_pmu) {
  2538. ia64_set_pmc(cnum, value);
  2539. }
  2540. #ifdef CONFIG_SMP
  2541. else {
  2542. /*
  2543. * per-task SMP only here
  2544. *
  2545. * we are guaranteed that the task is not running on the other CPU,
  2546. * we indicate that this PMD will need to be reloaded if the task
  2547. * is rescheduled on the CPU it ran last on.
  2548. */
  2549. ctx->ctx_reload_pmcs[0] |= 1UL << cnum;
  2550. }
  2551. #endif
  2552. }
  2553. DPRINT(("pmc[%u]=0x%lx ld=%d apmu=%d flags=0x%x all_pmcs=0x%lx used_pmds=0x%lx eventid=%ld smpl_pmds=0x%lx reset_pmds=0x%lx reloads_pmcs=0x%lx used_monitors=0x%lx ovfl_regs=0x%lx\n",
  2554. cnum,
  2555. value,
  2556. is_loaded,
  2557. can_access_pmu,
  2558. flags,
  2559. ctx->ctx_all_pmcs[0],
  2560. ctx->ctx_used_pmds[0],
  2561. ctx->ctx_pmds[cnum].eventid,
  2562. smpl_pmds,
  2563. reset_pmds,
  2564. ctx->ctx_reload_pmcs[0],
  2565. ctx->ctx_used_monitors[0],
  2566. ctx->ctx_ovfl_regs[0]));
  2567. }
  2568. /*
  2569. * make sure the changes are visible
  2570. */
  2571. if (can_access_pmu) ia64_srlz_d();
  2572. return 0;
  2573. error:
  2574. PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
  2575. return ret;
  2576. }
  2577. static int
  2578. pfm_write_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  2579. {
  2580. struct task_struct *task;
  2581. pfarg_reg_t *req = (pfarg_reg_t *)arg;
  2582. unsigned long value, hw_value, ovfl_mask;
  2583. unsigned int cnum;
  2584. int i, can_access_pmu = 0, state;
  2585. int is_counting, is_loaded, is_system, expert_mode;
  2586. int ret = -EINVAL;
  2587. pfm_reg_check_t wr_func;
  2588. state = ctx->ctx_state;
  2589. is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
  2590. is_system = ctx->ctx_fl_system;
  2591. ovfl_mask = pmu_conf->ovfl_val;
  2592. task = ctx->ctx_task;
  2593. if (unlikely(state == PFM_CTX_ZOMBIE)) return -EINVAL;
  2594. /*
  2595. * on both UP and SMP, we can only write to the PMC when the task is
  2596. * the owner of the local PMU.
  2597. */
  2598. if (likely(is_loaded)) {
  2599. /*
  2600. * In system wide and when the context is loaded, access can only happen
  2601. * when the caller is running on the CPU being monitored by the session.
  2602. * It does not have to be the owner (ctx_task) of the context per se.
  2603. */
  2604. if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
  2605. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  2606. return -EBUSY;
  2607. }
  2608. can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
  2609. }
  2610. expert_mode = pfm_sysctl.expert_mode;
  2611. for (i = 0; i < count; i++, req++) {
  2612. cnum = req->reg_num;
  2613. value = req->reg_value;
  2614. if (!PMD_IS_IMPL(cnum)) {
  2615. DPRINT(("pmd[%u] is unimplemented or invalid\n", cnum));
  2616. goto abort_mission;
  2617. }
  2618. is_counting = PMD_IS_COUNTING(cnum);
  2619. wr_func = pmu_conf->pmd_desc[cnum].write_check;
  2620. /*
  2621. * execute write checker, if any
  2622. */
  2623. if (unlikely(expert_mode == 0 && wr_func)) {
  2624. unsigned long v = value;
  2625. ret = (*wr_func)(task, ctx, cnum, &v, regs);
  2626. if (ret) goto abort_mission;
  2627. value = v;
  2628. ret = -EINVAL;
  2629. }
  2630. /*
  2631. * no error on this register
  2632. */
  2633. PFM_REG_RETFLAG_SET(req->reg_flags, 0);
  2634. /*
  2635. * now commit changes to software state
  2636. */
  2637. hw_value = value;
  2638. /*
  2639. * update virtualized (64bits) counter
  2640. */
  2641. if (is_counting) {
  2642. /*
  2643. * write context state
  2644. */
  2645. ctx->ctx_pmds[cnum].lval = value;
  2646. /*
  2647. * when context is load we use the split value
  2648. */
  2649. if (is_loaded) {
  2650. hw_value = value & ovfl_mask;
  2651. value = value & ~ovfl_mask;
  2652. }
  2653. }
  2654. /*
  2655. * update reset values (not just for counters)
  2656. */
  2657. ctx->ctx_pmds[cnum].long_reset = req->reg_long_reset;
  2658. ctx->ctx_pmds[cnum].short_reset = req->reg_short_reset;
  2659. /*
  2660. * update randomization parameters (not just for counters)
  2661. */
  2662. ctx->ctx_pmds[cnum].seed = req->reg_random_seed;
  2663. ctx->ctx_pmds[cnum].mask = req->reg_random_mask;
  2664. /*
  2665. * update context value
  2666. */
  2667. ctx->ctx_pmds[cnum].val = value;
  2668. /*
  2669. * Keep track of what we use
  2670. *
  2671. * We do not keep track of PMC because we have to
  2672. * systematically restore ALL of them.
  2673. */
  2674. CTX_USED_PMD(ctx, PMD_PMD_DEP(cnum));
  2675. /*
  2676. * mark this PMD register used as well
  2677. */
  2678. CTX_USED_PMD(ctx, RDEP(cnum));
  2679. /*
  2680. * make sure we do not try to reset on
  2681. * restart because we have established new values
  2682. */
  2683. if (is_counting && state == PFM_CTX_MASKED) {
  2684. ctx->ctx_ovfl_regs[0] &= ~1UL << cnum;
  2685. }
  2686. if (is_loaded) {
  2687. /*
  2688. * write thread state
  2689. */
  2690. if (is_system == 0) ctx->th_pmds[cnum] = hw_value;
  2691. /*
  2692. * write hardware register if we can
  2693. */
  2694. if (can_access_pmu) {
  2695. ia64_set_pmd(cnum, hw_value);
  2696. } else {
  2697. #ifdef CONFIG_SMP
  2698. /*
  2699. * we are guaranteed that the task is not running on the other CPU,
  2700. * we indicate that this PMD will need to be reloaded if the task
  2701. * is rescheduled on the CPU it ran last on.
  2702. */
  2703. ctx->ctx_reload_pmds[0] |= 1UL << cnum;
  2704. #endif
  2705. }
  2706. }
  2707. DPRINT(("pmd[%u]=0x%lx ld=%d apmu=%d, hw_value=0x%lx ctx_pmd=0x%lx short_reset=0x%lx "
  2708. "long_reset=0x%lx notify=%c seed=0x%lx mask=0x%lx used_pmds=0x%lx reset_pmds=0x%lx reload_pmds=0x%lx all_pmds=0x%lx ovfl_regs=0x%lx\n",
  2709. cnum,
  2710. value,
  2711. is_loaded,
  2712. can_access_pmu,
  2713. hw_value,
  2714. ctx->ctx_pmds[cnum].val,
  2715. ctx->ctx_pmds[cnum].short_reset,
  2716. ctx->ctx_pmds[cnum].long_reset,
  2717. PMC_OVFL_NOTIFY(ctx, cnum) ? 'Y':'N',
  2718. ctx->ctx_pmds[cnum].seed,
  2719. ctx->ctx_pmds[cnum].mask,
  2720. ctx->ctx_used_pmds[0],
  2721. ctx->ctx_pmds[cnum].reset_pmds[0],
  2722. ctx->ctx_reload_pmds[0],
  2723. ctx->ctx_all_pmds[0],
  2724. ctx->ctx_ovfl_regs[0]));
  2725. }
  2726. /*
  2727. * make changes visible
  2728. */
  2729. if (can_access_pmu) ia64_srlz_d();
  2730. return 0;
  2731. abort_mission:
  2732. /*
  2733. * for now, we have only one possibility for error
  2734. */
  2735. PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
  2736. return ret;
  2737. }
  2738. /*
  2739. * By the way of PROTECT_CONTEXT(), interrupts are masked while we are in this function.
  2740. * Therefore we know, we do not have to worry about the PMU overflow interrupt. If an
  2741. * interrupt is delivered during the call, it will be kept pending until we leave, making
  2742. * it appears as if it had been generated at the UNPROTECT_CONTEXT(). At least we are
  2743. * guaranteed to return consistent data to the user, it may simply be old. It is not
  2744. * trivial to treat the overflow while inside the call because you may end up in
  2745. * some module sampling buffer code causing deadlocks.
  2746. */
  2747. static int
  2748. pfm_read_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  2749. {
  2750. struct task_struct *task;
  2751. unsigned long val = 0UL, lval, ovfl_mask, sval;
  2752. pfarg_reg_t *req = (pfarg_reg_t *)arg;
  2753. unsigned int cnum, reg_flags = 0;
  2754. int i, can_access_pmu = 0, state;
  2755. int is_loaded, is_system, is_counting, expert_mode;
  2756. int ret = -EINVAL;
  2757. pfm_reg_check_t rd_func;
  2758. /*
  2759. * access is possible when loaded only for
  2760. * self-monitoring tasks or in UP mode
  2761. */
  2762. state = ctx->ctx_state;
  2763. is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
  2764. is_system = ctx->ctx_fl_system;
  2765. ovfl_mask = pmu_conf->ovfl_val;
  2766. task = ctx->ctx_task;
  2767. if (state == PFM_CTX_ZOMBIE) return -EINVAL;
  2768. if (likely(is_loaded)) {
  2769. /*
  2770. * In system wide and when the context is loaded, access can only happen
  2771. * when the caller is running on the CPU being monitored by the session.
  2772. * It does not have to be the owner (ctx_task) of the context per se.
  2773. */
  2774. if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
  2775. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  2776. return -EBUSY;
  2777. }
  2778. /*
  2779. * this can be true when not self-monitoring only in UP
  2780. */
  2781. can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
  2782. if (can_access_pmu) ia64_srlz_d();
  2783. }
  2784. expert_mode = pfm_sysctl.expert_mode;
  2785. DPRINT(("ld=%d apmu=%d ctx_state=%d\n",
  2786. is_loaded,
  2787. can_access_pmu,
  2788. state));
  2789. /*
  2790. * on both UP and SMP, we can only read the PMD from the hardware register when
  2791. * the task is the owner of the local PMU.
  2792. */
  2793. for (i = 0; i < count; i++, req++) {
  2794. cnum = req->reg_num;
  2795. reg_flags = req->reg_flags;
  2796. if (unlikely(!PMD_IS_IMPL(cnum))) goto error;
  2797. /*
  2798. * we can only read the register that we use. That includes
  2799. * the one we explicitly initialize AND the one we want included
  2800. * in the sampling buffer (smpl_regs).
  2801. *
  2802. * Having this restriction allows optimization in the ctxsw routine
  2803. * without compromising security (leaks)
  2804. */
  2805. if (unlikely(!CTX_IS_USED_PMD(ctx, cnum))) goto error;
  2806. sval = ctx->ctx_pmds[cnum].val;
  2807. lval = ctx->ctx_pmds[cnum].lval;
  2808. is_counting = PMD_IS_COUNTING(cnum);
  2809. /*
  2810. * If the task is not the current one, then we check if the
  2811. * PMU state is still in the local live register due to lazy ctxsw.
  2812. * If true, then we read directly from the registers.
  2813. */
  2814. if (can_access_pmu){
  2815. val = ia64_get_pmd(cnum);
  2816. } else {
  2817. /*
  2818. * context has been saved
  2819. * if context is zombie, then task does not exist anymore.
  2820. * In this case, we use the full value saved in the context (pfm_flush_regs()).
  2821. */
  2822. val = is_loaded ? ctx->th_pmds[cnum] : 0UL;
  2823. }
  2824. rd_func = pmu_conf->pmd_desc[cnum].read_check;
  2825. if (is_counting) {
  2826. /*
  2827. * XXX: need to check for overflow when loaded
  2828. */
  2829. val &= ovfl_mask;
  2830. val += sval;
  2831. }
  2832. /*
  2833. * execute read checker, if any
  2834. */
  2835. if (unlikely(expert_mode == 0 && rd_func)) {
  2836. unsigned long v = val;
  2837. ret = (*rd_func)(ctx->ctx_task, ctx, cnum, &v, regs);
  2838. if (ret) goto error;
  2839. val = v;
  2840. ret = -EINVAL;
  2841. }
  2842. PFM_REG_RETFLAG_SET(reg_flags, 0);
  2843. DPRINT(("pmd[%u]=0x%lx\n", cnum, val));
  2844. /*
  2845. * update register return value, abort all if problem during copy.
  2846. * we only modify the reg_flags field. no check mode is fine because
  2847. * access has been verified upfront in sys_perfmonctl().
  2848. */
  2849. req->reg_value = val;
  2850. req->reg_flags = reg_flags;
  2851. req->reg_last_reset_val = lval;
  2852. }
  2853. return 0;
  2854. error:
  2855. PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
  2856. return ret;
  2857. }
  2858. int
  2859. pfm_mod_write_pmcs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
  2860. {
  2861. pfm_context_t *ctx;
  2862. if (req == NULL) return -EINVAL;
  2863. ctx = GET_PMU_CTX();
  2864. if (ctx == NULL) return -EINVAL;
  2865. /*
  2866. * for now limit to current task, which is enough when calling
  2867. * from overflow handler
  2868. */
  2869. if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
  2870. return pfm_write_pmcs(ctx, req, nreq, regs);
  2871. }
  2872. EXPORT_SYMBOL(pfm_mod_write_pmcs);
  2873. int
  2874. pfm_mod_read_pmds(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
  2875. {
  2876. pfm_context_t *ctx;
  2877. if (req == NULL) return -EINVAL;
  2878. ctx = GET_PMU_CTX();
  2879. if (ctx == NULL) return -EINVAL;
  2880. /*
  2881. * for now limit to current task, which is enough when calling
  2882. * from overflow handler
  2883. */
  2884. if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
  2885. return pfm_read_pmds(ctx, req, nreq, regs);
  2886. }
  2887. EXPORT_SYMBOL(pfm_mod_read_pmds);
  2888. /*
  2889. * Only call this function when a process it trying to
  2890. * write the debug registers (reading is always allowed)
  2891. */
  2892. int
  2893. pfm_use_debug_registers(struct task_struct *task)
  2894. {
  2895. pfm_context_t *ctx = task->thread.pfm_context;
  2896. unsigned long flags;
  2897. int ret = 0;
  2898. if (pmu_conf->use_rr_dbregs == 0) return 0;
  2899. DPRINT(("called for [%d]\n", task_pid_nr(task)));
  2900. /*
  2901. * do it only once
  2902. */
  2903. if (task->thread.flags & IA64_THREAD_DBG_VALID) return 0;
  2904. /*
  2905. * Even on SMP, we do not need to use an atomic here because
  2906. * the only way in is via ptrace() and this is possible only when the
  2907. * process is stopped. Even in the case where the ctxsw out is not totally
  2908. * completed by the time we come here, there is no way the 'stopped' process
  2909. * could be in the middle of fiddling with the pfm_write_ibr_dbr() routine.
  2910. * So this is always safe.
  2911. */
  2912. if (ctx && ctx->ctx_fl_using_dbreg == 1) return -1;
  2913. LOCK_PFS(flags);
  2914. /*
  2915. * We cannot allow setting breakpoints when system wide monitoring
  2916. * sessions are using the debug registers.
  2917. */
  2918. if (pfm_sessions.pfs_sys_use_dbregs> 0)
  2919. ret = -1;
  2920. else
  2921. pfm_sessions.pfs_ptrace_use_dbregs++;
  2922. DPRINT(("ptrace_use_dbregs=%u sys_use_dbregs=%u by [%d] ret = %d\n",
  2923. pfm_sessions.pfs_ptrace_use_dbregs,
  2924. pfm_sessions.pfs_sys_use_dbregs,
  2925. task_pid_nr(task), ret));
  2926. UNLOCK_PFS(flags);
  2927. return ret;
  2928. }
  2929. /*
  2930. * This function is called for every task that exits with the
  2931. * IA64_THREAD_DBG_VALID set. This indicates a task which was
  2932. * able to use the debug registers for debugging purposes via
  2933. * ptrace(). Therefore we know it was not using them for
  2934. * perfmormance monitoring, so we only decrement the number
  2935. * of "ptraced" debug register users to keep the count up to date
  2936. */
  2937. int
  2938. pfm_release_debug_registers(struct task_struct *task)
  2939. {
  2940. unsigned long flags;
  2941. int ret;
  2942. if (pmu_conf->use_rr_dbregs == 0) return 0;
  2943. LOCK_PFS(flags);
  2944. if (pfm_sessions.pfs_ptrace_use_dbregs == 0) {
  2945. printk(KERN_ERR "perfmon: invalid release for [%d] ptrace_use_dbregs=0\n", task_pid_nr(task));
  2946. ret = -1;
  2947. } else {
  2948. pfm_sessions.pfs_ptrace_use_dbregs--;
  2949. ret = 0;
  2950. }
  2951. UNLOCK_PFS(flags);
  2952. return ret;
  2953. }
  2954. static int
  2955. pfm_restart(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  2956. {
  2957. struct task_struct *task;
  2958. pfm_buffer_fmt_t *fmt;
  2959. pfm_ovfl_ctrl_t rst_ctrl;
  2960. int state, is_system;
  2961. int ret = 0;
  2962. state = ctx->ctx_state;
  2963. fmt = ctx->ctx_buf_fmt;
  2964. is_system = ctx->ctx_fl_system;
  2965. task = PFM_CTX_TASK(ctx);
  2966. switch(state) {
  2967. case PFM_CTX_MASKED:
  2968. break;
  2969. case PFM_CTX_LOADED:
  2970. if (CTX_HAS_SMPL(ctx) && fmt->fmt_restart_active) break;
  2971. /* fall through */
  2972. case PFM_CTX_UNLOADED:
  2973. case PFM_CTX_ZOMBIE:
  2974. DPRINT(("invalid state=%d\n", state));
  2975. return -EBUSY;
  2976. default:
  2977. DPRINT(("state=%d, cannot operate (no active_restart handler)\n", state));
  2978. return -EINVAL;
  2979. }
  2980. /*
  2981. * In system wide and when the context is loaded, access can only happen
  2982. * when the caller is running on the CPU being monitored by the session.
  2983. * It does not have to be the owner (ctx_task) of the context per se.
  2984. */
  2985. if (is_system && ctx->ctx_cpu != smp_processor_id()) {
  2986. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  2987. return -EBUSY;
  2988. }
  2989. /* sanity check */
  2990. if (unlikely(task == NULL)) {
  2991. printk(KERN_ERR "perfmon: [%d] pfm_restart no task\n", task_pid_nr(current));
  2992. return -EINVAL;
  2993. }
  2994. if (task == current || is_system) {
  2995. fmt = ctx->ctx_buf_fmt;
  2996. DPRINT(("restarting self %d ovfl=0x%lx\n",
  2997. task_pid_nr(task),
  2998. ctx->ctx_ovfl_regs[0]));
  2999. if (CTX_HAS_SMPL(ctx)) {
  3000. prefetch(ctx->ctx_smpl_hdr);
  3001. rst_ctrl.bits.mask_monitoring = 0;
  3002. rst_ctrl.bits.reset_ovfl_pmds = 0;
  3003. if (state == PFM_CTX_LOADED)
  3004. ret = pfm_buf_fmt_restart_active(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
  3005. else
  3006. ret = pfm_buf_fmt_restart(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
  3007. } else {
  3008. rst_ctrl.bits.mask_monitoring = 0;
  3009. rst_ctrl.bits.reset_ovfl_pmds = 1;
  3010. }
  3011. if (ret == 0) {
  3012. if (rst_ctrl.bits.reset_ovfl_pmds)
  3013. pfm_reset_regs(ctx, ctx->ctx_ovfl_regs, PFM_PMD_LONG_RESET);
  3014. if (rst_ctrl.bits.mask_monitoring == 0) {
  3015. DPRINT(("resuming monitoring for [%d]\n", task_pid_nr(task)));
  3016. if (state == PFM_CTX_MASKED) pfm_restore_monitoring(task);
  3017. } else {
  3018. DPRINT(("keeping monitoring stopped for [%d]\n", task_pid_nr(task)));
  3019. // cannot use pfm_stop_monitoring(task, regs);
  3020. }
  3021. }
  3022. /*
  3023. * clear overflowed PMD mask to remove any stale information
  3024. */
  3025. ctx->ctx_ovfl_regs[0] = 0UL;
  3026. /*
  3027. * back to LOADED state
  3028. */
  3029. ctx->ctx_state = PFM_CTX_LOADED;
  3030. /*
  3031. * XXX: not really useful for self monitoring
  3032. */
  3033. ctx->ctx_fl_can_restart = 0;
  3034. return 0;
  3035. }
  3036. /*
  3037. * restart another task
  3038. */
  3039. /*
  3040. * When PFM_CTX_MASKED, we cannot issue a restart before the previous
  3041. * one is seen by the task.
  3042. */
  3043. if (state == PFM_CTX_MASKED) {
  3044. if (ctx->ctx_fl_can_restart == 0) return -EINVAL;
  3045. /*
  3046. * will prevent subsequent restart before this one is
  3047. * seen by other task
  3048. */
  3049. ctx->ctx_fl_can_restart = 0;
  3050. }
  3051. /*
  3052. * if blocking, then post the semaphore is PFM_CTX_MASKED, i.e.
  3053. * the task is blocked or on its way to block. That's the normal
  3054. * restart path. If the monitoring is not masked, then the task
  3055. * can be actively monitoring and we cannot directly intervene.
  3056. * Therefore we use the trap mechanism to catch the task and
  3057. * force it to reset the buffer/reset PMDs.
  3058. *
  3059. * if non-blocking, then we ensure that the task will go into
  3060. * pfm_handle_work() before returning to user mode.
  3061. *
  3062. * We cannot explicitly reset another task, it MUST always
  3063. * be done by the task itself. This works for system wide because
  3064. * the tool that is controlling the session is logically doing
  3065. * "self-monitoring".
  3066. */
  3067. if (CTX_OVFL_NOBLOCK(ctx) == 0 && state == PFM_CTX_MASKED) {
  3068. DPRINT(("unblocking [%d] \n", task_pid_nr(task)));
  3069. complete(&ctx->ctx_restart_done);
  3070. } else {
  3071. DPRINT(("[%d] armed exit trap\n", task_pid_nr(task)));
  3072. ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_RESET;
  3073. PFM_SET_WORK_PENDING(task, 1);
  3074. tsk_set_notify_resume(task);
  3075. /*
  3076. * XXX: send reschedule if task runs on another CPU
  3077. */
  3078. }
  3079. return 0;
  3080. }
  3081. static int
  3082. pfm_debug(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3083. {
  3084. unsigned int m = *(unsigned int *)arg;
  3085. pfm_sysctl.debug = m == 0 ? 0 : 1;
  3086. printk(KERN_INFO "perfmon debugging %s (timing reset)\n", pfm_sysctl.debug ? "on" : "off");
  3087. if (m == 0) {
  3088. memset(pfm_stats, 0, sizeof(pfm_stats));
  3089. for(m=0; m < NR_CPUS; m++) pfm_stats[m].pfm_ovfl_intr_cycles_min = ~0UL;
  3090. }
  3091. return 0;
  3092. }
  3093. /*
  3094. * arg can be NULL and count can be zero for this function
  3095. */
  3096. static int
  3097. pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3098. {
  3099. struct thread_struct *thread = NULL;
  3100. struct task_struct *task;
  3101. pfarg_dbreg_t *req = (pfarg_dbreg_t *)arg;
  3102. unsigned long flags;
  3103. dbreg_t dbreg;
  3104. unsigned int rnum;
  3105. int first_time;
  3106. int ret = 0, state;
  3107. int i, can_access_pmu = 0;
  3108. int is_system, is_loaded;
  3109. if (pmu_conf->use_rr_dbregs == 0) return -EINVAL;
  3110. state = ctx->ctx_state;
  3111. is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
  3112. is_system = ctx->ctx_fl_system;
  3113. task = ctx->ctx_task;
  3114. if (state == PFM_CTX_ZOMBIE) return -EINVAL;
  3115. /*
  3116. * on both UP and SMP, we can only write to the PMC when the task is
  3117. * the owner of the local PMU.
  3118. */
  3119. if (is_loaded) {
  3120. thread = &task->thread;
  3121. /*
  3122. * In system wide and when the context is loaded, access can only happen
  3123. * when the caller is running on the CPU being monitored by the session.
  3124. * It does not have to be the owner (ctx_task) of the context per se.
  3125. */
  3126. if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
  3127. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  3128. return -EBUSY;
  3129. }
  3130. can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
  3131. }
  3132. /*
  3133. * we do not need to check for ipsr.db because we do clear ibr.x, dbr.r, and dbr.w
  3134. * ensuring that no real breakpoint can be installed via this call.
  3135. *
  3136. * IMPORTANT: regs can be NULL in this function
  3137. */
  3138. first_time = ctx->ctx_fl_using_dbreg == 0;
  3139. /*
  3140. * don't bother if we are loaded and task is being debugged
  3141. */
  3142. if (is_loaded && (thread->flags & IA64_THREAD_DBG_VALID) != 0) {
  3143. DPRINT(("debug registers already in use for [%d]\n", task_pid_nr(task)));
  3144. return -EBUSY;
  3145. }
  3146. /*
  3147. * check for debug registers in system wide mode
  3148. *
  3149. * If though a check is done in pfm_context_load(),
  3150. * we must repeat it here, in case the registers are
  3151. * written after the context is loaded
  3152. */
  3153. if (is_loaded) {
  3154. LOCK_PFS(flags);
  3155. if (first_time && is_system) {
  3156. if (pfm_sessions.pfs_ptrace_use_dbregs)
  3157. ret = -EBUSY;
  3158. else
  3159. pfm_sessions.pfs_sys_use_dbregs++;
  3160. }
  3161. UNLOCK_PFS(flags);
  3162. }
  3163. if (ret != 0) return ret;
  3164. /*
  3165. * mark ourself as user of the debug registers for
  3166. * perfmon purposes.
  3167. */
  3168. ctx->ctx_fl_using_dbreg = 1;
  3169. /*
  3170. * clear hardware registers to make sure we don't
  3171. * pick up stale state.
  3172. *
  3173. * for a system wide session, we do not use
  3174. * thread.dbr, thread.ibr because this process
  3175. * never leaves the current CPU and the state
  3176. * is shared by all processes running on it
  3177. */
  3178. if (first_time && can_access_pmu) {
  3179. DPRINT(("[%d] clearing ibrs, dbrs\n", task_pid_nr(task)));
  3180. for (i=0; i < pmu_conf->num_ibrs; i++) {
  3181. ia64_set_ibr(i, 0UL);
  3182. ia64_dv_serialize_instruction();
  3183. }
  3184. ia64_srlz_i();
  3185. for (i=0; i < pmu_conf->num_dbrs; i++) {
  3186. ia64_set_dbr(i, 0UL);
  3187. ia64_dv_serialize_data();
  3188. }
  3189. ia64_srlz_d();
  3190. }
  3191. /*
  3192. * Now install the values into the registers
  3193. */
  3194. for (i = 0; i < count; i++, req++) {
  3195. rnum = req->dbreg_num;
  3196. dbreg.val = req->dbreg_value;
  3197. ret = -EINVAL;
  3198. if ((mode == PFM_CODE_RR && rnum >= PFM_NUM_IBRS) || ((mode == PFM_DATA_RR) && rnum >= PFM_NUM_DBRS)) {
  3199. DPRINT(("invalid register %u val=0x%lx mode=%d i=%d count=%d\n",
  3200. rnum, dbreg.val, mode, i, count));
  3201. goto abort_mission;
  3202. }
  3203. /*
  3204. * make sure we do not install enabled breakpoint
  3205. */
  3206. if (rnum & 0x1) {
  3207. if (mode == PFM_CODE_RR)
  3208. dbreg.ibr.ibr_x = 0;
  3209. else
  3210. dbreg.dbr.dbr_r = dbreg.dbr.dbr_w = 0;
  3211. }
  3212. PFM_REG_RETFLAG_SET(req->dbreg_flags, 0);
  3213. /*
  3214. * Debug registers, just like PMC, can only be modified
  3215. * by a kernel call. Moreover, perfmon() access to those
  3216. * registers are centralized in this routine. The hardware
  3217. * does not modify the value of these registers, therefore,
  3218. * if we save them as they are written, we can avoid having
  3219. * to save them on context switch out. This is made possible
  3220. * by the fact that when perfmon uses debug registers, ptrace()
  3221. * won't be able to modify them concurrently.
  3222. */
  3223. if (mode == PFM_CODE_RR) {
  3224. CTX_USED_IBR(ctx, rnum);
  3225. if (can_access_pmu) {
  3226. ia64_set_ibr(rnum, dbreg.val);
  3227. ia64_dv_serialize_instruction();
  3228. }
  3229. ctx->ctx_ibrs[rnum] = dbreg.val;
  3230. DPRINT(("write ibr%u=0x%lx used_ibrs=0x%x ld=%d apmu=%d\n",
  3231. rnum, dbreg.val, ctx->ctx_used_ibrs[0], is_loaded, can_access_pmu));
  3232. } else {
  3233. CTX_USED_DBR(ctx, rnum);
  3234. if (can_access_pmu) {
  3235. ia64_set_dbr(rnum, dbreg.val);
  3236. ia64_dv_serialize_data();
  3237. }
  3238. ctx->ctx_dbrs[rnum] = dbreg.val;
  3239. DPRINT(("write dbr%u=0x%lx used_dbrs=0x%x ld=%d apmu=%d\n",
  3240. rnum, dbreg.val, ctx->ctx_used_dbrs[0], is_loaded, can_access_pmu));
  3241. }
  3242. }
  3243. return 0;
  3244. abort_mission:
  3245. /*
  3246. * in case it was our first attempt, we undo the global modifications
  3247. */
  3248. if (first_time) {
  3249. LOCK_PFS(flags);
  3250. if (ctx->ctx_fl_system) {
  3251. pfm_sessions.pfs_sys_use_dbregs--;
  3252. }
  3253. UNLOCK_PFS(flags);
  3254. ctx->ctx_fl_using_dbreg = 0;
  3255. }
  3256. /*
  3257. * install error return flag
  3258. */
  3259. PFM_REG_RETFLAG_SET(req->dbreg_flags, PFM_REG_RETFL_EINVAL);
  3260. return ret;
  3261. }
  3262. static int
  3263. pfm_write_ibrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3264. {
  3265. return pfm_write_ibr_dbr(PFM_CODE_RR, ctx, arg, count, regs);
  3266. }
  3267. static int
  3268. pfm_write_dbrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3269. {
  3270. return pfm_write_ibr_dbr(PFM_DATA_RR, ctx, arg, count, regs);
  3271. }
  3272. int
  3273. pfm_mod_write_ibrs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
  3274. {
  3275. pfm_context_t *ctx;
  3276. if (req == NULL) return -EINVAL;
  3277. ctx = GET_PMU_CTX();
  3278. if (ctx == NULL) return -EINVAL;
  3279. /*
  3280. * for now limit to current task, which is enough when calling
  3281. * from overflow handler
  3282. */
  3283. if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
  3284. return pfm_write_ibrs(ctx, req, nreq, regs);
  3285. }
  3286. EXPORT_SYMBOL(pfm_mod_write_ibrs);
  3287. int
  3288. pfm_mod_write_dbrs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
  3289. {
  3290. pfm_context_t *ctx;
  3291. if (req == NULL) return -EINVAL;
  3292. ctx = GET_PMU_CTX();
  3293. if (ctx == NULL) return -EINVAL;
  3294. /*
  3295. * for now limit to current task, which is enough when calling
  3296. * from overflow handler
  3297. */
  3298. if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
  3299. return pfm_write_dbrs(ctx, req, nreq, regs);
  3300. }
  3301. EXPORT_SYMBOL(pfm_mod_write_dbrs);
  3302. static int
  3303. pfm_get_features(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3304. {
  3305. pfarg_features_t *req = (pfarg_features_t *)arg;
  3306. req->ft_version = PFM_VERSION;
  3307. return 0;
  3308. }
  3309. static int
  3310. pfm_stop(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3311. {
  3312. struct pt_regs *tregs;
  3313. struct task_struct *task = PFM_CTX_TASK(ctx);
  3314. int state, is_system;
  3315. state = ctx->ctx_state;
  3316. is_system = ctx->ctx_fl_system;
  3317. /*
  3318. * context must be attached to issue the stop command (includes LOADED,MASKED,ZOMBIE)
  3319. */
  3320. if (state == PFM_CTX_UNLOADED) return -EINVAL;
  3321. /*
  3322. * In system wide and when the context is loaded, access can only happen
  3323. * when the caller is running on the CPU being monitored by the session.
  3324. * It does not have to be the owner (ctx_task) of the context per se.
  3325. */
  3326. if (is_system && ctx->ctx_cpu != smp_processor_id()) {
  3327. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  3328. return -EBUSY;
  3329. }
  3330. DPRINT(("task [%d] ctx_state=%d is_system=%d\n",
  3331. task_pid_nr(PFM_CTX_TASK(ctx)),
  3332. state,
  3333. is_system));
  3334. /*
  3335. * in system mode, we need to update the PMU directly
  3336. * and the user level state of the caller, which may not
  3337. * necessarily be the creator of the context.
  3338. */
  3339. if (is_system) {
  3340. /*
  3341. * Update local PMU first
  3342. *
  3343. * disable dcr pp
  3344. */
  3345. ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) & ~IA64_DCR_PP);
  3346. ia64_srlz_i();
  3347. /*
  3348. * update local cpuinfo
  3349. */
  3350. PFM_CPUINFO_CLEAR(PFM_CPUINFO_DCR_PP);
  3351. /*
  3352. * stop monitoring, does srlz.i
  3353. */
  3354. pfm_clear_psr_pp();
  3355. /*
  3356. * stop monitoring in the caller
  3357. */
  3358. ia64_psr(regs)->pp = 0;
  3359. return 0;
  3360. }
  3361. /*
  3362. * per-task mode
  3363. */
  3364. if (task == current) {
  3365. /* stop monitoring at kernel level */
  3366. pfm_clear_psr_up();
  3367. /*
  3368. * stop monitoring at the user level
  3369. */
  3370. ia64_psr(regs)->up = 0;
  3371. } else {
  3372. tregs = task_pt_regs(task);
  3373. /*
  3374. * stop monitoring at the user level
  3375. */
  3376. ia64_psr(tregs)->up = 0;
  3377. /*
  3378. * monitoring disabled in kernel at next reschedule
  3379. */
  3380. ctx->ctx_saved_psr_up = 0;
  3381. DPRINT(("task=[%d]\n", task_pid_nr(task)));
  3382. }
  3383. return 0;
  3384. }
  3385. static int
  3386. pfm_start(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3387. {
  3388. struct pt_regs *tregs;
  3389. int state, is_system;
  3390. state = ctx->ctx_state;
  3391. is_system = ctx->ctx_fl_system;
  3392. if (state != PFM_CTX_LOADED) return -EINVAL;
  3393. /*
  3394. * In system wide and when the context is loaded, access can only happen
  3395. * when the caller is running on the CPU being monitored by the session.
  3396. * It does not have to be the owner (ctx_task) of the context per se.
  3397. */
  3398. if (is_system && ctx->ctx_cpu != smp_processor_id()) {
  3399. DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
  3400. return -EBUSY;
  3401. }
  3402. /*
  3403. * in system mode, we need to update the PMU directly
  3404. * and the user level state of the caller, which may not
  3405. * necessarily be the creator of the context.
  3406. */
  3407. if (is_system) {
  3408. /*
  3409. * set user level psr.pp for the caller
  3410. */
  3411. ia64_psr(regs)->pp = 1;
  3412. /*
  3413. * now update the local PMU and cpuinfo
  3414. */
  3415. PFM_CPUINFO_SET(PFM_CPUINFO_DCR_PP);
  3416. /*
  3417. * start monitoring at kernel level
  3418. */
  3419. pfm_set_psr_pp();
  3420. /* enable dcr pp */
  3421. ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) | IA64_DCR_PP);
  3422. ia64_srlz_i();
  3423. return 0;
  3424. }
  3425. /*
  3426. * per-process mode
  3427. */
  3428. if (ctx->ctx_task == current) {
  3429. /* start monitoring at kernel level */
  3430. pfm_set_psr_up();
  3431. /*
  3432. * activate monitoring at user level
  3433. */
  3434. ia64_psr(regs)->up = 1;
  3435. } else {
  3436. tregs = task_pt_regs(ctx->ctx_task);
  3437. /*
  3438. * start monitoring at the kernel level the next
  3439. * time the task is scheduled
  3440. */
  3441. ctx->ctx_saved_psr_up = IA64_PSR_UP;
  3442. /*
  3443. * activate monitoring at user level
  3444. */
  3445. ia64_psr(tregs)->up = 1;
  3446. }
  3447. return 0;
  3448. }
  3449. static int
  3450. pfm_get_pmc_reset(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3451. {
  3452. pfarg_reg_t *req = (pfarg_reg_t *)arg;
  3453. unsigned int cnum;
  3454. int i;
  3455. int ret = -EINVAL;
  3456. for (i = 0; i < count; i++, req++) {
  3457. cnum = req->reg_num;
  3458. if (!PMC_IS_IMPL(cnum)) goto abort_mission;
  3459. req->reg_value = PMC_DFL_VAL(cnum);
  3460. PFM_REG_RETFLAG_SET(req->reg_flags, 0);
  3461. DPRINT(("pmc_reset_val pmc[%u]=0x%lx\n", cnum, req->reg_value));
  3462. }
  3463. return 0;
  3464. abort_mission:
  3465. PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
  3466. return ret;
  3467. }
  3468. static int
  3469. pfm_check_task_exist(pfm_context_t *ctx)
  3470. {
  3471. struct task_struct *g, *t;
  3472. int ret = -ESRCH;
  3473. read_lock(&tasklist_lock);
  3474. do_each_thread (g, t) {
  3475. if (t->thread.pfm_context == ctx) {
  3476. ret = 0;
  3477. goto out;
  3478. }
  3479. } while_each_thread (g, t);
  3480. out:
  3481. read_unlock(&tasklist_lock);
  3482. DPRINT(("pfm_check_task_exist: ret=%d ctx=%p\n", ret, ctx));
  3483. return ret;
  3484. }
  3485. static int
  3486. pfm_context_load(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3487. {
  3488. struct task_struct *task;
  3489. struct thread_struct *thread;
  3490. struct pfm_context_t *old;
  3491. unsigned long flags;
  3492. #ifndef CONFIG_SMP
  3493. struct task_struct *owner_task = NULL;
  3494. #endif
  3495. pfarg_load_t *req = (pfarg_load_t *)arg;
  3496. unsigned long *pmcs_source, *pmds_source;
  3497. int the_cpu;
  3498. int ret = 0;
  3499. int state, is_system, set_dbregs = 0;
  3500. state = ctx->ctx_state;
  3501. is_system = ctx->ctx_fl_system;
  3502. /*
  3503. * can only load from unloaded or terminated state
  3504. */
  3505. if (state != PFM_CTX_UNLOADED) {
  3506. DPRINT(("cannot load to [%d], invalid ctx_state=%d\n",
  3507. req->load_pid,
  3508. ctx->ctx_state));
  3509. return -EBUSY;
  3510. }
  3511. DPRINT(("load_pid [%d] using_dbreg=%d\n", req->load_pid, ctx->ctx_fl_using_dbreg));
  3512. if (CTX_OVFL_NOBLOCK(ctx) == 0 && req->load_pid == current->pid) {
  3513. DPRINT(("cannot use blocking mode on self\n"));
  3514. return -EINVAL;
  3515. }
  3516. ret = pfm_get_task(ctx, req->load_pid, &task);
  3517. if (ret) {
  3518. DPRINT(("load_pid [%d] get_task=%d\n", req->load_pid, ret));
  3519. return ret;
  3520. }
  3521. ret = -EINVAL;
  3522. /*
  3523. * system wide is self monitoring only
  3524. */
  3525. if (is_system && task != current) {
  3526. DPRINT(("system wide is self monitoring only load_pid=%d\n",
  3527. req->load_pid));
  3528. goto error;
  3529. }
  3530. thread = &task->thread;
  3531. ret = 0;
  3532. /*
  3533. * cannot load a context which is using range restrictions,
  3534. * into a task that is being debugged.
  3535. */
  3536. if (ctx->ctx_fl_using_dbreg) {
  3537. if (thread->flags & IA64_THREAD_DBG_VALID) {
  3538. ret = -EBUSY;
  3539. DPRINT(("load_pid [%d] task is debugged, cannot load range restrictions\n", req->load_pid));
  3540. goto error;
  3541. }
  3542. LOCK_PFS(flags);
  3543. if (is_system) {
  3544. if (pfm_sessions.pfs_ptrace_use_dbregs) {
  3545. DPRINT(("cannot load [%d] dbregs in use\n",
  3546. task_pid_nr(task)));
  3547. ret = -EBUSY;
  3548. } else {
  3549. pfm_sessions.pfs_sys_use_dbregs++;
  3550. DPRINT(("load [%d] increased sys_use_dbreg=%u\n", task_pid_nr(task), pfm_sessions.pfs_sys_use_dbregs));
  3551. set_dbregs = 1;
  3552. }
  3553. }
  3554. UNLOCK_PFS(flags);
  3555. if (ret) goto error;
  3556. }
  3557. /*
  3558. * SMP system-wide monitoring implies self-monitoring.
  3559. *
  3560. * The programming model expects the task to
  3561. * be pinned on a CPU throughout the session.
  3562. * Here we take note of the current CPU at the
  3563. * time the context is loaded. No call from
  3564. * another CPU will be allowed.
  3565. *
  3566. * The pinning via shed_setaffinity()
  3567. * must be done by the calling task prior
  3568. * to this call.
  3569. *
  3570. * systemwide: keep track of CPU this session is supposed to run on
  3571. */
  3572. the_cpu = ctx->ctx_cpu = smp_processor_id();
  3573. ret = -EBUSY;
  3574. /*
  3575. * now reserve the session
  3576. */
  3577. ret = pfm_reserve_session(current, is_system, the_cpu);
  3578. if (ret) goto error;
  3579. /*
  3580. * task is necessarily stopped at this point.
  3581. *
  3582. * If the previous context was zombie, then it got removed in
  3583. * pfm_save_regs(). Therefore we should not see it here.
  3584. * If we see a context, then this is an active context
  3585. *
  3586. * XXX: needs to be atomic
  3587. */
  3588. DPRINT(("before cmpxchg() old_ctx=%p new_ctx=%p\n",
  3589. thread->pfm_context, ctx));
  3590. ret = -EBUSY;
  3591. old = ia64_cmpxchg(acq, &thread->pfm_context, NULL, ctx, sizeof(pfm_context_t *));
  3592. if (old != NULL) {
  3593. DPRINT(("load_pid [%d] already has a context\n", req->load_pid));
  3594. goto error_unres;
  3595. }
  3596. pfm_reset_msgq(ctx);
  3597. ctx->ctx_state = PFM_CTX_LOADED;
  3598. /*
  3599. * link context to task
  3600. */
  3601. ctx->ctx_task = task;
  3602. if (is_system) {
  3603. /*
  3604. * we load as stopped
  3605. */
  3606. PFM_CPUINFO_SET(PFM_CPUINFO_SYST_WIDE);
  3607. PFM_CPUINFO_CLEAR(PFM_CPUINFO_DCR_PP);
  3608. if (ctx->ctx_fl_excl_idle) PFM_CPUINFO_SET(PFM_CPUINFO_EXCL_IDLE);
  3609. } else {
  3610. thread->flags |= IA64_THREAD_PM_VALID;
  3611. }
  3612. /*
  3613. * propagate into thread-state
  3614. */
  3615. pfm_copy_pmds(task, ctx);
  3616. pfm_copy_pmcs(task, ctx);
  3617. pmcs_source = ctx->th_pmcs;
  3618. pmds_source = ctx->th_pmds;
  3619. /*
  3620. * always the case for system-wide
  3621. */
  3622. if (task == current) {
  3623. if (is_system == 0) {
  3624. /* allow user level control */
  3625. ia64_psr(regs)->sp = 0;
  3626. DPRINT(("clearing psr.sp for [%d]\n", task_pid_nr(task)));
  3627. SET_LAST_CPU(ctx, smp_processor_id());
  3628. INC_ACTIVATION();
  3629. SET_ACTIVATION(ctx);
  3630. #ifndef CONFIG_SMP
  3631. /*
  3632. * push the other task out, if any
  3633. */
  3634. owner_task = GET_PMU_OWNER();
  3635. if (owner_task) pfm_lazy_save_regs(owner_task);
  3636. #endif
  3637. }
  3638. /*
  3639. * load all PMD from ctx to PMU (as opposed to thread state)
  3640. * restore all PMC from ctx to PMU
  3641. */
  3642. pfm_restore_pmds(pmds_source, ctx->ctx_all_pmds[0]);
  3643. pfm_restore_pmcs(pmcs_source, ctx->ctx_all_pmcs[0]);
  3644. ctx->ctx_reload_pmcs[0] = 0UL;
  3645. ctx->ctx_reload_pmds[0] = 0UL;
  3646. /*
  3647. * guaranteed safe by earlier check against DBG_VALID
  3648. */
  3649. if (ctx->ctx_fl_using_dbreg) {
  3650. pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
  3651. pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
  3652. }
  3653. /*
  3654. * set new ownership
  3655. */
  3656. SET_PMU_OWNER(task, ctx);
  3657. DPRINT(("context loaded on PMU for [%d]\n", task_pid_nr(task)));
  3658. } else {
  3659. /*
  3660. * when not current, task MUST be stopped, so this is safe
  3661. */
  3662. regs = task_pt_regs(task);
  3663. /* force a full reload */
  3664. ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
  3665. SET_LAST_CPU(ctx, -1);
  3666. /* initial saved psr (stopped) */
  3667. ctx->ctx_saved_psr_up = 0UL;
  3668. ia64_psr(regs)->up = ia64_psr(regs)->pp = 0;
  3669. }
  3670. ret = 0;
  3671. error_unres:
  3672. if (ret) pfm_unreserve_session(ctx, ctx->ctx_fl_system, the_cpu);
  3673. error:
  3674. /*
  3675. * we must undo the dbregs setting (for system-wide)
  3676. */
  3677. if (ret && set_dbregs) {
  3678. LOCK_PFS(flags);
  3679. pfm_sessions.pfs_sys_use_dbregs--;
  3680. UNLOCK_PFS(flags);
  3681. }
  3682. /*
  3683. * release task, there is now a link with the context
  3684. */
  3685. if (is_system == 0 && task != current) {
  3686. pfm_put_task(task);
  3687. if (ret == 0) {
  3688. ret = pfm_check_task_exist(ctx);
  3689. if (ret) {
  3690. ctx->ctx_state = PFM_CTX_UNLOADED;
  3691. ctx->ctx_task = NULL;
  3692. }
  3693. }
  3694. }
  3695. return ret;
  3696. }
  3697. /*
  3698. * in this function, we do not need to increase the use count
  3699. * for the task via get_task_struct(), because we hold the
  3700. * context lock. If the task were to disappear while having
  3701. * a context attached, it would go through pfm_exit_thread()
  3702. * which also grabs the context lock and would therefore be blocked
  3703. * until we are here.
  3704. */
  3705. static void pfm_flush_pmds(struct task_struct *, pfm_context_t *ctx);
  3706. static int
  3707. pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
  3708. {
  3709. struct task_struct *task = PFM_CTX_TASK(ctx);
  3710. struct pt_regs *tregs;
  3711. int prev_state, is_system;
  3712. int ret;
  3713. DPRINT(("ctx_state=%d task [%d]\n", ctx->ctx_state, task ? task_pid_nr(task) : -1));
  3714. prev_state = ctx->ctx_state;
  3715. is_system = ctx->ctx_fl_system;
  3716. /*
  3717. * unload only when necessary
  3718. */
  3719. if (prev_state == PFM_CTX_UNLOADED) {
  3720. DPRINT(("ctx_state=%d, nothing to do\n", prev_state));
  3721. return 0;
  3722. }
  3723. /*
  3724. * clear psr and dcr bits
  3725. */
  3726. ret = pfm_stop(ctx, NULL, 0, regs);
  3727. if (ret) return ret;
  3728. ctx->ctx_state = PFM_CTX_UNLOADED;
  3729. /*
  3730. * in system mode, we need to update the PMU directly
  3731. * and the user level state of the caller, which may not
  3732. * necessarily be the creator of the context.
  3733. */
  3734. if (is_system) {
  3735. /*
  3736. * Update cpuinfo
  3737. *
  3738. * local PMU is taken care of in pfm_stop()
  3739. */
  3740. PFM_CPUINFO_CLEAR(PFM_CPUINFO_SYST_WIDE);
  3741. PFM_CPUINFO_CLEAR(PFM_CPUINFO_EXCL_IDLE);
  3742. /*
  3743. * save PMDs in context
  3744. * release ownership
  3745. */
  3746. pfm_flush_pmds(current, ctx);
  3747. /*
  3748. * at this point we are done with the PMU
  3749. * so we can unreserve the resource.
  3750. */
  3751. if (prev_state != PFM_CTX_ZOMBIE)
  3752. pfm_unreserve_session(ctx, 1 , ctx->ctx_cpu);
  3753. /*
  3754. * disconnect context from task
  3755. */
  3756. task->thread.pfm_context = NULL;
  3757. /*
  3758. * disconnect task from context
  3759. */
  3760. ctx->ctx_task = NULL;
  3761. /*
  3762. * There is nothing more to cleanup here.
  3763. */
  3764. return 0;
  3765. }
  3766. /*
  3767. * per-task mode
  3768. */
  3769. tregs = task == current ? regs : task_pt_regs(task);
  3770. if (task == current) {
  3771. /*
  3772. * cancel user level control
  3773. */
  3774. ia64_psr(regs)->sp = 1;
  3775. DPRINT(("setting psr.sp for [%d]\n", task_pid_nr(task)));
  3776. }
  3777. /*
  3778. * save PMDs to context
  3779. * release ownership
  3780. */
  3781. pfm_flush_pmds(task, ctx);
  3782. /*
  3783. * at this point we are done with the PMU
  3784. * so we can unreserve the resource.
  3785. *
  3786. * when state was ZOMBIE, we have already unreserved.
  3787. */
  3788. if (prev_state != PFM_CTX_ZOMBIE)
  3789. pfm_unreserve_session(ctx, 0 , ctx->ctx_cpu);
  3790. /*
  3791. * reset activation counter and psr
  3792. */
  3793. ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
  3794. SET_LAST_CPU(ctx, -1);
  3795. /*
  3796. * PMU state will not be restored
  3797. */
  3798. task->thread.flags &= ~IA64_THREAD_PM_VALID;
  3799. /*
  3800. * break links between context and task
  3801. */
  3802. task->thread.pfm_context = NULL;
  3803. ctx->ctx_task = NULL;
  3804. PFM_SET_WORK_PENDING(task, 0);
  3805. ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE;
  3806. ctx->ctx_fl_can_restart = 0;
  3807. ctx->ctx_fl_going_zombie = 0;
  3808. DPRINT(("disconnected [%d] from context\n", task_pid_nr(task)));
  3809. return 0;
  3810. }
  3811. /*
  3812. * called only from exit_thread(): task == current
  3813. * we come here only if current has a context attached (loaded or masked)
  3814. */
  3815. void
  3816. pfm_exit_thread(struct task_struct *task)
  3817. {
  3818. pfm_context_t *ctx;
  3819. unsigned long flags;
  3820. struct pt_regs *regs = task_pt_regs(task);
  3821. int ret, state;
  3822. int free_ok = 0;
  3823. ctx = PFM_GET_CTX(task);
  3824. PROTECT_CTX(ctx, flags);
  3825. DPRINT(("state=%d task [%d]\n", ctx->ctx_state, task_pid_nr(task)));
  3826. state = ctx->ctx_state;
  3827. switch(state) {
  3828. case PFM_CTX_UNLOADED:
  3829. /*
  3830. * only comes to this function if pfm_context is not NULL, i.e., cannot
  3831. * be in unloaded state
  3832. */
  3833. printk(KERN_ERR "perfmon: pfm_exit_thread [%d] ctx unloaded\n", task_pid_nr(task));
  3834. break;
  3835. case PFM_CTX_LOADED:
  3836. case PFM_CTX_MASKED:
  3837. ret = pfm_context_unload(ctx, NULL, 0, regs);
  3838. if (ret) {
  3839. printk(KERN_ERR "perfmon: pfm_exit_thread [%d] state=%d unload failed %d\n", task_pid_nr(task), state, ret);
  3840. }
  3841. DPRINT(("ctx unloaded for current state was %d\n", state));
  3842. pfm_end_notify_user(ctx);
  3843. break;
  3844. case PFM_CTX_ZOMBIE:
  3845. ret = pfm_context_unload(ctx, NULL, 0, regs);
  3846. if (ret) {
  3847. printk(KERN_ERR "perfmon: pfm_exit_thread [%d] state=%d unload failed %d\n", task_pid_nr(task), state, ret);
  3848. }
  3849. free_ok = 1;
  3850. break;
  3851. default:
  3852. printk(KERN_ERR "perfmon: pfm_exit_thread [%d] unexpected state=%d\n", task_pid_nr(task), state);
  3853. break;
  3854. }
  3855. UNPROTECT_CTX(ctx, flags);
  3856. { u64 psr = pfm_get_psr();
  3857. BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
  3858. BUG_ON(GET_PMU_OWNER());
  3859. BUG_ON(ia64_psr(regs)->up);
  3860. BUG_ON(ia64_psr(regs)->pp);
  3861. }
  3862. /*
  3863. * All memory free operations (especially for vmalloc'ed memory)
  3864. * MUST be done with interrupts ENABLED.
  3865. */
  3866. if (free_ok) pfm_context_free(ctx);
  3867. }
  3868. /*
  3869. * functions MUST be listed in the increasing order of their index (see permfon.h)
  3870. */
  3871. #define PFM_CMD(name, flags, arg_count, arg_type, getsz) { name, #name, flags, arg_count, sizeof(arg_type), getsz }
  3872. #define PFM_CMD_S(name, flags) { name, #name, flags, 0, 0, NULL }
  3873. #define PFM_CMD_PCLRWS (PFM_CMD_FD|PFM_CMD_ARG_RW|PFM_CMD_STOP)
  3874. #define PFM_CMD_PCLRW (PFM_CMD_FD|PFM_CMD_ARG_RW)
  3875. #define PFM_CMD_NONE { NULL, "no-cmd", 0, 0, 0, NULL}
  3876. static pfm_cmd_desc_t pfm_cmd_tab[]={
  3877. /* 0 */PFM_CMD_NONE,
  3878. /* 1 */PFM_CMD(pfm_write_pmcs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
  3879. /* 2 */PFM_CMD(pfm_write_pmds, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
  3880. /* 3 */PFM_CMD(pfm_read_pmds, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
  3881. /* 4 */PFM_CMD_S(pfm_stop, PFM_CMD_PCLRWS),
  3882. /* 5 */PFM_CMD_S(pfm_start, PFM_CMD_PCLRWS),
  3883. /* 6 */PFM_CMD_NONE,
  3884. /* 7 */PFM_CMD_NONE,
  3885. /* 8 */PFM_CMD(pfm_context_create, PFM_CMD_ARG_RW, 1, pfarg_context_t, pfm_ctx_getsize),
  3886. /* 9 */PFM_CMD_NONE,
  3887. /* 10 */PFM_CMD_S(pfm_restart, PFM_CMD_PCLRW),
  3888. /* 11 */PFM_CMD_NONE,
  3889. /* 12 */PFM_CMD(pfm_get_features, PFM_CMD_ARG_RW, 1, pfarg_features_t, NULL),
  3890. /* 13 */PFM_CMD(pfm_debug, 0, 1, unsigned int, NULL),
  3891. /* 14 */PFM_CMD_NONE,
  3892. /* 15 */PFM_CMD(pfm_get_pmc_reset, PFM_CMD_ARG_RW, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
  3893. /* 16 */PFM_CMD(pfm_context_load, PFM_CMD_PCLRWS, 1, pfarg_load_t, NULL),
  3894. /* 17 */PFM_CMD_S(pfm_context_unload, PFM_CMD_PCLRWS),
  3895. /* 18 */PFM_CMD_NONE,
  3896. /* 19 */PFM_CMD_NONE,
  3897. /* 20 */PFM_CMD_NONE,
  3898. /* 21 */PFM_CMD_NONE,
  3899. /* 22 */PFM_CMD_NONE,
  3900. /* 23 */PFM_CMD_NONE,
  3901. /* 24 */PFM_CMD_NONE,
  3902. /* 25 */PFM_CMD_NONE,
  3903. /* 26 */PFM_CMD_NONE,
  3904. /* 27 */PFM_CMD_NONE,
  3905. /* 28 */PFM_CMD_NONE,
  3906. /* 29 */PFM_CMD_NONE,
  3907. /* 30 */PFM_CMD_NONE,
  3908. /* 31 */PFM_CMD_NONE,
  3909. /* 32 */PFM_CMD(pfm_write_ibrs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_dbreg_t, NULL),
  3910. /* 33 */PFM_CMD(pfm_write_dbrs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_dbreg_t, NULL)
  3911. };
  3912. #define PFM_CMD_COUNT (sizeof(pfm_cmd_tab)/sizeof(pfm_cmd_desc_t))
  3913. static int
  3914. pfm_check_task_state(pfm_context_t *ctx, int cmd, unsigned long flags)
  3915. {
  3916. struct task_struct *task;
  3917. int state, old_state;
  3918. recheck:
  3919. state = ctx->ctx_state;
  3920. task = ctx->ctx_task;
  3921. if (task == NULL) {
  3922. DPRINT(("context %d no task, state=%d\n", ctx->ctx_fd, state));
  3923. return 0;
  3924. }
  3925. DPRINT(("context %d state=%d [%d] task_state=%ld must_stop=%d\n",
  3926. ctx->ctx_fd,
  3927. state,
  3928. task_pid_nr(task),
  3929. task->state, PFM_CMD_STOPPED(cmd)));
  3930. /*
  3931. * self-monitoring always ok.
  3932. *
  3933. * for system-wide the caller can either be the creator of the
  3934. * context (to one to which the context is attached to) OR
  3935. * a task running on the same CPU as the session.
  3936. */
  3937. if (task == current || ctx->ctx_fl_system) return 0;
  3938. /*
  3939. * we are monitoring another thread
  3940. */
  3941. switch(state) {
  3942. case PFM_CTX_UNLOADED:
  3943. /*
  3944. * if context is UNLOADED we are safe to go
  3945. */
  3946. return 0;
  3947. case PFM_CTX_ZOMBIE:
  3948. /*
  3949. * no command can operate on a zombie context
  3950. */
  3951. DPRINT(("cmd %d state zombie cannot operate on context\n", cmd));
  3952. return -EINVAL;
  3953. case PFM_CTX_MASKED:
  3954. /*
  3955. * PMU state has been saved to software even though
  3956. * the thread may still be running.
  3957. */
  3958. if (cmd != PFM_UNLOAD_CONTEXT) return 0;
  3959. }
  3960. /*
  3961. * context is LOADED or MASKED. Some commands may need to have
  3962. * the task stopped.
  3963. *
  3964. * We could lift this restriction for UP but it would mean that
  3965. * the user has no guarantee the task would not run between
  3966. * two successive calls to perfmonctl(). That's probably OK.
  3967. * If this user wants to ensure the task does not run, then
  3968. * the task must be stopped.
  3969. */
  3970. if (PFM_CMD_STOPPED(cmd)) {
  3971. if (!task_is_stopped_or_traced(task)) {
  3972. DPRINT(("[%d] task not in stopped state\n", task_pid_nr(task)));
  3973. return -EBUSY;
  3974. }
  3975. /*
  3976. * task is now stopped, wait for ctxsw out
  3977. *
  3978. * This is an interesting point in the code.
  3979. * We need to unprotect the context because
  3980. * the pfm_save_regs() routines needs to grab
  3981. * the same lock. There are danger in doing
  3982. * this because it leaves a window open for
  3983. * another task to get access to the context
  3984. * and possibly change its state. The one thing
  3985. * that is not possible is for the context to disappear
  3986. * because we are protected by the VFS layer, i.e.,
  3987. * get_fd()/put_fd().
  3988. */
  3989. old_state = state;
  3990. UNPROTECT_CTX(ctx, flags);
  3991. wait_task_inactive(task);
  3992. PROTECT_CTX(ctx, flags);
  3993. /*
  3994. * we must recheck to verify if state has changed
  3995. */
  3996. if (ctx->ctx_state != old_state) {
  3997. DPRINT(("old_state=%d new_state=%d\n", old_state, ctx->ctx_state));
  3998. goto recheck;
  3999. }
  4000. }
  4001. return 0;
  4002. }
  4003. /*
  4004. * system-call entry point (must return long)
  4005. */
  4006. asmlinkage long
  4007. sys_perfmonctl (int fd, int cmd, void __user *arg, int count)
  4008. {
  4009. struct file *file = NULL;
  4010. pfm_context_t *ctx = NULL;
  4011. unsigned long flags = 0UL;
  4012. void *args_k = NULL;
  4013. long ret; /* will expand int return types */
  4014. size_t base_sz, sz, xtra_sz = 0;
  4015. int narg, completed_args = 0, call_made = 0, cmd_flags;
  4016. int (*func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
  4017. int (*getsize)(void *arg, size_t *sz);
  4018. #define PFM_MAX_ARGSIZE 4096
  4019. /*
  4020. * reject any call if perfmon was disabled at initialization
  4021. */
  4022. if (unlikely(pmu_conf == NULL)) return -ENOSYS;
  4023. if (unlikely(cmd < 0 || cmd >= PFM_CMD_COUNT)) {
  4024. DPRINT(("invalid cmd=%d\n", cmd));
  4025. return -EINVAL;
  4026. }
  4027. func = pfm_cmd_tab[cmd].cmd_func;
  4028. narg = pfm_cmd_tab[cmd].cmd_narg;
  4029. base_sz = pfm_cmd_tab[cmd].cmd_argsize;
  4030. getsize = pfm_cmd_tab[cmd].cmd_getsize;
  4031. cmd_flags = pfm_cmd_tab[cmd].cmd_flags;
  4032. if (unlikely(func == NULL)) {
  4033. DPRINT(("invalid cmd=%d\n", cmd));
  4034. return -EINVAL;
  4035. }
  4036. DPRINT(("cmd=%s idx=%d narg=0x%x argsz=%lu count=%d\n",
  4037. PFM_CMD_NAME(cmd),
  4038. cmd,
  4039. narg,
  4040. base_sz,
  4041. count));
  4042. /*
  4043. * check if number of arguments matches what the command expects
  4044. */
  4045. if (unlikely((narg == PFM_CMD_ARG_MANY && count <= 0) || (narg > 0 && narg != count)))
  4046. return -EINVAL;
  4047. restart_args:
  4048. sz = xtra_sz + base_sz*count;
  4049. /*
  4050. * limit abuse to min page size
  4051. */
  4052. if (unlikely(sz > PFM_MAX_ARGSIZE)) {
  4053. printk(KERN_ERR "perfmon: [%d] argument too big %lu\n", task_pid_nr(current), sz);
  4054. return -E2BIG;
  4055. }
  4056. /*
  4057. * allocate default-sized argument buffer
  4058. */
  4059. if (likely(count && args_k == NULL)) {
  4060. args_k = kmalloc(PFM_MAX_ARGSIZE, GFP_KERNEL);
  4061. if (args_k == NULL) return -ENOMEM;
  4062. }
  4063. ret = -EFAULT;
  4064. /*
  4065. * copy arguments
  4066. *
  4067. * assume sz = 0 for command without parameters
  4068. */
  4069. if (sz && copy_from_user(args_k, arg, sz)) {
  4070. DPRINT(("cannot copy_from_user %lu bytes @%p\n", sz, arg));
  4071. goto error_args;
  4072. }
  4073. /*
  4074. * check if command supports extra parameters
  4075. */
  4076. if (completed_args == 0 && getsize) {
  4077. /*
  4078. * get extra parameters size (based on main argument)
  4079. */
  4080. ret = (*getsize)(args_k, &xtra_sz);
  4081. if (ret) goto error_args;
  4082. completed_args = 1;
  4083. DPRINT(("restart_args sz=%lu xtra_sz=%lu\n", sz, xtra_sz));
  4084. /* retry if necessary */
  4085. if (likely(xtra_sz)) goto restart_args;
  4086. }
  4087. if (unlikely((cmd_flags & PFM_CMD_FD) == 0)) goto skip_fd;
  4088. ret = -EBADF;
  4089. file = fget(fd);
  4090. if (unlikely(file == NULL)) {
  4091. DPRINT(("invalid fd %d\n", fd));
  4092. goto error_args;
  4093. }
  4094. if (unlikely(PFM_IS_FILE(file) == 0)) {
  4095. DPRINT(("fd %d not related to perfmon\n", fd));
  4096. goto error_args;
  4097. }
  4098. ctx = (pfm_context_t *)file->private_data;
  4099. if (unlikely(ctx == NULL)) {
  4100. DPRINT(("no context for fd %d\n", fd));
  4101. goto error_args;
  4102. }
  4103. prefetch(&ctx->ctx_state);
  4104. PROTECT_CTX(ctx, flags);
  4105. /*
  4106. * check task is stopped
  4107. */
  4108. ret = pfm_check_task_state(ctx, cmd, flags);
  4109. if (unlikely(ret)) goto abort_locked;
  4110. skip_fd:
  4111. ret = (*func)(ctx, args_k, count, task_pt_regs(current));
  4112. call_made = 1;
  4113. abort_locked:
  4114. if (likely(ctx)) {
  4115. DPRINT(("context unlocked\n"));
  4116. UNPROTECT_CTX(ctx, flags);
  4117. }
  4118. /* copy argument back to user, if needed */
  4119. if (call_made && PFM_CMD_RW_ARG(cmd) && copy_to_user(arg, args_k, base_sz*count)) ret = -EFAULT;
  4120. error_args:
  4121. if (file)
  4122. fput(file);
  4123. kfree(args_k);
  4124. DPRINT(("cmd=%s ret=%ld\n", PFM_CMD_NAME(cmd), ret));
  4125. return ret;
  4126. }
  4127. static void
  4128. pfm_resume_after_ovfl(pfm_context_t *ctx, unsigned long ovfl_regs, struct pt_regs *regs)
  4129. {
  4130. pfm_buffer_fmt_t *fmt = ctx->ctx_buf_fmt;
  4131. pfm_ovfl_ctrl_t rst_ctrl;
  4132. int state;
  4133. int ret = 0;
  4134. state = ctx->ctx_state;
  4135. /*
  4136. * Unlock sampling buffer and reset index atomically
  4137. * XXX: not really needed when blocking
  4138. */
  4139. if (CTX_HAS_SMPL(ctx)) {
  4140. rst_ctrl.bits.mask_monitoring = 0;
  4141. rst_ctrl.bits.reset_ovfl_pmds = 0;
  4142. if (state == PFM_CTX_LOADED)
  4143. ret = pfm_buf_fmt_restart_active(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
  4144. else
  4145. ret = pfm_buf_fmt_restart(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
  4146. } else {
  4147. rst_ctrl.bits.mask_monitoring = 0;
  4148. rst_ctrl.bits.reset_ovfl_pmds = 1;
  4149. }
  4150. if (ret == 0) {
  4151. if (rst_ctrl.bits.reset_ovfl_pmds) {
  4152. pfm_reset_regs(ctx, &ovfl_regs, PFM_PMD_LONG_RESET);
  4153. }
  4154. if (rst_ctrl.bits.mask_monitoring == 0) {
  4155. DPRINT(("resuming monitoring\n"));
  4156. if (ctx->ctx_state == PFM_CTX_MASKED) pfm_restore_monitoring(current);
  4157. } else {
  4158. DPRINT(("stopping monitoring\n"));
  4159. //pfm_stop_monitoring(current, regs);
  4160. }
  4161. ctx->ctx_state = PFM_CTX_LOADED;
  4162. }
  4163. }
  4164. /*
  4165. * context MUST BE LOCKED when calling
  4166. * can only be called for current
  4167. */
  4168. static void
  4169. pfm_context_force_terminate(pfm_context_t *ctx, struct pt_regs *regs)
  4170. {
  4171. int ret;
  4172. DPRINT(("entering for [%d]\n", task_pid_nr(current)));
  4173. ret = pfm_context_unload(ctx, NULL, 0, regs);
  4174. if (ret) {
  4175. printk(KERN_ERR "pfm_context_force_terminate: [%d] unloaded failed with %d\n", task_pid_nr(current), ret);
  4176. }
  4177. /*
  4178. * and wakeup controlling task, indicating we are now disconnected
  4179. */
  4180. wake_up_interruptible(&ctx->ctx_zombieq);
  4181. /*
  4182. * given that context is still locked, the controlling
  4183. * task will only get access when we return from
  4184. * pfm_handle_work().
  4185. */
  4186. }
  4187. static int pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds);
  4188. /*
  4189. * pfm_handle_work() can be called with interrupts enabled
  4190. * (TIF_NEED_RESCHED) or disabled. The down_interruptible
  4191. * call may sleep, therefore we must re-enable interrupts
  4192. * to avoid deadlocks. It is safe to do so because this function
  4193. * is called ONLY when returning to user level (PUStk=1), in which case
  4194. * there is no risk of kernel stack overflow due to deep
  4195. * interrupt nesting.
  4196. */
  4197. void
  4198. pfm_handle_work(void)
  4199. {
  4200. pfm_context_t *ctx;
  4201. struct pt_regs *regs;
  4202. unsigned long flags, dummy_flags;
  4203. unsigned long ovfl_regs;
  4204. unsigned int reason;
  4205. int ret;
  4206. ctx = PFM_GET_CTX(current);
  4207. if (ctx == NULL) {
  4208. printk(KERN_ERR "perfmon: [%d] has no PFM context\n", task_pid_nr(current));
  4209. return;
  4210. }
  4211. PROTECT_CTX(ctx, flags);
  4212. PFM_SET_WORK_PENDING(current, 0);
  4213. tsk_clear_notify_resume(current);
  4214. regs = task_pt_regs(current);
  4215. /*
  4216. * extract reason for being here and clear
  4217. */
  4218. reason = ctx->ctx_fl_trap_reason;
  4219. ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE;
  4220. ovfl_regs = ctx->ctx_ovfl_regs[0];
  4221. DPRINT(("reason=%d state=%d\n", reason, ctx->ctx_state));
  4222. /*
  4223. * must be done before we check for simple-reset mode
  4224. */
  4225. if (ctx->ctx_fl_going_zombie || ctx->ctx_state == PFM_CTX_ZOMBIE) goto do_zombie;
  4226. //if (CTX_OVFL_NOBLOCK(ctx)) goto skip_blocking;
  4227. if (reason == PFM_TRAP_REASON_RESET) goto skip_blocking;
  4228. /*
  4229. * restore interrupt mask to what it was on entry.
  4230. * Could be enabled/diasbled.
  4231. */
  4232. UNPROTECT_CTX(ctx, flags);
  4233. /*
  4234. * force interrupt enable because of down_interruptible()
  4235. */
  4236. local_irq_enable();
  4237. DPRINT(("before block sleeping\n"));
  4238. /*
  4239. * may go through without blocking on SMP systems
  4240. * if restart has been received already by the time we call down()
  4241. */
  4242. ret = wait_for_completion_interruptible(&ctx->ctx_restart_done);
  4243. DPRINT(("after block sleeping ret=%d\n", ret));
  4244. /*
  4245. * lock context and mask interrupts again
  4246. * We save flags into a dummy because we may have
  4247. * altered interrupts mask compared to entry in this
  4248. * function.
  4249. */
  4250. PROTECT_CTX(ctx, dummy_flags);
  4251. /*
  4252. * we need to read the ovfl_regs only after wake-up
  4253. * because we may have had pfm_write_pmds() in between
  4254. * and that can changed PMD values and therefore
  4255. * ovfl_regs is reset for these new PMD values.
  4256. */
  4257. ovfl_regs = ctx->ctx_ovfl_regs[0];
  4258. if (ctx->ctx_fl_going_zombie) {
  4259. do_zombie:
  4260. DPRINT(("context is zombie, bailing out\n"));
  4261. pfm_context_force_terminate(ctx, regs);
  4262. goto nothing_to_do;
  4263. }
  4264. /*
  4265. * in case of interruption of down() we don't restart anything
  4266. */
  4267. if (ret < 0) goto nothing_to_do;
  4268. skip_blocking:
  4269. pfm_resume_after_ovfl(ctx, ovfl_regs, regs);
  4270. ctx->ctx_ovfl_regs[0] = 0UL;
  4271. nothing_to_do:
  4272. /*
  4273. * restore flags as they were upon entry
  4274. */
  4275. UNPROTECT_CTX(ctx, flags);
  4276. }
  4277. static int
  4278. pfm_notify_user(pfm_context_t *ctx, pfm_msg_t *msg)
  4279. {
  4280. if (ctx->ctx_state == PFM_CTX_ZOMBIE) {
  4281. DPRINT(("ignoring overflow notification, owner is zombie\n"));
  4282. return 0;
  4283. }
  4284. DPRINT(("waking up somebody\n"));
  4285. if (msg) wake_up_interruptible(&ctx->ctx_msgq_wait);
  4286. /*
  4287. * safe, we are not in intr handler, nor in ctxsw when
  4288. * we come here
  4289. */
  4290. kill_fasync (&ctx->ctx_async_queue, SIGIO, POLL_IN);
  4291. return 0;
  4292. }
  4293. static int
  4294. pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds)
  4295. {
  4296. pfm_msg_t *msg = NULL;
  4297. if (ctx->ctx_fl_no_msg == 0) {
  4298. msg = pfm_get_new_msg(ctx);
  4299. if (msg == NULL) {
  4300. printk(KERN_ERR "perfmon: pfm_ovfl_notify_user no more notification msgs\n");
  4301. return -1;
  4302. }
  4303. msg->pfm_ovfl_msg.msg_type = PFM_MSG_OVFL;
  4304. msg->pfm_ovfl_msg.msg_ctx_fd = ctx->ctx_fd;
  4305. msg->pfm_ovfl_msg.msg_active_set = 0;
  4306. msg->pfm_ovfl_msg.msg_ovfl_pmds[0] = ovfl_pmds;
  4307. msg->pfm_ovfl_msg.msg_ovfl_pmds[1] = 0UL;
  4308. msg->pfm_ovfl_msg.msg_ovfl_pmds[2] = 0UL;
  4309. msg->pfm_ovfl_msg.msg_ovfl_pmds[3] = 0UL;
  4310. msg->pfm_ovfl_msg.msg_tstamp = 0UL;
  4311. }
  4312. DPRINT(("ovfl msg: msg=%p no_msg=%d fd=%d ovfl_pmds=0x%lx\n",
  4313. msg,
  4314. ctx->ctx_fl_no_msg,
  4315. ctx->ctx_fd,
  4316. ovfl_pmds));
  4317. return pfm_notify_user(ctx, msg);
  4318. }
  4319. static int
  4320. pfm_end_notify_user(pfm_context_t *ctx)
  4321. {
  4322. pfm_msg_t *msg;
  4323. msg = pfm_get_new_msg(ctx);
  4324. if (msg == NULL) {
  4325. printk(KERN_ERR "perfmon: pfm_end_notify_user no more notification msgs\n");
  4326. return -1;
  4327. }
  4328. /* no leak */
  4329. memset(msg, 0, sizeof(*msg));
  4330. msg->pfm_end_msg.msg_type = PFM_MSG_END;
  4331. msg->pfm_end_msg.msg_ctx_fd = ctx->ctx_fd;
  4332. msg->pfm_ovfl_msg.msg_tstamp = 0UL;
  4333. DPRINT(("end msg: msg=%p no_msg=%d ctx_fd=%d\n",
  4334. msg,
  4335. ctx->ctx_fl_no_msg,
  4336. ctx->ctx_fd));
  4337. return pfm_notify_user(ctx, msg);
  4338. }
  4339. /*
  4340. * main overflow processing routine.
  4341. * it can be called from the interrupt path or explicitly during the context switch code
  4342. */
  4343. static void
  4344. pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx, u64 pmc0, struct pt_regs *regs)
  4345. {
  4346. pfm_ovfl_arg_t *ovfl_arg;
  4347. unsigned long mask;
  4348. unsigned long old_val, ovfl_val, new_val;
  4349. unsigned long ovfl_notify = 0UL, ovfl_pmds = 0UL, smpl_pmds = 0UL, reset_pmds;
  4350. unsigned long tstamp;
  4351. pfm_ovfl_ctrl_t ovfl_ctrl;
  4352. unsigned int i, has_smpl;
  4353. int must_notify = 0;
  4354. if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) goto stop_monitoring;
  4355. /*
  4356. * sanity test. Should never happen
  4357. */
  4358. if (unlikely((pmc0 & 0x1) == 0)) goto sanity_check;
  4359. tstamp = ia64_get_itc();
  4360. mask = pmc0 >> PMU_FIRST_COUNTER;
  4361. ovfl_val = pmu_conf->ovfl_val;
  4362. has_smpl = CTX_HAS_SMPL(ctx);
  4363. DPRINT_ovfl(("pmc0=0x%lx pid=%d iip=0x%lx, %s "
  4364. "used_pmds=0x%lx\n",
  4365. pmc0,
  4366. task ? task_pid_nr(task): -1,
  4367. (regs ? regs->cr_iip : 0),
  4368. CTX_OVFL_NOBLOCK(ctx) ? "nonblocking" : "blocking",
  4369. ctx->ctx_used_pmds[0]));
  4370. /*
  4371. * first we update the virtual counters
  4372. * assume there was a prior ia64_srlz_d() issued
  4373. */
  4374. for (i = PMU_FIRST_COUNTER; mask ; i++, mask >>= 1) {
  4375. /* skip pmd which did not overflow */
  4376. if ((mask & 0x1) == 0) continue;
  4377. /*
  4378. * Note that the pmd is not necessarily 0 at this point as qualified events
  4379. * may have happened before the PMU was frozen. The residual count is not
  4380. * taken into consideration here but will be with any read of the pmd via
  4381. * pfm_read_pmds().
  4382. */
  4383. old_val = new_val = ctx->ctx_pmds[i].val;
  4384. new_val += 1 + ovfl_val;
  4385. ctx->ctx_pmds[i].val = new_val;
  4386. /*
  4387. * check for overflow condition
  4388. */
  4389. if (likely(old_val > new_val)) {
  4390. ovfl_pmds |= 1UL << i;
  4391. if (PMC_OVFL_NOTIFY(ctx, i)) ovfl_notify |= 1UL << i;
  4392. }
  4393. DPRINT_ovfl(("ctx_pmd[%d].val=0x%lx old_val=0x%lx pmd=0x%lx ovfl_pmds=0x%lx ovfl_notify=0x%lx\n",
  4394. i,
  4395. new_val,
  4396. old_val,
  4397. ia64_get_pmd(i) & ovfl_val,
  4398. ovfl_pmds,
  4399. ovfl_notify));
  4400. }
  4401. /*
  4402. * there was no 64-bit overflow, nothing else to do
  4403. */
  4404. if (ovfl_pmds == 0UL) return;
  4405. /*
  4406. * reset all control bits
  4407. */
  4408. ovfl_ctrl.val = 0;
  4409. reset_pmds = 0UL;
  4410. /*
  4411. * if a sampling format module exists, then we "cache" the overflow by
  4412. * calling the module's handler() routine.
  4413. */
  4414. if (has_smpl) {
  4415. unsigned long start_cycles, end_cycles;
  4416. unsigned long pmd_mask;
  4417. int j, k, ret = 0;
  4418. int this_cpu = smp_processor_id();
  4419. pmd_mask = ovfl_pmds >> PMU_FIRST_COUNTER;
  4420. ovfl_arg = &ctx->ctx_ovfl_arg;
  4421. prefetch(ctx->ctx_smpl_hdr);
  4422. for(i=PMU_FIRST_COUNTER; pmd_mask && ret == 0; i++, pmd_mask >>=1) {
  4423. mask = 1UL << i;
  4424. if ((pmd_mask & 0x1) == 0) continue;
  4425. ovfl_arg->ovfl_pmd = (unsigned char )i;
  4426. ovfl_arg->ovfl_notify = ovfl_notify & mask ? 1 : 0;
  4427. ovfl_arg->active_set = 0;
  4428. ovfl_arg->ovfl_ctrl.val = 0; /* module must fill in all fields */
  4429. ovfl_arg->smpl_pmds[0] = smpl_pmds = ctx->ctx_pmds[i].smpl_pmds[0];
  4430. ovfl_arg->pmd_value = ctx->ctx_pmds[i].val;
  4431. ovfl_arg->pmd_last_reset = ctx->ctx_pmds[i].lval;
  4432. ovfl_arg->pmd_eventid = ctx->ctx_pmds[i].eventid;
  4433. /*
  4434. * copy values of pmds of interest. Sampling format may copy them
  4435. * into sampling buffer.
  4436. */
  4437. if (smpl_pmds) {
  4438. for(j=0, k=0; smpl_pmds; j++, smpl_pmds >>=1) {
  4439. if ((smpl_pmds & 0x1) == 0) continue;
  4440. ovfl_arg->smpl_pmds_values[k++] = PMD_IS_COUNTING(j) ? pfm_read_soft_counter(ctx, j) : ia64_get_pmd(j);
  4441. DPRINT_ovfl(("smpl_pmd[%d]=pmd%u=0x%lx\n", k-1, j, ovfl_arg->smpl_pmds_values[k-1]));
  4442. }
  4443. }
  4444. pfm_stats[this_cpu].pfm_smpl_handler_calls++;
  4445. start_cycles = ia64_get_itc();
  4446. /*
  4447. * call custom buffer format record (handler) routine
  4448. */
  4449. ret = (*ctx->ctx_buf_fmt->fmt_handler)(task, ctx->ctx_smpl_hdr, ovfl_arg, regs, tstamp);
  4450. end_cycles = ia64_get_itc();
  4451. /*
  4452. * For those controls, we take the union because they have
  4453. * an all or nothing behavior.
  4454. */
  4455. ovfl_ctrl.bits.notify_user |= ovfl_arg->ovfl_ctrl.bits.notify_user;
  4456. ovfl_ctrl.bits.block_task |= ovfl_arg->ovfl_ctrl.bits.block_task;
  4457. ovfl_ctrl.bits.mask_monitoring |= ovfl_arg->ovfl_ctrl.bits.mask_monitoring;
  4458. /*
  4459. * build the bitmask of pmds to reset now
  4460. */
  4461. if (ovfl_arg->ovfl_ctrl.bits.reset_ovfl_pmds) reset_pmds |= mask;
  4462. pfm_stats[this_cpu].pfm_smpl_handler_cycles += end_cycles - start_cycles;
  4463. }
  4464. /*
  4465. * when the module cannot handle the rest of the overflows, we abort right here
  4466. */
  4467. if (ret && pmd_mask) {
  4468. DPRINT(("handler aborts leftover ovfl_pmds=0x%lx\n",
  4469. pmd_mask<<PMU_FIRST_COUNTER));
  4470. }
  4471. /*
  4472. * remove the pmds we reset now from the set of pmds to reset in pfm_restart()
  4473. */
  4474. ovfl_pmds &= ~reset_pmds;
  4475. } else {
  4476. /*
  4477. * when no sampling module is used, then the default
  4478. * is to notify on overflow if requested by user
  4479. */
  4480. ovfl_ctrl.bits.notify_user = ovfl_notify ? 1 : 0;
  4481. ovfl_ctrl.bits.block_task = ovfl_notify ? 1 : 0;
  4482. ovfl_ctrl.bits.mask_monitoring = ovfl_notify ? 1 : 0; /* XXX: change for saturation */
  4483. ovfl_ctrl.bits.reset_ovfl_pmds = ovfl_notify ? 0 : 1;
  4484. /*
  4485. * if needed, we reset all overflowed pmds
  4486. */
  4487. if (ovfl_notify == 0) reset_pmds = ovfl_pmds;
  4488. }
  4489. DPRINT_ovfl(("ovfl_pmds=0x%lx reset_pmds=0x%lx\n", ovfl_pmds, reset_pmds));
  4490. /*
  4491. * reset the requested PMD registers using the short reset values
  4492. */
  4493. if (reset_pmds) {
  4494. unsigned long bm = reset_pmds;
  4495. pfm_reset_regs(ctx, &bm, PFM_PMD_SHORT_RESET);
  4496. }
  4497. if (ovfl_notify && ovfl_ctrl.bits.notify_user) {
  4498. /*
  4499. * keep track of what to reset when unblocking
  4500. */
  4501. ctx->ctx_ovfl_regs[0] = ovfl_pmds;
  4502. /*
  4503. * check for blocking context
  4504. */
  4505. if (CTX_OVFL_NOBLOCK(ctx) == 0 && ovfl_ctrl.bits.block_task) {
  4506. ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_BLOCK;
  4507. /*
  4508. * set the perfmon specific checking pending work for the task
  4509. */
  4510. PFM_SET_WORK_PENDING(task, 1);
  4511. /*
  4512. * when coming from ctxsw, current still points to the
  4513. * previous task, therefore we must work with task and not current.
  4514. */
  4515. tsk_set_notify_resume(task);
  4516. }
  4517. /*
  4518. * defer until state is changed (shorten spin window). the context is locked
  4519. * anyway, so the signal receiver would come spin for nothing.
  4520. */
  4521. must_notify = 1;
  4522. }
  4523. DPRINT_ovfl(("owner [%d] pending=%ld reason=%u ovfl_pmds=0x%lx ovfl_notify=0x%lx masked=%d\n",
  4524. GET_PMU_OWNER() ? task_pid_nr(GET_PMU_OWNER()) : -1,
  4525. PFM_GET_WORK_PENDING(task),
  4526. ctx->ctx_fl_trap_reason,
  4527. ovfl_pmds,
  4528. ovfl_notify,
  4529. ovfl_ctrl.bits.mask_monitoring ? 1 : 0));
  4530. /*
  4531. * in case monitoring must be stopped, we toggle the psr bits
  4532. */
  4533. if (ovfl_ctrl.bits.mask_monitoring) {
  4534. pfm_mask_monitoring(task);
  4535. ctx->ctx_state = PFM_CTX_MASKED;
  4536. ctx->ctx_fl_can_restart = 1;
  4537. }
  4538. /*
  4539. * send notification now
  4540. */
  4541. if (must_notify) pfm_ovfl_notify_user(ctx, ovfl_notify);
  4542. return;
  4543. sanity_check:
  4544. printk(KERN_ERR "perfmon: CPU%d overflow handler [%d] pmc0=0x%lx\n",
  4545. smp_processor_id(),
  4546. task ? task_pid_nr(task) : -1,
  4547. pmc0);
  4548. return;
  4549. stop_monitoring:
  4550. /*
  4551. * in SMP, zombie context is never restored but reclaimed in pfm_load_regs().
  4552. * Moreover, zombies are also reclaimed in pfm_save_regs(). Therefore we can
  4553. * come here as zombie only if the task is the current task. In which case, we
  4554. * can access the PMU hardware directly.
  4555. *
  4556. * Note that zombies do have PM_VALID set. So here we do the minimal.
  4557. *
  4558. * In case the context was zombified it could not be reclaimed at the time
  4559. * the monitoring program exited. At this point, the PMU reservation has been
  4560. * returned, the sampiing buffer has been freed. We must convert this call
  4561. * into a spurious interrupt. However, we must also avoid infinite overflows
  4562. * by stopping monitoring for this task. We can only come here for a per-task
  4563. * context. All we need to do is to stop monitoring using the psr bits which
  4564. * are always task private. By re-enabling secure montioring, we ensure that
  4565. * the monitored task will not be able to re-activate monitoring.
  4566. * The task will eventually be context switched out, at which point the context
  4567. * will be reclaimed (that includes releasing ownership of the PMU).
  4568. *
  4569. * So there might be a window of time where the number of per-task session is zero
  4570. * yet one PMU might have a owner and get at most one overflow interrupt for a zombie
  4571. * context. This is safe because if a per-task session comes in, it will push this one
  4572. * out and by the virtue on pfm_save_regs(), this one will disappear. If a system wide
  4573. * session is force on that CPU, given that we use task pinning, pfm_save_regs() will
  4574. * also push our zombie context out.
  4575. *
  4576. * Overall pretty hairy stuff....
  4577. */
  4578. DPRINT(("ctx is zombie for [%d], converted to spurious\n", task ? task_pid_nr(task): -1));
  4579. pfm_clear_psr_up();
  4580. ia64_psr(regs)->up = 0;
  4581. ia64_psr(regs)->sp = 1;
  4582. return;
  4583. }
  4584. static int
  4585. pfm_do_interrupt_handler(void *arg, struct pt_regs *regs)
  4586. {
  4587. struct task_struct *task;
  4588. pfm_context_t *ctx;
  4589. unsigned long flags;
  4590. u64 pmc0;
  4591. int this_cpu = smp_processor_id();
  4592. int retval = 0;
  4593. pfm_stats[this_cpu].pfm_ovfl_intr_count++;
  4594. /*
  4595. * srlz.d done before arriving here
  4596. */
  4597. pmc0 = ia64_get_pmc(0);
  4598. task = GET_PMU_OWNER();
  4599. ctx = GET_PMU_CTX();
  4600. /*
  4601. * if we have some pending bits set
  4602. * assumes : if any PMC0.bit[63-1] is set, then PMC0.fr = 1
  4603. */
  4604. if (PMC0_HAS_OVFL(pmc0) && task) {
  4605. /*
  4606. * we assume that pmc0.fr is always set here
  4607. */
  4608. /* sanity check */
  4609. if (!ctx) goto report_spurious1;
  4610. if (ctx->ctx_fl_system == 0 && (task->thread.flags & IA64_THREAD_PM_VALID) == 0)
  4611. goto report_spurious2;
  4612. PROTECT_CTX_NOPRINT(ctx, flags);
  4613. pfm_overflow_handler(task, ctx, pmc0, regs);
  4614. UNPROTECT_CTX_NOPRINT(ctx, flags);
  4615. } else {
  4616. pfm_stats[this_cpu].pfm_spurious_ovfl_intr_count++;
  4617. retval = -1;
  4618. }
  4619. /*
  4620. * keep it unfrozen at all times
  4621. */
  4622. pfm_unfreeze_pmu();
  4623. return retval;
  4624. report_spurious1:
  4625. printk(KERN_INFO "perfmon: spurious overflow interrupt on CPU%d: process %d has no PFM context\n",
  4626. this_cpu, task_pid_nr(task));
  4627. pfm_unfreeze_pmu();
  4628. return -1;
  4629. report_spurious2:
  4630. printk(KERN_INFO "perfmon: spurious overflow interrupt on CPU%d: process %d, invalid flag\n",
  4631. this_cpu,
  4632. task_pid_nr(task));
  4633. pfm_unfreeze_pmu();
  4634. return -1;
  4635. }
  4636. static irqreturn_t
  4637. pfm_interrupt_handler(int irq, void *arg)
  4638. {
  4639. unsigned long start_cycles, total_cycles;
  4640. unsigned long min, max;
  4641. int this_cpu;
  4642. int ret;
  4643. struct pt_regs *regs = get_irq_regs();
  4644. this_cpu = get_cpu();
  4645. if (likely(!pfm_alt_intr_handler)) {
  4646. min = pfm_stats[this_cpu].pfm_ovfl_intr_cycles_min;
  4647. max = pfm_stats[this_cpu].pfm_ovfl_intr_cycles_max;
  4648. start_cycles = ia64_get_itc();
  4649. ret = pfm_do_interrupt_handler(arg, regs);
  4650. total_cycles = ia64_get_itc();
  4651. /*
  4652. * don't measure spurious interrupts
  4653. */
  4654. if (likely(ret == 0)) {
  4655. total_cycles -= start_cycles;
  4656. if (total_cycles < min) pfm_stats[this_cpu].pfm_ovfl_intr_cycles_min = total_cycles;
  4657. if (total_cycles > max) pfm_stats[this_cpu].pfm_ovfl_intr_cycles_max = total_cycles;
  4658. pfm_stats[this_cpu].pfm_ovfl_intr_cycles += total_cycles;
  4659. }
  4660. }
  4661. else {
  4662. (*pfm_alt_intr_handler->handler)(irq, arg, regs);
  4663. }
  4664. put_cpu_no_resched();
  4665. return IRQ_HANDLED;
  4666. }
  4667. /*
  4668. * /proc/perfmon interface, for debug only
  4669. */
  4670. #define PFM_PROC_SHOW_HEADER ((void *)NR_CPUS+1)
  4671. static void *
  4672. pfm_proc_start(struct seq_file *m, loff_t *pos)
  4673. {
  4674. if (*pos == 0) {
  4675. return PFM_PROC_SHOW_HEADER;
  4676. }
  4677. while (*pos <= NR_CPUS) {
  4678. if (cpu_online(*pos - 1)) {
  4679. return (void *)*pos;
  4680. }
  4681. ++*pos;
  4682. }
  4683. return NULL;
  4684. }
  4685. static void *
  4686. pfm_proc_next(struct seq_file *m, void *v, loff_t *pos)
  4687. {
  4688. ++*pos;
  4689. return pfm_proc_start(m, pos);
  4690. }
  4691. static void
  4692. pfm_proc_stop(struct seq_file *m, void *v)
  4693. {
  4694. }
  4695. static void
  4696. pfm_proc_show_header(struct seq_file *m)
  4697. {
  4698. struct list_head * pos;
  4699. pfm_buffer_fmt_t * entry;
  4700. unsigned long flags;
  4701. seq_printf(m,
  4702. "perfmon version : %u.%u\n"
  4703. "model : %s\n"
  4704. "fastctxsw : %s\n"
  4705. "expert mode : %s\n"
  4706. "ovfl_mask : 0x%lx\n"
  4707. "PMU flags : 0x%x\n",
  4708. PFM_VERSION_MAJ, PFM_VERSION_MIN,
  4709. pmu_conf->pmu_name,
  4710. pfm_sysctl.fastctxsw > 0 ? "Yes": "No",
  4711. pfm_sysctl.expert_mode > 0 ? "Yes": "No",
  4712. pmu_conf->ovfl_val,
  4713. pmu_conf->flags);
  4714. LOCK_PFS(flags);
  4715. seq_printf(m,
  4716. "proc_sessions : %u\n"
  4717. "sys_sessions : %u\n"
  4718. "sys_use_dbregs : %u\n"
  4719. "ptrace_use_dbregs : %u\n",
  4720. pfm_sessions.pfs_task_sessions,
  4721. pfm_sessions.pfs_sys_sessions,
  4722. pfm_sessions.pfs_sys_use_dbregs,
  4723. pfm_sessions.pfs_ptrace_use_dbregs);
  4724. UNLOCK_PFS(flags);
  4725. spin_lock(&pfm_buffer_fmt_lock);
  4726. list_for_each(pos, &pfm_buffer_fmt_list) {
  4727. entry = list_entry(pos, pfm_buffer_fmt_t, fmt_list);
  4728. seq_printf(m, "format : %02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x %s\n",
  4729. entry->fmt_uuid[0],
  4730. entry->fmt_uuid[1],
  4731. entry->fmt_uuid[2],
  4732. entry->fmt_uuid[3],
  4733. entry->fmt_uuid[4],
  4734. entry->fmt_uuid[5],
  4735. entry->fmt_uuid[6],
  4736. entry->fmt_uuid[7],
  4737. entry->fmt_uuid[8],
  4738. entry->fmt_uuid[9],
  4739. entry->fmt_uuid[10],
  4740. entry->fmt_uuid[11],
  4741. entry->fmt_uuid[12],
  4742. entry->fmt_uuid[13],
  4743. entry->fmt_uuid[14],
  4744. entry->fmt_uuid[15],
  4745. entry->fmt_name);
  4746. }
  4747. spin_unlock(&pfm_buffer_fmt_lock);
  4748. }
  4749. static int
  4750. pfm_proc_show(struct seq_file *m, void *v)
  4751. {
  4752. unsigned long psr;
  4753. unsigned int i;
  4754. int cpu;
  4755. if (v == PFM_PROC_SHOW_HEADER) {
  4756. pfm_proc_show_header(m);
  4757. return 0;
  4758. }
  4759. /* show info for CPU (v - 1) */
  4760. cpu = (long)v - 1;
  4761. seq_printf(m,
  4762. "CPU%-2d overflow intrs : %lu\n"
  4763. "CPU%-2d overflow cycles : %lu\n"
  4764. "CPU%-2d overflow min : %lu\n"
  4765. "CPU%-2d overflow max : %lu\n"
  4766. "CPU%-2d smpl handler calls : %lu\n"
  4767. "CPU%-2d smpl handler cycles : %lu\n"
  4768. "CPU%-2d spurious intrs : %lu\n"
  4769. "CPU%-2d replay intrs : %lu\n"
  4770. "CPU%-2d syst_wide : %d\n"
  4771. "CPU%-2d dcr_pp : %d\n"
  4772. "CPU%-2d exclude idle : %d\n"
  4773. "CPU%-2d owner : %d\n"
  4774. "CPU%-2d context : %p\n"
  4775. "CPU%-2d activations : %lu\n",
  4776. cpu, pfm_stats[cpu].pfm_ovfl_intr_count,
  4777. cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles,
  4778. cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles_min,
  4779. cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles_max,
  4780. cpu, pfm_stats[cpu].pfm_smpl_handler_calls,
  4781. cpu, pfm_stats[cpu].pfm_smpl_handler_cycles,
  4782. cpu, pfm_stats[cpu].pfm_spurious_ovfl_intr_count,
  4783. cpu, pfm_stats[cpu].pfm_replay_ovfl_intr_count,
  4784. cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_SYST_WIDE ? 1 : 0,
  4785. cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_DCR_PP ? 1 : 0,
  4786. cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_EXCL_IDLE ? 1 : 0,
  4787. cpu, pfm_get_cpu_data(pmu_owner, cpu) ? pfm_get_cpu_data(pmu_owner, cpu)->pid: -1,
  4788. cpu, pfm_get_cpu_data(pmu_ctx, cpu),
  4789. cpu, pfm_get_cpu_data(pmu_activation_number, cpu));
  4790. if (num_online_cpus() == 1 && pfm_sysctl.debug > 0) {
  4791. psr = pfm_get_psr();
  4792. ia64_srlz_d();
  4793. seq_printf(m,
  4794. "CPU%-2d psr : 0x%lx\n"
  4795. "CPU%-2d pmc0 : 0x%lx\n",
  4796. cpu, psr,
  4797. cpu, ia64_get_pmc(0));
  4798. for (i=0; PMC_IS_LAST(i) == 0; i++) {
  4799. if (PMC_IS_COUNTING(i) == 0) continue;
  4800. seq_printf(m,
  4801. "CPU%-2d pmc%u : 0x%lx\n"
  4802. "CPU%-2d pmd%u : 0x%lx\n",
  4803. cpu, i, ia64_get_pmc(i),
  4804. cpu, i, ia64_get_pmd(i));
  4805. }
  4806. }
  4807. return 0;
  4808. }
  4809. const struct seq_operations pfm_seq_ops = {
  4810. .start = pfm_proc_start,
  4811. .next = pfm_proc_next,
  4812. .stop = pfm_proc_stop,
  4813. .show = pfm_proc_show
  4814. };
  4815. static int
  4816. pfm_proc_open(struct inode *inode, struct file *file)
  4817. {
  4818. return seq_open(file, &pfm_seq_ops);
  4819. }
  4820. /*
  4821. * we come here as soon as local_cpu_data->pfm_syst_wide is set. this happens
  4822. * during pfm_enable() hence before pfm_start(). We cannot assume monitoring
  4823. * is active or inactive based on mode. We must rely on the value in
  4824. * local_cpu_data->pfm_syst_info
  4825. */
  4826. void
  4827. pfm_syst_wide_update_task(struct task_struct *task, unsigned long info, int is_ctxswin)
  4828. {
  4829. struct pt_regs *regs;
  4830. unsigned long dcr;
  4831. unsigned long dcr_pp;
  4832. dcr_pp = info & PFM_CPUINFO_DCR_PP ? 1 : 0;
  4833. /*
  4834. * pid 0 is guaranteed to be the idle task. There is one such task with pid 0
  4835. * on every CPU, so we can rely on the pid to identify the idle task.
  4836. */
  4837. if ((info & PFM_CPUINFO_EXCL_IDLE) == 0 || task->pid) {
  4838. regs = task_pt_regs(task);
  4839. ia64_psr(regs)->pp = is_ctxswin ? dcr_pp : 0;
  4840. return;
  4841. }
  4842. /*
  4843. * if monitoring has started
  4844. */
  4845. if (dcr_pp) {
  4846. dcr = ia64_getreg(_IA64_REG_CR_DCR);
  4847. /*
  4848. * context switching in?
  4849. */
  4850. if (is_ctxswin) {
  4851. /* mask monitoring for the idle task */
  4852. ia64_setreg(_IA64_REG_CR_DCR, dcr & ~IA64_DCR_PP);
  4853. pfm_clear_psr_pp();
  4854. ia64_srlz_i();
  4855. return;
  4856. }
  4857. /*
  4858. * context switching out
  4859. * restore monitoring for next task
  4860. *
  4861. * Due to inlining this odd if-then-else construction generates
  4862. * better code.
  4863. */
  4864. ia64_setreg(_IA64_REG_CR_DCR, dcr |IA64_DCR_PP);
  4865. pfm_set_psr_pp();
  4866. ia64_srlz_i();
  4867. }
  4868. }
  4869. #ifdef CONFIG_SMP
  4870. static void
  4871. pfm_force_cleanup(pfm_context_t *ctx, struct pt_regs *regs)
  4872. {
  4873. struct task_struct *task = ctx->ctx_task;
  4874. ia64_psr(regs)->up = 0;
  4875. ia64_psr(regs)->sp = 1;
  4876. if (GET_PMU_OWNER() == task) {
  4877. DPRINT(("cleared ownership for [%d]\n",
  4878. task_pid_nr(ctx->ctx_task)));
  4879. SET_PMU_OWNER(NULL, NULL);
  4880. }
  4881. /*
  4882. * disconnect the task from the context and vice-versa
  4883. */
  4884. PFM_SET_WORK_PENDING(task, 0);
  4885. task->thread.pfm_context = NULL;
  4886. task->thread.flags &= ~IA64_THREAD_PM_VALID;
  4887. DPRINT(("force cleanup for [%d]\n", task_pid_nr(task)));
  4888. }
  4889. /*
  4890. * in 2.6, interrupts are masked when we come here and the runqueue lock is held
  4891. */
  4892. void
  4893. pfm_save_regs(struct task_struct *task)
  4894. {
  4895. pfm_context_t *ctx;
  4896. unsigned long flags;
  4897. u64 psr;
  4898. ctx = PFM_GET_CTX(task);
  4899. if (ctx == NULL) return;
  4900. /*
  4901. * we always come here with interrupts ALREADY disabled by
  4902. * the scheduler. So we simply need to protect against concurrent
  4903. * access, not CPU concurrency.
  4904. */
  4905. flags = pfm_protect_ctx_ctxsw(ctx);
  4906. if (ctx->ctx_state == PFM_CTX_ZOMBIE) {
  4907. struct pt_regs *regs = task_pt_regs(task);
  4908. pfm_clear_psr_up();
  4909. pfm_force_cleanup(ctx, regs);
  4910. BUG_ON(ctx->ctx_smpl_hdr);
  4911. pfm_unprotect_ctx_ctxsw(ctx, flags);
  4912. pfm_context_free(ctx);
  4913. return;
  4914. }
  4915. /*
  4916. * save current PSR: needed because we modify it
  4917. */
  4918. ia64_srlz_d();
  4919. psr = pfm_get_psr();
  4920. BUG_ON(psr & (IA64_PSR_I));
  4921. /*
  4922. * stop monitoring:
  4923. * This is the last instruction which may generate an overflow
  4924. *
  4925. * We do not need to set psr.sp because, it is irrelevant in kernel.
  4926. * It will be restored from ipsr when going back to user level
  4927. */
  4928. pfm_clear_psr_up();
  4929. /*
  4930. * keep a copy of psr.up (for reload)
  4931. */
  4932. ctx->ctx_saved_psr_up = psr & IA64_PSR_UP;
  4933. /*
  4934. * release ownership of this PMU.
  4935. * PM interrupts are masked, so nothing
  4936. * can happen.
  4937. */
  4938. SET_PMU_OWNER(NULL, NULL);
  4939. /*
  4940. * we systematically save the PMD as we have no
  4941. * guarantee we will be schedule at that same
  4942. * CPU again.
  4943. */
  4944. pfm_save_pmds(ctx->th_pmds, ctx->ctx_used_pmds[0]);
  4945. /*
  4946. * save pmc0 ia64_srlz_d() done in pfm_save_pmds()
  4947. * we will need it on the restore path to check
  4948. * for pending overflow.
  4949. */
  4950. ctx->th_pmcs[0] = ia64_get_pmc(0);
  4951. /*
  4952. * unfreeze PMU if had pending overflows
  4953. */
  4954. if (ctx->th_pmcs[0] & ~0x1UL) pfm_unfreeze_pmu();
  4955. /*
  4956. * finally, allow context access.
  4957. * interrupts will still be masked after this call.
  4958. */
  4959. pfm_unprotect_ctx_ctxsw(ctx, flags);
  4960. }
  4961. #else /* !CONFIG_SMP */
  4962. void
  4963. pfm_save_regs(struct task_struct *task)
  4964. {
  4965. pfm_context_t *ctx;
  4966. u64 psr;
  4967. ctx = PFM_GET_CTX(task);
  4968. if (ctx == NULL) return;
  4969. /*
  4970. * save current PSR: needed because we modify it
  4971. */
  4972. psr = pfm_get_psr();
  4973. BUG_ON(psr & (IA64_PSR_I));
  4974. /*
  4975. * stop monitoring:
  4976. * This is the last instruction which may generate an overflow
  4977. *
  4978. * We do not need to set psr.sp because, it is irrelevant in kernel.
  4979. * It will be restored from ipsr when going back to user level
  4980. */
  4981. pfm_clear_psr_up();
  4982. /*
  4983. * keep a copy of psr.up (for reload)
  4984. */
  4985. ctx->ctx_saved_psr_up = psr & IA64_PSR_UP;
  4986. }
  4987. static void
  4988. pfm_lazy_save_regs (struct task_struct *task)
  4989. {
  4990. pfm_context_t *ctx;
  4991. unsigned long flags;
  4992. { u64 psr = pfm_get_psr();
  4993. BUG_ON(psr & IA64_PSR_UP);
  4994. }
  4995. ctx = PFM_GET_CTX(task);
  4996. /*
  4997. * we need to mask PMU overflow here to
  4998. * make sure that we maintain pmc0 until
  4999. * we save it. overflow interrupts are
  5000. * treated as spurious if there is no
  5001. * owner.
  5002. *
  5003. * XXX: I don't think this is necessary
  5004. */
  5005. PROTECT_CTX(ctx,flags);
  5006. /*
  5007. * release ownership of this PMU.
  5008. * must be done before we save the registers.
  5009. *
  5010. * after this call any PMU interrupt is treated
  5011. * as spurious.
  5012. */
  5013. SET_PMU_OWNER(NULL, NULL);
  5014. /*
  5015. * save all the pmds we use
  5016. */
  5017. pfm_save_pmds(ctx->th_pmds, ctx->ctx_used_pmds[0]);
  5018. /*
  5019. * save pmc0 ia64_srlz_d() done in pfm_save_pmds()
  5020. * it is needed to check for pended overflow
  5021. * on the restore path
  5022. */
  5023. ctx->th_pmcs[0] = ia64_get_pmc(0);
  5024. /*
  5025. * unfreeze PMU if had pending overflows
  5026. */
  5027. if (ctx->th_pmcs[0] & ~0x1UL) pfm_unfreeze_pmu();
  5028. /*
  5029. * now get can unmask PMU interrupts, they will
  5030. * be treated as purely spurious and we will not
  5031. * lose any information
  5032. */
  5033. UNPROTECT_CTX(ctx,flags);
  5034. }
  5035. #endif /* CONFIG_SMP */
  5036. #ifdef CONFIG_SMP
  5037. /*
  5038. * in 2.6, interrupts are masked when we come here and the runqueue lock is held
  5039. */
  5040. void
  5041. pfm_load_regs (struct task_struct *task)
  5042. {
  5043. pfm_context_t *ctx;
  5044. unsigned long pmc_mask = 0UL, pmd_mask = 0UL;
  5045. unsigned long flags;
  5046. u64 psr, psr_up;
  5047. int need_irq_resend;
  5048. ctx = PFM_GET_CTX(task);
  5049. if (unlikely(ctx == NULL)) return;
  5050. BUG_ON(GET_PMU_OWNER());
  5051. /*
  5052. * possible on unload
  5053. */
  5054. if (unlikely((task->thread.flags & IA64_THREAD_PM_VALID) == 0)) return;
  5055. /*
  5056. * we always come here with interrupts ALREADY disabled by
  5057. * the scheduler. So we simply need to protect against concurrent
  5058. * access, not CPU concurrency.
  5059. */
  5060. flags = pfm_protect_ctx_ctxsw(ctx);
  5061. psr = pfm_get_psr();
  5062. need_irq_resend = pmu_conf->flags & PFM_PMU_IRQ_RESEND;
  5063. BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
  5064. BUG_ON(psr & IA64_PSR_I);
  5065. if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) {
  5066. struct pt_regs *regs = task_pt_regs(task);
  5067. BUG_ON(ctx->ctx_smpl_hdr);
  5068. pfm_force_cleanup(ctx, regs);
  5069. pfm_unprotect_ctx_ctxsw(ctx, flags);
  5070. /*
  5071. * this one (kmalloc'ed) is fine with interrupts disabled
  5072. */
  5073. pfm_context_free(ctx);
  5074. return;
  5075. }
  5076. /*
  5077. * we restore ALL the debug registers to avoid picking up
  5078. * stale state.
  5079. */
  5080. if (ctx->ctx_fl_using_dbreg) {
  5081. pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
  5082. pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
  5083. }
  5084. /*
  5085. * retrieve saved psr.up
  5086. */
  5087. psr_up = ctx->ctx_saved_psr_up;
  5088. /*
  5089. * if we were the last user of the PMU on that CPU,
  5090. * then nothing to do except restore psr
  5091. */
  5092. if (GET_LAST_CPU(ctx) == smp_processor_id() && ctx->ctx_last_activation == GET_ACTIVATION()) {
  5093. /*
  5094. * retrieve partial reload masks (due to user modifications)
  5095. */
  5096. pmc_mask = ctx->ctx_reload_pmcs[0];
  5097. pmd_mask = ctx->ctx_reload_pmds[0];
  5098. } else {
  5099. /*
  5100. * To avoid leaking information to the user level when psr.sp=0,
  5101. * we must reload ALL implemented pmds (even the ones we don't use).
  5102. * In the kernel we only allow PFM_READ_PMDS on registers which
  5103. * we initialized or requested (sampling) so there is no risk there.
  5104. */
  5105. pmd_mask = pfm_sysctl.fastctxsw ? ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0];
  5106. /*
  5107. * ALL accessible PMCs are systematically reloaded, unused registers
  5108. * get their default (from pfm_reset_pmu_state()) values to avoid picking
  5109. * up stale configuration.
  5110. *
  5111. * PMC0 is never in the mask. It is always restored separately.
  5112. */
  5113. pmc_mask = ctx->ctx_all_pmcs[0];
  5114. }
  5115. /*
  5116. * when context is MASKED, we will restore PMC with plm=0
  5117. * and PMD with stale information, but that's ok, nothing
  5118. * will be captured.
  5119. *
  5120. * XXX: optimize here
  5121. */
  5122. if (pmd_mask) pfm_restore_pmds(ctx->th_pmds, pmd_mask);
  5123. if (pmc_mask) pfm_restore_pmcs(ctx->th_pmcs, pmc_mask);
  5124. /*
  5125. * check for pending overflow at the time the state
  5126. * was saved.
  5127. */
  5128. if (unlikely(PMC0_HAS_OVFL(ctx->th_pmcs[0]))) {
  5129. /*
  5130. * reload pmc0 with the overflow information
  5131. * On McKinley PMU, this will trigger a PMU interrupt
  5132. */
  5133. ia64_set_pmc(0, ctx->th_pmcs[0]);
  5134. ia64_srlz_d();
  5135. ctx->th_pmcs[0] = 0UL;
  5136. /*
  5137. * will replay the PMU interrupt
  5138. */
  5139. if (need_irq_resend) ia64_resend_irq(IA64_PERFMON_VECTOR);
  5140. pfm_stats[smp_processor_id()].pfm_replay_ovfl_intr_count++;
  5141. }
  5142. /*
  5143. * we just did a reload, so we reset the partial reload fields
  5144. */
  5145. ctx->ctx_reload_pmcs[0] = 0UL;
  5146. ctx->ctx_reload_pmds[0] = 0UL;
  5147. SET_LAST_CPU(ctx, smp_processor_id());
  5148. /*
  5149. * dump activation value for this PMU
  5150. */
  5151. INC_ACTIVATION();
  5152. /*
  5153. * record current activation for this context
  5154. */
  5155. SET_ACTIVATION(ctx);
  5156. /*
  5157. * establish new ownership.
  5158. */
  5159. SET_PMU_OWNER(task, ctx);
  5160. /*
  5161. * restore the psr.up bit. measurement
  5162. * is active again.
  5163. * no PMU interrupt can happen at this point
  5164. * because we still have interrupts disabled.
  5165. */
  5166. if (likely(psr_up)) pfm_set_psr_up();
  5167. /*
  5168. * allow concurrent access to context
  5169. */
  5170. pfm_unprotect_ctx_ctxsw(ctx, flags);
  5171. }
  5172. #else /* !CONFIG_SMP */
  5173. /*
  5174. * reload PMU state for UP kernels
  5175. * in 2.5 we come here with interrupts disabled
  5176. */
  5177. void
  5178. pfm_load_regs (struct task_struct *task)
  5179. {
  5180. pfm_context_t *ctx;
  5181. struct task_struct *owner;
  5182. unsigned long pmd_mask, pmc_mask;
  5183. u64 psr, psr_up;
  5184. int need_irq_resend;
  5185. owner = GET_PMU_OWNER();
  5186. ctx = PFM_GET_CTX(task);
  5187. psr = pfm_get_psr();
  5188. BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
  5189. BUG_ON(psr & IA64_PSR_I);
  5190. /*
  5191. * we restore ALL the debug registers to avoid picking up
  5192. * stale state.
  5193. *
  5194. * This must be done even when the task is still the owner
  5195. * as the registers may have been modified via ptrace()
  5196. * (not perfmon) by the previous task.
  5197. */
  5198. if (ctx->ctx_fl_using_dbreg) {
  5199. pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
  5200. pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
  5201. }
  5202. /*
  5203. * retrieved saved psr.up
  5204. */
  5205. psr_up = ctx->ctx_saved_psr_up;
  5206. need_irq_resend = pmu_conf->flags & PFM_PMU_IRQ_RESEND;
  5207. /*
  5208. * short path, our state is still there, just
  5209. * need to restore psr and we go
  5210. *
  5211. * we do not touch either PMC nor PMD. the psr is not touched
  5212. * by the overflow_handler. So we are safe w.r.t. to interrupt
  5213. * concurrency even without interrupt masking.
  5214. */
  5215. if (likely(owner == task)) {
  5216. if (likely(psr_up)) pfm_set_psr_up();
  5217. return;
  5218. }
  5219. /*
  5220. * someone else is still using the PMU, first push it out and
  5221. * then we'll be able to install our stuff !
  5222. *
  5223. * Upon return, there will be no owner for the current PMU
  5224. */
  5225. if (owner) pfm_lazy_save_regs(owner);
  5226. /*
  5227. * To avoid leaking information to the user level when psr.sp=0,
  5228. * we must reload ALL implemented pmds (even the ones we don't use).
  5229. * In the kernel we only allow PFM_READ_PMDS on registers which
  5230. * we initialized or requested (sampling) so there is no risk there.
  5231. */
  5232. pmd_mask = pfm_sysctl.fastctxsw ? ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0];
  5233. /*
  5234. * ALL accessible PMCs are systematically reloaded, unused registers
  5235. * get their default (from pfm_reset_pmu_state()) values to avoid picking
  5236. * up stale configuration.
  5237. *
  5238. * PMC0 is never in the mask. It is always restored separately
  5239. */
  5240. pmc_mask = ctx->ctx_all_pmcs[0];
  5241. pfm_restore_pmds(ctx->th_pmds, pmd_mask);
  5242. pfm_restore_pmcs(ctx->th_pmcs, pmc_mask);
  5243. /*
  5244. * check for pending overflow at the time the state
  5245. * was saved.
  5246. */
  5247. if (unlikely(PMC0_HAS_OVFL(ctx->th_pmcs[0]))) {
  5248. /*
  5249. * reload pmc0 with the overflow information
  5250. * On McKinley PMU, this will trigger a PMU interrupt
  5251. */
  5252. ia64_set_pmc(0, ctx->th_pmcs[0]);
  5253. ia64_srlz_d();
  5254. ctx->th_pmcs[0] = 0UL;
  5255. /*
  5256. * will replay the PMU interrupt
  5257. */
  5258. if (need_irq_resend) ia64_resend_irq(IA64_PERFMON_VECTOR);
  5259. pfm_stats[smp_processor_id()].pfm_replay_ovfl_intr_count++;
  5260. }
  5261. /*
  5262. * establish new ownership.
  5263. */
  5264. SET_PMU_OWNER(task, ctx);
  5265. /*
  5266. * restore the psr.up bit. measurement
  5267. * is active again.
  5268. * no PMU interrupt can happen at this point
  5269. * because we still have interrupts disabled.
  5270. */
  5271. if (likely(psr_up)) pfm_set_psr_up();
  5272. }
  5273. #endif /* CONFIG_SMP */
  5274. /*
  5275. * this function assumes monitoring is stopped
  5276. */
  5277. static void
  5278. pfm_flush_pmds(struct task_struct *task, pfm_context_t *ctx)
  5279. {
  5280. u64 pmc0;
  5281. unsigned long mask2, val, pmd_val, ovfl_val;
  5282. int i, can_access_pmu = 0;
  5283. int is_self;
  5284. /*
  5285. * is the caller the task being monitored (or which initiated the
  5286. * session for system wide measurements)
  5287. */
  5288. is_self = ctx->ctx_task == task ? 1 : 0;
  5289. /*
  5290. * can access PMU is task is the owner of the PMU state on the current CPU
  5291. * or if we are running on the CPU bound to the context in system-wide mode
  5292. * (that is not necessarily the task the context is attached to in this mode).
  5293. * In system-wide we always have can_access_pmu true because a task running on an
  5294. * invalid processor is flagged earlier in the call stack (see pfm_stop).
  5295. */
  5296. can_access_pmu = (GET_PMU_OWNER() == task) || (ctx->ctx_fl_system && ctx->ctx_cpu == smp_processor_id());
  5297. if (can_access_pmu) {
  5298. /*
  5299. * Mark the PMU as not owned
  5300. * This will cause the interrupt handler to do nothing in case an overflow
  5301. * interrupt was in-flight
  5302. * This also guarantees that pmc0 will contain the final state
  5303. * It virtually gives us full control on overflow processing from that point
  5304. * on.
  5305. */
  5306. SET_PMU_OWNER(NULL, NULL);
  5307. DPRINT(("releasing ownership\n"));
  5308. /*
  5309. * read current overflow status:
  5310. *
  5311. * we are guaranteed to read the final stable state
  5312. */
  5313. ia64_srlz_d();
  5314. pmc0 = ia64_get_pmc(0); /* slow */
  5315. /*
  5316. * reset freeze bit, overflow status information destroyed
  5317. */
  5318. pfm_unfreeze_pmu();
  5319. } else {
  5320. pmc0 = ctx->th_pmcs[0];
  5321. /*
  5322. * clear whatever overflow status bits there were
  5323. */
  5324. ctx->th_pmcs[0] = 0;
  5325. }
  5326. ovfl_val = pmu_conf->ovfl_val;
  5327. /*
  5328. * we save all the used pmds
  5329. * we take care of overflows for counting PMDs
  5330. *
  5331. * XXX: sampling situation is not taken into account here
  5332. */
  5333. mask2 = ctx->ctx_used_pmds[0];
  5334. DPRINT(("is_self=%d ovfl_val=0x%lx mask2=0x%lx\n", is_self, ovfl_val, mask2));
  5335. for (i = 0; mask2; i++, mask2>>=1) {
  5336. /* skip non used pmds */
  5337. if ((mask2 & 0x1) == 0) continue;
  5338. /*
  5339. * can access PMU always true in system wide mode
  5340. */
  5341. val = pmd_val = can_access_pmu ? ia64_get_pmd(i) : ctx->th_pmds[i];
  5342. if (PMD_IS_COUNTING(i)) {
  5343. DPRINT(("[%d] pmd[%d] ctx_pmd=0x%lx hw_pmd=0x%lx\n",
  5344. task_pid_nr(task),
  5345. i,
  5346. ctx->ctx_pmds[i].val,
  5347. val & ovfl_val));
  5348. /*
  5349. * we rebuild the full 64 bit value of the counter
  5350. */
  5351. val = ctx->ctx_pmds[i].val + (val & ovfl_val);
  5352. /*
  5353. * now everything is in ctx_pmds[] and we need
  5354. * to clear the saved context from save_regs() such that
  5355. * pfm_read_pmds() gets the correct value
  5356. */
  5357. pmd_val = 0UL;
  5358. /*
  5359. * take care of overflow inline
  5360. */
  5361. if (pmc0 & (1UL << i)) {
  5362. val += 1 + ovfl_val;
  5363. DPRINT(("[%d] pmd[%d] overflowed\n", task_pid_nr(task), i));
  5364. }
  5365. }
  5366. DPRINT(("[%d] ctx_pmd[%d]=0x%lx pmd_val=0x%lx\n", task_pid_nr(task), i, val, pmd_val));
  5367. if (is_self) ctx->th_pmds[i] = pmd_val;
  5368. ctx->ctx_pmds[i].val = val;
  5369. }
  5370. }
  5371. static struct irqaction perfmon_irqaction = {
  5372. .handler = pfm_interrupt_handler,
  5373. .flags = IRQF_DISABLED,
  5374. .name = "perfmon"
  5375. };
  5376. static void
  5377. pfm_alt_save_pmu_state(void *data)
  5378. {
  5379. struct pt_regs *regs;
  5380. regs = task_pt_regs(current);
  5381. DPRINT(("called\n"));
  5382. /*
  5383. * should not be necessary but
  5384. * let's take not risk
  5385. */
  5386. pfm_clear_psr_up();
  5387. pfm_clear_psr_pp();
  5388. ia64_psr(regs)->pp = 0;
  5389. /*
  5390. * This call is required
  5391. * May cause a spurious interrupt on some processors
  5392. */
  5393. pfm_freeze_pmu();
  5394. ia64_srlz_d();
  5395. }
  5396. void
  5397. pfm_alt_restore_pmu_state(void *data)
  5398. {
  5399. struct pt_regs *regs;
  5400. regs = task_pt_regs(current);
  5401. DPRINT(("called\n"));
  5402. /*
  5403. * put PMU back in state expected
  5404. * by perfmon
  5405. */
  5406. pfm_clear_psr_up();
  5407. pfm_clear_psr_pp();
  5408. ia64_psr(regs)->pp = 0;
  5409. /*
  5410. * perfmon runs with PMU unfrozen at all times
  5411. */
  5412. pfm_unfreeze_pmu();
  5413. ia64_srlz_d();
  5414. }
  5415. int
  5416. pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
  5417. {
  5418. int ret, i;
  5419. int reserve_cpu;
  5420. /* some sanity checks */
  5421. if (hdl == NULL || hdl->handler == NULL) return -EINVAL;
  5422. /* do the easy test first */
  5423. if (pfm_alt_intr_handler) return -EBUSY;
  5424. /* one at a time in the install or remove, just fail the others */
  5425. if (!spin_trylock(&pfm_alt_install_check)) {
  5426. return -EBUSY;
  5427. }
  5428. /* reserve our session */
  5429. for_each_online_cpu(reserve_cpu) {
  5430. ret = pfm_reserve_session(NULL, 1, reserve_cpu);
  5431. if (ret) goto cleanup_reserve;
  5432. }
  5433. /* save the current system wide pmu states */
  5434. ret = on_each_cpu(pfm_alt_save_pmu_state, NULL, 0, 1);
  5435. if (ret) {
  5436. DPRINT(("on_each_cpu() failed: %d\n", ret));
  5437. goto cleanup_reserve;
  5438. }
  5439. /* officially change to the alternate interrupt handler */
  5440. pfm_alt_intr_handler = hdl;
  5441. spin_unlock(&pfm_alt_install_check);
  5442. return 0;
  5443. cleanup_reserve:
  5444. for_each_online_cpu(i) {
  5445. /* don't unreserve more than we reserved */
  5446. if (i >= reserve_cpu) break;
  5447. pfm_unreserve_session(NULL, 1, i);
  5448. }
  5449. spin_unlock(&pfm_alt_install_check);
  5450. return ret;
  5451. }
  5452. EXPORT_SYMBOL_GPL(pfm_install_alt_pmu_interrupt);
  5453. int
  5454. pfm_remove_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
  5455. {
  5456. int i;
  5457. int ret;
  5458. if (hdl == NULL) return -EINVAL;
  5459. /* cannot remove someone else's handler! */
  5460. if (pfm_alt_intr_handler != hdl) return -EINVAL;
  5461. /* one at a time in the install or remove, just fail the others */
  5462. if (!spin_trylock(&pfm_alt_install_check)) {
  5463. return -EBUSY;
  5464. }
  5465. pfm_alt_intr_handler = NULL;
  5466. ret = on_each_cpu(pfm_alt_restore_pmu_state, NULL, 0, 1);
  5467. if (ret) {
  5468. DPRINT(("on_each_cpu() failed: %d\n", ret));
  5469. }
  5470. for_each_online_cpu(i) {
  5471. pfm_unreserve_session(NULL, 1, i);
  5472. }
  5473. spin_unlock(&pfm_alt_install_check);
  5474. return 0;
  5475. }
  5476. EXPORT_SYMBOL_GPL(pfm_remove_alt_pmu_interrupt);
  5477. /*
  5478. * perfmon initialization routine, called from the initcall() table
  5479. */
  5480. static int init_pfm_fs(void);
  5481. static int __init
  5482. pfm_probe_pmu(void)
  5483. {
  5484. pmu_config_t **p;
  5485. int family;
  5486. family = local_cpu_data->family;
  5487. p = pmu_confs;
  5488. while(*p) {
  5489. if ((*p)->probe) {
  5490. if ((*p)->probe() == 0) goto found;
  5491. } else if ((*p)->pmu_family == family || (*p)->pmu_family == 0xff) {
  5492. goto found;
  5493. }
  5494. p++;
  5495. }
  5496. return -1;
  5497. found:
  5498. pmu_conf = *p;
  5499. return 0;
  5500. }
  5501. static const struct file_operations pfm_proc_fops = {
  5502. .open = pfm_proc_open,
  5503. .read = seq_read,
  5504. .llseek = seq_lseek,
  5505. .release = seq_release,
  5506. };
  5507. int __init
  5508. pfm_init(void)
  5509. {
  5510. unsigned int n, n_counters, i;
  5511. printk("perfmon: version %u.%u IRQ %u\n",
  5512. PFM_VERSION_MAJ,
  5513. PFM_VERSION_MIN,
  5514. IA64_PERFMON_VECTOR);
  5515. if (pfm_probe_pmu()) {
  5516. printk(KERN_INFO "perfmon: disabled, there is no support for processor family %d\n",
  5517. local_cpu_data->family);
  5518. return -ENODEV;
  5519. }
  5520. /*
  5521. * compute the number of implemented PMD/PMC from the
  5522. * description tables
  5523. */
  5524. n = 0;
  5525. for (i=0; PMC_IS_LAST(i) == 0; i++) {
  5526. if (PMC_IS_IMPL(i) == 0) continue;
  5527. pmu_conf->impl_pmcs[i>>6] |= 1UL << (i&63);
  5528. n++;
  5529. }
  5530. pmu_conf->num_pmcs = n;
  5531. n = 0; n_counters = 0;
  5532. for (i=0; PMD_IS_LAST(i) == 0; i++) {
  5533. if (PMD_IS_IMPL(i) == 0) continue;
  5534. pmu_conf->impl_pmds[i>>6] |= 1UL << (i&63);
  5535. n++;
  5536. if (PMD_IS_COUNTING(i)) n_counters++;
  5537. }
  5538. pmu_conf->num_pmds = n;
  5539. pmu_conf->num_counters = n_counters;
  5540. /*
  5541. * sanity checks on the number of debug registers
  5542. */
  5543. if (pmu_conf->use_rr_dbregs) {
  5544. if (pmu_conf->num_ibrs > IA64_NUM_DBG_REGS) {
  5545. printk(KERN_INFO "perfmon: unsupported number of code debug registers (%u)\n", pmu_conf->num_ibrs);
  5546. pmu_conf = NULL;
  5547. return -1;
  5548. }
  5549. if (pmu_conf->num_dbrs > IA64_NUM_DBG_REGS) {
  5550. printk(KERN_INFO "perfmon: unsupported number of data debug registers (%u)\n", pmu_conf->num_ibrs);
  5551. pmu_conf = NULL;
  5552. return -1;
  5553. }
  5554. }
  5555. printk("perfmon: %s PMU detected, %u PMCs, %u PMDs, %u counters (%lu bits)\n",
  5556. pmu_conf->pmu_name,
  5557. pmu_conf->num_pmcs,
  5558. pmu_conf->num_pmds,
  5559. pmu_conf->num_counters,
  5560. ffz(pmu_conf->ovfl_val));
  5561. /* sanity check */
  5562. if (pmu_conf->num_pmds >= PFM_NUM_PMD_REGS || pmu_conf->num_pmcs >= PFM_NUM_PMC_REGS) {
  5563. printk(KERN_ERR "perfmon: not enough pmc/pmd, perfmon disabled\n");
  5564. pmu_conf = NULL;
  5565. return -1;
  5566. }
  5567. /*
  5568. * create /proc/perfmon (mostly for debugging purposes)
  5569. */
  5570. perfmon_dir = proc_create("perfmon", S_IRUGO, NULL, &pfm_proc_fops);
  5571. if (perfmon_dir == NULL) {
  5572. printk(KERN_ERR "perfmon: cannot create /proc entry, perfmon disabled\n");
  5573. pmu_conf = NULL;
  5574. return -1;
  5575. }
  5576. /*
  5577. * create /proc/sys/kernel/perfmon (for debugging purposes)
  5578. */
  5579. pfm_sysctl_header = register_sysctl_table(pfm_sysctl_root);
  5580. /*
  5581. * initialize all our spinlocks
  5582. */
  5583. spin_lock_init(&pfm_sessions.pfs_lock);
  5584. spin_lock_init(&pfm_buffer_fmt_lock);
  5585. init_pfm_fs();
  5586. for(i=0; i < NR_CPUS; i++) pfm_stats[i].pfm_ovfl_intr_cycles_min = ~0UL;
  5587. return 0;
  5588. }
  5589. __initcall(pfm_init);
  5590. /*
  5591. * this function is called before pfm_init()
  5592. */
  5593. void
  5594. pfm_init_percpu (void)
  5595. {
  5596. static int first_time=1;
  5597. /*
  5598. * make sure no measurement is active
  5599. * (may inherit programmed PMCs from EFI).
  5600. */
  5601. pfm_clear_psr_pp();
  5602. pfm_clear_psr_up();
  5603. /*
  5604. * we run with the PMU not frozen at all times
  5605. */
  5606. pfm_unfreeze_pmu();
  5607. if (first_time) {
  5608. register_percpu_irq(IA64_PERFMON_VECTOR, &perfmon_irqaction);
  5609. first_time=0;
  5610. }
  5611. ia64_setreg(_IA64_REG_CR_PMV, IA64_PERFMON_VECTOR);
  5612. ia64_srlz_d();
  5613. }
  5614. /*
  5615. * used for debug purposes only
  5616. */
  5617. void
  5618. dump_pmu_state(const char *from)
  5619. {
  5620. struct task_struct *task;
  5621. struct pt_regs *regs;
  5622. pfm_context_t *ctx;
  5623. unsigned long psr, dcr, info, flags;
  5624. int i, this_cpu;
  5625. local_irq_save(flags);
  5626. this_cpu = smp_processor_id();
  5627. regs = task_pt_regs(current);
  5628. info = PFM_CPUINFO_GET();
  5629. dcr = ia64_getreg(_IA64_REG_CR_DCR);
  5630. if (info == 0 && ia64_psr(regs)->pp == 0 && (dcr & IA64_DCR_PP) == 0) {
  5631. local_irq_restore(flags);
  5632. return;
  5633. }
  5634. printk("CPU%d from %s() current [%d] iip=0x%lx %s\n",
  5635. this_cpu,
  5636. from,
  5637. task_pid_nr(current),
  5638. regs->cr_iip,
  5639. current->comm);
  5640. task = GET_PMU_OWNER();
  5641. ctx = GET_PMU_CTX();
  5642. printk("->CPU%d owner [%d] ctx=%p\n", this_cpu, task ? task_pid_nr(task) : -1, ctx);
  5643. psr = pfm_get_psr();
  5644. printk("->CPU%d pmc0=0x%lx psr.pp=%d psr.up=%d dcr.pp=%d syst_info=0x%lx user_psr.up=%d user_psr.pp=%d\n",
  5645. this_cpu,
  5646. ia64_get_pmc(0),
  5647. psr & IA64_PSR_PP ? 1 : 0,
  5648. psr & IA64_PSR_UP ? 1 : 0,
  5649. dcr & IA64_DCR_PP ? 1 : 0,
  5650. info,
  5651. ia64_psr(regs)->up,
  5652. ia64_psr(regs)->pp);
  5653. ia64_psr(regs)->up = 0;
  5654. ia64_psr(regs)->pp = 0;
  5655. for (i=1; PMC_IS_LAST(i) == 0; i++) {
  5656. if (PMC_IS_IMPL(i) == 0) continue;
  5657. printk("->CPU%d pmc[%d]=0x%lx thread_pmc[%d]=0x%lx\n", this_cpu, i, ia64_get_pmc(i), i, ctx->th_pmcs[i]);
  5658. }
  5659. for (i=1; PMD_IS_LAST(i) == 0; i++) {
  5660. if (PMD_IS_IMPL(i) == 0) continue;
  5661. printk("->CPU%d pmd[%d]=0x%lx thread_pmd[%d]=0x%lx\n", this_cpu, i, ia64_get_pmd(i), i, ctx->th_pmds[i]);
  5662. }
  5663. if (ctx) {
  5664. printk("->CPU%d ctx_state=%d vaddr=%p addr=%p fd=%d ctx_task=[%d] saved_psr_up=0x%lx\n",
  5665. this_cpu,
  5666. ctx->ctx_state,
  5667. ctx->ctx_smpl_vaddr,
  5668. ctx->ctx_smpl_hdr,
  5669. ctx->ctx_msgq_head,
  5670. ctx->ctx_msgq_tail,
  5671. ctx->ctx_saved_psr_up);
  5672. }
  5673. local_irq_restore(flags);
  5674. }
  5675. /*
  5676. * called from process.c:copy_thread(). task is new child.
  5677. */
  5678. void
  5679. pfm_inherit(struct task_struct *task, struct pt_regs *regs)
  5680. {
  5681. struct thread_struct *thread;
  5682. DPRINT(("perfmon: pfm_inherit clearing state for [%d]\n", task_pid_nr(task)));
  5683. thread = &task->thread;
  5684. /*
  5685. * cut links inherited from parent (current)
  5686. */
  5687. thread->pfm_context = NULL;
  5688. PFM_SET_WORK_PENDING(task, 0);
  5689. /*
  5690. * the psr bits are already set properly in copy_threads()
  5691. */
  5692. }
  5693. #else /* !CONFIG_PERFMON */
  5694. asmlinkage long
  5695. sys_perfmonctl (int fd, int cmd, void *arg, int count)
  5696. {
  5697. return -ENOSYS;
  5698. }
  5699. #endif /* CONFIG_PERFMON */