sched.c 246 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107
  1. /*
  2. * kernel/sched.c
  3. *
  4. * Kernel scheduler and related syscalls
  5. *
  6. * Copyright (C) 1991-2002 Linus Torvalds
  7. *
  8. * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and
  9. * make semaphores SMP safe
  10. * 1998-11-19 Implemented schedule_timeout() and related stuff
  11. * by Andrea Arcangeli
  12. * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:
  13. * hybrid priority-list and round-robin design with
  14. * an array-switch method of distributing timeslices
  15. * and per-CPU runqueues. Cleanups and useful suggestions
  16. * by Davide Libenzi, preemptible kernel bits by Robert Love.
  17. * 2003-09-03 Interactivity tuning by Con Kolivas.
  18. * 2004-04-02 Scheduler domains code by Nick Piggin
  19. * 2007-04-15 Work begun on replacing all interactivity tuning with a
  20. * fair scheduling design by Con Kolivas.
  21. * 2007-05-05 Load balancing (smp-nice) and other improvements
  22. * by Peter Williams
  23. * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith
  24. * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri
  25. * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins,
  26. * Thomas Gleixner, Mike Kravetz
  27. */
  28. #include <linux/mm.h>
  29. #include <linux/module.h>
  30. #include <linux/nmi.h>
  31. #include <linux/init.h>
  32. #include <linux/uaccess.h>
  33. #include <linux/highmem.h>
  34. #include <linux/smp_lock.h>
  35. #include <asm/mmu_context.h>
  36. #include <linux/interrupt.h>
  37. #include <linux/capability.h>
  38. #include <linux/completion.h>
  39. #include <linux/kernel_stat.h>
  40. #include <linux/debug_locks.h>
  41. #include <linux/security.h>
  42. #include <linux/notifier.h>
  43. #include <linux/profile.h>
  44. #include <linux/freezer.h>
  45. #include <linux/vmalloc.h>
  46. #include <linux/blkdev.h>
  47. #include <linux/delay.h>
  48. #include <linux/pid_namespace.h>
  49. #include <linux/smp.h>
  50. #include <linux/threads.h>
  51. #include <linux/timer.h>
  52. #include <linux/rcupdate.h>
  53. #include <linux/cpu.h>
  54. #include <linux/cpuset.h>
  55. #include <linux/percpu.h>
  56. #include <linux/kthread.h>
  57. #include <linux/proc_fs.h>
  58. #include <linux/seq_file.h>
  59. #include <linux/sysctl.h>
  60. #include <linux/syscalls.h>
  61. #include <linux/times.h>
  62. #include <linux/tsacct_kern.h>
  63. #include <linux/kprobes.h>
  64. #include <linux/delayacct.h>
  65. #include <linux/reciprocal_div.h>
  66. #include <linux/unistd.h>
  67. #include <linux/pagemap.h>
  68. #include <linux/hrtimer.h>
  69. #include <linux/tick.h>
  70. #include <linux/bootmem.h>
  71. #include <linux/debugfs.h>
  72. #include <linux/ctype.h>
  73. #include <linux/ftrace.h>
  74. #include <trace/sched.h>
  75. #include <asm/tlb.h>
  76. #include <asm/irq_regs.h>
  77. #include "sched_cpupri.h"
  78. /*
  79. * Convert user-nice values [ -20 ... 0 ... 19 ]
  80. * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
  81. * and back.
  82. */
  83. #define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20)
  84. #define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20)
  85. #define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio)
  86. /*
  87. * 'User priority' is the nice value converted to something we
  88. * can work with better when scaling various scheduler parameters,
  89. * it's a [ 0 ... 39 ] range.
  90. */
  91. #define USER_PRIO(p) ((p)-MAX_RT_PRIO)
  92. #define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)
  93. #define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
  94. /*
  95. * Helpers for converting nanosecond timing to jiffy resolution
  96. */
  97. #define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
  98. #define NICE_0_LOAD SCHED_LOAD_SCALE
  99. #define NICE_0_SHIFT SCHED_LOAD_SHIFT
  100. /*
  101. * These are the 'tuning knobs' of the scheduler:
  102. *
  103. * default timeslice is 100 msecs (used only for SCHED_RR tasks).
  104. * Timeslices get refilled after they expire.
  105. */
  106. #define DEF_TIMESLICE (100 * HZ / 1000)
  107. /*
  108. * single value that denotes runtime == period, ie unlimited time.
  109. */
  110. #define RUNTIME_INF ((u64)~0ULL)
  111. DEFINE_TRACE(sched_wait_task);
  112. DEFINE_TRACE(sched_wakeup);
  113. DEFINE_TRACE(sched_wakeup_new);
  114. DEFINE_TRACE(sched_switch);
  115. DEFINE_TRACE(sched_migrate_task);
  116. #ifdef CONFIG_SMP
  117. static void double_rq_lock(struct rq *rq1, struct rq *rq2);
  118. /*
  119. * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
  120. * Since cpu_power is a 'constant', we can use a reciprocal divide.
  121. */
  122. static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
  123. {
  124. return reciprocal_divide(load, sg->reciprocal_cpu_power);
  125. }
  126. /*
  127. * Each time a sched group cpu_power is changed,
  128. * we must compute its reciprocal value
  129. */
  130. static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
  131. {
  132. sg->__cpu_power += val;
  133. sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
  134. }
  135. #endif
  136. static inline int rt_policy(int policy)
  137. {
  138. if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
  139. return 1;
  140. return 0;
  141. }
  142. static inline int task_has_rt_policy(struct task_struct *p)
  143. {
  144. return rt_policy(p->policy);
  145. }
  146. /*
  147. * This is the priority-queue data structure of the RT scheduling class:
  148. */
  149. struct rt_prio_array {
  150. DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
  151. struct list_head queue[MAX_RT_PRIO];
  152. };
  153. struct rt_bandwidth {
  154. /* nests inside the rq lock: */
  155. spinlock_t rt_runtime_lock;
  156. ktime_t rt_period;
  157. u64 rt_runtime;
  158. struct hrtimer rt_period_timer;
  159. };
  160. static struct rt_bandwidth def_rt_bandwidth;
  161. static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
  162. static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
  163. {
  164. struct rt_bandwidth *rt_b =
  165. container_of(timer, struct rt_bandwidth, rt_period_timer);
  166. ktime_t now;
  167. int overrun;
  168. int idle = 0;
  169. for (;;) {
  170. now = hrtimer_cb_get_time(timer);
  171. overrun = hrtimer_forward(timer, now, rt_b->rt_period);
  172. if (!overrun)
  173. break;
  174. idle = do_sched_rt_period_timer(rt_b, overrun);
  175. }
  176. return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
  177. }
  178. static
  179. void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
  180. {
  181. rt_b->rt_period = ns_to_ktime(period);
  182. rt_b->rt_runtime = runtime;
  183. spin_lock_init(&rt_b->rt_runtime_lock);
  184. hrtimer_init(&rt_b->rt_period_timer,
  185. CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  186. rt_b->rt_period_timer.function = sched_rt_period_timer;
  187. }
  188. static inline int rt_bandwidth_enabled(void)
  189. {
  190. return sysctl_sched_rt_runtime >= 0;
  191. }
  192. static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
  193. {
  194. ktime_t now;
  195. if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
  196. return;
  197. if (hrtimer_active(&rt_b->rt_period_timer))
  198. return;
  199. spin_lock(&rt_b->rt_runtime_lock);
  200. for (;;) {
  201. unsigned long delta;
  202. ktime_t soft, hard;
  203. if (hrtimer_active(&rt_b->rt_period_timer))
  204. break;
  205. now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
  206. hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
  207. soft = hrtimer_get_softexpires(&rt_b->rt_period_timer);
  208. hard = hrtimer_get_expires(&rt_b->rt_period_timer);
  209. delta = ktime_to_ns(ktime_sub(hard, soft));
  210. __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,
  211. HRTIMER_MODE_ABS, 0);
  212. }
  213. spin_unlock(&rt_b->rt_runtime_lock);
  214. }
  215. #ifdef CONFIG_RT_GROUP_SCHED
  216. static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
  217. {
  218. hrtimer_cancel(&rt_b->rt_period_timer);
  219. }
  220. #endif
  221. /*
  222. * sched_domains_mutex serializes calls to arch_init_sched_domains,
  223. * detach_destroy_domains and partition_sched_domains.
  224. */
  225. static DEFINE_MUTEX(sched_domains_mutex);
  226. #ifdef CONFIG_GROUP_SCHED
  227. #include <linux/cgroup.h>
  228. struct cfs_rq;
  229. static LIST_HEAD(task_groups);
  230. /* task group related information */
  231. struct task_group {
  232. #ifdef CONFIG_CGROUP_SCHED
  233. struct cgroup_subsys_state css;
  234. #endif
  235. #ifdef CONFIG_USER_SCHED
  236. uid_t uid;
  237. #endif
  238. #ifdef CONFIG_FAIR_GROUP_SCHED
  239. /* schedulable entities of this group on each cpu */
  240. struct sched_entity **se;
  241. /* runqueue "owned" by this group on each cpu */
  242. struct cfs_rq **cfs_rq;
  243. unsigned long shares;
  244. #endif
  245. #ifdef CONFIG_RT_GROUP_SCHED
  246. struct sched_rt_entity **rt_se;
  247. struct rt_rq **rt_rq;
  248. struct rt_bandwidth rt_bandwidth;
  249. #endif
  250. struct rcu_head rcu;
  251. struct list_head list;
  252. struct task_group *parent;
  253. struct list_head siblings;
  254. struct list_head children;
  255. };
  256. #ifdef CONFIG_USER_SCHED
  257. /* Helper function to pass uid information to create_sched_user() */
  258. void set_tg_uid(struct user_struct *user)
  259. {
  260. user->tg->uid = user->uid;
  261. }
  262. /*
  263. * Root task group.
  264. * Every UID task group (including init_task_group aka UID-0) will
  265. * be a child to this group.
  266. */
  267. struct task_group root_task_group;
  268. #ifdef CONFIG_FAIR_GROUP_SCHED
  269. /* Default task group's sched entity on each cpu */
  270. static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
  271. /* Default task group's cfs_rq on each cpu */
  272. static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
  273. #endif /* CONFIG_FAIR_GROUP_SCHED */
  274. #ifdef CONFIG_RT_GROUP_SCHED
  275. static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
  276. static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
  277. #endif /* CONFIG_RT_GROUP_SCHED */
  278. #else /* !CONFIG_USER_SCHED */
  279. #define root_task_group init_task_group
  280. #endif /* CONFIG_USER_SCHED */
  281. /* task_group_lock serializes add/remove of task groups and also changes to
  282. * a task group's cpu shares.
  283. */
  284. static DEFINE_SPINLOCK(task_group_lock);
  285. #ifdef CONFIG_SMP
  286. static int root_task_group_empty(void)
  287. {
  288. return list_empty(&root_task_group.children);
  289. }
  290. #endif
  291. #ifdef CONFIG_FAIR_GROUP_SCHED
  292. #ifdef CONFIG_USER_SCHED
  293. # define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD)
  294. #else /* !CONFIG_USER_SCHED */
  295. # define INIT_TASK_GROUP_LOAD NICE_0_LOAD
  296. #endif /* CONFIG_USER_SCHED */
  297. /*
  298. * A weight of 0 or 1 can cause arithmetics problems.
  299. * A weight of a cfs_rq is the sum of weights of which entities
  300. * are queued on this cfs_rq, so a weight of a entity should not be
  301. * too large, so as the shares value of a task group.
  302. * (The default weight is 1024 - so there's no practical
  303. * limitation from this.)
  304. */
  305. #define MIN_SHARES 2
  306. #define MAX_SHARES (1UL << 18)
  307. static int init_task_group_load = INIT_TASK_GROUP_LOAD;
  308. #endif
  309. /* Default task group.
  310. * Every task in system belong to this group at bootup.
  311. */
  312. struct task_group init_task_group;
  313. /* return group to which a task belongs */
  314. static inline struct task_group *task_group(struct task_struct *p)
  315. {
  316. struct task_group *tg;
  317. #ifdef CONFIG_USER_SCHED
  318. rcu_read_lock();
  319. tg = __task_cred(p)->user->tg;
  320. rcu_read_unlock();
  321. #elif defined(CONFIG_CGROUP_SCHED)
  322. tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
  323. struct task_group, css);
  324. #else
  325. tg = &init_task_group;
  326. #endif
  327. return tg;
  328. }
  329. /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
  330. static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
  331. {
  332. #ifdef CONFIG_FAIR_GROUP_SCHED
  333. p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
  334. p->se.parent = task_group(p)->se[cpu];
  335. #endif
  336. #ifdef CONFIG_RT_GROUP_SCHED
  337. p->rt.rt_rq = task_group(p)->rt_rq[cpu];
  338. p->rt.parent = task_group(p)->rt_se[cpu];
  339. #endif
  340. }
  341. #else
  342. #ifdef CONFIG_SMP
  343. static int root_task_group_empty(void)
  344. {
  345. return 1;
  346. }
  347. #endif
  348. static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
  349. static inline struct task_group *task_group(struct task_struct *p)
  350. {
  351. return NULL;
  352. }
  353. #endif /* CONFIG_GROUP_SCHED */
  354. /* CFS-related fields in a runqueue */
  355. struct cfs_rq {
  356. struct load_weight load;
  357. unsigned long nr_running;
  358. u64 exec_clock;
  359. u64 min_vruntime;
  360. struct rb_root tasks_timeline;
  361. struct rb_node *rb_leftmost;
  362. struct list_head tasks;
  363. struct list_head *balance_iterator;
  364. /*
  365. * 'curr' points to currently running entity on this cfs_rq.
  366. * It is set to NULL otherwise (i.e when none are currently running).
  367. */
  368. struct sched_entity *curr, *next, *last;
  369. unsigned int nr_spread_over;
  370. #ifdef CONFIG_FAIR_GROUP_SCHED
  371. struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */
  372. /*
  373. * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
  374. * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
  375. * (like users, containers etc.)
  376. *
  377. * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
  378. * list is used during load balance.
  379. */
  380. struct list_head leaf_cfs_rq_list;
  381. struct task_group *tg; /* group that "owns" this runqueue */
  382. #ifdef CONFIG_SMP
  383. /*
  384. * the part of load.weight contributed by tasks
  385. */
  386. unsigned long task_weight;
  387. /*
  388. * h_load = weight * f(tg)
  389. *
  390. * Where f(tg) is the recursive weight fraction assigned to
  391. * this group.
  392. */
  393. unsigned long h_load;
  394. /*
  395. * this cpu's part of tg->shares
  396. */
  397. unsigned long shares;
  398. /*
  399. * load.weight at the time we set shares
  400. */
  401. unsigned long rq_weight;
  402. #endif
  403. #endif
  404. };
  405. /* Real-Time classes' related field in a runqueue: */
  406. struct rt_rq {
  407. struct rt_prio_array active;
  408. unsigned long rt_nr_running;
  409. #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
  410. struct {
  411. int curr; /* highest queued rt task prio */
  412. #ifdef CONFIG_SMP
  413. int next; /* next highest */
  414. #endif
  415. } highest_prio;
  416. #endif
  417. #ifdef CONFIG_SMP
  418. unsigned long rt_nr_migratory;
  419. int overloaded;
  420. struct plist_head pushable_tasks;
  421. #endif
  422. int rt_throttled;
  423. u64 rt_time;
  424. u64 rt_runtime;
  425. /* Nests inside the rq lock: */
  426. spinlock_t rt_runtime_lock;
  427. #ifdef CONFIG_RT_GROUP_SCHED
  428. unsigned long rt_nr_boosted;
  429. struct rq *rq;
  430. struct list_head leaf_rt_rq_list;
  431. struct task_group *tg;
  432. struct sched_rt_entity *rt_se;
  433. #endif
  434. };
  435. #ifdef CONFIG_SMP
  436. /*
  437. * We add the notion of a root-domain which will be used to define per-domain
  438. * variables. Each exclusive cpuset essentially defines an island domain by
  439. * fully partitioning the member cpus from any other cpuset. Whenever a new
  440. * exclusive cpuset is created, we also create and attach a new root-domain
  441. * object.
  442. *
  443. */
  444. struct root_domain {
  445. atomic_t refcount;
  446. cpumask_var_t span;
  447. cpumask_var_t online;
  448. /*
  449. * The "RT overload" flag: it gets set if a CPU has more than
  450. * one runnable RT task.
  451. */
  452. cpumask_var_t rto_mask;
  453. atomic_t rto_count;
  454. #ifdef CONFIG_SMP
  455. struct cpupri cpupri;
  456. #endif
  457. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  458. /*
  459. * Preferred wake up cpu nominated by sched_mc balance that will be
  460. * used when most cpus are idle in the system indicating overall very
  461. * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
  462. */
  463. unsigned int sched_mc_preferred_wakeup_cpu;
  464. #endif
  465. };
  466. /*
  467. * By default the system creates a single root-domain with all cpus as
  468. * members (mimicking the global state we have today).
  469. */
  470. static struct root_domain def_root_domain;
  471. #endif
  472. /*
  473. * This is the main, per-CPU runqueue data structure.
  474. *
  475. * Locking rule: those places that want to lock multiple runqueues
  476. * (such as the load balancing or the thread migration code), lock
  477. * acquire operations must be ordered by ascending &runqueue.
  478. */
  479. struct rq {
  480. /* runqueue lock: */
  481. spinlock_t lock;
  482. /*
  483. * nr_running and cpu_load should be in the same cacheline because
  484. * remote CPUs use both these fields when doing load calculation.
  485. */
  486. unsigned long nr_running;
  487. #define CPU_LOAD_IDX_MAX 5
  488. unsigned long cpu_load[CPU_LOAD_IDX_MAX];
  489. #ifdef CONFIG_NO_HZ
  490. unsigned long last_tick_seen;
  491. unsigned char in_nohz_recently;
  492. #endif
  493. /* capture load from *all* tasks on this cpu: */
  494. struct load_weight load;
  495. unsigned long nr_load_updates;
  496. u64 nr_switches;
  497. struct cfs_rq cfs;
  498. struct rt_rq rt;
  499. #ifdef CONFIG_FAIR_GROUP_SCHED
  500. /* list of leaf cfs_rq on this cpu: */
  501. struct list_head leaf_cfs_rq_list;
  502. #endif
  503. #ifdef CONFIG_RT_GROUP_SCHED
  504. struct list_head leaf_rt_rq_list;
  505. #endif
  506. /*
  507. * This is part of a global counter where only the total sum
  508. * over all CPUs matters. A task can increase this counter on
  509. * one CPU and if it got migrated afterwards it may decrease
  510. * it on another CPU. Always updated under the runqueue lock:
  511. */
  512. unsigned long nr_uninterruptible;
  513. struct task_struct *curr, *idle;
  514. unsigned long next_balance;
  515. struct mm_struct *prev_mm;
  516. u64 clock;
  517. atomic_t nr_iowait;
  518. #ifdef CONFIG_SMP
  519. struct root_domain *rd;
  520. struct sched_domain *sd;
  521. unsigned char idle_at_tick;
  522. /* For active balancing */
  523. int active_balance;
  524. int push_cpu;
  525. /* cpu of this runqueue: */
  526. int cpu;
  527. int online;
  528. unsigned long avg_load_per_task;
  529. struct task_struct *migration_thread;
  530. struct list_head migration_queue;
  531. #endif
  532. #ifdef CONFIG_SCHED_HRTICK
  533. #ifdef CONFIG_SMP
  534. int hrtick_csd_pending;
  535. struct call_single_data hrtick_csd;
  536. #endif
  537. struct hrtimer hrtick_timer;
  538. #endif
  539. #ifdef CONFIG_SCHEDSTATS
  540. /* latency stats */
  541. struct sched_info rq_sched_info;
  542. unsigned long long rq_cpu_time;
  543. /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
  544. /* sys_sched_yield() stats */
  545. unsigned int yld_count;
  546. /* schedule() stats */
  547. unsigned int sched_switch;
  548. unsigned int sched_count;
  549. unsigned int sched_goidle;
  550. /* try_to_wake_up() stats */
  551. unsigned int ttwu_count;
  552. unsigned int ttwu_local;
  553. /* BKL stats */
  554. unsigned int bkl_count;
  555. #endif
  556. };
  557. static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
  558. static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync)
  559. {
  560. rq->curr->sched_class->check_preempt_curr(rq, p, sync);
  561. }
  562. static inline int cpu_of(struct rq *rq)
  563. {
  564. #ifdef CONFIG_SMP
  565. return rq->cpu;
  566. #else
  567. return 0;
  568. #endif
  569. }
  570. /*
  571. * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
  572. * See detach_destroy_domains: synchronize_sched for details.
  573. *
  574. * The domain tree of any CPU may only be accessed from within
  575. * preempt-disabled sections.
  576. */
  577. #define for_each_domain(cpu, __sd) \
  578. for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
  579. #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
  580. #define this_rq() (&__get_cpu_var(runqueues))
  581. #define task_rq(p) cpu_rq(task_cpu(p))
  582. #define cpu_curr(cpu) (cpu_rq(cpu)->curr)
  583. static inline void update_rq_clock(struct rq *rq)
  584. {
  585. rq->clock = sched_clock_cpu(cpu_of(rq));
  586. }
  587. /*
  588. * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
  589. */
  590. #ifdef CONFIG_SCHED_DEBUG
  591. # define const_debug __read_mostly
  592. #else
  593. # define const_debug static const
  594. #endif
  595. /**
  596. * runqueue_is_locked
  597. *
  598. * Returns true if the current cpu runqueue is locked.
  599. * This interface allows printk to be called with the runqueue lock
  600. * held and know whether or not it is OK to wake up the klogd.
  601. */
  602. int runqueue_is_locked(void)
  603. {
  604. int cpu = get_cpu();
  605. struct rq *rq = cpu_rq(cpu);
  606. int ret;
  607. ret = spin_is_locked(&rq->lock);
  608. put_cpu();
  609. return ret;
  610. }
  611. /*
  612. * Debugging: various feature bits
  613. */
  614. #define SCHED_FEAT(name, enabled) \
  615. __SCHED_FEAT_##name ,
  616. enum {
  617. #include "sched_features.h"
  618. };
  619. #undef SCHED_FEAT
  620. #define SCHED_FEAT(name, enabled) \
  621. (1UL << __SCHED_FEAT_##name) * enabled |
  622. const_debug unsigned int sysctl_sched_features =
  623. #include "sched_features.h"
  624. 0;
  625. #undef SCHED_FEAT
  626. #ifdef CONFIG_SCHED_DEBUG
  627. #define SCHED_FEAT(name, enabled) \
  628. #name ,
  629. static __read_mostly char *sched_feat_names[] = {
  630. #include "sched_features.h"
  631. NULL
  632. };
  633. #undef SCHED_FEAT
  634. static int sched_feat_show(struct seq_file *m, void *v)
  635. {
  636. int i;
  637. for (i = 0; sched_feat_names[i]; i++) {
  638. if (!(sysctl_sched_features & (1UL << i)))
  639. seq_puts(m, "NO_");
  640. seq_printf(m, "%s ", sched_feat_names[i]);
  641. }
  642. seq_puts(m, "\n");
  643. return 0;
  644. }
  645. static ssize_t
  646. sched_feat_write(struct file *filp, const char __user *ubuf,
  647. size_t cnt, loff_t *ppos)
  648. {
  649. char buf[64];
  650. char *cmp = buf;
  651. int neg = 0;
  652. int i;
  653. if (cnt > 63)
  654. cnt = 63;
  655. if (copy_from_user(&buf, ubuf, cnt))
  656. return -EFAULT;
  657. buf[cnt] = 0;
  658. if (strncmp(buf, "NO_", 3) == 0) {
  659. neg = 1;
  660. cmp += 3;
  661. }
  662. for (i = 0; sched_feat_names[i]; i++) {
  663. int len = strlen(sched_feat_names[i]);
  664. if (strncmp(cmp, sched_feat_names[i], len) == 0) {
  665. if (neg)
  666. sysctl_sched_features &= ~(1UL << i);
  667. else
  668. sysctl_sched_features |= (1UL << i);
  669. break;
  670. }
  671. }
  672. if (!sched_feat_names[i])
  673. return -EINVAL;
  674. filp->f_pos += cnt;
  675. return cnt;
  676. }
  677. static int sched_feat_open(struct inode *inode, struct file *filp)
  678. {
  679. return single_open(filp, sched_feat_show, NULL);
  680. }
  681. static struct file_operations sched_feat_fops = {
  682. .open = sched_feat_open,
  683. .write = sched_feat_write,
  684. .read = seq_read,
  685. .llseek = seq_lseek,
  686. .release = single_release,
  687. };
  688. static __init int sched_init_debug(void)
  689. {
  690. debugfs_create_file("sched_features", 0644, NULL, NULL,
  691. &sched_feat_fops);
  692. return 0;
  693. }
  694. late_initcall(sched_init_debug);
  695. #endif
  696. #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
  697. /*
  698. * Number of tasks to iterate in a single balance run.
  699. * Limited because this is done with IRQs disabled.
  700. */
  701. const_debug unsigned int sysctl_sched_nr_migrate = 32;
  702. /*
  703. * ratelimit for updating the group shares.
  704. * default: 0.25ms
  705. */
  706. unsigned int sysctl_sched_shares_ratelimit = 250000;
  707. /*
  708. * Inject some fuzzyness into changing the per-cpu group shares
  709. * this avoids remote rq-locks at the expense of fairness.
  710. * default: 4
  711. */
  712. unsigned int sysctl_sched_shares_thresh = 4;
  713. /*
  714. * period over which we measure -rt task cpu usage in us.
  715. * default: 1s
  716. */
  717. unsigned int sysctl_sched_rt_period = 1000000;
  718. static __read_mostly int scheduler_running;
  719. /*
  720. * part of the period that we allow rt tasks to run in us.
  721. * default: 0.95s
  722. */
  723. int sysctl_sched_rt_runtime = 950000;
  724. static inline u64 global_rt_period(void)
  725. {
  726. return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
  727. }
  728. static inline u64 global_rt_runtime(void)
  729. {
  730. if (sysctl_sched_rt_runtime < 0)
  731. return RUNTIME_INF;
  732. return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
  733. }
  734. #ifndef prepare_arch_switch
  735. # define prepare_arch_switch(next) do { } while (0)
  736. #endif
  737. #ifndef finish_arch_switch
  738. # define finish_arch_switch(prev) do { } while (0)
  739. #endif
  740. static inline int task_current(struct rq *rq, struct task_struct *p)
  741. {
  742. return rq->curr == p;
  743. }
  744. #ifndef __ARCH_WANT_UNLOCKED_CTXSW
  745. static inline int task_running(struct rq *rq, struct task_struct *p)
  746. {
  747. return task_current(rq, p);
  748. }
  749. static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
  750. {
  751. }
  752. static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
  753. {
  754. #ifdef CONFIG_DEBUG_SPINLOCK
  755. /* this is a valid case when another task releases the spinlock */
  756. rq->lock.owner = current;
  757. #endif
  758. /*
  759. * If we are tracking spinlock dependencies then we have to
  760. * fix up the runqueue lock - which gets 'carried over' from
  761. * prev into current:
  762. */
  763. spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
  764. spin_unlock_irq(&rq->lock);
  765. }
  766. #else /* __ARCH_WANT_UNLOCKED_CTXSW */
  767. static inline int task_running(struct rq *rq, struct task_struct *p)
  768. {
  769. #ifdef CONFIG_SMP
  770. return p->oncpu;
  771. #else
  772. return task_current(rq, p);
  773. #endif
  774. }
  775. static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
  776. {
  777. #ifdef CONFIG_SMP
  778. /*
  779. * We can optimise this out completely for !SMP, because the
  780. * SMP rebalancing from interrupt is the only thing that cares
  781. * here.
  782. */
  783. next->oncpu = 1;
  784. #endif
  785. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  786. spin_unlock_irq(&rq->lock);
  787. #else
  788. spin_unlock(&rq->lock);
  789. #endif
  790. }
  791. static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
  792. {
  793. #ifdef CONFIG_SMP
  794. /*
  795. * After ->oncpu is cleared, the task can be moved to a different CPU.
  796. * We must ensure this doesn't happen until the switch is completely
  797. * finished.
  798. */
  799. smp_wmb();
  800. prev->oncpu = 0;
  801. #endif
  802. #ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  803. local_irq_enable();
  804. #endif
  805. }
  806. #endif /* __ARCH_WANT_UNLOCKED_CTXSW */
  807. /*
  808. * __task_rq_lock - lock the runqueue a given task resides on.
  809. * Must be called interrupts disabled.
  810. */
  811. static inline struct rq *__task_rq_lock(struct task_struct *p)
  812. __acquires(rq->lock)
  813. {
  814. for (;;) {
  815. struct rq *rq = task_rq(p);
  816. spin_lock(&rq->lock);
  817. if (likely(rq == task_rq(p)))
  818. return rq;
  819. spin_unlock(&rq->lock);
  820. }
  821. }
  822. /*
  823. * task_rq_lock - lock the runqueue a given task resides on and disable
  824. * interrupts. Note the ordering: we can safely lookup the task_rq without
  825. * explicitly disabling preemption.
  826. */
  827. static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
  828. __acquires(rq->lock)
  829. {
  830. struct rq *rq;
  831. for (;;) {
  832. local_irq_save(*flags);
  833. rq = task_rq(p);
  834. spin_lock(&rq->lock);
  835. if (likely(rq == task_rq(p)))
  836. return rq;
  837. spin_unlock_irqrestore(&rq->lock, *flags);
  838. }
  839. }
  840. void task_rq_unlock_wait(struct task_struct *p)
  841. {
  842. struct rq *rq = task_rq(p);
  843. smp_mb(); /* spin-unlock-wait is not a full memory barrier */
  844. spin_unlock_wait(&rq->lock);
  845. }
  846. static void __task_rq_unlock(struct rq *rq)
  847. __releases(rq->lock)
  848. {
  849. spin_unlock(&rq->lock);
  850. }
  851. static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
  852. __releases(rq->lock)
  853. {
  854. spin_unlock_irqrestore(&rq->lock, *flags);
  855. }
  856. /*
  857. * this_rq_lock - lock this runqueue and disable interrupts.
  858. */
  859. static struct rq *this_rq_lock(void)
  860. __acquires(rq->lock)
  861. {
  862. struct rq *rq;
  863. local_irq_disable();
  864. rq = this_rq();
  865. spin_lock(&rq->lock);
  866. return rq;
  867. }
  868. #ifdef CONFIG_SCHED_HRTICK
  869. /*
  870. * Use HR-timers to deliver accurate preemption points.
  871. *
  872. * Its all a bit involved since we cannot program an hrt while holding the
  873. * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
  874. * reschedule event.
  875. *
  876. * When we get rescheduled we reprogram the hrtick_timer outside of the
  877. * rq->lock.
  878. */
  879. /*
  880. * Use hrtick when:
  881. * - enabled by features
  882. * - hrtimer is actually high res
  883. */
  884. static inline int hrtick_enabled(struct rq *rq)
  885. {
  886. if (!sched_feat(HRTICK))
  887. return 0;
  888. if (!cpu_active(cpu_of(rq)))
  889. return 0;
  890. return hrtimer_is_hres_active(&rq->hrtick_timer);
  891. }
  892. static void hrtick_clear(struct rq *rq)
  893. {
  894. if (hrtimer_active(&rq->hrtick_timer))
  895. hrtimer_cancel(&rq->hrtick_timer);
  896. }
  897. /*
  898. * High-resolution timer tick.
  899. * Runs from hardirq context with interrupts disabled.
  900. */
  901. static enum hrtimer_restart hrtick(struct hrtimer *timer)
  902. {
  903. struct rq *rq = container_of(timer, struct rq, hrtick_timer);
  904. WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
  905. spin_lock(&rq->lock);
  906. update_rq_clock(rq);
  907. rq->curr->sched_class->task_tick(rq, rq->curr, 1);
  908. spin_unlock(&rq->lock);
  909. return HRTIMER_NORESTART;
  910. }
  911. #ifdef CONFIG_SMP
  912. /*
  913. * called from hardirq (IPI) context
  914. */
  915. static void __hrtick_start(void *arg)
  916. {
  917. struct rq *rq = arg;
  918. spin_lock(&rq->lock);
  919. hrtimer_restart(&rq->hrtick_timer);
  920. rq->hrtick_csd_pending = 0;
  921. spin_unlock(&rq->lock);
  922. }
  923. /*
  924. * Called to set the hrtick timer state.
  925. *
  926. * called with rq->lock held and irqs disabled
  927. */
  928. static void hrtick_start(struct rq *rq, u64 delay)
  929. {
  930. struct hrtimer *timer = &rq->hrtick_timer;
  931. ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
  932. hrtimer_set_expires(timer, time);
  933. if (rq == this_rq()) {
  934. hrtimer_restart(timer);
  935. } else if (!rq->hrtick_csd_pending) {
  936. __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);
  937. rq->hrtick_csd_pending = 1;
  938. }
  939. }
  940. static int
  941. hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
  942. {
  943. int cpu = (int)(long)hcpu;
  944. switch (action) {
  945. case CPU_UP_CANCELED:
  946. case CPU_UP_CANCELED_FROZEN:
  947. case CPU_DOWN_PREPARE:
  948. case CPU_DOWN_PREPARE_FROZEN:
  949. case CPU_DEAD:
  950. case CPU_DEAD_FROZEN:
  951. hrtick_clear(cpu_rq(cpu));
  952. return NOTIFY_OK;
  953. }
  954. return NOTIFY_DONE;
  955. }
  956. static __init void init_hrtick(void)
  957. {
  958. hotcpu_notifier(hotplug_hrtick, 0);
  959. }
  960. #else
  961. /*
  962. * Called to set the hrtick timer state.
  963. *
  964. * called with rq->lock held and irqs disabled
  965. */
  966. static void hrtick_start(struct rq *rq, u64 delay)
  967. {
  968. __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
  969. HRTIMER_MODE_REL, 0);
  970. }
  971. static inline void init_hrtick(void)
  972. {
  973. }
  974. #endif /* CONFIG_SMP */
  975. static void init_rq_hrtick(struct rq *rq)
  976. {
  977. #ifdef CONFIG_SMP
  978. rq->hrtick_csd_pending = 0;
  979. rq->hrtick_csd.flags = 0;
  980. rq->hrtick_csd.func = __hrtick_start;
  981. rq->hrtick_csd.info = rq;
  982. #endif
  983. hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  984. rq->hrtick_timer.function = hrtick;
  985. }
  986. #else /* CONFIG_SCHED_HRTICK */
  987. static inline void hrtick_clear(struct rq *rq)
  988. {
  989. }
  990. static inline void init_rq_hrtick(struct rq *rq)
  991. {
  992. }
  993. static inline void init_hrtick(void)
  994. {
  995. }
  996. #endif /* CONFIG_SCHED_HRTICK */
  997. /*
  998. * resched_task - mark a task 'to be rescheduled now'.
  999. *
  1000. * On UP this means the setting of the need_resched flag, on SMP it
  1001. * might also involve a cross-CPU call to trigger the scheduler on
  1002. * the target CPU.
  1003. */
  1004. #ifdef CONFIG_SMP
  1005. #ifndef tsk_is_polling
  1006. #define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
  1007. #endif
  1008. static void resched_task(struct task_struct *p)
  1009. {
  1010. int cpu;
  1011. assert_spin_locked(&task_rq(p)->lock);
  1012. if (test_tsk_need_resched(p))
  1013. return;
  1014. set_tsk_need_resched(p);
  1015. cpu = task_cpu(p);
  1016. if (cpu == smp_processor_id())
  1017. return;
  1018. /* NEED_RESCHED must be visible before we test polling */
  1019. smp_mb();
  1020. if (!tsk_is_polling(p))
  1021. smp_send_reschedule(cpu);
  1022. }
  1023. static void resched_cpu(int cpu)
  1024. {
  1025. struct rq *rq = cpu_rq(cpu);
  1026. unsigned long flags;
  1027. if (!spin_trylock_irqsave(&rq->lock, flags))
  1028. return;
  1029. resched_task(cpu_curr(cpu));
  1030. spin_unlock_irqrestore(&rq->lock, flags);
  1031. }
  1032. #ifdef CONFIG_NO_HZ
  1033. /*
  1034. * When add_timer_on() enqueues a timer into the timer wheel of an
  1035. * idle CPU then this timer might expire before the next timer event
  1036. * which is scheduled to wake up that CPU. In case of a completely
  1037. * idle system the next event might even be infinite time into the
  1038. * future. wake_up_idle_cpu() ensures that the CPU is woken up and
  1039. * leaves the inner idle loop so the newly added timer is taken into
  1040. * account when the CPU goes back to idle and evaluates the timer
  1041. * wheel for the next timer event.
  1042. */
  1043. void wake_up_idle_cpu(int cpu)
  1044. {
  1045. struct rq *rq = cpu_rq(cpu);
  1046. if (cpu == smp_processor_id())
  1047. return;
  1048. /*
  1049. * This is safe, as this function is called with the timer
  1050. * wheel base lock of (cpu) held. When the CPU is on the way
  1051. * to idle and has not yet set rq->curr to idle then it will
  1052. * be serialized on the timer wheel base lock and take the new
  1053. * timer into account automatically.
  1054. */
  1055. if (rq->curr != rq->idle)
  1056. return;
  1057. /*
  1058. * We can set TIF_RESCHED on the idle task of the other CPU
  1059. * lockless. The worst case is that the other CPU runs the
  1060. * idle task through an additional NOOP schedule()
  1061. */
  1062. set_tsk_need_resched(rq->idle);
  1063. /* NEED_RESCHED must be visible before we test polling */
  1064. smp_mb();
  1065. if (!tsk_is_polling(rq->idle))
  1066. smp_send_reschedule(cpu);
  1067. }
  1068. #endif /* CONFIG_NO_HZ */
  1069. #else /* !CONFIG_SMP */
  1070. static void resched_task(struct task_struct *p)
  1071. {
  1072. assert_spin_locked(&task_rq(p)->lock);
  1073. set_tsk_need_resched(p);
  1074. }
  1075. #endif /* CONFIG_SMP */
  1076. #if BITS_PER_LONG == 32
  1077. # define WMULT_CONST (~0UL)
  1078. #else
  1079. # define WMULT_CONST (1UL << 32)
  1080. #endif
  1081. #define WMULT_SHIFT 32
  1082. /*
  1083. * Shift right and round:
  1084. */
  1085. #define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
  1086. /*
  1087. * delta *= weight / lw
  1088. */
  1089. static unsigned long
  1090. calc_delta_mine(unsigned long delta_exec, unsigned long weight,
  1091. struct load_weight *lw)
  1092. {
  1093. u64 tmp;
  1094. if (!lw->inv_weight) {
  1095. if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
  1096. lw->inv_weight = 1;
  1097. else
  1098. lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
  1099. / (lw->weight+1);
  1100. }
  1101. tmp = (u64)delta_exec * weight;
  1102. /*
  1103. * Check whether we'd overflow the 64-bit multiplication:
  1104. */
  1105. if (unlikely(tmp > WMULT_CONST))
  1106. tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
  1107. WMULT_SHIFT/2);
  1108. else
  1109. tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
  1110. return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
  1111. }
  1112. static inline void update_load_add(struct load_weight *lw, unsigned long inc)
  1113. {
  1114. lw->weight += inc;
  1115. lw->inv_weight = 0;
  1116. }
  1117. static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
  1118. {
  1119. lw->weight -= dec;
  1120. lw->inv_weight = 0;
  1121. }
  1122. /*
  1123. * To aid in avoiding the subversion of "niceness" due to uneven distribution
  1124. * of tasks with abnormal "nice" values across CPUs the contribution that
  1125. * each task makes to its run queue's load is weighted according to its
  1126. * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
  1127. * scaled version of the new time slice allocation that they receive on time
  1128. * slice expiry etc.
  1129. */
  1130. #define WEIGHT_IDLEPRIO 3
  1131. #define WMULT_IDLEPRIO 1431655765
  1132. /*
  1133. * Nice levels are multiplicative, with a gentle 10% change for every
  1134. * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
  1135. * nice 1, it will get ~10% less CPU time than another CPU-bound task
  1136. * that remained on nice 0.
  1137. *
  1138. * The "10% effect" is relative and cumulative: from _any_ nice level,
  1139. * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
  1140. * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
  1141. * If a task goes up by ~10% and another task goes down by ~10% then
  1142. * the relative distance between them is ~25%.)
  1143. */
  1144. static const int prio_to_weight[40] = {
  1145. /* -20 */ 88761, 71755, 56483, 46273, 36291,
  1146. /* -15 */ 29154, 23254, 18705, 14949, 11916,
  1147. /* -10 */ 9548, 7620, 6100, 4904, 3906,
  1148. /* -5 */ 3121, 2501, 1991, 1586, 1277,
  1149. /* 0 */ 1024, 820, 655, 526, 423,
  1150. /* 5 */ 335, 272, 215, 172, 137,
  1151. /* 10 */ 110, 87, 70, 56, 45,
  1152. /* 15 */ 36, 29, 23, 18, 15,
  1153. };
  1154. /*
  1155. * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
  1156. *
  1157. * In cases where the weight does not change often, we can use the
  1158. * precalculated inverse to speed up arithmetics by turning divisions
  1159. * into multiplications:
  1160. */
  1161. static const u32 prio_to_wmult[40] = {
  1162. /* -20 */ 48388, 59856, 76040, 92818, 118348,
  1163. /* -15 */ 147320, 184698, 229616, 287308, 360437,
  1164. /* -10 */ 449829, 563644, 704093, 875809, 1099582,
  1165. /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
  1166. /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
  1167. /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
  1168. /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
  1169. /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
  1170. };
  1171. static void activate_task(struct rq *rq, struct task_struct *p, int wakeup);
  1172. /*
  1173. * runqueue iterator, to support SMP load-balancing between different
  1174. * scheduling classes, without having to expose their internal data
  1175. * structures to the load-balancing proper:
  1176. */
  1177. struct rq_iterator {
  1178. void *arg;
  1179. struct task_struct *(*start)(void *);
  1180. struct task_struct *(*next)(void *);
  1181. };
  1182. #ifdef CONFIG_SMP
  1183. static unsigned long
  1184. balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
  1185. unsigned long max_load_move, struct sched_domain *sd,
  1186. enum cpu_idle_type idle, int *all_pinned,
  1187. int *this_best_prio, struct rq_iterator *iterator);
  1188. static int
  1189. iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
  1190. struct sched_domain *sd, enum cpu_idle_type idle,
  1191. struct rq_iterator *iterator);
  1192. #endif
  1193. #ifdef CONFIG_CGROUP_CPUACCT
  1194. static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
  1195. #else
  1196. static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
  1197. #endif
  1198. static inline void inc_cpu_load(struct rq *rq, unsigned long load)
  1199. {
  1200. update_load_add(&rq->load, load);
  1201. }
  1202. static inline void dec_cpu_load(struct rq *rq, unsigned long load)
  1203. {
  1204. update_load_sub(&rq->load, load);
  1205. }
  1206. #if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) || defined(CONFIG_RT_GROUP_SCHED)
  1207. typedef int (*tg_visitor)(struct task_group *, void *);
  1208. /*
  1209. * Iterate the full tree, calling @down when first entering a node and @up when
  1210. * leaving it for the final time.
  1211. */
  1212. static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
  1213. {
  1214. struct task_group *parent, *child;
  1215. int ret;
  1216. rcu_read_lock();
  1217. parent = &root_task_group;
  1218. down:
  1219. ret = (*down)(parent, data);
  1220. if (ret)
  1221. goto out_unlock;
  1222. list_for_each_entry_rcu(child, &parent->children, siblings) {
  1223. parent = child;
  1224. goto down;
  1225. up:
  1226. continue;
  1227. }
  1228. ret = (*up)(parent, data);
  1229. if (ret)
  1230. goto out_unlock;
  1231. child = parent;
  1232. parent = parent->parent;
  1233. if (parent)
  1234. goto up;
  1235. out_unlock:
  1236. rcu_read_unlock();
  1237. return ret;
  1238. }
  1239. static int tg_nop(struct task_group *tg, void *data)
  1240. {
  1241. return 0;
  1242. }
  1243. #endif
  1244. #ifdef CONFIG_SMP
  1245. static unsigned long source_load(int cpu, int type);
  1246. static unsigned long target_load(int cpu, int type);
  1247. static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
  1248. static unsigned long cpu_avg_load_per_task(int cpu)
  1249. {
  1250. struct rq *rq = cpu_rq(cpu);
  1251. unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
  1252. if (nr_running)
  1253. rq->avg_load_per_task = rq->load.weight / nr_running;
  1254. else
  1255. rq->avg_load_per_task = 0;
  1256. return rq->avg_load_per_task;
  1257. }
  1258. #ifdef CONFIG_FAIR_GROUP_SCHED
  1259. static void __set_se_shares(struct sched_entity *se, unsigned long shares);
  1260. /*
  1261. * Calculate and set the cpu's group shares.
  1262. */
  1263. static void
  1264. update_group_shares_cpu(struct task_group *tg, int cpu,
  1265. unsigned long sd_shares, unsigned long sd_rq_weight)
  1266. {
  1267. unsigned long shares;
  1268. unsigned long rq_weight;
  1269. if (!tg->se[cpu])
  1270. return;
  1271. rq_weight = tg->cfs_rq[cpu]->rq_weight;
  1272. /*
  1273. * \Sum shares * rq_weight
  1274. * shares = -----------------------
  1275. * \Sum rq_weight
  1276. *
  1277. */
  1278. shares = (sd_shares * rq_weight) / sd_rq_weight;
  1279. shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
  1280. if (abs(shares - tg->se[cpu]->load.weight) >
  1281. sysctl_sched_shares_thresh) {
  1282. struct rq *rq = cpu_rq(cpu);
  1283. unsigned long flags;
  1284. spin_lock_irqsave(&rq->lock, flags);
  1285. tg->cfs_rq[cpu]->shares = shares;
  1286. __set_se_shares(tg->se[cpu], shares);
  1287. spin_unlock_irqrestore(&rq->lock, flags);
  1288. }
  1289. }
  1290. /*
  1291. * Re-compute the task group their per cpu shares over the given domain.
  1292. * This needs to be done in a bottom-up fashion because the rq weight of a
  1293. * parent group depends on the shares of its child groups.
  1294. */
  1295. static int tg_shares_up(struct task_group *tg, void *data)
  1296. {
  1297. unsigned long weight, rq_weight = 0;
  1298. unsigned long shares = 0;
  1299. struct sched_domain *sd = data;
  1300. int i;
  1301. for_each_cpu(i, sched_domain_span(sd)) {
  1302. /*
  1303. * If there are currently no tasks on the cpu pretend there
  1304. * is one of average load so that when a new task gets to
  1305. * run here it will not get delayed by group starvation.
  1306. */
  1307. weight = tg->cfs_rq[i]->load.weight;
  1308. if (!weight)
  1309. weight = NICE_0_LOAD;
  1310. tg->cfs_rq[i]->rq_weight = weight;
  1311. rq_weight += weight;
  1312. shares += tg->cfs_rq[i]->shares;
  1313. }
  1314. if ((!shares && rq_weight) || shares > tg->shares)
  1315. shares = tg->shares;
  1316. if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
  1317. shares = tg->shares;
  1318. for_each_cpu(i, sched_domain_span(sd))
  1319. update_group_shares_cpu(tg, i, shares, rq_weight);
  1320. return 0;
  1321. }
  1322. /*
  1323. * Compute the cpu's hierarchical load factor for each task group.
  1324. * This needs to be done in a top-down fashion because the load of a child
  1325. * group is a fraction of its parents load.
  1326. */
  1327. static int tg_load_down(struct task_group *tg, void *data)
  1328. {
  1329. unsigned long load;
  1330. long cpu = (long)data;
  1331. if (!tg->parent) {
  1332. load = cpu_rq(cpu)->load.weight;
  1333. } else {
  1334. load = tg->parent->cfs_rq[cpu]->h_load;
  1335. load *= tg->cfs_rq[cpu]->shares;
  1336. load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
  1337. }
  1338. tg->cfs_rq[cpu]->h_load = load;
  1339. return 0;
  1340. }
  1341. static void update_shares(struct sched_domain *sd)
  1342. {
  1343. u64 now = cpu_clock(raw_smp_processor_id());
  1344. s64 elapsed = now - sd->last_update;
  1345. if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
  1346. sd->last_update = now;
  1347. walk_tg_tree(tg_nop, tg_shares_up, sd);
  1348. }
  1349. }
  1350. static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
  1351. {
  1352. spin_unlock(&rq->lock);
  1353. update_shares(sd);
  1354. spin_lock(&rq->lock);
  1355. }
  1356. static void update_h_load(long cpu)
  1357. {
  1358. walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
  1359. }
  1360. #else
  1361. static inline void update_shares(struct sched_domain *sd)
  1362. {
  1363. }
  1364. static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
  1365. {
  1366. }
  1367. #endif
  1368. #ifdef CONFIG_PREEMPT
  1369. /*
  1370. * fair double_lock_balance: Safely acquires both rq->locks in a fair
  1371. * way at the expense of forcing extra atomic operations in all
  1372. * invocations. This assures that the double_lock is acquired using the
  1373. * same underlying policy as the spinlock_t on this architecture, which
  1374. * reduces latency compared to the unfair variant below. However, it
  1375. * also adds more overhead and therefore may reduce throughput.
  1376. */
  1377. static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1378. __releases(this_rq->lock)
  1379. __acquires(busiest->lock)
  1380. __acquires(this_rq->lock)
  1381. {
  1382. spin_unlock(&this_rq->lock);
  1383. double_rq_lock(this_rq, busiest);
  1384. return 1;
  1385. }
  1386. #else
  1387. /*
  1388. * Unfair double_lock_balance: Optimizes throughput at the expense of
  1389. * latency by eliminating extra atomic operations when the locks are
  1390. * already in proper order on entry. This favors lower cpu-ids and will
  1391. * grant the double lock to lower cpus over higher ids under contention,
  1392. * regardless of entry order into the function.
  1393. */
  1394. static int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1395. __releases(this_rq->lock)
  1396. __acquires(busiest->lock)
  1397. __acquires(this_rq->lock)
  1398. {
  1399. int ret = 0;
  1400. if (unlikely(!spin_trylock(&busiest->lock))) {
  1401. if (busiest < this_rq) {
  1402. spin_unlock(&this_rq->lock);
  1403. spin_lock(&busiest->lock);
  1404. spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
  1405. ret = 1;
  1406. } else
  1407. spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
  1408. }
  1409. return ret;
  1410. }
  1411. #endif /* CONFIG_PREEMPT */
  1412. /*
  1413. * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
  1414. */
  1415. static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1416. {
  1417. if (unlikely(!irqs_disabled())) {
  1418. /* printk() doesn't work good under rq->lock */
  1419. spin_unlock(&this_rq->lock);
  1420. BUG_ON(1);
  1421. }
  1422. return _double_lock_balance(this_rq, busiest);
  1423. }
  1424. static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
  1425. __releases(busiest->lock)
  1426. {
  1427. spin_unlock(&busiest->lock);
  1428. lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
  1429. }
  1430. #endif
  1431. #ifdef CONFIG_FAIR_GROUP_SCHED
  1432. static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
  1433. {
  1434. #ifdef CONFIG_SMP
  1435. cfs_rq->shares = shares;
  1436. #endif
  1437. }
  1438. #endif
  1439. #include "sched_stats.h"
  1440. #include "sched_idletask.c"
  1441. #include "sched_fair.c"
  1442. #include "sched_rt.c"
  1443. #ifdef CONFIG_SCHED_DEBUG
  1444. # include "sched_debug.c"
  1445. #endif
  1446. #define sched_class_highest (&rt_sched_class)
  1447. #define for_each_class(class) \
  1448. for (class = sched_class_highest; class; class = class->next)
  1449. static void inc_nr_running(struct rq *rq)
  1450. {
  1451. rq->nr_running++;
  1452. }
  1453. static void dec_nr_running(struct rq *rq)
  1454. {
  1455. rq->nr_running--;
  1456. }
  1457. static void set_load_weight(struct task_struct *p)
  1458. {
  1459. if (task_has_rt_policy(p)) {
  1460. p->se.load.weight = prio_to_weight[0] * 2;
  1461. p->se.load.inv_weight = prio_to_wmult[0] >> 1;
  1462. return;
  1463. }
  1464. /*
  1465. * SCHED_IDLE tasks get minimal weight:
  1466. */
  1467. if (p->policy == SCHED_IDLE) {
  1468. p->se.load.weight = WEIGHT_IDLEPRIO;
  1469. p->se.load.inv_weight = WMULT_IDLEPRIO;
  1470. return;
  1471. }
  1472. p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
  1473. p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
  1474. }
  1475. static void update_avg(u64 *avg, u64 sample)
  1476. {
  1477. s64 diff = sample - *avg;
  1478. *avg += diff >> 3;
  1479. }
  1480. static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
  1481. {
  1482. if (wakeup)
  1483. p->se.start_runtime = p->se.sum_exec_runtime;
  1484. sched_info_queued(p);
  1485. p->sched_class->enqueue_task(rq, p, wakeup);
  1486. p->se.on_rq = 1;
  1487. }
  1488. static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
  1489. {
  1490. if (sleep) {
  1491. if (p->se.last_wakeup) {
  1492. update_avg(&p->se.avg_overlap,
  1493. p->se.sum_exec_runtime - p->se.last_wakeup);
  1494. p->se.last_wakeup = 0;
  1495. } else {
  1496. update_avg(&p->se.avg_wakeup,
  1497. sysctl_sched_wakeup_granularity);
  1498. }
  1499. }
  1500. sched_info_dequeued(p);
  1501. p->sched_class->dequeue_task(rq, p, sleep);
  1502. p->se.on_rq = 0;
  1503. }
  1504. /*
  1505. * __normal_prio - return the priority that is based on the static prio
  1506. */
  1507. static inline int __normal_prio(struct task_struct *p)
  1508. {
  1509. return p->static_prio;
  1510. }
  1511. /*
  1512. * Calculate the expected normal priority: i.e. priority
  1513. * without taking RT-inheritance into account. Might be
  1514. * boosted by interactivity modifiers. Changes upon fork,
  1515. * setprio syscalls, and whenever the interactivity
  1516. * estimator recalculates.
  1517. */
  1518. static inline int normal_prio(struct task_struct *p)
  1519. {
  1520. int prio;
  1521. if (task_has_rt_policy(p))
  1522. prio = MAX_RT_PRIO-1 - p->rt_priority;
  1523. else
  1524. prio = __normal_prio(p);
  1525. return prio;
  1526. }
  1527. /*
  1528. * Calculate the current priority, i.e. the priority
  1529. * taken into account by the scheduler. This value might
  1530. * be boosted by RT tasks, or might be boosted by
  1531. * interactivity modifiers. Will be RT if the task got
  1532. * RT-boosted. If not then it returns p->normal_prio.
  1533. */
  1534. static int effective_prio(struct task_struct *p)
  1535. {
  1536. p->normal_prio = normal_prio(p);
  1537. /*
  1538. * If we are RT tasks or we were boosted to RT priority,
  1539. * keep the priority unchanged. Otherwise, update priority
  1540. * to the normal priority:
  1541. */
  1542. if (!rt_prio(p->prio))
  1543. return p->normal_prio;
  1544. return p->prio;
  1545. }
  1546. /*
  1547. * activate_task - move a task to the runqueue.
  1548. */
  1549. static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
  1550. {
  1551. if (task_contributes_to_load(p))
  1552. rq->nr_uninterruptible--;
  1553. enqueue_task(rq, p, wakeup);
  1554. inc_nr_running(rq);
  1555. }
  1556. /*
  1557. * deactivate_task - remove a task from the runqueue.
  1558. */
  1559. static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
  1560. {
  1561. if (task_contributes_to_load(p))
  1562. rq->nr_uninterruptible++;
  1563. dequeue_task(rq, p, sleep);
  1564. dec_nr_running(rq);
  1565. }
  1566. /**
  1567. * task_curr - is this task currently executing on a CPU?
  1568. * @p: the task in question.
  1569. */
  1570. inline int task_curr(const struct task_struct *p)
  1571. {
  1572. return cpu_curr(task_cpu(p)) == p;
  1573. }
  1574. static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
  1575. {
  1576. set_task_rq(p, cpu);
  1577. #ifdef CONFIG_SMP
  1578. /*
  1579. * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
  1580. * successfuly executed on another CPU. We must ensure that updates of
  1581. * per-task data have been completed by this moment.
  1582. */
  1583. smp_wmb();
  1584. task_thread_info(p)->cpu = cpu;
  1585. #endif
  1586. }
  1587. static inline void check_class_changed(struct rq *rq, struct task_struct *p,
  1588. const struct sched_class *prev_class,
  1589. int oldprio, int running)
  1590. {
  1591. if (prev_class != p->sched_class) {
  1592. if (prev_class->switched_from)
  1593. prev_class->switched_from(rq, p, running);
  1594. p->sched_class->switched_to(rq, p, running);
  1595. } else
  1596. p->sched_class->prio_changed(rq, p, oldprio, running);
  1597. }
  1598. #ifdef CONFIG_SMP
  1599. /* Used instead of source_load when we know the type == 0 */
  1600. static unsigned long weighted_cpuload(const int cpu)
  1601. {
  1602. return cpu_rq(cpu)->load.weight;
  1603. }
  1604. /*
  1605. * Is this task likely cache-hot:
  1606. */
  1607. static int
  1608. task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
  1609. {
  1610. s64 delta;
  1611. /*
  1612. * Buddy candidates are cache hot:
  1613. */
  1614. if (sched_feat(CACHE_HOT_BUDDY) &&
  1615. (&p->se == cfs_rq_of(&p->se)->next ||
  1616. &p->se == cfs_rq_of(&p->se)->last))
  1617. return 1;
  1618. if (p->sched_class != &fair_sched_class)
  1619. return 0;
  1620. if (sysctl_sched_migration_cost == -1)
  1621. return 1;
  1622. if (sysctl_sched_migration_cost == 0)
  1623. return 0;
  1624. delta = now - p->se.exec_start;
  1625. return delta < (s64)sysctl_sched_migration_cost;
  1626. }
  1627. void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
  1628. {
  1629. int old_cpu = task_cpu(p);
  1630. struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
  1631. struct cfs_rq *old_cfsrq = task_cfs_rq(p),
  1632. *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
  1633. u64 clock_offset;
  1634. clock_offset = old_rq->clock - new_rq->clock;
  1635. trace_sched_migrate_task(p, task_cpu(p), new_cpu);
  1636. #ifdef CONFIG_SCHEDSTATS
  1637. if (p->se.wait_start)
  1638. p->se.wait_start -= clock_offset;
  1639. if (p->se.sleep_start)
  1640. p->se.sleep_start -= clock_offset;
  1641. if (p->se.block_start)
  1642. p->se.block_start -= clock_offset;
  1643. if (old_cpu != new_cpu) {
  1644. schedstat_inc(p, se.nr_migrations);
  1645. if (task_hot(p, old_rq->clock, NULL))
  1646. schedstat_inc(p, se.nr_forced2_migrations);
  1647. }
  1648. #endif
  1649. p->se.vruntime -= old_cfsrq->min_vruntime -
  1650. new_cfsrq->min_vruntime;
  1651. __set_task_cpu(p, new_cpu);
  1652. }
  1653. struct migration_req {
  1654. struct list_head list;
  1655. struct task_struct *task;
  1656. int dest_cpu;
  1657. struct completion done;
  1658. };
  1659. /*
  1660. * The task's runqueue lock must be held.
  1661. * Returns true if you have to wait for migration thread.
  1662. */
  1663. static int
  1664. migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
  1665. {
  1666. struct rq *rq = task_rq(p);
  1667. /*
  1668. * If the task is not on a runqueue (and not running), then
  1669. * it is sufficient to simply update the task's cpu field.
  1670. */
  1671. if (!p->se.on_rq && !task_running(rq, p)) {
  1672. set_task_cpu(p, dest_cpu);
  1673. return 0;
  1674. }
  1675. init_completion(&req->done);
  1676. req->task = p;
  1677. req->dest_cpu = dest_cpu;
  1678. list_add(&req->list, &rq->migration_queue);
  1679. return 1;
  1680. }
  1681. /*
  1682. * wait_task_inactive - wait for a thread to unschedule.
  1683. *
  1684. * If @match_state is nonzero, it's the @p->state value just checked and
  1685. * not expected to change. If it changes, i.e. @p might have woken up,
  1686. * then return zero. When we succeed in waiting for @p to be off its CPU,
  1687. * we return a positive number (its total switch count). If a second call
  1688. * a short while later returns the same number, the caller can be sure that
  1689. * @p has remained unscheduled the whole time.
  1690. *
  1691. * The caller must ensure that the task *will* unschedule sometime soon,
  1692. * else this function might spin for a *long* time. This function can't
  1693. * be called with interrupts off, or it may introduce deadlock with
  1694. * smp_call_function() if an IPI is sent by the same process we are
  1695. * waiting to become inactive.
  1696. */
  1697. unsigned long wait_task_inactive(struct task_struct *p, long match_state)
  1698. {
  1699. unsigned long flags;
  1700. int running, on_rq;
  1701. unsigned long ncsw;
  1702. struct rq *rq;
  1703. for (;;) {
  1704. /*
  1705. * We do the initial early heuristics without holding
  1706. * any task-queue locks at all. We'll only try to get
  1707. * the runqueue lock when things look like they will
  1708. * work out!
  1709. */
  1710. rq = task_rq(p);
  1711. /*
  1712. * If the task is actively running on another CPU
  1713. * still, just relax and busy-wait without holding
  1714. * any locks.
  1715. *
  1716. * NOTE! Since we don't hold any locks, it's not
  1717. * even sure that "rq" stays as the right runqueue!
  1718. * But we don't care, since "task_running()" will
  1719. * return false if the runqueue has changed and p
  1720. * is actually now running somewhere else!
  1721. */
  1722. while (task_running(rq, p)) {
  1723. if (match_state && unlikely(p->state != match_state))
  1724. return 0;
  1725. cpu_relax();
  1726. }
  1727. /*
  1728. * Ok, time to look more closely! We need the rq
  1729. * lock now, to be *sure*. If we're wrong, we'll
  1730. * just go back and repeat.
  1731. */
  1732. rq = task_rq_lock(p, &flags);
  1733. trace_sched_wait_task(rq, p);
  1734. running = task_running(rq, p);
  1735. on_rq = p->se.on_rq;
  1736. ncsw = 0;
  1737. if (!match_state || p->state == match_state)
  1738. ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
  1739. task_rq_unlock(rq, &flags);
  1740. /*
  1741. * If it changed from the expected state, bail out now.
  1742. */
  1743. if (unlikely(!ncsw))
  1744. break;
  1745. /*
  1746. * Was it really running after all now that we
  1747. * checked with the proper locks actually held?
  1748. *
  1749. * Oops. Go back and try again..
  1750. */
  1751. if (unlikely(running)) {
  1752. cpu_relax();
  1753. continue;
  1754. }
  1755. /*
  1756. * It's not enough that it's not actively running,
  1757. * it must be off the runqueue _entirely_, and not
  1758. * preempted!
  1759. *
  1760. * So if it was still runnable (but just not actively
  1761. * running right now), it's preempted, and we should
  1762. * yield - it could be a while.
  1763. */
  1764. if (unlikely(on_rq)) {
  1765. schedule_timeout_uninterruptible(1);
  1766. continue;
  1767. }
  1768. /*
  1769. * Ahh, all good. It wasn't running, and it wasn't
  1770. * runnable, which means that it will never become
  1771. * running in the future either. We're all done!
  1772. */
  1773. break;
  1774. }
  1775. return ncsw;
  1776. }
  1777. /***
  1778. * kick_process - kick a running thread to enter/exit the kernel
  1779. * @p: the to-be-kicked thread
  1780. *
  1781. * Cause a process which is running on another CPU to enter
  1782. * kernel-mode, without any delay. (to get signals handled.)
  1783. *
  1784. * NOTE: this function doesnt have to take the runqueue lock,
  1785. * because all it wants to ensure is that the remote task enters
  1786. * the kernel. If the IPI races and the task has been migrated
  1787. * to another CPU then no harm is done and the purpose has been
  1788. * achieved as well.
  1789. */
  1790. void kick_process(struct task_struct *p)
  1791. {
  1792. int cpu;
  1793. preempt_disable();
  1794. cpu = task_cpu(p);
  1795. if ((cpu != smp_processor_id()) && task_curr(p))
  1796. smp_send_reschedule(cpu);
  1797. preempt_enable();
  1798. }
  1799. /*
  1800. * Return a low guess at the load of a migration-source cpu weighted
  1801. * according to the scheduling class and "nice" value.
  1802. *
  1803. * We want to under-estimate the load of migration sources, to
  1804. * balance conservatively.
  1805. */
  1806. static unsigned long source_load(int cpu, int type)
  1807. {
  1808. struct rq *rq = cpu_rq(cpu);
  1809. unsigned long total = weighted_cpuload(cpu);
  1810. if (type == 0 || !sched_feat(LB_BIAS))
  1811. return total;
  1812. return min(rq->cpu_load[type-1], total);
  1813. }
  1814. /*
  1815. * Return a high guess at the load of a migration-target cpu weighted
  1816. * according to the scheduling class and "nice" value.
  1817. */
  1818. static unsigned long target_load(int cpu, int type)
  1819. {
  1820. struct rq *rq = cpu_rq(cpu);
  1821. unsigned long total = weighted_cpuload(cpu);
  1822. if (type == 0 || !sched_feat(LB_BIAS))
  1823. return total;
  1824. return max(rq->cpu_load[type-1], total);
  1825. }
  1826. /*
  1827. * find_idlest_group finds and returns the least busy CPU group within the
  1828. * domain.
  1829. */
  1830. static struct sched_group *
  1831. find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
  1832. {
  1833. struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups;
  1834. unsigned long min_load = ULONG_MAX, this_load = 0;
  1835. int load_idx = sd->forkexec_idx;
  1836. int imbalance = 100 + (sd->imbalance_pct-100)/2;
  1837. do {
  1838. unsigned long load, avg_load;
  1839. int local_group;
  1840. int i;
  1841. /* Skip over this group if it has no CPUs allowed */
  1842. if (!cpumask_intersects(sched_group_cpus(group),
  1843. &p->cpus_allowed))
  1844. continue;
  1845. local_group = cpumask_test_cpu(this_cpu,
  1846. sched_group_cpus(group));
  1847. /* Tally up the load of all CPUs in the group */
  1848. avg_load = 0;
  1849. for_each_cpu(i, sched_group_cpus(group)) {
  1850. /* Bias balancing toward cpus of our domain */
  1851. if (local_group)
  1852. load = source_load(i, load_idx);
  1853. else
  1854. load = target_load(i, load_idx);
  1855. avg_load += load;
  1856. }
  1857. /* Adjust by relative CPU power of the group */
  1858. avg_load = sg_div_cpu_power(group,
  1859. avg_load * SCHED_LOAD_SCALE);
  1860. if (local_group) {
  1861. this_load = avg_load;
  1862. this = group;
  1863. } else if (avg_load < min_load) {
  1864. min_load = avg_load;
  1865. idlest = group;
  1866. }
  1867. } while (group = group->next, group != sd->groups);
  1868. if (!idlest || 100*this_load < imbalance*min_load)
  1869. return NULL;
  1870. return idlest;
  1871. }
  1872. /*
  1873. * find_idlest_cpu - find the idlest cpu among the cpus in group.
  1874. */
  1875. static int
  1876. find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
  1877. {
  1878. unsigned long load, min_load = ULONG_MAX;
  1879. int idlest = -1;
  1880. int i;
  1881. /* Traverse only the allowed CPUs */
  1882. for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
  1883. load = weighted_cpuload(i);
  1884. if (load < min_load || (load == min_load && i == this_cpu)) {
  1885. min_load = load;
  1886. idlest = i;
  1887. }
  1888. }
  1889. return idlest;
  1890. }
  1891. /*
  1892. * sched_balance_self: balance the current task (running on cpu) in domains
  1893. * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
  1894. * SD_BALANCE_EXEC.
  1895. *
  1896. * Balance, ie. select the least loaded group.
  1897. *
  1898. * Returns the target CPU number, or the same CPU if no balancing is needed.
  1899. *
  1900. * preempt must be disabled.
  1901. */
  1902. static int sched_balance_self(int cpu, int flag)
  1903. {
  1904. struct task_struct *t = current;
  1905. struct sched_domain *tmp, *sd = NULL;
  1906. for_each_domain(cpu, tmp) {
  1907. /*
  1908. * If power savings logic is enabled for a domain, stop there.
  1909. */
  1910. if (tmp->flags & SD_POWERSAVINGS_BALANCE)
  1911. break;
  1912. if (tmp->flags & flag)
  1913. sd = tmp;
  1914. }
  1915. if (sd)
  1916. update_shares(sd);
  1917. while (sd) {
  1918. struct sched_group *group;
  1919. int new_cpu, weight;
  1920. if (!(sd->flags & flag)) {
  1921. sd = sd->child;
  1922. continue;
  1923. }
  1924. group = find_idlest_group(sd, t, cpu);
  1925. if (!group) {
  1926. sd = sd->child;
  1927. continue;
  1928. }
  1929. new_cpu = find_idlest_cpu(group, t, cpu);
  1930. if (new_cpu == -1 || new_cpu == cpu) {
  1931. /* Now try balancing at a lower domain level of cpu */
  1932. sd = sd->child;
  1933. continue;
  1934. }
  1935. /* Now try balancing at a lower domain level of new_cpu */
  1936. cpu = new_cpu;
  1937. weight = cpumask_weight(sched_domain_span(sd));
  1938. sd = NULL;
  1939. for_each_domain(cpu, tmp) {
  1940. if (weight <= cpumask_weight(sched_domain_span(tmp)))
  1941. break;
  1942. if (tmp->flags & flag)
  1943. sd = tmp;
  1944. }
  1945. /* while loop will break here if sd == NULL */
  1946. }
  1947. return cpu;
  1948. }
  1949. #endif /* CONFIG_SMP */
  1950. /***
  1951. * try_to_wake_up - wake up a thread
  1952. * @p: the to-be-woken-up thread
  1953. * @state: the mask of task states that can be woken
  1954. * @sync: do a synchronous wakeup?
  1955. *
  1956. * Put it on the run-queue if it's not already there. The "current"
  1957. * thread is always on the run-queue (except when the actual
  1958. * re-schedule is in progress), and as such you're allowed to do
  1959. * the simpler "current->state = TASK_RUNNING" to mark yourself
  1960. * runnable without the overhead of this.
  1961. *
  1962. * returns failure only if the task is already active.
  1963. */
  1964. static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
  1965. {
  1966. int cpu, orig_cpu, this_cpu, success = 0;
  1967. unsigned long flags;
  1968. long old_state;
  1969. struct rq *rq;
  1970. if (!sched_feat(SYNC_WAKEUPS))
  1971. sync = 0;
  1972. #ifdef CONFIG_SMP
  1973. if (sched_feat(LB_WAKEUP_UPDATE) && !root_task_group_empty()) {
  1974. struct sched_domain *sd;
  1975. this_cpu = raw_smp_processor_id();
  1976. cpu = task_cpu(p);
  1977. for_each_domain(this_cpu, sd) {
  1978. if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
  1979. update_shares(sd);
  1980. break;
  1981. }
  1982. }
  1983. }
  1984. #endif
  1985. smp_wmb();
  1986. rq = task_rq_lock(p, &flags);
  1987. update_rq_clock(rq);
  1988. old_state = p->state;
  1989. if (!(old_state & state))
  1990. goto out;
  1991. if (p->se.on_rq)
  1992. goto out_running;
  1993. cpu = task_cpu(p);
  1994. orig_cpu = cpu;
  1995. this_cpu = smp_processor_id();
  1996. #ifdef CONFIG_SMP
  1997. if (unlikely(task_running(rq, p)))
  1998. goto out_activate;
  1999. cpu = p->sched_class->select_task_rq(p, sync);
  2000. if (cpu != orig_cpu) {
  2001. set_task_cpu(p, cpu);
  2002. task_rq_unlock(rq, &flags);
  2003. /* might preempt at this point */
  2004. rq = task_rq_lock(p, &flags);
  2005. old_state = p->state;
  2006. if (!(old_state & state))
  2007. goto out;
  2008. if (p->se.on_rq)
  2009. goto out_running;
  2010. this_cpu = smp_processor_id();
  2011. cpu = task_cpu(p);
  2012. }
  2013. #ifdef CONFIG_SCHEDSTATS
  2014. schedstat_inc(rq, ttwu_count);
  2015. if (cpu == this_cpu)
  2016. schedstat_inc(rq, ttwu_local);
  2017. else {
  2018. struct sched_domain *sd;
  2019. for_each_domain(this_cpu, sd) {
  2020. if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
  2021. schedstat_inc(sd, ttwu_wake_remote);
  2022. break;
  2023. }
  2024. }
  2025. }
  2026. #endif /* CONFIG_SCHEDSTATS */
  2027. out_activate:
  2028. #endif /* CONFIG_SMP */
  2029. schedstat_inc(p, se.nr_wakeups);
  2030. if (sync)
  2031. schedstat_inc(p, se.nr_wakeups_sync);
  2032. if (orig_cpu != cpu)
  2033. schedstat_inc(p, se.nr_wakeups_migrate);
  2034. if (cpu == this_cpu)
  2035. schedstat_inc(p, se.nr_wakeups_local);
  2036. else
  2037. schedstat_inc(p, se.nr_wakeups_remote);
  2038. activate_task(rq, p, 1);
  2039. success = 1;
  2040. /*
  2041. * Only attribute actual wakeups done by this task.
  2042. */
  2043. if (!in_interrupt()) {
  2044. struct sched_entity *se = &current->se;
  2045. u64 sample = se->sum_exec_runtime;
  2046. if (se->last_wakeup)
  2047. sample -= se->last_wakeup;
  2048. else
  2049. sample -= se->start_runtime;
  2050. update_avg(&se->avg_wakeup, sample);
  2051. se->last_wakeup = se->sum_exec_runtime;
  2052. }
  2053. out_running:
  2054. trace_sched_wakeup(rq, p, success);
  2055. check_preempt_curr(rq, p, sync);
  2056. p->state = TASK_RUNNING;
  2057. #ifdef CONFIG_SMP
  2058. if (p->sched_class->task_wake_up)
  2059. p->sched_class->task_wake_up(rq, p);
  2060. #endif
  2061. out:
  2062. task_rq_unlock(rq, &flags);
  2063. return success;
  2064. }
  2065. int wake_up_process(struct task_struct *p)
  2066. {
  2067. return try_to_wake_up(p, TASK_ALL, 0);
  2068. }
  2069. EXPORT_SYMBOL(wake_up_process);
  2070. int wake_up_state(struct task_struct *p, unsigned int state)
  2071. {
  2072. return try_to_wake_up(p, state, 0);
  2073. }
  2074. /*
  2075. * Perform scheduler related setup for a newly forked process p.
  2076. * p is forked by current.
  2077. *
  2078. * __sched_fork() is basic setup used by init_idle() too:
  2079. */
  2080. static void __sched_fork(struct task_struct *p)
  2081. {
  2082. p->se.exec_start = 0;
  2083. p->se.sum_exec_runtime = 0;
  2084. p->se.prev_sum_exec_runtime = 0;
  2085. p->se.last_wakeup = 0;
  2086. p->se.avg_overlap = 0;
  2087. p->se.start_runtime = 0;
  2088. p->se.avg_wakeup = sysctl_sched_wakeup_granularity;
  2089. #ifdef CONFIG_SCHEDSTATS
  2090. p->se.wait_start = 0;
  2091. p->se.sum_sleep_runtime = 0;
  2092. p->se.sleep_start = 0;
  2093. p->se.block_start = 0;
  2094. p->se.sleep_max = 0;
  2095. p->se.block_max = 0;
  2096. p->se.exec_max = 0;
  2097. p->se.slice_max = 0;
  2098. p->se.wait_max = 0;
  2099. #endif
  2100. INIT_LIST_HEAD(&p->rt.run_list);
  2101. p->se.on_rq = 0;
  2102. INIT_LIST_HEAD(&p->se.group_node);
  2103. #ifdef CONFIG_PREEMPT_NOTIFIERS
  2104. INIT_HLIST_HEAD(&p->preempt_notifiers);
  2105. #endif
  2106. /*
  2107. * We mark the process as running here, but have not actually
  2108. * inserted it onto the runqueue yet. This guarantees that
  2109. * nobody will actually run it, and a signal or other external
  2110. * event cannot wake it up and insert it on the runqueue either.
  2111. */
  2112. p->state = TASK_RUNNING;
  2113. }
  2114. /*
  2115. * fork()/clone()-time setup:
  2116. */
  2117. void sched_fork(struct task_struct *p, int clone_flags)
  2118. {
  2119. int cpu = get_cpu();
  2120. __sched_fork(p);
  2121. #ifdef CONFIG_SMP
  2122. cpu = sched_balance_self(cpu, SD_BALANCE_FORK);
  2123. #endif
  2124. set_task_cpu(p, cpu);
  2125. /*
  2126. * Make sure we do not leak PI boosting priority to the child:
  2127. */
  2128. p->prio = current->normal_prio;
  2129. if (!rt_prio(p->prio))
  2130. p->sched_class = &fair_sched_class;
  2131. #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
  2132. if (likely(sched_info_on()))
  2133. memset(&p->sched_info, 0, sizeof(p->sched_info));
  2134. #endif
  2135. #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
  2136. p->oncpu = 0;
  2137. #endif
  2138. #ifdef CONFIG_PREEMPT
  2139. /* Want to start with kernel preemption disabled. */
  2140. task_thread_info(p)->preempt_count = 1;
  2141. #endif
  2142. plist_node_init(&p->pushable_tasks, MAX_PRIO);
  2143. put_cpu();
  2144. }
  2145. /*
  2146. * wake_up_new_task - wake up a newly created task for the first time.
  2147. *
  2148. * This function will do some initial scheduler statistics housekeeping
  2149. * that must be done for every newly created context, then puts the task
  2150. * on the runqueue and wakes it.
  2151. */
  2152. void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
  2153. {
  2154. unsigned long flags;
  2155. struct rq *rq;
  2156. rq = task_rq_lock(p, &flags);
  2157. BUG_ON(p->state != TASK_RUNNING);
  2158. update_rq_clock(rq);
  2159. p->prio = effective_prio(p);
  2160. if (!p->sched_class->task_new || !current->se.on_rq) {
  2161. activate_task(rq, p, 0);
  2162. } else {
  2163. /*
  2164. * Let the scheduling class do new task startup
  2165. * management (if any):
  2166. */
  2167. p->sched_class->task_new(rq, p);
  2168. inc_nr_running(rq);
  2169. }
  2170. trace_sched_wakeup_new(rq, p, 1);
  2171. check_preempt_curr(rq, p, 0);
  2172. #ifdef CONFIG_SMP
  2173. if (p->sched_class->task_wake_up)
  2174. p->sched_class->task_wake_up(rq, p);
  2175. #endif
  2176. task_rq_unlock(rq, &flags);
  2177. }
  2178. #ifdef CONFIG_PREEMPT_NOTIFIERS
  2179. /**
  2180. * preempt_notifier_register - tell me when current is being preempted & rescheduled
  2181. * @notifier: notifier struct to register
  2182. */
  2183. void preempt_notifier_register(struct preempt_notifier *notifier)
  2184. {
  2185. hlist_add_head(&notifier->link, &current->preempt_notifiers);
  2186. }
  2187. EXPORT_SYMBOL_GPL(preempt_notifier_register);
  2188. /**
  2189. * preempt_notifier_unregister - no longer interested in preemption notifications
  2190. * @notifier: notifier struct to unregister
  2191. *
  2192. * This is safe to call from within a preemption notifier.
  2193. */
  2194. void preempt_notifier_unregister(struct preempt_notifier *notifier)
  2195. {
  2196. hlist_del(&notifier->link);
  2197. }
  2198. EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
  2199. static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
  2200. {
  2201. struct preempt_notifier *notifier;
  2202. struct hlist_node *node;
  2203. hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
  2204. notifier->ops->sched_in(notifier, raw_smp_processor_id());
  2205. }
  2206. static void
  2207. fire_sched_out_preempt_notifiers(struct task_struct *curr,
  2208. struct task_struct *next)
  2209. {
  2210. struct preempt_notifier *notifier;
  2211. struct hlist_node *node;
  2212. hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
  2213. notifier->ops->sched_out(notifier, next);
  2214. }
  2215. #else /* !CONFIG_PREEMPT_NOTIFIERS */
  2216. static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
  2217. {
  2218. }
  2219. static void
  2220. fire_sched_out_preempt_notifiers(struct task_struct *curr,
  2221. struct task_struct *next)
  2222. {
  2223. }
  2224. #endif /* CONFIG_PREEMPT_NOTIFIERS */
  2225. /**
  2226. * prepare_task_switch - prepare to switch tasks
  2227. * @rq: the runqueue preparing to switch
  2228. * @prev: the current task that is being switched out
  2229. * @next: the task we are going to switch to.
  2230. *
  2231. * This is called with the rq lock held and interrupts off. It must
  2232. * be paired with a subsequent finish_task_switch after the context
  2233. * switch.
  2234. *
  2235. * prepare_task_switch sets up locking and calls architecture specific
  2236. * hooks.
  2237. */
  2238. static inline void
  2239. prepare_task_switch(struct rq *rq, struct task_struct *prev,
  2240. struct task_struct *next)
  2241. {
  2242. fire_sched_out_preempt_notifiers(prev, next);
  2243. prepare_lock_switch(rq, next);
  2244. prepare_arch_switch(next);
  2245. }
  2246. /**
  2247. * finish_task_switch - clean up after a task-switch
  2248. * @rq: runqueue associated with task-switch
  2249. * @prev: the thread we just switched away from.
  2250. *
  2251. * finish_task_switch must be called after the context switch, paired
  2252. * with a prepare_task_switch call before the context switch.
  2253. * finish_task_switch will reconcile locking set up by prepare_task_switch,
  2254. * and do any other architecture-specific cleanup actions.
  2255. *
  2256. * Note that we may have delayed dropping an mm in context_switch(). If
  2257. * so, we finish that here outside of the runqueue lock. (Doing it
  2258. * with the lock held can cause deadlocks; see schedule() for
  2259. * details.)
  2260. */
  2261. static void finish_task_switch(struct rq *rq, struct task_struct *prev)
  2262. __releases(rq->lock)
  2263. {
  2264. struct mm_struct *mm = rq->prev_mm;
  2265. long prev_state;
  2266. #ifdef CONFIG_SMP
  2267. int post_schedule = 0;
  2268. if (current->sched_class->needs_post_schedule)
  2269. post_schedule = current->sched_class->needs_post_schedule(rq);
  2270. #endif
  2271. rq->prev_mm = NULL;
  2272. /*
  2273. * A task struct has one reference for the use as "current".
  2274. * If a task dies, then it sets TASK_DEAD in tsk->state and calls
  2275. * schedule one last time. The schedule call will never return, and
  2276. * the scheduled task must drop that reference.
  2277. * The test for TASK_DEAD must occur while the runqueue locks are
  2278. * still held, otherwise prev could be scheduled on another cpu, die
  2279. * there before we look at prev->state, and then the reference would
  2280. * be dropped twice.
  2281. * Manfred Spraul <manfred@colorfullife.com>
  2282. */
  2283. prev_state = prev->state;
  2284. finish_arch_switch(prev);
  2285. finish_lock_switch(rq, prev);
  2286. #ifdef CONFIG_SMP
  2287. if (post_schedule)
  2288. current->sched_class->post_schedule(rq);
  2289. #endif
  2290. fire_sched_in_preempt_notifiers(current);
  2291. if (mm)
  2292. mmdrop(mm);
  2293. if (unlikely(prev_state == TASK_DEAD)) {
  2294. /*
  2295. * Remove function-return probe instances associated with this
  2296. * task and put them back on the free list.
  2297. */
  2298. kprobe_flush_task(prev);
  2299. put_task_struct(prev);
  2300. }
  2301. }
  2302. /**
  2303. * schedule_tail - first thing a freshly forked thread must call.
  2304. * @prev: the thread we just switched away from.
  2305. */
  2306. asmlinkage void schedule_tail(struct task_struct *prev)
  2307. __releases(rq->lock)
  2308. {
  2309. struct rq *rq = this_rq();
  2310. finish_task_switch(rq, prev);
  2311. #ifdef __ARCH_WANT_UNLOCKED_CTXSW
  2312. /* In this case, finish_task_switch does not reenable preemption */
  2313. preempt_enable();
  2314. #endif
  2315. if (current->set_child_tid)
  2316. put_user(task_pid_vnr(current), current->set_child_tid);
  2317. }
  2318. /*
  2319. * context_switch - switch to the new MM and the new
  2320. * thread's register state.
  2321. */
  2322. static inline void
  2323. context_switch(struct rq *rq, struct task_struct *prev,
  2324. struct task_struct *next)
  2325. {
  2326. struct mm_struct *mm, *oldmm;
  2327. prepare_task_switch(rq, prev, next);
  2328. trace_sched_switch(rq, prev, next);
  2329. mm = next->mm;
  2330. oldmm = prev->active_mm;
  2331. /*
  2332. * For paravirt, this is coupled with an exit in switch_to to
  2333. * combine the page table reload and the switch backend into
  2334. * one hypercall.
  2335. */
  2336. arch_enter_lazy_cpu_mode();
  2337. if (unlikely(!mm)) {
  2338. next->active_mm = oldmm;
  2339. atomic_inc(&oldmm->mm_count);
  2340. enter_lazy_tlb(oldmm, next);
  2341. } else
  2342. switch_mm(oldmm, mm, next);
  2343. if (unlikely(!prev->mm)) {
  2344. prev->active_mm = NULL;
  2345. rq->prev_mm = oldmm;
  2346. }
  2347. /*
  2348. * Since the runqueue lock will be released by the next
  2349. * task (which is an invalid locking op but in the case
  2350. * of the scheduler it's an obvious special-case), so we
  2351. * do an early lockdep release here:
  2352. */
  2353. #ifndef __ARCH_WANT_UNLOCKED_CTXSW
  2354. spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
  2355. #endif
  2356. /* Here we just switch the register state and the stack. */
  2357. switch_to(prev, next, prev);
  2358. barrier();
  2359. /*
  2360. * this_rq must be evaluated again because prev may have moved
  2361. * CPUs since it called schedule(), thus the 'rq' on its stack
  2362. * frame will be invalid.
  2363. */
  2364. finish_task_switch(this_rq(), prev);
  2365. }
  2366. /*
  2367. * nr_running, nr_uninterruptible and nr_context_switches:
  2368. *
  2369. * externally visible scheduler statistics: current number of runnable
  2370. * threads, current number of uninterruptible-sleeping threads, total
  2371. * number of context switches performed since bootup.
  2372. */
  2373. unsigned long nr_running(void)
  2374. {
  2375. unsigned long i, sum = 0;
  2376. for_each_online_cpu(i)
  2377. sum += cpu_rq(i)->nr_running;
  2378. return sum;
  2379. }
  2380. unsigned long nr_uninterruptible(void)
  2381. {
  2382. unsigned long i, sum = 0;
  2383. for_each_possible_cpu(i)
  2384. sum += cpu_rq(i)->nr_uninterruptible;
  2385. /*
  2386. * Since we read the counters lockless, it might be slightly
  2387. * inaccurate. Do not allow it to go below zero though:
  2388. */
  2389. if (unlikely((long)sum < 0))
  2390. sum = 0;
  2391. return sum;
  2392. }
  2393. unsigned long long nr_context_switches(void)
  2394. {
  2395. int i;
  2396. unsigned long long sum = 0;
  2397. for_each_possible_cpu(i)
  2398. sum += cpu_rq(i)->nr_switches;
  2399. return sum;
  2400. }
  2401. unsigned long nr_iowait(void)
  2402. {
  2403. unsigned long i, sum = 0;
  2404. for_each_possible_cpu(i)
  2405. sum += atomic_read(&cpu_rq(i)->nr_iowait);
  2406. return sum;
  2407. }
  2408. unsigned long nr_active(void)
  2409. {
  2410. unsigned long i, running = 0, uninterruptible = 0;
  2411. for_each_online_cpu(i) {
  2412. running += cpu_rq(i)->nr_running;
  2413. uninterruptible += cpu_rq(i)->nr_uninterruptible;
  2414. }
  2415. if (unlikely((long)uninterruptible < 0))
  2416. uninterruptible = 0;
  2417. return running + uninterruptible;
  2418. }
  2419. /*
  2420. * Update rq->cpu_load[] statistics. This function is usually called every
  2421. * scheduler tick (TICK_NSEC).
  2422. */
  2423. static void update_cpu_load(struct rq *this_rq)
  2424. {
  2425. unsigned long this_load = this_rq->load.weight;
  2426. int i, scale;
  2427. this_rq->nr_load_updates++;
  2428. /* Update our load: */
  2429. for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
  2430. unsigned long old_load, new_load;
  2431. /* scale is effectively 1 << i now, and >> i divides by scale */
  2432. old_load = this_rq->cpu_load[i];
  2433. new_load = this_load;
  2434. /*
  2435. * Round up the averaging division if load is increasing. This
  2436. * prevents us from getting stuck on 9 if the load is 10, for
  2437. * example.
  2438. */
  2439. if (new_load > old_load)
  2440. new_load += scale-1;
  2441. this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
  2442. }
  2443. }
  2444. #ifdef CONFIG_SMP
  2445. /*
  2446. * double_rq_lock - safely lock two runqueues
  2447. *
  2448. * Note this does not disable interrupts like task_rq_lock,
  2449. * you need to do so manually before calling.
  2450. */
  2451. static void double_rq_lock(struct rq *rq1, struct rq *rq2)
  2452. __acquires(rq1->lock)
  2453. __acquires(rq2->lock)
  2454. {
  2455. BUG_ON(!irqs_disabled());
  2456. if (rq1 == rq2) {
  2457. spin_lock(&rq1->lock);
  2458. __acquire(rq2->lock); /* Fake it out ;) */
  2459. } else {
  2460. if (rq1 < rq2) {
  2461. spin_lock(&rq1->lock);
  2462. spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
  2463. } else {
  2464. spin_lock(&rq2->lock);
  2465. spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
  2466. }
  2467. }
  2468. update_rq_clock(rq1);
  2469. update_rq_clock(rq2);
  2470. }
  2471. /*
  2472. * double_rq_unlock - safely unlock two runqueues
  2473. *
  2474. * Note this does not restore interrupts like task_rq_unlock,
  2475. * you need to do so manually after calling.
  2476. */
  2477. static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
  2478. __releases(rq1->lock)
  2479. __releases(rq2->lock)
  2480. {
  2481. spin_unlock(&rq1->lock);
  2482. if (rq1 != rq2)
  2483. spin_unlock(&rq2->lock);
  2484. else
  2485. __release(rq2->lock);
  2486. }
  2487. /*
  2488. * If dest_cpu is allowed for this process, migrate the task to it.
  2489. * This is accomplished by forcing the cpu_allowed mask to only
  2490. * allow dest_cpu, which will force the cpu onto dest_cpu. Then
  2491. * the cpu_allowed mask is restored.
  2492. */
  2493. static void sched_migrate_task(struct task_struct *p, int dest_cpu)
  2494. {
  2495. struct migration_req req;
  2496. unsigned long flags;
  2497. struct rq *rq;
  2498. rq = task_rq_lock(p, &flags);
  2499. if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
  2500. || unlikely(!cpu_active(dest_cpu)))
  2501. goto out;
  2502. /* force the process onto the specified CPU */
  2503. if (migrate_task(p, dest_cpu, &req)) {
  2504. /* Need to wait for migration thread (might exit: take ref). */
  2505. struct task_struct *mt = rq->migration_thread;
  2506. get_task_struct(mt);
  2507. task_rq_unlock(rq, &flags);
  2508. wake_up_process(mt);
  2509. put_task_struct(mt);
  2510. wait_for_completion(&req.done);
  2511. return;
  2512. }
  2513. out:
  2514. task_rq_unlock(rq, &flags);
  2515. }
  2516. /*
  2517. * sched_exec - execve() is a valuable balancing opportunity, because at
  2518. * this point the task has the smallest effective memory and cache footprint.
  2519. */
  2520. void sched_exec(void)
  2521. {
  2522. int new_cpu, this_cpu = get_cpu();
  2523. new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC);
  2524. put_cpu();
  2525. if (new_cpu != this_cpu)
  2526. sched_migrate_task(current, new_cpu);
  2527. }
  2528. /*
  2529. * pull_task - move a task from a remote runqueue to the local runqueue.
  2530. * Both runqueues must be locked.
  2531. */
  2532. static void pull_task(struct rq *src_rq, struct task_struct *p,
  2533. struct rq *this_rq, int this_cpu)
  2534. {
  2535. deactivate_task(src_rq, p, 0);
  2536. set_task_cpu(p, this_cpu);
  2537. activate_task(this_rq, p, 0);
  2538. /*
  2539. * Note that idle threads have a prio of MAX_PRIO, for this test
  2540. * to be always true for them.
  2541. */
  2542. check_preempt_curr(this_rq, p, 0);
  2543. }
  2544. /*
  2545. * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
  2546. */
  2547. static
  2548. int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
  2549. struct sched_domain *sd, enum cpu_idle_type idle,
  2550. int *all_pinned)
  2551. {
  2552. int tsk_cache_hot = 0;
  2553. /*
  2554. * We do not migrate tasks that are:
  2555. * 1) running (obviously), or
  2556. * 2) cannot be migrated to this CPU due to cpus_allowed, or
  2557. * 3) are cache-hot on their current CPU.
  2558. */
  2559. if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
  2560. schedstat_inc(p, se.nr_failed_migrations_affine);
  2561. return 0;
  2562. }
  2563. *all_pinned = 0;
  2564. if (task_running(rq, p)) {
  2565. schedstat_inc(p, se.nr_failed_migrations_running);
  2566. return 0;
  2567. }
  2568. /*
  2569. * Aggressive migration if:
  2570. * 1) task is cache cold, or
  2571. * 2) too many balance attempts have failed.
  2572. */
  2573. tsk_cache_hot = task_hot(p, rq->clock, sd);
  2574. if (!tsk_cache_hot ||
  2575. sd->nr_balance_failed > sd->cache_nice_tries) {
  2576. #ifdef CONFIG_SCHEDSTATS
  2577. if (tsk_cache_hot) {
  2578. schedstat_inc(sd, lb_hot_gained[idle]);
  2579. schedstat_inc(p, se.nr_forced_migrations);
  2580. }
  2581. #endif
  2582. return 1;
  2583. }
  2584. if (tsk_cache_hot) {
  2585. schedstat_inc(p, se.nr_failed_migrations_hot);
  2586. return 0;
  2587. }
  2588. return 1;
  2589. }
  2590. static unsigned long
  2591. balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
  2592. unsigned long max_load_move, struct sched_domain *sd,
  2593. enum cpu_idle_type idle, int *all_pinned,
  2594. int *this_best_prio, struct rq_iterator *iterator)
  2595. {
  2596. int loops = 0, pulled = 0, pinned = 0;
  2597. struct task_struct *p;
  2598. long rem_load_move = max_load_move;
  2599. if (max_load_move == 0)
  2600. goto out;
  2601. pinned = 1;
  2602. /*
  2603. * Start the load-balancing iterator:
  2604. */
  2605. p = iterator->start(iterator->arg);
  2606. next:
  2607. if (!p || loops++ > sysctl_sched_nr_migrate)
  2608. goto out;
  2609. if ((p->se.load.weight >> 1) > rem_load_move ||
  2610. !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
  2611. p = iterator->next(iterator->arg);
  2612. goto next;
  2613. }
  2614. pull_task(busiest, p, this_rq, this_cpu);
  2615. pulled++;
  2616. rem_load_move -= p->se.load.weight;
  2617. #ifdef CONFIG_PREEMPT
  2618. /*
  2619. * NEWIDLE balancing is a source of latency, so preemptible kernels
  2620. * will stop after the first task is pulled to minimize the critical
  2621. * section.
  2622. */
  2623. if (idle == CPU_NEWLY_IDLE)
  2624. goto out;
  2625. #endif
  2626. /*
  2627. * We only want to steal up to the prescribed amount of weighted load.
  2628. */
  2629. if (rem_load_move > 0) {
  2630. if (p->prio < *this_best_prio)
  2631. *this_best_prio = p->prio;
  2632. p = iterator->next(iterator->arg);
  2633. goto next;
  2634. }
  2635. out:
  2636. /*
  2637. * Right now, this is one of only two places pull_task() is called,
  2638. * so we can safely collect pull_task() stats here rather than
  2639. * inside pull_task().
  2640. */
  2641. schedstat_add(sd, lb_gained[idle], pulled);
  2642. if (all_pinned)
  2643. *all_pinned = pinned;
  2644. return max_load_move - rem_load_move;
  2645. }
  2646. /*
  2647. * move_tasks tries to move up to max_load_move weighted load from busiest to
  2648. * this_rq, as part of a balancing operation within domain "sd".
  2649. * Returns 1 if successful and 0 otherwise.
  2650. *
  2651. * Called with both runqueues locked.
  2652. */
  2653. static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
  2654. unsigned long max_load_move,
  2655. struct sched_domain *sd, enum cpu_idle_type idle,
  2656. int *all_pinned)
  2657. {
  2658. const struct sched_class *class = sched_class_highest;
  2659. unsigned long total_load_moved = 0;
  2660. int this_best_prio = this_rq->curr->prio;
  2661. do {
  2662. total_load_moved +=
  2663. class->load_balance(this_rq, this_cpu, busiest,
  2664. max_load_move - total_load_moved,
  2665. sd, idle, all_pinned, &this_best_prio);
  2666. class = class->next;
  2667. #ifdef CONFIG_PREEMPT
  2668. /*
  2669. * NEWIDLE balancing is a source of latency, so preemptible
  2670. * kernels will stop after the first task is pulled to minimize
  2671. * the critical section.
  2672. */
  2673. if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
  2674. break;
  2675. #endif
  2676. } while (class && max_load_move > total_load_moved);
  2677. return total_load_moved > 0;
  2678. }
  2679. static int
  2680. iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
  2681. struct sched_domain *sd, enum cpu_idle_type idle,
  2682. struct rq_iterator *iterator)
  2683. {
  2684. struct task_struct *p = iterator->start(iterator->arg);
  2685. int pinned = 0;
  2686. while (p) {
  2687. if (can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
  2688. pull_task(busiest, p, this_rq, this_cpu);
  2689. /*
  2690. * Right now, this is only the second place pull_task()
  2691. * is called, so we can safely collect pull_task()
  2692. * stats here rather than inside pull_task().
  2693. */
  2694. schedstat_inc(sd, lb_gained[idle]);
  2695. return 1;
  2696. }
  2697. p = iterator->next(iterator->arg);
  2698. }
  2699. return 0;
  2700. }
  2701. /*
  2702. * move_one_task tries to move exactly one task from busiest to this_rq, as
  2703. * part of active balancing operations within "domain".
  2704. * Returns 1 if successful and 0 otherwise.
  2705. *
  2706. * Called with both runqueues locked.
  2707. */
  2708. static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
  2709. struct sched_domain *sd, enum cpu_idle_type idle)
  2710. {
  2711. const struct sched_class *class;
  2712. for (class = sched_class_highest; class; class = class->next)
  2713. if (class->move_one_task(this_rq, this_cpu, busiest, sd, idle))
  2714. return 1;
  2715. return 0;
  2716. }
  2717. /********** Helpers for find_busiest_group ************************/
  2718. /*
  2719. * sd_lb_stats - Structure to store the statistics of a sched_domain
  2720. * during load balancing.
  2721. */
  2722. struct sd_lb_stats {
  2723. struct sched_group *busiest; /* Busiest group in this sd */
  2724. struct sched_group *this; /* Local group in this sd */
  2725. unsigned long total_load; /* Total load of all groups in sd */
  2726. unsigned long total_pwr; /* Total power of all groups in sd */
  2727. unsigned long avg_load; /* Average load across all groups in sd */
  2728. /** Statistics of this group */
  2729. unsigned long this_load;
  2730. unsigned long this_load_per_task;
  2731. unsigned long this_nr_running;
  2732. /* Statistics of the busiest group */
  2733. unsigned long max_load;
  2734. unsigned long busiest_load_per_task;
  2735. unsigned long busiest_nr_running;
  2736. int group_imb; /* Is there imbalance in this sd */
  2737. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  2738. int power_savings_balance; /* Is powersave balance needed for this sd */
  2739. struct sched_group *group_min; /* Least loaded group in sd */
  2740. struct sched_group *group_leader; /* Group which relieves group_min */
  2741. unsigned long min_load_per_task; /* load_per_task in group_min */
  2742. unsigned long leader_nr_running; /* Nr running of group_leader */
  2743. unsigned long min_nr_running; /* Nr running of group_min */
  2744. #endif
  2745. };
  2746. /*
  2747. * sg_lb_stats - stats of a sched_group required for load_balancing
  2748. */
  2749. struct sg_lb_stats {
  2750. unsigned long avg_load; /*Avg load across the CPUs of the group */
  2751. unsigned long group_load; /* Total load over the CPUs of the group */
  2752. unsigned long sum_nr_running; /* Nr tasks running in the group */
  2753. unsigned long sum_weighted_load; /* Weighted load of group's tasks */
  2754. unsigned long group_capacity;
  2755. int group_imb; /* Is there an imbalance in the group ? */
  2756. };
  2757. /**
  2758. * group_first_cpu - Returns the first cpu in the cpumask of a sched_group.
  2759. * @group: The group whose first cpu is to be returned.
  2760. */
  2761. static inline unsigned int group_first_cpu(struct sched_group *group)
  2762. {
  2763. return cpumask_first(sched_group_cpus(group));
  2764. }
  2765. /**
  2766. * get_sd_load_idx - Obtain the load index for a given sched domain.
  2767. * @sd: The sched_domain whose load_idx is to be obtained.
  2768. * @idle: The Idle status of the CPU for whose sd load_icx is obtained.
  2769. */
  2770. static inline int get_sd_load_idx(struct sched_domain *sd,
  2771. enum cpu_idle_type idle)
  2772. {
  2773. int load_idx;
  2774. switch (idle) {
  2775. case CPU_NOT_IDLE:
  2776. load_idx = sd->busy_idx;
  2777. break;
  2778. case CPU_NEWLY_IDLE:
  2779. load_idx = sd->newidle_idx;
  2780. break;
  2781. default:
  2782. load_idx = sd->idle_idx;
  2783. break;
  2784. }
  2785. return load_idx;
  2786. }
  2787. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  2788. /**
  2789. * init_sd_power_savings_stats - Initialize power savings statistics for
  2790. * the given sched_domain, during load balancing.
  2791. *
  2792. * @sd: Sched domain whose power-savings statistics are to be initialized.
  2793. * @sds: Variable containing the statistics for sd.
  2794. * @idle: Idle status of the CPU at which we're performing load-balancing.
  2795. */
  2796. static inline void init_sd_power_savings_stats(struct sched_domain *sd,
  2797. struct sd_lb_stats *sds, enum cpu_idle_type idle)
  2798. {
  2799. /*
  2800. * Busy processors will not participate in power savings
  2801. * balance.
  2802. */
  2803. if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
  2804. sds->power_savings_balance = 0;
  2805. else {
  2806. sds->power_savings_balance = 1;
  2807. sds->min_nr_running = ULONG_MAX;
  2808. sds->leader_nr_running = 0;
  2809. }
  2810. }
  2811. /**
  2812. * update_sd_power_savings_stats - Update the power saving stats for a
  2813. * sched_domain while performing load balancing.
  2814. *
  2815. * @group: sched_group belonging to the sched_domain under consideration.
  2816. * @sds: Variable containing the statistics of the sched_domain
  2817. * @local_group: Does group contain the CPU for which we're performing
  2818. * load balancing ?
  2819. * @sgs: Variable containing the statistics of the group.
  2820. */
  2821. static inline void update_sd_power_savings_stats(struct sched_group *group,
  2822. struct sd_lb_stats *sds, int local_group, struct sg_lb_stats *sgs)
  2823. {
  2824. if (!sds->power_savings_balance)
  2825. return;
  2826. /*
  2827. * If the local group is idle or completely loaded
  2828. * no need to do power savings balance at this domain
  2829. */
  2830. if (local_group && (sds->this_nr_running >= sgs->group_capacity ||
  2831. !sds->this_nr_running))
  2832. sds->power_savings_balance = 0;
  2833. /*
  2834. * If a group is already running at full capacity or idle,
  2835. * don't include that group in power savings calculations
  2836. */
  2837. if (!sds->power_savings_balance ||
  2838. sgs->sum_nr_running >= sgs->group_capacity ||
  2839. !sgs->sum_nr_running)
  2840. return;
  2841. /*
  2842. * Calculate the group which has the least non-idle load.
  2843. * This is the group from where we need to pick up the load
  2844. * for saving power
  2845. */
  2846. if ((sgs->sum_nr_running < sds->min_nr_running) ||
  2847. (sgs->sum_nr_running == sds->min_nr_running &&
  2848. group_first_cpu(group) > group_first_cpu(sds->group_min))) {
  2849. sds->group_min = group;
  2850. sds->min_nr_running = sgs->sum_nr_running;
  2851. sds->min_load_per_task = sgs->sum_weighted_load /
  2852. sgs->sum_nr_running;
  2853. }
  2854. /*
  2855. * Calculate the group which is almost near its
  2856. * capacity but still has some space to pick up some load
  2857. * from other group and save more power
  2858. */
  2859. if (sgs->sum_nr_running > sgs->group_capacity - 1)
  2860. return;
  2861. if (sgs->sum_nr_running > sds->leader_nr_running ||
  2862. (sgs->sum_nr_running == sds->leader_nr_running &&
  2863. group_first_cpu(group) < group_first_cpu(sds->group_leader))) {
  2864. sds->group_leader = group;
  2865. sds->leader_nr_running = sgs->sum_nr_running;
  2866. }
  2867. }
  2868. /**
  2869. * check_power_save_busiest_group - see if there is potential for some power-savings balance
  2870. * @sds: Variable containing the statistics of the sched_domain
  2871. * under consideration.
  2872. * @this_cpu: Cpu at which we're currently performing load-balancing.
  2873. * @imbalance: Variable to store the imbalance.
  2874. *
  2875. * Description:
  2876. * Check if we have potential to perform some power-savings balance.
  2877. * If yes, set the busiest group to be the least loaded group in the
  2878. * sched_domain, so that it's CPUs can be put to idle.
  2879. *
  2880. * Returns 1 if there is potential to perform power-savings balance.
  2881. * Else returns 0.
  2882. */
  2883. static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
  2884. int this_cpu, unsigned long *imbalance)
  2885. {
  2886. if (!sds->power_savings_balance)
  2887. return 0;
  2888. if (sds->this != sds->group_leader ||
  2889. sds->group_leader == sds->group_min)
  2890. return 0;
  2891. *imbalance = sds->min_load_per_task;
  2892. sds->busiest = sds->group_min;
  2893. if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
  2894. cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
  2895. group_first_cpu(sds->group_leader);
  2896. }
  2897. return 1;
  2898. }
  2899. #else /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
  2900. static inline void init_sd_power_savings_stats(struct sched_domain *sd,
  2901. struct sd_lb_stats *sds, enum cpu_idle_type idle)
  2902. {
  2903. return;
  2904. }
  2905. static inline void update_sd_power_savings_stats(struct sched_group *group,
  2906. struct sd_lb_stats *sds, int local_group, struct sg_lb_stats *sgs)
  2907. {
  2908. return;
  2909. }
  2910. static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
  2911. int this_cpu, unsigned long *imbalance)
  2912. {
  2913. return 0;
  2914. }
  2915. #endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
  2916. /**
  2917. * update_sg_lb_stats - Update sched_group's statistics for load balancing.
  2918. * @group: sched_group whose statistics are to be updated.
  2919. * @this_cpu: Cpu for which load balance is currently performed.
  2920. * @idle: Idle status of this_cpu
  2921. * @load_idx: Load index of sched_domain of this_cpu for load calc.
  2922. * @sd_idle: Idle status of the sched_domain containing group.
  2923. * @local_group: Does group contain this_cpu.
  2924. * @cpus: Set of cpus considered for load balancing.
  2925. * @balance: Should we balance.
  2926. * @sgs: variable to hold the statistics for this group.
  2927. */
  2928. static inline void update_sg_lb_stats(struct sched_group *group, int this_cpu,
  2929. enum cpu_idle_type idle, int load_idx, int *sd_idle,
  2930. int local_group, const struct cpumask *cpus,
  2931. int *balance, struct sg_lb_stats *sgs)
  2932. {
  2933. unsigned long load, max_cpu_load, min_cpu_load;
  2934. int i;
  2935. unsigned int balance_cpu = -1, first_idle_cpu = 0;
  2936. unsigned long sum_avg_load_per_task;
  2937. unsigned long avg_load_per_task;
  2938. if (local_group)
  2939. balance_cpu = group_first_cpu(group);
  2940. /* Tally up the load of all CPUs in the group */
  2941. sum_avg_load_per_task = avg_load_per_task = 0;
  2942. max_cpu_load = 0;
  2943. min_cpu_load = ~0UL;
  2944. for_each_cpu_and(i, sched_group_cpus(group), cpus) {
  2945. struct rq *rq = cpu_rq(i);
  2946. if (*sd_idle && rq->nr_running)
  2947. *sd_idle = 0;
  2948. /* Bias balancing toward cpus of our domain */
  2949. if (local_group) {
  2950. if (idle_cpu(i) && !first_idle_cpu) {
  2951. first_idle_cpu = 1;
  2952. balance_cpu = i;
  2953. }
  2954. load = target_load(i, load_idx);
  2955. } else {
  2956. load = source_load(i, load_idx);
  2957. if (load > max_cpu_load)
  2958. max_cpu_load = load;
  2959. if (min_cpu_load > load)
  2960. min_cpu_load = load;
  2961. }
  2962. sgs->group_load += load;
  2963. sgs->sum_nr_running += rq->nr_running;
  2964. sgs->sum_weighted_load += weighted_cpuload(i);
  2965. sum_avg_load_per_task += cpu_avg_load_per_task(i);
  2966. }
  2967. /*
  2968. * First idle cpu or the first cpu(busiest) in this sched group
  2969. * is eligible for doing load balancing at this and above
  2970. * domains. In the newly idle case, we will allow all the cpu's
  2971. * to do the newly idle load balance.
  2972. */
  2973. if (idle != CPU_NEWLY_IDLE && local_group &&
  2974. balance_cpu != this_cpu && balance) {
  2975. *balance = 0;
  2976. return;
  2977. }
  2978. /* Adjust by relative CPU power of the group */
  2979. sgs->avg_load = sg_div_cpu_power(group,
  2980. sgs->group_load * SCHED_LOAD_SCALE);
  2981. /*
  2982. * Consider the group unbalanced when the imbalance is larger
  2983. * than the average weight of two tasks.
  2984. *
  2985. * APZ: with cgroup the avg task weight can vary wildly and
  2986. * might not be a suitable number - should we keep a
  2987. * normalized nr_running number somewhere that negates
  2988. * the hierarchy?
  2989. */
  2990. avg_load_per_task = sg_div_cpu_power(group,
  2991. sum_avg_load_per_task * SCHED_LOAD_SCALE);
  2992. if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
  2993. sgs->group_imb = 1;
  2994. sgs->group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
  2995. }
  2996. /**
  2997. * update_sd_lb_stats - Update sched_group's statistics for load balancing.
  2998. * @sd: sched_domain whose statistics are to be updated.
  2999. * @this_cpu: Cpu for which load balance is currently performed.
  3000. * @idle: Idle status of this_cpu
  3001. * @sd_idle: Idle status of the sched_domain containing group.
  3002. * @cpus: Set of cpus considered for load balancing.
  3003. * @balance: Should we balance.
  3004. * @sds: variable to hold the statistics for this sched_domain.
  3005. */
  3006. static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
  3007. enum cpu_idle_type idle, int *sd_idle,
  3008. const struct cpumask *cpus, int *balance,
  3009. struct sd_lb_stats *sds)
  3010. {
  3011. struct sched_group *group = sd->groups;
  3012. struct sg_lb_stats sgs;
  3013. int load_idx;
  3014. init_sd_power_savings_stats(sd, sds, idle);
  3015. load_idx = get_sd_load_idx(sd, idle);
  3016. do {
  3017. int local_group;
  3018. local_group = cpumask_test_cpu(this_cpu,
  3019. sched_group_cpus(group));
  3020. memset(&sgs, 0, sizeof(sgs));
  3021. update_sg_lb_stats(group, this_cpu, idle, load_idx, sd_idle,
  3022. local_group, cpus, balance, &sgs);
  3023. if (local_group && balance && !(*balance))
  3024. return;
  3025. sds->total_load += sgs.group_load;
  3026. sds->total_pwr += group->__cpu_power;
  3027. if (local_group) {
  3028. sds->this_load = sgs.avg_load;
  3029. sds->this = group;
  3030. sds->this_nr_running = sgs.sum_nr_running;
  3031. sds->this_load_per_task = sgs.sum_weighted_load;
  3032. } else if (sgs.avg_load > sds->max_load &&
  3033. (sgs.sum_nr_running > sgs.group_capacity ||
  3034. sgs.group_imb)) {
  3035. sds->max_load = sgs.avg_load;
  3036. sds->busiest = group;
  3037. sds->busiest_nr_running = sgs.sum_nr_running;
  3038. sds->busiest_load_per_task = sgs.sum_weighted_load;
  3039. sds->group_imb = sgs.group_imb;
  3040. }
  3041. update_sd_power_savings_stats(group, sds, local_group, &sgs);
  3042. group = group->next;
  3043. } while (group != sd->groups);
  3044. }
  3045. /**
  3046. * fix_small_imbalance - Calculate the minor imbalance that exists
  3047. * amongst the groups of a sched_domain, during
  3048. * load balancing.
  3049. * @sds: Statistics of the sched_domain whose imbalance is to be calculated.
  3050. * @this_cpu: The cpu at whose sched_domain we're performing load-balance.
  3051. * @imbalance: Variable to store the imbalance.
  3052. */
  3053. static inline void fix_small_imbalance(struct sd_lb_stats *sds,
  3054. int this_cpu, unsigned long *imbalance)
  3055. {
  3056. unsigned long tmp, pwr_now = 0, pwr_move = 0;
  3057. unsigned int imbn = 2;
  3058. if (sds->this_nr_running) {
  3059. sds->this_load_per_task /= sds->this_nr_running;
  3060. if (sds->busiest_load_per_task >
  3061. sds->this_load_per_task)
  3062. imbn = 1;
  3063. } else
  3064. sds->this_load_per_task =
  3065. cpu_avg_load_per_task(this_cpu);
  3066. if (sds->max_load - sds->this_load + sds->busiest_load_per_task >=
  3067. sds->busiest_load_per_task * imbn) {
  3068. *imbalance = sds->busiest_load_per_task;
  3069. return;
  3070. }
  3071. /*
  3072. * OK, we don't have enough imbalance to justify moving tasks,
  3073. * however we may be able to increase total CPU power used by
  3074. * moving them.
  3075. */
  3076. pwr_now += sds->busiest->__cpu_power *
  3077. min(sds->busiest_load_per_task, sds->max_load);
  3078. pwr_now += sds->this->__cpu_power *
  3079. min(sds->this_load_per_task, sds->this_load);
  3080. pwr_now /= SCHED_LOAD_SCALE;
  3081. /* Amount of load we'd subtract */
  3082. tmp = sg_div_cpu_power(sds->busiest,
  3083. sds->busiest_load_per_task * SCHED_LOAD_SCALE);
  3084. if (sds->max_load > tmp)
  3085. pwr_move += sds->busiest->__cpu_power *
  3086. min(sds->busiest_load_per_task, sds->max_load - tmp);
  3087. /* Amount of load we'd add */
  3088. if (sds->max_load * sds->busiest->__cpu_power <
  3089. sds->busiest_load_per_task * SCHED_LOAD_SCALE)
  3090. tmp = sg_div_cpu_power(sds->this,
  3091. sds->max_load * sds->busiest->__cpu_power);
  3092. else
  3093. tmp = sg_div_cpu_power(sds->this,
  3094. sds->busiest_load_per_task * SCHED_LOAD_SCALE);
  3095. pwr_move += sds->this->__cpu_power *
  3096. min(sds->this_load_per_task, sds->this_load + tmp);
  3097. pwr_move /= SCHED_LOAD_SCALE;
  3098. /* Move if we gain throughput */
  3099. if (pwr_move > pwr_now)
  3100. *imbalance = sds->busiest_load_per_task;
  3101. }
  3102. /**
  3103. * calculate_imbalance - Calculate the amount of imbalance present within the
  3104. * groups of a given sched_domain during load balance.
  3105. * @sds: statistics of the sched_domain whose imbalance is to be calculated.
  3106. * @this_cpu: Cpu for which currently load balance is being performed.
  3107. * @imbalance: The variable to store the imbalance.
  3108. */
  3109. static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu,
  3110. unsigned long *imbalance)
  3111. {
  3112. unsigned long max_pull;
  3113. /*
  3114. * In the presence of smp nice balancing, certain scenarios can have
  3115. * max load less than avg load(as we skip the groups at or below
  3116. * its cpu_power, while calculating max_load..)
  3117. */
  3118. if (sds->max_load < sds->avg_load) {
  3119. *imbalance = 0;
  3120. return fix_small_imbalance(sds, this_cpu, imbalance);
  3121. }
  3122. /* Don't want to pull so many tasks that a group would go idle */
  3123. max_pull = min(sds->max_load - sds->avg_load,
  3124. sds->max_load - sds->busiest_load_per_task);
  3125. /* How much load to actually move to equalise the imbalance */
  3126. *imbalance = min(max_pull * sds->busiest->__cpu_power,
  3127. (sds->avg_load - sds->this_load) * sds->this->__cpu_power)
  3128. / SCHED_LOAD_SCALE;
  3129. /*
  3130. * if *imbalance is less than the average load per runnable task
  3131. * there is no gaurantee that any tasks will be moved so we'll have
  3132. * a think about bumping its value to force at least one task to be
  3133. * moved
  3134. */
  3135. if (*imbalance < sds->busiest_load_per_task)
  3136. return fix_small_imbalance(sds, this_cpu, imbalance);
  3137. }
  3138. /******* find_busiest_group() helpers end here *********************/
  3139. /**
  3140. * find_busiest_group - Returns the busiest group within the sched_domain
  3141. * if there is an imbalance. If there isn't an imbalance, and
  3142. * the user has opted for power-savings, it returns a group whose
  3143. * CPUs can be put to idle by rebalancing those tasks elsewhere, if
  3144. * such a group exists.
  3145. *
  3146. * Also calculates the amount of weighted load which should be moved
  3147. * to restore balance.
  3148. *
  3149. * @sd: The sched_domain whose busiest group is to be returned.
  3150. * @this_cpu: The cpu for which load balancing is currently being performed.
  3151. * @imbalance: Variable which stores amount of weighted load which should
  3152. * be moved to restore balance/put a group to idle.
  3153. * @idle: The idle status of this_cpu.
  3154. * @sd_idle: The idleness of sd
  3155. * @cpus: The set of CPUs under consideration for load-balancing.
  3156. * @balance: Pointer to a variable indicating if this_cpu
  3157. * is the appropriate cpu to perform load balancing at this_level.
  3158. *
  3159. * Returns: - the busiest group if imbalance exists.
  3160. * - If no imbalance and user has opted for power-savings balance,
  3161. * return the least loaded group whose CPUs can be
  3162. * put to idle by rebalancing its tasks onto our group.
  3163. */
  3164. static struct sched_group *
  3165. find_busiest_group(struct sched_domain *sd, int this_cpu,
  3166. unsigned long *imbalance, enum cpu_idle_type idle,
  3167. int *sd_idle, const struct cpumask *cpus, int *balance)
  3168. {
  3169. struct sd_lb_stats sds;
  3170. memset(&sds, 0, sizeof(sds));
  3171. /*
  3172. * Compute the various statistics relavent for load balancing at
  3173. * this level.
  3174. */
  3175. update_sd_lb_stats(sd, this_cpu, idle, sd_idle, cpus,
  3176. balance, &sds);
  3177. /* Cases where imbalance does not exist from POV of this_cpu */
  3178. /* 1) this_cpu is not the appropriate cpu to perform load balancing
  3179. * at this level.
  3180. * 2) There is no busy sibling group to pull from.
  3181. * 3) This group is the busiest group.
  3182. * 4) This group is more busy than the avg busieness at this
  3183. * sched_domain.
  3184. * 5) The imbalance is within the specified limit.
  3185. * 6) Any rebalance would lead to ping-pong
  3186. */
  3187. if (balance && !(*balance))
  3188. goto ret;
  3189. if (!sds.busiest || sds.busiest_nr_running == 0)
  3190. goto out_balanced;
  3191. if (sds.this_load >= sds.max_load)
  3192. goto out_balanced;
  3193. sds.avg_load = (SCHED_LOAD_SCALE * sds.total_load) / sds.total_pwr;
  3194. if (sds.this_load >= sds.avg_load)
  3195. goto out_balanced;
  3196. if (100 * sds.max_load <= sd->imbalance_pct * sds.this_load)
  3197. goto out_balanced;
  3198. sds.busiest_load_per_task /= sds.busiest_nr_running;
  3199. if (sds.group_imb)
  3200. sds.busiest_load_per_task =
  3201. min(sds.busiest_load_per_task, sds.avg_load);
  3202. /*
  3203. * We're trying to get all the cpus to the average_load, so we don't
  3204. * want to push ourselves above the average load, nor do we wish to
  3205. * reduce the max loaded cpu below the average load, as either of these
  3206. * actions would just result in more rebalancing later, and ping-pong
  3207. * tasks around. Thus we look for the minimum possible imbalance.
  3208. * Negative imbalances (*we* are more loaded than anyone else) will
  3209. * be counted as no imbalance for these purposes -- we can't fix that
  3210. * by pulling tasks to us. Be careful of negative numbers as they'll
  3211. * appear as very large values with unsigned longs.
  3212. */
  3213. if (sds.max_load <= sds.busiest_load_per_task)
  3214. goto out_balanced;
  3215. /* Looks like there is an imbalance. Compute it */
  3216. calculate_imbalance(&sds, this_cpu, imbalance);
  3217. return sds.busiest;
  3218. out_balanced:
  3219. /*
  3220. * There is no obvious imbalance. But check if we can do some balancing
  3221. * to save power.
  3222. */
  3223. if (check_power_save_busiest_group(&sds, this_cpu, imbalance))
  3224. return sds.busiest;
  3225. ret:
  3226. *imbalance = 0;
  3227. return NULL;
  3228. }
  3229. /*
  3230. * find_busiest_queue - find the busiest runqueue among the cpus in group.
  3231. */
  3232. static struct rq *
  3233. find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
  3234. unsigned long imbalance, const struct cpumask *cpus)
  3235. {
  3236. struct rq *busiest = NULL, *rq;
  3237. unsigned long max_load = 0;
  3238. int i;
  3239. for_each_cpu(i, sched_group_cpus(group)) {
  3240. unsigned long wl;
  3241. if (!cpumask_test_cpu(i, cpus))
  3242. continue;
  3243. rq = cpu_rq(i);
  3244. wl = weighted_cpuload(i);
  3245. if (rq->nr_running == 1 && wl > imbalance)
  3246. continue;
  3247. if (wl > max_load) {
  3248. max_load = wl;
  3249. busiest = rq;
  3250. }
  3251. }
  3252. return busiest;
  3253. }
  3254. /*
  3255. * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
  3256. * so long as it is large enough.
  3257. */
  3258. #define MAX_PINNED_INTERVAL 512
  3259. /* Working cpumask for load_balance and load_balance_newidle. */
  3260. static DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask);
  3261. /*
  3262. * Check this_cpu to ensure it is balanced within domain. Attempt to move
  3263. * tasks if there is an imbalance.
  3264. */
  3265. static int load_balance(int this_cpu, struct rq *this_rq,
  3266. struct sched_domain *sd, enum cpu_idle_type idle,
  3267. int *balance)
  3268. {
  3269. int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
  3270. struct sched_group *group;
  3271. unsigned long imbalance;
  3272. struct rq *busiest;
  3273. unsigned long flags;
  3274. struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
  3275. cpumask_setall(cpus);
  3276. /*
  3277. * When power savings policy is enabled for the parent domain, idle
  3278. * sibling can pick up load irrespective of busy siblings. In this case,
  3279. * let the state of idle sibling percolate up as CPU_IDLE, instead of
  3280. * portraying it as CPU_NOT_IDLE.
  3281. */
  3282. if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
  3283. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3284. sd_idle = 1;
  3285. schedstat_inc(sd, lb_count[idle]);
  3286. redo:
  3287. update_shares(sd);
  3288. group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
  3289. cpus, balance);
  3290. if (*balance == 0)
  3291. goto out_balanced;
  3292. if (!group) {
  3293. schedstat_inc(sd, lb_nobusyg[idle]);
  3294. goto out_balanced;
  3295. }
  3296. busiest = find_busiest_queue(group, idle, imbalance, cpus);
  3297. if (!busiest) {
  3298. schedstat_inc(sd, lb_nobusyq[idle]);
  3299. goto out_balanced;
  3300. }
  3301. BUG_ON(busiest == this_rq);
  3302. schedstat_add(sd, lb_imbalance[idle], imbalance);
  3303. ld_moved = 0;
  3304. if (busiest->nr_running > 1) {
  3305. /*
  3306. * Attempt to move tasks. If find_busiest_group has found
  3307. * an imbalance but busiest->nr_running <= 1, the group is
  3308. * still unbalanced. ld_moved simply stays zero, so it is
  3309. * correctly treated as an imbalance.
  3310. */
  3311. local_irq_save(flags);
  3312. double_rq_lock(this_rq, busiest);
  3313. ld_moved = move_tasks(this_rq, this_cpu, busiest,
  3314. imbalance, sd, idle, &all_pinned);
  3315. double_rq_unlock(this_rq, busiest);
  3316. local_irq_restore(flags);
  3317. /*
  3318. * some other cpu did the load balance for us.
  3319. */
  3320. if (ld_moved && this_cpu != smp_processor_id())
  3321. resched_cpu(this_cpu);
  3322. /* All tasks on this runqueue were pinned by CPU affinity */
  3323. if (unlikely(all_pinned)) {
  3324. cpumask_clear_cpu(cpu_of(busiest), cpus);
  3325. if (!cpumask_empty(cpus))
  3326. goto redo;
  3327. goto out_balanced;
  3328. }
  3329. }
  3330. if (!ld_moved) {
  3331. schedstat_inc(sd, lb_failed[idle]);
  3332. sd->nr_balance_failed++;
  3333. if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {
  3334. spin_lock_irqsave(&busiest->lock, flags);
  3335. /* don't kick the migration_thread, if the curr
  3336. * task on busiest cpu can't be moved to this_cpu
  3337. */
  3338. if (!cpumask_test_cpu(this_cpu,
  3339. &busiest->curr->cpus_allowed)) {
  3340. spin_unlock_irqrestore(&busiest->lock, flags);
  3341. all_pinned = 1;
  3342. goto out_one_pinned;
  3343. }
  3344. if (!busiest->active_balance) {
  3345. busiest->active_balance = 1;
  3346. busiest->push_cpu = this_cpu;
  3347. active_balance = 1;
  3348. }
  3349. spin_unlock_irqrestore(&busiest->lock, flags);
  3350. if (active_balance)
  3351. wake_up_process(busiest->migration_thread);
  3352. /*
  3353. * We've kicked active balancing, reset the failure
  3354. * counter.
  3355. */
  3356. sd->nr_balance_failed = sd->cache_nice_tries+1;
  3357. }
  3358. } else
  3359. sd->nr_balance_failed = 0;
  3360. if (likely(!active_balance)) {
  3361. /* We were unbalanced, so reset the balancing interval */
  3362. sd->balance_interval = sd->min_interval;
  3363. } else {
  3364. /*
  3365. * If we've begun active balancing, start to back off. This
  3366. * case may not be covered by the all_pinned logic if there
  3367. * is only 1 task on the busy runqueue (because we don't call
  3368. * move_tasks).
  3369. */
  3370. if (sd->balance_interval < sd->max_interval)
  3371. sd->balance_interval *= 2;
  3372. }
  3373. if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
  3374. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3375. ld_moved = -1;
  3376. goto out;
  3377. out_balanced:
  3378. schedstat_inc(sd, lb_balanced[idle]);
  3379. sd->nr_balance_failed = 0;
  3380. out_one_pinned:
  3381. /* tune up the balancing interval */
  3382. if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
  3383. (sd->balance_interval < sd->max_interval))
  3384. sd->balance_interval *= 2;
  3385. if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
  3386. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3387. ld_moved = -1;
  3388. else
  3389. ld_moved = 0;
  3390. out:
  3391. if (ld_moved)
  3392. update_shares(sd);
  3393. return ld_moved;
  3394. }
  3395. /*
  3396. * Check this_cpu to ensure it is balanced within domain. Attempt to move
  3397. * tasks if there is an imbalance.
  3398. *
  3399. * Called from schedule when this_rq is about to become idle (CPU_NEWLY_IDLE).
  3400. * this_rq is locked.
  3401. */
  3402. static int
  3403. load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
  3404. {
  3405. struct sched_group *group;
  3406. struct rq *busiest = NULL;
  3407. unsigned long imbalance;
  3408. int ld_moved = 0;
  3409. int sd_idle = 0;
  3410. int all_pinned = 0;
  3411. struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
  3412. cpumask_setall(cpus);
  3413. /*
  3414. * When power savings policy is enabled for the parent domain, idle
  3415. * sibling can pick up load irrespective of busy siblings. In this case,
  3416. * let the state of idle sibling percolate up as IDLE, instead of
  3417. * portraying it as CPU_NOT_IDLE.
  3418. */
  3419. if (sd->flags & SD_SHARE_CPUPOWER &&
  3420. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3421. sd_idle = 1;
  3422. schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
  3423. redo:
  3424. update_shares_locked(this_rq, sd);
  3425. group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
  3426. &sd_idle, cpus, NULL);
  3427. if (!group) {
  3428. schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
  3429. goto out_balanced;
  3430. }
  3431. busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance, cpus);
  3432. if (!busiest) {
  3433. schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
  3434. goto out_balanced;
  3435. }
  3436. BUG_ON(busiest == this_rq);
  3437. schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
  3438. ld_moved = 0;
  3439. if (busiest->nr_running > 1) {
  3440. /* Attempt to move tasks */
  3441. double_lock_balance(this_rq, busiest);
  3442. /* this_rq->clock is already updated */
  3443. update_rq_clock(busiest);
  3444. ld_moved = move_tasks(this_rq, this_cpu, busiest,
  3445. imbalance, sd, CPU_NEWLY_IDLE,
  3446. &all_pinned);
  3447. double_unlock_balance(this_rq, busiest);
  3448. if (unlikely(all_pinned)) {
  3449. cpumask_clear_cpu(cpu_of(busiest), cpus);
  3450. if (!cpumask_empty(cpus))
  3451. goto redo;
  3452. }
  3453. }
  3454. if (!ld_moved) {
  3455. int active_balance = 0;
  3456. schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
  3457. if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
  3458. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3459. return -1;
  3460. if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
  3461. return -1;
  3462. if (sd->nr_balance_failed++ < 2)
  3463. return -1;
  3464. /*
  3465. * The only task running in a non-idle cpu can be moved to this
  3466. * cpu in an attempt to completely freeup the other CPU
  3467. * package. The same method used to move task in load_balance()
  3468. * have been extended for load_balance_newidle() to speedup
  3469. * consolidation at sched_mc=POWERSAVINGS_BALANCE_WAKEUP (2)
  3470. *
  3471. * The package power saving logic comes from
  3472. * find_busiest_group(). If there are no imbalance, then
  3473. * f_b_g() will return NULL. However when sched_mc={1,2} then
  3474. * f_b_g() will select a group from which a running task may be
  3475. * pulled to this cpu in order to make the other package idle.
  3476. * If there is no opportunity to make a package idle and if
  3477. * there are no imbalance, then f_b_g() will return NULL and no
  3478. * action will be taken in load_balance_newidle().
  3479. *
  3480. * Under normal task pull operation due to imbalance, there
  3481. * will be more than one task in the source run queue and
  3482. * move_tasks() will succeed. ld_moved will be true and this
  3483. * active balance code will not be triggered.
  3484. */
  3485. /* Lock busiest in correct order while this_rq is held */
  3486. double_lock_balance(this_rq, busiest);
  3487. /*
  3488. * don't kick the migration_thread, if the curr
  3489. * task on busiest cpu can't be moved to this_cpu
  3490. */
  3491. if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
  3492. double_unlock_balance(this_rq, busiest);
  3493. all_pinned = 1;
  3494. return ld_moved;
  3495. }
  3496. if (!busiest->active_balance) {
  3497. busiest->active_balance = 1;
  3498. busiest->push_cpu = this_cpu;
  3499. active_balance = 1;
  3500. }
  3501. double_unlock_balance(this_rq, busiest);
  3502. /*
  3503. * Should not call ttwu while holding a rq->lock
  3504. */
  3505. spin_unlock(&this_rq->lock);
  3506. if (active_balance)
  3507. wake_up_process(busiest->migration_thread);
  3508. spin_lock(&this_rq->lock);
  3509. } else
  3510. sd->nr_balance_failed = 0;
  3511. update_shares_locked(this_rq, sd);
  3512. return ld_moved;
  3513. out_balanced:
  3514. schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
  3515. if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
  3516. !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
  3517. return -1;
  3518. sd->nr_balance_failed = 0;
  3519. return 0;
  3520. }
  3521. /*
  3522. * idle_balance is called by schedule() if this_cpu is about to become
  3523. * idle. Attempts to pull tasks from other CPUs.
  3524. */
  3525. static void idle_balance(int this_cpu, struct rq *this_rq)
  3526. {
  3527. struct sched_domain *sd;
  3528. int pulled_task = 0;
  3529. unsigned long next_balance = jiffies + HZ;
  3530. for_each_domain(this_cpu, sd) {
  3531. unsigned long interval;
  3532. if (!(sd->flags & SD_LOAD_BALANCE))
  3533. continue;
  3534. if (sd->flags & SD_BALANCE_NEWIDLE)
  3535. /* If we've pulled tasks over stop searching: */
  3536. pulled_task = load_balance_newidle(this_cpu, this_rq,
  3537. sd);
  3538. interval = msecs_to_jiffies(sd->balance_interval);
  3539. if (time_after(next_balance, sd->last_balance + interval))
  3540. next_balance = sd->last_balance + interval;
  3541. if (pulled_task)
  3542. break;
  3543. }
  3544. if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
  3545. /*
  3546. * We are going idle. next_balance may be set based on
  3547. * a busy processor. So reset next_balance.
  3548. */
  3549. this_rq->next_balance = next_balance;
  3550. }
  3551. }
  3552. /*
  3553. * active_load_balance is run by migration threads. It pushes running tasks
  3554. * off the busiest CPU onto idle CPUs. It requires at least 1 task to be
  3555. * running on each physical CPU where possible, and avoids physical /
  3556. * logical imbalances.
  3557. *
  3558. * Called with busiest_rq locked.
  3559. */
  3560. static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
  3561. {
  3562. int target_cpu = busiest_rq->push_cpu;
  3563. struct sched_domain *sd;
  3564. struct rq *target_rq;
  3565. /* Is there any task to move? */
  3566. if (busiest_rq->nr_running <= 1)
  3567. return;
  3568. target_rq = cpu_rq(target_cpu);
  3569. /*
  3570. * This condition is "impossible", if it occurs
  3571. * we need to fix it. Originally reported by
  3572. * Bjorn Helgaas on a 128-cpu setup.
  3573. */
  3574. BUG_ON(busiest_rq == target_rq);
  3575. /* move a task from busiest_rq to target_rq */
  3576. double_lock_balance(busiest_rq, target_rq);
  3577. update_rq_clock(busiest_rq);
  3578. update_rq_clock(target_rq);
  3579. /* Search for an sd spanning us and the target CPU. */
  3580. for_each_domain(target_cpu, sd) {
  3581. if ((sd->flags & SD_LOAD_BALANCE) &&
  3582. cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
  3583. break;
  3584. }
  3585. if (likely(sd)) {
  3586. schedstat_inc(sd, alb_count);
  3587. if (move_one_task(target_rq, target_cpu, busiest_rq,
  3588. sd, CPU_IDLE))
  3589. schedstat_inc(sd, alb_pushed);
  3590. else
  3591. schedstat_inc(sd, alb_failed);
  3592. }
  3593. double_unlock_balance(busiest_rq, target_rq);
  3594. }
  3595. #ifdef CONFIG_NO_HZ
  3596. static struct {
  3597. atomic_t load_balancer;
  3598. cpumask_var_t cpu_mask;
  3599. } nohz ____cacheline_aligned = {
  3600. .load_balancer = ATOMIC_INIT(-1),
  3601. };
  3602. /*
  3603. * This routine will try to nominate the ilb (idle load balancing)
  3604. * owner among the cpus whose ticks are stopped. ilb owner will do the idle
  3605. * load balancing on behalf of all those cpus. If all the cpus in the system
  3606. * go into this tickless mode, then there will be no ilb owner (as there is
  3607. * no need for one) and all the cpus will sleep till the next wakeup event
  3608. * arrives...
  3609. *
  3610. * For the ilb owner, tick is not stopped. And this tick will be used
  3611. * for idle load balancing. ilb owner will still be part of
  3612. * nohz.cpu_mask..
  3613. *
  3614. * While stopping the tick, this cpu will become the ilb owner if there
  3615. * is no other owner. And will be the owner till that cpu becomes busy
  3616. * or if all cpus in the system stop their ticks at which point
  3617. * there is no need for ilb owner.
  3618. *
  3619. * When the ilb owner becomes busy, it nominates another owner, during the
  3620. * next busy scheduler_tick()
  3621. */
  3622. int select_nohz_load_balancer(int stop_tick)
  3623. {
  3624. int cpu = smp_processor_id();
  3625. if (stop_tick) {
  3626. cpu_rq(cpu)->in_nohz_recently = 1;
  3627. if (!cpu_active(cpu)) {
  3628. if (atomic_read(&nohz.load_balancer) != cpu)
  3629. return 0;
  3630. /*
  3631. * If we are going offline and still the leader,
  3632. * give up!
  3633. */
  3634. if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
  3635. BUG();
  3636. return 0;
  3637. }
  3638. cpumask_set_cpu(cpu, nohz.cpu_mask);
  3639. /* time for ilb owner also to sleep */
  3640. if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
  3641. if (atomic_read(&nohz.load_balancer) == cpu)
  3642. atomic_set(&nohz.load_balancer, -1);
  3643. return 0;
  3644. }
  3645. if (atomic_read(&nohz.load_balancer) == -1) {
  3646. /* make me the ilb owner */
  3647. if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
  3648. return 1;
  3649. } else if (atomic_read(&nohz.load_balancer) == cpu)
  3650. return 1;
  3651. } else {
  3652. if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
  3653. return 0;
  3654. cpumask_clear_cpu(cpu, nohz.cpu_mask);
  3655. if (atomic_read(&nohz.load_balancer) == cpu)
  3656. if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
  3657. BUG();
  3658. }
  3659. return 0;
  3660. }
  3661. #endif
  3662. static DEFINE_SPINLOCK(balancing);
  3663. /*
  3664. * It checks each scheduling domain to see if it is due to be balanced,
  3665. * and initiates a balancing operation if so.
  3666. *
  3667. * Balancing parameters are set up in arch_init_sched_domains.
  3668. */
  3669. static void rebalance_domains(int cpu, enum cpu_idle_type idle)
  3670. {
  3671. int balance = 1;
  3672. struct rq *rq = cpu_rq(cpu);
  3673. unsigned long interval;
  3674. struct sched_domain *sd;
  3675. /* Earliest time when we have to do rebalance again */
  3676. unsigned long next_balance = jiffies + 60*HZ;
  3677. int update_next_balance = 0;
  3678. int need_serialize;
  3679. for_each_domain(cpu, sd) {
  3680. if (!(sd->flags & SD_LOAD_BALANCE))
  3681. continue;
  3682. interval = sd->balance_interval;
  3683. if (idle != CPU_IDLE)
  3684. interval *= sd->busy_factor;
  3685. /* scale ms to jiffies */
  3686. interval = msecs_to_jiffies(interval);
  3687. if (unlikely(!interval))
  3688. interval = 1;
  3689. if (interval > HZ*NR_CPUS/10)
  3690. interval = HZ*NR_CPUS/10;
  3691. need_serialize = sd->flags & SD_SERIALIZE;
  3692. if (need_serialize) {
  3693. if (!spin_trylock(&balancing))
  3694. goto out;
  3695. }
  3696. if (time_after_eq(jiffies, sd->last_balance + interval)) {
  3697. if (load_balance(cpu, rq, sd, idle, &balance)) {
  3698. /*
  3699. * We've pulled tasks over so either we're no
  3700. * longer idle, or one of our SMT siblings is
  3701. * not idle.
  3702. */
  3703. idle = CPU_NOT_IDLE;
  3704. }
  3705. sd->last_balance = jiffies;
  3706. }
  3707. if (need_serialize)
  3708. spin_unlock(&balancing);
  3709. out:
  3710. if (time_after(next_balance, sd->last_balance + interval)) {
  3711. next_balance = sd->last_balance + interval;
  3712. update_next_balance = 1;
  3713. }
  3714. /*
  3715. * Stop the load balance at this level. There is another
  3716. * CPU in our sched group which is doing load balancing more
  3717. * actively.
  3718. */
  3719. if (!balance)
  3720. break;
  3721. }
  3722. /*
  3723. * next_balance will be updated only when there is a need.
  3724. * When the cpu is attached to null domain for ex, it will not be
  3725. * updated.
  3726. */
  3727. if (likely(update_next_balance))
  3728. rq->next_balance = next_balance;
  3729. }
  3730. /*
  3731. * run_rebalance_domains is triggered when needed from the scheduler tick.
  3732. * In CONFIG_NO_HZ case, the idle load balance owner will do the
  3733. * rebalancing for all the cpus for whom scheduler ticks are stopped.
  3734. */
  3735. static void run_rebalance_domains(struct softirq_action *h)
  3736. {
  3737. int this_cpu = smp_processor_id();
  3738. struct rq *this_rq = cpu_rq(this_cpu);
  3739. enum cpu_idle_type idle = this_rq->idle_at_tick ?
  3740. CPU_IDLE : CPU_NOT_IDLE;
  3741. rebalance_domains(this_cpu, idle);
  3742. #ifdef CONFIG_NO_HZ
  3743. /*
  3744. * If this cpu is the owner for idle load balancing, then do the
  3745. * balancing on behalf of the other idle cpus whose ticks are
  3746. * stopped.
  3747. */
  3748. if (this_rq->idle_at_tick &&
  3749. atomic_read(&nohz.load_balancer) == this_cpu) {
  3750. struct rq *rq;
  3751. int balance_cpu;
  3752. for_each_cpu(balance_cpu, nohz.cpu_mask) {
  3753. if (balance_cpu == this_cpu)
  3754. continue;
  3755. /*
  3756. * If this cpu gets work to do, stop the load balancing
  3757. * work being done for other cpus. Next load
  3758. * balancing owner will pick it up.
  3759. */
  3760. if (need_resched())
  3761. break;
  3762. rebalance_domains(balance_cpu, CPU_IDLE);
  3763. rq = cpu_rq(balance_cpu);
  3764. if (time_after(this_rq->next_balance, rq->next_balance))
  3765. this_rq->next_balance = rq->next_balance;
  3766. }
  3767. }
  3768. #endif
  3769. }
  3770. static inline int on_null_domain(int cpu)
  3771. {
  3772. return !rcu_dereference(cpu_rq(cpu)->sd);
  3773. }
  3774. /*
  3775. * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
  3776. *
  3777. * In case of CONFIG_NO_HZ, this is the place where we nominate a new
  3778. * idle load balancing owner or decide to stop the periodic load balancing,
  3779. * if the whole system is idle.
  3780. */
  3781. static inline void trigger_load_balance(struct rq *rq, int cpu)
  3782. {
  3783. #ifdef CONFIG_NO_HZ
  3784. /*
  3785. * If we were in the nohz mode recently and busy at the current
  3786. * scheduler tick, then check if we need to nominate new idle
  3787. * load balancer.
  3788. */
  3789. if (rq->in_nohz_recently && !rq->idle_at_tick) {
  3790. rq->in_nohz_recently = 0;
  3791. if (atomic_read(&nohz.load_balancer) == cpu) {
  3792. cpumask_clear_cpu(cpu, nohz.cpu_mask);
  3793. atomic_set(&nohz.load_balancer, -1);
  3794. }
  3795. if (atomic_read(&nohz.load_balancer) == -1) {
  3796. /*
  3797. * simple selection for now: Nominate the
  3798. * first cpu in the nohz list to be the next
  3799. * ilb owner.
  3800. *
  3801. * TBD: Traverse the sched domains and nominate
  3802. * the nearest cpu in the nohz.cpu_mask.
  3803. */
  3804. int ilb = cpumask_first(nohz.cpu_mask);
  3805. if (ilb < nr_cpu_ids)
  3806. resched_cpu(ilb);
  3807. }
  3808. }
  3809. /*
  3810. * If this cpu is idle and doing idle load balancing for all the
  3811. * cpus with ticks stopped, is it time for that to stop?
  3812. */
  3813. if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
  3814. cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
  3815. resched_cpu(cpu);
  3816. return;
  3817. }
  3818. /*
  3819. * If this cpu is idle and the idle load balancing is done by
  3820. * someone else, then no need raise the SCHED_SOFTIRQ
  3821. */
  3822. if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
  3823. cpumask_test_cpu(cpu, nohz.cpu_mask))
  3824. return;
  3825. #endif
  3826. /* Don't need to rebalance while attached to NULL domain */
  3827. if (time_after_eq(jiffies, rq->next_balance) &&
  3828. likely(!on_null_domain(cpu)))
  3829. raise_softirq(SCHED_SOFTIRQ);
  3830. }
  3831. #else /* CONFIG_SMP */
  3832. /*
  3833. * on UP we do not need to balance between CPUs:
  3834. */
  3835. static inline void idle_balance(int cpu, struct rq *rq)
  3836. {
  3837. }
  3838. #endif
  3839. DEFINE_PER_CPU(struct kernel_stat, kstat);
  3840. EXPORT_PER_CPU_SYMBOL(kstat);
  3841. /*
  3842. * Return any ns on the sched_clock that have not yet been banked in
  3843. * @p in case that task is currently running.
  3844. */
  3845. unsigned long long task_delta_exec(struct task_struct *p)
  3846. {
  3847. unsigned long flags;
  3848. struct rq *rq;
  3849. u64 ns = 0;
  3850. rq = task_rq_lock(p, &flags);
  3851. if (task_current(rq, p)) {
  3852. u64 delta_exec;
  3853. update_rq_clock(rq);
  3854. delta_exec = rq->clock - p->se.exec_start;
  3855. if ((s64)delta_exec > 0)
  3856. ns = delta_exec;
  3857. }
  3858. task_rq_unlock(rq, &flags);
  3859. return ns;
  3860. }
  3861. /*
  3862. * Account user cpu time to a process.
  3863. * @p: the process that the cpu time gets accounted to
  3864. * @cputime: the cpu time spent in user space since the last update
  3865. * @cputime_scaled: cputime scaled by cpu frequency
  3866. */
  3867. void account_user_time(struct task_struct *p, cputime_t cputime,
  3868. cputime_t cputime_scaled)
  3869. {
  3870. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3871. cputime64_t tmp;
  3872. /* Add user time to process. */
  3873. p->utime = cputime_add(p->utime, cputime);
  3874. p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
  3875. account_group_user_time(p, cputime);
  3876. /* Add user time to cpustat. */
  3877. tmp = cputime_to_cputime64(cputime);
  3878. if (TASK_NICE(p) > 0)
  3879. cpustat->nice = cputime64_add(cpustat->nice, tmp);
  3880. else
  3881. cpustat->user = cputime64_add(cpustat->user, tmp);
  3882. /* Account for user time used */
  3883. acct_update_integrals(p);
  3884. }
  3885. /*
  3886. * Account guest cpu time to a process.
  3887. * @p: the process that the cpu time gets accounted to
  3888. * @cputime: the cpu time spent in virtual machine since the last update
  3889. * @cputime_scaled: cputime scaled by cpu frequency
  3890. */
  3891. static void account_guest_time(struct task_struct *p, cputime_t cputime,
  3892. cputime_t cputime_scaled)
  3893. {
  3894. cputime64_t tmp;
  3895. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3896. tmp = cputime_to_cputime64(cputime);
  3897. /* Add guest time to process. */
  3898. p->utime = cputime_add(p->utime, cputime);
  3899. p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
  3900. account_group_user_time(p, cputime);
  3901. p->gtime = cputime_add(p->gtime, cputime);
  3902. /* Add guest time to cpustat. */
  3903. cpustat->user = cputime64_add(cpustat->user, tmp);
  3904. cpustat->guest = cputime64_add(cpustat->guest, tmp);
  3905. }
  3906. /*
  3907. * Account system cpu time to a process.
  3908. * @p: the process that the cpu time gets accounted to
  3909. * @hardirq_offset: the offset to subtract from hardirq_count()
  3910. * @cputime: the cpu time spent in kernel space since the last update
  3911. * @cputime_scaled: cputime scaled by cpu frequency
  3912. */
  3913. void account_system_time(struct task_struct *p, int hardirq_offset,
  3914. cputime_t cputime, cputime_t cputime_scaled)
  3915. {
  3916. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3917. cputime64_t tmp;
  3918. if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
  3919. account_guest_time(p, cputime, cputime_scaled);
  3920. return;
  3921. }
  3922. /* Add system time to process. */
  3923. p->stime = cputime_add(p->stime, cputime);
  3924. p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
  3925. account_group_system_time(p, cputime);
  3926. /* Add system time to cpustat. */
  3927. tmp = cputime_to_cputime64(cputime);
  3928. if (hardirq_count() - hardirq_offset)
  3929. cpustat->irq = cputime64_add(cpustat->irq, tmp);
  3930. else if (softirq_count())
  3931. cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
  3932. else
  3933. cpustat->system = cputime64_add(cpustat->system, tmp);
  3934. /* Account for system time used */
  3935. acct_update_integrals(p);
  3936. }
  3937. /*
  3938. * Account for involuntary wait time.
  3939. * @steal: the cpu time spent in involuntary wait
  3940. */
  3941. void account_steal_time(cputime_t cputime)
  3942. {
  3943. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3944. cputime64_t cputime64 = cputime_to_cputime64(cputime);
  3945. cpustat->steal = cputime64_add(cpustat->steal, cputime64);
  3946. }
  3947. /*
  3948. * Account for idle time.
  3949. * @cputime: the cpu time spent in idle wait
  3950. */
  3951. void account_idle_time(cputime_t cputime)
  3952. {
  3953. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3954. cputime64_t cputime64 = cputime_to_cputime64(cputime);
  3955. struct rq *rq = this_rq();
  3956. if (atomic_read(&rq->nr_iowait) > 0)
  3957. cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
  3958. else
  3959. cpustat->idle = cputime64_add(cpustat->idle, cputime64);
  3960. }
  3961. #ifndef CONFIG_VIRT_CPU_ACCOUNTING
  3962. /*
  3963. * Account a single tick of cpu time.
  3964. * @p: the process that the cpu time gets accounted to
  3965. * @user_tick: indicates if the tick is a user or a system tick
  3966. */
  3967. void account_process_tick(struct task_struct *p, int user_tick)
  3968. {
  3969. cputime_t one_jiffy = jiffies_to_cputime(1);
  3970. cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy);
  3971. struct rq *rq = this_rq();
  3972. if (user_tick)
  3973. account_user_time(p, one_jiffy, one_jiffy_scaled);
  3974. else if (p != rq->idle)
  3975. account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
  3976. one_jiffy_scaled);
  3977. else
  3978. account_idle_time(one_jiffy);
  3979. }
  3980. /*
  3981. * Account multiple ticks of steal time.
  3982. * @p: the process from which the cpu time has been stolen
  3983. * @ticks: number of stolen ticks
  3984. */
  3985. void account_steal_ticks(unsigned long ticks)
  3986. {
  3987. account_steal_time(jiffies_to_cputime(ticks));
  3988. }
  3989. /*
  3990. * Account multiple ticks of idle time.
  3991. * @ticks: number of stolen ticks
  3992. */
  3993. void account_idle_ticks(unsigned long ticks)
  3994. {
  3995. account_idle_time(jiffies_to_cputime(ticks));
  3996. }
  3997. #endif
  3998. /*
  3999. * Use precise platform statistics if available:
  4000. */
  4001. #ifdef CONFIG_VIRT_CPU_ACCOUNTING
  4002. cputime_t task_utime(struct task_struct *p)
  4003. {
  4004. return p->utime;
  4005. }
  4006. cputime_t task_stime(struct task_struct *p)
  4007. {
  4008. return p->stime;
  4009. }
  4010. #else
  4011. cputime_t task_utime(struct task_struct *p)
  4012. {
  4013. clock_t utime = cputime_to_clock_t(p->utime),
  4014. total = utime + cputime_to_clock_t(p->stime);
  4015. u64 temp;
  4016. /*
  4017. * Use CFS's precise accounting:
  4018. */
  4019. temp = (u64)nsec_to_clock_t(p->se.sum_exec_runtime);
  4020. if (total) {
  4021. temp *= utime;
  4022. do_div(temp, total);
  4023. }
  4024. utime = (clock_t)temp;
  4025. p->prev_utime = max(p->prev_utime, clock_t_to_cputime(utime));
  4026. return p->prev_utime;
  4027. }
  4028. cputime_t task_stime(struct task_struct *p)
  4029. {
  4030. clock_t stime;
  4031. /*
  4032. * Use CFS's precise accounting. (we subtract utime from
  4033. * the total, to make sure the total observed by userspace
  4034. * grows monotonically - apps rely on that):
  4035. */
  4036. stime = nsec_to_clock_t(p->se.sum_exec_runtime) -
  4037. cputime_to_clock_t(task_utime(p));
  4038. if (stime >= 0)
  4039. p->prev_stime = max(p->prev_stime, clock_t_to_cputime(stime));
  4040. return p->prev_stime;
  4041. }
  4042. #endif
  4043. inline cputime_t task_gtime(struct task_struct *p)
  4044. {
  4045. return p->gtime;
  4046. }
  4047. /*
  4048. * This function gets called by the timer code, with HZ frequency.
  4049. * We call it with interrupts disabled.
  4050. *
  4051. * It also gets called by the fork code, when changing the parent's
  4052. * timeslices.
  4053. */
  4054. void scheduler_tick(void)
  4055. {
  4056. int cpu = smp_processor_id();
  4057. struct rq *rq = cpu_rq(cpu);
  4058. struct task_struct *curr = rq->curr;
  4059. sched_clock_tick();
  4060. spin_lock(&rq->lock);
  4061. update_rq_clock(rq);
  4062. update_cpu_load(rq);
  4063. curr->sched_class->task_tick(rq, curr, 0);
  4064. spin_unlock(&rq->lock);
  4065. #ifdef CONFIG_SMP
  4066. rq->idle_at_tick = idle_cpu(cpu);
  4067. trigger_load_balance(rq, cpu);
  4068. #endif
  4069. }
  4070. unsigned long get_parent_ip(unsigned long addr)
  4071. {
  4072. if (in_lock_functions(addr)) {
  4073. addr = CALLER_ADDR2;
  4074. if (in_lock_functions(addr))
  4075. addr = CALLER_ADDR3;
  4076. }
  4077. return addr;
  4078. }
  4079. #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
  4080. defined(CONFIG_PREEMPT_TRACER))
  4081. void __kprobes add_preempt_count(int val)
  4082. {
  4083. #ifdef CONFIG_DEBUG_PREEMPT
  4084. /*
  4085. * Underflow?
  4086. */
  4087. if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
  4088. return;
  4089. #endif
  4090. preempt_count() += val;
  4091. #ifdef CONFIG_DEBUG_PREEMPT
  4092. /*
  4093. * Spinlock count overflowing soon?
  4094. */
  4095. DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
  4096. PREEMPT_MASK - 10);
  4097. #endif
  4098. if (preempt_count() == val)
  4099. trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
  4100. }
  4101. EXPORT_SYMBOL(add_preempt_count);
  4102. void __kprobes sub_preempt_count(int val)
  4103. {
  4104. #ifdef CONFIG_DEBUG_PREEMPT
  4105. /*
  4106. * Underflow?
  4107. */
  4108. if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
  4109. return;
  4110. /*
  4111. * Is the spinlock portion underflowing?
  4112. */
  4113. if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
  4114. !(preempt_count() & PREEMPT_MASK)))
  4115. return;
  4116. #endif
  4117. if (preempt_count() == val)
  4118. trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
  4119. preempt_count() -= val;
  4120. }
  4121. EXPORT_SYMBOL(sub_preempt_count);
  4122. #endif
  4123. /*
  4124. * Print scheduling while atomic bug:
  4125. */
  4126. static noinline void __schedule_bug(struct task_struct *prev)
  4127. {
  4128. struct pt_regs *regs = get_irq_regs();
  4129. printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
  4130. prev->comm, prev->pid, preempt_count());
  4131. debug_show_held_locks(prev);
  4132. print_modules();
  4133. if (irqs_disabled())
  4134. print_irqtrace_events(prev);
  4135. if (regs)
  4136. show_regs(regs);
  4137. else
  4138. dump_stack();
  4139. }
  4140. /*
  4141. * Various schedule()-time debugging checks and statistics:
  4142. */
  4143. static inline void schedule_debug(struct task_struct *prev)
  4144. {
  4145. /*
  4146. * Test if we are atomic. Since do_exit() needs to call into
  4147. * schedule() atomically, we ignore that path for now.
  4148. * Otherwise, whine if we are scheduling when we should not be.
  4149. */
  4150. if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
  4151. __schedule_bug(prev);
  4152. profile_hit(SCHED_PROFILING, __builtin_return_address(0));
  4153. schedstat_inc(this_rq(), sched_count);
  4154. #ifdef CONFIG_SCHEDSTATS
  4155. if (unlikely(prev->lock_depth >= 0)) {
  4156. schedstat_inc(this_rq(), bkl_count);
  4157. schedstat_inc(prev, sched_info.bkl_count);
  4158. }
  4159. #endif
  4160. }
  4161. static void put_prev_task(struct rq *rq, struct task_struct *prev)
  4162. {
  4163. if (prev->state == TASK_RUNNING) {
  4164. u64 runtime = prev->se.sum_exec_runtime;
  4165. runtime -= prev->se.prev_sum_exec_runtime;
  4166. runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost);
  4167. /*
  4168. * In order to avoid avg_overlap growing stale when we are
  4169. * indeed overlapping and hence not getting put to sleep, grow
  4170. * the avg_overlap on preemption.
  4171. *
  4172. * We use the average preemption runtime because that
  4173. * correlates to the amount of cache footprint a task can
  4174. * build up.
  4175. */
  4176. update_avg(&prev->se.avg_overlap, runtime);
  4177. }
  4178. prev->sched_class->put_prev_task(rq, prev);
  4179. }
  4180. /*
  4181. * Pick up the highest-prio task:
  4182. */
  4183. static inline struct task_struct *
  4184. pick_next_task(struct rq *rq)
  4185. {
  4186. const struct sched_class *class;
  4187. struct task_struct *p;
  4188. /*
  4189. * Optimization: we know that if all tasks are in
  4190. * the fair class we can call that function directly:
  4191. */
  4192. if (likely(rq->nr_running == rq->cfs.nr_running)) {
  4193. p = fair_sched_class.pick_next_task(rq);
  4194. if (likely(p))
  4195. return p;
  4196. }
  4197. class = sched_class_highest;
  4198. for ( ; ; ) {
  4199. p = class->pick_next_task(rq);
  4200. if (p)
  4201. return p;
  4202. /*
  4203. * Will never be NULL as the idle class always
  4204. * returns a non-NULL p:
  4205. */
  4206. class = class->next;
  4207. }
  4208. }
  4209. /*
  4210. * schedule() is the main scheduler function.
  4211. */
  4212. asmlinkage void __sched __schedule(void)
  4213. {
  4214. struct task_struct *prev, *next;
  4215. unsigned long *switch_count;
  4216. struct rq *rq;
  4217. int cpu;
  4218. cpu = smp_processor_id();
  4219. rq = cpu_rq(cpu);
  4220. rcu_qsctr_inc(cpu);
  4221. prev = rq->curr;
  4222. switch_count = &prev->nivcsw;
  4223. release_kernel_lock(prev);
  4224. need_resched_nonpreemptible:
  4225. schedule_debug(prev);
  4226. if (sched_feat(HRTICK))
  4227. hrtick_clear(rq);
  4228. spin_lock_irq(&rq->lock);
  4229. update_rq_clock(rq);
  4230. clear_tsk_need_resched(prev);
  4231. if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
  4232. if (unlikely(signal_pending_state(prev->state, prev)))
  4233. prev->state = TASK_RUNNING;
  4234. else
  4235. deactivate_task(rq, prev, 1);
  4236. switch_count = &prev->nvcsw;
  4237. }
  4238. #ifdef CONFIG_SMP
  4239. if (prev->sched_class->pre_schedule)
  4240. prev->sched_class->pre_schedule(rq, prev);
  4241. #endif
  4242. if (unlikely(!rq->nr_running))
  4243. idle_balance(cpu, rq);
  4244. put_prev_task(rq, prev);
  4245. next = pick_next_task(rq);
  4246. if (likely(prev != next)) {
  4247. sched_info_switch(prev, next);
  4248. rq->nr_switches++;
  4249. rq->curr = next;
  4250. ++*switch_count;
  4251. context_switch(rq, prev, next); /* unlocks the rq */
  4252. /*
  4253. * the context switch might have flipped the stack from under
  4254. * us, hence refresh the local variables.
  4255. */
  4256. cpu = smp_processor_id();
  4257. rq = cpu_rq(cpu);
  4258. } else
  4259. spin_unlock_irq(&rq->lock);
  4260. if (unlikely(reacquire_kernel_lock(current) < 0))
  4261. goto need_resched_nonpreemptible;
  4262. }
  4263. asmlinkage void __sched schedule(void)
  4264. {
  4265. need_resched:
  4266. preempt_disable();
  4267. __schedule();
  4268. preempt_enable_no_resched();
  4269. if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
  4270. goto need_resched;
  4271. }
  4272. EXPORT_SYMBOL(schedule);
  4273. #ifdef CONFIG_SMP
  4274. /*
  4275. * Look out! "owner" is an entirely speculative pointer
  4276. * access and not reliable.
  4277. */
  4278. int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner)
  4279. {
  4280. unsigned int cpu;
  4281. struct rq *rq;
  4282. if (!sched_feat(OWNER_SPIN))
  4283. return 0;
  4284. #ifdef CONFIG_DEBUG_PAGEALLOC
  4285. /*
  4286. * Need to access the cpu field knowing that
  4287. * DEBUG_PAGEALLOC could have unmapped it if
  4288. * the mutex owner just released it and exited.
  4289. */
  4290. if (probe_kernel_address(&owner->cpu, cpu))
  4291. goto out;
  4292. #else
  4293. cpu = owner->cpu;
  4294. #endif
  4295. /*
  4296. * Even if the access succeeded (likely case),
  4297. * the cpu field may no longer be valid.
  4298. */
  4299. if (cpu >= nr_cpumask_bits)
  4300. goto out;
  4301. /*
  4302. * We need to validate that we can do a
  4303. * get_cpu() and that we have the percpu area.
  4304. */
  4305. if (!cpu_online(cpu))
  4306. goto out;
  4307. rq = cpu_rq(cpu);
  4308. for (;;) {
  4309. /*
  4310. * Owner changed, break to re-assess state.
  4311. */
  4312. if (lock->owner != owner)
  4313. break;
  4314. /*
  4315. * Is that owner really running on that cpu?
  4316. */
  4317. if (task_thread_info(rq->curr) != owner || need_resched())
  4318. return 0;
  4319. cpu_relax();
  4320. }
  4321. out:
  4322. return 1;
  4323. }
  4324. #endif
  4325. #ifdef CONFIG_PREEMPT
  4326. /*
  4327. * this is the entry point to schedule() from in-kernel preemption
  4328. * off of preempt_enable. Kernel preemptions off return from interrupt
  4329. * occur there and call schedule directly.
  4330. */
  4331. asmlinkage void __sched preempt_schedule(void)
  4332. {
  4333. struct thread_info *ti = current_thread_info();
  4334. /*
  4335. * If there is a non-zero preempt_count or interrupts are disabled,
  4336. * we do not want to preempt the current task. Just return..
  4337. */
  4338. if (likely(ti->preempt_count || irqs_disabled()))
  4339. return;
  4340. do {
  4341. add_preempt_count(PREEMPT_ACTIVE);
  4342. schedule();
  4343. sub_preempt_count(PREEMPT_ACTIVE);
  4344. /*
  4345. * Check again in case we missed a preemption opportunity
  4346. * between schedule and now.
  4347. */
  4348. barrier();
  4349. } while (need_resched());
  4350. }
  4351. EXPORT_SYMBOL(preempt_schedule);
  4352. /*
  4353. * this is the entry point to schedule() from kernel preemption
  4354. * off of irq context.
  4355. * Note, that this is called and return with irqs disabled. This will
  4356. * protect us against recursive calling from irq.
  4357. */
  4358. asmlinkage void __sched preempt_schedule_irq(void)
  4359. {
  4360. struct thread_info *ti = current_thread_info();
  4361. /* Catch callers which need to be fixed */
  4362. BUG_ON(ti->preempt_count || !irqs_disabled());
  4363. do {
  4364. add_preempt_count(PREEMPT_ACTIVE);
  4365. local_irq_enable();
  4366. schedule();
  4367. local_irq_disable();
  4368. sub_preempt_count(PREEMPT_ACTIVE);
  4369. /*
  4370. * Check again in case we missed a preemption opportunity
  4371. * between schedule and now.
  4372. */
  4373. barrier();
  4374. } while (need_resched());
  4375. }
  4376. #endif /* CONFIG_PREEMPT */
  4377. int default_wake_function(wait_queue_t *curr, unsigned mode, int sync,
  4378. void *key)
  4379. {
  4380. return try_to_wake_up(curr->private, mode, sync);
  4381. }
  4382. EXPORT_SYMBOL(default_wake_function);
  4383. /*
  4384. * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
  4385. * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
  4386. * number) then we wake all the non-exclusive tasks and one exclusive task.
  4387. *
  4388. * There are circumstances in which we can try to wake a task which has already
  4389. * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
  4390. * zero in this (rare) case, and we handle it by continuing to scan the queue.
  4391. */
  4392. void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
  4393. int nr_exclusive, int sync, void *key)
  4394. {
  4395. wait_queue_t *curr, *next;
  4396. list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
  4397. unsigned flags = curr->flags;
  4398. if (curr->func(curr, mode, sync, key) &&
  4399. (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
  4400. break;
  4401. }
  4402. }
  4403. /**
  4404. * __wake_up - wake up threads blocked on a waitqueue.
  4405. * @q: the waitqueue
  4406. * @mode: which threads
  4407. * @nr_exclusive: how many wake-one or wake-many threads to wake up
  4408. * @key: is directly passed to the wakeup function
  4409. */
  4410. void __wake_up(wait_queue_head_t *q, unsigned int mode,
  4411. int nr_exclusive, void *key)
  4412. {
  4413. unsigned long flags;
  4414. spin_lock_irqsave(&q->lock, flags);
  4415. __wake_up_common(q, mode, nr_exclusive, 0, key);
  4416. spin_unlock_irqrestore(&q->lock, flags);
  4417. }
  4418. EXPORT_SYMBOL(__wake_up);
  4419. /*
  4420. * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
  4421. */
  4422. void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
  4423. {
  4424. __wake_up_common(q, mode, 1, 0, NULL);
  4425. }
  4426. void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)
  4427. {
  4428. __wake_up_common(q, mode, 1, 0, key);
  4429. }
  4430. /**
  4431. * __wake_up_sync_key - wake up threads blocked on a waitqueue.
  4432. * @q: the waitqueue
  4433. * @mode: which threads
  4434. * @nr_exclusive: how many wake-one or wake-many threads to wake up
  4435. * @key: opaque value to be passed to wakeup targets
  4436. *
  4437. * The sync wakeup differs that the waker knows that it will schedule
  4438. * away soon, so while the target thread will be woken up, it will not
  4439. * be migrated to another CPU - ie. the two threads are 'synchronized'
  4440. * with each other. This can prevent needless bouncing between CPUs.
  4441. *
  4442. * On UP it can prevent extra preemption.
  4443. */
  4444. void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
  4445. int nr_exclusive, void *key)
  4446. {
  4447. unsigned long flags;
  4448. int sync = 1;
  4449. if (unlikely(!q))
  4450. return;
  4451. if (unlikely(!nr_exclusive))
  4452. sync = 0;
  4453. spin_lock_irqsave(&q->lock, flags);
  4454. __wake_up_common(q, mode, nr_exclusive, sync, key);
  4455. spin_unlock_irqrestore(&q->lock, flags);
  4456. }
  4457. EXPORT_SYMBOL_GPL(__wake_up_sync_key);
  4458. /*
  4459. * __wake_up_sync - see __wake_up_sync_key()
  4460. */
  4461. void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
  4462. {
  4463. __wake_up_sync_key(q, mode, nr_exclusive, NULL);
  4464. }
  4465. EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */
  4466. /**
  4467. * complete: - signals a single thread waiting on this completion
  4468. * @x: holds the state of this particular completion
  4469. *
  4470. * This will wake up a single thread waiting on this completion. Threads will be
  4471. * awakened in the same order in which they were queued.
  4472. *
  4473. * See also complete_all(), wait_for_completion() and related routines.
  4474. */
  4475. void complete(struct completion *x)
  4476. {
  4477. unsigned long flags;
  4478. spin_lock_irqsave(&x->wait.lock, flags);
  4479. x->done++;
  4480. __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
  4481. spin_unlock_irqrestore(&x->wait.lock, flags);
  4482. }
  4483. EXPORT_SYMBOL(complete);
  4484. /**
  4485. * complete_all: - signals all threads waiting on this completion
  4486. * @x: holds the state of this particular completion
  4487. *
  4488. * This will wake up all threads waiting on this particular completion event.
  4489. */
  4490. void complete_all(struct completion *x)
  4491. {
  4492. unsigned long flags;
  4493. spin_lock_irqsave(&x->wait.lock, flags);
  4494. x->done += UINT_MAX/2;
  4495. __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
  4496. spin_unlock_irqrestore(&x->wait.lock, flags);
  4497. }
  4498. EXPORT_SYMBOL(complete_all);
  4499. static inline long __sched
  4500. do_wait_for_common(struct completion *x, long timeout, int state)
  4501. {
  4502. if (!x->done) {
  4503. DECLARE_WAITQUEUE(wait, current);
  4504. wait.flags |= WQ_FLAG_EXCLUSIVE;
  4505. __add_wait_queue_tail(&x->wait, &wait);
  4506. do {
  4507. if (signal_pending_state(state, current)) {
  4508. timeout = -ERESTARTSYS;
  4509. break;
  4510. }
  4511. __set_current_state(state);
  4512. spin_unlock_irq(&x->wait.lock);
  4513. timeout = schedule_timeout(timeout);
  4514. spin_lock_irq(&x->wait.lock);
  4515. } while (!x->done && timeout);
  4516. __remove_wait_queue(&x->wait, &wait);
  4517. if (!x->done)
  4518. return timeout;
  4519. }
  4520. x->done--;
  4521. return timeout ?: 1;
  4522. }
  4523. static long __sched
  4524. wait_for_common(struct completion *x, long timeout, int state)
  4525. {
  4526. might_sleep();
  4527. spin_lock_irq(&x->wait.lock);
  4528. timeout = do_wait_for_common(x, timeout, state);
  4529. spin_unlock_irq(&x->wait.lock);
  4530. return timeout;
  4531. }
  4532. /**
  4533. * wait_for_completion: - waits for completion of a task
  4534. * @x: holds the state of this particular completion
  4535. *
  4536. * This waits to be signaled for completion of a specific task. It is NOT
  4537. * interruptible and there is no timeout.
  4538. *
  4539. * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
  4540. * and interrupt capability. Also see complete().
  4541. */
  4542. void __sched wait_for_completion(struct completion *x)
  4543. {
  4544. wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
  4545. }
  4546. EXPORT_SYMBOL(wait_for_completion);
  4547. /**
  4548. * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
  4549. * @x: holds the state of this particular completion
  4550. * @timeout: timeout value in jiffies
  4551. *
  4552. * This waits for either a completion of a specific task to be signaled or for a
  4553. * specified timeout to expire. The timeout is in jiffies. It is not
  4554. * interruptible.
  4555. */
  4556. unsigned long __sched
  4557. wait_for_completion_timeout(struct completion *x, unsigned long timeout)
  4558. {
  4559. return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
  4560. }
  4561. EXPORT_SYMBOL(wait_for_completion_timeout);
  4562. /**
  4563. * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
  4564. * @x: holds the state of this particular completion
  4565. *
  4566. * This waits for completion of a specific task to be signaled. It is
  4567. * interruptible.
  4568. */
  4569. int __sched wait_for_completion_interruptible(struct completion *x)
  4570. {
  4571. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
  4572. if (t == -ERESTARTSYS)
  4573. return t;
  4574. return 0;
  4575. }
  4576. EXPORT_SYMBOL(wait_for_completion_interruptible);
  4577. /**
  4578. * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
  4579. * @x: holds the state of this particular completion
  4580. * @timeout: timeout value in jiffies
  4581. *
  4582. * This waits for either a completion of a specific task to be signaled or for a
  4583. * specified timeout to expire. It is interruptible. The timeout is in jiffies.
  4584. */
  4585. unsigned long __sched
  4586. wait_for_completion_interruptible_timeout(struct completion *x,
  4587. unsigned long timeout)
  4588. {
  4589. return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
  4590. }
  4591. EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
  4592. /**
  4593. * wait_for_completion_killable: - waits for completion of a task (killable)
  4594. * @x: holds the state of this particular completion
  4595. *
  4596. * This waits to be signaled for completion of a specific task. It can be
  4597. * interrupted by a kill signal.
  4598. */
  4599. int __sched wait_for_completion_killable(struct completion *x)
  4600. {
  4601. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
  4602. if (t == -ERESTARTSYS)
  4603. return t;
  4604. return 0;
  4605. }
  4606. EXPORT_SYMBOL(wait_for_completion_killable);
  4607. /**
  4608. * try_wait_for_completion - try to decrement a completion without blocking
  4609. * @x: completion structure
  4610. *
  4611. * Returns: 0 if a decrement cannot be done without blocking
  4612. * 1 if a decrement succeeded.
  4613. *
  4614. * If a completion is being used as a counting completion,
  4615. * attempt to decrement the counter without blocking. This
  4616. * enables us to avoid waiting if the resource the completion
  4617. * is protecting is not available.
  4618. */
  4619. bool try_wait_for_completion(struct completion *x)
  4620. {
  4621. int ret = 1;
  4622. spin_lock_irq(&x->wait.lock);
  4623. if (!x->done)
  4624. ret = 0;
  4625. else
  4626. x->done--;
  4627. spin_unlock_irq(&x->wait.lock);
  4628. return ret;
  4629. }
  4630. EXPORT_SYMBOL(try_wait_for_completion);
  4631. /**
  4632. * completion_done - Test to see if a completion has any waiters
  4633. * @x: completion structure
  4634. *
  4635. * Returns: 0 if there are waiters (wait_for_completion() in progress)
  4636. * 1 if there are no waiters.
  4637. *
  4638. */
  4639. bool completion_done(struct completion *x)
  4640. {
  4641. int ret = 1;
  4642. spin_lock_irq(&x->wait.lock);
  4643. if (!x->done)
  4644. ret = 0;
  4645. spin_unlock_irq(&x->wait.lock);
  4646. return ret;
  4647. }
  4648. EXPORT_SYMBOL(completion_done);
  4649. static long __sched
  4650. sleep_on_common(wait_queue_head_t *q, int state, long timeout)
  4651. {
  4652. unsigned long flags;
  4653. wait_queue_t wait;
  4654. init_waitqueue_entry(&wait, current);
  4655. __set_current_state(state);
  4656. spin_lock_irqsave(&q->lock, flags);
  4657. __add_wait_queue(q, &wait);
  4658. spin_unlock(&q->lock);
  4659. timeout = schedule_timeout(timeout);
  4660. spin_lock_irq(&q->lock);
  4661. __remove_wait_queue(q, &wait);
  4662. spin_unlock_irqrestore(&q->lock, flags);
  4663. return timeout;
  4664. }
  4665. void __sched interruptible_sleep_on(wait_queue_head_t *q)
  4666. {
  4667. sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
  4668. }
  4669. EXPORT_SYMBOL(interruptible_sleep_on);
  4670. long __sched
  4671. interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
  4672. {
  4673. return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
  4674. }
  4675. EXPORT_SYMBOL(interruptible_sleep_on_timeout);
  4676. void __sched sleep_on(wait_queue_head_t *q)
  4677. {
  4678. sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
  4679. }
  4680. EXPORT_SYMBOL(sleep_on);
  4681. long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
  4682. {
  4683. return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
  4684. }
  4685. EXPORT_SYMBOL(sleep_on_timeout);
  4686. #ifdef CONFIG_RT_MUTEXES
  4687. /*
  4688. * rt_mutex_setprio - set the current priority of a task
  4689. * @p: task
  4690. * @prio: prio value (kernel-internal form)
  4691. *
  4692. * This function changes the 'effective' priority of a task. It does
  4693. * not touch ->normal_prio like __setscheduler().
  4694. *
  4695. * Used by the rt_mutex code to implement priority inheritance logic.
  4696. */
  4697. void rt_mutex_setprio(struct task_struct *p, int prio)
  4698. {
  4699. unsigned long flags;
  4700. int oldprio, on_rq, running;
  4701. struct rq *rq;
  4702. const struct sched_class *prev_class = p->sched_class;
  4703. BUG_ON(prio < 0 || prio > MAX_PRIO);
  4704. rq = task_rq_lock(p, &flags);
  4705. update_rq_clock(rq);
  4706. oldprio = p->prio;
  4707. on_rq = p->se.on_rq;
  4708. running = task_current(rq, p);
  4709. if (on_rq)
  4710. dequeue_task(rq, p, 0);
  4711. if (running)
  4712. p->sched_class->put_prev_task(rq, p);
  4713. if (rt_prio(prio))
  4714. p->sched_class = &rt_sched_class;
  4715. else
  4716. p->sched_class = &fair_sched_class;
  4717. p->prio = prio;
  4718. if (running)
  4719. p->sched_class->set_curr_task(rq);
  4720. if (on_rq) {
  4721. enqueue_task(rq, p, 0);
  4722. check_class_changed(rq, p, prev_class, oldprio, running);
  4723. }
  4724. task_rq_unlock(rq, &flags);
  4725. }
  4726. #endif
  4727. void set_user_nice(struct task_struct *p, long nice)
  4728. {
  4729. int old_prio, delta, on_rq;
  4730. unsigned long flags;
  4731. struct rq *rq;
  4732. if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
  4733. return;
  4734. /*
  4735. * We have to be careful, if called from sys_setpriority(),
  4736. * the task might be in the middle of scheduling on another CPU.
  4737. */
  4738. rq = task_rq_lock(p, &flags);
  4739. update_rq_clock(rq);
  4740. /*
  4741. * The RT priorities are set via sched_setscheduler(), but we still
  4742. * allow the 'normal' nice value to be set - but as expected
  4743. * it wont have any effect on scheduling until the task is
  4744. * SCHED_FIFO/SCHED_RR:
  4745. */
  4746. if (task_has_rt_policy(p)) {
  4747. p->static_prio = NICE_TO_PRIO(nice);
  4748. goto out_unlock;
  4749. }
  4750. on_rq = p->se.on_rq;
  4751. if (on_rq)
  4752. dequeue_task(rq, p, 0);
  4753. p->static_prio = NICE_TO_PRIO(nice);
  4754. set_load_weight(p);
  4755. old_prio = p->prio;
  4756. p->prio = effective_prio(p);
  4757. delta = p->prio - old_prio;
  4758. if (on_rq) {
  4759. enqueue_task(rq, p, 0);
  4760. /*
  4761. * If the task increased its priority or is running and
  4762. * lowered its priority, then reschedule its CPU:
  4763. */
  4764. if (delta < 0 || (delta > 0 && task_running(rq, p)))
  4765. resched_task(rq->curr);
  4766. }
  4767. out_unlock:
  4768. task_rq_unlock(rq, &flags);
  4769. }
  4770. EXPORT_SYMBOL(set_user_nice);
  4771. /*
  4772. * can_nice - check if a task can reduce its nice value
  4773. * @p: task
  4774. * @nice: nice value
  4775. */
  4776. int can_nice(const struct task_struct *p, const int nice)
  4777. {
  4778. /* convert nice value [19,-20] to rlimit style value [1,40] */
  4779. int nice_rlim = 20 - nice;
  4780. return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur ||
  4781. capable(CAP_SYS_NICE));
  4782. }
  4783. #ifdef __ARCH_WANT_SYS_NICE
  4784. /*
  4785. * sys_nice - change the priority of the current process.
  4786. * @increment: priority increment
  4787. *
  4788. * sys_setpriority is a more generic, but much slower function that
  4789. * does similar things.
  4790. */
  4791. SYSCALL_DEFINE1(nice, int, increment)
  4792. {
  4793. long nice, retval;
  4794. /*
  4795. * Setpriority might change our priority at the same moment.
  4796. * We don't have to worry. Conceptually one call occurs first
  4797. * and we have a single winner.
  4798. */
  4799. if (increment < -40)
  4800. increment = -40;
  4801. if (increment > 40)
  4802. increment = 40;
  4803. nice = TASK_NICE(current) + increment;
  4804. if (nice < -20)
  4805. nice = -20;
  4806. if (nice > 19)
  4807. nice = 19;
  4808. if (increment < 0 && !can_nice(current, nice))
  4809. return -EPERM;
  4810. retval = security_task_setnice(current, nice);
  4811. if (retval)
  4812. return retval;
  4813. set_user_nice(current, nice);
  4814. return 0;
  4815. }
  4816. #endif
  4817. /**
  4818. * task_prio - return the priority value of a given task.
  4819. * @p: the task in question.
  4820. *
  4821. * This is the priority value as seen by users in /proc.
  4822. * RT tasks are offset by -200. Normal tasks are centered
  4823. * around 0, value goes from -16 to +15.
  4824. */
  4825. int task_prio(const struct task_struct *p)
  4826. {
  4827. return p->prio - MAX_RT_PRIO;
  4828. }
  4829. /**
  4830. * task_nice - return the nice value of a given task.
  4831. * @p: the task in question.
  4832. */
  4833. int task_nice(const struct task_struct *p)
  4834. {
  4835. return TASK_NICE(p);
  4836. }
  4837. EXPORT_SYMBOL(task_nice);
  4838. /**
  4839. * idle_cpu - is a given cpu idle currently?
  4840. * @cpu: the processor in question.
  4841. */
  4842. int idle_cpu(int cpu)
  4843. {
  4844. return cpu_curr(cpu) == cpu_rq(cpu)->idle;
  4845. }
  4846. /**
  4847. * idle_task - return the idle task for a given cpu.
  4848. * @cpu: the processor in question.
  4849. */
  4850. struct task_struct *idle_task(int cpu)
  4851. {
  4852. return cpu_rq(cpu)->idle;
  4853. }
  4854. /**
  4855. * find_process_by_pid - find a process with a matching PID value.
  4856. * @pid: the pid in question.
  4857. */
  4858. static struct task_struct *find_process_by_pid(pid_t pid)
  4859. {
  4860. return pid ? find_task_by_vpid(pid) : current;
  4861. }
  4862. /* Actually do priority change: must hold rq lock. */
  4863. static void
  4864. __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
  4865. {
  4866. BUG_ON(p->se.on_rq);
  4867. p->policy = policy;
  4868. switch (p->policy) {
  4869. case SCHED_NORMAL:
  4870. case SCHED_BATCH:
  4871. case SCHED_IDLE:
  4872. p->sched_class = &fair_sched_class;
  4873. break;
  4874. case SCHED_FIFO:
  4875. case SCHED_RR:
  4876. p->sched_class = &rt_sched_class;
  4877. break;
  4878. }
  4879. p->rt_priority = prio;
  4880. p->normal_prio = normal_prio(p);
  4881. /* we are holding p->pi_lock already */
  4882. p->prio = rt_mutex_getprio(p);
  4883. set_load_weight(p);
  4884. }
  4885. /*
  4886. * check the target process has a UID that matches the current process's
  4887. */
  4888. static bool check_same_owner(struct task_struct *p)
  4889. {
  4890. const struct cred *cred = current_cred(), *pcred;
  4891. bool match;
  4892. rcu_read_lock();
  4893. pcred = __task_cred(p);
  4894. match = (cred->euid == pcred->euid ||
  4895. cred->euid == pcred->uid);
  4896. rcu_read_unlock();
  4897. return match;
  4898. }
  4899. static int __sched_setscheduler(struct task_struct *p, int policy,
  4900. struct sched_param *param, bool user)
  4901. {
  4902. int retval, oldprio, oldpolicy = -1, on_rq, running;
  4903. unsigned long flags;
  4904. const struct sched_class *prev_class = p->sched_class;
  4905. struct rq *rq;
  4906. /* may grab non-irq protected spin_locks */
  4907. BUG_ON(in_interrupt());
  4908. recheck:
  4909. /* double check policy once rq lock held */
  4910. if (policy < 0)
  4911. policy = oldpolicy = p->policy;
  4912. else if (policy != SCHED_FIFO && policy != SCHED_RR &&
  4913. policy != SCHED_NORMAL && policy != SCHED_BATCH &&
  4914. policy != SCHED_IDLE)
  4915. return -EINVAL;
  4916. /*
  4917. * Valid priorities for SCHED_FIFO and SCHED_RR are
  4918. * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
  4919. * SCHED_BATCH and SCHED_IDLE is 0.
  4920. */
  4921. if (param->sched_priority < 0 ||
  4922. (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
  4923. (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
  4924. return -EINVAL;
  4925. if (rt_policy(policy) != (param->sched_priority != 0))
  4926. return -EINVAL;
  4927. /*
  4928. * Allow unprivileged RT tasks to decrease priority:
  4929. */
  4930. if (user && !capable(CAP_SYS_NICE)) {
  4931. if (rt_policy(policy)) {
  4932. unsigned long rlim_rtprio;
  4933. if (!lock_task_sighand(p, &flags))
  4934. return -ESRCH;
  4935. rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
  4936. unlock_task_sighand(p, &flags);
  4937. /* can't set/change the rt policy */
  4938. if (policy != p->policy && !rlim_rtprio)
  4939. return -EPERM;
  4940. /* can't increase priority */
  4941. if (param->sched_priority > p->rt_priority &&
  4942. param->sched_priority > rlim_rtprio)
  4943. return -EPERM;
  4944. }
  4945. /*
  4946. * Like positive nice levels, dont allow tasks to
  4947. * move out of SCHED_IDLE either:
  4948. */
  4949. if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
  4950. return -EPERM;
  4951. /* can't change other user's priorities */
  4952. if (!check_same_owner(p))
  4953. return -EPERM;
  4954. }
  4955. if (user) {
  4956. #ifdef CONFIG_RT_GROUP_SCHED
  4957. /*
  4958. * Do not allow realtime tasks into groups that have no runtime
  4959. * assigned.
  4960. */
  4961. if (rt_bandwidth_enabled() && rt_policy(policy) &&
  4962. task_group(p)->rt_bandwidth.rt_runtime == 0)
  4963. return -EPERM;
  4964. #endif
  4965. retval = security_task_setscheduler(p, policy, param);
  4966. if (retval)
  4967. return retval;
  4968. }
  4969. /*
  4970. * make sure no PI-waiters arrive (or leave) while we are
  4971. * changing the priority of the task:
  4972. */
  4973. spin_lock_irqsave(&p->pi_lock, flags);
  4974. /*
  4975. * To be able to change p->policy safely, the apropriate
  4976. * runqueue lock must be held.
  4977. */
  4978. rq = __task_rq_lock(p);
  4979. /* recheck policy now with rq lock held */
  4980. if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
  4981. policy = oldpolicy = -1;
  4982. __task_rq_unlock(rq);
  4983. spin_unlock_irqrestore(&p->pi_lock, flags);
  4984. goto recheck;
  4985. }
  4986. update_rq_clock(rq);
  4987. on_rq = p->se.on_rq;
  4988. running = task_current(rq, p);
  4989. if (on_rq)
  4990. deactivate_task(rq, p, 0);
  4991. if (running)
  4992. p->sched_class->put_prev_task(rq, p);
  4993. oldprio = p->prio;
  4994. __setscheduler(rq, p, policy, param->sched_priority);
  4995. if (running)
  4996. p->sched_class->set_curr_task(rq);
  4997. if (on_rq) {
  4998. activate_task(rq, p, 0);
  4999. check_class_changed(rq, p, prev_class, oldprio, running);
  5000. }
  5001. __task_rq_unlock(rq);
  5002. spin_unlock_irqrestore(&p->pi_lock, flags);
  5003. rt_mutex_adjust_pi(p);
  5004. return 0;
  5005. }
  5006. /**
  5007. * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
  5008. * @p: the task in question.
  5009. * @policy: new policy.
  5010. * @param: structure containing the new RT priority.
  5011. *
  5012. * NOTE that the task may be already dead.
  5013. */
  5014. int sched_setscheduler(struct task_struct *p, int policy,
  5015. struct sched_param *param)
  5016. {
  5017. return __sched_setscheduler(p, policy, param, true);
  5018. }
  5019. EXPORT_SYMBOL_GPL(sched_setscheduler);
  5020. /**
  5021. * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
  5022. * @p: the task in question.
  5023. * @policy: new policy.
  5024. * @param: structure containing the new RT priority.
  5025. *
  5026. * Just like sched_setscheduler, only don't bother checking if the
  5027. * current context has permission. For example, this is needed in
  5028. * stop_machine(): we create temporary high priority worker threads,
  5029. * but our caller might not have that capability.
  5030. */
  5031. int sched_setscheduler_nocheck(struct task_struct *p, int policy,
  5032. struct sched_param *param)
  5033. {
  5034. return __sched_setscheduler(p, policy, param, false);
  5035. }
  5036. static int
  5037. do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
  5038. {
  5039. struct sched_param lparam;
  5040. struct task_struct *p;
  5041. int retval;
  5042. if (!param || pid < 0)
  5043. return -EINVAL;
  5044. if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
  5045. return -EFAULT;
  5046. rcu_read_lock();
  5047. retval = -ESRCH;
  5048. p = find_process_by_pid(pid);
  5049. if (p != NULL)
  5050. retval = sched_setscheduler(p, policy, &lparam);
  5051. rcu_read_unlock();
  5052. return retval;
  5053. }
  5054. /**
  5055. * sys_sched_setscheduler - set/change the scheduler policy and RT priority
  5056. * @pid: the pid in question.
  5057. * @policy: new policy.
  5058. * @param: structure containing the new RT priority.
  5059. */
  5060. SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
  5061. struct sched_param __user *, param)
  5062. {
  5063. /* negative values for policy are not valid */
  5064. if (policy < 0)
  5065. return -EINVAL;
  5066. return do_sched_setscheduler(pid, policy, param);
  5067. }
  5068. /**
  5069. * sys_sched_setparam - set/change the RT priority of a thread
  5070. * @pid: the pid in question.
  5071. * @param: structure containing the new RT priority.
  5072. */
  5073. SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
  5074. {
  5075. return do_sched_setscheduler(pid, -1, param);
  5076. }
  5077. /**
  5078. * sys_sched_getscheduler - get the policy (scheduling class) of a thread
  5079. * @pid: the pid in question.
  5080. */
  5081. SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
  5082. {
  5083. struct task_struct *p;
  5084. int retval;
  5085. if (pid < 0)
  5086. return -EINVAL;
  5087. retval = -ESRCH;
  5088. read_lock(&tasklist_lock);
  5089. p = find_process_by_pid(pid);
  5090. if (p) {
  5091. retval = security_task_getscheduler(p);
  5092. if (!retval)
  5093. retval = p->policy;
  5094. }
  5095. read_unlock(&tasklist_lock);
  5096. return retval;
  5097. }
  5098. /**
  5099. * sys_sched_getscheduler - get the RT priority of a thread
  5100. * @pid: the pid in question.
  5101. * @param: structure containing the RT priority.
  5102. */
  5103. SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
  5104. {
  5105. struct sched_param lp;
  5106. struct task_struct *p;
  5107. int retval;
  5108. if (!param || pid < 0)
  5109. return -EINVAL;
  5110. read_lock(&tasklist_lock);
  5111. p = find_process_by_pid(pid);
  5112. retval = -ESRCH;
  5113. if (!p)
  5114. goto out_unlock;
  5115. retval = security_task_getscheduler(p);
  5116. if (retval)
  5117. goto out_unlock;
  5118. lp.sched_priority = p->rt_priority;
  5119. read_unlock(&tasklist_lock);
  5120. /*
  5121. * This one might sleep, we cannot do it with a spinlock held ...
  5122. */
  5123. retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
  5124. return retval;
  5125. out_unlock:
  5126. read_unlock(&tasklist_lock);
  5127. return retval;
  5128. }
  5129. long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
  5130. {
  5131. cpumask_var_t cpus_allowed, new_mask;
  5132. struct task_struct *p;
  5133. int retval;
  5134. get_online_cpus();
  5135. read_lock(&tasklist_lock);
  5136. p = find_process_by_pid(pid);
  5137. if (!p) {
  5138. read_unlock(&tasklist_lock);
  5139. put_online_cpus();
  5140. return -ESRCH;
  5141. }
  5142. /*
  5143. * It is not safe to call set_cpus_allowed with the
  5144. * tasklist_lock held. We will bump the task_struct's
  5145. * usage count and then drop tasklist_lock.
  5146. */
  5147. get_task_struct(p);
  5148. read_unlock(&tasklist_lock);
  5149. if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
  5150. retval = -ENOMEM;
  5151. goto out_put_task;
  5152. }
  5153. if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
  5154. retval = -ENOMEM;
  5155. goto out_free_cpus_allowed;
  5156. }
  5157. retval = -EPERM;
  5158. if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
  5159. goto out_unlock;
  5160. retval = security_task_setscheduler(p, 0, NULL);
  5161. if (retval)
  5162. goto out_unlock;
  5163. cpuset_cpus_allowed(p, cpus_allowed);
  5164. cpumask_and(new_mask, in_mask, cpus_allowed);
  5165. again:
  5166. retval = set_cpus_allowed_ptr(p, new_mask);
  5167. if (!retval) {
  5168. cpuset_cpus_allowed(p, cpus_allowed);
  5169. if (!cpumask_subset(new_mask, cpus_allowed)) {
  5170. /*
  5171. * We must have raced with a concurrent cpuset
  5172. * update. Just reset the cpus_allowed to the
  5173. * cpuset's cpus_allowed
  5174. */
  5175. cpumask_copy(new_mask, cpus_allowed);
  5176. goto again;
  5177. }
  5178. }
  5179. out_unlock:
  5180. free_cpumask_var(new_mask);
  5181. out_free_cpus_allowed:
  5182. free_cpumask_var(cpus_allowed);
  5183. out_put_task:
  5184. put_task_struct(p);
  5185. put_online_cpus();
  5186. return retval;
  5187. }
  5188. static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
  5189. struct cpumask *new_mask)
  5190. {
  5191. if (len < cpumask_size())
  5192. cpumask_clear(new_mask);
  5193. else if (len > cpumask_size())
  5194. len = cpumask_size();
  5195. return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
  5196. }
  5197. /**
  5198. * sys_sched_setaffinity - set the cpu affinity of a process
  5199. * @pid: pid of the process
  5200. * @len: length in bytes of the bitmask pointed to by user_mask_ptr
  5201. * @user_mask_ptr: user-space pointer to the new cpu mask
  5202. */
  5203. SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
  5204. unsigned long __user *, user_mask_ptr)
  5205. {
  5206. cpumask_var_t new_mask;
  5207. int retval;
  5208. if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
  5209. return -ENOMEM;
  5210. retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
  5211. if (retval == 0)
  5212. retval = sched_setaffinity(pid, new_mask);
  5213. free_cpumask_var(new_mask);
  5214. return retval;
  5215. }
  5216. long sched_getaffinity(pid_t pid, struct cpumask *mask)
  5217. {
  5218. struct task_struct *p;
  5219. int retval;
  5220. get_online_cpus();
  5221. read_lock(&tasklist_lock);
  5222. retval = -ESRCH;
  5223. p = find_process_by_pid(pid);
  5224. if (!p)
  5225. goto out_unlock;
  5226. retval = security_task_getscheduler(p);
  5227. if (retval)
  5228. goto out_unlock;
  5229. cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
  5230. out_unlock:
  5231. read_unlock(&tasklist_lock);
  5232. put_online_cpus();
  5233. return retval;
  5234. }
  5235. /**
  5236. * sys_sched_getaffinity - get the cpu affinity of a process
  5237. * @pid: pid of the process
  5238. * @len: length in bytes of the bitmask pointed to by user_mask_ptr
  5239. * @user_mask_ptr: user-space pointer to hold the current cpu mask
  5240. */
  5241. SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
  5242. unsigned long __user *, user_mask_ptr)
  5243. {
  5244. int ret;
  5245. cpumask_var_t mask;
  5246. if (len < cpumask_size())
  5247. return -EINVAL;
  5248. if (!alloc_cpumask_var(&mask, GFP_KERNEL))
  5249. return -ENOMEM;
  5250. ret = sched_getaffinity(pid, mask);
  5251. if (ret == 0) {
  5252. if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
  5253. ret = -EFAULT;
  5254. else
  5255. ret = cpumask_size();
  5256. }
  5257. free_cpumask_var(mask);
  5258. return ret;
  5259. }
  5260. /**
  5261. * sys_sched_yield - yield the current processor to other threads.
  5262. *
  5263. * This function yields the current CPU to other tasks. If there are no
  5264. * other threads running on this CPU then this function will return.
  5265. */
  5266. SYSCALL_DEFINE0(sched_yield)
  5267. {
  5268. struct rq *rq = this_rq_lock();
  5269. schedstat_inc(rq, yld_count);
  5270. current->sched_class->yield_task(rq);
  5271. /*
  5272. * Since we are going to call schedule() anyway, there's
  5273. * no need to preempt or enable interrupts:
  5274. */
  5275. __release(rq->lock);
  5276. spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
  5277. _raw_spin_unlock(&rq->lock);
  5278. preempt_enable_no_resched();
  5279. schedule();
  5280. return 0;
  5281. }
  5282. static void __cond_resched(void)
  5283. {
  5284. #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
  5285. __might_sleep(__FILE__, __LINE__);
  5286. #endif
  5287. /*
  5288. * The BKS might be reacquired before we have dropped
  5289. * PREEMPT_ACTIVE, which could trigger a second
  5290. * cond_resched() call.
  5291. */
  5292. do {
  5293. add_preempt_count(PREEMPT_ACTIVE);
  5294. schedule();
  5295. sub_preempt_count(PREEMPT_ACTIVE);
  5296. } while (need_resched());
  5297. }
  5298. int __sched _cond_resched(void)
  5299. {
  5300. if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
  5301. system_state == SYSTEM_RUNNING) {
  5302. __cond_resched();
  5303. return 1;
  5304. }
  5305. return 0;
  5306. }
  5307. EXPORT_SYMBOL(_cond_resched);
  5308. /*
  5309. * cond_resched_lock() - if a reschedule is pending, drop the given lock,
  5310. * call schedule, and on return reacquire the lock.
  5311. *
  5312. * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
  5313. * operations here to prevent schedule() from being called twice (once via
  5314. * spin_unlock(), once by hand).
  5315. */
  5316. int cond_resched_lock(spinlock_t *lock)
  5317. {
  5318. int resched = need_resched() && system_state == SYSTEM_RUNNING;
  5319. int ret = 0;
  5320. if (spin_needbreak(lock) || resched) {
  5321. spin_unlock(lock);
  5322. if (resched && need_resched())
  5323. __cond_resched();
  5324. else
  5325. cpu_relax();
  5326. ret = 1;
  5327. spin_lock(lock);
  5328. }
  5329. return ret;
  5330. }
  5331. EXPORT_SYMBOL(cond_resched_lock);
  5332. int __sched cond_resched_softirq(void)
  5333. {
  5334. BUG_ON(!in_softirq());
  5335. if (need_resched() && system_state == SYSTEM_RUNNING) {
  5336. local_bh_enable();
  5337. __cond_resched();
  5338. local_bh_disable();
  5339. return 1;
  5340. }
  5341. return 0;
  5342. }
  5343. EXPORT_SYMBOL(cond_resched_softirq);
  5344. /**
  5345. * yield - yield the current processor to other threads.
  5346. *
  5347. * This is a shortcut for kernel-space yielding - it marks the
  5348. * thread runnable and calls sys_sched_yield().
  5349. */
  5350. void __sched yield(void)
  5351. {
  5352. set_current_state(TASK_RUNNING);
  5353. sys_sched_yield();
  5354. }
  5355. EXPORT_SYMBOL(yield);
  5356. /*
  5357. * This task is about to go to sleep on IO. Increment rq->nr_iowait so
  5358. * that process accounting knows that this is a task in IO wait state.
  5359. *
  5360. * But don't do that if it is a deliberate, throttling IO wait (this task
  5361. * has set its backing_dev_info: the queue against which it should throttle)
  5362. */
  5363. void __sched io_schedule(void)
  5364. {
  5365. struct rq *rq = &__raw_get_cpu_var(runqueues);
  5366. delayacct_blkio_start();
  5367. atomic_inc(&rq->nr_iowait);
  5368. schedule();
  5369. atomic_dec(&rq->nr_iowait);
  5370. delayacct_blkio_end();
  5371. }
  5372. EXPORT_SYMBOL(io_schedule);
  5373. long __sched io_schedule_timeout(long timeout)
  5374. {
  5375. struct rq *rq = &__raw_get_cpu_var(runqueues);
  5376. long ret;
  5377. delayacct_blkio_start();
  5378. atomic_inc(&rq->nr_iowait);
  5379. ret = schedule_timeout(timeout);
  5380. atomic_dec(&rq->nr_iowait);
  5381. delayacct_blkio_end();
  5382. return ret;
  5383. }
  5384. /**
  5385. * sys_sched_get_priority_max - return maximum RT priority.
  5386. * @policy: scheduling class.
  5387. *
  5388. * this syscall returns the maximum rt_priority that can be used
  5389. * by a given scheduling class.
  5390. */
  5391. SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
  5392. {
  5393. int ret = -EINVAL;
  5394. switch (policy) {
  5395. case SCHED_FIFO:
  5396. case SCHED_RR:
  5397. ret = MAX_USER_RT_PRIO-1;
  5398. break;
  5399. case SCHED_NORMAL:
  5400. case SCHED_BATCH:
  5401. case SCHED_IDLE:
  5402. ret = 0;
  5403. break;
  5404. }
  5405. return ret;
  5406. }
  5407. /**
  5408. * sys_sched_get_priority_min - return minimum RT priority.
  5409. * @policy: scheduling class.
  5410. *
  5411. * this syscall returns the minimum rt_priority that can be used
  5412. * by a given scheduling class.
  5413. */
  5414. SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
  5415. {
  5416. int ret = -EINVAL;
  5417. switch (policy) {
  5418. case SCHED_FIFO:
  5419. case SCHED_RR:
  5420. ret = 1;
  5421. break;
  5422. case SCHED_NORMAL:
  5423. case SCHED_BATCH:
  5424. case SCHED_IDLE:
  5425. ret = 0;
  5426. }
  5427. return ret;
  5428. }
  5429. /**
  5430. * sys_sched_rr_get_interval - return the default timeslice of a process.
  5431. * @pid: pid of the process.
  5432. * @interval: userspace pointer to the timeslice value.
  5433. *
  5434. * this syscall writes the default timeslice value of a given process
  5435. * into the user-space timespec buffer. A value of '0' means infinity.
  5436. */
  5437. SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
  5438. struct timespec __user *, interval)
  5439. {
  5440. struct task_struct *p;
  5441. unsigned int time_slice;
  5442. int retval;
  5443. struct timespec t;
  5444. if (pid < 0)
  5445. return -EINVAL;
  5446. retval = -ESRCH;
  5447. read_lock(&tasklist_lock);
  5448. p = find_process_by_pid(pid);
  5449. if (!p)
  5450. goto out_unlock;
  5451. retval = security_task_getscheduler(p);
  5452. if (retval)
  5453. goto out_unlock;
  5454. /*
  5455. * Time slice is 0 for SCHED_FIFO tasks and for SCHED_OTHER
  5456. * tasks that are on an otherwise idle runqueue:
  5457. */
  5458. time_slice = 0;
  5459. if (p->policy == SCHED_RR) {
  5460. time_slice = DEF_TIMESLICE;
  5461. } else if (p->policy != SCHED_FIFO) {
  5462. struct sched_entity *se = &p->se;
  5463. unsigned long flags;
  5464. struct rq *rq;
  5465. rq = task_rq_lock(p, &flags);
  5466. if (rq->cfs.load.weight)
  5467. time_slice = NS_TO_JIFFIES(sched_slice(&rq->cfs, se));
  5468. task_rq_unlock(rq, &flags);
  5469. }
  5470. read_unlock(&tasklist_lock);
  5471. jiffies_to_timespec(time_slice, &t);
  5472. retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
  5473. return retval;
  5474. out_unlock:
  5475. read_unlock(&tasklist_lock);
  5476. return retval;
  5477. }
  5478. static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
  5479. void sched_show_task(struct task_struct *p)
  5480. {
  5481. unsigned long free = 0;
  5482. unsigned state;
  5483. state = p->state ? __ffs(p->state) + 1 : 0;
  5484. printk(KERN_INFO "%-13.13s %c", p->comm,
  5485. state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
  5486. #if BITS_PER_LONG == 32
  5487. if (state == TASK_RUNNING)
  5488. printk(KERN_CONT " running ");
  5489. else
  5490. printk(KERN_CONT " %08lx ", thread_saved_pc(p));
  5491. #else
  5492. if (state == TASK_RUNNING)
  5493. printk(KERN_CONT " running task ");
  5494. else
  5495. printk(KERN_CONT " %016lx ", thread_saved_pc(p));
  5496. #endif
  5497. #ifdef CONFIG_DEBUG_STACK_USAGE
  5498. free = stack_not_used(p);
  5499. #endif
  5500. printk(KERN_CONT "%5lu %5d %6d\n", free,
  5501. task_pid_nr(p), task_pid_nr(p->real_parent));
  5502. show_stack(p, NULL);
  5503. }
  5504. void show_state_filter(unsigned long state_filter)
  5505. {
  5506. struct task_struct *g, *p;
  5507. #if BITS_PER_LONG == 32
  5508. printk(KERN_INFO
  5509. " task PC stack pid father\n");
  5510. #else
  5511. printk(KERN_INFO
  5512. " task PC stack pid father\n");
  5513. #endif
  5514. read_lock(&tasklist_lock);
  5515. do_each_thread(g, p) {
  5516. /*
  5517. * reset the NMI-timeout, listing all files on a slow
  5518. * console might take alot of time:
  5519. */
  5520. touch_nmi_watchdog();
  5521. if (!state_filter || (p->state & state_filter))
  5522. sched_show_task(p);
  5523. } while_each_thread(g, p);
  5524. touch_all_softlockup_watchdogs();
  5525. #ifdef CONFIG_SCHED_DEBUG
  5526. sysrq_sched_debug_show();
  5527. #endif
  5528. read_unlock(&tasklist_lock);
  5529. /*
  5530. * Only show locks if all tasks are dumped:
  5531. */
  5532. if (state_filter == -1)
  5533. debug_show_all_locks();
  5534. }
  5535. void __cpuinit init_idle_bootup_task(struct task_struct *idle)
  5536. {
  5537. idle->sched_class = &idle_sched_class;
  5538. }
  5539. /**
  5540. * init_idle - set up an idle thread for a given CPU
  5541. * @idle: task in question
  5542. * @cpu: cpu the idle task belongs to
  5543. *
  5544. * NOTE: this function does not set the idle thread's NEED_RESCHED
  5545. * flag, to make booting more robust.
  5546. */
  5547. void __cpuinit init_idle(struct task_struct *idle, int cpu)
  5548. {
  5549. struct rq *rq = cpu_rq(cpu);
  5550. unsigned long flags;
  5551. spin_lock_irqsave(&rq->lock, flags);
  5552. __sched_fork(idle);
  5553. idle->se.exec_start = sched_clock();
  5554. idle->prio = idle->normal_prio = MAX_PRIO;
  5555. cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
  5556. __set_task_cpu(idle, cpu);
  5557. rq->curr = rq->idle = idle;
  5558. #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
  5559. idle->oncpu = 1;
  5560. #endif
  5561. spin_unlock_irqrestore(&rq->lock, flags);
  5562. /* Set the preempt count _outside_ the spinlocks! */
  5563. #if defined(CONFIG_PREEMPT)
  5564. task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
  5565. #else
  5566. task_thread_info(idle)->preempt_count = 0;
  5567. #endif
  5568. /*
  5569. * The idle tasks have their own, simple scheduling class:
  5570. */
  5571. idle->sched_class = &idle_sched_class;
  5572. ftrace_graph_init_task(idle);
  5573. }
  5574. /*
  5575. * In a system that switches off the HZ timer nohz_cpu_mask
  5576. * indicates which cpus entered this state. This is used
  5577. * in the rcu update to wait only for active cpus. For system
  5578. * which do not switch off the HZ timer nohz_cpu_mask should
  5579. * always be CPU_BITS_NONE.
  5580. */
  5581. cpumask_var_t nohz_cpu_mask;
  5582. /*
  5583. * Increase the granularity value when there are more CPUs,
  5584. * because with more CPUs the 'effective latency' as visible
  5585. * to users decreases. But the relationship is not linear,
  5586. * so pick a second-best guess by going with the log2 of the
  5587. * number of CPUs.
  5588. *
  5589. * This idea comes from the SD scheduler of Con Kolivas:
  5590. */
  5591. static inline void sched_init_granularity(void)
  5592. {
  5593. unsigned int factor = 1 + ilog2(num_online_cpus());
  5594. const unsigned long limit = 200000000;
  5595. sysctl_sched_min_granularity *= factor;
  5596. if (sysctl_sched_min_granularity > limit)
  5597. sysctl_sched_min_granularity = limit;
  5598. sysctl_sched_latency *= factor;
  5599. if (sysctl_sched_latency > limit)
  5600. sysctl_sched_latency = limit;
  5601. sysctl_sched_wakeup_granularity *= factor;
  5602. sysctl_sched_shares_ratelimit *= factor;
  5603. }
  5604. #ifdef CONFIG_SMP
  5605. /*
  5606. * This is how migration works:
  5607. *
  5608. * 1) we queue a struct migration_req structure in the source CPU's
  5609. * runqueue and wake up that CPU's migration thread.
  5610. * 2) we down() the locked semaphore => thread blocks.
  5611. * 3) migration thread wakes up (implicitly it forces the migrated
  5612. * thread off the CPU)
  5613. * 4) it gets the migration request and checks whether the migrated
  5614. * task is still in the wrong runqueue.
  5615. * 5) if it's in the wrong runqueue then the migration thread removes
  5616. * it and puts it into the right queue.
  5617. * 6) migration thread up()s the semaphore.
  5618. * 7) we wake up and the migration is done.
  5619. */
  5620. /*
  5621. * Change a given task's CPU affinity. Migrate the thread to a
  5622. * proper CPU and schedule it away if the CPU it's executing on
  5623. * is removed from the allowed bitmask.
  5624. *
  5625. * NOTE: the caller must have a valid reference to the task, the
  5626. * task must not exit() & deallocate itself prematurely. The
  5627. * call is not atomic; no spinlocks may be held.
  5628. */
  5629. int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
  5630. {
  5631. struct migration_req req;
  5632. unsigned long flags;
  5633. struct rq *rq;
  5634. int ret = 0;
  5635. rq = task_rq_lock(p, &flags);
  5636. if (!cpumask_intersects(new_mask, cpu_online_mask)) {
  5637. ret = -EINVAL;
  5638. goto out;
  5639. }
  5640. if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
  5641. !cpumask_equal(&p->cpus_allowed, new_mask))) {
  5642. ret = -EINVAL;
  5643. goto out;
  5644. }
  5645. if (p->sched_class->set_cpus_allowed)
  5646. p->sched_class->set_cpus_allowed(p, new_mask);
  5647. else {
  5648. cpumask_copy(&p->cpus_allowed, new_mask);
  5649. p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
  5650. }
  5651. /* Can the task run on the task's current CPU? If so, we're done */
  5652. if (cpumask_test_cpu(task_cpu(p), new_mask))
  5653. goto out;
  5654. if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
  5655. /* Need help from migration thread: drop lock and wait. */
  5656. task_rq_unlock(rq, &flags);
  5657. wake_up_process(rq->migration_thread);
  5658. wait_for_completion(&req.done);
  5659. tlb_migrate_finish(p->mm);
  5660. return 0;
  5661. }
  5662. out:
  5663. task_rq_unlock(rq, &flags);
  5664. return ret;
  5665. }
  5666. EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
  5667. /*
  5668. * Move (not current) task off this cpu, onto dest cpu. We're doing
  5669. * this because either it can't run here any more (set_cpus_allowed()
  5670. * away from this CPU, or CPU going down), or because we're
  5671. * attempting to rebalance this task on exec (sched_exec).
  5672. *
  5673. * So we race with normal scheduler movements, but that's OK, as long
  5674. * as the task is no longer on this CPU.
  5675. *
  5676. * Returns non-zero if task was successfully migrated.
  5677. */
  5678. static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
  5679. {
  5680. struct rq *rq_dest, *rq_src;
  5681. int ret = 0, on_rq;
  5682. if (unlikely(!cpu_active(dest_cpu)))
  5683. return ret;
  5684. rq_src = cpu_rq(src_cpu);
  5685. rq_dest = cpu_rq(dest_cpu);
  5686. double_rq_lock(rq_src, rq_dest);
  5687. /* Already moved. */
  5688. if (task_cpu(p) != src_cpu)
  5689. goto done;
  5690. /* Affinity changed (again). */
  5691. if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
  5692. goto fail;
  5693. on_rq = p->se.on_rq;
  5694. if (on_rq)
  5695. deactivate_task(rq_src, p, 0);
  5696. set_task_cpu(p, dest_cpu);
  5697. if (on_rq) {
  5698. activate_task(rq_dest, p, 0);
  5699. check_preempt_curr(rq_dest, p, 0);
  5700. }
  5701. done:
  5702. ret = 1;
  5703. fail:
  5704. double_rq_unlock(rq_src, rq_dest);
  5705. return ret;
  5706. }
  5707. /*
  5708. * migration_thread - this is a highprio system thread that performs
  5709. * thread migration by bumping thread off CPU then 'pushing' onto
  5710. * another runqueue.
  5711. */
  5712. static int migration_thread(void *data)
  5713. {
  5714. int cpu = (long)data;
  5715. struct rq *rq;
  5716. rq = cpu_rq(cpu);
  5717. BUG_ON(rq->migration_thread != current);
  5718. set_current_state(TASK_INTERRUPTIBLE);
  5719. while (!kthread_should_stop()) {
  5720. struct migration_req *req;
  5721. struct list_head *head;
  5722. spin_lock_irq(&rq->lock);
  5723. if (cpu_is_offline(cpu)) {
  5724. spin_unlock_irq(&rq->lock);
  5725. goto wait_to_die;
  5726. }
  5727. if (rq->active_balance) {
  5728. active_load_balance(rq, cpu);
  5729. rq->active_balance = 0;
  5730. }
  5731. head = &rq->migration_queue;
  5732. if (list_empty(head)) {
  5733. spin_unlock_irq(&rq->lock);
  5734. schedule();
  5735. set_current_state(TASK_INTERRUPTIBLE);
  5736. continue;
  5737. }
  5738. req = list_entry(head->next, struct migration_req, list);
  5739. list_del_init(head->next);
  5740. spin_unlock(&rq->lock);
  5741. __migrate_task(req->task, cpu, req->dest_cpu);
  5742. local_irq_enable();
  5743. complete(&req->done);
  5744. }
  5745. __set_current_state(TASK_RUNNING);
  5746. return 0;
  5747. wait_to_die:
  5748. /* Wait for kthread_stop */
  5749. set_current_state(TASK_INTERRUPTIBLE);
  5750. while (!kthread_should_stop()) {
  5751. schedule();
  5752. set_current_state(TASK_INTERRUPTIBLE);
  5753. }
  5754. __set_current_state(TASK_RUNNING);
  5755. return 0;
  5756. }
  5757. #ifdef CONFIG_HOTPLUG_CPU
  5758. static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
  5759. {
  5760. int ret;
  5761. local_irq_disable();
  5762. ret = __migrate_task(p, src_cpu, dest_cpu);
  5763. local_irq_enable();
  5764. return ret;
  5765. }
  5766. /*
  5767. * Figure out where task on dead CPU should go, use force if necessary.
  5768. */
  5769. static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
  5770. {
  5771. int dest_cpu;
  5772. const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
  5773. again:
  5774. /* Look for allowed, online CPU in same node. */
  5775. for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
  5776. if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
  5777. goto move;
  5778. /* Any allowed, online CPU? */
  5779. dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
  5780. if (dest_cpu < nr_cpu_ids)
  5781. goto move;
  5782. /* No more Mr. Nice Guy. */
  5783. if (dest_cpu >= nr_cpu_ids) {
  5784. cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
  5785. dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
  5786. /*
  5787. * Don't tell them about moving exiting tasks or
  5788. * kernel threads (both mm NULL), since they never
  5789. * leave kernel.
  5790. */
  5791. if (p->mm && printk_ratelimit()) {
  5792. printk(KERN_INFO "process %d (%s) no "
  5793. "longer affine to cpu%d\n",
  5794. task_pid_nr(p), p->comm, dead_cpu);
  5795. }
  5796. }
  5797. move:
  5798. /* It can have affinity changed while we were choosing. */
  5799. if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
  5800. goto again;
  5801. }
  5802. /*
  5803. * While a dead CPU has no uninterruptible tasks queued at this point,
  5804. * it might still have a nonzero ->nr_uninterruptible counter, because
  5805. * for performance reasons the counter is not stricly tracking tasks to
  5806. * their home CPUs. So we just add the counter to another CPU's counter,
  5807. * to keep the global sum constant after CPU-down:
  5808. */
  5809. static void migrate_nr_uninterruptible(struct rq *rq_src)
  5810. {
  5811. struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
  5812. unsigned long flags;
  5813. local_irq_save(flags);
  5814. double_rq_lock(rq_src, rq_dest);
  5815. rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
  5816. rq_src->nr_uninterruptible = 0;
  5817. double_rq_unlock(rq_src, rq_dest);
  5818. local_irq_restore(flags);
  5819. }
  5820. /* Run through task list and migrate tasks from the dead cpu. */
  5821. static void migrate_live_tasks(int src_cpu)
  5822. {
  5823. struct task_struct *p, *t;
  5824. read_lock(&tasklist_lock);
  5825. do_each_thread(t, p) {
  5826. if (p == current)
  5827. continue;
  5828. if (task_cpu(p) == src_cpu)
  5829. move_task_off_dead_cpu(src_cpu, p);
  5830. } while_each_thread(t, p);
  5831. read_unlock(&tasklist_lock);
  5832. }
  5833. /*
  5834. * Schedules idle task to be the next runnable task on current CPU.
  5835. * It does so by boosting its priority to highest possible.
  5836. * Used by CPU offline code.
  5837. */
  5838. void sched_idle_next(void)
  5839. {
  5840. int this_cpu = smp_processor_id();
  5841. struct rq *rq = cpu_rq(this_cpu);
  5842. struct task_struct *p = rq->idle;
  5843. unsigned long flags;
  5844. /* cpu has to be offline */
  5845. BUG_ON(cpu_online(this_cpu));
  5846. /*
  5847. * Strictly not necessary since rest of the CPUs are stopped by now
  5848. * and interrupts disabled on the current cpu.
  5849. */
  5850. spin_lock_irqsave(&rq->lock, flags);
  5851. __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
  5852. update_rq_clock(rq);
  5853. activate_task(rq, p, 0);
  5854. spin_unlock_irqrestore(&rq->lock, flags);
  5855. }
  5856. /*
  5857. * Ensures that the idle task is using init_mm right before its cpu goes
  5858. * offline.
  5859. */
  5860. void idle_task_exit(void)
  5861. {
  5862. struct mm_struct *mm = current->active_mm;
  5863. BUG_ON(cpu_online(smp_processor_id()));
  5864. if (mm != &init_mm)
  5865. switch_mm(mm, &init_mm, current);
  5866. mmdrop(mm);
  5867. }
  5868. /* called under rq->lock with disabled interrupts */
  5869. static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
  5870. {
  5871. struct rq *rq = cpu_rq(dead_cpu);
  5872. /* Must be exiting, otherwise would be on tasklist. */
  5873. BUG_ON(!p->exit_state);
  5874. /* Cannot have done final schedule yet: would have vanished. */
  5875. BUG_ON(p->state == TASK_DEAD);
  5876. get_task_struct(p);
  5877. /*
  5878. * Drop lock around migration; if someone else moves it,
  5879. * that's OK. No task can be added to this CPU, so iteration is
  5880. * fine.
  5881. */
  5882. spin_unlock_irq(&rq->lock);
  5883. move_task_off_dead_cpu(dead_cpu, p);
  5884. spin_lock_irq(&rq->lock);
  5885. put_task_struct(p);
  5886. }
  5887. /* release_task() removes task from tasklist, so we won't find dead tasks. */
  5888. static void migrate_dead_tasks(unsigned int dead_cpu)
  5889. {
  5890. struct rq *rq = cpu_rq(dead_cpu);
  5891. struct task_struct *next;
  5892. for ( ; ; ) {
  5893. if (!rq->nr_running)
  5894. break;
  5895. update_rq_clock(rq);
  5896. next = pick_next_task(rq);
  5897. if (!next)
  5898. break;
  5899. next->sched_class->put_prev_task(rq, next);
  5900. migrate_dead(dead_cpu, next);
  5901. }
  5902. }
  5903. #endif /* CONFIG_HOTPLUG_CPU */
  5904. #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
  5905. static struct ctl_table sd_ctl_dir[] = {
  5906. {
  5907. .procname = "sched_domain",
  5908. .mode = 0555,
  5909. },
  5910. {0, },
  5911. };
  5912. static struct ctl_table sd_ctl_root[] = {
  5913. {
  5914. .ctl_name = CTL_KERN,
  5915. .procname = "kernel",
  5916. .mode = 0555,
  5917. .child = sd_ctl_dir,
  5918. },
  5919. {0, },
  5920. };
  5921. static struct ctl_table *sd_alloc_ctl_entry(int n)
  5922. {
  5923. struct ctl_table *entry =
  5924. kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
  5925. return entry;
  5926. }
  5927. static void sd_free_ctl_entry(struct ctl_table **tablep)
  5928. {
  5929. struct ctl_table *entry;
  5930. /*
  5931. * In the intermediate directories, both the child directory and
  5932. * procname are dynamically allocated and could fail but the mode
  5933. * will always be set. In the lowest directory the names are
  5934. * static strings and all have proc handlers.
  5935. */
  5936. for (entry = *tablep; entry->mode; entry++) {
  5937. if (entry->child)
  5938. sd_free_ctl_entry(&entry->child);
  5939. if (entry->proc_handler == NULL)
  5940. kfree(entry->procname);
  5941. }
  5942. kfree(*tablep);
  5943. *tablep = NULL;
  5944. }
  5945. static void
  5946. set_table_entry(struct ctl_table *entry,
  5947. const char *procname, void *data, int maxlen,
  5948. mode_t mode, proc_handler *proc_handler)
  5949. {
  5950. entry->procname = procname;
  5951. entry->data = data;
  5952. entry->maxlen = maxlen;
  5953. entry->mode = mode;
  5954. entry->proc_handler = proc_handler;
  5955. }
  5956. static struct ctl_table *
  5957. sd_alloc_ctl_domain_table(struct sched_domain *sd)
  5958. {
  5959. struct ctl_table *table = sd_alloc_ctl_entry(13);
  5960. if (table == NULL)
  5961. return NULL;
  5962. set_table_entry(&table[0], "min_interval", &sd->min_interval,
  5963. sizeof(long), 0644, proc_doulongvec_minmax);
  5964. set_table_entry(&table[1], "max_interval", &sd->max_interval,
  5965. sizeof(long), 0644, proc_doulongvec_minmax);
  5966. set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
  5967. sizeof(int), 0644, proc_dointvec_minmax);
  5968. set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
  5969. sizeof(int), 0644, proc_dointvec_minmax);
  5970. set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
  5971. sizeof(int), 0644, proc_dointvec_minmax);
  5972. set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
  5973. sizeof(int), 0644, proc_dointvec_minmax);
  5974. set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
  5975. sizeof(int), 0644, proc_dointvec_minmax);
  5976. set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
  5977. sizeof(int), 0644, proc_dointvec_minmax);
  5978. set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
  5979. sizeof(int), 0644, proc_dointvec_minmax);
  5980. set_table_entry(&table[9], "cache_nice_tries",
  5981. &sd->cache_nice_tries,
  5982. sizeof(int), 0644, proc_dointvec_minmax);
  5983. set_table_entry(&table[10], "flags", &sd->flags,
  5984. sizeof(int), 0644, proc_dointvec_minmax);
  5985. set_table_entry(&table[11], "name", sd->name,
  5986. CORENAME_MAX_SIZE, 0444, proc_dostring);
  5987. /* &table[12] is terminator */
  5988. return table;
  5989. }
  5990. static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
  5991. {
  5992. struct ctl_table *entry, *table;
  5993. struct sched_domain *sd;
  5994. int domain_num = 0, i;
  5995. char buf[32];
  5996. for_each_domain(cpu, sd)
  5997. domain_num++;
  5998. entry = table = sd_alloc_ctl_entry(domain_num + 1);
  5999. if (table == NULL)
  6000. return NULL;
  6001. i = 0;
  6002. for_each_domain(cpu, sd) {
  6003. snprintf(buf, 32, "domain%d", i);
  6004. entry->procname = kstrdup(buf, GFP_KERNEL);
  6005. entry->mode = 0555;
  6006. entry->child = sd_alloc_ctl_domain_table(sd);
  6007. entry++;
  6008. i++;
  6009. }
  6010. return table;
  6011. }
  6012. static struct ctl_table_header *sd_sysctl_header;
  6013. static void register_sched_domain_sysctl(void)
  6014. {
  6015. int i, cpu_num = num_online_cpus();
  6016. struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
  6017. char buf[32];
  6018. WARN_ON(sd_ctl_dir[0].child);
  6019. sd_ctl_dir[0].child = entry;
  6020. if (entry == NULL)
  6021. return;
  6022. for_each_online_cpu(i) {
  6023. snprintf(buf, 32, "cpu%d", i);
  6024. entry->procname = kstrdup(buf, GFP_KERNEL);
  6025. entry->mode = 0555;
  6026. entry->child = sd_alloc_ctl_cpu_table(i);
  6027. entry++;
  6028. }
  6029. WARN_ON(sd_sysctl_header);
  6030. sd_sysctl_header = register_sysctl_table(sd_ctl_root);
  6031. }
  6032. /* may be called multiple times per register */
  6033. static void unregister_sched_domain_sysctl(void)
  6034. {
  6035. if (sd_sysctl_header)
  6036. unregister_sysctl_table(sd_sysctl_header);
  6037. sd_sysctl_header = NULL;
  6038. if (sd_ctl_dir[0].child)
  6039. sd_free_ctl_entry(&sd_ctl_dir[0].child);
  6040. }
  6041. #else
  6042. static void register_sched_domain_sysctl(void)
  6043. {
  6044. }
  6045. static void unregister_sched_domain_sysctl(void)
  6046. {
  6047. }
  6048. #endif
  6049. static void set_rq_online(struct rq *rq)
  6050. {
  6051. if (!rq->online) {
  6052. const struct sched_class *class;
  6053. cpumask_set_cpu(rq->cpu, rq->rd->online);
  6054. rq->online = 1;
  6055. for_each_class(class) {
  6056. if (class->rq_online)
  6057. class->rq_online(rq);
  6058. }
  6059. }
  6060. }
  6061. static void set_rq_offline(struct rq *rq)
  6062. {
  6063. if (rq->online) {
  6064. const struct sched_class *class;
  6065. for_each_class(class) {
  6066. if (class->rq_offline)
  6067. class->rq_offline(rq);
  6068. }
  6069. cpumask_clear_cpu(rq->cpu, rq->rd->online);
  6070. rq->online = 0;
  6071. }
  6072. }
  6073. /*
  6074. * migration_call - callback that gets triggered when a CPU is added.
  6075. * Here we can start up the necessary migration thread for the new CPU.
  6076. */
  6077. static int __cpuinit
  6078. migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
  6079. {
  6080. struct task_struct *p;
  6081. int cpu = (long)hcpu;
  6082. unsigned long flags;
  6083. struct rq *rq;
  6084. switch (action) {
  6085. case CPU_UP_PREPARE:
  6086. case CPU_UP_PREPARE_FROZEN:
  6087. p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
  6088. if (IS_ERR(p))
  6089. return NOTIFY_BAD;
  6090. kthread_bind(p, cpu);
  6091. /* Must be high prio: stop_machine expects to yield to it. */
  6092. rq = task_rq_lock(p, &flags);
  6093. __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
  6094. task_rq_unlock(rq, &flags);
  6095. cpu_rq(cpu)->migration_thread = p;
  6096. break;
  6097. case CPU_ONLINE:
  6098. case CPU_ONLINE_FROZEN:
  6099. /* Strictly unnecessary, as first user will wake it. */
  6100. wake_up_process(cpu_rq(cpu)->migration_thread);
  6101. /* Update our root-domain */
  6102. rq = cpu_rq(cpu);
  6103. spin_lock_irqsave(&rq->lock, flags);
  6104. if (rq->rd) {
  6105. BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
  6106. set_rq_online(rq);
  6107. }
  6108. spin_unlock_irqrestore(&rq->lock, flags);
  6109. break;
  6110. #ifdef CONFIG_HOTPLUG_CPU
  6111. case CPU_UP_CANCELED:
  6112. case CPU_UP_CANCELED_FROZEN:
  6113. if (!cpu_rq(cpu)->migration_thread)
  6114. break;
  6115. /* Unbind it from offline cpu so it can run. Fall thru. */
  6116. kthread_bind(cpu_rq(cpu)->migration_thread,
  6117. cpumask_any(cpu_online_mask));
  6118. kthread_stop(cpu_rq(cpu)->migration_thread);
  6119. cpu_rq(cpu)->migration_thread = NULL;
  6120. break;
  6121. case CPU_DEAD:
  6122. case CPU_DEAD_FROZEN:
  6123. cpuset_lock(); /* around calls to cpuset_cpus_allowed_lock() */
  6124. migrate_live_tasks(cpu);
  6125. rq = cpu_rq(cpu);
  6126. kthread_stop(rq->migration_thread);
  6127. rq->migration_thread = NULL;
  6128. /* Idle task back to normal (off runqueue, low prio) */
  6129. spin_lock_irq(&rq->lock);
  6130. update_rq_clock(rq);
  6131. deactivate_task(rq, rq->idle, 0);
  6132. rq->idle->static_prio = MAX_PRIO;
  6133. __setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
  6134. rq->idle->sched_class = &idle_sched_class;
  6135. migrate_dead_tasks(cpu);
  6136. spin_unlock_irq(&rq->lock);
  6137. cpuset_unlock();
  6138. migrate_nr_uninterruptible(rq);
  6139. BUG_ON(rq->nr_running != 0);
  6140. /*
  6141. * No need to migrate the tasks: it was best-effort if
  6142. * they didn't take sched_hotcpu_mutex. Just wake up
  6143. * the requestors.
  6144. */
  6145. spin_lock_irq(&rq->lock);
  6146. while (!list_empty(&rq->migration_queue)) {
  6147. struct migration_req *req;
  6148. req = list_entry(rq->migration_queue.next,
  6149. struct migration_req, list);
  6150. list_del_init(&req->list);
  6151. spin_unlock_irq(&rq->lock);
  6152. complete(&req->done);
  6153. spin_lock_irq(&rq->lock);
  6154. }
  6155. spin_unlock_irq(&rq->lock);
  6156. break;
  6157. case CPU_DYING:
  6158. case CPU_DYING_FROZEN:
  6159. /* Update our root-domain */
  6160. rq = cpu_rq(cpu);
  6161. spin_lock_irqsave(&rq->lock, flags);
  6162. if (rq->rd) {
  6163. BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
  6164. set_rq_offline(rq);
  6165. }
  6166. spin_unlock_irqrestore(&rq->lock, flags);
  6167. break;
  6168. #endif
  6169. }
  6170. return NOTIFY_OK;
  6171. }
  6172. /* Register at highest priority so that task migration (migrate_all_tasks)
  6173. * happens before everything else.
  6174. */
  6175. static struct notifier_block __cpuinitdata migration_notifier = {
  6176. .notifier_call = migration_call,
  6177. .priority = 10
  6178. };
  6179. static int __init migration_init(void)
  6180. {
  6181. void *cpu = (void *)(long)smp_processor_id();
  6182. int err;
  6183. /* Start one for the boot CPU: */
  6184. err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
  6185. BUG_ON(err == NOTIFY_BAD);
  6186. migration_call(&migration_notifier, CPU_ONLINE, cpu);
  6187. register_cpu_notifier(&migration_notifier);
  6188. return err;
  6189. }
  6190. early_initcall(migration_init);
  6191. #endif
  6192. #ifdef CONFIG_SMP
  6193. #ifdef CONFIG_SCHED_DEBUG
  6194. static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
  6195. struct cpumask *groupmask)
  6196. {
  6197. struct sched_group *group = sd->groups;
  6198. char str[256];
  6199. cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
  6200. cpumask_clear(groupmask);
  6201. printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
  6202. if (!(sd->flags & SD_LOAD_BALANCE)) {
  6203. printk("does not load-balance\n");
  6204. if (sd->parent)
  6205. printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
  6206. " has parent");
  6207. return -1;
  6208. }
  6209. printk(KERN_CONT "span %s level %s\n", str, sd->name);
  6210. if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
  6211. printk(KERN_ERR "ERROR: domain->span does not contain "
  6212. "CPU%d\n", cpu);
  6213. }
  6214. if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
  6215. printk(KERN_ERR "ERROR: domain->groups does not contain"
  6216. " CPU%d\n", cpu);
  6217. }
  6218. printk(KERN_DEBUG "%*s groups:", level + 1, "");
  6219. do {
  6220. if (!group) {
  6221. printk("\n");
  6222. printk(KERN_ERR "ERROR: group is NULL\n");
  6223. break;
  6224. }
  6225. if (!group->__cpu_power) {
  6226. printk(KERN_CONT "\n");
  6227. printk(KERN_ERR "ERROR: domain->cpu_power not "
  6228. "set\n");
  6229. break;
  6230. }
  6231. if (!cpumask_weight(sched_group_cpus(group))) {
  6232. printk(KERN_CONT "\n");
  6233. printk(KERN_ERR "ERROR: empty group\n");
  6234. break;
  6235. }
  6236. if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
  6237. printk(KERN_CONT "\n");
  6238. printk(KERN_ERR "ERROR: repeated CPUs\n");
  6239. break;
  6240. }
  6241. cpumask_or(groupmask, groupmask, sched_group_cpus(group));
  6242. cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
  6243. printk(KERN_CONT " %s", str);
  6244. group = group->next;
  6245. } while (group != sd->groups);
  6246. printk(KERN_CONT "\n");
  6247. if (!cpumask_equal(sched_domain_span(sd), groupmask))
  6248. printk(KERN_ERR "ERROR: groups don't span domain->span\n");
  6249. if (sd->parent &&
  6250. !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
  6251. printk(KERN_ERR "ERROR: parent span is not a superset "
  6252. "of domain->span\n");
  6253. return 0;
  6254. }
  6255. static void sched_domain_debug(struct sched_domain *sd, int cpu)
  6256. {
  6257. cpumask_var_t groupmask;
  6258. int level = 0;
  6259. if (!sd) {
  6260. printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
  6261. return;
  6262. }
  6263. printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
  6264. if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
  6265. printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
  6266. return;
  6267. }
  6268. for (;;) {
  6269. if (sched_domain_debug_one(sd, cpu, level, groupmask))
  6270. break;
  6271. level++;
  6272. sd = sd->parent;
  6273. if (!sd)
  6274. break;
  6275. }
  6276. free_cpumask_var(groupmask);
  6277. }
  6278. #else /* !CONFIG_SCHED_DEBUG */
  6279. # define sched_domain_debug(sd, cpu) do { } while (0)
  6280. #endif /* CONFIG_SCHED_DEBUG */
  6281. static int sd_degenerate(struct sched_domain *sd)
  6282. {
  6283. if (cpumask_weight(sched_domain_span(sd)) == 1)
  6284. return 1;
  6285. /* Following flags need at least 2 groups */
  6286. if (sd->flags & (SD_LOAD_BALANCE |
  6287. SD_BALANCE_NEWIDLE |
  6288. SD_BALANCE_FORK |
  6289. SD_BALANCE_EXEC |
  6290. SD_SHARE_CPUPOWER |
  6291. SD_SHARE_PKG_RESOURCES)) {
  6292. if (sd->groups != sd->groups->next)
  6293. return 0;
  6294. }
  6295. /* Following flags don't use groups */
  6296. if (sd->flags & (SD_WAKE_IDLE |
  6297. SD_WAKE_AFFINE |
  6298. SD_WAKE_BALANCE))
  6299. return 0;
  6300. return 1;
  6301. }
  6302. static int
  6303. sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
  6304. {
  6305. unsigned long cflags = sd->flags, pflags = parent->flags;
  6306. if (sd_degenerate(parent))
  6307. return 1;
  6308. if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
  6309. return 0;
  6310. /* Does parent contain flags not in child? */
  6311. /* WAKE_BALANCE is a subset of WAKE_AFFINE */
  6312. if (cflags & SD_WAKE_AFFINE)
  6313. pflags &= ~SD_WAKE_BALANCE;
  6314. /* Flags needing groups don't count if only 1 group in parent */
  6315. if (parent->groups == parent->groups->next) {
  6316. pflags &= ~(SD_LOAD_BALANCE |
  6317. SD_BALANCE_NEWIDLE |
  6318. SD_BALANCE_FORK |
  6319. SD_BALANCE_EXEC |
  6320. SD_SHARE_CPUPOWER |
  6321. SD_SHARE_PKG_RESOURCES);
  6322. if (nr_node_ids == 1)
  6323. pflags &= ~SD_SERIALIZE;
  6324. }
  6325. if (~cflags & pflags)
  6326. return 0;
  6327. return 1;
  6328. }
  6329. static void free_rootdomain(struct root_domain *rd)
  6330. {
  6331. cpupri_cleanup(&rd->cpupri);
  6332. free_cpumask_var(rd->rto_mask);
  6333. free_cpumask_var(rd->online);
  6334. free_cpumask_var(rd->span);
  6335. kfree(rd);
  6336. }
  6337. static void rq_attach_root(struct rq *rq, struct root_domain *rd)
  6338. {
  6339. struct root_domain *old_rd = NULL;
  6340. unsigned long flags;
  6341. spin_lock_irqsave(&rq->lock, flags);
  6342. if (rq->rd) {
  6343. old_rd = rq->rd;
  6344. if (cpumask_test_cpu(rq->cpu, old_rd->online))
  6345. set_rq_offline(rq);
  6346. cpumask_clear_cpu(rq->cpu, old_rd->span);
  6347. /*
  6348. * If we dont want to free the old_rt yet then
  6349. * set old_rd to NULL to skip the freeing later
  6350. * in this function:
  6351. */
  6352. if (!atomic_dec_and_test(&old_rd->refcount))
  6353. old_rd = NULL;
  6354. }
  6355. atomic_inc(&rd->refcount);
  6356. rq->rd = rd;
  6357. cpumask_set_cpu(rq->cpu, rd->span);
  6358. if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
  6359. set_rq_online(rq);
  6360. spin_unlock_irqrestore(&rq->lock, flags);
  6361. if (old_rd)
  6362. free_rootdomain(old_rd);
  6363. }
  6364. static int __init_refok init_rootdomain(struct root_domain *rd, bool bootmem)
  6365. {
  6366. memset(rd, 0, sizeof(*rd));
  6367. if (bootmem) {
  6368. alloc_bootmem_cpumask_var(&def_root_domain.span);
  6369. alloc_bootmem_cpumask_var(&def_root_domain.online);
  6370. alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
  6371. cpupri_init(&rd->cpupri, true);
  6372. return 0;
  6373. }
  6374. if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
  6375. goto out;
  6376. if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
  6377. goto free_span;
  6378. if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
  6379. goto free_online;
  6380. if (cpupri_init(&rd->cpupri, false) != 0)
  6381. goto free_rto_mask;
  6382. return 0;
  6383. free_rto_mask:
  6384. free_cpumask_var(rd->rto_mask);
  6385. free_online:
  6386. free_cpumask_var(rd->online);
  6387. free_span:
  6388. free_cpumask_var(rd->span);
  6389. out:
  6390. return -ENOMEM;
  6391. }
  6392. static void init_defrootdomain(void)
  6393. {
  6394. init_rootdomain(&def_root_domain, true);
  6395. atomic_set(&def_root_domain.refcount, 1);
  6396. }
  6397. static struct root_domain *alloc_rootdomain(void)
  6398. {
  6399. struct root_domain *rd;
  6400. rd = kmalloc(sizeof(*rd), GFP_KERNEL);
  6401. if (!rd)
  6402. return NULL;
  6403. if (init_rootdomain(rd, false) != 0) {
  6404. kfree(rd);
  6405. return NULL;
  6406. }
  6407. return rd;
  6408. }
  6409. /*
  6410. * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
  6411. * hold the hotplug lock.
  6412. */
  6413. static void
  6414. cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
  6415. {
  6416. struct rq *rq = cpu_rq(cpu);
  6417. struct sched_domain *tmp;
  6418. /* Remove the sched domains which do not contribute to scheduling. */
  6419. for (tmp = sd; tmp; ) {
  6420. struct sched_domain *parent = tmp->parent;
  6421. if (!parent)
  6422. break;
  6423. if (sd_parent_degenerate(tmp, parent)) {
  6424. tmp->parent = parent->parent;
  6425. if (parent->parent)
  6426. parent->parent->child = tmp;
  6427. } else
  6428. tmp = tmp->parent;
  6429. }
  6430. if (sd && sd_degenerate(sd)) {
  6431. sd = sd->parent;
  6432. if (sd)
  6433. sd->child = NULL;
  6434. }
  6435. sched_domain_debug(sd, cpu);
  6436. rq_attach_root(rq, rd);
  6437. rcu_assign_pointer(rq->sd, sd);
  6438. }
  6439. /* cpus with isolated domains */
  6440. static cpumask_var_t cpu_isolated_map;
  6441. /* Setup the mask of cpus configured for isolated domains */
  6442. static int __init isolated_cpu_setup(char *str)
  6443. {
  6444. cpulist_parse(str, cpu_isolated_map);
  6445. return 1;
  6446. }
  6447. __setup("isolcpus=", isolated_cpu_setup);
  6448. /*
  6449. * init_sched_build_groups takes the cpumask we wish to span, and a pointer
  6450. * to a function which identifies what group(along with sched group) a CPU
  6451. * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
  6452. * (due to the fact that we keep track of groups covered with a struct cpumask).
  6453. *
  6454. * init_sched_build_groups will build a circular linked list of the groups
  6455. * covered by the given span, and will set each group's ->cpumask correctly,
  6456. * and ->cpu_power to 0.
  6457. */
  6458. static void
  6459. init_sched_build_groups(const struct cpumask *span,
  6460. const struct cpumask *cpu_map,
  6461. int (*group_fn)(int cpu, const struct cpumask *cpu_map,
  6462. struct sched_group **sg,
  6463. struct cpumask *tmpmask),
  6464. struct cpumask *covered, struct cpumask *tmpmask)
  6465. {
  6466. struct sched_group *first = NULL, *last = NULL;
  6467. int i;
  6468. cpumask_clear(covered);
  6469. for_each_cpu(i, span) {
  6470. struct sched_group *sg;
  6471. int group = group_fn(i, cpu_map, &sg, tmpmask);
  6472. int j;
  6473. if (cpumask_test_cpu(i, covered))
  6474. continue;
  6475. cpumask_clear(sched_group_cpus(sg));
  6476. sg->__cpu_power = 0;
  6477. for_each_cpu(j, span) {
  6478. if (group_fn(j, cpu_map, NULL, tmpmask) != group)
  6479. continue;
  6480. cpumask_set_cpu(j, covered);
  6481. cpumask_set_cpu(j, sched_group_cpus(sg));
  6482. }
  6483. if (!first)
  6484. first = sg;
  6485. if (last)
  6486. last->next = sg;
  6487. last = sg;
  6488. }
  6489. last->next = first;
  6490. }
  6491. #define SD_NODES_PER_DOMAIN 16
  6492. #ifdef CONFIG_NUMA
  6493. /**
  6494. * find_next_best_node - find the next node to include in a sched_domain
  6495. * @node: node whose sched_domain we're building
  6496. * @used_nodes: nodes already in the sched_domain
  6497. *
  6498. * Find the next node to include in a given scheduling domain. Simply
  6499. * finds the closest node not already in the @used_nodes map.
  6500. *
  6501. * Should use nodemask_t.
  6502. */
  6503. static int find_next_best_node(int node, nodemask_t *used_nodes)
  6504. {
  6505. int i, n, val, min_val, best_node = 0;
  6506. min_val = INT_MAX;
  6507. for (i = 0; i < nr_node_ids; i++) {
  6508. /* Start at @node */
  6509. n = (node + i) % nr_node_ids;
  6510. if (!nr_cpus_node(n))
  6511. continue;
  6512. /* Skip already used nodes */
  6513. if (node_isset(n, *used_nodes))
  6514. continue;
  6515. /* Simple min distance search */
  6516. val = node_distance(node, n);
  6517. if (val < min_val) {
  6518. min_val = val;
  6519. best_node = n;
  6520. }
  6521. }
  6522. node_set(best_node, *used_nodes);
  6523. return best_node;
  6524. }
  6525. /**
  6526. * sched_domain_node_span - get a cpumask for a node's sched_domain
  6527. * @node: node whose cpumask we're constructing
  6528. * @span: resulting cpumask
  6529. *
  6530. * Given a node, construct a good cpumask for its sched_domain to span. It
  6531. * should be one that prevents unnecessary balancing, but also spreads tasks
  6532. * out optimally.
  6533. */
  6534. static void sched_domain_node_span(int node, struct cpumask *span)
  6535. {
  6536. nodemask_t used_nodes;
  6537. int i;
  6538. cpumask_clear(span);
  6539. nodes_clear(used_nodes);
  6540. cpumask_or(span, span, cpumask_of_node(node));
  6541. node_set(node, used_nodes);
  6542. for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
  6543. int next_node = find_next_best_node(node, &used_nodes);
  6544. cpumask_or(span, span, cpumask_of_node(next_node));
  6545. }
  6546. }
  6547. #endif /* CONFIG_NUMA */
  6548. int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
  6549. /*
  6550. * The cpus mask in sched_group and sched_domain hangs off the end.
  6551. * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
  6552. * for nr_cpu_ids < CONFIG_NR_CPUS.
  6553. */
  6554. struct static_sched_group {
  6555. struct sched_group sg;
  6556. DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
  6557. };
  6558. struct static_sched_domain {
  6559. struct sched_domain sd;
  6560. DECLARE_BITMAP(span, CONFIG_NR_CPUS);
  6561. };
  6562. /*
  6563. * SMT sched-domains:
  6564. */
  6565. #ifdef CONFIG_SCHED_SMT
  6566. static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
  6567. static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
  6568. static int
  6569. cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
  6570. struct sched_group **sg, struct cpumask *unused)
  6571. {
  6572. if (sg)
  6573. *sg = &per_cpu(sched_group_cpus, cpu).sg;
  6574. return cpu;
  6575. }
  6576. #endif /* CONFIG_SCHED_SMT */
  6577. /*
  6578. * multi-core sched-domains:
  6579. */
  6580. #ifdef CONFIG_SCHED_MC
  6581. static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
  6582. static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
  6583. #endif /* CONFIG_SCHED_MC */
  6584. #if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
  6585. static int
  6586. cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
  6587. struct sched_group **sg, struct cpumask *mask)
  6588. {
  6589. int group;
  6590. cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
  6591. group = cpumask_first(mask);
  6592. if (sg)
  6593. *sg = &per_cpu(sched_group_core, group).sg;
  6594. return group;
  6595. }
  6596. #elif defined(CONFIG_SCHED_MC)
  6597. static int
  6598. cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
  6599. struct sched_group **sg, struct cpumask *unused)
  6600. {
  6601. if (sg)
  6602. *sg = &per_cpu(sched_group_core, cpu).sg;
  6603. return cpu;
  6604. }
  6605. #endif
  6606. static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
  6607. static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
  6608. static int
  6609. cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
  6610. struct sched_group **sg, struct cpumask *mask)
  6611. {
  6612. int group;
  6613. #ifdef CONFIG_SCHED_MC
  6614. cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
  6615. group = cpumask_first(mask);
  6616. #elif defined(CONFIG_SCHED_SMT)
  6617. cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
  6618. group = cpumask_first(mask);
  6619. #else
  6620. group = cpu;
  6621. #endif
  6622. if (sg)
  6623. *sg = &per_cpu(sched_group_phys, group).sg;
  6624. return group;
  6625. }
  6626. #ifdef CONFIG_NUMA
  6627. /*
  6628. * The init_sched_build_groups can't handle what we want to do with node
  6629. * groups, so roll our own. Now each node has its own list of groups which
  6630. * gets dynamically allocated.
  6631. */
  6632. static DEFINE_PER_CPU(struct static_sched_domain, node_domains);
  6633. static struct sched_group ***sched_group_nodes_bycpu;
  6634. static DEFINE_PER_CPU(struct static_sched_domain, allnodes_domains);
  6635. static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
  6636. static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
  6637. struct sched_group **sg,
  6638. struct cpumask *nodemask)
  6639. {
  6640. int group;
  6641. cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
  6642. group = cpumask_first(nodemask);
  6643. if (sg)
  6644. *sg = &per_cpu(sched_group_allnodes, group).sg;
  6645. return group;
  6646. }
  6647. static void init_numa_sched_groups_power(struct sched_group *group_head)
  6648. {
  6649. struct sched_group *sg = group_head;
  6650. int j;
  6651. if (!sg)
  6652. return;
  6653. do {
  6654. for_each_cpu(j, sched_group_cpus(sg)) {
  6655. struct sched_domain *sd;
  6656. sd = &per_cpu(phys_domains, j).sd;
  6657. if (j != cpumask_first(sched_group_cpus(sd->groups))) {
  6658. /*
  6659. * Only add "power" once for each
  6660. * physical package.
  6661. */
  6662. continue;
  6663. }
  6664. sg_inc_cpu_power(sg, sd->groups->__cpu_power);
  6665. }
  6666. sg = sg->next;
  6667. } while (sg != group_head);
  6668. }
  6669. #endif /* CONFIG_NUMA */
  6670. #ifdef CONFIG_NUMA
  6671. /* Free memory allocated for various sched_group structures */
  6672. static void free_sched_groups(const struct cpumask *cpu_map,
  6673. struct cpumask *nodemask)
  6674. {
  6675. int cpu, i;
  6676. for_each_cpu(cpu, cpu_map) {
  6677. struct sched_group **sched_group_nodes
  6678. = sched_group_nodes_bycpu[cpu];
  6679. if (!sched_group_nodes)
  6680. continue;
  6681. for (i = 0; i < nr_node_ids; i++) {
  6682. struct sched_group *oldsg, *sg = sched_group_nodes[i];
  6683. cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
  6684. if (cpumask_empty(nodemask))
  6685. continue;
  6686. if (sg == NULL)
  6687. continue;
  6688. sg = sg->next;
  6689. next_sg:
  6690. oldsg = sg;
  6691. sg = sg->next;
  6692. kfree(oldsg);
  6693. if (oldsg != sched_group_nodes[i])
  6694. goto next_sg;
  6695. }
  6696. kfree(sched_group_nodes);
  6697. sched_group_nodes_bycpu[cpu] = NULL;
  6698. }
  6699. }
  6700. #else /* !CONFIG_NUMA */
  6701. static void free_sched_groups(const struct cpumask *cpu_map,
  6702. struct cpumask *nodemask)
  6703. {
  6704. }
  6705. #endif /* CONFIG_NUMA */
  6706. /*
  6707. * Initialize sched groups cpu_power.
  6708. *
  6709. * cpu_power indicates the capacity of sched group, which is used while
  6710. * distributing the load between different sched groups in a sched domain.
  6711. * Typically cpu_power for all the groups in a sched domain will be same unless
  6712. * there are asymmetries in the topology. If there are asymmetries, group
  6713. * having more cpu_power will pickup more load compared to the group having
  6714. * less cpu_power.
  6715. *
  6716. * cpu_power will be a multiple of SCHED_LOAD_SCALE. This multiple represents
  6717. * the maximum number of tasks a group can handle in the presence of other idle
  6718. * or lightly loaded groups in the same sched domain.
  6719. */
  6720. static void init_sched_groups_power(int cpu, struct sched_domain *sd)
  6721. {
  6722. struct sched_domain *child;
  6723. struct sched_group *group;
  6724. WARN_ON(!sd || !sd->groups);
  6725. if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
  6726. return;
  6727. child = sd->child;
  6728. sd->groups->__cpu_power = 0;
  6729. /*
  6730. * For perf policy, if the groups in child domain share resources
  6731. * (for example cores sharing some portions of the cache hierarchy
  6732. * or SMT), then set this domain groups cpu_power such that each group
  6733. * can handle only one task, when there are other idle groups in the
  6734. * same sched domain.
  6735. */
  6736. if (!child || (!(sd->flags & SD_POWERSAVINGS_BALANCE) &&
  6737. (child->flags &
  6738. (SD_SHARE_CPUPOWER | SD_SHARE_PKG_RESOURCES)))) {
  6739. sg_inc_cpu_power(sd->groups, SCHED_LOAD_SCALE);
  6740. return;
  6741. }
  6742. /*
  6743. * add cpu_power of each child group to this groups cpu_power
  6744. */
  6745. group = child->groups;
  6746. do {
  6747. sg_inc_cpu_power(sd->groups, group->__cpu_power);
  6748. group = group->next;
  6749. } while (group != child->groups);
  6750. }
  6751. /*
  6752. * Initializers for schedule domains
  6753. * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
  6754. */
  6755. #ifdef CONFIG_SCHED_DEBUG
  6756. # define SD_INIT_NAME(sd, type) sd->name = #type
  6757. #else
  6758. # define SD_INIT_NAME(sd, type) do { } while (0)
  6759. #endif
  6760. #define SD_INIT(sd, type) sd_init_##type(sd)
  6761. #define SD_INIT_FUNC(type) \
  6762. static noinline void sd_init_##type(struct sched_domain *sd) \
  6763. { \
  6764. memset(sd, 0, sizeof(*sd)); \
  6765. *sd = SD_##type##_INIT; \
  6766. sd->level = SD_LV_##type; \
  6767. SD_INIT_NAME(sd, type); \
  6768. }
  6769. SD_INIT_FUNC(CPU)
  6770. #ifdef CONFIG_NUMA
  6771. SD_INIT_FUNC(ALLNODES)
  6772. SD_INIT_FUNC(NODE)
  6773. #endif
  6774. #ifdef CONFIG_SCHED_SMT
  6775. SD_INIT_FUNC(SIBLING)
  6776. #endif
  6777. #ifdef CONFIG_SCHED_MC
  6778. SD_INIT_FUNC(MC)
  6779. #endif
  6780. static int default_relax_domain_level = -1;
  6781. static int __init setup_relax_domain_level(char *str)
  6782. {
  6783. unsigned long val;
  6784. val = simple_strtoul(str, NULL, 0);
  6785. if (val < SD_LV_MAX)
  6786. default_relax_domain_level = val;
  6787. return 1;
  6788. }
  6789. __setup("relax_domain_level=", setup_relax_domain_level);
  6790. static void set_domain_attribute(struct sched_domain *sd,
  6791. struct sched_domain_attr *attr)
  6792. {
  6793. int request;
  6794. if (!attr || attr->relax_domain_level < 0) {
  6795. if (default_relax_domain_level < 0)
  6796. return;
  6797. else
  6798. request = default_relax_domain_level;
  6799. } else
  6800. request = attr->relax_domain_level;
  6801. if (request < sd->level) {
  6802. /* turn off idle balance on this domain */
  6803. sd->flags &= ~(SD_WAKE_IDLE|SD_BALANCE_NEWIDLE);
  6804. } else {
  6805. /* turn on idle balance on this domain */
  6806. sd->flags |= (SD_WAKE_IDLE_FAR|SD_BALANCE_NEWIDLE);
  6807. }
  6808. }
  6809. /*
  6810. * Build sched domains for a given set of cpus and attach the sched domains
  6811. * to the individual cpus
  6812. */
  6813. static int __build_sched_domains(const struct cpumask *cpu_map,
  6814. struct sched_domain_attr *attr)
  6815. {
  6816. int i, err = -ENOMEM;
  6817. struct root_domain *rd;
  6818. cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
  6819. tmpmask;
  6820. #ifdef CONFIG_NUMA
  6821. cpumask_var_t domainspan, covered, notcovered;
  6822. struct sched_group **sched_group_nodes = NULL;
  6823. int sd_allnodes = 0;
  6824. if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
  6825. goto out;
  6826. if (!alloc_cpumask_var(&covered, GFP_KERNEL))
  6827. goto free_domainspan;
  6828. if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
  6829. goto free_covered;
  6830. #endif
  6831. if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
  6832. goto free_notcovered;
  6833. if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
  6834. goto free_nodemask;
  6835. if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
  6836. goto free_this_sibling_map;
  6837. if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
  6838. goto free_this_core_map;
  6839. if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
  6840. goto free_send_covered;
  6841. #ifdef CONFIG_NUMA
  6842. /*
  6843. * Allocate the per-node list of sched groups
  6844. */
  6845. sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *),
  6846. GFP_KERNEL);
  6847. if (!sched_group_nodes) {
  6848. printk(KERN_WARNING "Can not alloc sched group node list\n");
  6849. goto free_tmpmask;
  6850. }
  6851. #endif
  6852. rd = alloc_rootdomain();
  6853. if (!rd) {
  6854. printk(KERN_WARNING "Cannot alloc root domain\n");
  6855. goto free_sched_groups;
  6856. }
  6857. #ifdef CONFIG_NUMA
  6858. sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
  6859. #endif
  6860. /*
  6861. * Set up domains for cpus specified by the cpu_map.
  6862. */
  6863. for_each_cpu(i, cpu_map) {
  6864. struct sched_domain *sd = NULL, *p;
  6865. cpumask_and(nodemask, cpumask_of_node(cpu_to_node(i)), cpu_map);
  6866. #ifdef CONFIG_NUMA
  6867. if (cpumask_weight(cpu_map) >
  6868. SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
  6869. sd = &per_cpu(allnodes_domains, i).sd;
  6870. SD_INIT(sd, ALLNODES);
  6871. set_domain_attribute(sd, attr);
  6872. cpumask_copy(sched_domain_span(sd), cpu_map);
  6873. cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
  6874. p = sd;
  6875. sd_allnodes = 1;
  6876. } else
  6877. p = NULL;
  6878. sd = &per_cpu(node_domains, i).sd;
  6879. SD_INIT(sd, NODE);
  6880. set_domain_attribute(sd, attr);
  6881. sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
  6882. sd->parent = p;
  6883. if (p)
  6884. p->child = sd;
  6885. cpumask_and(sched_domain_span(sd),
  6886. sched_domain_span(sd), cpu_map);
  6887. #endif
  6888. p = sd;
  6889. sd = &per_cpu(phys_domains, i).sd;
  6890. SD_INIT(sd, CPU);
  6891. set_domain_attribute(sd, attr);
  6892. cpumask_copy(sched_domain_span(sd), nodemask);
  6893. sd->parent = p;
  6894. if (p)
  6895. p->child = sd;
  6896. cpu_to_phys_group(i, cpu_map, &sd->groups, tmpmask);
  6897. #ifdef CONFIG_SCHED_MC
  6898. p = sd;
  6899. sd = &per_cpu(core_domains, i).sd;
  6900. SD_INIT(sd, MC);
  6901. set_domain_attribute(sd, attr);
  6902. cpumask_and(sched_domain_span(sd), cpu_map,
  6903. cpu_coregroup_mask(i));
  6904. sd->parent = p;
  6905. p->child = sd;
  6906. cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
  6907. #endif
  6908. #ifdef CONFIG_SCHED_SMT
  6909. p = sd;
  6910. sd = &per_cpu(cpu_domains, i).sd;
  6911. SD_INIT(sd, SIBLING);
  6912. set_domain_attribute(sd, attr);
  6913. cpumask_and(sched_domain_span(sd),
  6914. topology_thread_cpumask(i), cpu_map);
  6915. sd->parent = p;
  6916. p->child = sd;
  6917. cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
  6918. #endif
  6919. }
  6920. #ifdef CONFIG_SCHED_SMT
  6921. /* Set up CPU (sibling) groups */
  6922. for_each_cpu(i, cpu_map) {
  6923. cpumask_and(this_sibling_map,
  6924. topology_thread_cpumask(i), cpu_map);
  6925. if (i != cpumask_first(this_sibling_map))
  6926. continue;
  6927. init_sched_build_groups(this_sibling_map, cpu_map,
  6928. &cpu_to_cpu_group,
  6929. send_covered, tmpmask);
  6930. }
  6931. #endif
  6932. #ifdef CONFIG_SCHED_MC
  6933. /* Set up multi-core groups */
  6934. for_each_cpu(i, cpu_map) {
  6935. cpumask_and(this_core_map, cpu_coregroup_mask(i), cpu_map);
  6936. if (i != cpumask_first(this_core_map))
  6937. continue;
  6938. init_sched_build_groups(this_core_map, cpu_map,
  6939. &cpu_to_core_group,
  6940. send_covered, tmpmask);
  6941. }
  6942. #endif
  6943. /* Set up physical groups */
  6944. for (i = 0; i < nr_node_ids; i++) {
  6945. cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
  6946. if (cpumask_empty(nodemask))
  6947. continue;
  6948. init_sched_build_groups(nodemask, cpu_map,
  6949. &cpu_to_phys_group,
  6950. send_covered, tmpmask);
  6951. }
  6952. #ifdef CONFIG_NUMA
  6953. /* Set up node groups */
  6954. if (sd_allnodes) {
  6955. init_sched_build_groups(cpu_map, cpu_map,
  6956. &cpu_to_allnodes_group,
  6957. send_covered, tmpmask);
  6958. }
  6959. for (i = 0; i < nr_node_ids; i++) {
  6960. /* Set up node groups */
  6961. struct sched_group *sg, *prev;
  6962. int j;
  6963. cpumask_clear(covered);
  6964. cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
  6965. if (cpumask_empty(nodemask)) {
  6966. sched_group_nodes[i] = NULL;
  6967. continue;
  6968. }
  6969. sched_domain_node_span(i, domainspan);
  6970. cpumask_and(domainspan, domainspan, cpu_map);
  6971. sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
  6972. GFP_KERNEL, i);
  6973. if (!sg) {
  6974. printk(KERN_WARNING "Can not alloc domain group for "
  6975. "node %d\n", i);
  6976. goto error;
  6977. }
  6978. sched_group_nodes[i] = sg;
  6979. for_each_cpu(j, nodemask) {
  6980. struct sched_domain *sd;
  6981. sd = &per_cpu(node_domains, j).sd;
  6982. sd->groups = sg;
  6983. }
  6984. sg->__cpu_power = 0;
  6985. cpumask_copy(sched_group_cpus(sg), nodemask);
  6986. sg->next = sg;
  6987. cpumask_or(covered, covered, nodemask);
  6988. prev = sg;
  6989. for (j = 0; j < nr_node_ids; j++) {
  6990. int n = (i + j) % nr_node_ids;
  6991. cpumask_complement(notcovered, covered);
  6992. cpumask_and(tmpmask, notcovered, cpu_map);
  6993. cpumask_and(tmpmask, tmpmask, domainspan);
  6994. if (cpumask_empty(tmpmask))
  6995. break;
  6996. cpumask_and(tmpmask, tmpmask, cpumask_of_node(n));
  6997. if (cpumask_empty(tmpmask))
  6998. continue;
  6999. sg = kmalloc_node(sizeof(struct sched_group) +
  7000. cpumask_size(),
  7001. GFP_KERNEL, i);
  7002. if (!sg) {
  7003. printk(KERN_WARNING
  7004. "Can not alloc domain group for node %d\n", j);
  7005. goto error;
  7006. }
  7007. sg->__cpu_power = 0;
  7008. cpumask_copy(sched_group_cpus(sg), tmpmask);
  7009. sg->next = prev->next;
  7010. cpumask_or(covered, covered, tmpmask);
  7011. prev->next = sg;
  7012. prev = sg;
  7013. }
  7014. }
  7015. #endif
  7016. /* Calculate CPU power for physical packages and nodes */
  7017. #ifdef CONFIG_SCHED_SMT
  7018. for_each_cpu(i, cpu_map) {
  7019. struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
  7020. init_sched_groups_power(i, sd);
  7021. }
  7022. #endif
  7023. #ifdef CONFIG_SCHED_MC
  7024. for_each_cpu(i, cpu_map) {
  7025. struct sched_domain *sd = &per_cpu(core_domains, i).sd;
  7026. init_sched_groups_power(i, sd);
  7027. }
  7028. #endif
  7029. for_each_cpu(i, cpu_map) {
  7030. struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
  7031. init_sched_groups_power(i, sd);
  7032. }
  7033. #ifdef CONFIG_NUMA
  7034. for (i = 0; i < nr_node_ids; i++)
  7035. init_numa_sched_groups_power(sched_group_nodes[i]);
  7036. if (sd_allnodes) {
  7037. struct sched_group *sg;
  7038. cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
  7039. tmpmask);
  7040. init_numa_sched_groups_power(sg);
  7041. }
  7042. #endif
  7043. /* Attach the domains */
  7044. for_each_cpu(i, cpu_map) {
  7045. struct sched_domain *sd;
  7046. #ifdef CONFIG_SCHED_SMT
  7047. sd = &per_cpu(cpu_domains, i).sd;
  7048. #elif defined(CONFIG_SCHED_MC)
  7049. sd = &per_cpu(core_domains, i).sd;
  7050. #else
  7051. sd = &per_cpu(phys_domains, i).sd;
  7052. #endif
  7053. cpu_attach_domain(sd, rd, i);
  7054. }
  7055. err = 0;
  7056. free_tmpmask:
  7057. free_cpumask_var(tmpmask);
  7058. free_send_covered:
  7059. free_cpumask_var(send_covered);
  7060. free_this_core_map:
  7061. free_cpumask_var(this_core_map);
  7062. free_this_sibling_map:
  7063. free_cpumask_var(this_sibling_map);
  7064. free_nodemask:
  7065. free_cpumask_var(nodemask);
  7066. free_notcovered:
  7067. #ifdef CONFIG_NUMA
  7068. free_cpumask_var(notcovered);
  7069. free_covered:
  7070. free_cpumask_var(covered);
  7071. free_domainspan:
  7072. free_cpumask_var(domainspan);
  7073. out:
  7074. #endif
  7075. return err;
  7076. free_sched_groups:
  7077. #ifdef CONFIG_NUMA
  7078. kfree(sched_group_nodes);
  7079. #endif
  7080. goto free_tmpmask;
  7081. #ifdef CONFIG_NUMA
  7082. error:
  7083. free_sched_groups(cpu_map, tmpmask);
  7084. free_rootdomain(rd);
  7085. goto free_tmpmask;
  7086. #endif
  7087. }
  7088. static int build_sched_domains(const struct cpumask *cpu_map)
  7089. {
  7090. return __build_sched_domains(cpu_map, NULL);
  7091. }
  7092. static struct cpumask *doms_cur; /* current sched domains */
  7093. static int ndoms_cur; /* number of sched domains in 'doms_cur' */
  7094. static struct sched_domain_attr *dattr_cur;
  7095. /* attribues of custom domains in 'doms_cur' */
  7096. /*
  7097. * Special case: If a kmalloc of a doms_cur partition (array of
  7098. * cpumask) fails, then fallback to a single sched domain,
  7099. * as determined by the single cpumask fallback_doms.
  7100. */
  7101. static cpumask_var_t fallback_doms;
  7102. /*
  7103. * arch_update_cpu_topology lets virtualized architectures update the
  7104. * cpu core maps. It is supposed to return 1 if the topology changed
  7105. * or 0 if it stayed the same.
  7106. */
  7107. int __attribute__((weak)) arch_update_cpu_topology(void)
  7108. {
  7109. return 0;
  7110. }
  7111. /*
  7112. * Set up scheduler domains and groups. Callers must hold the hotplug lock.
  7113. * For now this just excludes isolated cpus, but could be used to
  7114. * exclude other special cases in the future.
  7115. */
  7116. static int arch_init_sched_domains(const struct cpumask *cpu_map)
  7117. {
  7118. int err;
  7119. arch_update_cpu_topology();
  7120. ndoms_cur = 1;
  7121. doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
  7122. if (!doms_cur)
  7123. doms_cur = fallback_doms;
  7124. cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
  7125. dattr_cur = NULL;
  7126. err = build_sched_domains(doms_cur);
  7127. register_sched_domain_sysctl();
  7128. return err;
  7129. }
  7130. static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
  7131. struct cpumask *tmpmask)
  7132. {
  7133. free_sched_groups(cpu_map, tmpmask);
  7134. }
  7135. /*
  7136. * Detach sched domains from a group of cpus specified in cpu_map
  7137. * These cpus will now be attached to the NULL domain
  7138. */
  7139. static void detach_destroy_domains(const struct cpumask *cpu_map)
  7140. {
  7141. /* Save because hotplug lock held. */
  7142. static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
  7143. int i;
  7144. for_each_cpu(i, cpu_map)
  7145. cpu_attach_domain(NULL, &def_root_domain, i);
  7146. synchronize_sched();
  7147. arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
  7148. }
  7149. /* handle null as "default" */
  7150. static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
  7151. struct sched_domain_attr *new, int idx_new)
  7152. {
  7153. struct sched_domain_attr tmp;
  7154. /* fast path */
  7155. if (!new && !cur)
  7156. return 1;
  7157. tmp = SD_ATTR_INIT;
  7158. return !memcmp(cur ? (cur + idx_cur) : &tmp,
  7159. new ? (new + idx_new) : &tmp,
  7160. sizeof(struct sched_domain_attr));
  7161. }
  7162. /*
  7163. * Partition sched domains as specified by the 'ndoms_new'
  7164. * cpumasks in the array doms_new[] of cpumasks. This compares
  7165. * doms_new[] to the current sched domain partitioning, doms_cur[].
  7166. * It destroys each deleted domain and builds each new domain.
  7167. *
  7168. * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
  7169. * The masks don't intersect (don't overlap.) We should setup one
  7170. * sched domain for each mask. CPUs not in any of the cpumasks will
  7171. * not be load balanced. If the same cpumask appears both in the
  7172. * current 'doms_cur' domains and in the new 'doms_new', we can leave
  7173. * it as it is.
  7174. *
  7175. * The passed in 'doms_new' should be kmalloc'd. This routine takes
  7176. * ownership of it and will kfree it when done with it. If the caller
  7177. * failed the kmalloc call, then it can pass in doms_new == NULL &&
  7178. * ndoms_new == 1, and partition_sched_domains() will fallback to
  7179. * the single partition 'fallback_doms', it also forces the domains
  7180. * to be rebuilt.
  7181. *
  7182. * If doms_new == NULL it will be replaced with cpu_online_mask.
  7183. * ndoms_new == 0 is a special case for destroying existing domains,
  7184. * and it will not create the default domain.
  7185. *
  7186. * Call with hotplug lock held
  7187. */
  7188. /* FIXME: Change to struct cpumask *doms_new[] */
  7189. void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
  7190. struct sched_domain_attr *dattr_new)
  7191. {
  7192. int i, j, n;
  7193. int new_topology;
  7194. mutex_lock(&sched_domains_mutex);
  7195. /* always unregister in case we don't destroy any domains */
  7196. unregister_sched_domain_sysctl();
  7197. /* Let architecture update cpu core mappings. */
  7198. new_topology = arch_update_cpu_topology();
  7199. n = doms_new ? ndoms_new : 0;
  7200. /* Destroy deleted domains */
  7201. for (i = 0; i < ndoms_cur; i++) {
  7202. for (j = 0; j < n && !new_topology; j++) {
  7203. if (cpumask_equal(&doms_cur[i], &doms_new[j])
  7204. && dattrs_equal(dattr_cur, i, dattr_new, j))
  7205. goto match1;
  7206. }
  7207. /* no match - a current sched domain not in new doms_new[] */
  7208. detach_destroy_domains(doms_cur + i);
  7209. match1:
  7210. ;
  7211. }
  7212. if (doms_new == NULL) {
  7213. ndoms_cur = 0;
  7214. doms_new = fallback_doms;
  7215. cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
  7216. WARN_ON_ONCE(dattr_new);
  7217. }
  7218. /* Build new domains */
  7219. for (i = 0; i < ndoms_new; i++) {
  7220. for (j = 0; j < ndoms_cur && !new_topology; j++) {
  7221. if (cpumask_equal(&doms_new[i], &doms_cur[j])
  7222. && dattrs_equal(dattr_new, i, dattr_cur, j))
  7223. goto match2;
  7224. }
  7225. /* no match - add a new doms_new */
  7226. __build_sched_domains(doms_new + i,
  7227. dattr_new ? dattr_new + i : NULL);
  7228. match2:
  7229. ;
  7230. }
  7231. /* Remember the new sched domains */
  7232. if (doms_cur != fallback_doms)
  7233. kfree(doms_cur);
  7234. kfree(dattr_cur); /* kfree(NULL) is safe */
  7235. doms_cur = doms_new;
  7236. dattr_cur = dattr_new;
  7237. ndoms_cur = ndoms_new;
  7238. register_sched_domain_sysctl();
  7239. mutex_unlock(&sched_domains_mutex);
  7240. }
  7241. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  7242. static void arch_reinit_sched_domains(void)
  7243. {
  7244. get_online_cpus();
  7245. /* Destroy domains first to force the rebuild */
  7246. partition_sched_domains(0, NULL, NULL);
  7247. rebuild_sched_domains();
  7248. put_online_cpus();
  7249. }
  7250. static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
  7251. {
  7252. unsigned int level = 0;
  7253. if (sscanf(buf, "%u", &level) != 1)
  7254. return -EINVAL;
  7255. /*
  7256. * level is always be positive so don't check for
  7257. * level < POWERSAVINGS_BALANCE_NONE which is 0
  7258. * What happens on 0 or 1 byte write,
  7259. * need to check for count as well?
  7260. */
  7261. if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
  7262. return -EINVAL;
  7263. if (smt)
  7264. sched_smt_power_savings = level;
  7265. else
  7266. sched_mc_power_savings = level;
  7267. arch_reinit_sched_domains();
  7268. return count;
  7269. }
  7270. #ifdef CONFIG_SCHED_MC
  7271. static ssize_t sched_mc_power_savings_show(struct sysdev_class *class,
  7272. char *page)
  7273. {
  7274. return sprintf(page, "%u\n", sched_mc_power_savings);
  7275. }
  7276. static ssize_t sched_mc_power_savings_store(struct sysdev_class *class,
  7277. const char *buf, size_t count)
  7278. {
  7279. return sched_power_savings_store(buf, count, 0);
  7280. }
  7281. static SYSDEV_CLASS_ATTR(sched_mc_power_savings, 0644,
  7282. sched_mc_power_savings_show,
  7283. sched_mc_power_savings_store);
  7284. #endif
  7285. #ifdef CONFIG_SCHED_SMT
  7286. static ssize_t sched_smt_power_savings_show(struct sysdev_class *dev,
  7287. char *page)
  7288. {
  7289. return sprintf(page, "%u\n", sched_smt_power_savings);
  7290. }
  7291. static ssize_t sched_smt_power_savings_store(struct sysdev_class *dev,
  7292. const char *buf, size_t count)
  7293. {
  7294. return sched_power_savings_store(buf, count, 1);
  7295. }
  7296. static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
  7297. sched_smt_power_savings_show,
  7298. sched_smt_power_savings_store);
  7299. #endif
  7300. int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
  7301. {
  7302. int err = 0;
  7303. #ifdef CONFIG_SCHED_SMT
  7304. if (smt_capable())
  7305. err = sysfs_create_file(&cls->kset.kobj,
  7306. &attr_sched_smt_power_savings.attr);
  7307. #endif
  7308. #ifdef CONFIG_SCHED_MC
  7309. if (!err && mc_capable())
  7310. err = sysfs_create_file(&cls->kset.kobj,
  7311. &attr_sched_mc_power_savings.attr);
  7312. #endif
  7313. return err;
  7314. }
  7315. #endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
  7316. #ifndef CONFIG_CPUSETS
  7317. /*
  7318. * Add online and remove offline CPUs from the scheduler domains.
  7319. * When cpusets are enabled they take over this function.
  7320. */
  7321. static int update_sched_domains(struct notifier_block *nfb,
  7322. unsigned long action, void *hcpu)
  7323. {
  7324. switch (action) {
  7325. case CPU_ONLINE:
  7326. case CPU_ONLINE_FROZEN:
  7327. case CPU_DEAD:
  7328. case CPU_DEAD_FROZEN:
  7329. partition_sched_domains(1, NULL, NULL);
  7330. return NOTIFY_OK;
  7331. default:
  7332. return NOTIFY_DONE;
  7333. }
  7334. }
  7335. #endif
  7336. static int update_runtime(struct notifier_block *nfb,
  7337. unsigned long action, void *hcpu)
  7338. {
  7339. int cpu = (int)(long)hcpu;
  7340. switch (action) {
  7341. case CPU_DOWN_PREPARE:
  7342. case CPU_DOWN_PREPARE_FROZEN:
  7343. disable_runtime(cpu_rq(cpu));
  7344. return NOTIFY_OK;
  7345. case CPU_DOWN_FAILED:
  7346. case CPU_DOWN_FAILED_FROZEN:
  7347. case CPU_ONLINE:
  7348. case CPU_ONLINE_FROZEN:
  7349. enable_runtime(cpu_rq(cpu));
  7350. return NOTIFY_OK;
  7351. default:
  7352. return NOTIFY_DONE;
  7353. }
  7354. }
  7355. void __init sched_init_smp(void)
  7356. {
  7357. cpumask_var_t non_isolated_cpus;
  7358. alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
  7359. #if defined(CONFIG_NUMA)
  7360. sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
  7361. GFP_KERNEL);
  7362. BUG_ON(sched_group_nodes_bycpu == NULL);
  7363. #endif
  7364. get_online_cpus();
  7365. mutex_lock(&sched_domains_mutex);
  7366. arch_init_sched_domains(cpu_online_mask);
  7367. cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
  7368. if (cpumask_empty(non_isolated_cpus))
  7369. cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
  7370. mutex_unlock(&sched_domains_mutex);
  7371. put_online_cpus();
  7372. #ifndef CONFIG_CPUSETS
  7373. /* XXX: Theoretical race here - CPU may be hotplugged now */
  7374. hotcpu_notifier(update_sched_domains, 0);
  7375. #endif
  7376. /* RT runtime code needs to handle some hotplug events */
  7377. hotcpu_notifier(update_runtime, 0);
  7378. init_hrtick();
  7379. /* Move init over to a non-isolated CPU */
  7380. if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
  7381. BUG();
  7382. sched_init_granularity();
  7383. free_cpumask_var(non_isolated_cpus);
  7384. alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
  7385. init_sched_rt_class();
  7386. }
  7387. #else
  7388. void __init sched_init_smp(void)
  7389. {
  7390. sched_init_granularity();
  7391. }
  7392. #endif /* CONFIG_SMP */
  7393. int in_sched_functions(unsigned long addr)
  7394. {
  7395. return in_lock_functions(addr) ||
  7396. (addr >= (unsigned long)__sched_text_start
  7397. && addr < (unsigned long)__sched_text_end);
  7398. }
  7399. static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
  7400. {
  7401. cfs_rq->tasks_timeline = RB_ROOT;
  7402. INIT_LIST_HEAD(&cfs_rq->tasks);
  7403. #ifdef CONFIG_FAIR_GROUP_SCHED
  7404. cfs_rq->rq = rq;
  7405. #endif
  7406. cfs_rq->min_vruntime = (u64)(-(1LL << 20));
  7407. }
  7408. static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
  7409. {
  7410. struct rt_prio_array *array;
  7411. int i;
  7412. array = &rt_rq->active;
  7413. for (i = 0; i < MAX_RT_PRIO; i++) {
  7414. INIT_LIST_HEAD(array->queue + i);
  7415. __clear_bit(i, array->bitmap);
  7416. }
  7417. /* delimiter for bitsearch: */
  7418. __set_bit(MAX_RT_PRIO, array->bitmap);
  7419. #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
  7420. rt_rq->highest_prio.curr = MAX_RT_PRIO;
  7421. #ifdef CONFIG_SMP
  7422. rt_rq->highest_prio.next = MAX_RT_PRIO;
  7423. #endif
  7424. #endif
  7425. #ifdef CONFIG_SMP
  7426. rt_rq->rt_nr_migratory = 0;
  7427. rt_rq->overloaded = 0;
  7428. plist_head_init(&rq->rt.pushable_tasks, &rq->lock);
  7429. #endif
  7430. rt_rq->rt_time = 0;
  7431. rt_rq->rt_throttled = 0;
  7432. rt_rq->rt_runtime = 0;
  7433. spin_lock_init(&rt_rq->rt_runtime_lock);
  7434. #ifdef CONFIG_RT_GROUP_SCHED
  7435. rt_rq->rt_nr_boosted = 0;
  7436. rt_rq->rq = rq;
  7437. #endif
  7438. }
  7439. #ifdef CONFIG_FAIR_GROUP_SCHED
  7440. static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
  7441. struct sched_entity *se, int cpu, int add,
  7442. struct sched_entity *parent)
  7443. {
  7444. struct rq *rq = cpu_rq(cpu);
  7445. tg->cfs_rq[cpu] = cfs_rq;
  7446. init_cfs_rq(cfs_rq, rq);
  7447. cfs_rq->tg = tg;
  7448. if (add)
  7449. list_add(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
  7450. tg->se[cpu] = se;
  7451. /* se could be NULL for init_task_group */
  7452. if (!se)
  7453. return;
  7454. if (!parent)
  7455. se->cfs_rq = &rq->cfs;
  7456. else
  7457. se->cfs_rq = parent->my_q;
  7458. se->my_q = cfs_rq;
  7459. se->load.weight = tg->shares;
  7460. se->load.inv_weight = 0;
  7461. se->parent = parent;
  7462. }
  7463. #endif
  7464. #ifdef CONFIG_RT_GROUP_SCHED
  7465. static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
  7466. struct sched_rt_entity *rt_se, int cpu, int add,
  7467. struct sched_rt_entity *parent)
  7468. {
  7469. struct rq *rq = cpu_rq(cpu);
  7470. tg->rt_rq[cpu] = rt_rq;
  7471. init_rt_rq(rt_rq, rq);
  7472. rt_rq->tg = tg;
  7473. rt_rq->rt_se = rt_se;
  7474. rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
  7475. if (add)
  7476. list_add(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);
  7477. tg->rt_se[cpu] = rt_se;
  7478. if (!rt_se)
  7479. return;
  7480. if (!parent)
  7481. rt_se->rt_rq = &rq->rt;
  7482. else
  7483. rt_se->rt_rq = parent->my_q;
  7484. rt_se->my_q = rt_rq;
  7485. rt_se->parent = parent;
  7486. INIT_LIST_HEAD(&rt_se->run_list);
  7487. }
  7488. #endif
  7489. void __init sched_init(void)
  7490. {
  7491. int i, j;
  7492. unsigned long alloc_size = 0, ptr;
  7493. #ifdef CONFIG_FAIR_GROUP_SCHED
  7494. alloc_size += 2 * nr_cpu_ids * sizeof(void **);
  7495. #endif
  7496. #ifdef CONFIG_RT_GROUP_SCHED
  7497. alloc_size += 2 * nr_cpu_ids * sizeof(void **);
  7498. #endif
  7499. #ifdef CONFIG_USER_SCHED
  7500. alloc_size *= 2;
  7501. #endif
  7502. #ifdef CONFIG_CPUMASK_OFFSTACK
  7503. alloc_size += num_possible_cpus() * cpumask_size();
  7504. #endif
  7505. /*
  7506. * As sched_init() is called before page_alloc is setup,
  7507. * we use alloc_bootmem().
  7508. */
  7509. if (alloc_size) {
  7510. ptr = (unsigned long)alloc_bootmem(alloc_size);
  7511. #ifdef CONFIG_FAIR_GROUP_SCHED
  7512. init_task_group.se = (struct sched_entity **)ptr;
  7513. ptr += nr_cpu_ids * sizeof(void **);
  7514. init_task_group.cfs_rq = (struct cfs_rq **)ptr;
  7515. ptr += nr_cpu_ids * sizeof(void **);
  7516. #ifdef CONFIG_USER_SCHED
  7517. root_task_group.se = (struct sched_entity **)ptr;
  7518. ptr += nr_cpu_ids * sizeof(void **);
  7519. root_task_group.cfs_rq = (struct cfs_rq **)ptr;
  7520. ptr += nr_cpu_ids * sizeof(void **);
  7521. #endif /* CONFIG_USER_SCHED */
  7522. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7523. #ifdef CONFIG_RT_GROUP_SCHED
  7524. init_task_group.rt_se = (struct sched_rt_entity **)ptr;
  7525. ptr += nr_cpu_ids * sizeof(void **);
  7526. init_task_group.rt_rq = (struct rt_rq **)ptr;
  7527. ptr += nr_cpu_ids * sizeof(void **);
  7528. #ifdef CONFIG_USER_SCHED
  7529. root_task_group.rt_se = (struct sched_rt_entity **)ptr;
  7530. ptr += nr_cpu_ids * sizeof(void **);
  7531. root_task_group.rt_rq = (struct rt_rq **)ptr;
  7532. ptr += nr_cpu_ids * sizeof(void **);
  7533. #endif /* CONFIG_USER_SCHED */
  7534. #endif /* CONFIG_RT_GROUP_SCHED */
  7535. #ifdef CONFIG_CPUMASK_OFFSTACK
  7536. for_each_possible_cpu(i) {
  7537. per_cpu(load_balance_tmpmask, i) = (void *)ptr;
  7538. ptr += cpumask_size();
  7539. }
  7540. #endif /* CONFIG_CPUMASK_OFFSTACK */
  7541. }
  7542. #ifdef CONFIG_SMP
  7543. init_defrootdomain();
  7544. #endif
  7545. init_rt_bandwidth(&def_rt_bandwidth,
  7546. global_rt_period(), global_rt_runtime());
  7547. #ifdef CONFIG_RT_GROUP_SCHED
  7548. init_rt_bandwidth(&init_task_group.rt_bandwidth,
  7549. global_rt_period(), global_rt_runtime());
  7550. #ifdef CONFIG_USER_SCHED
  7551. init_rt_bandwidth(&root_task_group.rt_bandwidth,
  7552. global_rt_period(), RUNTIME_INF);
  7553. #endif /* CONFIG_USER_SCHED */
  7554. #endif /* CONFIG_RT_GROUP_SCHED */
  7555. #ifdef CONFIG_GROUP_SCHED
  7556. list_add(&init_task_group.list, &task_groups);
  7557. INIT_LIST_HEAD(&init_task_group.children);
  7558. #ifdef CONFIG_USER_SCHED
  7559. INIT_LIST_HEAD(&root_task_group.children);
  7560. init_task_group.parent = &root_task_group;
  7561. list_add(&init_task_group.siblings, &root_task_group.children);
  7562. #endif /* CONFIG_USER_SCHED */
  7563. #endif /* CONFIG_GROUP_SCHED */
  7564. for_each_possible_cpu(i) {
  7565. struct rq *rq;
  7566. rq = cpu_rq(i);
  7567. spin_lock_init(&rq->lock);
  7568. rq->nr_running = 0;
  7569. init_cfs_rq(&rq->cfs, rq);
  7570. init_rt_rq(&rq->rt, rq);
  7571. #ifdef CONFIG_FAIR_GROUP_SCHED
  7572. init_task_group.shares = init_task_group_load;
  7573. INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
  7574. #ifdef CONFIG_CGROUP_SCHED
  7575. /*
  7576. * How much cpu bandwidth does init_task_group get?
  7577. *
  7578. * In case of task-groups formed thr' the cgroup filesystem, it
  7579. * gets 100% of the cpu resources in the system. This overall
  7580. * system cpu resource is divided among the tasks of
  7581. * init_task_group and its child task-groups in a fair manner,
  7582. * based on each entity's (task or task-group's) weight
  7583. * (se->load.weight).
  7584. *
  7585. * In other words, if init_task_group has 10 tasks of weight
  7586. * 1024) and two child groups A0 and A1 (of weight 1024 each),
  7587. * then A0's share of the cpu resource is:
  7588. *
  7589. * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
  7590. *
  7591. * We achieve this by letting init_task_group's tasks sit
  7592. * directly in rq->cfs (i.e init_task_group->se[] = NULL).
  7593. */
  7594. init_tg_cfs_entry(&init_task_group, &rq->cfs, NULL, i, 1, NULL);
  7595. #elif defined CONFIG_USER_SCHED
  7596. root_task_group.shares = NICE_0_LOAD;
  7597. init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, 0, NULL);
  7598. /*
  7599. * In case of task-groups formed thr' the user id of tasks,
  7600. * init_task_group represents tasks belonging to root user.
  7601. * Hence it forms a sibling of all subsequent groups formed.
  7602. * In this case, init_task_group gets only a fraction of overall
  7603. * system cpu resource, based on the weight assigned to root
  7604. * user's cpu share (INIT_TASK_GROUP_LOAD). This is accomplished
  7605. * by letting tasks of init_task_group sit in a separate cfs_rq
  7606. * (init_cfs_rq) and having one entity represent this group of
  7607. * tasks in rq->cfs (i.e init_task_group->se[] != NULL).
  7608. */
  7609. init_tg_cfs_entry(&init_task_group,
  7610. &per_cpu(init_cfs_rq, i),
  7611. &per_cpu(init_sched_entity, i), i, 1,
  7612. root_task_group.se[i]);
  7613. #endif
  7614. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7615. rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
  7616. #ifdef CONFIG_RT_GROUP_SCHED
  7617. INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
  7618. #ifdef CONFIG_CGROUP_SCHED
  7619. init_tg_rt_entry(&init_task_group, &rq->rt, NULL, i, 1, NULL);
  7620. #elif defined CONFIG_USER_SCHED
  7621. init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL);
  7622. init_tg_rt_entry(&init_task_group,
  7623. &per_cpu(init_rt_rq, i),
  7624. &per_cpu(init_sched_rt_entity, i), i, 1,
  7625. root_task_group.rt_se[i]);
  7626. #endif
  7627. #endif
  7628. for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
  7629. rq->cpu_load[j] = 0;
  7630. #ifdef CONFIG_SMP
  7631. rq->sd = NULL;
  7632. rq->rd = NULL;
  7633. rq->active_balance = 0;
  7634. rq->next_balance = jiffies;
  7635. rq->push_cpu = 0;
  7636. rq->cpu = i;
  7637. rq->online = 0;
  7638. rq->migration_thread = NULL;
  7639. INIT_LIST_HEAD(&rq->migration_queue);
  7640. rq_attach_root(rq, &def_root_domain);
  7641. #endif
  7642. init_rq_hrtick(rq);
  7643. atomic_set(&rq->nr_iowait, 0);
  7644. }
  7645. set_load_weight(&init_task);
  7646. #ifdef CONFIG_PREEMPT_NOTIFIERS
  7647. INIT_HLIST_HEAD(&init_task.preempt_notifiers);
  7648. #endif
  7649. #ifdef CONFIG_SMP
  7650. open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
  7651. #endif
  7652. #ifdef CONFIG_RT_MUTEXES
  7653. plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
  7654. #endif
  7655. /*
  7656. * The boot idle thread does lazy MMU switching as well:
  7657. */
  7658. atomic_inc(&init_mm.mm_count);
  7659. enter_lazy_tlb(&init_mm, current);
  7660. /*
  7661. * Make us the idle thread. Technically, schedule() should not be
  7662. * called from this thread, however somewhere below it might be,
  7663. * but because we are the idle thread, we just pick up running again
  7664. * when this runqueue becomes "idle".
  7665. */
  7666. init_idle(current, smp_processor_id());
  7667. /*
  7668. * During early bootup we pretend to be a normal task:
  7669. */
  7670. current->sched_class = &fair_sched_class;
  7671. /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
  7672. alloc_bootmem_cpumask_var(&nohz_cpu_mask);
  7673. #ifdef CONFIG_SMP
  7674. #ifdef CONFIG_NO_HZ
  7675. alloc_bootmem_cpumask_var(&nohz.cpu_mask);
  7676. #endif
  7677. alloc_bootmem_cpumask_var(&cpu_isolated_map);
  7678. #endif /* SMP */
  7679. scheduler_running = 1;
  7680. }
  7681. #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
  7682. void __might_sleep(char *file, int line)
  7683. {
  7684. #ifdef in_atomic
  7685. static unsigned long prev_jiffy; /* ratelimiting */
  7686. if ((!in_atomic() && !irqs_disabled()) ||
  7687. system_state != SYSTEM_RUNNING || oops_in_progress)
  7688. return;
  7689. if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
  7690. return;
  7691. prev_jiffy = jiffies;
  7692. printk(KERN_ERR
  7693. "BUG: sleeping function called from invalid context at %s:%d\n",
  7694. file, line);
  7695. printk(KERN_ERR
  7696. "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
  7697. in_atomic(), irqs_disabled(),
  7698. current->pid, current->comm);
  7699. debug_show_held_locks(current);
  7700. if (irqs_disabled())
  7701. print_irqtrace_events(current);
  7702. dump_stack();
  7703. #endif
  7704. }
  7705. EXPORT_SYMBOL(__might_sleep);
  7706. #endif
  7707. #ifdef CONFIG_MAGIC_SYSRQ
  7708. static void normalize_task(struct rq *rq, struct task_struct *p)
  7709. {
  7710. int on_rq;
  7711. update_rq_clock(rq);
  7712. on_rq = p->se.on_rq;
  7713. if (on_rq)
  7714. deactivate_task(rq, p, 0);
  7715. __setscheduler(rq, p, SCHED_NORMAL, 0);
  7716. if (on_rq) {
  7717. activate_task(rq, p, 0);
  7718. resched_task(rq->curr);
  7719. }
  7720. }
  7721. void normalize_rt_tasks(void)
  7722. {
  7723. struct task_struct *g, *p;
  7724. unsigned long flags;
  7725. struct rq *rq;
  7726. read_lock_irqsave(&tasklist_lock, flags);
  7727. do_each_thread(g, p) {
  7728. /*
  7729. * Only normalize user tasks:
  7730. */
  7731. if (!p->mm)
  7732. continue;
  7733. p->se.exec_start = 0;
  7734. #ifdef CONFIG_SCHEDSTATS
  7735. p->se.wait_start = 0;
  7736. p->se.sleep_start = 0;
  7737. p->se.block_start = 0;
  7738. #endif
  7739. if (!rt_task(p)) {
  7740. /*
  7741. * Renice negative nice level userspace
  7742. * tasks back to 0:
  7743. */
  7744. if (TASK_NICE(p) < 0 && p->mm)
  7745. set_user_nice(p, 0);
  7746. continue;
  7747. }
  7748. spin_lock(&p->pi_lock);
  7749. rq = __task_rq_lock(p);
  7750. normalize_task(rq, p);
  7751. __task_rq_unlock(rq);
  7752. spin_unlock(&p->pi_lock);
  7753. } while_each_thread(g, p);
  7754. read_unlock_irqrestore(&tasklist_lock, flags);
  7755. }
  7756. #endif /* CONFIG_MAGIC_SYSRQ */
  7757. #ifdef CONFIG_IA64
  7758. /*
  7759. * These functions are only useful for the IA64 MCA handling.
  7760. *
  7761. * They can only be called when the whole system has been
  7762. * stopped - every CPU needs to be quiescent, and no scheduling
  7763. * activity can take place. Using them for anything else would
  7764. * be a serious bug, and as a result, they aren't even visible
  7765. * under any other configuration.
  7766. */
  7767. /**
  7768. * curr_task - return the current task for a given cpu.
  7769. * @cpu: the processor in question.
  7770. *
  7771. * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
  7772. */
  7773. struct task_struct *curr_task(int cpu)
  7774. {
  7775. return cpu_curr(cpu);
  7776. }
  7777. /**
  7778. * set_curr_task - set the current task for a given cpu.
  7779. * @cpu: the processor in question.
  7780. * @p: the task pointer to set.
  7781. *
  7782. * Description: This function must only be used when non-maskable interrupts
  7783. * are serviced on a separate stack. It allows the architecture to switch the
  7784. * notion of the current task on a cpu in a non-blocking manner. This function
  7785. * must be called with all CPU's synchronized, and interrupts disabled, the
  7786. * and caller must save the original value of the current task (see
  7787. * curr_task() above) and restore that value before reenabling interrupts and
  7788. * re-starting the system.
  7789. *
  7790. * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
  7791. */
  7792. void set_curr_task(int cpu, struct task_struct *p)
  7793. {
  7794. cpu_curr(cpu) = p;
  7795. }
  7796. #endif
  7797. #ifdef CONFIG_FAIR_GROUP_SCHED
  7798. static void free_fair_sched_group(struct task_group *tg)
  7799. {
  7800. int i;
  7801. for_each_possible_cpu(i) {
  7802. if (tg->cfs_rq)
  7803. kfree(tg->cfs_rq[i]);
  7804. if (tg->se)
  7805. kfree(tg->se[i]);
  7806. }
  7807. kfree(tg->cfs_rq);
  7808. kfree(tg->se);
  7809. }
  7810. static
  7811. int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
  7812. {
  7813. struct cfs_rq *cfs_rq;
  7814. struct sched_entity *se;
  7815. struct rq *rq;
  7816. int i;
  7817. tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
  7818. if (!tg->cfs_rq)
  7819. goto err;
  7820. tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
  7821. if (!tg->se)
  7822. goto err;
  7823. tg->shares = NICE_0_LOAD;
  7824. for_each_possible_cpu(i) {
  7825. rq = cpu_rq(i);
  7826. cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
  7827. GFP_KERNEL, cpu_to_node(i));
  7828. if (!cfs_rq)
  7829. goto err;
  7830. se = kzalloc_node(sizeof(struct sched_entity),
  7831. GFP_KERNEL, cpu_to_node(i));
  7832. if (!se)
  7833. goto err;
  7834. init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
  7835. }
  7836. return 1;
  7837. err:
  7838. return 0;
  7839. }
  7840. static inline void register_fair_sched_group(struct task_group *tg, int cpu)
  7841. {
  7842. list_add_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list,
  7843. &cpu_rq(cpu)->leaf_cfs_rq_list);
  7844. }
  7845. static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
  7846. {
  7847. list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
  7848. }
  7849. #else /* !CONFG_FAIR_GROUP_SCHED */
  7850. static inline void free_fair_sched_group(struct task_group *tg)
  7851. {
  7852. }
  7853. static inline
  7854. int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
  7855. {
  7856. return 1;
  7857. }
  7858. static inline void register_fair_sched_group(struct task_group *tg, int cpu)
  7859. {
  7860. }
  7861. static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
  7862. {
  7863. }
  7864. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7865. #ifdef CONFIG_RT_GROUP_SCHED
  7866. static void free_rt_sched_group(struct task_group *tg)
  7867. {
  7868. int i;
  7869. destroy_rt_bandwidth(&tg->rt_bandwidth);
  7870. for_each_possible_cpu(i) {
  7871. if (tg->rt_rq)
  7872. kfree(tg->rt_rq[i]);
  7873. if (tg->rt_se)
  7874. kfree(tg->rt_se[i]);
  7875. }
  7876. kfree(tg->rt_rq);
  7877. kfree(tg->rt_se);
  7878. }
  7879. static
  7880. int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
  7881. {
  7882. struct rt_rq *rt_rq;
  7883. struct sched_rt_entity *rt_se;
  7884. struct rq *rq;
  7885. int i;
  7886. tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
  7887. if (!tg->rt_rq)
  7888. goto err;
  7889. tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
  7890. if (!tg->rt_se)
  7891. goto err;
  7892. init_rt_bandwidth(&tg->rt_bandwidth,
  7893. ktime_to_ns(def_rt_bandwidth.rt_period), 0);
  7894. for_each_possible_cpu(i) {
  7895. rq = cpu_rq(i);
  7896. rt_rq = kzalloc_node(sizeof(struct rt_rq),
  7897. GFP_KERNEL, cpu_to_node(i));
  7898. if (!rt_rq)
  7899. goto err;
  7900. rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
  7901. GFP_KERNEL, cpu_to_node(i));
  7902. if (!rt_se)
  7903. goto err;
  7904. init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
  7905. }
  7906. return 1;
  7907. err:
  7908. return 0;
  7909. }
  7910. static inline void register_rt_sched_group(struct task_group *tg, int cpu)
  7911. {
  7912. list_add_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list,
  7913. &cpu_rq(cpu)->leaf_rt_rq_list);
  7914. }
  7915. static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
  7916. {
  7917. list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
  7918. }
  7919. #else /* !CONFIG_RT_GROUP_SCHED */
  7920. static inline void free_rt_sched_group(struct task_group *tg)
  7921. {
  7922. }
  7923. static inline
  7924. int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
  7925. {
  7926. return 1;
  7927. }
  7928. static inline void register_rt_sched_group(struct task_group *tg, int cpu)
  7929. {
  7930. }
  7931. static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
  7932. {
  7933. }
  7934. #endif /* CONFIG_RT_GROUP_SCHED */
  7935. #ifdef CONFIG_GROUP_SCHED
  7936. static void free_sched_group(struct task_group *tg)
  7937. {
  7938. free_fair_sched_group(tg);
  7939. free_rt_sched_group(tg);
  7940. kfree(tg);
  7941. }
  7942. /* allocate runqueue etc for a new task group */
  7943. struct task_group *sched_create_group(struct task_group *parent)
  7944. {
  7945. struct task_group *tg;
  7946. unsigned long flags;
  7947. int i;
  7948. tg = kzalloc(sizeof(*tg), GFP_KERNEL);
  7949. if (!tg)
  7950. return ERR_PTR(-ENOMEM);
  7951. if (!alloc_fair_sched_group(tg, parent))
  7952. goto err;
  7953. if (!alloc_rt_sched_group(tg, parent))
  7954. goto err;
  7955. spin_lock_irqsave(&task_group_lock, flags);
  7956. for_each_possible_cpu(i) {
  7957. register_fair_sched_group(tg, i);
  7958. register_rt_sched_group(tg, i);
  7959. }
  7960. list_add_rcu(&tg->list, &task_groups);
  7961. WARN_ON(!parent); /* root should already exist */
  7962. tg->parent = parent;
  7963. INIT_LIST_HEAD(&tg->children);
  7964. list_add_rcu(&tg->siblings, &parent->children);
  7965. spin_unlock_irqrestore(&task_group_lock, flags);
  7966. return tg;
  7967. err:
  7968. free_sched_group(tg);
  7969. return ERR_PTR(-ENOMEM);
  7970. }
  7971. /* rcu callback to free various structures associated with a task group */
  7972. static void free_sched_group_rcu(struct rcu_head *rhp)
  7973. {
  7974. /* now it should be safe to free those cfs_rqs */
  7975. free_sched_group(container_of(rhp, struct task_group, rcu));
  7976. }
  7977. /* Destroy runqueue etc associated with a task group */
  7978. void sched_destroy_group(struct task_group *tg)
  7979. {
  7980. unsigned long flags;
  7981. int i;
  7982. spin_lock_irqsave(&task_group_lock, flags);
  7983. for_each_possible_cpu(i) {
  7984. unregister_fair_sched_group(tg, i);
  7985. unregister_rt_sched_group(tg, i);
  7986. }
  7987. list_del_rcu(&tg->list);
  7988. list_del_rcu(&tg->siblings);
  7989. spin_unlock_irqrestore(&task_group_lock, flags);
  7990. /* wait for possible concurrent references to cfs_rqs complete */
  7991. call_rcu(&tg->rcu, free_sched_group_rcu);
  7992. }
  7993. /* change task's runqueue when it moves between groups.
  7994. * The caller of this function should have put the task in its new group
  7995. * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
  7996. * reflect its new group.
  7997. */
  7998. void sched_move_task(struct task_struct *tsk)
  7999. {
  8000. int on_rq, running;
  8001. unsigned long flags;
  8002. struct rq *rq;
  8003. rq = task_rq_lock(tsk, &flags);
  8004. update_rq_clock(rq);
  8005. running = task_current(rq, tsk);
  8006. on_rq = tsk->se.on_rq;
  8007. if (on_rq)
  8008. dequeue_task(rq, tsk, 0);
  8009. if (unlikely(running))
  8010. tsk->sched_class->put_prev_task(rq, tsk);
  8011. set_task_rq(tsk, task_cpu(tsk));
  8012. #ifdef CONFIG_FAIR_GROUP_SCHED
  8013. if (tsk->sched_class->moved_group)
  8014. tsk->sched_class->moved_group(tsk);
  8015. #endif
  8016. if (unlikely(running))
  8017. tsk->sched_class->set_curr_task(rq);
  8018. if (on_rq)
  8019. enqueue_task(rq, tsk, 0);
  8020. task_rq_unlock(rq, &flags);
  8021. }
  8022. #endif /* CONFIG_GROUP_SCHED */
  8023. #ifdef CONFIG_FAIR_GROUP_SCHED
  8024. static void __set_se_shares(struct sched_entity *se, unsigned long shares)
  8025. {
  8026. struct cfs_rq *cfs_rq = se->cfs_rq;
  8027. int on_rq;
  8028. on_rq = se->on_rq;
  8029. if (on_rq)
  8030. dequeue_entity(cfs_rq, se, 0);
  8031. se->load.weight = shares;
  8032. se->load.inv_weight = 0;
  8033. if (on_rq)
  8034. enqueue_entity(cfs_rq, se, 0);
  8035. }
  8036. static void set_se_shares(struct sched_entity *se, unsigned long shares)
  8037. {
  8038. struct cfs_rq *cfs_rq = se->cfs_rq;
  8039. struct rq *rq = cfs_rq->rq;
  8040. unsigned long flags;
  8041. spin_lock_irqsave(&rq->lock, flags);
  8042. __set_se_shares(se, shares);
  8043. spin_unlock_irqrestore(&rq->lock, flags);
  8044. }
  8045. static DEFINE_MUTEX(shares_mutex);
  8046. int sched_group_set_shares(struct task_group *tg, unsigned long shares)
  8047. {
  8048. int i;
  8049. unsigned long flags;
  8050. /*
  8051. * We can't change the weight of the root cgroup.
  8052. */
  8053. if (!tg->se[0])
  8054. return -EINVAL;
  8055. if (shares < MIN_SHARES)
  8056. shares = MIN_SHARES;
  8057. else if (shares > MAX_SHARES)
  8058. shares = MAX_SHARES;
  8059. mutex_lock(&shares_mutex);
  8060. if (tg->shares == shares)
  8061. goto done;
  8062. spin_lock_irqsave(&task_group_lock, flags);
  8063. for_each_possible_cpu(i)
  8064. unregister_fair_sched_group(tg, i);
  8065. list_del_rcu(&tg->siblings);
  8066. spin_unlock_irqrestore(&task_group_lock, flags);
  8067. /* wait for any ongoing reference to this group to finish */
  8068. synchronize_sched();
  8069. /*
  8070. * Now we are free to modify the group's share on each cpu
  8071. * w/o tripping rebalance_share or load_balance_fair.
  8072. */
  8073. tg->shares = shares;
  8074. for_each_possible_cpu(i) {
  8075. /*
  8076. * force a rebalance
  8077. */
  8078. cfs_rq_set_shares(tg->cfs_rq[i], 0);
  8079. set_se_shares(tg->se[i], shares);
  8080. }
  8081. /*
  8082. * Enable load balance activity on this group, by inserting it back on
  8083. * each cpu's rq->leaf_cfs_rq_list.
  8084. */
  8085. spin_lock_irqsave(&task_group_lock, flags);
  8086. for_each_possible_cpu(i)
  8087. register_fair_sched_group(tg, i);
  8088. list_add_rcu(&tg->siblings, &tg->parent->children);
  8089. spin_unlock_irqrestore(&task_group_lock, flags);
  8090. done:
  8091. mutex_unlock(&shares_mutex);
  8092. return 0;
  8093. }
  8094. unsigned long sched_group_shares(struct task_group *tg)
  8095. {
  8096. return tg->shares;
  8097. }
  8098. #endif
  8099. #ifdef CONFIG_RT_GROUP_SCHED
  8100. /*
  8101. * Ensure that the real time constraints are schedulable.
  8102. */
  8103. static DEFINE_MUTEX(rt_constraints_mutex);
  8104. static unsigned long to_ratio(u64 period, u64 runtime)
  8105. {
  8106. if (runtime == RUNTIME_INF)
  8107. return 1ULL << 20;
  8108. return div64_u64(runtime << 20, period);
  8109. }
  8110. /* Must be called with tasklist_lock held */
  8111. static inline int tg_has_rt_tasks(struct task_group *tg)
  8112. {
  8113. struct task_struct *g, *p;
  8114. do_each_thread(g, p) {
  8115. if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg)
  8116. return 1;
  8117. } while_each_thread(g, p);
  8118. return 0;
  8119. }
  8120. struct rt_schedulable_data {
  8121. struct task_group *tg;
  8122. u64 rt_period;
  8123. u64 rt_runtime;
  8124. };
  8125. static int tg_schedulable(struct task_group *tg, void *data)
  8126. {
  8127. struct rt_schedulable_data *d = data;
  8128. struct task_group *child;
  8129. unsigned long total, sum = 0;
  8130. u64 period, runtime;
  8131. period = ktime_to_ns(tg->rt_bandwidth.rt_period);
  8132. runtime = tg->rt_bandwidth.rt_runtime;
  8133. if (tg == d->tg) {
  8134. period = d->rt_period;
  8135. runtime = d->rt_runtime;
  8136. }
  8137. #ifdef CONFIG_USER_SCHED
  8138. if (tg == &root_task_group) {
  8139. period = global_rt_period();
  8140. runtime = global_rt_runtime();
  8141. }
  8142. #endif
  8143. /*
  8144. * Cannot have more runtime than the period.
  8145. */
  8146. if (runtime > period && runtime != RUNTIME_INF)
  8147. return -EINVAL;
  8148. /*
  8149. * Ensure we don't starve existing RT tasks.
  8150. */
  8151. if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
  8152. return -EBUSY;
  8153. total = to_ratio(period, runtime);
  8154. /*
  8155. * Nobody can have more than the global setting allows.
  8156. */
  8157. if (total > to_ratio(global_rt_period(), global_rt_runtime()))
  8158. return -EINVAL;
  8159. /*
  8160. * The sum of our children's runtime should not exceed our own.
  8161. */
  8162. list_for_each_entry_rcu(child, &tg->children, siblings) {
  8163. period = ktime_to_ns(child->rt_bandwidth.rt_period);
  8164. runtime = child->rt_bandwidth.rt_runtime;
  8165. if (child == d->tg) {
  8166. period = d->rt_period;
  8167. runtime = d->rt_runtime;
  8168. }
  8169. sum += to_ratio(period, runtime);
  8170. }
  8171. if (sum > total)
  8172. return -EINVAL;
  8173. return 0;
  8174. }
  8175. static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
  8176. {
  8177. struct rt_schedulable_data data = {
  8178. .tg = tg,
  8179. .rt_period = period,
  8180. .rt_runtime = runtime,
  8181. };
  8182. return walk_tg_tree(tg_schedulable, tg_nop, &data);
  8183. }
  8184. static int tg_set_bandwidth(struct task_group *tg,
  8185. u64 rt_period, u64 rt_runtime)
  8186. {
  8187. int i, err = 0;
  8188. mutex_lock(&rt_constraints_mutex);
  8189. read_lock(&tasklist_lock);
  8190. err = __rt_schedulable(tg, rt_period, rt_runtime);
  8191. if (err)
  8192. goto unlock;
  8193. spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  8194. tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
  8195. tg->rt_bandwidth.rt_runtime = rt_runtime;
  8196. for_each_possible_cpu(i) {
  8197. struct rt_rq *rt_rq = tg->rt_rq[i];
  8198. spin_lock(&rt_rq->rt_runtime_lock);
  8199. rt_rq->rt_runtime = rt_runtime;
  8200. spin_unlock(&rt_rq->rt_runtime_lock);
  8201. }
  8202. spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  8203. unlock:
  8204. read_unlock(&tasklist_lock);
  8205. mutex_unlock(&rt_constraints_mutex);
  8206. return err;
  8207. }
  8208. int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
  8209. {
  8210. u64 rt_runtime, rt_period;
  8211. rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
  8212. rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
  8213. if (rt_runtime_us < 0)
  8214. rt_runtime = RUNTIME_INF;
  8215. return tg_set_bandwidth(tg, rt_period, rt_runtime);
  8216. }
  8217. long sched_group_rt_runtime(struct task_group *tg)
  8218. {
  8219. u64 rt_runtime_us;
  8220. if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
  8221. return -1;
  8222. rt_runtime_us = tg->rt_bandwidth.rt_runtime;
  8223. do_div(rt_runtime_us, NSEC_PER_USEC);
  8224. return rt_runtime_us;
  8225. }
  8226. int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
  8227. {
  8228. u64 rt_runtime, rt_period;
  8229. rt_period = (u64)rt_period_us * NSEC_PER_USEC;
  8230. rt_runtime = tg->rt_bandwidth.rt_runtime;
  8231. if (rt_period == 0)
  8232. return -EINVAL;
  8233. return tg_set_bandwidth(tg, rt_period, rt_runtime);
  8234. }
  8235. long sched_group_rt_period(struct task_group *tg)
  8236. {
  8237. u64 rt_period_us;
  8238. rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
  8239. do_div(rt_period_us, NSEC_PER_USEC);
  8240. return rt_period_us;
  8241. }
  8242. static int sched_rt_global_constraints(void)
  8243. {
  8244. u64 runtime, period;
  8245. int ret = 0;
  8246. if (sysctl_sched_rt_period <= 0)
  8247. return -EINVAL;
  8248. runtime = global_rt_runtime();
  8249. period = global_rt_period();
  8250. /*
  8251. * Sanity check on the sysctl variables.
  8252. */
  8253. if (runtime > period && runtime != RUNTIME_INF)
  8254. return -EINVAL;
  8255. mutex_lock(&rt_constraints_mutex);
  8256. read_lock(&tasklist_lock);
  8257. ret = __rt_schedulable(NULL, 0, 0);
  8258. read_unlock(&tasklist_lock);
  8259. mutex_unlock(&rt_constraints_mutex);
  8260. return ret;
  8261. }
  8262. int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
  8263. {
  8264. /* Don't accept realtime tasks when there is no way for them to run */
  8265. if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
  8266. return 0;
  8267. return 1;
  8268. }
  8269. #else /* !CONFIG_RT_GROUP_SCHED */
  8270. static int sched_rt_global_constraints(void)
  8271. {
  8272. unsigned long flags;
  8273. int i;
  8274. if (sysctl_sched_rt_period <= 0)
  8275. return -EINVAL;
  8276. spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
  8277. for_each_possible_cpu(i) {
  8278. struct rt_rq *rt_rq = &cpu_rq(i)->rt;
  8279. spin_lock(&rt_rq->rt_runtime_lock);
  8280. rt_rq->rt_runtime = global_rt_runtime();
  8281. spin_unlock(&rt_rq->rt_runtime_lock);
  8282. }
  8283. spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
  8284. return 0;
  8285. }
  8286. #endif /* CONFIG_RT_GROUP_SCHED */
  8287. int sched_rt_handler(struct ctl_table *table, int write,
  8288. struct file *filp, void __user *buffer, size_t *lenp,
  8289. loff_t *ppos)
  8290. {
  8291. int ret;
  8292. int old_period, old_runtime;
  8293. static DEFINE_MUTEX(mutex);
  8294. mutex_lock(&mutex);
  8295. old_period = sysctl_sched_rt_period;
  8296. old_runtime = sysctl_sched_rt_runtime;
  8297. ret = proc_dointvec(table, write, filp, buffer, lenp, ppos);
  8298. if (!ret && write) {
  8299. ret = sched_rt_global_constraints();
  8300. if (ret) {
  8301. sysctl_sched_rt_period = old_period;
  8302. sysctl_sched_rt_runtime = old_runtime;
  8303. } else {
  8304. def_rt_bandwidth.rt_runtime = global_rt_runtime();
  8305. def_rt_bandwidth.rt_period =
  8306. ns_to_ktime(global_rt_period());
  8307. }
  8308. }
  8309. mutex_unlock(&mutex);
  8310. return ret;
  8311. }
  8312. #ifdef CONFIG_CGROUP_SCHED
  8313. /* return corresponding task_group object of a cgroup */
  8314. static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
  8315. {
  8316. return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
  8317. struct task_group, css);
  8318. }
  8319. static struct cgroup_subsys_state *
  8320. cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
  8321. {
  8322. struct task_group *tg, *parent;
  8323. if (!cgrp->parent) {
  8324. /* This is early initialization for the top cgroup */
  8325. return &init_task_group.css;
  8326. }
  8327. parent = cgroup_tg(cgrp->parent);
  8328. tg = sched_create_group(parent);
  8329. if (IS_ERR(tg))
  8330. return ERR_PTR(-ENOMEM);
  8331. return &tg->css;
  8332. }
  8333. static void
  8334. cpu_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
  8335. {
  8336. struct task_group *tg = cgroup_tg(cgrp);
  8337. sched_destroy_group(tg);
  8338. }
  8339. static int
  8340. cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
  8341. struct task_struct *tsk)
  8342. {
  8343. #ifdef CONFIG_RT_GROUP_SCHED
  8344. if (!sched_rt_can_attach(cgroup_tg(cgrp), tsk))
  8345. return -EINVAL;
  8346. #else
  8347. /* We don't support RT-tasks being in separate groups */
  8348. if (tsk->sched_class != &fair_sched_class)
  8349. return -EINVAL;
  8350. #endif
  8351. return 0;
  8352. }
  8353. static void
  8354. cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
  8355. struct cgroup *old_cont, struct task_struct *tsk)
  8356. {
  8357. sched_move_task(tsk);
  8358. }
  8359. #ifdef CONFIG_FAIR_GROUP_SCHED
  8360. static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
  8361. u64 shareval)
  8362. {
  8363. return sched_group_set_shares(cgroup_tg(cgrp), shareval);
  8364. }
  8365. static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
  8366. {
  8367. struct task_group *tg = cgroup_tg(cgrp);
  8368. return (u64) tg->shares;
  8369. }
  8370. #endif /* CONFIG_FAIR_GROUP_SCHED */
  8371. #ifdef CONFIG_RT_GROUP_SCHED
  8372. static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
  8373. s64 val)
  8374. {
  8375. return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
  8376. }
  8377. static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
  8378. {
  8379. return sched_group_rt_runtime(cgroup_tg(cgrp));
  8380. }
  8381. static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
  8382. u64 rt_period_us)
  8383. {
  8384. return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
  8385. }
  8386. static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
  8387. {
  8388. return sched_group_rt_period(cgroup_tg(cgrp));
  8389. }
  8390. #endif /* CONFIG_RT_GROUP_SCHED */
  8391. static struct cftype cpu_files[] = {
  8392. #ifdef CONFIG_FAIR_GROUP_SCHED
  8393. {
  8394. .name = "shares",
  8395. .read_u64 = cpu_shares_read_u64,
  8396. .write_u64 = cpu_shares_write_u64,
  8397. },
  8398. #endif
  8399. #ifdef CONFIG_RT_GROUP_SCHED
  8400. {
  8401. .name = "rt_runtime_us",
  8402. .read_s64 = cpu_rt_runtime_read,
  8403. .write_s64 = cpu_rt_runtime_write,
  8404. },
  8405. {
  8406. .name = "rt_period_us",
  8407. .read_u64 = cpu_rt_period_read_uint,
  8408. .write_u64 = cpu_rt_period_write_uint,
  8409. },
  8410. #endif
  8411. };
  8412. static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
  8413. {
  8414. return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files));
  8415. }
  8416. struct cgroup_subsys cpu_cgroup_subsys = {
  8417. .name = "cpu",
  8418. .create = cpu_cgroup_create,
  8419. .destroy = cpu_cgroup_destroy,
  8420. .can_attach = cpu_cgroup_can_attach,
  8421. .attach = cpu_cgroup_attach,
  8422. .populate = cpu_cgroup_populate,
  8423. .subsys_id = cpu_cgroup_subsys_id,
  8424. .early_init = 1,
  8425. };
  8426. #endif /* CONFIG_CGROUP_SCHED */
  8427. #ifdef CONFIG_CGROUP_CPUACCT
  8428. /*
  8429. * CPU accounting code for task groups.
  8430. *
  8431. * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
  8432. * (balbir@in.ibm.com).
  8433. */
  8434. /* track cpu usage of a group of tasks and its child groups */
  8435. struct cpuacct {
  8436. struct cgroup_subsys_state css;
  8437. /* cpuusage holds pointer to a u64-type object on every cpu */
  8438. u64 *cpuusage;
  8439. struct cpuacct *parent;
  8440. };
  8441. struct cgroup_subsys cpuacct_subsys;
  8442. /* return cpu accounting group corresponding to this container */
  8443. static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
  8444. {
  8445. return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
  8446. struct cpuacct, css);
  8447. }
  8448. /* return cpu accounting group to which this task belongs */
  8449. static inline struct cpuacct *task_ca(struct task_struct *tsk)
  8450. {
  8451. return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
  8452. struct cpuacct, css);
  8453. }
  8454. /* create a new cpu accounting group */
  8455. static struct cgroup_subsys_state *cpuacct_create(
  8456. struct cgroup_subsys *ss, struct cgroup *cgrp)
  8457. {
  8458. struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
  8459. if (!ca)
  8460. return ERR_PTR(-ENOMEM);
  8461. ca->cpuusage = alloc_percpu(u64);
  8462. if (!ca->cpuusage) {
  8463. kfree(ca);
  8464. return ERR_PTR(-ENOMEM);
  8465. }
  8466. if (cgrp->parent)
  8467. ca->parent = cgroup_ca(cgrp->parent);
  8468. return &ca->css;
  8469. }
  8470. /* destroy an existing cpu accounting group */
  8471. static void
  8472. cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
  8473. {
  8474. struct cpuacct *ca = cgroup_ca(cgrp);
  8475. free_percpu(ca->cpuusage);
  8476. kfree(ca);
  8477. }
  8478. static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
  8479. {
  8480. u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
  8481. u64 data;
  8482. #ifndef CONFIG_64BIT
  8483. /*
  8484. * Take rq->lock to make 64-bit read safe on 32-bit platforms.
  8485. */
  8486. spin_lock_irq(&cpu_rq(cpu)->lock);
  8487. data = *cpuusage;
  8488. spin_unlock_irq(&cpu_rq(cpu)->lock);
  8489. #else
  8490. data = *cpuusage;
  8491. #endif
  8492. return data;
  8493. }
  8494. static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
  8495. {
  8496. u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
  8497. #ifndef CONFIG_64BIT
  8498. /*
  8499. * Take rq->lock to make 64-bit write safe on 32-bit platforms.
  8500. */
  8501. spin_lock_irq(&cpu_rq(cpu)->lock);
  8502. *cpuusage = val;
  8503. spin_unlock_irq(&cpu_rq(cpu)->lock);
  8504. #else
  8505. *cpuusage = val;
  8506. #endif
  8507. }
  8508. /* return total cpu usage (in nanoseconds) of a group */
  8509. static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
  8510. {
  8511. struct cpuacct *ca = cgroup_ca(cgrp);
  8512. u64 totalcpuusage = 0;
  8513. int i;
  8514. for_each_present_cpu(i)
  8515. totalcpuusage += cpuacct_cpuusage_read(ca, i);
  8516. return totalcpuusage;
  8517. }
  8518. static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
  8519. u64 reset)
  8520. {
  8521. struct cpuacct *ca = cgroup_ca(cgrp);
  8522. int err = 0;
  8523. int i;
  8524. if (reset) {
  8525. err = -EINVAL;
  8526. goto out;
  8527. }
  8528. for_each_present_cpu(i)
  8529. cpuacct_cpuusage_write(ca, i, 0);
  8530. out:
  8531. return err;
  8532. }
  8533. static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
  8534. struct seq_file *m)
  8535. {
  8536. struct cpuacct *ca = cgroup_ca(cgroup);
  8537. u64 percpu;
  8538. int i;
  8539. for_each_present_cpu(i) {
  8540. percpu = cpuacct_cpuusage_read(ca, i);
  8541. seq_printf(m, "%llu ", (unsigned long long) percpu);
  8542. }
  8543. seq_printf(m, "\n");
  8544. return 0;
  8545. }
  8546. static struct cftype files[] = {
  8547. {
  8548. .name = "usage",
  8549. .read_u64 = cpuusage_read,
  8550. .write_u64 = cpuusage_write,
  8551. },
  8552. {
  8553. .name = "usage_percpu",
  8554. .read_seq_string = cpuacct_percpu_seq_read,
  8555. },
  8556. };
  8557. static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
  8558. {
  8559. return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
  8560. }
  8561. /*
  8562. * charge this task's execution time to its accounting group.
  8563. *
  8564. * called with rq->lock held.
  8565. */
  8566. static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
  8567. {
  8568. struct cpuacct *ca;
  8569. int cpu;
  8570. if (unlikely(!cpuacct_subsys.active))
  8571. return;
  8572. cpu = task_cpu(tsk);
  8573. ca = task_ca(tsk);
  8574. for (; ca; ca = ca->parent) {
  8575. u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
  8576. *cpuusage += cputime;
  8577. }
  8578. }
  8579. struct cgroup_subsys cpuacct_subsys = {
  8580. .name = "cpuacct",
  8581. .create = cpuacct_create,
  8582. .destroy = cpuacct_destroy,
  8583. .populate = cpuacct_populate,
  8584. .subsys_id = cpuacct_subsys_id,
  8585. };
  8586. #endif /* CONFIG_CGROUP_CPUACCT */