sched.c 232 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748
  1. /*
  2. * kernel/sched.c
  3. *
  4. * Kernel scheduler and related syscalls
  5. *
  6. * Copyright (C) 1991-2002 Linus Torvalds
  7. *
  8. * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and
  9. * make semaphores SMP safe
  10. * 1998-11-19 Implemented schedule_timeout() and related stuff
  11. * by Andrea Arcangeli
  12. * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:
  13. * hybrid priority-list and round-robin design with
  14. * an array-switch method of distributing timeslices
  15. * and per-CPU runqueues. Cleanups and useful suggestions
  16. * by Davide Libenzi, preemptible kernel bits by Robert Love.
  17. * 2003-09-03 Interactivity tuning by Con Kolivas.
  18. * 2004-04-02 Scheduler domains code by Nick Piggin
  19. * 2007-04-15 Work begun on replacing all interactivity tuning with a
  20. * fair scheduling design by Con Kolivas.
  21. * 2007-05-05 Load balancing (smp-nice) and other improvements
  22. * by Peter Williams
  23. * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith
  24. * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri
  25. * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins,
  26. * Thomas Gleixner, Mike Kravetz
  27. */
  28. #include <linux/mm.h>
  29. #include <linux/module.h>
  30. #include <linux/nmi.h>
  31. #include <linux/init.h>
  32. #include <linux/uaccess.h>
  33. #include <linux/highmem.h>
  34. #include <asm/mmu_context.h>
  35. #include <linux/interrupt.h>
  36. #include <linux/capability.h>
  37. #include <linux/completion.h>
  38. #include <linux/kernel_stat.h>
  39. #include <linux/debug_locks.h>
  40. #include <linux/perf_event.h>
  41. #include <linux/security.h>
  42. #include <linux/notifier.h>
  43. #include <linux/profile.h>
  44. #include <linux/freezer.h>
  45. #include <linux/vmalloc.h>
  46. #include <linux/blkdev.h>
  47. #include <linux/delay.h>
  48. #include <linux/pid_namespace.h>
  49. #include <linux/smp.h>
  50. #include <linux/threads.h>
  51. #include <linux/timer.h>
  52. #include <linux/rcupdate.h>
  53. #include <linux/cpu.h>
  54. #include <linux/cpuset.h>
  55. #include <linux/percpu.h>
  56. #include <linux/proc_fs.h>
  57. #include <linux/seq_file.h>
  58. #include <linux/stop_machine.h>
  59. #include <linux/sysctl.h>
  60. #include <linux/syscalls.h>
  61. #include <linux/times.h>
  62. #include <linux/tsacct_kern.h>
  63. #include <linux/kprobes.h>
  64. #include <linux/delayacct.h>
  65. #include <linux/unistd.h>
  66. #include <linux/pagemap.h>
  67. #include <linux/hrtimer.h>
  68. #include <linux/tick.h>
  69. #include <linux/debugfs.h>
  70. #include <linux/ctype.h>
  71. #include <linux/ftrace.h>
  72. #include <linux/slab.h>
  73. #include <asm/tlb.h>
  74. #include <asm/irq_regs.h>
  75. #include <asm/mutex.h>
  76. #ifdef CONFIG_PARAVIRT
  77. #include <asm/paravirt.h>
  78. #endif
  79. #include "sched_cpupri.h"
  80. #include "workqueue_sched.h"
  81. #include "sched_autogroup.h"
  82. #define CREATE_TRACE_POINTS
  83. #include <trace/events/sched.h>
  84. /*
  85. * Convert user-nice values [ -20 ... 0 ... 19 ]
  86. * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
  87. * and back.
  88. */
  89. #define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20)
  90. #define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20)
  91. #define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio)
  92. /*
  93. * 'User priority' is the nice value converted to something we
  94. * can work with better when scaling various scheduler parameters,
  95. * it's a [ 0 ... 39 ] range.
  96. */
  97. #define USER_PRIO(p) ((p)-MAX_RT_PRIO)
  98. #define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)
  99. #define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
  100. /*
  101. * Helpers for converting nanosecond timing to jiffy resolution
  102. */
  103. #define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
  104. #define NICE_0_LOAD SCHED_LOAD_SCALE
  105. #define NICE_0_SHIFT SCHED_LOAD_SHIFT
  106. /*
  107. * These are the 'tuning knobs' of the scheduler:
  108. *
  109. * default timeslice is 100 msecs (used only for SCHED_RR tasks).
  110. * Timeslices get refilled after they expire.
  111. */
  112. #define DEF_TIMESLICE (100 * HZ / 1000)
  113. /*
  114. * single value that denotes runtime == period, ie unlimited time.
  115. */
  116. #define RUNTIME_INF ((u64)~0ULL)
  117. static inline int rt_policy(int policy)
  118. {
  119. if (policy == SCHED_FIFO || policy == SCHED_RR)
  120. return 1;
  121. return 0;
  122. }
  123. static inline int task_has_rt_policy(struct task_struct *p)
  124. {
  125. return rt_policy(p->policy);
  126. }
  127. /*
  128. * This is the priority-queue data structure of the RT scheduling class:
  129. */
  130. struct rt_prio_array {
  131. DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
  132. struct list_head queue[MAX_RT_PRIO];
  133. };
  134. struct rt_bandwidth {
  135. /* nests inside the rq lock: */
  136. raw_spinlock_t rt_runtime_lock;
  137. ktime_t rt_period;
  138. u64 rt_runtime;
  139. struct hrtimer rt_period_timer;
  140. };
  141. static struct rt_bandwidth def_rt_bandwidth;
  142. static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
  143. static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
  144. {
  145. struct rt_bandwidth *rt_b =
  146. container_of(timer, struct rt_bandwidth, rt_period_timer);
  147. ktime_t now;
  148. int overrun;
  149. int idle = 0;
  150. for (;;) {
  151. now = hrtimer_cb_get_time(timer);
  152. overrun = hrtimer_forward(timer, now, rt_b->rt_period);
  153. if (!overrun)
  154. break;
  155. idle = do_sched_rt_period_timer(rt_b, overrun);
  156. }
  157. return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
  158. }
  159. static
  160. void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
  161. {
  162. rt_b->rt_period = ns_to_ktime(period);
  163. rt_b->rt_runtime = runtime;
  164. raw_spin_lock_init(&rt_b->rt_runtime_lock);
  165. hrtimer_init(&rt_b->rt_period_timer,
  166. CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  167. rt_b->rt_period_timer.function = sched_rt_period_timer;
  168. }
  169. static inline int rt_bandwidth_enabled(void)
  170. {
  171. return sysctl_sched_rt_runtime >= 0;
  172. }
  173. static void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)
  174. {
  175. unsigned long delta;
  176. ktime_t soft, hard, now;
  177. for (;;) {
  178. if (hrtimer_active(period_timer))
  179. break;
  180. now = hrtimer_cb_get_time(period_timer);
  181. hrtimer_forward(period_timer, now, period);
  182. soft = hrtimer_get_softexpires(period_timer);
  183. hard = hrtimer_get_expires(period_timer);
  184. delta = ktime_to_ns(ktime_sub(hard, soft));
  185. __hrtimer_start_range_ns(period_timer, soft, delta,
  186. HRTIMER_MODE_ABS_PINNED, 0);
  187. }
  188. }
  189. static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
  190. {
  191. if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
  192. return;
  193. if (hrtimer_active(&rt_b->rt_period_timer))
  194. return;
  195. raw_spin_lock(&rt_b->rt_runtime_lock);
  196. start_bandwidth_timer(&rt_b->rt_period_timer, rt_b->rt_period);
  197. raw_spin_unlock(&rt_b->rt_runtime_lock);
  198. }
  199. #ifdef CONFIG_RT_GROUP_SCHED
  200. static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
  201. {
  202. hrtimer_cancel(&rt_b->rt_period_timer);
  203. }
  204. #endif
  205. /*
  206. * sched_domains_mutex serializes calls to init_sched_domains,
  207. * detach_destroy_domains and partition_sched_domains.
  208. */
  209. static DEFINE_MUTEX(sched_domains_mutex);
  210. #ifdef CONFIG_CGROUP_SCHED
  211. #include <linux/cgroup.h>
  212. struct cfs_rq;
  213. static LIST_HEAD(task_groups);
  214. struct cfs_bandwidth {
  215. #ifdef CONFIG_CFS_BANDWIDTH
  216. raw_spinlock_t lock;
  217. ktime_t period;
  218. u64 quota, runtime;
  219. s64 hierarchal_quota;
  220. u64 runtime_expires;
  221. int idle, timer_active;
  222. struct hrtimer period_timer, slack_timer;
  223. struct list_head throttled_cfs_rq;
  224. /* statistics */
  225. int nr_periods, nr_throttled;
  226. u64 throttled_time;
  227. #endif
  228. };
  229. /* task group related information */
  230. struct task_group {
  231. struct cgroup_subsys_state css;
  232. #ifdef CONFIG_FAIR_GROUP_SCHED
  233. /* schedulable entities of this group on each cpu */
  234. struct sched_entity **se;
  235. /* runqueue "owned" by this group on each cpu */
  236. struct cfs_rq **cfs_rq;
  237. unsigned long shares;
  238. atomic_t load_weight;
  239. #endif
  240. #ifdef CONFIG_RT_GROUP_SCHED
  241. struct sched_rt_entity **rt_se;
  242. struct rt_rq **rt_rq;
  243. struct rt_bandwidth rt_bandwidth;
  244. #endif
  245. struct rcu_head rcu;
  246. struct list_head list;
  247. struct task_group *parent;
  248. struct list_head siblings;
  249. struct list_head children;
  250. #ifdef CONFIG_SCHED_AUTOGROUP
  251. struct autogroup *autogroup;
  252. #endif
  253. struct cfs_bandwidth cfs_bandwidth;
  254. };
  255. /* task_group_lock serializes the addition/removal of task groups */
  256. static DEFINE_SPINLOCK(task_group_lock);
  257. #ifdef CONFIG_FAIR_GROUP_SCHED
  258. # define ROOT_TASK_GROUP_LOAD NICE_0_LOAD
  259. /*
  260. * A weight of 0 or 1 can cause arithmetics problems.
  261. * A weight of a cfs_rq is the sum of weights of which entities
  262. * are queued on this cfs_rq, so a weight of a entity should not be
  263. * too large, so as the shares value of a task group.
  264. * (The default weight is 1024 - so there's no practical
  265. * limitation from this.)
  266. */
  267. #define MIN_SHARES (1UL << 1)
  268. #define MAX_SHARES (1UL << 18)
  269. static int root_task_group_load = ROOT_TASK_GROUP_LOAD;
  270. #endif
  271. /* Default task group.
  272. * Every task in system belong to this group at bootup.
  273. */
  274. struct task_group root_task_group;
  275. #endif /* CONFIG_CGROUP_SCHED */
  276. /* CFS-related fields in a runqueue */
  277. struct cfs_rq {
  278. struct load_weight load;
  279. unsigned long nr_running, h_nr_running;
  280. u64 exec_clock;
  281. u64 min_vruntime;
  282. #ifndef CONFIG_64BIT
  283. u64 min_vruntime_copy;
  284. #endif
  285. struct rb_root tasks_timeline;
  286. struct rb_node *rb_leftmost;
  287. struct list_head tasks;
  288. struct list_head *balance_iterator;
  289. /*
  290. * 'curr' points to currently running entity on this cfs_rq.
  291. * It is set to NULL otherwise (i.e when none are currently running).
  292. */
  293. struct sched_entity *curr, *next, *last, *skip;
  294. #ifdef CONFIG_SCHED_DEBUG
  295. unsigned int nr_spread_over;
  296. #endif
  297. #ifdef CONFIG_FAIR_GROUP_SCHED
  298. struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */
  299. /*
  300. * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
  301. * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
  302. * (like users, containers etc.)
  303. *
  304. * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
  305. * list is used during load balance.
  306. */
  307. int on_list;
  308. struct list_head leaf_cfs_rq_list;
  309. struct task_group *tg; /* group that "owns" this runqueue */
  310. #ifdef CONFIG_SMP
  311. /*
  312. * the part of load.weight contributed by tasks
  313. */
  314. unsigned long task_weight;
  315. /*
  316. * h_load = weight * f(tg)
  317. *
  318. * Where f(tg) is the recursive weight fraction assigned to
  319. * this group.
  320. */
  321. unsigned long h_load;
  322. /*
  323. * Maintaining per-cpu shares distribution for group scheduling
  324. *
  325. * load_stamp is the last time we updated the load average
  326. * load_last is the last time we updated the load average and saw load
  327. * load_unacc_exec_time is currently unaccounted execution time
  328. */
  329. u64 load_avg;
  330. u64 load_period;
  331. u64 load_stamp, load_last, load_unacc_exec_time;
  332. unsigned long load_contribution;
  333. #endif
  334. #ifdef CONFIG_CFS_BANDWIDTH
  335. int runtime_enabled;
  336. u64 runtime_expires;
  337. s64 runtime_remaining;
  338. u64 throttled_timestamp;
  339. int throttled, throttle_count;
  340. struct list_head throttled_list;
  341. #endif
  342. #endif
  343. };
  344. #ifdef CONFIG_FAIR_GROUP_SCHED
  345. #ifdef CONFIG_CFS_BANDWIDTH
  346. static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
  347. {
  348. return &tg->cfs_bandwidth;
  349. }
  350. static inline u64 default_cfs_period(void);
  351. static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun);
  352. static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b);
  353. static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
  354. {
  355. struct cfs_bandwidth *cfs_b =
  356. container_of(timer, struct cfs_bandwidth, slack_timer);
  357. do_sched_cfs_slack_timer(cfs_b);
  358. return HRTIMER_NORESTART;
  359. }
  360. static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
  361. {
  362. struct cfs_bandwidth *cfs_b =
  363. container_of(timer, struct cfs_bandwidth, period_timer);
  364. ktime_t now;
  365. int overrun;
  366. int idle = 0;
  367. for (;;) {
  368. now = hrtimer_cb_get_time(timer);
  369. overrun = hrtimer_forward(timer, now, cfs_b->period);
  370. if (!overrun)
  371. break;
  372. idle = do_sched_cfs_period_timer(cfs_b, overrun);
  373. }
  374. return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
  375. }
  376. static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
  377. {
  378. raw_spin_lock_init(&cfs_b->lock);
  379. cfs_b->runtime = 0;
  380. cfs_b->quota = RUNTIME_INF;
  381. cfs_b->period = ns_to_ktime(default_cfs_period());
  382. INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);
  383. hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  384. cfs_b->period_timer.function = sched_cfs_period_timer;
  385. hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  386. cfs_b->slack_timer.function = sched_cfs_slack_timer;
  387. }
  388. static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
  389. {
  390. cfs_rq->runtime_enabled = 0;
  391. INIT_LIST_HEAD(&cfs_rq->throttled_list);
  392. }
  393. /* requires cfs_b->lock, may release to reprogram timer */
  394. static void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
  395. {
  396. /*
  397. * The timer may be active because we're trying to set a new bandwidth
  398. * period or because we're racing with the tear-down path
  399. * (timer_active==0 becomes visible before the hrtimer call-back
  400. * terminates). In either case we ensure that it's re-programmed
  401. */
  402. while (unlikely(hrtimer_active(&cfs_b->period_timer))) {
  403. raw_spin_unlock(&cfs_b->lock);
  404. /* ensure cfs_b->lock is available while we wait */
  405. hrtimer_cancel(&cfs_b->period_timer);
  406. raw_spin_lock(&cfs_b->lock);
  407. /* if someone else restarted the timer then we're done */
  408. if (cfs_b->timer_active)
  409. return;
  410. }
  411. cfs_b->timer_active = 1;
  412. start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period);
  413. }
  414. static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
  415. {
  416. hrtimer_cancel(&cfs_b->period_timer);
  417. hrtimer_cancel(&cfs_b->slack_timer);
  418. }
  419. #else
  420. static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
  421. static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
  422. static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
  423. static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
  424. {
  425. return NULL;
  426. }
  427. #endif /* CONFIG_CFS_BANDWIDTH */
  428. #endif /* CONFIG_FAIR_GROUP_SCHED */
  429. /* Real-Time classes' related field in a runqueue: */
  430. struct rt_rq {
  431. struct rt_prio_array active;
  432. unsigned long rt_nr_running;
  433. #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
  434. struct {
  435. int curr; /* highest queued rt task prio */
  436. #ifdef CONFIG_SMP
  437. int next; /* next highest */
  438. #endif
  439. } highest_prio;
  440. #endif
  441. #ifdef CONFIG_SMP
  442. unsigned long rt_nr_migratory;
  443. unsigned long rt_nr_total;
  444. int overloaded;
  445. struct plist_head pushable_tasks;
  446. #endif
  447. int rt_throttled;
  448. u64 rt_time;
  449. u64 rt_runtime;
  450. /* Nests inside the rq lock: */
  451. raw_spinlock_t rt_runtime_lock;
  452. #ifdef CONFIG_RT_GROUP_SCHED
  453. unsigned long rt_nr_boosted;
  454. struct rq *rq;
  455. struct list_head leaf_rt_rq_list;
  456. struct task_group *tg;
  457. #endif
  458. };
  459. #ifdef CONFIG_SMP
  460. /*
  461. * We add the notion of a root-domain which will be used to define per-domain
  462. * variables. Each exclusive cpuset essentially defines an island domain by
  463. * fully partitioning the member cpus from any other cpuset. Whenever a new
  464. * exclusive cpuset is created, we also create and attach a new root-domain
  465. * object.
  466. *
  467. */
  468. struct root_domain {
  469. atomic_t refcount;
  470. atomic_t rto_count;
  471. struct rcu_head rcu;
  472. cpumask_var_t span;
  473. cpumask_var_t online;
  474. /*
  475. * The "RT overload" flag: it gets set if a CPU has more than
  476. * one runnable RT task.
  477. */
  478. cpumask_var_t rto_mask;
  479. struct cpupri cpupri;
  480. };
  481. /*
  482. * By default the system creates a single root-domain with all cpus as
  483. * members (mimicking the global state we have today).
  484. */
  485. static struct root_domain def_root_domain;
  486. #endif /* CONFIG_SMP */
  487. /*
  488. * This is the main, per-CPU runqueue data structure.
  489. *
  490. * Locking rule: those places that want to lock multiple runqueues
  491. * (such as the load balancing or the thread migration code), lock
  492. * acquire operations must be ordered by ascending &runqueue.
  493. */
  494. struct rq {
  495. /* runqueue lock: */
  496. raw_spinlock_t lock;
  497. /*
  498. * nr_running and cpu_load should be in the same cacheline because
  499. * remote CPUs use both these fields when doing load calculation.
  500. */
  501. unsigned long nr_running;
  502. #define CPU_LOAD_IDX_MAX 5
  503. unsigned long cpu_load[CPU_LOAD_IDX_MAX];
  504. unsigned long last_load_update_tick;
  505. #ifdef CONFIG_NO_HZ
  506. u64 nohz_stamp;
  507. unsigned char nohz_balance_kick;
  508. #endif
  509. int skip_clock_update;
  510. /* capture load from *all* tasks on this cpu: */
  511. struct load_weight load;
  512. unsigned long nr_load_updates;
  513. u64 nr_switches;
  514. struct cfs_rq cfs;
  515. struct rt_rq rt;
  516. #ifdef CONFIG_FAIR_GROUP_SCHED
  517. /* list of leaf cfs_rq on this cpu: */
  518. struct list_head leaf_cfs_rq_list;
  519. #endif
  520. #ifdef CONFIG_RT_GROUP_SCHED
  521. struct list_head leaf_rt_rq_list;
  522. #endif
  523. /*
  524. * This is part of a global counter where only the total sum
  525. * over all CPUs matters. A task can increase this counter on
  526. * one CPU and if it got migrated afterwards it may decrease
  527. * it on another CPU. Always updated under the runqueue lock:
  528. */
  529. unsigned long nr_uninterruptible;
  530. struct task_struct *curr, *idle, *stop;
  531. unsigned long next_balance;
  532. struct mm_struct *prev_mm;
  533. u64 clock;
  534. u64 clock_task;
  535. atomic_t nr_iowait;
  536. #ifdef CONFIG_SMP
  537. struct root_domain *rd;
  538. struct sched_domain *sd;
  539. unsigned long cpu_power;
  540. unsigned char idle_at_tick;
  541. /* For active balancing */
  542. int post_schedule;
  543. int active_balance;
  544. int push_cpu;
  545. struct cpu_stop_work active_balance_work;
  546. /* cpu of this runqueue: */
  547. int cpu;
  548. int online;
  549. u64 rt_avg;
  550. u64 age_stamp;
  551. u64 idle_stamp;
  552. u64 avg_idle;
  553. #endif
  554. #ifdef CONFIG_IRQ_TIME_ACCOUNTING
  555. u64 prev_irq_time;
  556. #endif
  557. #ifdef CONFIG_PARAVIRT
  558. u64 prev_steal_time;
  559. #endif
  560. #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
  561. u64 prev_steal_time_rq;
  562. #endif
  563. /* calc_load related fields */
  564. unsigned long calc_load_update;
  565. long calc_load_active;
  566. #ifdef CONFIG_SCHED_HRTICK
  567. #ifdef CONFIG_SMP
  568. int hrtick_csd_pending;
  569. struct call_single_data hrtick_csd;
  570. #endif
  571. struct hrtimer hrtick_timer;
  572. #endif
  573. #ifdef CONFIG_SCHEDSTATS
  574. /* latency stats */
  575. struct sched_info rq_sched_info;
  576. unsigned long long rq_cpu_time;
  577. /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
  578. /* sys_sched_yield() stats */
  579. unsigned int yld_count;
  580. /* schedule() stats */
  581. unsigned int sched_switch;
  582. unsigned int sched_count;
  583. unsigned int sched_goidle;
  584. /* try_to_wake_up() stats */
  585. unsigned int ttwu_count;
  586. unsigned int ttwu_local;
  587. #endif
  588. #ifdef CONFIG_SMP
  589. struct llist_head wake_list;
  590. #endif
  591. };
  592. static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
  593. static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
  594. static inline int cpu_of(struct rq *rq)
  595. {
  596. #ifdef CONFIG_SMP
  597. return rq->cpu;
  598. #else
  599. return 0;
  600. #endif
  601. }
  602. #define rcu_dereference_check_sched_domain(p) \
  603. rcu_dereference_check((p), \
  604. lockdep_is_held(&sched_domains_mutex))
  605. /*
  606. * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
  607. * See detach_destroy_domains: synchronize_sched for details.
  608. *
  609. * The domain tree of any CPU may only be accessed from within
  610. * preempt-disabled sections.
  611. */
  612. #define for_each_domain(cpu, __sd) \
  613. for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
  614. #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
  615. #define this_rq() (&__get_cpu_var(runqueues))
  616. #define task_rq(p) cpu_rq(task_cpu(p))
  617. #define cpu_curr(cpu) (cpu_rq(cpu)->curr)
  618. #define raw_rq() (&__raw_get_cpu_var(runqueues))
  619. #ifdef CONFIG_CGROUP_SCHED
  620. /*
  621. * Return the group to which this tasks belongs.
  622. *
  623. * We use task_subsys_state_check() and extend the RCU verification with
  624. * pi->lock and rq->lock because cpu_cgroup_attach() holds those locks for each
  625. * task it moves into the cgroup. Therefore by holding either of those locks,
  626. * we pin the task to the current cgroup.
  627. */
  628. static inline struct task_group *task_group(struct task_struct *p)
  629. {
  630. struct task_group *tg;
  631. struct cgroup_subsys_state *css;
  632. css = task_subsys_state_check(p, cpu_cgroup_subsys_id,
  633. lockdep_is_held(&p->pi_lock) ||
  634. lockdep_is_held(&task_rq(p)->lock));
  635. tg = container_of(css, struct task_group, css);
  636. return autogroup_task_group(p, tg);
  637. }
  638. /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
  639. static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
  640. {
  641. #ifdef CONFIG_FAIR_GROUP_SCHED
  642. p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
  643. p->se.parent = task_group(p)->se[cpu];
  644. #endif
  645. #ifdef CONFIG_RT_GROUP_SCHED
  646. p->rt.rt_rq = task_group(p)->rt_rq[cpu];
  647. p->rt.parent = task_group(p)->rt_se[cpu];
  648. #endif
  649. }
  650. #else /* CONFIG_CGROUP_SCHED */
  651. static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
  652. static inline struct task_group *task_group(struct task_struct *p)
  653. {
  654. return NULL;
  655. }
  656. #endif /* CONFIG_CGROUP_SCHED */
  657. static void update_rq_clock_task(struct rq *rq, s64 delta);
  658. static void update_rq_clock(struct rq *rq)
  659. {
  660. s64 delta;
  661. if (rq->skip_clock_update > 0)
  662. return;
  663. delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
  664. rq->clock += delta;
  665. update_rq_clock_task(rq, delta);
  666. }
  667. /*
  668. * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
  669. */
  670. #ifdef CONFIG_SCHED_DEBUG
  671. # define const_debug __read_mostly
  672. #else
  673. # define const_debug static const
  674. #endif
  675. /**
  676. * runqueue_is_locked - Returns true if the current cpu runqueue is locked
  677. * @cpu: the processor in question.
  678. *
  679. * This interface allows printk to be called with the runqueue lock
  680. * held and know whether or not it is OK to wake up the klogd.
  681. */
  682. int runqueue_is_locked(int cpu)
  683. {
  684. return raw_spin_is_locked(&cpu_rq(cpu)->lock);
  685. }
  686. /*
  687. * Debugging: various feature bits
  688. */
  689. #define SCHED_FEAT(name, enabled) \
  690. __SCHED_FEAT_##name ,
  691. enum {
  692. #include "sched_features.h"
  693. };
  694. #undef SCHED_FEAT
  695. #define SCHED_FEAT(name, enabled) \
  696. (1UL << __SCHED_FEAT_##name) * enabled |
  697. const_debug unsigned int sysctl_sched_features =
  698. #include "sched_features.h"
  699. 0;
  700. #undef SCHED_FEAT
  701. #ifdef CONFIG_SCHED_DEBUG
  702. #define SCHED_FEAT(name, enabled) \
  703. #name ,
  704. static __read_mostly char *sched_feat_names[] = {
  705. #include "sched_features.h"
  706. NULL
  707. };
  708. #undef SCHED_FEAT
  709. static int sched_feat_show(struct seq_file *m, void *v)
  710. {
  711. int i;
  712. for (i = 0; sched_feat_names[i]; i++) {
  713. if (!(sysctl_sched_features & (1UL << i)))
  714. seq_puts(m, "NO_");
  715. seq_printf(m, "%s ", sched_feat_names[i]);
  716. }
  717. seq_puts(m, "\n");
  718. return 0;
  719. }
  720. static ssize_t
  721. sched_feat_write(struct file *filp, const char __user *ubuf,
  722. size_t cnt, loff_t *ppos)
  723. {
  724. char buf[64];
  725. char *cmp;
  726. int neg = 0;
  727. int i;
  728. if (cnt > 63)
  729. cnt = 63;
  730. if (copy_from_user(&buf, ubuf, cnt))
  731. return -EFAULT;
  732. buf[cnt] = 0;
  733. cmp = strstrip(buf);
  734. if (strncmp(cmp, "NO_", 3) == 0) {
  735. neg = 1;
  736. cmp += 3;
  737. }
  738. for (i = 0; sched_feat_names[i]; i++) {
  739. if (strcmp(cmp, sched_feat_names[i]) == 0) {
  740. if (neg)
  741. sysctl_sched_features &= ~(1UL << i);
  742. else
  743. sysctl_sched_features |= (1UL << i);
  744. break;
  745. }
  746. }
  747. if (!sched_feat_names[i])
  748. return -EINVAL;
  749. *ppos += cnt;
  750. return cnt;
  751. }
  752. static int sched_feat_open(struct inode *inode, struct file *filp)
  753. {
  754. return single_open(filp, sched_feat_show, NULL);
  755. }
  756. static const struct file_operations sched_feat_fops = {
  757. .open = sched_feat_open,
  758. .write = sched_feat_write,
  759. .read = seq_read,
  760. .llseek = seq_lseek,
  761. .release = single_release,
  762. };
  763. static __init int sched_init_debug(void)
  764. {
  765. debugfs_create_file("sched_features", 0644, NULL, NULL,
  766. &sched_feat_fops);
  767. return 0;
  768. }
  769. late_initcall(sched_init_debug);
  770. #endif
  771. #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
  772. /*
  773. * Number of tasks to iterate in a single balance run.
  774. * Limited because this is done with IRQs disabled.
  775. */
  776. const_debug unsigned int sysctl_sched_nr_migrate = 32;
  777. /*
  778. * period over which we average the RT time consumption, measured
  779. * in ms.
  780. *
  781. * default: 1s
  782. */
  783. const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
  784. /*
  785. * period over which we measure -rt task cpu usage in us.
  786. * default: 1s
  787. */
  788. unsigned int sysctl_sched_rt_period = 1000000;
  789. static __read_mostly int scheduler_running;
  790. /*
  791. * part of the period that we allow rt tasks to run in us.
  792. * default: 0.95s
  793. */
  794. int sysctl_sched_rt_runtime = 950000;
  795. static inline u64 global_rt_period(void)
  796. {
  797. return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
  798. }
  799. static inline u64 global_rt_runtime(void)
  800. {
  801. if (sysctl_sched_rt_runtime < 0)
  802. return RUNTIME_INF;
  803. return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
  804. }
  805. #ifndef prepare_arch_switch
  806. # define prepare_arch_switch(next) do { } while (0)
  807. #endif
  808. #ifndef finish_arch_switch
  809. # define finish_arch_switch(prev) do { } while (0)
  810. #endif
  811. static inline int task_current(struct rq *rq, struct task_struct *p)
  812. {
  813. return rq->curr == p;
  814. }
  815. static inline int task_running(struct rq *rq, struct task_struct *p)
  816. {
  817. #ifdef CONFIG_SMP
  818. return p->on_cpu;
  819. #else
  820. return task_current(rq, p);
  821. #endif
  822. }
  823. #ifndef __ARCH_WANT_UNLOCKED_CTXSW
  824. static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
  825. {
  826. #ifdef CONFIG_SMP
  827. /*
  828. * We can optimise this out completely for !SMP, because the
  829. * SMP rebalancing from interrupt is the only thing that cares
  830. * here.
  831. */
  832. next->on_cpu = 1;
  833. #endif
  834. }
  835. static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
  836. {
  837. #ifdef CONFIG_SMP
  838. /*
  839. * After ->on_cpu is cleared, the task can be moved to a different CPU.
  840. * We must ensure this doesn't happen until the switch is completely
  841. * finished.
  842. */
  843. smp_wmb();
  844. prev->on_cpu = 0;
  845. #endif
  846. #ifdef CONFIG_DEBUG_SPINLOCK
  847. /* this is a valid case when another task releases the spinlock */
  848. rq->lock.owner = current;
  849. #endif
  850. /*
  851. * If we are tracking spinlock dependencies then we have to
  852. * fix up the runqueue lock - which gets 'carried over' from
  853. * prev into current:
  854. */
  855. spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
  856. raw_spin_unlock_irq(&rq->lock);
  857. }
  858. #else /* __ARCH_WANT_UNLOCKED_CTXSW */
  859. static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
  860. {
  861. #ifdef CONFIG_SMP
  862. /*
  863. * We can optimise this out completely for !SMP, because the
  864. * SMP rebalancing from interrupt is the only thing that cares
  865. * here.
  866. */
  867. next->on_cpu = 1;
  868. #endif
  869. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  870. raw_spin_unlock_irq(&rq->lock);
  871. #else
  872. raw_spin_unlock(&rq->lock);
  873. #endif
  874. }
  875. static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
  876. {
  877. #ifdef CONFIG_SMP
  878. /*
  879. * After ->on_cpu is cleared, the task can be moved to a different CPU.
  880. * We must ensure this doesn't happen until the switch is completely
  881. * finished.
  882. */
  883. smp_wmb();
  884. prev->on_cpu = 0;
  885. #endif
  886. #ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  887. local_irq_enable();
  888. #endif
  889. }
  890. #endif /* __ARCH_WANT_UNLOCKED_CTXSW */
  891. /*
  892. * __task_rq_lock - lock the rq @p resides on.
  893. */
  894. static inline struct rq *__task_rq_lock(struct task_struct *p)
  895. __acquires(rq->lock)
  896. {
  897. struct rq *rq;
  898. lockdep_assert_held(&p->pi_lock);
  899. for (;;) {
  900. rq = task_rq(p);
  901. raw_spin_lock(&rq->lock);
  902. if (likely(rq == task_rq(p)))
  903. return rq;
  904. raw_spin_unlock(&rq->lock);
  905. }
  906. }
  907. /*
  908. * task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
  909. */
  910. static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
  911. __acquires(p->pi_lock)
  912. __acquires(rq->lock)
  913. {
  914. struct rq *rq;
  915. for (;;) {
  916. raw_spin_lock_irqsave(&p->pi_lock, *flags);
  917. rq = task_rq(p);
  918. raw_spin_lock(&rq->lock);
  919. if (likely(rq == task_rq(p)))
  920. return rq;
  921. raw_spin_unlock(&rq->lock);
  922. raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
  923. }
  924. }
  925. static void __task_rq_unlock(struct rq *rq)
  926. __releases(rq->lock)
  927. {
  928. raw_spin_unlock(&rq->lock);
  929. }
  930. static inline void
  931. task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags)
  932. __releases(rq->lock)
  933. __releases(p->pi_lock)
  934. {
  935. raw_spin_unlock(&rq->lock);
  936. raw_spin_unlock_irqrestore(&p->pi_lock, *flags);
  937. }
  938. /*
  939. * this_rq_lock - lock this runqueue and disable interrupts.
  940. */
  941. static struct rq *this_rq_lock(void)
  942. __acquires(rq->lock)
  943. {
  944. struct rq *rq;
  945. local_irq_disable();
  946. rq = this_rq();
  947. raw_spin_lock(&rq->lock);
  948. return rq;
  949. }
  950. #ifdef CONFIG_SCHED_HRTICK
  951. /*
  952. * Use HR-timers to deliver accurate preemption points.
  953. *
  954. * Its all a bit involved since we cannot program an hrt while holding the
  955. * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
  956. * reschedule event.
  957. *
  958. * When we get rescheduled we reprogram the hrtick_timer outside of the
  959. * rq->lock.
  960. */
  961. /*
  962. * Use hrtick when:
  963. * - enabled by features
  964. * - hrtimer is actually high res
  965. */
  966. static inline int hrtick_enabled(struct rq *rq)
  967. {
  968. if (!sched_feat(HRTICK))
  969. return 0;
  970. if (!cpu_active(cpu_of(rq)))
  971. return 0;
  972. return hrtimer_is_hres_active(&rq->hrtick_timer);
  973. }
  974. static void hrtick_clear(struct rq *rq)
  975. {
  976. if (hrtimer_active(&rq->hrtick_timer))
  977. hrtimer_cancel(&rq->hrtick_timer);
  978. }
  979. /*
  980. * High-resolution timer tick.
  981. * Runs from hardirq context with interrupts disabled.
  982. */
  983. static enum hrtimer_restart hrtick(struct hrtimer *timer)
  984. {
  985. struct rq *rq = container_of(timer, struct rq, hrtick_timer);
  986. WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
  987. raw_spin_lock(&rq->lock);
  988. update_rq_clock(rq);
  989. rq->curr->sched_class->task_tick(rq, rq->curr, 1);
  990. raw_spin_unlock(&rq->lock);
  991. return HRTIMER_NORESTART;
  992. }
  993. #ifdef CONFIG_SMP
  994. /*
  995. * called from hardirq (IPI) context
  996. */
  997. static void __hrtick_start(void *arg)
  998. {
  999. struct rq *rq = arg;
  1000. raw_spin_lock(&rq->lock);
  1001. hrtimer_restart(&rq->hrtick_timer);
  1002. rq->hrtick_csd_pending = 0;
  1003. raw_spin_unlock(&rq->lock);
  1004. }
  1005. /*
  1006. * Called to set the hrtick timer state.
  1007. *
  1008. * called with rq->lock held and irqs disabled
  1009. */
  1010. static void hrtick_start(struct rq *rq, u64 delay)
  1011. {
  1012. struct hrtimer *timer = &rq->hrtick_timer;
  1013. ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
  1014. hrtimer_set_expires(timer, time);
  1015. if (rq == this_rq()) {
  1016. hrtimer_restart(timer);
  1017. } else if (!rq->hrtick_csd_pending) {
  1018. __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);
  1019. rq->hrtick_csd_pending = 1;
  1020. }
  1021. }
  1022. static int
  1023. hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
  1024. {
  1025. int cpu = (int)(long)hcpu;
  1026. switch (action) {
  1027. case CPU_UP_CANCELED:
  1028. case CPU_UP_CANCELED_FROZEN:
  1029. case CPU_DOWN_PREPARE:
  1030. case CPU_DOWN_PREPARE_FROZEN:
  1031. case CPU_DEAD:
  1032. case CPU_DEAD_FROZEN:
  1033. hrtick_clear(cpu_rq(cpu));
  1034. return NOTIFY_OK;
  1035. }
  1036. return NOTIFY_DONE;
  1037. }
  1038. static __init void init_hrtick(void)
  1039. {
  1040. hotcpu_notifier(hotplug_hrtick, 0);
  1041. }
  1042. #else
  1043. /*
  1044. * Called to set the hrtick timer state.
  1045. *
  1046. * called with rq->lock held and irqs disabled
  1047. */
  1048. static void hrtick_start(struct rq *rq, u64 delay)
  1049. {
  1050. __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
  1051. HRTIMER_MODE_REL_PINNED, 0);
  1052. }
  1053. static inline void init_hrtick(void)
  1054. {
  1055. }
  1056. #endif /* CONFIG_SMP */
  1057. static void init_rq_hrtick(struct rq *rq)
  1058. {
  1059. #ifdef CONFIG_SMP
  1060. rq->hrtick_csd_pending = 0;
  1061. rq->hrtick_csd.flags = 0;
  1062. rq->hrtick_csd.func = __hrtick_start;
  1063. rq->hrtick_csd.info = rq;
  1064. #endif
  1065. hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  1066. rq->hrtick_timer.function = hrtick;
  1067. }
  1068. #else /* CONFIG_SCHED_HRTICK */
  1069. static inline void hrtick_clear(struct rq *rq)
  1070. {
  1071. }
  1072. static inline void init_rq_hrtick(struct rq *rq)
  1073. {
  1074. }
  1075. static inline void init_hrtick(void)
  1076. {
  1077. }
  1078. #endif /* CONFIG_SCHED_HRTICK */
  1079. /*
  1080. * resched_task - mark a task 'to be rescheduled now'.
  1081. *
  1082. * On UP this means the setting of the need_resched flag, on SMP it
  1083. * might also involve a cross-CPU call to trigger the scheduler on
  1084. * the target CPU.
  1085. */
  1086. #ifdef CONFIG_SMP
  1087. #ifndef tsk_is_polling
  1088. #define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
  1089. #endif
  1090. static void resched_task(struct task_struct *p)
  1091. {
  1092. int cpu;
  1093. assert_raw_spin_locked(&task_rq(p)->lock);
  1094. if (test_tsk_need_resched(p))
  1095. return;
  1096. set_tsk_need_resched(p);
  1097. cpu = task_cpu(p);
  1098. if (cpu == smp_processor_id())
  1099. return;
  1100. /* NEED_RESCHED must be visible before we test polling */
  1101. smp_mb();
  1102. if (!tsk_is_polling(p))
  1103. smp_send_reschedule(cpu);
  1104. }
  1105. static void resched_cpu(int cpu)
  1106. {
  1107. struct rq *rq = cpu_rq(cpu);
  1108. unsigned long flags;
  1109. if (!raw_spin_trylock_irqsave(&rq->lock, flags))
  1110. return;
  1111. resched_task(cpu_curr(cpu));
  1112. raw_spin_unlock_irqrestore(&rq->lock, flags);
  1113. }
  1114. #ifdef CONFIG_NO_HZ
  1115. /*
  1116. * In the semi idle case, use the nearest busy cpu for migrating timers
  1117. * from an idle cpu. This is good for power-savings.
  1118. *
  1119. * We don't do similar optimization for completely idle system, as
  1120. * selecting an idle cpu will add more delays to the timers than intended
  1121. * (as that cpu's timer base may not be uptodate wrt jiffies etc).
  1122. */
  1123. int get_nohz_timer_target(void)
  1124. {
  1125. int cpu = smp_processor_id();
  1126. int i;
  1127. struct sched_domain *sd;
  1128. rcu_read_lock();
  1129. for_each_domain(cpu, sd) {
  1130. for_each_cpu(i, sched_domain_span(sd)) {
  1131. if (!idle_cpu(i)) {
  1132. cpu = i;
  1133. goto unlock;
  1134. }
  1135. }
  1136. }
  1137. unlock:
  1138. rcu_read_unlock();
  1139. return cpu;
  1140. }
  1141. /*
  1142. * When add_timer_on() enqueues a timer into the timer wheel of an
  1143. * idle CPU then this timer might expire before the next timer event
  1144. * which is scheduled to wake up that CPU. In case of a completely
  1145. * idle system the next event might even be infinite time into the
  1146. * future. wake_up_idle_cpu() ensures that the CPU is woken up and
  1147. * leaves the inner idle loop so the newly added timer is taken into
  1148. * account when the CPU goes back to idle and evaluates the timer
  1149. * wheel for the next timer event.
  1150. */
  1151. void wake_up_idle_cpu(int cpu)
  1152. {
  1153. struct rq *rq = cpu_rq(cpu);
  1154. if (cpu == smp_processor_id())
  1155. return;
  1156. /*
  1157. * This is safe, as this function is called with the timer
  1158. * wheel base lock of (cpu) held. When the CPU is on the way
  1159. * to idle and has not yet set rq->curr to idle then it will
  1160. * be serialized on the timer wheel base lock and take the new
  1161. * timer into account automatically.
  1162. */
  1163. if (rq->curr != rq->idle)
  1164. return;
  1165. /*
  1166. * We can set TIF_RESCHED on the idle task of the other CPU
  1167. * lockless. The worst case is that the other CPU runs the
  1168. * idle task through an additional NOOP schedule()
  1169. */
  1170. set_tsk_need_resched(rq->idle);
  1171. /* NEED_RESCHED must be visible before we test polling */
  1172. smp_mb();
  1173. if (!tsk_is_polling(rq->idle))
  1174. smp_send_reschedule(cpu);
  1175. }
  1176. #endif /* CONFIG_NO_HZ */
  1177. static u64 sched_avg_period(void)
  1178. {
  1179. return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2;
  1180. }
  1181. static void sched_avg_update(struct rq *rq)
  1182. {
  1183. s64 period = sched_avg_period();
  1184. while ((s64)(rq->clock - rq->age_stamp) > period) {
  1185. /*
  1186. * Inline assembly required to prevent the compiler
  1187. * optimising this loop into a divmod call.
  1188. * See __iter_div_u64_rem() for another example of this.
  1189. */
  1190. asm("" : "+rm" (rq->age_stamp));
  1191. rq->age_stamp += period;
  1192. rq->rt_avg /= 2;
  1193. }
  1194. }
  1195. static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
  1196. {
  1197. rq->rt_avg += rt_delta;
  1198. sched_avg_update(rq);
  1199. }
  1200. #else /* !CONFIG_SMP */
  1201. static void resched_task(struct task_struct *p)
  1202. {
  1203. assert_raw_spin_locked(&task_rq(p)->lock);
  1204. set_tsk_need_resched(p);
  1205. }
  1206. static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
  1207. {
  1208. }
  1209. static void sched_avg_update(struct rq *rq)
  1210. {
  1211. }
  1212. #endif /* CONFIG_SMP */
  1213. #if BITS_PER_LONG == 32
  1214. # define WMULT_CONST (~0UL)
  1215. #else
  1216. # define WMULT_CONST (1UL << 32)
  1217. #endif
  1218. #define WMULT_SHIFT 32
  1219. /*
  1220. * Shift right and round:
  1221. */
  1222. #define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
  1223. /*
  1224. * delta *= weight / lw
  1225. */
  1226. static unsigned long
  1227. calc_delta_mine(unsigned long delta_exec, unsigned long weight,
  1228. struct load_weight *lw)
  1229. {
  1230. u64 tmp;
  1231. /*
  1232. * weight can be less than 2^SCHED_LOAD_RESOLUTION for task group sched
  1233. * entities since MIN_SHARES = 2. Treat weight as 1 if less than
  1234. * 2^SCHED_LOAD_RESOLUTION.
  1235. */
  1236. if (likely(weight > (1UL << SCHED_LOAD_RESOLUTION)))
  1237. tmp = (u64)delta_exec * scale_load_down(weight);
  1238. else
  1239. tmp = (u64)delta_exec;
  1240. if (!lw->inv_weight) {
  1241. unsigned long w = scale_load_down(lw->weight);
  1242. if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
  1243. lw->inv_weight = 1;
  1244. else if (unlikely(!w))
  1245. lw->inv_weight = WMULT_CONST;
  1246. else
  1247. lw->inv_weight = WMULT_CONST / w;
  1248. }
  1249. /*
  1250. * Check whether we'd overflow the 64-bit multiplication:
  1251. */
  1252. if (unlikely(tmp > WMULT_CONST))
  1253. tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
  1254. WMULT_SHIFT/2);
  1255. else
  1256. tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
  1257. return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
  1258. }
  1259. static inline void update_load_add(struct load_weight *lw, unsigned long inc)
  1260. {
  1261. lw->weight += inc;
  1262. lw->inv_weight = 0;
  1263. }
  1264. static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
  1265. {
  1266. lw->weight -= dec;
  1267. lw->inv_weight = 0;
  1268. }
  1269. static inline void update_load_set(struct load_weight *lw, unsigned long w)
  1270. {
  1271. lw->weight = w;
  1272. lw->inv_weight = 0;
  1273. }
  1274. /*
  1275. * To aid in avoiding the subversion of "niceness" due to uneven distribution
  1276. * of tasks with abnormal "nice" values across CPUs the contribution that
  1277. * each task makes to its run queue's load is weighted according to its
  1278. * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
  1279. * scaled version of the new time slice allocation that they receive on time
  1280. * slice expiry etc.
  1281. */
  1282. #define WEIGHT_IDLEPRIO 3
  1283. #define WMULT_IDLEPRIO 1431655765
  1284. /*
  1285. * Nice levels are multiplicative, with a gentle 10% change for every
  1286. * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
  1287. * nice 1, it will get ~10% less CPU time than another CPU-bound task
  1288. * that remained on nice 0.
  1289. *
  1290. * The "10% effect" is relative and cumulative: from _any_ nice level,
  1291. * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
  1292. * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
  1293. * If a task goes up by ~10% and another task goes down by ~10% then
  1294. * the relative distance between them is ~25%.)
  1295. */
  1296. static const int prio_to_weight[40] = {
  1297. /* -20 */ 88761, 71755, 56483, 46273, 36291,
  1298. /* -15 */ 29154, 23254, 18705, 14949, 11916,
  1299. /* -10 */ 9548, 7620, 6100, 4904, 3906,
  1300. /* -5 */ 3121, 2501, 1991, 1586, 1277,
  1301. /* 0 */ 1024, 820, 655, 526, 423,
  1302. /* 5 */ 335, 272, 215, 172, 137,
  1303. /* 10 */ 110, 87, 70, 56, 45,
  1304. /* 15 */ 36, 29, 23, 18, 15,
  1305. };
  1306. /*
  1307. * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
  1308. *
  1309. * In cases where the weight does not change often, we can use the
  1310. * precalculated inverse to speed up arithmetics by turning divisions
  1311. * into multiplications:
  1312. */
  1313. static const u32 prio_to_wmult[40] = {
  1314. /* -20 */ 48388, 59856, 76040, 92818, 118348,
  1315. /* -15 */ 147320, 184698, 229616, 287308, 360437,
  1316. /* -10 */ 449829, 563644, 704093, 875809, 1099582,
  1317. /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
  1318. /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
  1319. /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
  1320. /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
  1321. /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
  1322. };
  1323. /* Time spent by the tasks of the cpu accounting group executing in ... */
  1324. enum cpuacct_stat_index {
  1325. CPUACCT_STAT_USER, /* ... user mode */
  1326. CPUACCT_STAT_SYSTEM, /* ... kernel mode */
  1327. CPUACCT_STAT_NSTATS,
  1328. };
  1329. #ifdef CONFIG_CGROUP_CPUACCT
  1330. static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
  1331. static void cpuacct_update_stats(struct task_struct *tsk,
  1332. enum cpuacct_stat_index idx, cputime_t val);
  1333. #else
  1334. static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
  1335. static inline void cpuacct_update_stats(struct task_struct *tsk,
  1336. enum cpuacct_stat_index idx, cputime_t val) {}
  1337. #endif
  1338. static inline void inc_cpu_load(struct rq *rq, unsigned long load)
  1339. {
  1340. update_load_add(&rq->load, load);
  1341. }
  1342. static inline void dec_cpu_load(struct rq *rq, unsigned long load)
  1343. {
  1344. update_load_sub(&rq->load, load);
  1345. }
  1346. #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
  1347. (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
  1348. typedef int (*tg_visitor)(struct task_group *, void *);
  1349. /*
  1350. * Iterate task_group tree rooted at *from, calling @down when first entering a
  1351. * node and @up when leaving it for the final time.
  1352. *
  1353. * Caller must hold rcu_lock or sufficient equivalent.
  1354. */
  1355. static int walk_tg_tree_from(struct task_group *from,
  1356. tg_visitor down, tg_visitor up, void *data)
  1357. {
  1358. struct task_group *parent, *child;
  1359. int ret;
  1360. parent = from;
  1361. down:
  1362. ret = (*down)(parent, data);
  1363. if (ret)
  1364. goto out;
  1365. list_for_each_entry_rcu(child, &parent->children, siblings) {
  1366. parent = child;
  1367. goto down;
  1368. up:
  1369. continue;
  1370. }
  1371. ret = (*up)(parent, data);
  1372. if (ret || parent == from)
  1373. goto out;
  1374. child = parent;
  1375. parent = parent->parent;
  1376. if (parent)
  1377. goto up;
  1378. out:
  1379. return ret;
  1380. }
  1381. /*
  1382. * Iterate the full tree, calling @down when first entering a node and @up when
  1383. * leaving it for the final time.
  1384. *
  1385. * Caller must hold rcu_lock or sufficient equivalent.
  1386. */
  1387. static inline int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
  1388. {
  1389. return walk_tg_tree_from(&root_task_group, down, up, data);
  1390. }
  1391. static int tg_nop(struct task_group *tg, void *data)
  1392. {
  1393. return 0;
  1394. }
  1395. #endif
  1396. #ifdef CONFIG_SMP
  1397. /* Used instead of source_load when we know the type == 0 */
  1398. static unsigned long weighted_cpuload(const int cpu)
  1399. {
  1400. return cpu_rq(cpu)->load.weight;
  1401. }
  1402. /*
  1403. * Return a low guess at the load of a migration-source cpu weighted
  1404. * according to the scheduling class and "nice" value.
  1405. *
  1406. * We want to under-estimate the load of migration sources, to
  1407. * balance conservatively.
  1408. */
  1409. static unsigned long source_load(int cpu, int type)
  1410. {
  1411. struct rq *rq = cpu_rq(cpu);
  1412. unsigned long total = weighted_cpuload(cpu);
  1413. if (type == 0 || !sched_feat(LB_BIAS))
  1414. return total;
  1415. return min(rq->cpu_load[type-1], total);
  1416. }
  1417. /*
  1418. * Return a high guess at the load of a migration-target cpu weighted
  1419. * according to the scheduling class and "nice" value.
  1420. */
  1421. static unsigned long target_load(int cpu, int type)
  1422. {
  1423. struct rq *rq = cpu_rq(cpu);
  1424. unsigned long total = weighted_cpuload(cpu);
  1425. if (type == 0 || !sched_feat(LB_BIAS))
  1426. return total;
  1427. return max(rq->cpu_load[type-1], total);
  1428. }
  1429. static unsigned long power_of(int cpu)
  1430. {
  1431. return cpu_rq(cpu)->cpu_power;
  1432. }
  1433. static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
  1434. static unsigned long cpu_avg_load_per_task(int cpu)
  1435. {
  1436. struct rq *rq = cpu_rq(cpu);
  1437. unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
  1438. if (nr_running)
  1439. return rq->load.weight / nr_running;
  1440. return 0;
  1441. }
  1442. #ifdef CONFIG_PREEMPT
  1443. static void double_rq_lock(struct rq *rq1, struct rq *rq2);
  1444. /*
  1445. * fair double_lock_balance: Safely acquires both rq->locks in a fair
  1446. * way at the expense of forcing extra atomic operations in all
  1447. * invocations. This assures that the double_lock is acquired using the
  1448. * same underlying policy as the spinlock_t on this architecture, which
  1449. * reduces latency compared to the unfair variant below. However, it
  1450. * also adds more overhead and therefore may reduce throughput.
  1451. */
  1452. static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1453. __releases(this_rq->lock)
  1454. __acquires(busiest->lock)
  1455. __acquires(this_rq->lock)
  1456. {
  1457. raw_spin_unlock(&this_rq->lock);
  1458. double_rq_lock(this_rq, busiest);
  1459. return 1;
  1460. }
  1461. #else
  1462. /*
  1463. * Unfair double_lock_balance: Optimizes throughput at the expense of
  1464. * latency by eliminating extra atomic operations when the locks are
  1465. * already in proper order on entry. This favors lower cpu-ids and will
  1466. * grant the double lock to lower cpus over higher ids under contention,
  1467. * regardless of entry order into the function.
  1468. */
  1469. static int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1470. __releases(this_rq->lock)
  1471. __acquires(busiest->lock)
  1472. __acquires(this_rq->lock)
  1473. {
  1474. int ret = 0;
  1475. if (unlikely(!raw_spin_trylock(&busiest->lock))) {
  1476. if (busiest < this_rq) {
  1477. raw_spin_unlock(&this_rq->lock);
  1478. raw_spin_lock(&busiest->lock);
  1479. raw_spin_lock_nested(&this_rq->lock,
  1480. SINGLE_DEPTH_NESTING);
  1481. ret = 1;
  1482. } else
  1483. raw_spin_lock_nested(&busiest->lock,
  1484. SINGLE_DEPTH_NESTING);
  1485. }
  1486. return ret;
  1487. }
  1488. #endif /* CONFIG_PREEMPT */
  1489. /*
  1490. * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
  1491. */
  1492. static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
  1493. {
  1494. if (unlikely(!irqs_disabled())) {
  1495. /* printk() doesn't work good under rq->lock */
  1496. raw_spin_unlock(&this_rq->lock);
  1497. BUG_ON(1);
  1498. }
  1499. return _double_lock_balance(this_rq, busiest);
  1500. }
  1501. static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
  1502. __releases(busiest->lock)
  1503. {
  1504. raw_spin_unlock(&busiest->lock);
  1505. lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
  1506. }
  1507. /*
  1508. * double_rq_lock - safely lock two runqueues
  1509. *
  1510. * Note this does not disable interrupts like task_rq_lock,
  1511. * you need to do so manually before calling.
  1512. */
  1513. static void double_rq_lock(struct rq *rq1, struct rq *rq2)
  1514. __acquires(rq1->lock)
  1515. __acquires(rq2->lock)
  1516. {
  1517. BUG_ON(!irqs_disabled());
  1518. if (rq1 == rq2) {
  1519. raw_spin_lock(&rq1->lock);
  1520. __acquire(rq2->lock); /* Fake it out ;) */
  1521. } else {
  1522. if (rq1 < rq2) {
  1523. raw_spin_lock(&rq1->lock);
  1524. raw_spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
  1525. } else {
  1526. raw_spin_lock(&rq2->lock);
  1527. raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
  1528. }
  1529. }
  1530. }
  1531. /*
  1532. * double_rq_unlock - safely unlock two runqueues
  1533. *
  1534. * Note this does not restore interrupts like task_rq_unlock,
  1535. * you need to do so manually after calling.
  1536. */
  1537. static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
  1538. __releases(rq1->lock)
  1539. __releases(rq2->lock)
  1540. {
  1541. raw_spin_unlock(&rq1->lock);
  1542. if (rq1 != rq2)
  1543. raw_spin_unlock(&rq2->lock);
  1544. else
  1545. __release(rq2->lock);
  1546. }
  1547. #else /* CONFIG_SMP */
  1548. /*
  1549. * double_rq_lock - safely lock two runqueues
  1550. *
  1551. * Note this does not disable interrupts like task_rq_lock,
  1552. * you need to do so manually before calling.
  1553. */
  1554. static void double_rq_lock(struct rq *rq1, struct rq *rq2)
  1555. __acquires(rq1->lock)
  1556. __acquires(rq2->lock)
  1557. {
  1558. BUG_ON(!irqs_disabled());
  1559. BUG_ON(rq1 != rq2);
  1560. raw_spin_lock(&rq1->lock);
  1561. __acquire(rq2->lock); /* Fake it out ;) */
  1562. }
  1563. /*
  1564. * double_rq_unlock - safely unlock two runqueues
  1565. *
  1566. * Note this does not restore interrupts like task_rq_unlock,
  1567. * you need to do so manually after calling.
  1568. */
  1569. static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
  1570. __releases(rq1->lock)
  1571. __releases(rq2->lock)
  1572. {
  1573. BUG_ON(rq1 != rq2);
  1574. raw_spin_unlock(&rq1->lock);
  1575. __release(rq2->lock);
  1576. }
  1577. #endif
  1578. static void calc_load_account_idle(struct rq *this_rq);
  1579. static void update_sysctl(void);
  1580. static int get_update_sysctl_factor(void);
  1581. static void update_cpu_load(struct rq *this_rq);
  1582. static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
  1583. {
  1584. set_task_rq(p, cpu);
  1585. #ifdef CONFIG_SMP
  1586. /*
  1587. * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
  1588. * successfuly executed on another CPU. We must ensure that updates of
  1589. * per-task data have been completed by this moment.
  1590. */
  1591. smp_wmb();
  1592. task_thread_info(p)->cpu = cpu;
  1593. #endif
  1594. }
  1595. static const struct sched_class rt_sched_class;
  1596. #define sched_class_highest (&stop_sched_class)
  1597. #define for_each_class(class) \
  1598. for (class = sched_class_highest; class; class = class->next)
  1599. #include "sched_stats.h"
  1600. static void inc_nr_running(struct rq *rq)
  1601. {
  1602. rq->nr_running++;
  1603. }
  1604. static void dec_nr_running(struct rq *rq)
  1605. {
  1606. rq->nr_running--;
  1607. }
  1608. static void set_load_weight(struct task_struct *p)
  1609. {
  1610. int prio = p->static_prio - MAX_RT_PRIO;
  1611. struct load_weight *load = &p->se.load;
  1612. /*
  1613. * SCHED_IDLE tasks get minimal weight:
  1614. */
  1615. if (p->policy == SCHED_IDLE) {
  1616. load->weight = scale_load(WEIGHT_IDLEPRIO);
  1617. load->inv_weight = WMULT_IDLEPRIO;
  1618. return;
  1619. }
  1620. load->weight = scale_load(prio_to_weight[prio]);
  1621. load->inv_weight = prio_to_wmult[prio];
  1622. }
  1623. static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
  1624. {
  1625. update_rq_clock(rq);
  1626. sched_info_queued(p);
  1627. p->sched_class->enqueue_task(rq, p, flags);
  1628. }
  1629. static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
  1630. {
  1631. update_rq_clock(rq);
  1632. sched_info_dequeued(p);
  1633. p->sched_class->dequeue_task(rq, p, flags);
  1634. }
  1635. /*
  1636. * activate_task - move a task to the runqueue.
  1637. */
  1638. static void activate_task(struct rq *rq, struct task_struct *p, int flags)
  1639. {
  1640. if (task_contributes_to_load(p))
  1641. rq->nr_uninterruptible--;
  1642. enqueue_task(rq, p, flags);
  1643. }
  1644. /*
  1645. * deactivate_task - remove a task from the runqueue.
  1646. */
  1647. static void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
  1648. {
  1649. if (task_contributes_to_load(p))
  1650. rq->nr_uninterruptible++;
  1651. dequeue_task(rq, p, flags);
  1652. }
  1653. #ifdef CONFIG_IRQ_TIME_ACCOUNTING
  1654. /*
  1655. * There are no locks covering percpu hardirq/softirq time.
  1656. * They are only modified in account_system_vtime, on corresponding CPU
  1657. * with interrupts disabled. So, writes are safe.
  1658. * They are read and saved off onto struct rq in update_rq_clock().
  1659. * This may result in other CPU reading this CPU's irq time and can
  1660. * race with irq/account_system_vtime on this CPU. We would either get old
  1661. * or new value with a side effect of accounting a slice of irq time to wrong
  1662. * task when irq is in progress while we read rq->clock. That is a worthy
  1663. * compromise in place of having locks on each irq in account_system_time.
  1664. */
  1665. static DEFINE_PER_CPU(u64, cpu_hardirq_time);
  1666. static DEFINE_PER_CPU(u64, cpu_softirq_time);
  1667. static DEFINE_PER_CPU(u64, irq_start_time);
  1668. static int sched_clock_irqtime;
  1669. void enable_sched_clock_irqtime(void)
  1670. {
  1671. sched_clock_irqtime = 1;
  1672. }
  1673. void disable_sched_clock_irqtime(void)
  1674. {
  1675. sched_clock_irqtime = 0;
  1676. }
  1677. #ifndef CONFIG_64BIT
  1678. static DEFINE_PER_CPU(seqcount_t, irq_time_seq);
  1679. static inline void irq_time_write_begin(void)
  1680. {
  1681. __this_cpu_inc(irq_time_seq.sequence);
  1682. smp_wmb();
  1683. }
  1684. static inline void irq_time_write_end(void)
  1685. {
  1686. smp_wmb();
  1687. __this_cpu_inc(irq_time_seq.sequence);
  1688. }
  1689. static inline u64 irq_time_read(int cpu)
  1690. {
  1691. u64 irq_time;
  1692. unsigned seq;
  1693. do {
  1694. seq = read_seqcount_begin(&per_cpu(irq_time_seq, cpu));
  1695. irq_time = per_cpu(cpu_softirq_time, cpu) +
  1696. per_cpu(cpu_hardirq_time, cpu);
  1697. } while (read_seqcount_retry(&per_cpu(irq_time_seq, cpu), seq));
  1698. return irq_time;
  1699. }
  1700. #else /* CONFIG_64BIT */
  1701. static inline void irq_time_write_begin(void)
  1702. {
  1703. }
  1704. static inline void irq_time_write_end(void)
  1705. {
  1706. }
  1707. static inline u64 irq_time_read(int cpu)
  1708. {
  1709. return per_cpu(cpu_softirq_time, cpu) + per_cpu(cpu_hardirq_time, cpu);
  1710. }
  1711. #endif /* CONFIG_64BIT */
  1712. /*
  1713. * Called before incrementing preempt_count on {soft,}irq_enter
  1714. * and before decrementing preempt_count on {soft,}irq_exit.
  1715. */
  1716. void account_system_vtime(struct task_struct *curr)
  1717. {
  1718. unsigned long flags;
  1719. s64 delta;
  1720. int cpu;
  1721. if (!sched_clock_irqtime)
  1722. return;
  1723. local_irq_save(flags);
  1724. cpu = smp_processor_id();
  1725. delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time);
  1726. __this_cpu_add(irq_start_time, delta);
  1727. irq_time_write_begin();
  1728. /*
  1729. * We do not account for softirq time from ksoftirqd here.
  1730. * We want to continue accounting softirq time to ksoftirqd thread
  1731. * in that case, so as not to confuse scheduler with a special task
  1732. * that do not consume any time, but still wants to run.
  1733. */
  1734. if (hardirq_count())
  1735. __this_cpu_add(cpu_hardirq_time, delta);
  1736. else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
  1737. __this_cpu_add(cpu_softirq_time, delta);
  1738. irq_time_write_end();
  1739. local_irq_restore(flags);
  1740. }
  1741. EXPORT_SYMBOL_GPL(account_system_vtime);
  1742. #endif /* CONFIG_IRQ_TIME_ACCOUNTING */
  1743. #ifdef CONFIG_PARAVIRT
  1744. static inline u64 steal_ticks(u64 steal)
  1745. {
  1746. if (unlikely(steal > NSEC_PER_SEC))
  1747. return div_u64(steal, TICK_NSEC);
  1748. return __iter_div_u64_rem(steal, TICK_NSEC, &steal);
  1749. }
  1750. #endif
  1751. static void update_rq_clock_task(struct rq *rq, s64 delta)
  1752. {
  1753. /*
  1754. * In theory, the compile should just see 0 here, and optimize out the call
  1755. * to sched_rt_avg_update. But I don't trust it...
  1756. */
  1757. #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
  1758. s64 steal = 0, irq_delta = 0;
  1759. #endif
  1760. #ifdef CONFIG_IRQ_TIME_ACCOUNTING
  1761. irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
  1762. /*
  1763. * Since irq_time is only updated on {soft,}irq_exit, we might run into
  1764. * this case when a previous update_rq_clock() happened inside a
  1765. * {soft,}irq region.
  1766. *
  1767. * When this happens, we stop ->clock_task and only update the
  1768. * prev_irq_time stamp to account for the part that fit, so that a next
  1769. * update will consume the rest. This ensures ->clock_task is
  1770. * monotonic.
  1771. *
  1772. * It does however cause some slight miss-attribution of {soft,}irq
  1773. * time, a more accurate solution would be to update the irq_time using
  1774. * the current rq->clock timestamp, except that would require using
  1775. * atomic ops.
  1776. */
  1777. if (irq_delta > delta)
  1778. irq_delta = delta;
  1779. rq->prev_irq_time += irq_delta;
  1780. delta -= irq_delta;
  1781. #endif
  1782. #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
  1783. if (static_branch((&paravirt_steal_rq_enabled))) {
  1784. u64 st;
  1785. steal = paravirt_steal_clock(cpu_of(rq));
  1786. steal -= rq->prev_steal_time_rq;
  1787. if (unlikely(steal > delta))
  1788. steal = delta;
  1789. st = steal_ticks(steal);
  1790. steal = st * TICK_NSEC;
  1791. rq->prev_steal_time_rq += steal;
  1792. delta -= steal;
  1793. }
  1794. #endif
  1795. rq->clock_task += delta;
  1796. #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
  1797. if ((irq_delta + steal) && sched_feat(NONTASK_POWER))
  1798. sched_rt_avg_update(rq, irq_delta + steal);
  1799. #endif
  1800. }
  1801. #ifdef CONFIG_IRQ_TIME_ACCOUNTING
  1802. static int irqtime_account_hi_update(void)
  1803. {
  1804. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  1805. unsigned long flags;
  1806. u64 latest_ns;
  1807. int ret = 0;
  1808. local_irq_save(flags);
  1809. latest_ns = this_cpu_read(cpu_hardirq_time);
  1810. if (cputime64_gt(nsecs_to_cputime64(latest_ns), cpustat->irq))
  1811. ret = 1;
  1812. local_irq_restore(flags);
  1813. return ret;
  1814. }
  1815. static int irqtime_account_si_update(void)
  1816. {
  1817. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  1818. unsigned long flags;
  1819. u64 latest_ns;
  1820. int ret = 0;
  1821. local_irq_save(flags);
  1822. latest_ns = this_cpu_read(cpu_softirq_time);
  1823. if (cputime64_gt(nsecs_to_cputime64(latest_ns), cpustat->softirq))
  1824. ret = 1;
  1825. local_irq_restore(flags);
  1826. return ret;
  1827. }
  1828. #else /* CONFIG_IRQ_TIME_ACCOUNTING */
  1829. #define sched_clock_irqtime (0)
  1830. #endif
  1831. #include "sched_idletask.c"
  1832. #include "sched_fair.c"
  1833. #include "sched_rt.c"
  1834. #include "sched_autogroup.c"
  1835. #include "sched_stoptask.c"
  1836. #ifdef CONFIG_SCHED_DEBUG
  1837. # include "sched_debug.c"
  1838. #endif
  1839. void sched_set_stop_task(int cpu, struct task_struct *stop)
  1840. {
  1841. struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
  1842. struct task_struct *old_stop = cpu_rq(cpu)->stop;
  1843. if (stop) {
  1844. /*
  1845. * Make it appear like a SCHED_FIFO task, its something
  1846. * userspace knows about and won't get confused about.
  1847. *
  1848. * Also, it will make PI more or less work without too
  1849. * much confusion -- but then, stop work should not
  1850. * rely on PI working anyway.
  1851. */
  1852. sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
  1853. stop->sched_class = &stop_sched_class;
  1854. }
  1855. cpu_rq(cpu)->stop = stop;
  1856. if (old_stop) {
  1857. /*
  1858. * Reset it back to a normal scheduling class so that
  1859. * it can die in pieces.
  1860. */
  1861. old_stop->sched_class = &rt_sched_class;
  1862. }
  1863. }
  1864. /*
  1865. * __normal_prio - return the priority that is based on the static prio
  1866. */
  1867. static inline int __normal_prio(struct task_struct *p)
  1868. {
  1869. return p->static_prio;
  1870. }
  1871. /*
  1872. * Calculate the expected normal priority: i.e. priority
  1873. * without taking RT-inheritance into account. Might be
  1874. * boosted by interactivity modifiers. Changes upon fork,
  1875. * setprio syscalls, and whenever the interactivity
  1876. * estimator recalculates.
  1877. */
  1878. static inline int normal_prio(struct task_struct *p)
  1879. {
  1880. int prio;
  1881. if (task_has_rt_policy(p))
  1882. prio = MAX_RT_PRIO-1 - p->rt_priority;
  1883. else
  1884. prio = __normal_prio(p);
  1885. return prio;
  1886. }
  1887. /*
  1888. * Calculate the current priority, i.e. the priority
  1889. * taken into account by the scheduler. This value might
  1890. * be boosted by RT tasks, or might be boosted by
  1891. * interactivity modifiers. Will be RT if the task got
  1892. * RT-boosted. If not then it returns p->normal_prio.
  1893. */
  1894. static int effective_prio(struct task_struct *p)
  1895. {
  1896. p->normal_prio = normal_prio(p);
  1897. /*
  1898. * If we are RT tasks or we were boosted to RT priority,
  1899. * keep the priority unchanged. Otherwise, update priority
  1900. * to the normal priority:
  1901. */
  1902. if (!rt_prio(p->prio))
  1903. return p->normal_prio;
  1904. return p->prio;
  1905. }
  1906. /**
  1907. * task_curr - is this task currently executing on a CPU?
  1908. * @p: the task in question.
  1909. */
  1910. inline int task_curr(const struct task_struct *p)
  1911. {
  1912. return cpu_curr(task_cpu(p)) == p;
  1913. }
  1914. static inline void check_class_changed(struct rq *rq, struct task_struct *p,
  1915. const struct sched_class *prev_class,
  1916. int oldprio)
  1917. {
  1918. if (prev_class != p->sched_class) {
  1919. if (prev_class->switched_from)
  1920. prev_class->switched_from(rq, p);
  1921. p->sched_class->switched_to(rq, p);
  1922. } else if (oldprio != p->prio)
  1923. p->sched_class->prio_changed(rq, p, oldprio);
  1924. }
  1925. static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
  1926. {
  1927. const struct sched_class *class;
  1928. if (p->sched_class == rq->curr->sched_class) {
  1929. rq->curr->sched_class->check_preempt_curr(rq, p, flags);
  1930. } else {
  1931. for_each_class(class) {
  1932. if (class == rq->curr->sched_class)
  1933. break;
  1934. if (class == p->sched_class) {
  1935. resched_task(rq->curr);
  1936. break;
  1937. }
  1938. }
  1939. }
  1940. /*
  1941. * A queue event has occurred, and we're going to schedule. In
  1942. * this case, we can save a useless back to back clock update.
  1943. */
  1944. if (rq->curr->on_rq && test_tsk_need_resched(rq->curr))
  1945. rq->skip_clock_update = 1;
  1946. }
  1947. #ifdef CONFIG_SMP
  1948. /*
  1949. * Is this task likely cache-hot:
  1950. */
  1951. static int
  1952. task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
  1953. {
  1954. s64 delta;
  1955. if (p->sched_class != &fair_sched_class)
  1956. return 0;
  1957. if (unlikely(p->policy == SCHED_IDLE))
  1958. return 0;
  1959. /*
  1960. * Buddy candidates are cache hot:
  1961. */
  1962. if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running &&
  1963. (&p->se == cfs_rq_of(&p->se)->next ||
  1964. &p->se == cfs_rq_of(&p->se)->last))
  1965. return 1;
  1966. if (sysctl_sched_migration_cost == -1)
  1967. return 1;
  1968. if (sysctl_sched_migration_cost == 0)
  1969. return 0;
  1970. delta = now - p->se.exec_start;
  1971. return delta < (s64)sysctl_sched_migration_cost;
  1972. }
  1973. void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
  1974. {
  1975. #ifdef CONFIG_SCHED_DEBUG
  1976. /*
  1977. * We should never call set_task_cpu() on a blocked task,
  1978. * ttwu() will sort out the placement.
  1979. */
  1980. WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
  1981. !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE));
  1982. #ifdef CONFIG_LOCKDEP
  1983. /*
  1984. * The caller should hold either p->pi_lock or rq->lock, when changing
  1985. * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
  1986. *
  1987. * sched_move_task() holds both and thus holding either pins the cgroup,
  1988. * see set_task_rq().
  1989. *
  1990. * Furthermore, all task_rq users should acquire both locks, see
  1991. * task_rq_lock().
  1992. */
  1993. WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
  1994. lockdep_is_held(&task_rq(p)->lock)));
  1995. #endif
  1996. #endif
  1997. trace_sched_migrate_task(p, new_cpu);
  1998. if (task_cpu(p) != new_cpu) {
  1999. p->se.nr_migrations++;
  2000. perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0);
  2001. }
  2002. __set_task_cpu(p, new_cpu);
  2003. }
  2004. struct migration_arg {
  2005. struct task_struct *task;
  2006. int dest_cpu;
  2007. };
  2008. static int migration_cpu_stop(void *data);
  2009. /*
  2010. * wait_task_inactive - wait for a thread to unschedule.
  2011. *
  2012. * If @match_state is nonzero, it's the @p->state value just checked and
  2013. * not expected to change. If it changes, i.e. @p might have woken up,
  2014. * then return zero. When we succeed in waiting for @p to be off its CPU,
  2015. * we return a positive number (its total switch count). If a second call
  2016. * a short while later returns the same number, the caller can be sure that
  2017. * @p has remained unscheduled the whole time.
  2018. *
  2019. * The caller must ensure that the task *will* unschedule sometime soon,
  2020. * else this function might spin for a *long* time. This function can't
  2021. * be called with interrupts off, or it may introduce deadlock with
  2022. * smp_call_function() if an IPI is sent by the same process we are
  2023. * waiting to become inactive.
  2024. */
  2025. unsigned long wait_task_inactive(struct task_struct *p, long match_state)
  2026. {
  2027. unsigned long flags;
  2028. int running, on_rq;
  2029. unsigned long ncsw;
  2030. struct rq *rq;
  2031. for (;;) {
  2032. /*
  2033. * We do the initial early heuristics without holding
  2034. * any task-queue locks at all. We'll only try to get
  2035. * the runqueue lock when things look like they will
  2036. * work out!
  2037. */
  2038. rq = task_rq(p);
  2039. /*
  2040. * If the task is actively running on another CPU
  2041. * still, just relax and busy-wait without holding
  2042. * any locks.
  2043. *
  2044. * NOTE! Since we don't hold any locks, it's not
  2045. * even sure that "rq" stays as the right runqueue!
  2046. * But we don't care, since "task_running()" will
  2047. * return false if the runqueue has changed and p
  2048. * is actually now running somewhere else!
  2049. */
  2050. while (task_running(rq, p)) {
  2051. if (match_state && unlikely(p->state != match_state))
  2052. return 0;
  2053. cpu_relax();
  2054. }
  2055. /*
  2056. * Ok, time to look more closely! We need the rq
  2057. * lock now, to be *sure*. If we're wrong, we'll
  2058. * just go back and repeat.
  2059. */
  2060. rq = task_rq_lock(p, &flags);
  2061. trace_sched_wait_task(p);
  2062. running = task_running(rq, p);
  2063. on_rq = p->on_rq;
  2064. ncsw = 0;
  2065. if (!match_state || p->state == match_state)
  2066. ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
  2067. task_rq_unlock(rq, p, &flags);
  2068. /*
  2069. * If it changed from the expected state, bail out now.
  2070. */
  2071. if (unlikely(!ncsw))
  2072. break;
  2073. /*
  2074. * Was it really running after all now that we
  2075. * checked with the proper locks actually held?
  2076. *
  2077. * Oops. Go back and try again..
  2078. */
  2079. if (unlikely(running)) {
  2080. cpu_relax();
  2081. continue;
  2082. }
  2083. /*
  2084. * It's not enough that it's not actively running,
  2085. * it must be off the runqueue _entirely_, and not
  2086. * preempted!
  2087. *
  2088. * So if it was still runnable (but just not actively
  2089. * running right now), it's preempted, and we should
  2090. * yield - it could be a while.
  2091. */
  2092. if (unlikely(on_rq)) {
  2093. ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);
  2094. set_current_state(TASK_UNINTERRUPTIBLE);
  2095. schedule_hrtimeout(&to, HRTIMER_MODE_REL);
  2096. continue;
  2097. }
  2098. /*
  2099. * Ahh, all good. It wasn't running, and it wasn't
  2100. * runnable, which means that it will never become
  2101. * running in the future either. We're all done!
  2102. */
  2103. break;
  2104. }
  2105. return ncsw;
  2106. }
  2107. /***
  2108. * kick_process - kick a running thread to enter/exit the kernel
  2109. * @p: the to-be-kicked thread
  2110. *
  2111. * Cause a process which is running on another CPU to enter
  2112. * kernel-mode, without any delay. (to get signals handled.)
  2113. *
  2114. * NOTE: this function doesn't have to take the runqueue lock,
  2115. * because all it wants to ensure is that the remote task enters
  2116. * the kernel. If the IPI races and the task has been migrated
  2117. * to another CPU then no harm is done and the purpose has been
  2118. * achieved as well.
  2119. */
  2120. void kick_process(struct task_struct *p)
  2121. {
  2122. int cpu;
  2123. preempt_disable();
  2124. cpu = task_cpu(p);
  2125. if ((cpu != smp_processor_id()) && task_curr(p))
  2126. smp_send_reschedule(cpu);
  2127. preempt_enable();
  2128. }
  2129. EXPORT_SYMBOL_GPL(kick_process);
  2130. #endif /* CONFIG_SMP */
  2131. #ifdef CONFIG_SMP
  2132. /*
  2133. * ->cpus_allowed is protected by both rq->lock and p->pi_lock
  2134. */
  2135. static int select_fallback_rq(int cpu, struct task_struct *p)
  2136. {
  2137. int dest_cpu;
  2138. const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(cpu));
  2139. /* Look for allowed, online CPU in same node. */
  2140. for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
  2141. if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
  2142. return dest_cpu;
  2143. /* Any allowed, online CPU? */
  2144. dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
  2145. if (dest_cpu < nr_cpu_ids)
  2146. return dest_cpu;
  2147. /* No more Mr. Nice Guy. */
  2148. dest_cpu = cpuset_cpus_allowed_fallback(p);
  2149. /*
  2150. * Don't tell them about moving exiting tasks or
  2151. * kernel threads (both mm NULL), since they never
  2152. * leave kernel.
  2153. */
  2154. if (p->mm && printk_ratelimit()) {
  2155. printk(KERN_INFO "process %d (%s) no longer affine to cpu%d\n",
  2156. task_pid_nr(p), p->comm, cpu);
  2157. }
  2158. return dest_cpu;
  2159. }
  2160. /*
  2161. * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
  2162. */
  2163. static inline
  2164. int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
  2165. {
  2166. int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags);
  2167. /*
  2168. * In order not to call set_task_cpu() on a blocking task we need
  2169. * to rely on ttwu() to place the task on a valid ->cpus_allowed
  2170. * cpu.
  2171. *
  2172. * Since this is common to all placement strategies, this lives here.
  2173. *
  2174. * [ this allows ->select_task() to simply return task_cpu(p) and
  2175. * not worry about this generic constraint ]
  2176. */
  2177. if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed) ||
  2178. !cpu_online(cpu)))
  2179. cpu = select_fallback_rq(task_cpu(p), p);
  2180. return cpu;
  2181. }
  2182. static void update_avg(u64 *avg, u64 sample)
  2183. {
  2184. s64 diff = sample - *avg;
  2185. *avg += diff >> 3;
  2186. }
  2187. #endif
  2188. static void
  2189. ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
  2190. {
  2191. #ifdef CONFIG_SCHEDSTATS
  2192. struct rq *rq = this_rq();
  2193. #ifdef CONFIG_SMP
  2194. int this_cpu = smp_processor_id();
  2195. if (cpu == this_cpu) {
  2196. schedstat_inc(rq, ttwu_local);
  2197. schedstat_inc(p, se.statistics.nr_wakeups_local);
  2198. } else {
  2199. struct sched_domain *sd;
  2200. schedstat_inc(p, se.statistics.nr_wakeups_remote);
  2201. rcu_read_lock();
  2202. for_each_domain(this_cpu, sd) {
  2203. if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
  2204. schedstat_inc(sd, ttwu_wake_remote);
  2205. break;
  2206. }
  2207. }
  2208. rcu_read_unlock();
  2209. }
  2210. if (wake_flags & WF_MIGRATED)
  2211. schedstat_inc(p, se.statistics.nr_wakeups_migrate);
  2212. #endif /* CONFIG_SMP */
  2213. schedstat_inc(rq, ttwu_count);
  2214. schedstat_inc(p, se.statistics.nr_wakeups);
  2215. if (wake_flags & WF_SYNC)
  2216. schedstat_inc(p, se.statistics.nr_wakeups_sync);
  2217. #endif /* CONFIG_SCHEDSTATS */
  2218. }
  2219. static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
  2220. {
  2221. activate_task(rq, p, en_flags);
  2222. p->on_rq = 1;
  2223. /* if a worker is waking up, notify workqueue */
  2224. if (p->flags & PF_WQ_WORKER)
  2225. wq_worker_waking_up(p, cpu_of(rq));
  2226. }
  2227. /*
  2228. * Mark the task runnable and perform wakeup-preemption.
  2229. */
  2230. static void
  2231. ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
  2232. {
  2233. trace_sched_wakeup(p, true);
  2234. check_preempt_curr(rq, p, wake_flags);
  2235. p->state = TASK_RUNNING;
  2236. #ifdef CONFIG_SMP
  2237. if (p->sched_class->task_woken)
  2238. p->sched_class->task_woken(rq, p);
  2239. if (rq->idle_stamp) {
  2240. u64 delta = rq->clock - rq->idle_stamp;
  2241. u64 max = 2*sysctl_sched_migration_cost;
  2242. if (delta > max)
  2243. rq->avg_idle = max;
  2244. else
  2245. update_avg(&rq->avg_idle, delta);
  2246. rq->idle_stamp = 0;
  2247. }
  2248. #endif
  2249. }
  2250. static void
  2251. ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
  2252. {
  2253. #ifdef CONFIG_SMP
  2254. if (p->sched_contributes_to_load)
  2255. rq->nr_uninterruptible--;
  2256. #endif
  2257. ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
  2258. ttwu_do_wakeup(rq, p, wake_flags);
  2259. }
  2260. /*
  2261. * Called in case the task @p isn't fully descheduled from its runqueue,
  2262. * in this case we must do a remote wakeup. Its a 'light' wakeup though,
  2263. * since all we need to do is flip p->state to TASK_RUNNING, since
  2264. * the task is still ->on_rq.
  2265. */
  2266. static int ttwu_remote(struct task_struct *p, int wake_flags)
  2267. {
  2268. struct rq *rq;
  2269. int ret = 0;
  2270. rq = __task_rq_lock(p);
  2271. if (p->on_rq) {
  2272. ttwu_do_wakeup(rq, p, wake_flags);
  2273. ret = 1;
  2274. }
  2275. __task_rq_unlock(rq);
  2276. return ret;
  2277. }
  2278. #ifdef CONFIG_SMP
  2279. static void sched_ttwu_pending(void)
  2280. {
  2281. struct rq *rq = this_rq();
  2282. struct llist_node *llist = llist_del_all(&rq->wake_list);
  2283. struct task_struct *p;
  2284. raw_spin_lock(&rq->lock);
  2285. while (llist) {
  2286. p = llist_entry(llist, struct task_struct, wake_entry);
  2287. llist = llist_next(llist);
  2288. ttwu_do_activate(rq, p, 0);
  2289. }
  2290. raw_spin_unlock(&rq->lock);
  2291. }
  2292. void scheduler_ipi(void)
  2293. {
  2294. if (llist_empty(&this_rq()->wake_list))
  2295. return;
  2296. /*
  2297. * Not all reschedule IPI handlers call irq_enter/irq_exit, since
  2298. * traditionally all their work was done from the interrupt return
  2299. * path. Now that we actually do some work, we need to make sure
  2300. * we do call them.
  2301. *
  2302. * Some archs already do call them, luckily irq_enter/exit nest
  2303. * properly.
  2304. *
  2305. * Arguably we should visit all archs and update all handlers,
  2306. * however a fair share of IPIs are still resched only so this would
  2307. * somewhat pessimize the simple resched case.
  2308. */
  2309. irq_enter();
  2310. sched_ttwu_pending();
  2311. irq_exit();
  2312. }
  2313. static void ttwu_queue_remote(struct task_struct *p, int cpu)
  2314. {
  2315. if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list))
  2316. smp_send_reschedule(cpu);
  2317. }
  2318. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  2319. static int ttwu_activate_remote(struct task_struct *p, int wake_flags)
  2320. {
  2321. struct rq *rq;
  2322. int ret = 0;
  2323. rq = __task_rq_lock(p);
  2324. if (p->on_cpu) {
  2325. ttwu_activate(rq, p, ENQUEUE_WAKEUP);
  2326. ttwu_do_wakeup(rq, p, wake_flags);
  2327. ret = 1;
  2328. }
  2329. __task_rq_unlock(rq);
  2330. return ret;
  2331. }
  2332. #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
  2333. #endif /* CONFIG_SMP */
  2334. static void ttwu_queue(struct task_struct *p, int cpu)
  2335. {
  2336. struct rq *rq = cpu_rq(cpu);
  2337. #if defined(CONFIG_SMP)
  2338. if (sched_feat(TTWU_QUEUE) && cpu != smp_processor_id()) {
  2339. sched_clock_cpu(cpu); /* sync clocks x-cpu */
  2340. ttwu_queue_remote(p, cpu);
  2341. return;
  2342. }
  2343. #endif
  2344. raw_spin_lock(&rq->lock);
  2345. ttwu_do_activate(rq, p, 0);
  2346. raw_spin_unlock(&rq->lock);
  2347. }
  2348. /**
  2349. * try_to_wake_up - wake up a thread
  2350. * @p: the thread to be awakened
  2351. * @state: the mask of task states that can be woken
  2352. * @wake_flags: wake modifier flags (WF_*)
  2353. *
  2354. * Put it on the run-queue if it's not already there. The "current"
  2355. * thread is always on the run-queue (except when the actual
  2356. * re-schedule is in progress), and as such you're allowed to do
  2357. * the simpler "current->state = TASK_RUNNING" to mark yourself
  2358. * runnable without the overhead of this.
  2359. *
  2360. * Returns %true if @p was woken up, %false if it was already running
  2361. * or @state didn't match @p's state.
  2362. */
  2363. static int
  2364. try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
  2365. {
  2366. unsigned long flags;
  2367. int cpu, success = 0;
  2368. smp_wmb();
  2369. raw_spin_lock_irqsave(&p->pi_lock, flags);
  2370. if (!(p->state & state))
  2371. goto out;
  2372. success = 1; /* we're going to change ->state */
  2373. cpu = task_cpu(p);
  2374. if (p->on_rq && ttwu_remote(p, wake_flags))
  2375. goto stat;
  2376. #ifdef CONFIG_SMP
  2377. /*
  2378. * If the owning (remote) cpu is still in the middle of schedule() with
  2379. * this task as prev, wait until its done referencing the task.
  2380. */
  2381. while (p->on_cpu) {
  2382. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  2383. /*
  2384. * In case the architecture enables interrupts in
  2385. * context_switch(), we cannot busy wait, since that
  2386. * would lead to deadlocks when an interrupt hits and
  2387. * tries to wake up @prev. So bail and do a complete
  2388. * remote wakeup.
  2389. */
  2390. if (ttwu_activate_remote(p, wake_flags))
  2391. goto stat;
  2392. #else
  2393. cpu_relax();
  2394. #endif
  2395. }
  2396. /*
  2397. * Pairs with the smp_wmb() in finish_lock_switch().
  2398. */
  2399. smp_rmb();
  2400. p->sched_contributes_to_load = !!task_contributes_to_load(p);
  2401. p->state = TASK_WAKING;
  2402. if (p->sched_class->task_waking)
  2403. p->sched_class->task_waking(p);
  2404. cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
  2405. if (task_cpu(p) != cpu) {
  2406. wake_flags |= WF_MIGRATED;
  2407. set_task_cpu(p, cpu);
  2408. }
  2409. #endif /* CONFIG_SMP */
  2410. ttwu_queue(p, cpu);
  2411. stat:
  2412. ttwu_stat(p, cpu, wake_flags);
  2413. out:
  2414. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  2415. return success;
  2416. }
  2417. /**
  2418. * try_to_wake_up_local - try to wake up a local task with rq lock held
  2419. * @p: the thread to be awakened
  2420. *
  2421. * Put @p on the run-queue if it's not already there. The caller must
  2422. * ensure that this_rq() is locked, @p is bound to this_rq() and not
  2423. * the current task.
  2424. */
  2425. static void try_to_wake_up_local(struct task_struct *p)
  2426. {
  2427. struct rq *rq = task_rq(p);
  2428. BUG_ON(rq != this_rq());
  2429. BUG_ON(p == current);
  2430. lockdep_assert_held(&rq->lock);
  2431. if (!raw_spin_trylock(&p->pi_lock)) {
  2432. raw_spin_unlock(&rq->lock);
  2433. raw_spin_lock(&p->pi_lock);
  2434. raw_spin_lock(&rq->lock);
  2435. }
  2436. if (!(p->state & TASK_NORMAL))
  2437. goto out;
  2438. if (!p->on_rq)
  2439. ttwu_activate(rq, p, ENQUEUE_WAKEUP);
  2440. ttwu_do_wakeup(rq, p, 0);
  2441. ttwu_stat(p, smp_processor_id(), 0);
  2442. out:
  2443. raw_spin_unlock(&p->pi_lock);
  2444. }
  2445. /**
  2446. * wake_up_process - Wake up a specific process
  2447. * @p: The process to be woken up.
  2448. *
  2449. * Attempt to wake up the nominated process and move it to the set of runnable
  2450. * processes. Returns 1 if the process was woken up, 0 if it was already
  2451. * running.
  2452. *
  2453. * It may be assumed that this function implies a write memory barrier before
  2454. * changing the task state if and only if any tasks are woken up.
  2455. */
  2456. int wake_up_process(struct task_struct *p)
  2457. {
  2458. return try_to_wake_up(p, TASK_ALL, 0);
  2459. }
  2460. EXPORT_SYMBOL(wake_up_process);
  2461. int wake_up_state(struct task_struct *p, unsigned int state)
  2462. {
  2463. return try_to_wake_up(p, state, 0);
  2464. }
  2465. /*
  2466. * Perform scheduler related setup for a newly forked process p.
  2467. * p is forked by current.
  2468. *
  2469. * __sched_fork() is basic setup used by init_idle() too:
  2470. */
  2471. static void __sched_fork(struct task_struct *p)
  2472. {
  2473. p->on_rq = 0;
  2474. p->se.on_rq = 0;
  2475. p->se.exec_start = 0;
  2476. p->se.sum_exec_runtime = 0;
  2477. p->se.prev_sum_exec_runtime = 0;
  2478. p->se.nr_migrations = 0;
  2479. p->se.vruntime = 0;
  2480. INIT_LIST_HEAD(&p->se.group_node);
  2481. #ifdef CONFIG_SCHEDSTATS
  2482. memset(&p->se.statistics, 0, sizeof(p->se.statistics));
  2483. #endif
  2484. INIT_LIST_HEAD(&p->rt.run_list);
  2485. #ifdef CONFIG_PREEMPT_NOTIFIERS
  2486. INIT_HLIST_HEAD(&p->preempt_notifiers);
  2487. #endif
  2488. }
  2489. /*
  2490. * fork()/clone()-time setup:
  2491. */
  2492. void sched_fork(struct task_struct *p)
  2493. {
  2494. unsigned long flags;
  2495. int cpu = get_cpu();
  2496. __sched_fork(p);
  2497. /*
  2498. * We mark the process as running here. This guarantees that
  2499. * nobody will actually run it, and a signal or other external
  2500. * event cannot wake it up and insert it on the runqueue either.
  2501. */
  2502. p->state = TASK_RUNNING;
  2503. /*
  2504. * Make sure we do not leak PI boosting priority to the child.
  2505. */
  2506. p->prio = current->normal_prio;
  2507. /*
  2508. * Revert to default priority/policy on fork if requested.
  2509. */
  2510. if (unlikely(p->sched_reset_on_fork)) {
  2511. if (task_has_rt_policy(p)) {
  2512. p->policy = SCHED_NORMAL;
  2513. p->static_prio = NICE_TO_PRIO(0);
  2514. p->rt_priority = 0;
  2515. } else if (PRIO_TO_NICE(p->static_prio) < 0)
  2516. p->static_prio = NICE_TO_PRIO(0);
  2517. p->prio = p->normal_prio = __normal_prio(p);
  2518. set_load_weight(p);
  2519. /*
  2520. * We don't need the reset flag anymore after the fork. It has
  2521. * fulfilled its duty:
  2522. */
  2523. p->sched_reset_on_fork = 0;
  2524. }
  2525. if (!rt_prio(p->prio))
  2526. p->sched_class = &fair_sched_class;
  2527. if (p->sched_class->task_fork)
  2528. p->sched_class->task_fork(p);
  2529. /*
  2530. * The child is not yet in the pid-hash so no cgroup attach races,
  2531. * and the cgroup is pinned to this child due to cgroup_fork()
  2532. * is ran before sched_fork().
  2533. *
  2534. * Silence PROVE_RCU.
  2535. */
  2536. raw_spin_lock_irqsave(&p->pi_lock, flags);
  2537. set_task_cpu(p, cpu);
  2538. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  2539. #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
  2540. if (likely(sched_info_on()))
  2541. memset(&p->sched_info, 0, sizeof(p->sched_info));
  2542. #endif
  2543. #if defined(CONFIG_SMP)
  2544. p->on_cpu = 0;
  2545. #endif
  2546. #ifdef CONFIG_PREEMPT_COUNT
  2547. /* Want to start with kernel preemption disabled. */
  2548. task_thread_info(p)->preempt_count = 1;
  2549. #endif
  2550. #ifdef CONFIG_SMP
  2551. plist_node_init(&p->pushable_tasks, MAX_PRIO);
  2552. #endif
  2553. put_cpu();
  2554. }
  2555. /*
  2556. * wake_up_new_task - wake up a newly created task for the first time.
  2557. *
  2558. * This function will do some initial scheduler statistics housekeeping
  2559. * that must be done for every newly created context, then puts the task
  2560. * on the runqueue and wakes it.
  2561. */
  2562. void wake_up_new_task(struct task_struct *p)
  2563. {
  2564. unsigned long flags;
  2565. struct rq *rq;
  2566. raw_spin_lock_irqsave(&p->pi_lock, flags);
  2567. #ifdef CONFIG_SMP
  2568. /*
  2569. * Fork balancing, do it here and not earlier because:
  2570. * - cpus_allowed can change in the fork path
  2571. * - any previously selected cpu might disappear through hotplug
  2572. */
  2573. set_task_cpu(p, select_task_rq(p, SD_BALANCE_FORK, 0));
  2574. #endif
  2575. rq = __task_rq_lock(p);
  2576. activate_task(rq, p, 0);
  2577. p->on_rq = 1;
  2578. trace_sched_wakeup_new(p, true);
  2579. check_preempt_curr(rq, p, WF_FORK);
  2580. #ifdef CONFIG_SMP
  2581. if (p->sched_class->task_woken)
  2582. p->sched_class->task_woken(rq, p);
  2583. #endif
  2584. task_rq_unlock(rq, p, &flags);
  2585. }
  2586. #ifdef CONFIG_PREEMPT_NOTIFIERS
  2587. /**
  2588. * preempt_notifier_register - tell me when current is being preempted & rescheduled
  2589. * @notifier: notifier struct to register
  2590. */
  2591. void preempt_notifier_register(struct preempt_notifier *notifier)
  2592. {
  2593. hlist_add_head(&notifier->link, &current->preempt_notifiers);
  2594. }
  2595. EXPORT_SYMBOL_GPL(preempt_notifier_register);
  2596. /**
  2597. * preempt_notifier_unregister - no longer interested in preemption notifications
  2598. * @notifier: notifier struct to unregister
  2599. *
  2600. * This is safe to call from within a preemption notifier.
  2601. */
  2602. void preempt_notifier_unregister(struct preempt_notifier *notifier)
  2603. {
  2604. hlist_del(&notifier->link);
  2605. }
  2606. EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
  2607. static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
  2608. {
  2609. struct preempt_notifier *notifier;
  2610. struct hlist_node *node;
  2611. hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
  2612. notifier->ops->sched_in(notifier, raw_smp_processor_id());
  2613. }
  2614. static void
  2615. fire_sched_out_preempt_notifiers(struct task_struct *curr,
  2616. struct task_struct *next)
  2617. {
  2618. struct preempt_notifier *notifier;
  2619. struct hlist_node *node;
  2620. hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
  2621. notifier->ops->sched_out(notifier, next);
  2622. }
  2623. #else /* !CONFIG_PREEMPT_NOTIFIERS */
  2624. static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
  2625. {
  2626. }
  2627. static void
  2628. fire_sched_out_preempt_notifiers(struct task_struct *curr,
  2629. struct task_struct *next)
  2630. {
  2631. }
  2632. #endif /* CONFIG_PREEMPT_NOTIFIERS */
  2633. /**
  2634. * prepare_task_switch - prepare to switch tasks
  2635. * @rq: the runqueue preparing to switch
  2636. * @prev: the current task that is being switched out
  2637. * @next: the task we are going to switch to.
  2638. *
  2639. * This is called with the rq lock held and interrupts off. It must
  2640. * be paired with a subsequent finish_task_switch after the context
  2641. * switch.
  2642. *
  2643. * prepare_task_switch sets up locking and calls architecture specific
  2644. * hooks.
  2645. */
  2646. static inline void
  2647. prepare_task_switch(struct rq *rq, struct task_struct *prev,
  2648. struct task_struct *next)
  2649. {
  2650. sched_info_switch(prev, next);
  2651. perf_event_task_sched_out(prev, next);
  2652. fire_sched_out_preempt_notifiers(prev, next);
  2653. prepare_lock_switch(rq, next);
  2654. prepare_arch_switch(next);
  2655. trace_sched_switch(prev, next);
  2656. }
  2657. /**
  2658. * finish_task_switch - clean up after a task-switch
  2659. * @rq: runqueue associated with task-switch
  2660. * @prev: the thread we just switched away from.
  2661. *
  2662. * finish_task_switch must be called after the context switch, paired
  2663. * with a prepare_task_switch call before the context switch.
  2664. * finish_task_switch will reconcile locking set up by prepare_task_switch,
  2665. * and do any other architecture-specific cleanup actions.
  2666. *
  2667. * Note that we may have delayed dropping an mm in context_switch(). If
  2668. * so, we finish that here outside of the runqueue lock. (Doing it
  2669. * with the lock held can cause deadlocks; see schedule() for
  2670. * details.)
  2671. */
  2672. static void finish_task_switch(struct rq *rq, struct task_struct *prev)
  2673. __releases(rq->lock)
  2674. {
  2675. struct mm_struct *mm = rq->prev_mm;
  2676. long prev_state;
  2677. rq->prev_mm = NULL;
  2678. /*
  2679. * A task struct has one reference for the use as "current".
  2680. * If a task dies, then it sets TASK_DEAD in tsk->state and calls
  2681. * schedule one last time. The schedule call will never return, and
  2682. * the scheduled task must drop that reference.
  2683. * The test for TASK_DEAD must occur while the runqueue locks are
  2684. * still held, otherwise prev could be scheduled on another cpu, die
  2685. * there before we look at prev->state, and then the reference would
  2686. * be dropped twice.
  2687. * Manfred Spraul <manfred@colorfullife.com>
  2688. */
  2689. prev_state = prev->state;
  2690. finish_arch_switch(prev);
  2691. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  2692. local_irq_disable();
  2693. #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
  2694. perf_event_task_sched_in(prev, current);
  2695. #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
  2696. local_irq_enable();
  2697. #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
  2698. finish_lock_switch(rq, prev);
  2699. fire_sched_in_preempt_notifiers(current);
  2700. if (mm)
  2701. mmdrop(mm);
  2702. if (unlikely(prev_state == TASK_DEAD)) {
  2703. /*
  2704. * Remove function-return probe instances associated with this
  2705. * task and put them back on the free list.
  2706. */
  2707. kprobe_flush_task(prev);
  2708. put_task_struct(prev);
  2709. }
  2710. }
  2711. #ifdef CONFIG_SMP
  2712. /* assumes rq->lock is held */
  2713. static inline void pre_schedule(struct rq *rq, struct task_struct *prev)
  2714. {
  2715. if (prev->sched_class->pre_schedule)
  2716. prev->sched_class->pre_schedule(rq, prev);
  2717. }
  2718. /* rq->lock is NOT held, but preemption is disabled */
  2719. static inline void post_schedule(struct rq *rq)
  2720. {
  2721. if (rq->post_schedule) {
  2722. unsigned long flags;
  2723. raw_spin_lock_irqsave(&rq->lock, flags);
  2724. if (rq->curr->sched_class->post_schedule)
  2725. rq->curr->sched_class->post_schedule(rq);
  2726. raw_spin_unlock_irqrestore(&rq->lock, flags);
  2727. rq->post_schedule = 0;
  2728. }
  2729. }
  2730. #else
  2731. static inline void pre_schedule(struct rq *rq, struct task_struct *p)
  2732. {
  2733. }
  2734. static inline void post_schedule(struct rq *rq)
  2735. {
  2736. }
  2737. #endif
  2738. /**
  2739. * schedule_tail - first thing a freshly forked thread must call.
  2740. * @prev: the thread we just switched away from.
  2741. */
  2742. asmlinkage void schedule_tail(struct task_struct *prev)
  2743. __releases(rq->lock)
  2744. {
  2745. struct rq *rq = this_rq();
  2746. finish_task_switch(rq, prev);
  2747. /*
  2748. * FIXME: do we need to worry about rq being invalidated by the
  2749. * task_switch?
  2750. */
  2751. post_schedule(rq);
  2752. #ifdef __ARCH_WANT_UNLOCKED_CTXSW
  2753. /* In this case, finish_task_switch does not reenable preemption */
  2754. preempt_enable();
  2755. #endif
  2756. if (current->set_child_tid)
  2757. put_user(task_pid_vnr(current), current->set_child_tid);
  2758. }
  2759. /*
  2760. * context_switch - switch to the new MM and the new
  2761. * thread's register state.
  2762. */
  2763. static inline void
  2764. context_switch(struct rq *rq, struct task_struct *prev,
  2765. struct task_struct *next)
  2766. {
  2767. struct mm_struct *mm, *oldmm;
  2768. prepare_task_switch(rq, prev, next);
  2769. mm = next->mm;
  2770. oldmm = prev->active_mm;
  2771. /*
  2772. * For paravirt, this is coupled with an exit in switch_to to
  2773. * combine the page table reload and the switch backend into
  2774. * one hypercall.
  2775. */
  2776. arch_start_context_switch(prev);
  2777. if (!mm) {
  2778. next->active_mm = oldmm;
  2779. atomic_inc(&oldmm->mm_count);
  2780. enter_lazy_tlb(oldmm, next);
  2781. } else
  2782. switch_mm(oldmm, mm, next);
  2783. if (!prev->mm) {
  2784. prev->active_mm = NULL;
  2785. rq->prev_mm = oldmm;
  2786. }
  2787. /*
  2788. * Since the runqueue lock will be released by the next
  2789. * task (which is an invalid locking op but in the case
  2790. * of the scheduler it's an obvious special-case), so we
  2791. * do an early lockdep release here:
  2792. */
  2793. #ifndef __ARCH_WANT_UNLOCKED_CTXSW
  2794. spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
  2795. #endif
  2796. /* Here we just switch the register state and the stack. */
  2797. switch_to(prev, next, prev);
  2798. barrier();
  2799. /*
  2800. * this_rq must be evaluated again because prev may have moved
  2801. * CPUs since it called schedule(), thus the 'rq' on its stack
  2802. * frame will be invalid.
  2803. */
  2804. finish_task_switch(this_rq(), prev);
  2805. }
  2806. /*
  2807. * nr_running, nr_uninterruptible and nr_context_switches:
  2808. *
  2809. * externally visible scheduler statistics: current number of runnable
  2810. * threads, current number of uninterruptible-sleeping threads, total
  2811. * number of context switches performed since bootup.
  2812. */
  2813. unsigned long nr_running(void)
  2814. {
  2815. unsigned long i, sum = 0;
  2816. for_each_online_cpu(i)
  2817. sum += cpu_rq(i)->nr_running;
  2818. return sum;
  2819. }
  2820. unsigned long nr_uninterruptible(void)
  2821. {
  2822. unsigned long i, sum = 0;
  2823. for_each_possible_cpu(i)
  2824. sum += cpu_rq(i)->nr_uninterruptible;
  2825. /*
  2826. * Since we read the counters lockless, it might be slightly
  2827. * inaccurate. Do not allow it to go below zero though:
  2828. */
  2829. if (unlikely((long)sum < 0))
  2830. sum = 0;
  2831. return sum;
  2832. }
  2833. unsigned long long nr_context_switches(void)
  2834. {
  2835. int i;
  2836. unsigned long long sum = 0;
  2837. for_each_possible_cpu(i)
  2838. sum += cpu_rq(i)->nr_switches;
  2839. return sum;
  2840. }
  2841. unsigned long nr_iowait(void)
  2842. {
  2843. unsigned long i, sum = 0;
  2844. for_each_possible_cpu(i)
  2845. sum += atomic_read(&cpu_rq(i)->nr_iowait);
  2846. return sum;
  2847. }
  2848. unsigned long nr_iowait_cpu(int cpu)
  2849. {
  2850. struct rq *this = cpu_rq(cpu);
  2851. return atomic_read(&this->nr_iowait);
  2852. }
  2853. unsigned long this_cpu_load(void)
  2854. {
  2855. struct rq *this = this_rq();
  2856. return this->cpu_load[0];
  2857. }
  2858. /* Variables and functions for calc_load */
  2859. static atomic_long_t calc_load_tasks;
  2860. static unsigned long calc_load_update;
  2861. unsigned long avenrun[3];
  2862. EXPORT_SYMBOL(avenrun);
  2863. static long calc_load_fold_active(struct rq *this_rq)
  2864. {
  2865. long nr_active, delta = 0;
  2866. nr_active = this_rq->nr_running;
  2867. nr_active += (long) this_rq->nr_uninterruptible;
  2868. if (nr_active != this_rq->calc_load_active) {
  2869. delta = nr_active - this_rq->calc_load_active;
  2870. this_rq->calc_load_active = nr_active;
  2871. }
  2872. return delta;
  2873. }
  2874. static unsigned long
  2875. calc_load(unsigned long load, unsigned long exp, unsigned long active)
  2876. {
  2877. load *= exp;
  2878. load += active * (FIXED_1 - exp);
  2879. load += 1UL << (FSHIFT - 1);
  2880. return load >> FSHIFT;
  2881. }
  2882. #ifdef CONFIG_NO_HZ
  2883. /*
  2884. * For NO_HZ we delay the active fold to the next LOAD_FREQ update.
  2885. *
  2886. * When making the ILB scale, we should try to pull this in as well.
  2887. */
  2888. static atomic_long_t calc_load_tasks_idle;
  2889. static void calc_load_account_idle(struct rq *this_rq)
  2890. {
  2891. long delta;
  2892. delta = calc_load_fold_active(this_rq);
  2893. if (delta)
  2894. atomic_long_add(delta, &calc_load_tasks_idle);
  2895. }
  2896. static long calc_load_fold_idle(void)
  2897. {
  2898. long delta = 0;
  2899. /*
  2900. * Its got a race, we don't care...
  2901. */
  2902. if (atomic_long_read(&calc_load_tasks_idle))
  2903. delta = atomic_long_xchg(&calc_load_tasks_idle, 0);
  2904. return delta;
  2905. }
  2906. /**
  2907. * fixed_power_int - compute: x^n, in O(log n) time
  2908. *
  2909. * @x: base of the power
  2910. * @frac_bits: fractional bits of @x
  2911. * @n: power to raise @x to.
  2912. *
  2913. * By exploiting the relation between the definition of the natural power
  2914. * function: x^n := x*x*...*x (x multiplied by itself for n times), and
  2915. * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i,
  2916. * (where: n_i \elem {0, 1}, the binary vector representing n),
  2917. * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is
  2918. * of course trivially computable in O(log_2 n), the length of our binary
  2919. * vector.
  2920. */
  2921. static unsigned long
  2922. fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n)
  2923. {
  2924. unsigned long result = 1UL << frac_bits;
  2925. if (n) for (;;) {
  2926. if (n & 1) {
  2927. result *= x;
  2928. result += 1UL << (frac_bits - 1);
  2929. result >>= frac_bits;
  2930. }
  2931. n >>= 1;
  2932. if (!n)
  2933. break;
  2934. x *= x;
  2935. x += 1UL << (frac_bits - 1);
  2936. x >>= frac_bits;
  2937. }
  2938. return result;
  2939. }
  2940. /*
  2941. * a1 = a0 * e + a * (1 - e)
  2942. *
  2943. * a2 = a1 * e + a * (1 - e)
  2944. * = (a0 * e + a * (1 - e)) * e + a * (1 - e)
  2945. * = a0 * e^2 + a * (1 - e) * (1 + e)
  2946. *
  2947. * a3 = a2 * e + a * (1 - e)
  2948. * = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e)
  2949. * = a0 * e^3 + a * (1 - e) * (1 + e + e^2)
  2950. *
  2951. * ...
  2952. *
  2953. * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1]
  2954. * = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e)
  2955. * = a0 * e^n + a * (1 - e^n)
  2956. *
  2957. * [1] application of the geometric series:
  2958. *
  2959. * n 1 - x^(n+1)
  2960. * S_n := \Sum x^i = -------------
  2961. * i=0 1 - x
  2962. */
  2963. static unsigned long
  2964. calc_load_n(unsigned long load, unsigned long exp,
  2965. unsigned long active, unsigned int n)
  2966. {
  2967. return calc_load(load, fixed_power_int(exp, FSHIFT, n), active);
  2968. }
  2969. /*
  2970. * NO_HZ can leave us missing all per-cpu ticks calling
  2971. * calc_load_account_active(), but since an idle CPU folds its delta into
  2972. * calc_load_tasks_idle per calc_load_account_idle(), all we need to do is fold
  2973. * in the pending idle delta if our idle period crossed a load cycle boundary.
  2974. *
  2975. * Once we've updated the global active value, we need to apply the exponential
  2976. * weights adjusted to the number of cycles missed.
  2977. */
  2978. static void calc_global_nohz(unsigned long ticks)
  2979. {
  2980. long delta, active, n;
  2981. if (time_before(jiffies, calc_load_update))
  2982. return;
  2983. /*
  2984. * If we crossed a calc_load_update boundary, make sure to fold
  2985. * any pending idle changes, the respective CPUs might have
  2986. * missed the tick driven calc_load_account_active() update
  2987. * due to NO_HZ.
  2988. */
  2989. delta = calc_load_fold_idle();
  2990. if (delta)
  2991. atomic_long_add(delta, &calc_load_tasks);
  2992. /*
  2993. * If we were idle for multiple load cycles, apply them.
  2994. */
  2995. if (ticks >= LOAD_FREQ) {
  2996. n = ticks / LOAD_FREQ;
  2997. active = atomic_long_read(&calc_load_tasks);
  2998. active = active > 0 ? active * FIXED_1 : 0;
  2999. avenrun[0] = calc_load_n(avenrun[0], EXP_1, active, n);
  3000. avenrun[1] = calc_load_n(avenrun[1], EXP_5, active, n);
  3001. avenrun[2] = calc_load_n(avenrun[2], EXP_15, active, n);
  3002. calc_load_update += n * LOAD_FREQ;
  3003. }
  3004. /*
  3005. * Its possible the remainder of the above division also crosses
  3006. * a LOAD_FREQ period, the regular check in calc_global_load()
  3007. * which comes after this will take care of that.
  3008. *
  3009. * Consider us being 11 ticks before a cycle completion, and us
  3010. * sleeping for 4*LOAD_FREQ + 22 ticks, then the above code will
  3011. * age us 4 cycles, and the test in calc_global_load() will
  3012. * pick up the final one.
  3013. */
  3014. }
  3015. #else
  3016. static void calc_load_account_idle(struct rq *this_rq)
  3017. {
  3018. }
  3019. static inline long calc_load_fold_idle(void)
  3020. {
  3021. return 0;
  3022. }
  3023. static void calc_global_nohz(unsigned long ticks)
  3024. {
  3025. }
  3026. #endif
  3027. /**
  3028. * get_avenrun - get the load average array
  3029. * @loads: pointer to dest load array
  3030. * @offset: offset to add
  3031. * @shift: shift count to shift the result left
  3032. *
  3033. * These values are estimates at best, so no need for locking.
  3034. */
  3035. void get_avenrun(unsigned long *loads, unsigned long offset, int shift)
  3036. {
  3037. loads[0] = (avenrun[0] + offset) << shift;
  3038. loads[1] = (avenrun[1] + offset) << shift;
  3039. loads[2] = (avenrun[2] + offset) << shift;
  3040. }
  3041. /*
  3042. * calc_load - update the avenrun load estimates 10 ticks after the
  3043. * CPUs have updated calc_load_tasks.
  3044. */
  3045. void calc_global_load(unsigned long ticks)
  3046. {
  3047. long active;
  3048. calc_global_nohz(ticks);
  3049. if (time_before(jiffies, calc_load_update + 10))
  3050. return;
  3051. active = atomic_long_read(&calc_load_tasks);
  3052. active = active > 0 ? active * FIXED_1 : 0;
  3053. avenrun[0] = calc_load(avenrun[0], EXP_1, active);
  3054. avenrun[1] = calc_load(avenrun[1], EXP_5, active);
  3055. avenrun[2] = calc_load(avenrun[2], EXP_15, active);
  3056. calc_load_update += LOAD_FREQ;
  3057. }
  3058. /*
  3059. * Called from update_cpu_load() to periodically update this CPU's
  3060. * active count.
  3061. */
  3062. static void calc_load_account_active(struct rq *this_rq)
  3063. {
  3064. long delta;
  3065. if (time_before(jiffies, this_rq->calc_load_update))
  3066. return;
  3067. delta = calc_load_fold_active(this_rq);
  3068. delta += calc_load_fold_idle();
  3069. if (delta)
  3070. atomic_long_add(delta, &calc_load_tasks);
  3071. this_rq->calc_load_update += LOAD_FREQ;
  3072. }
  3073. /*
  3074. * The exact cpuload at various idx values, calculated at every tick would be
  3075. * load = (2^idx - 1) / 2^idx * load + 1 / 2^idx * cur_load
  3076. *
  3077. * If a cpu misses updates for n-1 ticks (as it was idle) and update gets called
  3078. * on nth tick when cpu may be busy, then we have:
  3079. * load = ((2^idx - 1) / 2^idx)^(n-1) * load
  3080. * load = (2^idx - 1) / 2^idx) * load + 1 / 2^idx * cur_load
  3081. *
  3082. * decay_load_missed() below does efficient calculation of
  3083. * load = ((2^idx - 1) / 2^idx)^(n-1) * load
  3084. * avoiding 0..n-1 loop doing load = ((2^idx - 1) / 2^idx) * load
  3085. *
  3086. * The calculation is approximated on a 128 point scale.
  3087. * degrade_zero_ticks is the number of ticks after which load at any
  3088. * particular idx is approximated to be zero.
  3089. * degrade_factor is a precomputed table, a row for each load idx.
  3090. * Each column corresponds to degradation factor for a power of two ticks,
  3091. * based on 128 point scale.
  3092. * Example:
  3093. * row 2, col 3 (=12) says that the degradation at load idx 2 after
  3094. * 8 ticks is 12/128 (which is an approximation of exact factor 3^8/4^8).
  3095. *
  3096. * With this power of 2 load factors, we can degrade the load n times
  3097. * by looking at 1 bits in n and doing as many mult/shift instead of
  3098. * n mult/shifts needed by the exact degradation.
  3099. */
  3100. #define DEGRADE_SHIFT 7
  3101. static const unsigned char
  3102. degrade_zero_ticks[CPU_LOAD_IDX_MAX] = {0, 8, 32, 64, 128};
  3103. static const unsigned char
  3104. degrade_factor[CPU_LOAD_IDX_MAX][DEGRADE_SHIFT + 1] = {
  3105. {0, 0, 0, 0, 0, 0, 0, 0},
  3106. {64, 32, 8, 0, 0, 0, 0, 0},
  3107. {96, 72, 40, 12, 1, 0, 0},
  3108. {112, 98, 75, 43, 15, 1, 0},
  3109. {120, 112, 98, 76, 45, 16, 2} };
  3110. /*
  3111. * Update cpu_load for any missed ticks, due to tickless idle. The backlog
  3112. * would be when CPU is idle and so we just decay the old load without
  3113. * adding any new load.
  3114. */
  3115. static unsigned long
  3116. decay_load_missed(unsigned long load, unsigned long missed_updates, int idx)
  3117. {
  3118. int j = 0;
  3119. if (!missed_updates)
  3120. return load;
  3121. if (missed_updates >= degrade_zero_ticks[idx])
  3122. return 0;
  3123. if (idx == 1)
  3124. return load >> missed_updates;
  3125. while (missed_updates) {
  3126. if (missed_updates % 2)
  3127. load = (load * degrade_factor[idx][j]) >> DEGRADE_SHIFT;
  3128. missed_updates >>= 1;
  3129. j++;
  3130. }
  3131. return load;
  3132. }
  3133. /*
  3134. * Update rq->cpu_load[] statistics. This function is usually called every
  3135. * scheduler tick (TICK_NSEC). With tickless idle this will not be called
  3136. * every tick. We fix it up based on jiffies.
  3137. */
  3138. static void update_cpu_load(struct rq *this_rq)
  3139. {
  3140. unsigned long this_load = this_rq->load.weight;
  3141. unsigned long curr_jiffies = jiffies;
  3142. unsigned long pending_updates;
  3143. int i, scale;
  3144. this_rq->nr_load_updates++;
  3145. /* Avoid repeated calls on same jiffy, when moving in and out of idle */
  3146. if (curr_jiffies == this_rq->last_load_update_tick)
  3147. return;
  3148. pending_updates = curr_jiffies - this_rq->last_load_update_tick;
  3149. this_rq->last_load_update_tick = curr_jiffies;
  3150. /* Update our load: */
  3151. this_rq->cpu_load[0] = this_load; /* Fasttrack for idx 0 */
  3152. for (i = 1, scale = 2; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
  3153. unsigned long old_load, new_load;
  3154. /* scale is effectively 1 << i now, and >> i divides by scale */
  3155. old_load = this_rq->cpu_load[i];
  3156. old_load = decay_load_missed(old_load, pending_updates - 1, i);
  3157. new_load = this_load;
  3158. /*
  3159. * Round up the averaging division if load is increasing. This
  3160. * prevents us from getting stuck on 9 if the load is 10, for
  3161. * example.
  3162. */
  3163. if (new_load > old_load)
  3164. new_load += scale - 1;
  3165. this_rq->cpu_load[i] = (old_load * (scale - 1) + new_load) >> i;
  3166. }
  3167. sched_avg_update(this_rq);
  3168. }
  3169. static void update_cpu_load_active(struct rq *this_rq)
  3170. {
  3171. update_cpu_load(this_rq);
  3172. calc_load_account_active(this_rq);
  3173. }
  3174. #ifdef CONFIG_SMP
  3175. /*
  3176. * sched_exec - execve() is a valuable balancing opportunity, because at
  3177. * this point the task has the smallest effective memory and cache footprint.
  3178. */
  3179. void sched_exec(void)
  3180. {
  3181. struct task_struct *p = current;
  3182. unsigned long flags;
  3183. int dest_cpu;
  3184. raw_spin_lock_irqsave(&p->pi_lock, flags);
  3185. dest_cpu = p->sched_class->select_task_rq(p, SD_BALANCE_EXEC, 0);
  3186. if (dest_cpu == smp_processor_id())
  3187. goto unlock;
  3188. if (likely(cpu_active(dest_cpu))) {
  3189. struct migration_arg arg = { p, dest_cpu };
  3190. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  3191. stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
  3192. return;
  3193. }
  3194. unlock:
  3195. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  3196. }
  3197. #endif
  3198. DEFINE_PER_CPU(struct kernel_stat, kstat);
  3199. EXPORT_PER_CPU_SYMBOL(kstat);
  3200. /*
  3201. * Return any ns on the sched_clock that have not yet been accounted in
  3202. * @p in case that task is currently running.
  3203. *
  3204. * Called with task_rq_lock() held on @rq.
  3205. */
  3206. static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
  3207. {
  3208. u64 ns = 0;
  3209. if (task_current(rq, p)) {
  3210. update_rq_clock(rq);
  3211. ns = rq->clock_task - p->se.exec_start;
  3212. if ((s64)ns < 0)
  3213. ns = 0;
  3214. }
  3215. return ns;
  3216. }
  3217. unsigned long long task_delta_exec(struct task_struct *p)
  3218. {
  3219. unsigned long flags;
  3220. struct rq *rq;
  3221. u64 ns = 0;
  3222. rq = task_rq_lock(p, &flags);
  3223. ns = do_task_delta_exec(p, rq);
  3224. task_rq_unlock(rq, p, &flags);
  3225. return ns;
  3226. }
  3227. /*
  3228. * Return accounted runtime for the task.
  3229. * In case the task is currently running, return the runtime plus current's
  3230. * pending runtime that have not been accounted yet.
  3231. */
  3232. unsigned long long task_sched_runtime(struct task_struct *p)
  3233. {
  3234. unsigned long flags;
  3235. struct rq *rq;
  3236. u64 ns = 0;
  3237. rq = task_rq_lock(p, &flags);
  3238. ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
  3239. task_rq_unlock(rq, p, &flags);
  3240. return ns;
  3241. }
  3242. /*
  3243. * Account user cpu time to a process.
  3244. * @p: the process that the cpu time gets accounted to
  3245. * @cputime: the cpu time spent in user space since the last update
  3246. * @cputime_scaled: cputime scaled by cpu frequency
  3247. */
  3248. void account_user_time(struct task_struct *p, cputime_t cputime,
  3249. cputime_t cputime_scaled)
  3250. {
  3251. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3252. cputime64_t tmp;
  3253. /* Add user time to process. */
  3254. p->utime = cputime_add(p->utime, cputime);
  3255. p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
  3256. account_group_user_time(p, cputime);
  3257. /* Add user time to cpustat. */
  3258. tmp = cputime_to_cputime64(cputime);
  3259. if (TASK_NICE(p) > 0)
  3260. cpustat->nice = cputime64_add(cpustat->nice, tmp);
  3261. else
  3262. cpustat->user = cputime64_add(cpustat->user, tmp);
  3263. cpuacct_update_stats(p, CPUACCT_STAT_USER, cputime);
  3264. /* Account for user time used */
  3265. acct_update_integrals(p);
  3266. }
  3267. /*
  3268. * Account guest cpu time to a process.
  3269. * @p: the process that the cpu time gets accounted to
  3270. * @cputime: the cpu time spent in virtual machine since the last update
  3271. * @cputime_scaled: cputime scaled by cpu frequency
  3272. */
  3273. static void account_guest_time(struct task_struct *p, cputime_t cputime,
  3274. cputime_t cputime_scaled)
  3275. {
  3276. cputime64_t tmp;
  3277. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3278. tmp = cputime_to_cputime64(cputime);
  3279. /* Add guest time to process. */
  3280. p->utime = cputime_add(p->utime, cputime);
  3281. p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
  3282. account_group_user_time(p, cputime);
  3283. p->gtime = cputime_add(p->gtime, cputime);
  3284. /* Add guest time to cpustat. */
  3285. if (TASK_NICE(p) > 0) {
  3286. cpustat->nice = cputime64_add(cpustat->nice, tmp);
  3287. cpustat->guest_nice = cputime64_add(cpustat->guest_nice, tmp);
  3288. } else {
  3289. cpustat->user = cputime64_add(cpustat->user, tmp);
  3290. cpustat->guest = cputime64_add(cpustat->guest, tmp);
  3291. }
  3292. }
  3293. /*
  3294. * Account system cpu time to a process and desired cpustat field
  3295. * @p: the process that the cpu time gets accounted to
  3296. * @cputime: the cpu time spent in kernel space since the last update
  3297. * @cputime_scaled: cputime scaled by cpu frequency
  3298. * @target_cputime64: pointer to cpustat field that has to be updated
  3299. */
  3300. static inline
  3301. void __account_system_time(struct task_struct *p, cputime_t cputime,
  3302. cputime_t cputime_scaled, cputime64_t *target_cputime64)
  3303. {
  3304. cputime64_t tmp = cputime_to_cputime64(cputime);
  3305. /* Add system time to process. */
  3306. p->stime = cputime_add(p->stime, cputime);
  3307. p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
  3308. account_group_system_time(p, cputime);
  3309. /* Add system time to cpustat. */
  3310. *target_cputime64 = cputime64_add(*target_cputime64, tmp);
  3311. cpuacct_update_stats(p, CPUACCT_STAT_SYSTEM, cputime);
  3312. /* Account for system time used */
  3313. acct_update_integrals(p);
  3314. }
  3315. /*
  3316. * Account system cpu time to a process.
  3317. * @p: the process that the cpu time gets accounted to
  3318. * @hardirq_offset: the offset to subtract from hardirq_count()
  3319. * @cputime: the cpu time spent in kernel space since the last update
  3320. * @cputime_scaled: cputime scaled by cpu frequency
  3321. */
  3322. void account_system_time(struct task_struct *p, int hardirq_offset,
  3323. cputime_t cputime, cputime_t cputime_scaled)
  3324. {
  3325. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3326. cputime64_t *target_cputime64;
  3327. if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
  3328. account_guest_time(p, cputime, cputime_scaled);
  3329. return;
  3330. }
  3331. if (hardirq_count() - hardirq_offset)
  3332. target_cputime64 = &cpustat->irq;
  3333. else if (in_serving_softirq())
  3334. target_cputime64 = &cpustat->softirq;
  3335. else
  3336. target_cputime64 = &cpustat->system;
  3337. __account_system_time(p, cputime, cputime_scaled, target_cputime64);
  3338. }
  3339. /*
  3340. * Account for involuntary wait time.
  3341. * @cputime: the cpu time spent in involuntary wait
  3342. */
  3343. void account_steal_time(cputime_t cputime)
  3344. {
  3345. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3346. cputime64_t cputime64 = cputime_to_cputime64(cputime);
  3347. cpustat->steal = cputime64_add(cpustat->steal, cputime64);
  3348. }
  3349. /*
  3350. * Account for idle time.
  3351. * @cputime: the cpu time spent in idle wait
  3352. */
  3353. void account_idle_time(cputime_t cputime)
  3354. {
  3355. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3356. cputime64_t cputime64 = cputime_to_cputime64(cputime);
  3357. struct rq *rq = this_rq();
  3358. if (atomic_read(&rq->nr_iowait) > 0)
  3359. cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
  3360. else
  3361. cpustat->idle = cputime64_add(cpustat->idle, cputime64);
  3362. }
  3363. static __always_inline bool steal_account_process_tick(void)
  3364. {
  3365. #ifdef CONFIG_PARAVIRT
  3366. if (static_branch(&paravirt_steal_enabled)) {
  3367. u64 steal, st = 0;
  3368. steal = paravirt_steal_clock(smp_processor_id());
  3369. steal -= this_rq()->prev_steal_time;
  3370. st = steal_ticks(steal);
  3371. this_rq()->prev_steal_time += st * TICK_NSEC;
  3372. account_steal_time(st);
  3373. return st;
  3374. }
  3375. #endif
  3376. return false;
  3377. }
  3378. #ifndef CONFIG_VIRT_CPU_ACCOUNTING
  3379. #ifdef CONFIG_IRQ_TIME_ACCOUNTING
  3380. /*
  3381. * Account a tick to a process and cpustat
  3382. * @p: the process that the cpu time gets accounted to
  3383. * @user_tick: is the tick from userspace
  3384. * @rq: the pointer to rq
  3385. *
  3386. * Tick demultiplexing follows the order
  3387. * - pending hardirq update
  3388. * - pending softirq update
  3389. * - user_time
  3390. * - idle_time
  3391. * - system time
  3392. * - check for guest_time
  3393. * - else account as system_time
  3394. *
  3395. * Check for hardirq is done both for system and user time as there is
  3396. * no timer going off while we are on hardirq and hence we may never get an
  3397. * opportunity to update it solely in system time.
  3398. * p->stime and friends are only updated on system time and not on irq
  3399. * softirq as those do not count in task exec_runtime any more.
  3400. */
  3401. static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
  3402. struct rq *rq)
  3403. {
  3404. cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
  3405. cputime64_t tmp = cputime_to_cputime64(cputime_one_jiffy);
  3406. struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
  3407. if (steal_account_process_tick())
  3408. return;
  3409. if (irqtime_account_hi_update()) {
  3410. cpustat->irq = cputime64_add(cpustat->irq, tmp);
  3411. } else if (irqtime_account_si_update()) {
  3412. cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
  3413. } else if (this_cpu_ksoftirqd() == p) {
  3414. /*
  3415. * ksoftirqd time do not get accounted in cpu_softirq_time.
  3416. * So, we have to handle it separately here.
  3417. * Also, p->stime needs to be updated for ksoftirqd.
  3418. */
  3419. __account_system_time(p, cputime_one_jiffy, one_jiffy_scaled,
  3420. &cpustat->softirq);
  3421. } else if (user_tick) {
  3422. account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
  3423. } else if (p == rq->idle) {
  3424. account_idle_time(cputime_one_jiffy);
  3425. } else if (p->flags & PF_VCPU) { /* System time or guest time */
  3426. account_guest_time(p, cputime_one_jiffy, one_jiffy_scaled);
  3427. } else {
  3428. __account_system_time(p, cputime_one_jiffy, one_jiffy_scaled,
  3429. &cpustat->system);
  3430. }
  3431. }
  3432. static void irqtime_account_idle_ticks(int ticks)
  3433. {
  3434. int i;
  3435. struct rq *rq = this_rq();
  3436. for (i = 0; i < ticks; i++)
  3437. irqtime_account_process_tick(current, 0, rq);
  3438. }
  3439. #else /* CONFIG_IRQ_TIME_ACCOUNTING */
  3440. static void irqtime_account_idle_ticks(int ticks) {}
  3441. static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
  3442. struct rq *rq) {}
  3443. #endif /* CONFIG_IRQ_TIME_ACCOUNTING */
  3444. /*
  3445. * Account a single tick of cpu time.
  3446. * @p: the process that the cpu time gets accounted to
  3447. * @user_tick: indicates if the tick is a user or a system tick
  3448. */
  3449. void account_process_tick(struct task_struct *p, int user_tick)
  3450. {
  3451. cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
  3452. struct rq *rq = this_rq();
  3453. if (sched_clock_irqtime) {
  3454. irqtime_account_process_tick(p, user_tick, rq);
  3455. return;
  3456. }
  3457. if (steal_account_process_tick())
  3458. return;
  3459. if (user_tick)
  3460. account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
  3461. else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET))
  3462. account_system_time(p, HARDIRQ_OFFSET, cputime_one_jiffy,
  3463. one_jiffy_scaled);
  3464. else
  3465. account_idle_time(cputime_one_jiffy);
  3466. }
  3467. /*
  3468. * Account multiple ticks of steal time.
  3469. * @p: the process from which the cpu time has been stolen
  3470. * @ticks: number of stolen ticks
  3471. */
  3472. void account_steal_ticks(unsigned long ticks)
  3473. {
  3474. account_steal_time(jiffies_to_cputime(ticks));
  3475. }
  3476. /*
  3477. * Account multiple ticks of idle time.
  3478. * @ticks: number of stolen ticks
  3479. */
  3480. void account_idle_ticks(unsigned long ticks)
  3481. {
  3482. if (sched_clock_irqtime) {
  3483. irqtime_account_idle_ticks(ticks);
  3484. return;
  3485. }
  3486. account_idle_time(jiffies_to_cputime(ticks));
  3487. }
  3488. #endif
  3489. /*
  3490. * Use precise platform statistics if available:
  3491. */
  3492. #ifdef CONFIG_VIRT_CPU_ACCOUNTING
  3493. void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
  3494. {
  3495. *ut = p->utime;
  3496. *st = p->stime;
  3497. }
  3498. void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
  3499. {
  3500. struct task_cputime cputime;
  3501. thread_group_cputime(p, &cputime);
  3502. *ut = cputime.utime;
  3503. *st = cputime.stime;
  3504. }
  3505. #else
  3506. #ifndef nsecs_to_cputime
  3507. # define nsecs_to_cputime(__nsecs) nsecs_to_jiffies(__nsecs)
  3508. #endif
  3509. void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
  3510. {
  3511. cputime_t rtime, utime = p->utime, total = cputime_add(utime, p->stime);
  3512. /*
  3513. * Use CFS's precise accounting:
  3514. */
  3515. rtime = nsecs_to_cputime(p->se.sum_exec_runtime);
  3516. if (total) {
  3517. u64 temp = rtime;
  3518. temp *= utime;
  3519. do_div(temp, total);
  3520. utime = (cputime_t)temp;
  3521. } else
  3522. utime = rtime;
  3523. /*
  3524. * Compare with previous values, to keep monotonicity:
  3525. */
  3526. p->prev_utime = max(p->prev_utime, utime);
  3527. p->prev_stime = max(p->prev_stime, cputime_sub(rtime, p->prev_utime));
  3528. *ut = p->prev_utime;
  3529. *st = p->prev_stime;
  3530. }
  3531. /*
  3532. * Must be called with siglock held.
  3533. */
  3534. void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
  3535. {
  3536. struct signal_struct *sig = p->signal;
  3537. struct task_cputime cputime;
  3538. cputime_t rtime, utime, total;
  3539. thread_group_cputime(p, &cputime);
  3540. total = cputime_add(cputime.utime, cputime.stime);
  3541. rtime = nsecs_to_cputime(cputime.sum_exec_runtime);
  3542. if (total) {
  3543. u64 temp = rtime;
  3544. temp *= cputime.utime;
  3545. do_div(temp, total);
  3546. utime = (cputime_t)temp;
  3547. } else
  3548. utime = rtime;
  3549. sig->prev_utime = max(sig->prev_utime, utime);
  3550. sig->prev_stime = max(sig->prev_stime,
  3551. cputime_sub(rtime, sig->prev_utime));
  3552. *ut = sig->prev_utime;
  3553. *st = sig->prev_stime;
  3554. }
  3555. #endif
  3556. /*
  3557. * This function gets called by the timer code, with HZ frequency.
  3558. * We call it with interrupts disabled.
  3559. */
  3560. void scheduler_tick(void)
  3561. {
  3562. int cpu = smp_processor_id();
  3563. struct rq *rq = cpu_rq(cpu);
  3564. struct task_struct *curr = rq->curr;
  3565. sched_clock_tick();
  3566. raw_spin_lock(&rq->lock);
  3567. update_rq_clock(rq);
  3568. update_cpu_load_active(rq);
  3569. curr->sched_class->task_tick(rq, curr, 0);
  3570. raw_spin_unlock(&rq->lock);
  3571. perf_event_task_tick();
  3572. #ifdef CONFIG_SMP
  3573. rq->idle_at_tick = idle_cpu(cpu);
  3574. trigger_load_balance(rq, cpu);
  3575. #endif
  3576. }
  3577. notrace unsigned long get_parent_ip(unsigned long addr)
  3578. {
  3579. if (in_lock_functions(addr)) {
  3580. addr = CALLER_ADDR2;
  3581. if (in_lock_functions(addr))
  3582. addr = CALLER_ADDR3;
  3583. }
  3584. return addr;
  3585. }
  3586. #if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
  3587. defined(CONFIG_PREEMPT_TRACER))
  3588. void __kprobes add_preempt_count(int val)
  3589. {
  3590. #ifdef CONFIG_DEBUG_PREEMPT
  3591. /*
  3592. * Underflow?
  3593. */
  3594. if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
  3595. return;
  3596. #endif
  3597. preempt_count() += val;
  3598. #ifdef CONFIG_DEBUG_PREEMPT
  3599. /*
  3600. * Spinlock count overflowing soon?
  3601. */
  3602. DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
  3603. PREEMPT_MASK - 10);
  3604. #endif
  3605. if (preempt_count() == val)
  3606. trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
  3607. }
  3608. EXPORT_SYMBOL(add_preempt_count);
  3609. void __kprobes sub_preempt_count(int val)
  3610. {
  3611. #ifdef CONFIG_DEBUG_PREEMPT
  3612. /*
  3613. * Underflow?
  3614. */
  3615. if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
  3616. return;
  3617. /*
  3618. * Is the spinlock portion underflowing?
  3619. */
  3620. if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
  3621. !(preempt_count() & PREEMPT_MASK)))
  3622. return;
  3623. #endif
  3624. if (preempt_count() == val)
  3625. trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
  3626. preempt_count() -= val;
  3627. }
  3628. EXPORT_SYMBOL(sub_preempt_count);
  3629. #endif
  3630. /*
  3631. * Print scheduling while atomic bug:
  3632. */
  3633. static noinline void __schedule_bug(struct task_struct *prev)
  3634. {
  3635. struct pt_regs *regs = get_irq_regs();
  3636. printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
  3637. prev->comm, prev->pid, preempt_count());
  3638. debug_show_held_locks(prev);
  3639. print_modules();
  3640. if (irqs_disabled())
  3641. print_irqtrace_events(prev);
  3642. if (regs)
  3643. show_regs(regs);
  3644. else
  3645. dump_stack();
  3646. }
  3647. /*
  3648. * Various schedule()-time debugging checks and statistics:
  3649. */
  3650. static inline void schedule_debug(struct task_struct *prev)
  3651. {
  3652. /*
  3653. * Test if we are atomic. Since do_exit() needs to call into
  3654. * schedule() atomically, we ignore that path for now.
  3655. * Otherwise, whine if we are scheduling when we should not be.
  3656. */
  3657. if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
  3658. __schedule_bug(prev);
  3659. profile_hit(SCHED_PROFILING, __builtin_return_address(0));
  3660. schedstat_inc(this_rq(), sched_count);
  3661. }
  3662. static void put_prev_task(struct rq *rq, struct task_struct *prev)
  3663. {
  3664. if (prev->on_rq || rq->skip_clock_update < 0)
  3665. update_rq_clock(rq);
  3666. prev->sched_class->put_prev_task(rq, prev);
  3667. }
  3668. /*
  3669. * Pick up the highest-prio task:
  3670. */
  3671. static inline struct task_struct *
  3672. pick_next_task(struct rq *rq)
  3673. {
  3674. const struct sched_class *class;
  3675. struct task_struct *p;
  3676. /*
  3677. * Optimization: we know that if all tasks are in
  3678. * the fair class we can call that function directly:
  3679. */
  3680. if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
  3681. p = fair_sched_class.pick_next_task(rq);
  3682. if (likely(p))
  3683. return p;
  3684. }
  3685. for_each_class(class) {
  3686. p = class->pick_next_task(rq);
  3687. if (p)
  3688. return p;
  3689. }
  3690. BUG(); /* the idle class will always have a runnable task */
  3691. }
  3692. /*
  3693. * __schedule() is the main scheduler function.
  3694. */
  3695. static void __sched __schedule(void)
  3696. {
  3697. struct task_struct *prev, *next;
  3698. unsigned long *switch_count;
  3699. struct rq *rq;
  3700. int cpu;
  3701. need_resched:
  3702. preempt_disable();
  3703. cpu = smp_processor_id();
  3704. rq = cpu_rq(cpu);
  3705. rcu_note_context_switch(cpu);
  3706. prev = rq->curr;
  3707. schedule_debug(prev);
  3708. if (sched_feat(HRTICK))
  3709. hrtick_clear(rq);
  3710. raw_spin_lock_irq(&rq->lock);
  3711. switch_count = &prev->nivcsw;
  3712. if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
  3713. if (unlikely(signal_pending_state(prev->state, prev))) {
  3714. prev->state = TASK_RUNNING;
  3715. } else {
  3716. deactivate_task(rq, prev, DEQUEUE_SLEEP);
  3717. prev->on_rq = 0;
  3718. /*
  3719. * If a worker went to sleep, notify and ask workqueue
  3720. * whether it wants to wake up a task to maintain
  3721. * concurrency.
  3722. */
  3723. if (prev->flags & PF_WQ_WORKER) {
  3724. struct task_struct *to_wakeup;
  3725. to_wakeup = wq_worker_sleeping(prev, cpu);
  3726. if (to_wakeup)
  3727. try_to_wake_up_local(to_wakeup);
  3728. }
  3729. }
  3730. switch_count = &prev->nvcsw;
  3731. }
  3732. pre_schedule(rq, prev);
  3733. if (unlikely(!rq->nr_running))
  3734. idle_balance(cpu, rq);
  3735. put_prev_task(rq, prev);
  3736. next = pick_next_task(rq);
  3737. clear_tsk_need_resched(prev);
  3738. rq->skip_clock_update = 0;
  3739. if (likely(prev != next)) {
  3740. rq->nr_switches++;
  3741. rq->curr = next;
  3742. ++*switch_count;
  3743. context_switch(rq, prev, next); /* unlocks the rq */
  3744. /*
  3745. * The context switch have flipped the stack from under us
  3746. * and restored the local variables which were saved when
  3747. * this task called schedule() in the past. prev == current
  3748. * is still correct, but it can be moved to another cpu/rq.
  3749. */
  3750. cpu = smp_processor_id();
  3751. rq = cpu_rq(cpu);
  3752. } else
  3753. raw_spin_unlock_irq(&rq->lock);
  3754. post_schedule(rq);
  3755. preempt_enable_no_resched();
  3756. if (need_resched())
  3757. goto need_resched;
  3758. }
  3759. static inline void sched_submit_work(struct task_struct *tsk)
  3760. {
  3761. if (!tsk->state)
  3762. return;
  3763. /*
  3764. * If we are going to sleep and we have plugged IO queued,
  3765. * make sure to submit it to avoid deadlocks.
  3766. */
  3767. if (blk_needs_flush_plug(tsk))
  3768. blk_schedule_flush_plug(tsk);
  3769. }
  3770. asmlinkage void __sched schedule(void)
  3771. {
  3772. struct task_struct *tsk = current;
  3773. sched_submit_work(tsk);
  3774. __schedule();
  3775. }
  3776. EXPORT_SYMBOL(schedule);
  3777. #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
  3778. static inline bool owner_running(struct mutex *lock, struct task_struct *owner)
  3779. {
  3780. if (lock->owner != owner)
  3781. return false;
  3782. /*
  3783. * Ensure we emit the owner->on_cpu, dereference _after_ checking
  3784. * lock->owner still matches owner, if that fails, owner might
  3785. * point to free()d memory, if it still matches, the rcu_read_lock()
  3786. * ensures the memory stays valid.
  3787. */
  3788. barrier();
  3789. return owner->on_cpu;
  3790. }
  3791. /*
  3792. * Look out! "owner" is an entirely speculative pointer
  3793. * access and not reliable.
  3794. */
  3795. int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
  3796. {
  3797. if (!sched_feat(OWNER_SPIN))
  3798. return 0;
  3799. rcu_read_lock();
  3800. while (owner_running(lock, owner)) {
  3801. if (need_resched())
  3802. break;
  3803. arch_mutex_cpu_relax();
  3804. }
  3805. rcu_read_unlock();
  3806. /*
  3807. * We break out the loop above on need_resched() and when the
  3808. * owner changed, which is a sign for heavy contention. Return
  3809. * success only when lock->owner is NULL.
  3810. */
  3811. return lock->owner == NULL;
  3812. }
  3813. #endif
  3814. #ifdef CONFIG_PREEMPT
  3815. /*
  3816. * this is the entry point to schedule() from in-kernel preemption
  3817. * off of preempt_enable. Kernel preemptions off return from interrupt
  3818. * occur there and call schedule directly.
  3819. */
  3820. asmlinkage void __sched notrace preempt_schedule(void)
  3821. {
  3822. struct thread_info *ti = current_thread_info();
  3823. /*
  3824. * If there is a non-zero preempt_count or interrupts are disabled,
  3825. * we do not want to preempt the current task. Just return..
  3826. */
  3827. if (likely(ti->preempt_count || irqs_disabled()))
  3828. return;
  3829. do {
  3830. add_preempt_count_notrace(PREEMPT_ACTIVE);
  3831. __schedule();
  3832. sub_preempt_count_notrace(PREEMPT_ACTIVE);
  3833. /*
  3834. * Check again in case we missed a preemption opportunity
  3835. * between schedule and now.
  3836. */
  3837. barrier();
  3838. } while (need_resched());
  3839. }
  3840. EXPORT_SYMBOL(preempt_schedule);
  3841. /*
  3842. * this is the entry point to schedule() from kernel preemption
  3843. * off of irq context.
  3844. * Note, that this is called and return with irqs disabled. This will
  3845. * protect us against recursive calling from irq.
  3846. */
  3847. asmlinkage void __sched preempt_schedule_irq(void)
  3848. {
  3849. struct thread_info *ti = current_thread_info();
  3850. /* Catch callers which need to be fixed */
  3851. BUG_ON(ti->preempt_count || !irqs_disabled());
  3852. do {
  3853. add_preempt_count(PREEMPT_ACTIVE);
  3854. local_irq_enable();
  3855. __schedule();
  3856. local_irq_disable();
  3857. sub_preempt_count(PREEMPT_ACTIVE);
  3858. /*
  3859. * Check again in case we missed a preemption opportunity
  3860. * between schedule and now.
  3861. */
  3862. barrier();
  3863. } while (need_resched());
  3864. }
  3865. #endif /* CONFIG_PREEMPT */
  3866. int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
  3867. void *key)
  3868. {
  3869. return try_to_wake_up(curr->private, mode, wake_flags);
  3870. }
  3871. EXPORT_SYMBOL(default_wake_function);
  3872. /*
  3873. * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
  3874. * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
  3875. * number) then we wake all the non-exclusive tasks and one exclusive task.
  3876. *
  3877. * There are circumstances in which we can try to wake a task which has already
  3878. * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
  3879. * zero in this (rare) case, and we handle it by continuing to scan the queue.
  3880. */
  3881. static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
  3882. int nr_exclusive, int wake_flags, void *key)
  3883. {
  3884. wait_queue_t *curr, *next;
  3885. list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
  3886. unsigned flags = curr->flags;
  3887. if (curr->func(curr, mode, wake_flags, key) &&
  3888. (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
  3889. break;
  3890. }
  3891. }
  3892. /**
  3893. * __wake_up - wake up threads blocked on a waitqueue.
  3894. * @q: the waitqueue
  3895. * @mode: which threads
  3896. * @nr_exclusive: how many wake-one or wake-many threads to wake up
  3897. * @key: is directly passed to the wakeup function
  3898. *
  3899. * It may be assumed that this function implies a write memory barrier before
  3900. * changing the task state if and only if any tasks are woken up.
  3901. */
  3902. void __wake_up(wait_queue_head_t *q, unsigned int mode,
  3903. int nr_exclusive, void *key)
  3904. {
  3905. unsigned long flags;
  3906. spin_lock_irqsave(&q->lock, flags);
  3907. __wake_up_common(q, mode, nr_exclusive, 0, key);
  3908. spin_unlock_irqrestore(&q->lock, flags);
  3909. }
  3910. EXPORT_SYMBOL(__wake_up);
  3911. /*
  3912. * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
  3913. */
  3914. void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
  3915. {
  3916. __wake_up_common(q, mode, 1, 0, NULL);
  3917. }
  3918. EXPORT_SYMBOL_GPL(__wake_up_locked);
  3919. void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)
  3920. {
  3921. __wake_up_common(q, mode, 1, 0, key);
  3922. }
  3923. EXPORT_SYMBOL_GPL(__wake_up_locked_key);
  3924. /**
  3925. * __wake_up_sync_key - wake up threads blocked on a waitqueue.
  3926. * @q: the waitqueue
  3927. * @mode: which threads
  3928. * @nr_exclusive: how many wake-one or wake-many threads to wake up
  3929. * @key: opaque value to be passed to wakeup targets
  3930. *
  3931. * The sync wakeup differs that the waker knows that it will schedule
  3932. * away soon, so while the target thread will be woken up, it will not
  3933. * be migrated to another CPU - ie. the two threads are 'synchronized'
  3934. * with each other. This can prevent needless bouncing between CPUs.
  3935. *
  3936. * On UP it can prevent extra preemption.
  3937. *
  3938. * It may be assumed that this function implies a write memory barrier before
  3939. * changing the task state if and only if any tasks are woken up.
  3940. */
  3941. void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
  3942. int nr_exclusive, void *key)
  3943. {
  3944. unsigned long flags;
  3945. int wake_flags = WF_SYNC;
  3946. if (unlikely(!q))
  3947. return;
  3948. if (unlikely(!nr_exclusive))
  3949. wake_flags = 0;
  3950. spin_lock_irqsave(&q->lock, flags);
  3951. __wake_up_common(q, mode, nr_exclusive, wake_flags, key);
  3952. spin_unlock_irqrestore(&q->lock, flags);
  3953. }
  3954. EXPORT_SYMBOL_GPL(__wake_up_sync_key);
  3955. /*
  3956. * __wake_up_sync - see __wake_up_sync_key()
  3957. */
  3958. void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
  3959. {
  3960. __wake_up_sync_key(q, mode, nr_exclusive, NULL);
  3961. }
  3962. EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */
  3963. /**
  3964. * complete: - signals a single thread waiting on this completion
  3965. * @x: holds the state of this particular completion
  3966. *
  3967. * This will wake up a single thread waiting on this completion. Threads will be
  3968. * awakened in the same order in which they were queued.
  3969. *
  3970. * See also complete_all(), wait_for_completion() and related routines.
  3971. *
  3972. * It may be assumed that this function implies a write memory barrier before
  3973. * changing the task state if and only if any tasks are woken up.
  3974. */
  3975. void complete(struct completion *x)
  3976. {
  3977. unsigned long flags;
  3978. spin_lock_irqsave(&x->wait.lock, flags);
  3979. x->done++;
  3980. __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
  3981. spin_unlock_irqrestore(&x->wait.lock, flags);
  3982. }
  3983. EXPORT_SYMBOL(complete);
  3984. /**
  3985. * complete_all: - signals all threads waiting on this completion
  3986. * @x: holds the state of this particular completion
  3987. *
  3988. * This will wake up all threads waiting on this particular completion event.
  3989. *
  3990. * It may be assumed that this function implies a write memory barrier before
  3991. * changing the task state if and only if any tasks are woken up.
  3992. */
  3993. void complete_all(struct completion *x)
  3994. {
  3995. unsigned long flags;
  3996. spin_lock_irqsave(&x->wait.lock, flags);
  3997. x->done += UINT_MAX/2;
  3998. __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
  3999. spin_unlock_irqrestore(&x->wait.lock, flags);
  4000. }
  4001. EXPORT_SYMBOL(complete_all);
  4002. static inline long __sched
  4003. do_wait_for_common(struct completion *x, long timeout, int state)
  4004. {
  4005. if (!x->done) {
  4006. DECLARE_WAITQUEUE(wait, current);
  4007. __add_wait_queue_tail_exclusive(&x->wait, &wait);
  4008. do {
  4009. if (signal_pending_state(state, current)) {
  4010. timeout = -ERESTARTSYS;
  4011. break;
  4012. }
  4013. __set_current_state(state);
  4014. spin_unlock_irq(&x->wait.lock);
  4015. timeout = schedule_timeout(timeout);
  4016. spin_lock_irq(&x->wait.lock);
  4017. } while (!x->done && timeout);
  4018. __remove_wait_queue(&x->wait, &wait);
  4019. if (!x->done)
  4020. return timeout;
  4021. }
  4022. x->done--;
  4023. return timeout ?: 1;
  4024. }
  4025. static long __sched
  4026. wait_for_common(struct completion *x, long timeout, int state)
  4027. {
  4028. might_sleep();
  4029. spin_lock_irq(&x->wait.lock);
  4030. timeout = do_wait_for_common(x, timeout, state);
  4031. spin_unlock_irq(&x->wait.lock);
  4032. return timeout;
  4033. }
  4034. /**
  4035. * wait_for_completion: - waits for completion of a task
  4036. * @x: holds the state of this particular completion
  4037. *
  4038. * This waits to be signaled for completion of a specific task. It is NOT
  4039. * interruptible and there is no timeout.
  4040. *
  4041. * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
  4042. * and interrupt capability. Also see complete().
  4043. */
  4044. void __sched wait_for_completion(struct completion *x)
  4045. {
  4046. wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
  4047. }
  4048. EXPORT_SYMBOL(wait_for_completion);
  4049. /**
  4050. * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
  4051. * @x: holds the state of this particular completion
  4052. * @timeout: timeout value in jiffies
  4053. *
  4054. * This waits for either a completion of a specific task to be signaled or for a
  4055. * specified timeout to expire. The timeout is in jiffies. It is not
  4056. * interruptible.
  4057. */
  4058. unsigned long __sched
  4059. wait_for_completion_timeout(struct completion *x, unsigned long timeout)
  4060. {
  4061. return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
  4062. }
  4063. EXPORT_SYMBOL(wait_for_completion_timeout);
  4064. /**
  4065. * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
  4066. * @x: holds the state of this particular completion
  4067. *
  4068. * This waits for completion of a specific task to be signaled. It is
  4069. * interruptible.
  4070. */
  4071. int __sched wait_for_completion_interruptible(struct completion *x)
  4072. {
  4073. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
  4074. if (t == -ERESTARTSYS)
  4075. return t;
  4076. return 0;
  4077. }
  4078. EXPORT_SYMBOL(wait_for_completion_interruptible);
  4079. /**
  4080. * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
  4081. * @x: holds the state of this particular completion
  4082. * @timeout: timeout value in jiffies
  4083. *
  4084. * This waits for either a completion of a specific task to be signaled or for a
  4085. * specified timeout to expire. It is interruptible. The timeout is in jiffies.
  4086. */
  4087. long __sched
  4088. wait_for_completion_interruptible_timeout(struct completion *x,
  4089. unsigned long timeout)
  4090. {
  4091. return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
  4092. }
  4093. EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
  4094. /**
  4095. * wait_for_completion_killable: - waits for completion of a task (killable)
  4096. * @x: holds the state of this particular completion
  4097. *
  4098. * This waits to be signaled for completion of a specific task. It can be
  4099. * interrupted by a kill signal.
  4100. */
  4101. int __sched wait_for_completion_killable(struct completion *x)
  4102. {
  4103. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
  4104. if (t == -ERESTARTSYS)
  4105. return t;
  4106. return 0;
  4107. }
  4108. EXPORT_SYMBOL(wait_for_completion_killable);
  4109. /**
  4110. * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
  4111. * @x: holds the state of this particular completion
  4112. * @timeout: timeout value in jiffies
  4113. *
  4114. * This waits for either a completion of a specific task to be
  4115. * signaled or for a specified timeout to expire. It can be
  4116. * interrupted by a kill signal. The timeout is in jiffies.
  4117. */
  4118. long __sched
  4119. wait_for_completion_killable_timeout(struct completion *x,
  4120. unsigned long timeout)
  4121. {
  4122. return wait_for_common(x, timeout, TASK_KILLABLE);
  4123. }
  4124. EXPORT_SYMBOL(wait_for_completion_killable_timeout);
  4125. /**
  4126. * try_wait_for_completion - try to decrement a completion without blocking
  4127. * @x: completion structure
  4128. *
  4129. * Returns: 0 if a decrement cannot be done without blocking
  4130. * 1 if a decrement succeeded.
  4131. *
  4132. * If a completion is being used as a counting completion,
  4133. * attempt to decrement the counter without blocking. This
  4134. * enables us to avoid waiting if the resource the completion
  4135. * is protecting is not available.
  4136. */
  4137. bool try_wait_for_completion(struct completion *x)
  4138. {
  4139. unsigned long flags;
  4140. int ret = 1;
  4141. spin_lock_irqsave(&x->wait.lock, flags);
  4142. if (!x->done)
  4143. ret = 0;
  4144. else
  4145. x->done--;
  4146. spin_unlock_irqrestore(&x->wait.lock, flags);
  4147. return ret;
  4148. }
  4149. EXPORT_SYMBOL(try_wait_for_completion);
  4150. /**
  4151. * completion_done - Test to see if a completion has any waiters
  4152. * @x: completion structure
  4153. *
  4154. * Returns: 0 if there are waiters (wait_for_completion() in progress)
  4155. * 1 if there are no waiters.
  4156. *
  4157. */
  4158. bool completion_done(struct completion *x)
  4159. {
  4160. unsigned long flags;
  4161. int ret = 1;
  4162. spin_lock_irqsave(&x->wait.lock, flags);
  4163. if (!x->done)
  4164. ret = 0;
  4165. spin_unlock_irqrestore(&x->wait.lock, flags);
  4166. return ret;
  4167. }
  4168. EXPORT_SYMBOL(completion_done);
  4169. static long __sched
  4170. sleep_on_common(wait_queue_head_t *q, int state, long timeout)
  4171. {
  4172. unsigned long flags;
  4173. wait_queue_t wait;
  4174. init_waitqueue_entry(&wait, current);
  4175. __set_current_state(state);
  4176. spin_lock_irqsave(&q->lock, flags);
  4177. __add_wait_queue(q, &wait);
  4178. spin_unlock(&q->lock);
  4179. timeout = schedule_timeout(timeout);
  4180. spin_lock_irq(&q->lock);
  4181. __remove_wait_queue(q, &wait);
  4182. spin_unlock_irqrestore(&q->lock, flags);
  4183. return timeout;
  4184. }
  4185. void __sched interruptible_sleep_on(wait_queue_head_t *q)
  4186. {
  4187. sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
  4188. }
  4189. EXPORT_SYMBOL(interruptible_sleep_on);
  4190. long __sched
  4191. interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
  4192. {
  4193. return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
  4194. }
  4195. EXPORT_SYMBOL(interruptible_sleep_on_timeout);
  4196. void __sched sleep_on(wait_queue_head_t *q)
  4197. {
  4198. sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
  4199. }
  4200. EXPORT_SYMBOL(sleep_on);
  4201. long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
  4202. {
  4203. return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
  4204. }
  4205. EXPORT_SYMBOL(sleep_on_timeout);
  4206. #ifdef CONFIG_RT_MUTEXES
  4207. /*
  4208. * rt_mutex_setprio - set the current priority of a task
  4209. * @p: task
  4210. * @prio: prio value (kernel-internal form)
  4211. *
  4212. * This function changes the 'effective' priority of a task. It does
  4213. * not touch ->normal_prio like __setscheduler().
  4214. *
  4215. * Used by the rt_mutex code to implement priority inheritance logic.
  4216. */
  4217. void rt_mutex_setprio(struct task_struct *p, int prio)
  4218. {
  4219. int oldprio, on_rq, running;
  4220. struct rq *rq;
  4221. const struct sched_class *prev_class;
  4222. BUG_ON(prio < 0 || prio > MAX_PRIO);
  4223. rq = __task_rq_lock(p);
  4224. trace_sched_pi_setprio(p, prio);
  4225. oldprio = p->prio;
  4226. prev_class = p->sched_class;
  4227. on_rq = p->on_rq;
  4228. running = task_current(rq, p);
  4229. if (on_rq)
  4230. dequeue_task(rq, p, 0);
  4231. if (running)
  4232. p->sched_class->put_prev_task(rq, p);
  4233. if (rt_prio(prio))
  4234. p->sched_class = &rt_sched_class;
  4235. else
  4236. p->sched_class = &fair_sched_class;
  4237. p->prio = prio;
  4238. if (running)
  4239. p->sched_class->set_curr_task(rq);
  4240. if (on_rq)
  4241. enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0);
  4242. check_class_changed(rq, p, prev_class, oldprio);
  4243. __task_rq_unlock(rq);
  4244. }
  4245. #endif
  4246. void set_user_nice(struct task_struct *p, long nice)
  4247. {
  4248. int old_prio, delta, on_rq;
  4249. unsigned long flags;
  4250. struct rq *rq;
  4251. if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
  4252. return;
  4253. /*
  4254. * We have to be careful, if called from sys_setpriority(),
  4255. * the task might be in the middle of scheduling on another CPU.
  4256. */
  4257. rq = task_rq_lock(p, &flags);
  4258. /*
  4259. * The RT priorities are set via sched_setscheduler(), but we still
  4260. * allow the 'normal' nice value to be set - but as expected
  4261. * it wont have any effect on scheduling until the task is
  4262. * SCHED_FIFO/SCHED_RR:
  4263. */
  4264. if (task_has_rt_policy(p)) {
  4265. p->static_prio = NICE_TO_PRIO(nice);
  4266. goto out_unlock;
  4267. }
  4268. on_rq = p->on_rq;
  4269. if (on_rq)
  4270. dequeue_task(rq, p, 0);
  4271. p->static_prio = NICE_TO_PRIO(nice);
  4272. set_load_weight(p);
  4273. old_prio = p->prio;
  4274. p->prio = effective_prio(p);
  4275. delta = p->prio - old_prio;
  4276. if (on_rq) {
  4277. enqueue_task(rq, p, 0);
  4278. /*
  4279. * If the task increased its priority or is running and
  4280. * lowered its priority, then reschedule its CPU:
  4281. */
  4282. if (delta < 0 || (delta > 0 && task_running(rq, p)))
  4283. resched_task(rq->curr);
  4284. }
  4285. out_unlock:
  4286. task_rq_unlock(rq, p, &flags);
  4287. }
  4288. EXPORT_SYMBOL(set_user_nice);
  4289. /*
  4290. * can_nice - check if a task can reduce its nice value
  4291. * @p: task
  4292. * @nice: nice value
  4293. */
  4294. int can_nice(const struct task_struct *p, const int nice)
  4295. {
  4296. /* convert nice value [19,-20] to rlimit style value [1,40] */
  4297. int nice_rlim = 20 - nice;
  4298. return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
  4299. capable(CAP_SYS_NICE));
  4300. }
  4301. #ifdef __ARCH_WANT_SYS_NICE
  4302. /*
  4303. * sys_nice - change the priority of the current process.
  4304. * @increment: priority increment
  4305. *
  4306. * sys_setpriority is a more generic, but much slower function that
  4307. * does similar things.
  4308. */
  4309. SYSCALL_DEFINE1(nice, int, increment)
  4310. {
  4311. long nice, retval;
  4312. /*
  4313. * Setpriority might change our priority at the same moment.
  4314. * We don't have to worry. Conceptually one call occurs first
  4315. * and we have a single winner.
  4316. */
  4317. if (increment < -40)
  4318. increment = -40;
  4319. if (increment > 40)
  4320. increment = 40;
  4321. nice = TASK_NICE(current) + increment;
  4322. if (nice < -20)
  4323. nice = -20;
  4324. if (nice > 19)
  4325. nice = 19;
  4326. if (increment < 0 && !can_nice(current, nice))
  4327. return -EPERM;
  4328. retval = security_task_setnice(current, nice);
  4329. if (retval)
  4330. return retval;
  4331. set_user_nice(current, nice);
  4332. return 0;
  4333. }
  4334. #endif
  4335. /**
  4336. * task_prio - return the priority value of a given task.
  4337. * @p: the task in question.
  4338. *
  4339. * This is the priority value as seen by users in /proc.
  4340. * RT tasks are offset by -200. Normal tasks are centered
  4341. * around 0, value goes from -16 to +15.
  4342. */
  4343. int task_prio(const struct task_struct *p)
  4344. {
  4345. return p->prio - MAX_RT_PRIO;
  4346. }
  4347. /**
  4348. * task_nice - return the nice value of a given task.
  4349. * @p: the task in question.
  4350. */
  4351. int task_nice(const struct task_struct *p)
  4352. {
  4353. return TASK_NICE(p);
  4354. }
  4355. EXPORT_SYMBOL(task_nice);
  4356. /**
  4357. * idle_cpu - is a given cpu idle currently?
  4358. * @cpu: the processor in question.
  4359. */
  4360. int idle_cpu(int cpu)
  4361. {
  4362. return cpu_curr(cpu) == cpu_rq(cpu)->idle;
  4363. }
  4364. /**
  4365. * idle_task - return the idle task for a given cpu.
  4366. * @cpu: the processor in question.
  4367. */
  4368. struct task_struct *idle_task(int cpu)
  4369. {
  4370. return cpu_rq(cpu)->idle;
  4371. }
  4372. /**
  4373. * find_process_by_pid - find a process with a matching PID value.
  4374. * @pid: the pid in question.
  4375. */
  4376. static struct task_struct *find_process_by_pid(pid_t pid)
  4377. {
  4378. return pid ? find_task_by_vpid(pid) : current;
  4379. }
  4380. /* Actually do priority change: must hold rq lock. */
  4381. static void
  4382. __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
  4383. {
  4384. p->policy = policy;
  4385. p->rt_priority = prio;
  4386. p->normal_prio = normal_prio(p);
  4387. /* we are holding p->pi_lock already */
  4388. p->prio = rt_mutex_getprio(p);
  4389. if (rt_prio(p->prio))
  4390. p->sched_class = &rt_sched_class;
  4391. else
  4392. p->sched_class = &fair_sched_class;
  4393. set_load_weight(p);
  4394. }
  4395. /*
  4396. * check the target process has a UID that matches the current process's
  4397. */
  4398. static bool check_same_owner(struct task_struct *p)
  4399. {
  4400. const struct cred *cred = current_cred(), *pcred;
  4401. bool match;
  4402. rcu_read_lock();
  4403. pcred = __task_cred(p);
  4404. if (cred->user->user_ns == pcred->user->user_ns)
  4405. match = (cred->euid == pcred->euid ||
  4406. cred->euid == pcred->uid);
  4407. else
  4408. match = false;
  4409. rcu_read_unlock();
  4410. return match;
  4411. }
  4412. static int __sched_setscheduler(struct task_struct *p, int policy,
  4413. const struct sched_param *param, bool user)
  4414. {
  4415. int retval, oldprio, oldpolicy = -1, on_rq, running;
  4416. unsigned long flags;
  4417. const struct sched_class *prev_class;
  4418. struct rq *rq;
  4419. int reset_on_fork;
  4420. /* may grab non-irq protected spin_locks */
  4421. BUG_ON(in_interrupt());
  4422. recheck:
  4423. /* double check policy once rq lock held */
  4424. if (policy < 0) {
  4425. reset_on_fork = p->sched_reset_on_fork;
  4426. policy = oldpolicy = p->policy;
  4427. } else {
  4428. reset_on_fork = !!(policy & SCHED_RESET_ON_FORK);
  4429. policy &= ~SCHED_RESET_ON_FORK;
  4430. if (policy != SCHED_FIFO && policy != SCHED_RR &&
  4431. policy != SCHED_NORMAL && policy != SCHED_BATCH &&
  4432. policy != SCHED_IDLE)
  4433. return -EINVAL;
  4434. }
  4435. /*
  4436. * Valid priorities for SCHED_FIFO and SCHED_RR are
  4437. * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
  4438. * SCHED_BATCH and SCHED_IDLE is 0.
  4439. */
  4440. if (param->sched_priority < 0 ||
  4441. (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
  4442. (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
  4443. return -EINVAL;
  4444. if (rt_policy(policy) != (param->sched_priority != 0))
  4445. return -EINVAL;
  4446. /*
  4447. * Allow unprivileged RT tasks to decrease priority:
  4448. */
  4449. if (user && !capable(CAP_SYS_NICE)) {
  4450. if (rt_policy(policy)) {
  4451. unsigned long rlim_rtprio =
  4452. task_rlimit(p, RLIMIT_RTPRIO);
  4453. /* can't set/change the rt policy */
  4454. if (policy != p->policy && !rlim_rtprio)
  4455. return -EPERM;
  4456. /* can't increase priority */
  4457. if (param->sched_priority > p->rt_priority &&
  4458. param->sched_priority > rlim_rtprio)
  4459. return -EPERM;
  4460. }
  4461. /*
  4462. * Treat SCHED_IDLE as nice 20. Only allow a switch to
  4463. * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
  4464. */
  4465. if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) {
  4466. if (!can_nice(p, TASK_NICE(p)))
  4467. return -EPERM;
  4468. }
  4469. /* can't change other user's priorities */
  4470. if (!check_same_owner(p))
  4471. return -EPERM;
  4472. /* Normal users shall not reset the sched_reset_on_fork flag */
  4473. if (p->sched_reset_on_fork && !reset_on_fork)
  4474. return -EPERM;
  4475. }
  4476. if (user) {
  4477. retval = security_task_setscheduler(p);
  4478. if (retval)
  4479. return retval;
  4480. }
  4481. /*
  4482. * make sure no PI-waiters arrive (or leave) while we are
  4483. * changing the priority of the task:
  4484. *
  4485. * To be able to change p->policy safely, the appropriate
  4486. * runqueue lock must be held.
  4487. */
  4488. rq = task_rq_lock(p, &flags);
  4489. /*
  4490. * Changing the policy of the stop threads its a very bad idea
  4491. */
  4492. if (p == rq->stop) {
  4493. task_rq_unlock(rq, p, &flags);
  4494. return -EINVAL;
  4495. }
  4496. /*
  4497. * If not changing anything there's no need to proceed further:
  4498. */
  4499. if (unlikely(policy == p->policy && (!rt_policy(policy) ||
  4500. param->sched_priority == p->rt_priority))) {
  4501. __task_rq_unlock(rq);
  4502. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  4503. return 0;
  4504. }
  4505. #ifdef CONFIG_RT_GROUP_SCHED
  4506. if (user) {
  4507. /*
  4508. * Do not allow realtime tasks into groups that have no runtime
  4509. * assigned.
  4510. */
  4511. if (rt_bandwidth_enabled() && rt_policy(policy) &&
  4512. task_group(p)->rt_bandwidth.rt_runtime == 0 &&
  4513. !task_group_is_autogroup(task_group(p))) {
  4514. task_rq_unlock(rq, p, &flags);
  4515. return -EPERM;
  4516. }
  4517. }
  4518. #endif
  4519. /* recheck policy now with rq lock held */
  4520. if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
  4521. policy = oldpolicy = -1;
  4522. task_rq_unlock(rq, p, &flags);
  4523. goto recheck;
  4524. }
  4525. on_rq = p->on_rq;
  4526. running = task_current(rq, p);
  4527. if (on_rq)
  4528. deactivate_task(rq, p, 0);
  4529. if (running)
  4530. p->sched_class->put_prev_task(rq, p);
  4531. p->sched_reset_on_fork = reset_on_fork;
  4532. oldprio = p->prio;
  4533. prev_class = p->sched_class;
  4534. __setscheduler(rq, p, policy, param->sched_priority);
  4535. if (running)
  4536. p->sched_class->set_curr_task(rq);
  4537. if (on_rq)
  4538. activate_task(rq, p, 0);
  4539. check_class_changed(rq, p, prev_class, oldprio);
  4540. task_rq_unlock(rq, p, &flags);
  4541. rt_mutex_adjust_pi(p);
  4542. return 0;
  4543. }
  4544. /**
  4545. * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
  4546. * @p: the task in question.
  4547. * @policy: new policy.
  4548. * @param: structure containing the new RT priority.
  4549. *
  4550. * NOTE that the task may be already dead.
  4551. */
  4552. int sched_setscheduler(struct task_struct *p, int policy,
  4553. const struct sched_param *param)
  4554. {
  4555. return __sched_setscheduler(p, policy, param, true);
  4556. }
  4557. EXPORT_SYMBOL_GPL(sched_setscheduler);
  4558. /**
  4559. * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
  4560. * @p: the task in question.
  4561. * @policy: new policy.
  4562. * @param: structure containing the new RT priority.
  4563. *
  4564. * Just like sched_setscheduler, only don't bother checking if the
  4565. * current context has permission. For example, this is needed in
  4566. * stop_machine(): we create temporary high priority worker threads,
  4567. * but our caller might not have that capability.
  4568. */
  4569. int sched_setscheduler_nocheck(struct task_struct *p, int policy,
  4570. const struct sched_param *param)
  4571. {
  4572. return __sched_setscheduler(p, policy, param, false);
  4573. }
  4574. static int
  4575. do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
  4576. {
  4577. struct sched_param lparam;
  4578. struct task_struct *p;
  4579. int retval;
  4580. if (!param || pid < 0)
  4581. return -EINVAL;
  4582. if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
  4583. return -EFAULT;
  4584. rcu_read_lock();
  4585. retval = -ESRCH;
  4586. p = find_process_by_pid(pid);
  4587. if (p != NULL)
  4588. retval = sched_setscheduler(p, policy, &lparam);
  4589. rcu_read_unlock();
  4590. return retval;
  4591. }
  4592. /**
  4593. * sys_sched_setscheduler - set/change the scheduler policy and RT priority
  4594. * @pid: the pid in question.
  4595. * @policy: new policy.
  4596. * @param: structure containing the new RT priority.
  4597. */
  4598. SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
  4599. struct sched_param __user *, param)
  4600. {
  4601. /* negative values for policy are not valid */
  4602. if (policy < 0)
  4603. return -EINVAL;
  4604. return do_sched_setscheduler(pid, policy, param);
  4605. }
  4606. /**
  4607. * sys_sched_setparam - set/change the RT priority of a thread
  4608. * @pid: the pid in question.
  4609. * @param: structure containing the new RT priority.
  4610. */
  4611. SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
  4612. {
  4613. return do_sched_setscheduler(pid, -1, param);
  4614. }
  4615. /**
  4616. * sys_sched_getscheduler - get the policy (scheduling class) of a thread
  4617. * @pid: the pid in question.
  4618. */
  4619. SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
  4620. {
  4621. struct task_struct *p;
  4622. int retval;
  4623. if (pid < 0)
  4624. return -EINVAL;
  4625. retval = -ESRCH;
  4626. rcu_read_lock();
  4627. p = find_process_by_pid(pid);
  4628. if (p) {
  4629. retval = security_task_getscheduler(p);
  4630. if (!retval)
  4631. retval = p->policy
  4632. | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
  4633. }
  4634. rcu_read_unlock();
  4635. return retval;
  4636. }
  4637. /**
  4638. * sys_sched_getparam - get the RT priority of a thread
  4639. * @pid: the pid in question.
  4640. * @param: structure containing the RT priority.
  4641. */
  4642. SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
  4643. {
  4644. struct sched_param lp;
  4645. struct task_struct *p;
  4646. int retval;
  4647. if (!param || pid < 0)
  4648. return -EINVAL;
  4649. rcu_read_lock();
  4650. p = find_process_by_pid(pid);
  4651. retval = -ESRCH;
  4652. if (!p)
  4653. goto out_unlock;
  4654. retval = security_task_getscheduler(p);
  4655. if (retval)
  4656. goto out_unlock;
  4657. lp.sched_priority = p->rt_priority;
  4658. rcu_read_unlock();
  4659. /*
  4660. * This one might sleep, we cannot do it with a spinlock held ...
  4661. */
  4662. retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
  4663. return retval;
  4664. out_unlock:
  4665. rcu_read_unlock();
  4666. return retval;
  4667. }
  4668. long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
  4669. {
  4670. cpumask_var_t cpus_allowed, new_mask;
  4671. struct task_struct *p;
  4672. int retval;
  4673. get_online_cpus();
  4674. rcu_read_lock();
  4675. p = find_process_by_pid(pid);
  4676. if (!p) {
  4677. rcu_read_unlock();
  4678. put_online_cpus();
  4679. return -ESRCH;
  4680. }
  4681. /* Prevent p going away */
  4682. get_task_struct(p);
  4683. rcu_read_unlock();
  4684. if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
  4685. retval = -ENOMEM;
  4686. goto out_put_task;
  4687. }
  4688. if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
  4689. retval = -ENOMEM;
  4690. goto out_free_cpus_allowed;
  4691. }
  4692. retval = -EPERM;
  4693. if (!check_same_owner(p) && !task_ns_capable(p, CAP_SYS_NICE))
  4694. goto out_unlock;
  4695. retval = security_task_setscheduler(p);
  4696. if (retval)
  4697. goto out_unlock;
  4698. cpuset_cpus_allowed(p, cpus_allowed);
  4699. cpumask_and(new_mask, in_mask, cpus_allowed);
  4700. again:
  4701. retval = set_cpus_allowed_ptr(p, new_mask);
  4702. if (!retval) {
  4703. cpuset_cpus_allowed(p, cpus_allowed);
  4704. if (!cpumask_subset(new_mask, cpus_allowed)) {
  4705. /*
  4706. * We must have raced with a concurrent cpuset
  4707. * update. Just reset the cpus_allowed to the
  4708. * cpuset's cpus_allowed
  4709. */
  4710. cpumask_copy(new_mask, cpus_allowed);
  4711. goto again;
  4712. }
  4713. }
  4714. out_unlock:
  4715. free_cpumask_var(new_mask);
  4716. out_free_cpus_allowed:
  4717. free_cpumask_var(cpus_allowed);
  4718. out_put_task:
  4719. put_task_struct(p);
  4720. put_online_cpus();
  4721. return retval;
  4722. }
  4723. static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
  4724. struct cpumask *new_mask)
  4725. {
  4726. if (len < cpumask_size())
  4727. cpumask_clear(new_mask);
  4728. else if (len > cpumask_size())
  4729. len = cpumask_size();
  4730. return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
  4731. }
  4732. /**
  4733. * sys_sched_setaffinity - set the cpu affinity of a process
  4734. * @pid: pid of the process
  4735. * @len: length in bytes of the bitmask pointed to by user_mask_ptr
  4736. * @user_mask_ptr: user-space pointer to the new cpu mask
  4737. */
  4738. SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
  4739. unsigned long __user *, user_mask_ptr)
  4740. {
  4741. cpumask_var_t new_mask;
  4742. int retval;
  4743. if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
  4744. return -ENOMEM;
  4745. retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
  4746. if (retval == 0)
  4747. retval = sched_setaffinity(pid, new_mask);
  4748. free_cpumask_var(new_mask);
  4749. return retval;
  4750. }
  4751. long sched_getaffinity(pid_t pid, struct cpumask *mask)
  4752. {
  4753. struct task_struct *p;
  4754. unsigned long flags;
  4755. int retval;
  4756. get_online_cpus();
  4757. rcu_read_lock();
  4758. retval = -ESRCH;
  4759. p = find_process_by_pid(pid);
  4760. if (!p)
  4761. goto out_unlock;
  4762. retval = security_task_getscheduler(p);
  4763. if (retval)
  4764. goto out_unlock;
  4765. raw_spin_lock_irqsave(&p->pi_lock, flags);
  4766. cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
  4767. raw_spin_unlock_irqrestore(&p->pi_lock, flags);
  4768. out_unlock:
  4769. rcu_read_unlock();
  4770. put_online_cpus();
  4771. return retval;
  4772. }
  4773. /**
  4774. * sys_sched_getaffinity - get the cpu affinity of a process
  4775. * @pid: pid of the process
  4776. * @len: length in bytes of the bitmask pointed to by user_mask_ptr
  4777. * @user_mask_ptr: user-space pointer to hold the current cpu mask
  4778. */
  4779. SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
  4780. unsigned long __user *, user_mask_ptr)
  4781. {
  4782. int ret;
  4783. cpumask_var_t mask;
  4784. if ((len * BITS_PER_BYTE) < nr_cpu_ids)
  4785. return -EINVAL;
  4786. if (len & (sizeof(unsigned long)-1))
  4787. return -EINVAL;
  4788. if (!alloc_cpumask_var(&mask, GFP_KERNEL))
  4789. return -ENOMEM;
  4790. ret = sched_getaffinity(pid, mask);
  4791. if (ret == 0) {
  4792. size_t retlen = min_t(size_t, len, cpumask_size());
  4793. if (copy_to_user(user_mask_ptr, mask, retlen))
  4794. ret = -EFAULT;
  4795. else
  4796. ret = retlen;
  4797. }
  4798. free_cpumask_var(mask);
  4799. return ret;
  4800. }
  4801. /**
  4802. * sys_sched_yield - yield the current processor to other threads.
  4803. *
  4804. * This function yields the current CPU to other tasks. If there are no
  4805. * other threads running on this CPU then this function will return.
  4806. */
  4807. SYSCALL_DEFINE0(sched_yield)
  4808. {
  4809. struct rq *rq = this_rq_lock();
  4810. schedstat_inc(rq, yld_count);
  4811. current->sched_class->yield_task(rq);
  4812. /*
  4813. * Since we are going to call schedule() anyway, there's
  4814. * no need to preempt or enable interrupts:
  4815. */
  4816. __release(rq->lock);
  4817. spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
  4818. do_raw_spin_unlock(&rq->lock);
  4819. preempt_enable_no_resched();
  4820. schedule();
  4821. return 0;
  4822. }
  4823. static inline int should_resched(void)
  4824. {
  4825. return need_resched() && !(preempt_count() & PREEMPT_ACTIVE);
  4826. }
  4827. static void __cond_resched(void)
  4828. {
  4829. add_preempt_count(PREEMPT_ACTIVE);
  4830. __schedule();
  4831. sub_preempt_count(PREEMPT_ACTIVE);
  4832. }
  4833. int __sched _cond_resched(void)
  4834. {
  4835. if (should_resched()) {
  4836. __cond_resched();
  4837. return 1;
  4838. }
  4839. return 0;
  4840. }
  4841. EXPORT_SYMBOL(_cond_resched);
  4842. /*
  4843. * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
  4844. * call schedule, and on return reacquire the lock.
  4845. *
  4846. * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
  4847. * operations here to prevent schedule() from being called twice (once via
  4848. * spin_unlock(), once by hand).
  4849. */
  4850. int __cond_resched_lock(spinlock_t *lock)
  4851. {
  4852. int resched = should_resched();
  4853. int ret = 0;
  4854. lockdep_assert_held(lock);
  4855. if (spin_needbreak(lock) || resched) {
  4856. spin_unlock(lock);
  4857. if (resched)
  4858. __cond_resched();
  4859. else
  4860. cpu_relax();
  4861. ret = 1;
  4862. spin_lock(lock);
  4863. }
  4864. return ret;
  4865. }
  4866. EXPORT_SYMBOL(__cond_resched_lock);
  4867. int __sched __cond_resched_softirq(void)
  4868. {
  4869. BUG_ON(!in_softirq());
  4870. if (should_resched()) {
  4871. local_bh_enable();
  4872. __cond_resched();
  4873. local_bh_disable();
  4874. return 1;
  4875. }
  4876. return 0;
  4877. }
  4878. EXPORT_SYMBOL(__cond_resched_softirq);
  4879. /**
  4880. * yield - yield the current processor to other threads.
  4881. *
  4882. * This is a shortcut for kernel-space yielding - it marks the
  4883. * thread runnable and calls sys_sched_yield().
  4884. */
  4885. void __sched yield(void)
  4886. {
  4887. set_current_state(TASK_RUNNING);
  4888. sys_sched_yield();
  4889. }
  4890. EXPORT_SYMBOL(yield);
  4891. /**
  4892. * yield_to - yield the current processor to another thread in
  4893. * your thread group, or accelerate that thread toward the
  4894. * processor it's on.
  4895. * @p: target task
  4896. * @preempt: whether task preemption is allowed or not
  4897. *
  4898. * It's the caller's job to ensure that the target task struct
  4899. * can't go away on us before we can do any checks.
  4900. *
  4901. * Returns true if we indeed boosted the target task.
  4902. */
  4903. bool __sched yield_to(struct task_struct *p, bool preempt)
  4904. {
  4905. struct task_struct *curr = current;
  4906. struct rq *rq, *p_rq;
  4907. unsigned long flags;
  4908. bool yielded = 0;
  4909. local_irq_save(flags);
  4910. rq = this_rq();
  4911. again:
  4912. p_rq = task_rq(p);
  4913. double_rq_lock(rq, p_rq);
  4914. while (task_rq(p) != p_rq) {
  4915. double_rq_unlock(rq, p_rq);
  4916. goto again;
  4917. }
  4918. if (!curr->sched_class->yield_to_task)
  4919. goto out;
  4920. if (curr->sched_class != p->sched_class)
  4921. goto out;
  4922. if (task_running(p_rq, p) || p->state)
  4923. goto out;
  4924. yielded = curr->sched_class->yield_to_task(rq, p, preempt);
  4925. if (yielded) {
  4926. schedstat_inc(rq, yld_count);
  4927. /*
  4928. * Make p's CPU reschedule; pick_next_entity takes care of
  4929. * fairness.
  4930. */
  4931. if (preempt && rq != p_rq)
  4932. resched_task(p_rq->curr);
  4933. }
  4934. out:
  4935. double_rq_unlock(rq, p_rq);
  4936. local_irq_restore(flags);
  4937. if (yielded)
  4938. schedule();
  4939. return yielded;
  4940. }
  4941. EXPORT_SYMBOL_GPL(yield_to);
  4942. /*
  4943. * This task is about to go to sleep on IO. Increment rq->nr_iowait so
  4944. * that process accounting knows that this is a task in IO wait state.
  4945. */
  4946. void __sched io_schedule(void)
  4947. {
  4948. struct rq *rq = raw_rq();
  4949. delayacct_blkio_start();
  4950. atomic_inc(&rq->nr_iowait);
  4951. blk_flush_plug(current);
  4952. current->in_iowait = 1;
  4953. schedule();
  4954. current->in_iowait = 0;
  4955. atomic_dec(&rq->nr_iowait);
  4956. delayacct_blkio_end();
  4957. }
  4958. EXPORT_SYMBOL(io_schedule);
  4959. long __sched io_schedule_timeout(long timeout)
  4960. {
  4961. struct rq *rq = raw_rq();
  4962. long ret;
  4963. delayacct_blkio_start();
  4964. atomic_inc(&rq->nr_iowait);
  4965. blk_flush_plug(current);
  4966. current->in_iowait = 1;
  4967. ret = schedule_timeout(timeout);
  4968. current->in_iowait = 0;
  4969. atomic_dec(&rq->nr_iowait);
  4970. delayacct_blkio_end();
  4971. return ret;
  4972. }
  4973. /**
  4974. * sys_sched_get_priority_max - return maximum RT priority.
  4975. * @policy: scheduling class.
  4976. *
  4977. * this syscall returns the maximum rt_priority that can be used
  4978. * by a given scheduling class.
  4979. */
  4980. SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
  4981. {
  4982. int ret = -EINVAL;
  4983. switch (policy) {
  4984. case SCHED_FIFO:
  4985. case SCHED_RR:
  4986. ret = MAX_USER_RT_PRIO-1;
  4987. break;
  4988. case SCHED_NORMAL:
  4989. case SCHED_BATCH:
  4990. case SCHED_IDLE:
  4991. ret = 0;
  4992. break;
  4993. }
  4994. return ret;
  4995. }
  4996. /**
  4997. * sys_sched_get_priority_min - return minimum RT priority.
  4998. * @policy: scheduling class.
  4999. *
  5000. * this syscall returns the minimum rt_priority that can be used
  5001. * by a given scheduling class.
  5002. */
  5003. SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
  5004. {
  5005. int ret = -EINVAL;
  5006. switch (policy) {
  5007. case SCHED_FIFO:
  5008. case SCHED_RR:
  5009. ret = 1;
  5010. break;
  5011. case SCHED_NORMAL:
  5012. case SCHED_BATCH:
  5013. case SCHED_IDLE:
  5014. ret = 0;
  5015. }
  5016. return ret;
  5017. }
  5018. /**
  5019. * sys_sched_rr_get_interval - return the default timeslice of a process.
  5020. * @pid: pid of the process.
  5021. * @interval: userspace pointer to the timeslice value.
  5022. *
  5023. * this syscall writes the default timeslice value of a given process
  5024. * into the user-space timespec buffer. A value of '0' means infinity.
  5025. */
  5026. SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
  5027. struct timespec __user *, interval)
  5028. {
  5029. struct task_struct *p;
  5030. unsigned int time_slice;
  5031. unsigned long flags;
  5032. struct rq *rq;
  5033. int retval;
  5034. struct timespec t;
  5035. if (pid < 0)
  5036. return -EINVAL;
  5037. retval = -ESRCH;
  5038. rcu_read_lock();
  5039. p = find_process_by_pid(pid);
  5040. if (!p)
  5041. goto out_unlock;
  5042. retval = security_task_getscheduler(p);
  5043. if (retval)
  5044. goto out_unlock;
  5045. rq = task_rq_lock(p, &flags);
  5046. time_slice = p->sched_class->get_rr_interval(rq, p);
  5047. task_rq_unlock(rq, p, &flags);
  5048. rcu_read_unlock();
  5049. jiffies_to_timespec(time_slice, &t);
  5050. retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
  5051. return retval;
  5052. out_unlock:
  5053. rcu_read_unlock();
  5054. return retval;
  5055. }
  5056. static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
  5057. void sched_show_task(struct task_struct *p)
  5058. {
  5059. unsigned long free = 0;
  5060. unsigned state;
  5061. state = p->state ? __ffs(p->state) + 1 : 0;
  5062. printk(KERN_INFO "%-15.15s %c", p->comm,
  5063. state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
  5064. #if BITS_PER_LONG == 32
  5065. if (state == TASK_RUNNING)
  5066. printk(KERN_CONT " running ");
  5067. else
  5068. printk(KERN_CONT " %08lx ", thread_saved_pc(p));
  5069. #else
  5070. if (state == TASK_RUNNING)
  5071. printk(KERN_CONT " running task ");
  5072. else
  5073. printk(KERN_CONT " %016lx ", thread_saved_pc(p));
  5074. #endif
  5075. #ifdef CONFIG_DEBUG_STACK_USAGE
  5076. free = stack_not_used(p);
  5077. #endif
  5078. printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
  5079. task_pid_nr(p), task_pid_nr(p->real_parent),
  5080. (unsigned long)task_thread_info(p)->flags);
  5081. show_stack(p, NULL);
  5082. }
  5083. void show_state_filter(unsigned long state_filter)
  5084. {
  5085. struct task_struct *g, *p;
  5086. #if BITS_PER_LONG == 32
  5087. printk(KERN_INFO
  5088. " task PC stack pid father\n");
  5089. #else
  5090. printk(KERN_INFO
  5091. " task PC stack pid father\n");
  5092. #endif
  5093. read_lock(&tasklist_lock);
  5094. do_each_thread(g, p) {
  5095. /*
  5096. * reset the NMI-timeout, listing all files on a slow
  5097. * console might take a lot of time:
  5098. */
  5099. touch_nmi_watchdog();
  5100. if (!state_filter || (p->state & state_filter))
  5101. sched_show_task(p);
  5102. } while_each_thread(g, p);
  5103. touch_all_softlockup_watchdogs();
  5104. #ifdef CONFIG_SCHED_DEBUG
  5105. sysrq_sched_debug_show();
  5106. #endif
  5107. read_unlock(&tasklist_lock);
  5108. /*
  5109. * Only show locks if all tasks are dumped:
  5110. */
  5111. if (!state_filter)
  5112. debug_show_all_locks();
  5113. }
  5114. void __cpuinit init_idle_bootup_task(struct task_struct *idle)
  5115. {
  5116. idle->sched_class = &idle_sched_class;
  5117. }
  5118. /**
  5119. * init_idle - set up an idle thread for a given CPU
  5120. * @idle: task in question
  5121. * @cpu: cpu the idle task belongs to
  5122. *
  5123. * NOTE: this function does not set the idle thread's NEED_RESCHED
  5124. * flag, to make booting more robust.
  5125. */
  5126. void __cpuinit init_idle(struct task_struct *idle, int cpu)
  5127. {
  5128. struct rq *rq = cpu_rq(cpu);
  5129. unsigned long flags;
  5130. raw_spin_lock_irqsave(&rq->lock, flags);
  5131. __sched_fork(idle);
  5132. idle->state = TASK_RUNNING;
  5133. idle->se.exec_start = sched_clock();
  5134. do_set_cpus_allowed(idle, cpumask_of(cpu));
  5135. /*
  5136. * We're having a chicken and egg problem, even though we are
  5137. * holding rq->lock, the cpu isn't yet set to this cpu so the
  5138. * lockdep check in task_group() will fail.
  5139. *
  5140. * Similar case to sched_fork(). / Alternatively we could
  5141. * use task_rq_lock() here and obtain the other rq->lock.
  5142. *
  5143. * Silence PROVE_RCU
  5144. */
  5145. rcu_read_lock();
  5146. __set_task_cpu(idle, cpu);
  5147. rcu_read_unlock();
  5148. rq->curr = rq->idle = idle;
  5149. #if defined(CONFIG_SMP)
  5150. idle->on_cpu = 1;
  5151. #endif
  5152. raw_spin_unlock_irqrestore(&rq->lock, flags);
  5153. /* Set the preempt count _outside_ the spinlocks! */
  5154. task_thread_info(idle)->preempt_count = 0;
  5155. /*
  5156. * The idle tasks have their own, simple scheduling class:
  5157. */
  5158. idle->sched_class = &idle_sched_class;
  5159. ftrace_graph_init_idle_task(idle, cpu);
  5160. }
  5161. /*
  5162. * In a system that switches off the HZ timer nohz_cpu_mask
  5163. * indicates which cpus entered this state. This is used
  5164. * in the rcu update to wait only for active cpus. For system
  5165. * which do not switch off the HZ timer nohz_cpu_mask should
  5166. * always be CPU_BITS_NONE.
  5167. */
  5168. cpumask_var_t nohz_cpu_mask;
  5169. /*
  5170. * Increase the granularity value when there are more CPUs,
  5171. * because with more CPUs the 'effective latency' as visible
  5172. * to users decreases. But the relationship is not linear,
  5173. * so pick a second-best guess by going with the log2 of the
  5174. * number of CPUs.
  5175. *
  5176. * This idea comes from the SD scheduler of Con Kolivas:
  5177. */
  5178. static int get_update_sysctl_factor(void)
  5179. {
  5180. unsigned int cpus = min_t(int, num_online_cpus(), 8);
  5181. unsigned int factor;
  5182. switch (sysctl_sched_tunable_scaling) {
  5183. case SCHED_TUNABLESCALING_NONE:
  5184. factor = 1;
  5185. break;
  5186. case SCHED_TUNABLESCALING_LINEAR:
  5187. factor = cpus;
  5188. break;
  5189. case SCHED_TUNABLESCALING_LOG:
  5190. default:
  5191. factor = 1 + ilog2(cpus);
  5192. break;
  5193. }
  5194. return factor;
  5195. }
  5196. static void update_sysctl(void)
  5197. {
  5198. unsigned int factor = get_update_sysctl_factor();
  5199. #define SET_SYSCTL(name) \
  5200. (sysctl_##name = (factor) * normalized_sysctl_##name)
  5201. SET_SYSCTL(sched_min_granularity);
  5202. SET_SYSCTL(sched_latency);
  5203. SET_SYSCTL(sched_wakeup_granularity);
  5204. #undef SET_SYSCTL
  5205. }
  5206. static inline void sched_init_granularity(void)
  5207. {
  5208. update_sysctl();
  5209. }
  5210. #ifdef CONFIG_SMP
  5211. void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
  5212. {
  5213. if (p->sched_class && p->sched_class->set_cpus_allowed)
  5214. p->sched_class->set_cpus_allowed(p, new_mask);
  5215. else {
  5216. cpumask_copy(&p->cpus_allowed, new_mask);
  5217. p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
  5218. }
  5219. }
  5220. /*
  5221. * This is how migration works:
  5222. *
  5223. * 1) we invoke migration_cpu_stop() on the target CPU using
  5224. * stop_one_cpu().
  5225. * 2) stopper starts to run (implicitly forcing the migrated thread
  5226. * off the CPU)
  5227. * 3) it checks whether the migrated task is still in the wrong runqueue.
  5228. * 4) if it's in the wrong runqueue then the migration thread removes
  5229. * it and puts it into the right queue.
  5230. * 5) stopper completes and stop_one_cpu() returns and the migration
  5231. * is done.
  5232. */
  5233. /*
  5234. * Change a given task's CPU affinity. Migrate the thread to a
  5235. * proper CPU and schedule it away if the CPU it's executing on
  5236. * is removed from the allowed bitmask.
  5237. *
  5238. * NOTE: the caller must have a valid reference to the task, the
  5239. * task must not exit() & deallocate itself prematurely. The
  5240. * call is not atomic; no spinlocks may be held.
  5241. */
  5242. int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
  5243. {
  5244. unsigned long flags;
  5245. struct rq *rq;
  5246. unsigned int dest_cpu;
  5247. int ret = 0;
  5248. rq = task_rq_lock(p, &flags);
  5249. if (cpumask_equal(&p->cpus_allowed, new_mask))
  5250. goto out;
  5251. if (!cpumask_intersects(new_mask, cpu_active_mask)) {
  5252. ret = -EINVAL;
  5253. goto out;
  5254. }
  5255. if (unlikely((p->flags & PF_THREAD_BOUND) && p != current)) {
  5256. ret = -EINVAL;
  5257. goto out;
  5258. }
  5259. do_set_cpus_allowed(p, new_mask);
  5260. /* Can the task run on the task's current CPU? If so, we're done */
  5261. if (cpumask_test_cpu(task_cpu(p), new_mask))
  5262. goto out;
  5263. dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
  5264. if (p->on_rq) {
  5265. struct migration_arg arg = { p, dest_cpu };
  5266. /* Need help from migration thread: drop lock and wait. */
  5267. task_rq_unlock(rq, p, &flags);
  5268. stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
  5269. tlb_migrate_finish(p->mm);
  5270. return 0;
  5271. }
  5272. out:
  5273. task_rq_unlock(rq, p, &flags);
  5274. return ret;
  5275. }
  5276. EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
  5277. /*
  5278. * Move (not current) task off this cpu, onto dest cpu. We're doing
  5279. * this because either it can't run here any more (set_cpus_allowed()
  5280. * away from this CPU, or CPU going down), or because we're
  5281. * attempting to rebalance this task on exec (sched_exec).
  5282. *
  5283. * So we race with normal scheduler movements, but that's OK, as long
  5284. * as the task is no longer on this CPU.
  5285. *
  5286. * Returns non-zero if task was successfully migrated.
  5287. */
  5288. static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
  5289. {
  5290. struct rq *rq_dest, *rq_src;
  5291. int ret = 0;
  5292. if (unlikely(!cpu_active(dest_cpu)))
  5293. return ret;
  5294. rq_src = cpu_rq(src_cpu);
  5295. rq_dest = cpu_rq(dest_cpu);
  5296. raw_spin_lock(&p->pi_lock);
  5297. double_rq_lock(rq_src, rq_dest);
  5298. /* Already moved. */
  5299. if (task_cpu(p) != src_cpu)
  5300. goto done;
  5301. /* Affinity changed (again). */
  5302. if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
  5303. goto fail;
  5304. /*
  5305. * If we're not on a rq, the next wake-up will ensure we're
  5306. * placed properly.
  5307. */
  5308. if (p->on_rq) {
  5309. deactivate_task(rq_src, p, 0);
  5310. set_task_cpu(p, dest_cpu);
  5311. activate_task(rq_dest, p, 0);
  5312. check_preempt_curr(rq_dest, p, 0);
  5313. }
  5314. done:
  5315. ret = 1;
  5316. fail:
  5317. double_rq_unlock(rq_src, rq_dest);
  5318. raw_spin_unlock(&p->pi_lock);
  5319. return ret;
  5320. }
  5321. /*
  5322. * migration_cpu_stop - this will be executed by a highprio stopper thread
  5323. * and performs thread migration by bumping thread off CPU then
  5324. * 'pushing' onto another runqueue.
  5325. */
  5326. static int migration_cpu_stop(void *data)
  5327. {
  5328. struct migration_arg *arg = data;
  5329. /*
  5330. * The original target cpu might have gone down and we might
  5331. * be on another cpu but it doesn't matter.
  5332. */
  5333. local_irq_disable();
  5334. __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
  5335. local_irq_enable();
  5336. return 0;
  5337. }
  5338. #ifdef CONFIG_HOTPLUG_CPU
  5339. /*
  5340. * Ensures that the idle task is using init_mm right before its cpu goes
  5341. * offline.
  5342. */
  5343. void idle_task_exit(void)
  5344. {
  5345. struct mm_struct *mm = current->active_mm;
  5346. BUG_ON(cpu_online(smp_processor_id()));
  5347. if (mm != &init_mm)
  5348. switch_mm(mm, &init_mm, current);
  5349. mmdrop(mm);
  5350. }
  5351. /*
  5352. * While a dead CPU has no uninterruptible tasks queued at this point,
  5353. * it might still have a nonzero ->nr_uninterruptible counter, because
  5354. * for performance reasons the counter is not stricly tracking tasks to
  5355. * their home CPUs. So we just add the counter to another CPU's counter,
  5356. * to keep the global sum constant after CPU-down:
  5357. */
  5358. static void migrate_nr_uninterruptible(struct rq *rq_src)
  5359. {
  5360. struct rq *rq_dest = cpu_rq(cpumask_any(cpu_active_mask));
  5361. rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
  5362. rq_src->nr_uninterruptible = 0;
  5363. }
  5364. /*
  5365. * remove the tasks which were accounted by rq from calc_load_tasks.
  5366. */
  5367. static void calc_global_load_remove(struct rq *rq)
  5368. {
  5369. atomic_long_sub(rq->calc_load_active, &calc_load_tasks);
  5370. rq->calc_load_active = 0;
  5371. }
  5372. #ifdef CONFIG_CFS_BANDWIDTH
  5373. static void unthrottle_offline_cfs_rqs(struct rq *rq)
  5374. {
  5375. struct cfs_rq *cfs_rq;
  5376. for_each_leaf_cfs_rq(rq, cfs_rq) {
  5377. struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
  5378. if (!cfs_rq->runtime_enabled)
  5379. continue;
  5380. /*
  5381. * clock_task is not advancing so we just need to make sure
  5382. * there's some valid quota amount
  5383. */
  5384. cfs_rq->runtime_remaining = cfs_b->quota;
  5385. if (cfs_rq_throttled(cfs_rq))
  5386. unthrottle_cfs_rq(cfs_rq);
  5387. }
  5388. }
  5389. #else
  5390. static void unthrottle_offline_cfs_rqs(struct rq *rq) {}
  5391. #endif
  5392. /*
  5393. * Migrate all tasks from the rq, sleeping tasks will be migrated by
  5394. * try_to_wake_up()->select_task_rq().
  5395. *
  5396. * Called with rq->lock held even though we'er in stop_machine() and
  5397. * there's no concurrency possible, we hold the required locks anyway
  5398. * because of lock validation efforts.
  5399. */
  5400. static void migrate_tasks(unsigned int dead_cpu)
  5401. {
  5402. struct rq *rq = cpu_rq(dead_cpu);
  5403. struct task_struct *next, *stop = rq->stop;
  5404. int dest_cpu;
  5405. /*
  5406. * Fudge the rq selection such that the below task selection loop
  5407. * doesn't get stuck on the currently eligible stop task.
  5408. *
  5409. * We're currently inside stop_machine() and the rq is either stuck
  5410. * in the stop_machine_cpu_stop() loop, or we're executing this code,
  5411. * either way we should never end up calling schedule() until we're
  5412. * done here.
  5413. */
  5414. rq->stop = NULL;
  5415. /* Ensure any throttled groups are reachable by pick_next_task */
  5416. unthrottle_offline_cfs_rqs(rq);
  5417. for ( ; ; ) {
  5418. /*
  5419. * There's this thread running, bail when that's the only
  5420. * remaining thread.
  5421. */
  5422. if (rq->nr_running == 1)
  5423. break;
  5424. next = pick_next_task(rq);
  5425. BUG_ON(!next);
  5426. next->sched_class->put_prev_task(rq, next);
  5427. /* Find suitable destination for @next, with force if needed. */
  5428. dest_cpu = select_fallback_rq(dead_cpu, next);
  5429. raw_spin_unlock(&rq->lock);
  5430. __migrate_task(next, dead_cpu, dest_cpu);
  5431. raw_spin_lock(&rq->lock);
  5432. }
  5433. rq->stop = stop;
  5434. }
  5435. #endif /* CONFIG_HOTPLUG_CPU */
  5436. #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
  5437. static struct ctl_table sd_ctl_dir[] = {
  5438. {
  5439. .procname = "sched_domain",
  5440. .mode = 0555,
  5441. },
  5442. {}
  5443. };
  5444. static struct ctl_table sd_ctl_root[] = {
  5445. {
  5446. .procname = "kernel",
  5447. .mode = 0555,
  5448. .child = sd_ctl_dir,
  5449. },
  5450. {}
  5451. };
  5452. static struct ctl_table *sd_alloc_ctl_entry(int n)
  5453. {
  5454. struct ctl_table *entry =
  5455. kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
  5456. return entry;
  5457. }
  5458. static void sd_free_ctl_entry(struct ctl_table **tablep)
  5459. {
  5460. struct ctl_table *entry;
  5461. /*
  5462. * In the intermediate directories, both the child directory and
  5463. * procname are dynamically allocated and could fail but the mode
  5464. * will always be set. In the lowest directory the names are
  5465. * static strings and all have proc handlers.
  5466. */
  5467. for (entry = *tablep; entry->mode; entry++) {
  5468. if (entry->child)
  5469. sd_free_ctl_entry(&entry->child);
  5470. if (entry->proc_handler == NULL)
  5471. kfree(entry->procname);
  5472. }
  5473. kfree(*tablep);
  5474. *tablep = NULL;
  5475. }
  5476. static void
  5477. set_table_entry(struct ctl_table *entry,
  5478. const char *procname, void *data, int maxlen,
  5479. mode_t mode, proc_handler *proc_handler)
  5480. {
  5481. entry->procname = procname;
  5482. entry->data = data;
  5483. entry->maxlen = maxlen;
  5484. entry->mode = mode;
  5485. entry->proc_handler = proc_handler;
  5486. }
  5487. static struct ctl_table *
  5488. sd_alloc_ctl_domain_table(struct sched_domain *sd)
  5489. {
  5490. struct ctl_table *table = sd_alloc_ctl_entry(13);
  5491. if (table == NULL)
  5492. return NULL;
  5493. set_table_entry(&table[0], "min_interval", &sd->min_interval,
  5494. sizeof(long), 0644, proc_doulongvec_minmax);
  5495. set_table_entry(&table[1], "max_interval", &sd->max_interval,
  5496. sizeof(long), 0644, proc_doulongvec_minmax);
  5497. set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
  5498. sizeof(int), 0644, proc_dointvec_minmax);
  5499. set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
  5500. sizeof(int), 0644, proc_dointvec_minmax);
  5501. set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
  5502. sizeof(int), 0644, proc_dointvec_minmax);
  5503. set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
  5504. sizeof(int), 0644, proc_dointvec_minmax);
  5505. set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
  5506. sizeof(int), 0644, proc_dointvec_minmax);
  5507. set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
  5508. sizeof(int), 0644, proc_dointvec_minmax);
  5509. set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
  5510. sizeof(int), 0644, proc_dointvec_minmax);
  5511. set_table_entry(&table[9], "cache_nice_tries",
  5512. &sd->cache_nice_tries,
  5513. sizeof(int), 0644, proc_dointvec_minmax);
  5514. set_table_entry(&table[10], "flags", &sd->flags,
  5515. sizeof(int), 0644, proc_dointvec_minmax);
  5516. set_table_entry(&table[11], "name", sd->name,
  5517. CORENAME_MAX_SIZE, 0444, proc_dostring);
  5518. /* &table[12] is terminator */
  5519. return table;
  5520. }
  5521. static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
  5522. {
  5523. struct ctl_table *entry, *table;
  5524. struct sched_domain *sd;
  5525. int domain_num = 0, i;
  5526. char buf[32];
  5527. for_each_domain(cpu, sd)
  5528. domain_num++;
  5529. entry = table = sd_alloc_ctl_entry(domain_num + 1);
  5530. if (table == NULL)
  5531. return NULL;
  5532. i = 0;
  5533. for_each_domain(cpu, sd) {
  5534. snprintf(buf, 32, "domain%d", i);
  5535. entry->procname = kstrdup(buf, GFP_KERNEL);
  5536. entry->mode = 0555;
  5537. entry->child = sd_alloc_ctl_domain_table(sd);
  5538. entry++;
  5539. i++;
  5540. }
  5541. return table;
  5542. }
  5543. static struct ctl_table_header *sd_sysctl_header;
  5544. static void register_sched_domain_sysctl(void)
  5545. {
  5546. int i, cpu_num = num_possible_cpus();
  5547. struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
  5548. char buf[32];
  5549. WARN_ON(sd_ctl_dir[0].child);
  5550. sd_ctl_dir[0].child = entry;
  5551. if (entry == NULL)
  5552. return;
  5553. for_each_possible_cpu(i) {
  5554. snprintf(buf, 32, "cpu%d", i);
  5555. entry->procname = kstrdup(buf, GFP_KERNEL);
  5556. entry->mode = 0555;
  5557. entry->child = sd_alloc_ctl_cpu_table(i);
  5558. entry++;
  5559. }
  5560. WARN_ON(sd_sysctl_header);
  5561. sd_sysctl_header = register_sysctl_table(sd_ctl_root);
  5562. }
  5563. /* may be called multiple times per register */
  5564. static void unregister_sched_domain_sysctl(void)
  5565. {
  5566. if (sd_sysctl_header)
  5567. unregister_sysctl_table(sd_sysctl_header);
  5568. sd_sysctl_header = NULL;
  5569. if (sd_ctl_dir[0].child)
  5570. sd_free_ctl_entry(&sd_ctl_dir[0].child);
  5571. }
  5572. #else
  5573. static void register_sched_domain_sysctl(void)
  5574. {
  5575. }
  5576. static void unregister_sched_domain_sysctl(void)
  5577. {
  5578. }
  5579. #endif
  5580. static void set_rq_online(struct rq *rq)
  5581. {
  5582. if (!rq->online) {
  5583. const struct sched_class *class;
  5584. cpumask_set_cpu(rq->cpu, rq->rd->online);
  5585. rq->online = 1;
  5586. for_each_class(class) {
  5587. if (class->rq_online)
  5588. class->rq_online(rq);
  5589. }
  5590. }
  5591. }
  5592. static void set_rq_offline(struct rq *rq)
  5593. {
  5594. if (rq->online) {
  5595. const struct sched_class *class;
  5596. for_each_class(class) {
  5597. if (class->rq_offline)
  5598. class->rq_offline(rq);
  5599. }
  5600. cpumask_clear_cpu(rq->cpu, rq->rd->online);
  5601. rq->online = 0;
  5602. }
  5603. }
  5604. /*
  5605. * migration_call - callback that gets triggered when a CPU is added.
  5606. * Here we can start up the necessary migration thread for the new CPU.
  5607. */
  5608. static int __cpuinit
  5609. migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
  5610. {
  5611. int cpu = (long)hcpu;
  5612. unsigned long flags;
  5613. struct rq *rq = cpu_rq(cpu);
  5614. switch (action & ~CPU_TASKS_FROZEN) {
  5615. case CPU_UP_PREPARE:
  5616. rq->calc_load_update = calc_load_update;
  5617. break;
  5618. case CPU_ONLINE:
  5619. /* Update our root-domain */
  5620. raw_spin_lock_irqsave(&rq->lock, flags);
  5621. if (rq->rd) {
  5622. BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
  5623. set_rq_online(rq);
  5624. }
  5625. raw_spin_unlock_irqrestore(&rq->lock, flags);
  5626. break;
  5627. #ifdef CONFIG_HOTPLUG_CPU
  5628. case CPU_DYING:
  5629. sched_ttwu_pending();
  5630. /* Update our root-domain */
  5631. raw_spin_lock_irqsave(&rq->lock, flags);
  5632. if (rq->rd) {
  5633. BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
  5634. set_rq_offline(rq);
  5635. }
  5636. migrate_tasks(cpu);
  5637. BUG_ON(rq->nr_running != 1); /* the migration thread */
  5638. raw_spin_unlock_irqrestore(&rq->lock, flags);
  5639. migrate_nr_uninterruptible(rq);
  5640. calc_global_load_remove(rq);
  5641. break;
  5642. #endif
  5643. }
  5644. update_max_interval();
  5645. return NOTIFY_OK;
  5646. }
  5647. /*
  5648. * Register at high priority so that task migration (migrate_all_tasks)
  5649. * happens before everything else. This has to be lower priority than
  5650. * the notifier in the perf_event subsystem, though.
  5651. */
  5652. static struct notifier_block __cpuinitdata migration_notifier = {
  5653. .notifier_call = migration_call,
  5654. .priority = CPU_PRI_MIGRATION,
  5655. };
  5656. static int __cpuinit sched_cpu_active(struct notifier_block *nfb,
  5657. unsigned long action, void *hcpu)
  5658. {
  5659. switch (action & ~CPU_TASKS_FROZEN) {
  5660. case CPU_ONLINE:
  5661. case CPU_DOWN_FAILED:
  5662. set_cpu_active((long)hcpu, true);
  5663. return NOTIFY_OK;
  5664. default:
  5665. return NOTIFY_DONE;
  5666. }
  5667. }
  5668. static int __cpuinit sched_cpu_inactive(struct notifier_block *nfb,
  5669. unsigned long action, void *hcpu)
  5670. {
  5671. switch (action & ~CPU_TASKS_FROZEN) {
  5672. case CPU_DOWN_PREPARE:
  5673. set_cpu_active((long)hcpu, false);
  5674. return NOTIFY_OK;
  5675. default:
  5676. return NOTIFY_DONE;
  5677. }
  5678. }
  5679. static int __init migration_init(void)
  5680. {
  5681. void *cpu = (void *)(long)smp_processor_id();
  5682. int err;
  5683. /* Initialize migration for the boot CPU */
  5684. err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
  5685. BUG_ON(err == NOTIFY_BAD);
  5686. migration_call(&migration_notifier, CPU_ONLINE, cpu);
  5687. register_cpu_notifier(&migration_notifier);
  5688. /* Register cpu active notifiers */
  5689. cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
  5690. cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
  5691. return 0;
  5692. }
  5693. early_initcall(migration_init);
  5694. #endif
  5695. #ifdef CONFIG_SMP
  5696. static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */
  5697. #ifdef CONFIG_SCHED_DEBUG
  5698. static __read_mostly int sched_domain_debug_enabled;
  5699. static int __init sched_domain_debug_setup(char *str)
  5700. {
  5701. sched_domain_debug_enabled = 1;
  5702. return 0;
  5703. }
  5704. early_param("sched_debug", sched_domain_debug_setup);
  5705. static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
  5706. struct cpumask *groupmask)
  5707. {
  5708. struct sched_group *group = sd->groups;
  5709. char str[256];
  5710. cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
  5711. cpumask_clear(groupmask);
  5712. printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
  5713. if (!(sd->flags & SD_LOAD_BALANCE)) {
  5714. printk("does not load-balance\n");
  5715. if (sd->parent)
  5716. printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
  5717. " has parent");
  5718. return -1;
  5719. }
  5720. printk(KERN_CONT "span %s level %s\n", str, sd->name);
  5721. if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
  5722. printk(KERN_ERR "ERROR: domain->span does not contain "
  5723. "CPU%d\n", cpu);
  5724. }
  5725. if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
  5726. printk(KERN_ERR "ERROR: domain->groups does not contain"
  5727. " CPU%d\n", cpu);
  5728. }
  5729. printk(KERN_DEBUG "%*s groups:", level + 1, "");
  5730. do {
  5731. if (!group) {
  5732. printk("\n");
  5733. printk(KERN_ERR "ERROR: group is NULL\n");
  5734. break;
  5735. }
  5736. if (!group->sgp->power) {
  5737. printk(KERN_CONT "\n");
  5738. printk(KERN_ERR "ERROR: domain->cpu_power not "
  5739. "set\n");
  5740. break;
  5741. }
  5742. if (!cpumask_weight(sched_group_cpus(group))) {
  5743. printk(KERN_CONT "\n");
  5744. printk(KERN_ERR "ERROR: empty group\n");
  5745. break;
  5746. }
  5747. if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
  5748. printk(KERN_CONT "\n");
  5749. printk(KERN_ERR "ERROR: repeated CPUs\n");
  5750. break;
  5751. }
  5752. cpumask_or(groupmask, groupmask, sched_group_cpus(group));
  5753. cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
  5754. printk(KERN_CONT " %s", str);
  5755. if (group->sgp->power != SCHED_POWER_SCALE) {
  5756. printk(KERN_CONT " (cpu_power = %d)",
  5757. group->sgp->power);
  5758. }
  5759. group = group->next;
  5760. } while (group != sd->groups);
  5761. printk(KERN_CONT "\n");
  5762. if (!cpumask_equal(sched_domain_span(sd), groupmask))
  5763. printk(KERN_ERR "ERROR: groups don't span domain->span\n");
  5764. if (sd->parent &&
  5765. !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
  5766. printk(KERN_ERR "ERROR: parent span is not a superset "
  5767. "of domain->span\n");
  5768. return 0;
  5769. }
  5770. static void sched_domain_debug(struct sched_domain *sd, int cpu)
  5771. {
  5772. int level = 0;
  5773. if (!sched_domain_debug_enabled)
  5774. return;
  5775. if (!sd) {
  5776. printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
  5777. return;
  5778. }
  5779. printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
  5780. for (;;) {
  5781. if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
  5782. break;
  5783. level++;
  5784. sd = sd->parent;
  5785. if (!sd)
  5786. break;
  5787. }
  5788. }
  5789. #else /* !CONFIG_SCHED_DEBUG */
  5790. # define sched_domain_debug(sd, cpu) do { } while (0)
  5791. #endif /* CONFIG_SCHED_DEBUG */
  5792. static int sd_degenerate(struct sched_domain *sd)
  5793. {
  5794. if (cpumask_weight(sched_domain_span(sd)) == 1)
  5795. return 1;
  5796. /* Following flags need at least 2 groups */
  5797. if (sd->flags & (SD_LOAD_BALANCE |
  5798. SD_BALANCE_NEWIDLE |
  5799. SD_BALANCE_FORK |
  5800. SD_BALANCE_EXEC |
  5801. SD_SHARE_CPUPOWER |
  5802. SD_SHARE_PKG_RESOURCES)) {
  5803. if (sd->groups != sd->groups->next)
  5804. return 0;
  5805. }
  5806. /* Following flags don't use groups */
  5807. if (sd->flags & (SD_WAKE_AFFINE))
  5808. return 0;
  5809. return 1;
  5810. }
  5811. static int
  5812. sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
  5813. {
  5814. unsigned long cflags = sd->flags, pflags = parent->flags;
  5815. if (sd_degenerate(parent))
  5816. return 1;
  5817. if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
  5818. return 0;
  5819. /* Flags needing groups don't count if only 1 group in parent */
  5820. if (parent->groups == parent->groups->next) {
  5821. pflags &= ~(SD_LOAD_BALANCE |
  5822. SD_BALANCE_NEWIDLE |
  5823. SD_BALANCE_FORK |
  5824. SD_BALANCE_EXEC |
  5825. SD_SHARE_CPUPOWER |
  5826. SD_SHARE_PKG_RESOURCES);
  5827. if (nr_node_ids == 1)
  5828. pflags &= ~SD_SERIALIZE;
  5829. }
  5830. if (~cflags & pflags)
  5831. return 0;
  5832. return 1;
  5833. }
  5834. static void free_rootdomain(struct rcu_head *rcu)
  5835. {
  5836. struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
  5837. cpupri_cleanup(&rd->cpupri);
  5838. free_cpumask_var(rd->rto_mask);
  5839. free_cpumask_var(rd->online);
  5840. free_cpumask_var(rd->span);
  5841. kfree(rd);
  5842. }
  5843. static void rq_attach_root(struct rq *rq, struct root_domain *rd)
  5844. {
  5845. struct root_domain *old_rd = NULL;
  5846. unsigned long flags;
  5847. raw_spin_lock_irqsave(&rq->lock, flags);
  5848. if (rq->rd) {
  5849. old_rd = rq->rd;
  5850. if (cpumask_test_cpu(rq->cpu, old_rd->online))
  5851. set_rq_offline(rq);
  5852. cpumask_clear_cpu(rq->cpu, old_rd->span);
  5853. /*
  5854. * If we dont want to free the old_rt yet then
  5855. * set old_rd to NULL to skip the freeing later
  5856. * in this function:
  5857. */
  5858. if (!atomic_dec_and_test(&old_rd->refcount))
  5859. old_rd = NULL;
  5860. }
  5861. atomic_inc(&rd->refcount);
  5862. rq->rd = rd;
  5863. cpumask_set_cpu(rq->cpu, rd->span);
  5864. if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
  5865. set_rq_online(rq);
  5866. raw_spin_unlock_irqrestore(&rq->lock, flags);
  5867. if (old_rd)
  5868. call_rcu_sched(&old_rd->rcu, free_rootdomain);
  5869. }
  5870. static int init_rootdomain(struct root_domain *rd)
  5871. {
  5872. memset(rd, 0, sizeof(*rd));
  5873. if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
  5874. goto out;
  5875. if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
  5876. goto free_span;
  5877. if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
  5878. goto free_online;
  5879. if (cpupri_init(&rd->cpupri) != 0)
  5880. goto free_rto_mask;
  5881. return 0;
  5882. free_rto_mask:
  5883. free_cpumask_var(rd->rto_mask);
  5884. free_online:
  5885. free_cpumask_var(rd->online);
  5886. free_span:
  5887. free_cpumask_var(rd->span);
  5888. out:
  5889. return -ENOMEM;
  5890. }
  5891. static void init_defrootdomain(void)
  5892. {
  5893. init_rootdomain(&def_root_domain);
  5894. atomic_set(&def_root_domain.refcount, 1);
  5895. }
  5896. static struct root_domain *alloc_rootdomain(void)
  5897. {
  5898. struct root_domain *rd;
  5899. rd = kmalloc(sizeof(*rd), GFP_KERNEL);
  5900. if (!rd)
  5901. return NULL;
  5902. if (init_rootdomain(rd) != 0) {
  5903. kfree(rd);
  5904. return NULL;
  5905. }
  5906. return rd;
  5907. }
  5908. static void free_sched_groups(struct sched_group *sg, int free_sgp)
  5909. {
  5910. struct sched_group *tmp, *first;
  5911. if (!sg)
  5912. return;
  5913. first = sg;
  5914. do {
  5915. tmp = sg->next;
  5916. if (free_sgp && atomic_dec_and_test(&sg->sgp->ref))
  5917. kfree(sg->sgp);
  5918. kfree(sg);
  5919. sg = tmp;
  5920. } while (sg != first);
  5921. }
  5922. static void free_sched_domain(struct rcu_head *rcu)
  5923. {
  5924. struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
  5925. /*
  5926. * If its an overlapping domain it has private groups, iterate and
  5927. * nuke them all.
  5928. */
  5929. if (sd->flags & SD_OVERLAP) {
  5930. free_sched_groups(sd->groups, 1);
  5931. } else if (atomic_dec_and_test(&sd->groups->ref)) {
  5932. kfree(sd->groups->sgp);
  5933. kfree(sd->groups);
  5934. }
  5935. kfree(sd);
  5936. }
  5937. static void destroy_sched_domain(struct sched_domain *sd, int cpu)
  5938. {
  5939. call_rcu(&sd->rcu, free_sched_domain);
  5940. }
  5941. static void destroy_sched_domains(struct sched_domain *sd, int cpu)
  5942. {
  5943. for (; sd; sd = sd->parent)
  5944. destroy_sched_domain(sd, cpu);
  5945. }
  5946. /*
  5947. * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
  5948. * hold the hotplug lock.
  5949. */
  5950. static void
  5951. cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
  5952. {
  5953. struct rq *rq = cpu_rq(cpu);
  5954. struct sched_domain *tmp;
  5955. /* Remove the sched domains which do not contribute to scheduling. */
  5956. for (tmp = sd; tmp; ) {
  5957. struct sched_domain *parent = tmp->parent;
  5958. if (!parent)
  5959. break;
  5960. if (sd_parent_degenerate(tmp, parent)) {
  5961. tmp->parent = parent->parent;
  5962. if (parent->parent)
  5963. parent->parent->child = tmp;
  5964. destroy_sched_domain(parent, cpu);
  5965. } else
  5966. tmp = tmp->parent;
  5967. }
  5968. if (sd && sd_degenerate(sd)) {
  5969. tmp = sd;
  5970. sd = sd->parent;
  5971. destroy_sched_domain(tmp, cpu);
  5972. if (sd)
  5973. sd->child = NULL;
  5974. }
  5975. sched_domain_debug(sd, cpu);
  5976. rq_attach_root(rq, rd);
  5977. tmp = rq->sd;
  5978. rcu_assign_pointer(rq->sd, sd);
  5979. destroy_sched_domains(tmp, cpu);
  5980. }
  5981. /* cpus with isolated domains */
  5982. static cpumask_var_t cpu_isolated_map;
  5983. /* Setup the mask of cpus configured for isolated domains */
  5984. static int __init isolated_cpu_setup(char *str)
  5985. {
  5986. alloc_bootmem_cpumask_var(&cpu_isolated_map);
  5987. cpulist_parse(str, cpu_isolated_map);
  5988. return 1;
  5989. }
  5990. __setup("isolcpus=", isolated_cpu_setup);
  5991. #define SD_NODES_PER_DOMAIN 16
  5992. #ifdef CONFIG_NUMA
  5993. /**
  5994. * find_next_best_node - find the next node to include in a sched_domain
  5995. * @node: node whose sched_domain we're building
  5996. * @used_nodes: nodes already in the sched_domain
  5997. *
  5998. * Find the next node to include in a given scheduling domain. Simply
  5999. * finds the closest node not already in the @used_nodes map.
  6000. *
  6001. * Should use nodemask_t.
  6002. */
  6003. static int find_next_best_node(int node, nodemask_t *used_nodes)
  6004. {
  6005. int i, n, val, min_val, best_node = -1;
  6006. min_val = INT_MAX;
  6007. for (i = 0; i < nr_node_ids; i++) {
  6008. /* Start at @node */
  6009. n = (node + i) % nr_node_ids;
  6010. if (!nr_cpus_node(n))
  6011. continue;
  6012. /* Skip already used nodes */
  6013. if (node_isset(n, *used_nodes))
  6014. continue;
  6015. /* Simple min distance search */
  6016. val = node_distance(node, n);
  6017. if (val < min_val) {
  6018. min_val = val;
  6019. best_node = n;
  6020. }
  6021. }
  6022. if (best_node != -1)
  6023. node_set(best_node, *used_nodes);
  6024. return best_node;
  6025. }
  6026. /**
  6027. * sched_domain_node_span - get a cpumask for a node's sched_domain
  6028. * @node: node whose cpumask we're constructing
  6029. * @span: resulting cpumask
  6030. *
  6031. * Given a node, construct a good cpumask for its sched_domain to span. It
  6032. * should be one that prevents unnecessary balancing, but also spreads tasks
  6033. * out optimally.
  6034. */
  6035. static void sched_domain_node_span(int node, struct cpumask *span)
  6036. {
  6037. nodemask_t used_nodes;
  6038. int i;
  6039. cpumask_clear(span);
  6040. nodes_clear(used_nodes);
  6041. cpumask_or(span, span, cpumask_of_node(node));
  6042. node_set(node, used_nodes);
  6043. for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
  6044. int next_node = find_next_best_node(node, &used_nodes);
  6045. if (next_node < 0)
  6046. break;
  6047. cpumask_or(span, span, cpumask_of_node(next_node));
  6048. }
  6049. }
  6050. static const struct cpumask *cpu_node_mask(int cpu)
  6051. {
  6052. lockdep_assert_held(&sched_domains_mutex);
  6053. sched_domain_node_span(cpu_to_node(cpu), sched_domains_tmpmask);
  6054. return sched_domains_tmpmask;
  6055. }
  6056. static const struct cpumask *cpu_allnodes_mask(int cpu)
  6057. {
  6058. return cpu_possible_mask;
  6059. }
  6060. #endif /* CONFIG_NUMA */
  6061. static const struct cpumask *cpu_cpu_mask(int cpu)
  6062. {
  6063. return cpumask_of_node(cpu_to_node(cpu));
  6064. }
  6065. int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
  6066. struct sd_data {
  6067. struct sched_domain **__percpu sd;
  6068. struct sched_group **__percpu sg;
  6069. struct sched_group_power **__percpu sgp;
  6070. };
  6071. struct s_data {
  6072. struct sched_domain ** __percpu sd;
  6073. struct root_domain *rd;
  6074. };
  6075. enum s_alloc {
  6076. sa_rootdomain,
  6077. sa_sd,
  6078. sa_sd_storage,
  6079. sa_none,
  6080. };
  6081. struct sched_domain_topology_level;
  6082. typedef struct sched_domain *(*sched_domain_init_f)(struct sched_domain_topology_level *tl, int cpu);
  6083. typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);
  6084. #define SDTL_OVERLAP 0x01
  6085. struct sched_domain_topology_level {
  6086. sched_domain_init_f init;
  6087. sched_domain_mask_f mask;
  6088. int flags;
  6089. struct sd_data data;
  6090. };
  6091. static int
  6092. build_overlap_sched_groups(struct sched_domain *sd, int cpu)
  6093. {
  6094. struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg;
  6095. const struct cpumask *span = sched_domain_span(sd);
  6096. struct cpumask *covered = sched_domains_tmpmask;
  6097. struct sd_data *sdd = sd->private;
  6098. struct sched_domain *child;
  6099. int i;
  6100. cpumask_clear(covered);
  6101. for_each_cpu(i, span) {
  6102. struct cpumask *sg_span;
  6103. if (cpumask_test_cpu(i, covered))
  6104. continue;
  6105. sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
  6106. GFP_KERNEL, cpu_to_node(i));
  6107. if (!sg)
  6108. goto fail;
  6109. sg_span = sched_group_cpus(sg);
  6110. child = *per_cpu_ptr(sdd->sd, i);
  6111. if (child->child) {
  6112. child = child->child;
  6113. cpumask_copy(sg_span, sched_domain_span(child));
  6114. } else
  6115. cpumask_set_cpu(i, sg_span);
  6116. cpumask_or(covered, covered, sg_span);
  6117. sg->sgp = *per_cpu_ptr(sdd->sgp, cpumask_first(sg_span));
  6118. atomic_inc(&sg->sgp->ref);
  6119. if (cpumask_test_cpu(cpu, sg_span))
  6120. groups = sg;
  6121. if (!first)
  6122. first = sg;
  6123. if (last)
  6124. last->next = sg;
  6125. last = sg;
  6126. last->next = first;
  6127. }
  6128. sd->groups = groups;
  6129. return 0;
  6130. fail:
  6131. free_sched_groups(first, 0);
  6132. return -ENOMEM;
  6133. }
  6134. static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
  6135. {
  6136. struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
  6137. struct sched_domain *child = sd->child;
  6138. if (child)
  6139. cpu = cpumask_first(sched_domain_span(child));
  6140. if (sg) {
  6141. *sg = *per_cpu_ptr(sdd->sg, cpu);
  6142. (*sg)->sgp = *per_cpu_ptr(sdd->sgp, cpu);
  6143. atomic_set(&(*sg)->sgp->ref, 1); /* for claim_allocations */
  6144. }
  6145. return cpu;
  6146. }
  6147. /*
  6148. * build_sched_groups will build a circular linked list of the groups
  6149. * covered by the given span, and will set each group's ->cpumask correctly,
  6150. * and ->cpu_power to 0.
  6151. *
  6152. * Assumes the sched_domain tree is fully constructed
  6153. */
  6154. static int
  6155. build_sched_groups(struct sched_domain *sd, int cpu)
  6156. {
  6157. struct sched_group *first = NULL, *last = NULL;
  6158. struct sd_data *sdd = sd->private;
  6159. const struct cpumask *span = sched_domain_span(sd);
  6160. struct cpumask *covered;
  6161. int i;
  6162. get_group(cpu, sdd, &sd->groups);
  6163. atomic_inc(&sd->groups->ref);
  6164. if (cpu != cpumask_first(sched_domain_span(sd)))
  6165. return 0;
  6166. lockdep_assert_held(&sched_domains_mutex);
  6167. covered = sched_domains_tmpmask;
  6168. cpumask_clear(covered);
  6169. for_each_cpu(i, span) {
  6170. struct sched_group *sg;
  6171. int group = get_group(i, sdd, &sg);
  6172. int j;
  6173. if (cpumask_test_cpu(i, covered))
  6174. continue;
  6175. cpumask_clear(sched_group_cpus(sg));
  6176. sg->sgp->power = 0;
  6177. for_each_cpu(j, span) {
  6178. if (get_group(j, sdd, NULL) != group)
  6179. continue;
  6180. cpumask_set_cpu(j, covered);
  6181. cpumask_set_cpu(j, sched_group_cpus(sg));
  6182. }
  6183. if (!first)
  6184. first = sg;
  6185. if (last)
  6186. last->next = sg;
  6187. last = sg;
  6188. }
  6189. last->next = first;
  6190. return 0;
  6191. }
  6192. /*
  6193. * Initialize sched groups cpu_power.
  6194. *
  6195. * cpu_power indicates the capacity of sched group, which is used while
  6196. * distributing the load between different sched groups in a sched domain.
  6197. * Typically cpu_power for all the groups in a sched domain will be same unless
  6198. * there are asymmetries in the topology. If there are asymmetries, group
  6199. * having more cpu_power will pickup more load compared to the group having
  6200. * less cpu_power.
  6201. */
  6202. static void init_sched_groups_power(int cpu, struct sched_domain *sd)
  6203. {
  6204. struct sched_group *sg = sd->groups;
  6205. WARN_ON(!sd || !sg);
  6206. do {
  6207. sg->group_weight = cpumask_weight(sched_group_cpus(sg));
  6208. sg = sg->next;
  6209. } while (sg != sd->groups);
  6210. if (cpu != group_first_cpu(sg))
  6211. return;
  6212. update_group_power(sd, cpu);
  6213. }
  6214. /*
  6215. * Initializers for schedule domains
  6216. * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
  6217. */
  6218. #ifdef CONFIG_SCHED_DEBUG
  6219. # define SD_INIT_NAME(sd, type) sd->name = #type
  6220. #else
  6221. # define SD_INIT_NAME(sd, type) do { } while (0)
  6222. #endif
  6223. #define SD_INIT_FUNC(type) \
  6224. static noinline struct sched_domain * \
  6225. sd_init_##type(struct sched_domain_topology_level *tl, int cpu) \
  6226. { \
  6227. struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu); \
  6228. *sd = SD_##type##_INIT; \
  6229. SD_INIT_NAME(sd, type); \
  6230. sd->private = &tl->data; \
  6231. return sd; \
  6232. }
  6233. SD_INIT_FUNC(CPU)
  6234. #ifdef CONFIG_NUMA
  6235. SD_INIT_FUNC(ALLNODES)
  6236. SD_INIT_FUNC(NODE)
  6237. #endif
  6238. #ifdef CONFIG_SCHED_SMT
  6239. SD_INIT_FUNC(SIBLING)
  6240. #endif
  6241. #ifdef CONFIG_SCHED_MC
  6242. SD_INIT_FUNC(MC)
  6243. #endif
  6244. #ifdef CONFIG_SCHED_BOOK
  6245. SD_INIT_FUNC(BOOK)
  6246. #endif
  6247. static int default_relax_domain_level = -1;
  6248. int sched_domain_level_max;
  6249. static int __init setup_relax_domain_level(char *str)
  6250. {
  6251. unsigned long val;
  6252. val = simple_strtoul(str, NULL, 0);
  6253. if (val < sched_domain_level_max)
  6254. default_relax_domain_level = val;
  6255. return 1;
  6256. }
  6257. __setup("relax_domain_level=", setup_relax_domain_level);
  6258. static void set_domain_attribute(struct sched_domain *sd,
  6259. struct sched_domain_attr *attr)
  6260. {
  6261. int request;
  6262. if (!attr || attr->relax_domain_level < 0) {
  6263. if (default_relax_domain_level < 0)
  6264. return;
  6265. else
  6266. request = default_relax_domain_level;
  6267. } else
  6268. request = attr->relax_domain_level;
  6269. if (request < sd->level) {
  6270. /* turn off idle balance on this domain */
  6271. sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
  6272. } else {
  6273. /* turn on idle balance on this domain */
  6274. sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
  6275. }
  6276. }
  6277. static void __sdt_free(const struct cpumask *cpu_map);
  6278. static int __sdt_alloc(const struct cpumask *cpu_map);
  6279. static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
  6280. const struct cpumask *cpu_map)
  6281. {
  6282. switch (what) {
  6283. case sa_rootdomain:
  6284. if (!atomic_read(&d->rd->refcount))
  6285. free_rootdomain(&d->rd->rcu); /* fall through */
  6286. case sa_sd:
  6287. free_percpu(d->sd); /* fall through */
  6288. case sa_sd_storage:
  6289. __sdt_free(cpu_map); /* fall through */
  6290. case sa_none:
  6291. break;
  6292. }
  6293. }
  6294. static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
  6295. const struct cpumask *cpu_map)
  6296. {
  6297. memset(d, 0, sizeof(*d));
  6298. if (__sdt_alloc(cpu_map))
  6299. return sa_sd_storage;
  6300. d->sd = alloc_percpu(struct sched_domain *);
  6301. if (!d->sd)
  6302. return sa_sd_storage;
  6303. d->rd = alloc_rootdomain();
  6304. if (!d->rd)
  6305. return sa_sd;
  6306. return sa_rootdomain;
  6307. }
  6308. /*
  6309. * NULL the sd_data elements we've used to build the sched_domain and
  6310. * sched_group structure so that the subsequent __free_domain_allocs()
  6311. * will not free the data we're using.
  6312. */
  6313. static void claim_allocations(int cpu, struct sched_domain *sd)
  6314. {
  6315. struct sd_data *sdd = sd->private;
  6316. WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
  6317. *per_cpu_ptr(sdd->sd, cpu) = NULL;
  6318. if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
  6319. *per_cpu_ptr(sdd->sg, cpu) = NULL;
  6320. if (atomic_read(&(*per_cpu_ptr(sdd->sgp, cpu))->ref))
  6321. *per_cpu_ptr(sdd->sgp, cpu) = NULL;
  6322. }
  6323. #ifdef CONFIG_SCHED_SMT
  6324. static const struct cpumask *cpu_smt_mask(int cpu)
  6325. {
  6326. return topology_thread_cpumask(cpu);
  6327. }
  6328. #endif
  6329. /*
  6330. * Topology list, bottom-up.
  6331. */
  6332. static struct sched_domain_topology_level default_topology[] = {
  6333. #ifdef CONFIG_SCHED_SMT
  6334. { sd_init_SIBLING, cpu_smt_mask, },
  6335. #endif
  6336. #ifdef CONFIG_SCHED_MC
  6337. { sd_init_MC, cpu_coregroup_mask, },
  6338. #endif
  6339. #ifdef CONFIG_SCHED_BOOK
  6340. { sd_init_BOOK, cpu_book_mask, },
  6341. #endif
  6342. { sd_init_CPU, cpu_cpu_mask, },
  6343. #ifdef CONFIG_NUMA
  6344. { sd_init_NODE, cpu_node_mask, SDTL_OVERLAP, },
  6345. { sd_init_ALLNODES, cpu_allnodes_mask, },
  6346. #endif
  6347. { NULL, },
  6348. };
  6349. static struct sched_domain_topology_level *sched_domain_topology = default_topology;
  6350. static int __sdt_alloc(const struct cpumask *cpu_map)
  6351. {
  6352. struct sched_domain_topology_level *tl;
  6353. int j;
  6354. for (tl = sched_domain_topology; tl->init; tl++) {
  6355. struct sd_data *sdd = &tl->data;
  6356. sdd->sd = alloc_percpu(struct sched_domain *);
  6357. if (!sdd->sd)
  6358. return -ENOMEM;
  6359. sdd->sg = alloc_percpu(struct sched_group *);
  6360. if (!sdd->sg)
  6361. return -ENOMEM;
  6362. sdd->sgp = alloc_percpu(struct sched_group_power *);
  6363. if (!sdd->sgp)
  6364. return -ENOMEM;
  6365. for_each_cpu(j, cpu_map) {
  6366. struct sched_domain *sd;
  6367. struct sched_group *sg;
  6368. struct sched_group_power *sgp;
  6369. sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
  6370. GFP_KERNEL, cpu_to_node(j));
  6371. if (!sd)
  6372. return -ENOMEM;
  6373. *per_cpu_ptr(sdd->sd, j) = sd;
  6374. sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
  6375. GFP_KERNEL, cpu_to_node(j));
  6376. if (!sg)
  6377. return -ENOMEM;
  6378. *per_cpu_ptr(sdd->sg, j) = sg;
  6379. sgp = kzalloc_node(sizeof(struct sched_group_power),
  6380. GFP_KERNEL, cpu_to_node(j));
  6381. if (!sgp)
  6382. return -ENOMEM;
  6383. *per_cpu_ptr(sdd->sgp, j) = sgp;
  6384. }
  6385. }
  6386. return 0;
  6387. }
  6388. static void __sdt_free(const struct cpumask *cpu_map)
  6389. {
  6390. struct sched_domain_topology_level *tl;
  6391. int j;
  6392. for (tl = sched_domain_topology; tl->init; tl++) {
  6393. struct sd_data *sdd = &tl->data;
  6394. for_each_cpu(j, cpu_map) {
  6395. struct sched_domain *sd = *per_cpu_ptr(sdd->sd, j);
  6396. if (sd && (sd->flags & SD_OVERLAP))
  6397. free_sched_groups(sd->groups, 0);
  6398. kfree(*per_cpu_ptr(sdd->sd, j));
  6399. kfree(*per_cpu_ptr(sdd->sg, j));
  6400. kfree(*per_cpu_ptr(sdd->sgp, j));
  6401. }
  6402. free_percpu(sdd->sd);
  6403. free_percpu(sdd->sg);
  6404. free_percpu(sdd->sgp);
  6405. }
  6406. }
  6407. struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
  6408. struct s_data *d, const struct cpumask *cpu_map,
  6409. struct sched_domain_attr *attr, struct sched_domain *child,
  6410. int cpu)
  6411. {
  6412. struct sched_domain *sd = tl->init(tl, cpu);
  6413. if (!sd)
  6414. return child;
  6415. set_domain_attribute(sd, attr);
  6416. cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
  6417. if (child) {
  6418. sd->level = child->level + 1;
  6419. sched_domain_level_max = max(sched_domain_level_max, sd->level);
  6420. child->parent = sd;
  6421. }
  6422. sd->child = child;
  6423. return sd;
  6424. }
  6425. /*
  6426. * Build sched domains for a given set of cpus and attach the sched domains
  6427. * to the individual cpus
  6428. */
  6429. static int build_sched_domains(const struct cpumask *cpu_map,
  6430. struct sched_domain_attr *attr)
  6431. {
  6432. enum s_alloc alloc_state = sa_none;
  6433. struct sched_domain *sd;
  6434. struct s_data d;
  6435. int i, ret = -ENOMEM;
  6436. alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
  6437. if (alloc_state != sa_rootdomain)
  6438. goto error;
  6439. /* Set up domains for cpus specified by the cpu_map. */
  6440. for_each_cpu(i, cpu_map) {
  6441. struct sched_domain_topology_level *tl;
  6442. sd = NULL;
  6443. for (tl = sched_domain_topology; tl->init; tl++) {
  6444. sd = build_sched_domain(tl, &d, cpu_map, attr, sd, i);
  6445. if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP))
  6446. sd->flags |= SD_OVERLAP;
  6447. if (cpumask_equal(cpu_map, sched_domain_span(sd)))
  6448. break;
  6449. }
  6450. while (sd->child)
  6451. sd = sd->child;
  6452. *per_cpu_ptr(d.sd, i) = sd;
  6453. }
  6454. /* Build the groups for the domains */
  6455. for_each_cpu(i, cpu_map) {
  6456. for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
  6457. sd->span_weight = cpumask_weight(sched_domain_span(sd));
  6458. if (sd->flags & SD_OVERLAP) {
  6459. if (build_overlap_sched_groups(sd, i))
  6460. goto error;
  6461. } else {
  6462. if (build_sched_groups(sd, i))
  6463. goto error;
  6464. }
  6465. }
  6466. }
  6467. /* Calculate CPU power for physical packages and nodes */
  6468. for (i = nr_cpumask_bits-1; i >= 0; i--) {
  6469. if (!cpumask_test_cpu(i, cpu_map))
  6470. continue;
  6471. for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
  6472. claim_allocations(i, sd);
  6473. init_sched_groups_power(i, sd);
  6474. }
  6475. }
  6476. /* Attach the domains */
  6477. rcu_read_lock();
  6478. for_each_cpu(i, cpu_map) {
  6479. sd = *per_cpu_ptr(d.sd, i);
  6480. cpu_attach_domain(sd, d.rd, i);
  6481. }
  6482. rcu_read_unlock();
  6483. ret = 0;
  6484. error:
  6485. __free_domain_allocs(&d, alloc_state, cpu_map);
  6486. return ret;
  6487. }
  6488. static cpumask_var_t *doms_cur; /* current sched domains */
  6489. static int ndoms_cur; /* number of sched domains in 'doms_cur' */
  6490. static struct sched_domain_attr *dattr_cur;
  6491. /* attribues of custom domains in 'doms_cur' */
  6492. /*
  6493. * Special case: If a kmalloc of a doms_cur partition (array of
  6494. * cpumask) fails, then fallback to a single sched domain,
  6495. * as determined by the single cpumask fallback_doms.
  6496. */
  6497. static cpumask_var_t fallback_doms;
  6498. /*
  6499. * arch_update_cpu_topology lets virtualized architectures update the
  6500. * cpu core maps. It is supposed to return 1 if the topology changed
  6501. * or 0 if it stayed the same.
  6502. */
  6503. int __attribute__((weak)) arch_update_cpu_topology(void)
  6504. {
  6505. return 0;
  6506. }
  6507. cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
  6508. {
  6509. int i;
  6510. cpumask_var_t *doms;
  6511. doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
  6512. if (!doms)
  6513. return NULL;
  6514. for (i = 0; i < ndoms; i++) {
  6515. if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
  6516. free_sched_domains(doms, i);
  6517. return NULL;
  6518. }
  6519. }
  6520. return doms;
  6521. }
  6522. void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
  6523. {
  6524. unsigned int i;
  6525. for (i = 0; i < ndoms; i++)
  6526. free_cpumask_var(doms[i]);
  6527. kfree(doms);
  6528. }
  6529. /*
  6530. * Set up scheduler domains and groups. Callers must hold the hotplug lock.
  6531. * For now this just excludes isolated cpus, but could be used to
  6532. * exclude other special cases in the future.
  6533. */
  6534. static int init_sched_domains(const struct cpumask *cpu_map)
  6535. {
  6536. int err;
  6537. arch_update_cpu_topology();
  6538. ndoms_cur = 1;
  6539. doms_cur = alloc_sched_domains(ndoms_cur);
  6540. if (!doms_cur)
  6541. doms_cur = &fallback_doms;
  6542. cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
  6543. dattr_cur = NULL;
  6544. err = build_sched_domains(doms_cur[0], NULL);
  6545. register_sched_domain_sysctl();
  6546. return err;
  6547. }
  6548. /*
  6549. * Detach sched domains from a group of cpus specified in cpu_map
  6550. * These cpus will now be attached to the NULL domain
  6551. */
  6552. static void detach_destroy_domains(const struct cpumask *cpu_map)
  6553. {
  6554. int i;
  6555. rcu_read_lock();
  6556. for_each_cpu(i, cpu_map)
  6557. cpu_attach_domain(NULL, &def_root_domain, i);
  6558. rcu_read_unlock();
  6559. }
  6560. /* handle null as "default" */
  6561. static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
  6562. struct sched_domain_attr *new, int idx_new)
  6563. {
  6564. struct sched_domain_attr tmp;
  6565. /* fast path */
  6566. if (!new && !cur)
  6567. return 1;
  6568. tmp = SD_ATTR_INIT;
  6569. return !memcmp(cur ? (cur + idx_cur) : &tmp,
  6570. new ? (new + idx_new) : &tmp,
  6571. sizeof(struct sched_domain_attr));
  6572. }
  6573. /*
  6574. * Partition sched domains as specified by the 'ndoms_new'
  6575. * cpumasks in the array doms_new[] of cpumasks. This compares
  6576. * doms_new[] to the current sched domain partitioning, doms_cur[].
  6577. * It destroys each deleted domain and builds each new domain.
  6578. *
  6579. * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
  6580. * The masks don't intersect (don't overlap.) We should setup one
  6581. * sched domain for each mask. CPUs not in any of the cpumasks will
  6582. * not be load balanced. If the same cpumask appears both in the
  6583. * current 'doms_cur' domains and in the new 'doms_new', we can leave
  6584. * it as it is.
  6585. *
  6586. * The passed in 'doms_new' should be allocated using
  6587. * alloc_sched_domains. This routine takes ownership of it and will
  6588. * free_sched_domains it when done with it. If the caller failed the
  6589. * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
  6590. * and partition_sched_domains() will fallback to the single partition
  6591. * 'fallback_doms', it also forces the domains to be rebuilt.
  6592. *
  6593. * If doms_new == NULL it will be replaced with cpu_online_mask.
  6594. * ndoms_new == 0 is a special case for destroying existing domains,
  6595. * and it will not create the default domain.
  6596. *
  6597. * Call with hotplug lock held
  6598. */
  6599. void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
  6600. struct sched_domain_attr *dattr_new)
  6601. {
  6602. int i, j, n;
  6603. int new_topology;
  6604. mutex_lock(&sched_domains_mutex);
  6605. /* always unregister in case we don't destroy any domains */
  6606. unregister_sched_domain_sysctl();
  6607. /* Let architecture update cpu core mappings. */
  6608. new_topology = arch_update_cpu_topology();
  6609. n = doms_new ? ndoms_new : 0;
  6610. /* Destroy deleted domains */
  6611. for (i = 0; i < ndoms_cur; i++) {
  6612. for (j = 0; j < n && !new_topology; j++) {
  6613. if (cpumask_equal(doms_cur[i], doms_new[j])
  6614. && dattrs_equal(dattr_cur, i, dattr_new, j))
  6615. goto match1;
  6616. }
  6617. /* no match - a current sched domain not in new doms_new[] */
  6618. detach_destroy_domains(doms_cur[i]);
  6619. match1:
  6620. ;
  6621. }
  6622. if (doms_new == NULL) {
  6623. ndoms_cur = 0;
  6624. doms_new = &fallback_doms;
  6625. cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
  6626. WARN_ON_ONCE(dattr_new);
  6627. }
  6628. /* Build new domains */
  6629. for (i = 0; i < ndoms_new; i++) {
  6630. for (j = 0; j < ndoms_cur && !new_topology; j++) {
  6631. if (cpumask_equal(doms_new[i], doms_cur[j])
  6632. && dattrs_equal(dattr_new, i, dattr_cur, j))
  6633. goto match2;
  6634. }
  6635. /* no match - add a new doms_new */
  6636. build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
  6637. match2:
  6638. ;
  6639. }
  6640. /* Remember the new sched domains */
  6641. if (doms_cur != &fallback_doms)
  6642. free_sched_domains(doms_cur, ndoms_cur);
  6643. kfree(dattr_cur); /* kfree(NULL) is safe */
  6644. doms_cur = doms_new;
  6645. dattr_cur = dattr_new;
  6646. ndoms_cur = ndoms_new;
  6647. register_sched_domain_sysctl();
  6648. mutex_unlock(&sched_domains_mutex);
  6649. }
  6650. #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
  6651. static void reinit_sched_domains(void)
  6652. {
  6653. get_online_cpus();
  6654. /* Destroy domains first to force the rebuild */
  6655. partition_sched_domains(0, NULL, NULL);
  6656. rebuild_sched_domains();
  6657. put_online_cpus();
  6658. }
  6659. static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
  6660. {
  6661. unsigned int level = 0;
  6662. if (sscanf(buf, "%u", &level) != 1)
  6663. return -EINVAL;
  6664. /*
  6665. * level is always be positive so don't check for
  6666. * level < POWERSAVINGS_BALANCE_NONE which is 0
  6667. * What happens on 0 or 1 byte write,
  6668. * need to check for count as well?
  6669. */
  6670. if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
  6671. return -EINVAL;
  6672. if (smt)
  6673. sched_smt_power_savings = level;
  6674. else
  6675. sched_mc_power_savings = level;
  6676. reinit_sched_domains();
  6677. return count;
  6678. }
  6679. #ifdef CONFIG_SCHED_MC
  6680. static ssize_t sched_mc_power_savings_show(struct sysdev_class *class,
  6681. struct sysdev_class_attribute *attr,
  6682. char *page)
  6683. {
  6684. return sprintf(page, "%u\n", sched_mc_power_savings);
  6685. }
  6686. static ssize_t sched_mc_power_savings_store(struct sysdev_class *class,
  6687. struct sysdev_class_attribute *attr,
  6688. const char *buf, size_t count)
  6689. {
  6690. return sched_power_savings_store(buf, count, 0);
  6691. }
  6692. static SYSDEV_CLASS_ATTR(sched_mc_power_savings, 0644,
  6693. sched_mc_power_savings_show,
  6694. sched_mc_power_savings_store);
  6695. #endif
  6696. #ifdef CONFIG_SCHED_SMT
  6697. static ssize_t sched_smt_power_savings_show(struct sysdev_class *dev,
  6698. struct sysdev_class_attribute *attr,
  6699. char *page)
  6700. {
  6701. return sprintf(page, "%u\n", sched_smt_power_savings);
  6702. }
  6703. static ssize_t sched_smt_power_savings_store(struct sysdev_class *dev,
  6704. struct sysdev_class_attribute *attr,
  6705. const char *buf, size_t count)
  6706. {
  6707. return sched_power_savings_store(buf, count, 1);
  6708. }
  6709. static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
  6710. sched_smt_power_savings_show,
  6711. sched_smt_power_savings_store);
  6712. #endif
  6713. int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
  6714. {
  6715. int err = 0;
  6716. #ifdef CONFIG_SCHED_SMT
  6717. if (smt_capable())
  6718. err = sysfs_create_file(&cls->kset.kobj,
  6719. &attr_sched_smt_power_savings.attr);
  6720. #endif
  6721. #ifdef CONFIG_SCHED_MC
  6722. if (!err && mc_capable())
  6723. err = sysfs_create_file(&cls->kset.kobj,
  6724. &attr_sched_mc_power_savings.attr);
  6725. #endif
  6726. return err;
  6727. }
  6728. #endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
  6729. /*
  6730. * Update cpusets according to cpu_active mask. If cpusets are
  6731. * disabled, cpuset_update_active_cpus() becomes a simple wrapper
  6732. * around partition_sched_domains().
  6733. */
  6734. static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
  6735. void *hcpu)
  6736. {
  6737. switch (action & ~CPU_TASKS_FROZEN) {
  6738. case CPU_ONLINE:
  6739. case CPU_DOWN_FAILED:
  6740. cpuset_update_active_cpus();
  6741. return NOTIFY_OK;
  6742. default:
  6743. return NOTIFY_DONE;
  6744. }
  6745. }
  6746. static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
  6747. void *hcpu)
  6748. {
  6749. switch (action & ~CPU_TASKS_FROZEN) {
  6750. case CPU_DOWN_PREPARE:
  6751. cpuset_update_active_cpus();
  6752. return NOTIFY_OK;
  6753. default:
  6754. return NOTIFY_DONE;
  6755. }
  6756. }
  6757. static int update_runtime(struct notifier_block *nfb,
  6758. unsigned long action, void *hcpu)
  6759. {
  6760. int cpu = (int)(long)hcpu;
  6761. switch (action) {
  6762. case CPU_DOWN_PREPARE:
  6763. case CPU_DOWN_PREPARE_FROZEN:
  6764. disable_runtime(cpu_rq(cpu));
  6765. return NOTIFY_OK;
  6766. case CPU_DOWN_FAILED:
  6767. case CPU_DOWN_FAILED_FROZEN:
  6768. case CPU_ONLINE:
  6769. case CPU_ONLINE_FROZEN:
  6770. enable_runtime(cpu_rq(cpu));
  6771. return NOTIFY_OK;
  6772. default:
  6773. return NOTIFY_DONE;
  6774. }
  6775. }
  6776. void __init sched_init_smp(void)
  6777. {
  6778. cpumask_var_t non_isolated_cpus;
  6779. alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
  6780. alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
  6781. get_online_cpus();
  6782. mutex_lock(&sched_domains_mutex);
  6783. init_sched_domains(cpu_active_mask);
  6784. cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
  6785. if (cpumask_empty(non_isolated_cpus))
  6786. cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
  6787. mutex_unlock(&sched_domains_mutex);
  6788. put_online_cpus();
  6789. hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
  6790. hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
  6791. /* RT runtime code needs to handle some hotplug events */
  6792. hotcpu_notifier(update_runtime, 0);
  6793. init_hrtick();
  6794. /* Move init over to a non-isolated CPU */
  6795. if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
  6796. BUG();
  6797. sched_init_granularity();
  6798. free_cpumask_var(non_isolated_cpus);
  6799. init_sched_rt_class();
  6800. }
  6801. #else
  6802. void __init sched_init_smp(void)
  6803. {
  6804. sched_init_granularity();
  6805. }
  6806. #endif /* CONFIG_SMP */
  6807. const_debug unsigned int sysctl_timer_migration = 1;
  6808. int in_sched_functions(unsigned long addr)
  6809. {
  6810. return in_lock_functions(addr) ||
  6811. (addr >= (unsigned long)__sched_text_start
  6812. && addr < (unsigned long)__sched_text_end);
  6813. }
  6814. static void init_cfs_rq(struct cfs_rq *cfs_rq)
  6815. {
  6816. cfs_rq->tasks_timeline = RB_ROOT;
  6817. INIT_LIST_HEAD(&cfs_rq->tasks);
  6818. cfs_rq->min_vruntime = (u64)(-(1LL << 20));
  6819. #ifndef CONFIG_64BIT
  6820. cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
  6821. #endif
  6822. }
  6823. static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
  6824. {
  6825. struct rt_prio_array *array;
  6826. int i;
  6827. array = &rt_rq->active;
  6828. for (i = 0; i < MAX_RT_PRIO; i++) {
  6829. INIT_LIST_HEAD(array->queue + i);
  6830. __clear_bit(i, array->bitmap);
  6831. }
  6832. /* delimiter for bitsearch: */
  6833. __set_bit(MAX_RT_PRIO, array->bitmap);
  6834. #if defined CONFIG_SMP
  6835. rt_rq->highest_prio.curr = MAX_RT_PRIO;
  6836. rt_rq->highest_prio.next = MAX_RT_PRIO;
  6837. rt_rq->rt_nr_migratory = 0;
  6838. rt_rq->overloaded = 0;
  6839. plist_head_init(&rt_rq->pushable_tasks);
  6840. #endif
  6841. rt_rq->rt_time = 0;
  6842. rt_rq->rt_throttled = 0;
  6843. rt_rq->rt_runtime = 0;
  6844. raw_spin_lock_init(&rt_rq->rt_runtime_lock);
  6845. }
  6846. #ifdef CONFIG_FAIR_GROUP_SCHED
  6847. static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
  6848. struct sched_entity *se, int cpu,
  6849. struct sched_entity *parent)
  6850. {
  6851. struct rq *rq = cpu_rq(cpu);
  6852. cfs_rq->tg = tg;
  6853. cfs_rq->rq = rq;
  6854. #ifdef CONFIG_SMP
  6855. /* allow initial update_cfs_load() to truncate */
  6856. cfs_rq->load_stamp = 1;
  6857. #endif
  6858. init_cfs_rq_runtime(cfs_rq);
  6859. tg->cfs_rq[cpu] = cfs_rq;
  6860. tg->se[cpu] = se;
  6861. /* se could be NULL for root_task_group */
  6862. if (!se)
  6863. return;
  6864. if (!parent)
  6865. se->cfs_rq = &rq->cfs;
  6866. else
  6867. se->cfs_rq = parent->my_q;
  6868. se->my_q = cfs_rq;
  6869. update_load_set(&se->load, 0);
  6870. se->parent = parent;
  6871. }
  6872. #endif
  6873. #ifdef CONFIG_RT_GROUP_SCHED
  6874. static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
  6875. struct sched_rt_entity *rt_se, int cpu,
  6876. struct sched_rt_entity *parent)
  6877. {
  6878. struct rq *rq = cpu_rq(cpu);
  6879. rt_rq->highest_prio.curr = MAX_RT_PRIO;
  6880. rt_rq->rt_nr_boosted = 0;
  6881. rt_rq->rq = rq;
  6882. rt_rq->tg = tg;
  6883. tg->rt_rq[cpu] = rt_rq;
  6884. tg->rt_se[cpu] = rt_se;
  6885. if (!rt_se)
  6886. return;
  6887. if (!parent)
  6888. rt_se->rt_rq = &rq->rt;
  6889. else
  6890. rt_se->rt_rq = parent->my_q;
  6891. rt_se->my_q = rt_rq;
  6892. rt_se->parent = parent;
  6893. INIT_LIST_HEAD(&rt_se->run_list);
  6894. }
  6895. #endif
  6896. void __init sched_init(void)
  6897. {
  6898. int i, j;
  6899. unsigned long alloc_size = 0, ptr;
  6900. #ifdef CONFIG_FAIR_GROUP_SCHED
  6901. alloc_size += 2 * nr_cpu_ids * sizeof(void **);
  6902. #endif
  6903. #ifdef CONFIG_RT_GROUP_SCHED
  6904. alloc_size += 2 * nr_cpu_ids * sizeof(void **);
  6905. #endif
  6906. #ifdef CONFIG_CPUMASK_OFFSTACK
  6907. alloc_size += num_possible_cpus() * cpumask_size();
  6908. #endif
  6909. if (alloc_size) {
  6910. ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
  6911. #ifdef CONFIG_FAIR_GROUP_SCHED
  6912. root_task_group.se = (struct sched_entity **)ptr;
  6913. ptr += nr_cpu_ids * sizeof(void **);
  6914. root_task_group.cfs_rq = (struct cfs_rq **)ptr;
  6915. ptr += nr_cpu_ids * sizeof(void **);
  6916. #endif /* CONFIG_FAIR_GROUP_SCHED */
  6917. #ifdef CONFIG_RT_GROUP_SCHED
  6918. root_task_group.rt_se = (struct sched_rt_entity **)ptr;
  6919. ptr += nr_cpu_ids * sizeof(void **);
  6920. root_task_group.rt_rq = (struct rt_rq **)ptr;
  6921. ptr += nr_cpu_ids * sizeof(void **);
  6922. #endif /* CONFIG_RT_GROUP_SCHED */
  6923. #ifdef CONFIG_CPUMASK_OFFSTACK
  6924. for_each_possible_cpu(i) {
  6925. per_cpu(load_balance_tmpmask, i) = (void *)ptr;
  6926. ptr += cpumask_size();
  6927. }
  6928. #endif /* CONFIG_CPUMASK_OFFSTACK */
  6929. }
  6930. #ifdef CONFIG_SMP
  6931. init_defrootdomain();
  6932. #endif
  6933. init_rt_bandwidth(&def_rt_bandwidth,
  6934. global_rt_period(), global_rt_runtime());
  6935. #ifdef CONFIG_RT_GROUP_SCHED
  6936. init_rt_bandwidth(&root_task_group.rt_bandwidth,
  6937. global_rt_period(), global_rt_runtime());
  6938. #endif /* CONFIG_RT_GROUP_SCHED */
  6939. #ifdef CONFIG_CGROUP_SCHED
  6940. list_add(&root_task_group.list, &task_groups);
  6941. INIT_LIST_HEAD(&root_task_group.children);
  6942. autogroup_init(&init_task);
  6943. #endif /* CONFIG_CGROUP_SCHED */
  6944. for_each_possible_cpu(i) {
  6945. struct rq *rq;
  6946. rq = cpu_rq(i);
  6947. raw_spin_lock_init(&rq->lock);
  6948. rq->nr_running = 0;
  6949. rq->calc_load_active = 0;
  6950. rq->calc_load_update = jiffies + LOAD_FREQ;
  6951. init_cfs_rq(&rq->cfs);
  6952. init_rt_rq(&rq->rt, rq);
  6953. #ifdef CONFIG_FAIR_GROUP_SCHED
  6954. root_task_group.shares = root_task_group_load;
  6955. INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
  6956. /*
  6957. * How much cpu bandwidth does root_task_group get?
  6958. *
  6959. * In case of task-groups formed thr' the cgroup filesystem, it
  6960. * gets 100% of the cpu resources in the system. This overall
  6961. * system cpu resource is divided among the tasks of
  6962. * root_task_group and its child task-groups in a fair manner,
  6963. * based on each entity's (task or task-group's) weight
  6964. * (se->load.weight).
  6965. *
  6966. * In other words, if root_task_group has 10 tasks of weight
  6967. * 1024) and two child groups A0 and A1 (of weight 1024 each),
  6968. * then A0's share of the cpu resource is:
  6969. *
  6970. * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
  6971. *
  6972. * We achieve this by letting root_task_group's tasks sit
  6973. * directly in rq->cfs (i.e root_task_group->se[] = NULL).
  6974. */
  6975. init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
  6976. init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
  6977. #endif /* CONFIG_FAIR_GROUP_SCHED */
  6978. rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
  6979. #ifdef CONFIG_RT_GROUP_SCHED
  6980. INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
  6981. init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
  6982. #endif
  6983. for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
  6984. rq->cpu_load[j] = 0;
  6985. rq->last_load_update_tick = jiffies;
  6986. #ifdef CONFIG_SMP
  6987. rq->sd = NULL;
  6988. rq->rd = NULL;
  6989. rq->cpu_power = SCHED_POWER_SCALE;
  6990. rq->post_schedule = 0;
  6991. rq->active_balance = 0;
  6992. rq->next_balance = jiffies;
  6993. rq->push_cpu = 0;
  6994. rq->cpu = i;
  6995. rq->online = 0;
  6996. rq->idle_stamp = 0;
  6997. rq->avg_idle = 2*sysctl_sched_migration_cost;
  6998. rq_attach_root(rq, &def_root_domain);
  6999. #ifdef CONFIG_NO_HZ
  7000. rq->nohz_balance_kick = 0;
  7001. init_sched_softirq_csd(&per_cpu(remote_sched_softirq_cb, i));
  7002. #endif
  7003. #endif
  7004. init_rq_hrtick(rq);
  7005. atomic_set(&rq->nr_iowait, 0);
  7006. }
  7007. set_load_weight(&init_task);
  7008. #ifdef CONFIG_PREEMPT_NOTIFIERS
  7009. INIT_HLIST_HEAD(&init_task.preempt_notifiers);
  7010. #endif
  7011. #ifdef CONFIG_SMP
  7012. open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
  7013. #endif
  7014. #ifdef CONFIG_RT_MUTEXES
  7015. plist_head_init(&init_task.pi_waiters);
  7016. #endif
  7017. /*
  7018. * The boot idle thread does lazy MMU switching as well:
  7019. */
  7020. atomic_inc(&init_mm.mm_count);
  7021. enter_lazy_tlb(&init_mm, current);
  7022. /*
  7023. * Make us the idle thread. Technically, schedule() should not be
  7024. * called from this thread, however somewhere below it might be,
  7025. * but because we are the idle thread, we just pick up running again
  7026. * when this runqueue becomes "idle".
  7027. */
  7028. init_idle(current, smp_processor_id());
  7029. calc_load_update = jiffies + LOAD_FREQ;
  7030. /*
  7031. * During early bootup we pretend to be a normal task:
  7032. */
  7033. current->sched_class = &fair_sched_class;
  7034. /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
  7035. zalloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT);
  7036. #ifdef CONFIG_SMP
  7037. zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
  7038. #ifdef CONFIG_NO_HZ
  7039. zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);
  7040. alloc_cpumask_var(&nohz.grp_idle_mask, GFP_NOWAIT);
  7041. atomic_set(&nohz.load_balancer, nr_cpu_ids);
  7042. atomic_set(&nohz.first_pick_cpu, nr_cpu_ids);
  7043. atomic_set(&nohz.second_pick_cpu, nr_cpu_ids);
  7044. #endif
  7045. /* May be allocated at isolcpus cmdline parse time */
  7046. if (cpu_isolated_map == NULL)
  7047. zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
  7048. #endif /* SMP */
  7049. scheduler_running = 1;
  7050. }
  7051. #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
  7052. static inline int preempt_count_equals(int preempt_offset)
  7053. {
  7054. int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
  7055. return (nested == preempt_offset);
  7056. }
  7057. void __might_sleep(const char *file, int line, int preempt_offset)
  7058. {
  7059. static unsigned long prev_jiffy; /* ratelimiting */
  7060. if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) ||
  7061. system_state != SYSTEM_RUNNING || oops_in_progress)
  7062. return;
  7063. if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
  7064. return;
  7065. prev_jiffy = jiffies;
  7066. printk(KERN_ERR
  7067. "BUG: sleeping function called from invalid context at %s:%d\n",
  7068. file, line);
  7069. printk(KERN_ERR
  7070. "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
  7071. in_atomic(), irqs_disabled(),
  7072. current->pid, current->comm);
  7073. debug_show_held_locks(current);
  7074. if (irqs_disabled())
  7075. print_irqtrace_events(current);
  7076. dump_stack();
  7077. }
  7078. EXPORT_SYMBOL(__might_sleep);
  7079. #endif
  7080. #ifdef CONFIG_MAGIC_SYSRQ
  7081. static void normalize_task(struct rq *rq, struct task_struct *p)
  7082. {
  7083. const struct sched_class *prev_class = p->sched_class;
  7084. int old_prio = p->prio;
  7085. int on_rq;
  7086. on_rq = p->on_rq;
  7087. if (on_rq)
  7088. deactivate_task(rq, p, 0);
  7089. __setscheduler(rq, p, SCHED_NORMAL, 0);
  7090. if (on_rq) {
  7091. activate_task(rq, p, 0);
  7092. resched_task(rq->curr);
  7093. }
  7094. check_class_changed(rq, p, prev_class, old_prio);
  7095. }
  7096. void normalize_rt_tasks(void)
  7097. {
  7098. struct task_struct *g, *p;
  7099. unsigned long flags;
  7100. struct rq *rq;
  7101. read_lock_irqsave(&tasklist_lock, flags);
  7102. do_each_thread(g, p) {
  7103. /*
  7104. * Only normalize user tasks:
  7105. */
  7106. if (!p->mm)
  7107. continue;
  7108. p->se.exec_start = 0;
  7109. #ifdef CONFIG_SCHEDSTATS
  7110. p->se.statistics.wait_start = 0;
  7111. p->se.statistics.sleep_start = 0;
  7112. p->se.statistics.block_start = 0;
  7113. #endif
  7114. if (!rt_task(p)) {
  7115. /*
  7116. * Renice negative nice level userspace
  7117. * tasks back to 0:
  7118. */
  7119. if (TASK_NICE(p) < 0 && p->mm)
  7120. set_user_nice(p, 0);
  7121. continue;
  7122. }
  7123. raw_spin_lock(&p->pi_lock);
  7124. rq = __task_rq_lock(p);
  7125. normalize_task(rq, p);
  7126. __task_rq_unlock(rq);
  7127. raw_spin_unlock(&p->pi_lock);
  7128. } while_each_thread(g, p);
  7129. read_unlock_irqrestore(&tasklist_lock, flags);
  7130. }
  7131. #endif /* CONFIG_MAGIC_SYSRQ */
  7132. #if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
  7133. /*
  7134. * These functions are only useful for the IA64 MCA handling, or kdb.
  7135. *
  7136. * They can only be called when the whole system has been
  7137. * stopped - every CPU needs to be quiescent, and no scheduling
  7138. * activity can take place. Using them for anything else would
  7139. * be a serious bug, and as a result, they aren't even visible
  7140. * under any other configuration.
  7141. */
  7142. /**
  7143. * curr_task - return the current task for a given cpu.
  7144. * @cpu: the processor in question.
  7145. *
  7146. * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
  7147. */
  7148. struct task_struct *curr_task(int cpu)
  7149. {
  7150. return cpu_curr(cpu);
  7151. }
  7152. #endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
  7153. #ifdef CONFIG_IA64
  7154. /**
  7155. * set_curr_task - set the current task for a given cpu.
  7156. * @cpu: the processor in question.
  7157. * @p: the task pointer to set.
  7158. *
  7159. * Description: This function must only be used when non-maskable interrupts
  7160. * are serviced on a separate stack. It allows the architecture to switch the
  7161. * notion of the current task on a cpu in a non-blocking manner. This function
  7162. * must be called with all CPU's synchronized, and interrupts disabled, the
  7163. * and caller must save the original value of the current task (see
  7164. * curr_task() above) and restore that value before reenabling interrupts and
  7165. * re-starting the system.
  7166. *
  7167. * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
  7168. */
  7169. void set_curr_task(int cpu, struct task_struct *p)
  7170. {
  7171. cpu_curr(cpu) = p;
  7172. }
  7173. #endif
  7174. #ifdef CONFIG_FAIR_GROUP_SCHED
  7175. static void free_fair_sched_group(struct task_group *tg)
  7176. {
  7177. int i;
  7178. destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
  7179. for_each_possible_cpu(i) {
  7180. if (tg->cfs_rq)
  7181. kfree(tg->cfs_rq[i]);
  7182. if (tg->se)
  7183. kfree(tg->se[i]);
  7184. }
  7185. kfree(tg->cfs_rq);
  7186. kfree(tg->se);
  7187. }
  7188. static
  7189. int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
  7190. {
  7191. struct cfs_rq *cfs_rq;
  7192. struct sched_entity *se;
  7193. int i;
  7194. tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
  7195. if (!tg->cfs_rq)
  7196. goto err;
  7197. tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
  7198. if (!tg->se)
  7199. goto err;
  7200. tg->shares = NICE_0_LOAD;
  7201. init_cfs_bandwidth(tg_cfs_bandwidth(tg));
  7202. for_each_possible_cpu(i) {
  7203. cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
  7204. GFP_KERNEL, cpu_to_node(i));
  7205. if (!cfs_rq)
  7206. goto err;
  7207. se = kzalloc_node(sizeof(struct sched_entity),
  7208. GFP_KERNEL, cpu_to_node(i));
  7209. if (!se)
  7210. goto err_free_rq;
  7211. init_cfs_rq(cfs_rq);
  7212. init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
  7213. }
  7214. return 1;
  7215. err_free_rq:
  7216. kfree(cfs_rq);
  7217. err:
  7218. return 0;
  7219. }
  7220. static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
  7221. {
  7222. struct rq *rq = cpu_rq(cpu);
  7223. unsigned long flags;
  7224. /*
  7225. * Only empty task groups can be destroyed; so we can speculatively
  7226. * check on_list without danger of it being re-added.
  7227. */
  7228. if (!tg->cfs_rq[cpu]->on_list)
  7229. return;
  7230. raw_spin_lock_irqsave(&rq->lock, flags);
  7231. list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);
  7232. raw_spin_unlock_irqrestore(&rq->lock, flags);
  7233. }
  7234. #else /* !CONFIG_FAIR_GROUP_SCHED */
  7235. static inline void free_fair_sched_group(struct task_group *tg)
  7236. {
  7237. }
  7238. static inline
  7239. int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
  7240. {
  7241. return 1;
  7242. }
  7243. static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
  7244. {
  7245. }
  7246. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7247. #ifdef CONFIG_RT_GROUP_SCHED
  7248. static void free_rt_sched_group(struct task_group *tg)
  7249. {
  7250. int i;
  7251. if (tg->rt_se)
  7252. destroy_rt_bandwidth(&tg->rt_bandwidth);
  7253. for_each_possible_cpu(i) {
  7254. if (tg->rt_rq)
  7255. kfree(tg->rt_rq[i]);
  7256. if (tg->rt_se)
  7257. kfree(tg->rt_se[i]);
  7258. }
  7259. kfree(tg->rt_rq);
  7260. kfree(tg->rt_se);
  7261. }
  7262. static
  7263. int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
  7264. {
  7265. struct rt_rq *rt_rq;
  7266. struct sched_rt_entity *rt_se;
  7267. int i;
  7268. tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
  7269. if (!tg->rt_rq)
  7270. goto err;
  7271. tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
  7272. if (!tg->rt_se)
  7273. goto err;
  7274. init_rt_bandwidth(&tg->rt_bandwidth,
  7275. ktime_to_ns(def_rt_bandwidth.rt_period), 0);
  7276. for_each_possible_cpu(i) {
  7277. rt_rq = kzalloc_node(sizeof(struct rt_rq),
  7278. GFP_KERNEL, cpu_to_node(i));
  7279. if (!rt_rq)
  7280. goto err;
  7281. rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
  7282. GFP_KERNEL, cpu_to_node(i));
  7283. if (!rt_se)
  7284. goto err_free_rq;
  7285. init_rt_rq(rt_rq, cpu_rq(i));
  7286. rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
  7287. init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
  7288. }
  7289. return 1;
  7290. err_free_rq:
  7291. kfree(rt_rq);
  7292. err:
  7293. return 0;
  7294. }
  7295. #else /* !CONFIG_RT_GROUP_SCHED */
  7296. static inline void free_rt_sched_group(struct task_group *tg)
  7297. {
  7298. }
  7299. static inline
  7300. int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
  7301. {
  7302. return 1;
  7303. }
  7304. #endif /* CONFIG_RT_GROUP_SCHED */
  7305. #ifdef CONFIG_CGROUP_SCHED
  7306. static void free_sched_group(struct task_group *tg)
  7307. {
  7308. free_fair_sched_group(tg);
  7309. free_rt_sched_group(tg);
  7310. autogroup_free(tg);
  7311. kfree(tg);
  7312. }
  7313. /* allocate runqueue etc for a new task group */
  7314. struct task_group *sched_create_group(struct task_group *parent)
  7315. {
  7316. struct task_group *tg;
  7317. unsigned long flags;
  7318. tg = kzalloc(sizeof(*tg), GFP_KERNEL);
  7319. if (!tg)
  7320. return ERR_PTR(-ENOMEM);
  7321. if (!alloc_fair_sched_group(tg, parent))
  7322. goto err;
  7323. if (!alloc_rt_sched_group(tg, parent))
  7324. goto err;
  7325. spin_lock_irqsave(&task_group_lock, flags);
  7326. list_add_rcu(&tg->list, &task_groups);
  7327. WARN_ON(!parent); /* root should already exist */
  7328. tg->parent = parent;
  7329. INIT_LIST_HEAD(&tg->children);
  7330. list_add_rcu(&tg->siblings, &parent->children);
  7331. spin_unlock_irqrestore(&task_group_lock, flags);
  7332. return tg;
  7333. err:
  7334. free_sched_group(tg);
  7335. return ERR_PTR(-ENOMEM);
  7336. }
  7337. /* rcu callback to free various structures associated with a task group */
  7338. static void free_sched_group_rcu(struct rcu_head *rhp)
  7339. {
  7340. /* now it should be safe to free those cfs_rqs */
  7341. free_sched_group(container_of(rhp, struct task_group, rcu));
  7342. }
  7343. /* Destroy runqueue etc associated with a task group */
  7344. void sched_destroy_group(struct task_group *tg)
  7345. {
  7346. unsigned long flags;
  7347. int i;
  7348. /* end participation in shares distribution */
  7349. for_each_possible_cpu(i)
  7350. unregister_fair_sched_group(tg, i);
  7351. spin_lock_irqsave(&task_group_lock, flags);
  7352. list_del_rcu(&tg->list);
  7353. list_del_rcu(&tg->siblings);
  7354. spin_unlock_irqrestore(&task_group_lock, flags);
  7355. /* wait for possible concurrent references to cfs_rqs complete */
  7356. call_rcu(&tg->rcu, free_sched_group_rcu);
  7357. }
  7358. /* change task's runqueue when it moves between groups.
  7359. * The caller of this function should have put the task in its new group
  7360. * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
  7361. * reflect its new group.
  7362. */
  7363. void sched_move_task(struct task_struct *tsk)
  7364. {
  7365. int on_rq, running;
  7366. unsigned long flags;
  7367. struct rq *rq;
  7368. rq = task_rq_lock(tsk, &flags);
  7369. running = task_current(rq, tsk);
  7370. on_rq = tsk->on_rq;
  7371. if (on_rq)
  7372. dequeue_task(rq, tsk, 0);
  7373. if (unlikely(running))
  7374. tsk->sched_class->put_prev_task(rq, tsk);
  7375. #ifdef CONFIG_FAIR_GROUP_SCHED
  7376. if (tsk->sched_class->task_move_group)
  7377. tsk->sched_class->task_move_group(tsk, on_rq);
  7378. else
  7379. #endif
  7380. set_task_rq(tsk, task_cpu(tsk));
  7381. if (unlikely(running))
  7382. tsk->sched_class->set_curr_task(rq);
  7383. if (on_rq)
  7384. enqueue_task(rq, tsk, 0);
  7385. task_rq_unlock(rq, tsk, &flags);
  7386. }
  7387. #endif /* CONFIG_CGROUP_SCHED */
  7388. #ifdef CONFIG_FAIR_GROUP_SCHED
  7389. static DEFINE_MUTEX(shares_mutex);
  7390. int sched_group_set_shares(struct task_group *tg, unsigned long shares)
  7391. {
  7392. int i;
  7393. unsigned long flags;
  7394. /*
  7395. * We can't change the weight of the root cgroup.
  7396. */
  7397. if (!tg->se[0])
  7398. return -EINVAL;
  7399. shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
  7400. mutex_lock(&shares_mutex);
  7401. if (tg->shares == shares)
  7402. goto done;
  7403. tg->shares = shares;
  7404. for_each_possible_cpu(i) {
  7405. struct rq *rq = cpu_rq(i);
  7406. struct sched_entity *se;
  7407. se = tg->se[i];
  7408. /* Propagate contribution to hierarchy */
  7409. raw_spin_lock_irqsave(&rq->lock, flags);
  7410. for_each_sched_entity(se)
  7411. update_cfs_shares(group_cfs_rq(se));
  7412. raw_spin_unlock_irqrestore(&rq->lock, flags);
  7413. }
  7414. done:
  7415. mutex_unlock(&shares_mutex);
  7416. return 0;
  7417. }
  7418. unsigned long sched_group_shares(struct task_group *tg)
  7419. {
  7420. return tg->shares;
  7421. }
  7422. #endif
  7423. #if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_CFS_BANDWIDTH)
  7424. static unsigned long to_ratio(u64 period, u64 runtime)
  7425. {
  7426. if (runtime == RUNTIME_INF)
  7427. return 1ULL << 20;
  7428. return div64_u64(runtime << 20, period);
  7429. }
  7430. #endif
  7431. #ifdef CONFIG_RT_GROUP_SCHED
  7432. /*
  7433. * Ensure that the real time constraints are schedulable.
  7434. */
  7435. static DEFINE_MUTEX(rt_constraints_mutex);
  7436. /* Must be called with tasklist_lock held */
  7437. static inline int tg_has_rt_tasks(struct task_group *tg)
  7438. {
  7439. struct task_struct *g, *p;
  7440. do_each_thread(g, p) {
  7441. if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg)
  7442. return 1;
  7443. } while_each_thread(g, p);
  7444. return 0;
  7445. }
  7446. struct rt_schedulable_data {
  7447. struct task_group *tg;
  7448. u64 rt_period;
  7449. u64 rt_runtime;
  7450. };
  7451. static int tg_rt_schedulable(struct task_group *tg, void *data)
  7452. {
  7453. struct rt_schedulable_data *d = data;
  7454. struct task_group *child;
  7455. unsigned long total, sum = 0;
  7456. u64 period, runtime;
  7457. period = ktime_to_ns(tg->rt_bandwidth.rt_period);
  7458. runtime = tg->rt_bandwidth.rt_runtime;
  7459. if (tg == d->tg) {
  7460. period = d->rt_period;
  7461. runtime = d->rt_runtime;
  7462. }
  7463. /*
  7464. * Cannot have more runtime than the period.
  7465. */
  7466. if (runtime > period && runtime != RUNTIME_INF)
  7467. return -EINVAL;
  7468. /*
  7469. * Ensure we don't starve existing RT tasks.
  7470. */
  7471. if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
  7472. return -EBUSY;
  7473. total = to_ratio(period, runtime);
  7474. /*
  7475. * Nobody can have more than the global setting allows.
  7476. */
  7477. if (total > to_ratio(global_rt_period(), global_rt_runtime()))
  7478. return -EINVAL;
  7479. /*
  7480. * The sum of our children's runtime should not exceed our own.
  7481. */
  7482. list_for_each_entry_rcu(child, &tg->children, siblings) {
  7483. period = ktime_to_ns(child->rt_bandwidth.rt_period);
  7484. runtime = child->rt_bandwidth.rt_runtime;
  7485. if (child == d->tg) {
  7486. period = d->rt_period;
  7487. runtime = d->rt_runtime;
  7488. }
  7489. sum += to_ratio(period, runtime);
  7490. }
  7491. if (sum > total)
  7492. return -EINVAL;
  7493. return 0;
  7494. }
  7495. static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
  7496. {
  7497. int ret;
  7498. struct rt_schedulable_data data = {
  7499. .tg = tg,
  7500. .rt_period = period,
  7501. .rt_runtime = runtime,
  7502. };
  7503. rcu_read_lock();
  7504. ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
  7505. rcu_read_unlock();
  7506. return ret;
  7507. }
  7508. static int tg_set_rt_bandwidth(struct task_group *tg,
  7509. u64 rt_period, u64 rt_runtime)
  7510. {
  7511. int i, err = 0;
  7512. mutex_lock(&rt_constraints_mutex);
  7513. read_lock(&tasklist_lock);
  7514. err = __rt_schedulable(tg, rt_period, rt_runtime);
  7515. if (err)
  7516. goto unlock;
  7517. raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  7518. tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
  7519. tg->rt_bandwidth.rt_runtime = rt_runtime;
  7520. for_each_possible_cpu(i) {
  7521. struct rt_rq *rt_rq = tg->rt_rq[i];
  7522. raw_spin_lock(&rt_rq->rt_runtime_lock);
  7523. rt_rq->rt_runtime = rt_runtime;
  7524. raw_spin_unlock(&rt_rq->rt_runtime_lock);
  7525. }
  7526. raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  7527. unlock:
  7528. read_unlock(&tasklist_lock);
  7529. mutex_unlock(&rt_constraints_mutex);
  7530. return err;
  7531. }
  7532. int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
  7533. {
  7534. u64 rt_runtime, rt_period;
  7535. rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
  7536. rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
  7537. if (rt_runtime_us < 0)
  7538. rt_runtime = RUNTIME_INF;
  7539. return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
  7540. }
  7541. long sched_group_rt_runtime(struct task_group *tg)
  7542. {
  7543. u64 rt_runtime_us;
  7544. if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
  7545. return -1;
  7546. rt_runtime_us = tg->rt_bandwidth.rt_runtime;
  7547. do_div(rt_runtime_us, NSEC_PER_USEC);
  7548. return rt_runtime_us;
  7549. }
  7550. int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
  7551. {
  7552. u64 rt_runtime, rt_period;
  7553. rt_period = (u64)rt_period_us * NSEC_PER_USEC;
  7554. rt_runtime = tg->rt_bandwidth.rt_runtime;
  7555. if (rt_period == 0)
  7556. return -EINVAL;
  7557. return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
  7558. }
  7559. long sched_group_rt_period(struct task_group *tg)
  7560. {
  7561. u64 rt_period_us;
  7562. rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
  7563. do_div(rt_period_us, NSEC_PER_USEC);
  7564. return rt_period_us;
  7565. }
  7566. static int sched_rt_global_constraints(void)
  7567. {
  7568. u64 runtime, period;
  7569. int ret = 0;
  7570. if (sysctl_sched_rt_period <= 0)
  7571. return -EINVAL;
  7572. runtime = global_rt_runtime();
  7573. period = global_rt_period();
  7574. /*
  7575. * Sanity check on the sysctl variables.
  7576. */
  7577. if (runtime > period && runtime != RUNTIME_INF)
  7578. return -EINVAL;
  7579. mutex_lock(&rt_constraints_mutex);
  7580. read_lock(&tasklist_lock);
  7581. ret = __rt_schedulable(NULL, 0, 0);
  7582. read_unlock(&tasklist_lock);
  7583. mutex_unlock(&rt_constraints_mutex);
  7584. return ret;
  7585. }
  7586. int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
  7587. {
  7588. /* Don't accept realtime tasks when there is no way for them to run */
  7589. if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
  7590. return 0;
  7591. return 1;
  7592. }
  7593. #else /* !CONFIG_RT_GROUP_SCHED */
  7594. static int sched_rt_global_constraints(void)
  7595. {
  7596. unsigned long flags;
  7597. int i;
  7598. if (sysctl_sched_rt_period <= 0)
  7599. return -EINVAL;
  7600. /*
  7601. * There's always some RT tasks in the root group
  7602. * -- migration, kstopmachine etc..
  7603. */
  7604. if (sysctl_sched_rt_runtime == 0)
  7605. return -EBUSY;
  7606. raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
  7607. for_each_possible_cpu(i) {
  7608. struct rt_rq *rt_rq = &cpu_rq(i)->rt;
  7609. raw_spin_lock(&rt_rq->rt_runtime_lock);
  7610. rt_rq->rt_runtime = global_rt_runtime();
  7611. raw_spin_unlock(&rt_rq->rt_runtime_lock);
  7612. }
  7613. raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
  7614. return 0;
  7615. }
  7616. #endif /* CONFIG_RT_GROUP_SCHED */
  7617. int sched_rt_handler(struct ctl_table *table, int write,
  7618. void __user *buffer, size_t *lenp,
  7619. loff_t *ppos)
  7620. {
  7621. int ret;
  7622. int old_period, old_runtime;
  7623. static DEFINE_MUTEX(mutex);
  7624. mutex_lock(&mutex);
  7625. old_period = sysctl_sched_rt_period;
  7626. old_runtime = sysctl_sched_rt_runtime;
  7627. ret = proc_dointvec(table, write, buffer, lenp, ppos);
  7628. if (!ret && write) {
  7629. ret = sched_rt_global_constraints();
  7630. if (ret) {
  7631. sysctl_sched_rt_period = old_period;
  7632. sysctl_sched_rt_runtime = old_runtime;
  7633. } else {
  7634. def_rt_bandwidth.rt_runtime = global_rt_runtime();
  7635. def_rt_bandwidth.rt_period =
  7636. ns_to_ktime(global_rt_period());
  7637. }
  7638. }
  7639. mutex_unlock(&mutex);
  7640. return ret;
  7641. }
  7642. #ifdef CONFIG_CGROUP_SCHED
  7643. /* return corresponding task_group object of a cgroup */
  7644. static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
  7645. {
  7646. return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
  7647. struct task_group, css);
  7648. }
  7649. static struct cgroup_subsys_state *
  7650. cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
  7651. {
  7652. struct task_group *tg, *parent;
  7653. if (!cgrp->parent) {
  7654. /* This is early initialization for the top cgroup */
  7655. return &root_task_group.css;
  7656. }
  7657. parent = cgroup_tg(cgrp->parent);
  7658. tg = sched_create_group(parent);
  7659. if (IS_ERR(tg))
  7660. return ERR_PTR(-ENOMEM);
  7661. return &tg->css;
  7662. }
  7663. static void
  7664. cpu_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
  7665. {
  7666. struct task_group *tg = cgroup_tg(cgrp);
  7667. sched_destroy_group(tg);
  7668. }
  7669. static int
  7670. cpu_cgroup_can_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
  7671. {
  7672. #ifdef CONFIG_RT_GROUP_SCHED
  7673. if (!sched_rt_can_attach(cgroup_tg(cgrp), tsk))
  7674. return -EINVAL;
  7675. #else
  7676. /* We don't support RT-tasks being in separate groups */
  7677. if (tsk->sched_class != &fair_sched_class)
  7678. return -EINVAL;
  7679. #endif
  7680. return 0;
  7681. }
  7682. static void
  7683. cpu_cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
  7684. {
  7685. sched_move_task(tsk);
  7686. }
  7687. static void
  7688. cpu_cgroup_exit(struct cgroup_subsys *ss, struct cgroup *cgrp,
  7689. struct cgroup *old_cgrp, struct task_struct *task)
  7690. {
  7691. /*
  7692. * cgroup_exit() is called in the copy_process() failure path.
  7693. * Ignore this case since the task hasn't ran yet, this avoids
  7694. * trying to poke a half freed task state from generic code.
  7695. */
  7696. if (!(task->flags & PF_EXITING))
  7697. return;
  7698. sched_move_task(task);
  7699. }
  7700. #ifdef CONFIG_FAIR_GROUP_SCHED
  7701. static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
  7702. u64 shareval)
  7703. {
  7704. return sched_group_set_shares(cgroup_tg(cgrp), scale_load(shareval));
  7705. }
  7706. static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
  7707. {
  7708. struct task_group *tg = cgroup_tg(cgrp);
  7709. return (u64) scale_load_down(tg->shares);
  7710. }
  7711. #ifdef CONFIG_CFS_BANDWIDTH
  7712. static DEFINE_MUTEX(cfs_constraints_mutex);
  7713. const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
  7714. const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
  7715. static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
  7716. static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
  7717. {
  7718. int i, ret = 0, runtime_enabled;
  7719. struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
  7720. if (tg == &root_task_group)
  7721. return -EINVAL;
  7722. /*
  7723. * Ensure we have at some amount of bandwidth every period. This is
  7724. * to prevent reaching a state of large arrears when throttled via
  7725. * entity_tick() resulting in prolonged exit starvation.
  7726. */
  7727. if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
  7728. return -EINVAL;
  7729. /*
  7730. * Likewise, bound things on the otherside by preventing insane quota
  7731. * periods. This also allows us to normalize in computing quota
  7732. * feasibility.
  7733. */
  7734. if (period > max_cfs_quota_period)
  7735. return -EINVAL;
  7736. mutex_lock(&cfs_constraints_mutex);
  7737. ret = __cfs_schedulable(tg, period, quota);
  7738. if (ret)
  7739. goto out_unlock;
  7740. runtime_enabled = quota != RUNTIME_INF;
  7741. raw_spin_lock_irq(&cfs_b->lock);
  7742. cfs_b->period = ns_to_ktime(period);
  7743. cfs_b->quota = quota;
  7744. __refill_cfs_bandwidth_runtime(cfs_b);
  7745. /* restart the period timer (if active) to handle new period expiry */
  7746. if (runtime_enabled && cfs_b->timer_active) {
  7747. /* force a reprogram */
  7748. cfs_b->timer_active = 0;
  7749. __start_cfs_bandwidth(cfs_b);
  7750. }
  7751. raw_spin_unlock_irq(&cfs_b->lock);
  7752. for_each_possible_cpu(i) {
  7753. struct cfs_rq *cfs_rq = tg->cfs_rq[i];
  7754. struct rq *rq = rq_of(cfs_rq);
  7755. raw_spin_lock_irq(&rq->lock);
  7756. cfs_rq->runtime_enabled = runtime_enabled;
  7757. cfs_rq->runtime_remaining = 0;
  7758. if (cfs_rq_throttled(cfs_rq))
  7759. unthrottle_cfs_rq(cfs_rq);
  7760. raw_spin_unlock_irq(&rq->lock);
  7761. }
  7762. out_unlock:
  7763. mutex_unlock(&cfs_constraints_mutex);
  7764. return ret;
  7765. }
  7766. int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
  7767. {
  7768. u64 quota, period;
  7769. period = ktime_to_ns(tg_cfs_bandwidth(tg)->period);
  7770. if (cfs_quota_us < 0)
  7771. quota = RUNTIME_INF;
  7772. else
  7773. quota = (u64)cfs_quota_us * NSEC_PER_USEC;
  7774. return tg_set_cfs_bandwidth(tg, period, quota);
  7775. }
  7776. long tg_get_cfs_quota(struct task_group *tg)
  7777. {
  7778. u64 quota_us;
  7779. if (tg_cfs_bandwidth(tg)->quota == RUNTIME_INF)
  7780. return -1;
  7781. quota_us = tg_cfs_bandwidth(tg)->quota;
  7782. do_div(quota_us, NSEC_PER_USEC);
  7783. return quota_us;
  7784. }
  7785. int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
  7786. {
  7787. u64 quota, period;
  7788. period = (u64)cfs_period_us * NSEC_PER_USEC;
  7789. quota = tg_cfs_bandwidth(tg)->quota;
  7790. if (period <= 0)
  7791. return -EINVAL;
  7792. return tg_set_cfs_bandwidth(tg, period, quota);
  7793. }
  7794. long tg_get_cfs_period(struct task_group *tg)
  7795. {
  7796. u64 cfs_period_us;
  7797. cfs_period_us = ktime_to_ns(tg_cfs_bandwidth(tg)->period);
  7798. do_div(cfs_period_us, NSEC_PER_USEC);
  7799. return cfs_period_us;
  7800. }
  7801. static s64 cpu_cfs_quota_read_s64(struct cgroup *cgrp, struct cftype *cft)
  7802. {
  7803. return tg_get_cfs_quota(cgroup_tg(cgrp));
  7804. }
  7805. static int cpu_cfs_quota_write_s64(struct cgroup *cgrp, struct cftype *cftype,
  7806. s64 cfs_quota_us)
  7807. {
  7808. return tg_set_cfs_quota(cgroup_tg(cgrp), cfs_quota_us);
  7809. }
  7810. static u64 cpu_cfs_period_read_u64(struct cgroup *cgrp, struct cftype *cft)
  7811. {
  7812. return tg_get_cfs_period(cgroup_tg(cgrp));
  7813. }
  7814. static int cpu_cfs_period_write_u64(struct cgroup *cgrp, struct cftype *cftype,
  7815. u64 cfs_period_us)
  7816. {
  7817. return tg_set_cfs_period(cgroup_tg(cgrp), cfs_period_us);
  7818. }
  7819. struct cfs_schedulable_data {
  7820. struct task_group *tg;
  7821. u64 period, quota;
  7822. };
  7823. /*
  7824. * normalize group quota/period to be quota/max_period
  7825. * note: units are usecs
  7826. */
  7827. static u64 normalize_cfs_quota(struct task_group *tg,
  7828. struct cfs_schedulable_data *d)
  7829. {
  7830. u64 quota, period;
  7831. if (tg == d->tg) {
  7832. period = d->period;
  7833. quota = d->quota;
  7834. } else {
  7835. period = tg_get_cfs_period(tg);
  7836. quota = tg_get_cfs_quota(tg);
  7837. }
  7838. /* note: these should typically be equivalent */
  7839. if (quota == RUNTIME_INF || quota == -1)
  7840. return RUNTIME_INF;
  7841. return to_ratio(period, quota);
  7842. }
  7843. static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
  7844. {
  7845. struct cfs_schedulable_data *d = data;
  7846. struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
  7847. s64 quota = 0, parent_quota = -1;
  7848. if (!tg->parent) {
  7849. quota = RUNTIME_INF;
  7850. } else {
  7851. struct cfs_bandwidth *parent_b = tg_cfs_bandwidth(tg->parent);
  7852. quota = normalize_cfs_quota(tg, d);
  7853. parent_quota = parent_b->hierarchal_quota;
  7854. /*
  7855. * ensure max(child_quota) <= parent_quota, inherit when no
  7856. * limit is set
  7857. */
  7858. if (quota == RUNTIME_INF)
  7859. quota = parent_quota;
  7860. else if (parent_quota != RUNTIME_INF && quota > parent_quota)
  7861. return -EINVAL;
  7862. }
  7863. cfs_b->hierarchal_quota = quota;
  7864. return 0;
  7865. }
  7866. static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
  7867. {
  7868. int ret;
  7869. struct cfs_schedulable_data data = {
  7870. .tg = tg,
  7871. .period = period,
  7872. .quota = quota,
  7873. };
  7874. if (quota != RUNTIME_INF) {
  7875. do_div(data.period, NSEC_PER_USEC);
  7876. do_div(data.quota, NSEC_PER_USEC);
  7877. }
  7878. rcu_read_lock();
  7879. ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
  7880. rcu_read_unlock();
  7881. return ret;
  7882. }
  7883. static int cpu_stats_show(struct cgroup *cgrp, struct cftype *cft,
  7884. struct cgroup_map_cb *cb)
  7885. {
  7886. struct task_group *tg = cgroup_tg(cgrp);
  7887. struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
  7888. cb->fill(cb, "nr_periods", cfs_b->nr_periods);
  7889. cb->fill(cb, "nr_throttled", cfs_b->nr_throttled);
  7890. cb->fill(cb, "throttled_time", cfs_b->throttled_time);
  7891. return 0;
  7892. }
  7893. #endif /* CONFIG_CFS_BANDWIDTH */
  7894. #endif /* CONFIG_FAIR_GROUP_SCHED */
  7895. #ifdef CONFIG_RT_GROUP_SCHED
  7896. static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
  7897. s64 val)
  7898. {
  7899. return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
  7900. }
  7901. static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
  7902. {
  7903. return sched_group_rt_runtime(cgroup_tg(cgrp));
  7904. }
  7905. static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
  7906. u64 rt_period_us)
  7907. {
  7908. return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
  7909. }
  7910. static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
  7911. {
  7912. return sched_group_rt_period(cgroup_tg(cgrp));
  7913. }
  7914. #endif /* CONFIG_RT_GROUP_SCHED */
  7915. static struct cftype cpu_files[] = {
  7916. #ifdef CONFIG_FAIR_GROUP_SCHED
  7917. {
  7918. .name = "shares",
  7919. .read_u64 = cpu_shares_read_u64,
  7920. .write_u64 = cpu_shares_write_u64,
  7921. },
  7922. #endif
  7923. #ifdef CONFIG_CFS_BANDWIDTH
  7924. {
  7925. .name = "cfs_quota_us",
  7926. .read_s64 = cpu_cfs_quota_read_s64,
  7927. .write_s64 = cpu_cfs_quota_write_s64,
  7928. },
  7929. {
  7930. .name = "cfs_period_us",
  7931. .read_u64 = cpu_cfs_period_read_u64,
  7932. .write_u64 = cpu_cfs_period_write_u64,
  7933. },
  7934. {
  7935. .name = "stat",
  7936. .read_map = cpu_stats_show,
  7937. },
  7938. #endif
  7939. #ifdef CONFIG_RT_GROUP_SCHED
  7940. {
  7941. .name = "rt_runtime_us",
  7942. .read_s64 = cpu_rt_runtime_read,
  7943. .write_s64 = cpu_rt_runtime_write,
  7944. },
  7945. {
  7946. .name = "rt_period_us",
  7947. .read_u64 = cpu_rt_period_read_uint,
  7948. .write_u64 = cpu_rt_period_write_uint,
  7949. },
  7950. #endif
  7951. };
  7952. static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
  7953. {
  7954. return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files));
  7955. }
  7956. struct cgroup_subsys cpu_cgroup_subsys = {
  7957. .name = "cpu",
  7958. .create = cpu_cgroup_create,
  7959. .destroy = cpu_cgroup_destroy,
  7960. .can_attach_task = cpu_cgroup_can_attach_task,
  7961. .attach_task = cpu_cgroup_attach_task,
  7962. .exit = cpu_cgroup_exit,
  7963. .populate = cpu_cgroup_populate,
  7964. .subsys_id = cpu_cgroup_subsys_id,
  7965. .early_init = 1,
  7966. };
  7967. #endif /* CONFIG_CGROUP_SCHED */
  7968. #ifdef CONFIG_CGROUP_CPUACCT
  7969. /*
  7970. * CPU accounting code for task groups.
  7971. *
  7972. * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
  7973. * (balbir@in.ibm.com).
  7974. */
  7975. /* track cpu usage of a group of tasks and its child groups */
  7976. struct cpuacct {
  7977. struct cgroup_subsys_state css;
  7978. /* cpuusage holds pointer to a u64-type object on every cpu */
  7979. u64 __percpu *cpuusage;
  7980. struct percpu_counter cpustat[CPUACCT_STAT_NSTATS];
  7981. struct cpuacct *parent;
  7982. };
  7983. struct cgroup_subsys cpuacct_subsys;
  7984. /* return cpu accounting group corresponding to this container */
  7985. static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
  7986. {
  7987. return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
  7988. struct cpuacct, css);
  7989. }
  7990. /* return cpu accounting group to which this task belongs */
  7991. static inline struct cpuacct *task_ca(struct task_struct *tsk)
  7992. {
  7993. return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
  7994. struct cpuacct, css);
  7995. }
  7996. /* create a new cpu accounting group */
  7997. static struct cgroup_subsys_state *cpuacct_create(
  7998. struct cgroup_subsys *ss, struct cgroup *cgrp)
  7999. {
  8000. struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
  8001. int i;
  8002. if (!ca)
  8003. goto out;
  8004. ca->cpuusage = alloc_percpu(u64);
  8005. if (!ca->cpuusage)
  8006. goto out_free_ca;
  8007. for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
  8008. if (percpu_counter_init(&ca->cpustat[i], 0))
  8009. goto out_free_counters;
  8010. if (cgrp->parent)
  8011. ca->parent = cgroup_ca(cgrp->parent);
  8012. return &ca->css;
  8013. out_free_counters:
  8014. while (--i >= 0)
  8015. percpu_counter_destroy(&ca->cpustat[i]);
  8016. free_percpu(ca->cpuusage);
  8017. out_free_ca:
  8018. kfree(ca);
  8019. out:
  8020. return ERR_PTR(-ENOMEM);
  8021. }
  8022. /* destroy an existing cpu accounting group */
  8023. static void
  8024. cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
  8025. {
  8026. struct cpuacct *ca = cgroup_ca(cgrp);
  8027. int i;
  8028. for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
  8029. percpu_counter_destroy(&ca->cpustat[i]);
  8030. free_percpu(ca->cpuusage);
  8031. kfree(ca);
  8032. }
  8033. static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
  8034. {
  8035. u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
  8036. u64 data;
  8037. #ifndef CONFIG_64BIT
  8038. /*
  8039. * Take rq->lock to make 64-bit read safe on 32-bit platforms.
  8040. */
  8041. raw_spin_lock_irq(&cpu_rq(cpu)->lock);
  8042. data = *cpuusage;
  8043. raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
  8044. #else
  8045. data = *cpuusage;
  8046. #endif
  8047. return data;
  8048. }
  8049. static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
  8050. {
  8051. u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
  8052. #ifndef CONFIG_64BIT
  8053. /*
  8054. * Take rq->lock to make 64-bit write safe on 32-bit platforms.
  8055. */
  8056. raw_spin_lock_irq(&cpu_rq(cpu)->lock);
  8057. *cpuusage = val;
  8058. raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
  8059. #else
  8060. *cpuusage = val;
  8061. #endif
  8062. }
  8063. /* return total cpu usage (in nanoseconds) of a group */
  8064. static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
  8065. {
  8066. struct cpuacct *ca = cgroup_ca(cgrp);
  8067. u64 totalcpuusage = 0;
  8068. int i;
  8069. for_each_present_cpu(i)
  8070. totalcpuusage += cpuacct_cpuusage_read(ca, i);
  8071. return totalcpuusage;
  8072. }
  8073. static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
  8074. u64 reset)
  8075. {
  8076. struct cpuacct *ca = cgroup_ca(cgrp);
  8077. int err = 0;
  8078. int i;
  8079. if (reset) {
  8080. err = -EINVAL;
  8081. goto out;
  8082. }
  8083. for_each_present_cpu(i)
  8084. cpuacct_cpuusage_write(ca, i, 0);
  8085. out:
  8086. return err;
  8087. }
  8088. static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
  8089. struct seq_file *m)
  8090. {
  8091. struct cpuacct *ca = cgroup_ca(cgroup);
  8092. u64 percpu;
  8093. int i;
  8094. for_each_present_cpu(i) {
  8095. percpu = cpuacct_cpuusage_read(ca, i);
  8096. seq_printf(m, "%llu ", (unsigned long long) percpu);
  8097. }
  8098. seq_printf(m, "\n");
  8099. return 0;
  8100. }
  8101. static const char *cpuacct_stat_desc[] = {
  8102. [CPUACCT_STAT_USER] = "user",
  8103. [CPUACCT_STAT_SYSTEM] = "system",
  8104. };
  8105. static int cpuacct_stats_show(struct cgroup *cgrp, struct cftype *cft,
  8106. struct cgroup_map_cb *cb)
  8107. {
  8108. struct cpuacct *ca = cgroup_ca(cgrp);
  8109. int i;
  8110. for (i = 0; i < CPUACCT_STAT_NSTATS; i++) {
  8111. s64 val = percpu_counter_read(&ca->cpustat[i]);
  8112. val = cputime64_to_clock_t(val);
  8113. cb->fill(cb, cpuacct_stat_desc[i], val);
  8114. }
  8115. return 0;
  8116. }
  8117. static struct cftype files[] = {
  8118. {
  8119. .name = "usage",
  8120. .read_u64 = cpuusage_read,
  8121. .write_u64 = cpuusage_write,
  8122. },
  8123. {
  8124. .name = "usage_percpu",
  8125. .read_seq_string = cpuacct_percpu_seq_read,
  8126. },
  8127. {
  8128. .name = "stat",
  8129. .read_map = cpuacct_stats_show,
  8130. },
  8131. };
  8132. static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
  8133. {
  8134. return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
  8135. }
  8136. /*
  8137. * charge this task's execution time to its accounting group.
  8138. *
  8139. * called with rq->lock held.
  8140. */
  8141. static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
  8142. {
  8143. struct cpuacct *ca;
  8144. int cpu;
  8145. if (unlikely(!cpuacct_subsys.active))
  8146. return;
  8147. cpu = task_cpu(tsk);
  8148. rcu_read_lock();
  8149. ca = task_ca(tsk);
  8150. for (; ca; ca = ca->parent) {
  8151. u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
  8152. *cpuusage += cputime;
  8153. }
  8154. rcu_read_unlock();
  8155. }
  8156. /*
  8157. * When CONFIG_VIRT_CPU_ACCOUNTING is enabled one jiffy can be very large
  8158. * in cputime_t units. As a result, cpuacct_update_stats calls
  8159. * percpu_counter_add with values large enough to always overflow the
  8160. * per cpu batch limit causing bad SMP scalability.
  8161. *
  8162. * To fix this we scale percpu_counter_batch by cputime_one_jiffy so we
  8163. * batch the same amount of time with CONFIG_VIRT_CPU_ACCOUNTING disabled
  8164. * and enabled. We cap it at INT_MAX which is the largest allowed batch value.
  8165. */
  8166. #ifdef CONFIG_SMP
  8167. #define CPUACCT_BATCH \
  8168. min_t(long, percpu_counter_batch * cputime_one_jiffy, INT_MAX)
  8169. #else
  8170. #define CPUACCT_BATCH 0
  8171. #endif
  8172. /*
  8173. * Charge the system/user time to the task's accounting group.
  8174. */
  8175. static void cpuacct_update_stats(struct task_struct *tsk,
  8176. enum cpuacct_stat_index idx, cputime_t val)
  8177. {
  8178. struct cpuacct *ca;
  8179. int batch = CPUACCT_BATCH;
  8180. if (unlikely(!cpuacct_subsys.active))
  8181. return;
  8182. rcu_read_lock();
  8183. ca = task_ca(tsk);
  8184. do {
  8185. __percpu_counter_add(&ca->cpustat[idx], val, batch);
  8186. ca = ca->parent;
  8187. } while (ca);
  8188. rcu_read_unlock();
  8189. }
  8190. struct cgroup_subsys cpuacct_subsys = {
  8191. .name = "cpuacct",
  8192. .create = cpuacct_create,
  8193. .destroy = cpuacct_destroy,
  8194. .populate = cpuacct_populate,
  8195. .subsys_id = cpuacct_subsys_id,
  8196. };
  8197. #endif /* CONFIG_CGROUP_CPUACCT */