'generator' content='cgit v1.2.2'/>
aboutsummaryrefslogblamecommitdiffstats
path: root/kernel/sched.c
blob: 526618fe4a784e15450a041a8cb76399ebe8386a (plain) (tree)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

















                                                                          





                                                                       

                                                                             





                         
                          



                            
                             

                              
                              


                           
                          
                          

                         
                                







                           
                          
                           
                         

                           
                              
                          
                            
                                 
                         
                          
                          
                       
                          

                          
                         
                        
 
                    
                         
 

                         


















                                                                  
                                                               
   
                                                                             
 


                                                


                                                 
                                                                 

                                             
                                                 
 




                                                                  





                                 




















                                                                          

                                       
                                                                 








                                                           
  
                                                                        
   




                                                                                
                     




                                                
































                                                                            

                                               


                                                               
                                                                    

 


                                            





                                                         
                                                                      











                                                                              

                                                             










                                                           





                                                                   
                         
 

                         

              

                              
                                    
                   
                          

                                       

                              




                                                            





                                       
                                         
      
 
                            
                              



                                  

  
                        







                                                                       
                              



                                                                               
                                    



                                                                             
                                  
                              
                                       
                              
 
                                                                           

                             
                                        
 
                              

                                               
                              
                                           
                              
 
  



                                                                   


                                                        
                         
                                   
 


                                                       
                      
                                                            
   
                                  

                                          
                                                                  
 
                              
 
                        
                         
                                  

                                                                     
     
                              
      
                  


                                                                             
                                                                       
 
                              

                                                  
      
 
                            

                                                 
      



     
                                                                           



                                                                  
 
                                
 




                                      
                       
                         


                                      





                                                                    

                                                                             
                                                
 
                                    
 
                              

                                                                           

                                                                               





                                                                              

                                                                     

                 
          
                                                       
           
                                  
 






                                                                   
 



                                        




                                                
      

      
 


                                                     
                                    
                                                       

                                                           
                 
                                      
                       
      
                         
                    
                       
                                       
                                   
 
                            

                                    




                                         

  



                                                                             

                                                                           


                                                                           


                          

                             
 
          


                                                                     
                               
                           


                             

  



                                                                      



                                          
  





                                                                  
           

                            





                                                                          

                                                 
                                   
                   
                                     

                                       

                                                        



                                      
                        
 
                              

                                              

                            
                                         
      








                                                                     
                                        
                                   
                                  
 
                  
 


                           
                               




                                  

                                   
                   
 
                                        
 
                                             


                                         
                          



                                           


                                    




                                        



                                    

                              


                                  

                                    

                                

                       
                               


      
                                                           
 
                                                                                     
 
                                                               

 








                                       
  
                                                                             
                                                             



                                                              

                                                                                





                                                            




                                                 
  







                                                                 

















                                                                   


                                  



                                         
      
                           

  




                                                
                                                








                                         
                                                 





                           
                                                       
 


                                               


                                                          
         
                          
 
                 


















                                                            
                                          























                                                                     




                                                                  
                                                 




                                           



                                        









                                                                         

  





                                                      
                                           
                  
   
                                                    

  






                                                               
                                                         

              
                                              
 

                                           




                                                          
 






                                                           
                                        



                                                            
 
                           





                                                        




                                                                    
                                  
                                                                    
 
                                   

 
                                                                               


 
                                                                              
 



                                                                          






                                                                    



                                      
                                                                    



                        
                                   


      
                                                                               















                                                                      
                                                                              











                                                                              
      

                                       

  


                                                              
                                                              

                            




                                             
                                       
         


  
                                                                       
                                                                          

                                   
                                                                           

                            
                      
 





                                             
                                                          
         

 







                                                                     
                                           




                               
                                                                      




                                                  
  
                                                            
   
                                    

                            
                      







                             










                                                                            









                                               
                                    
                         


                                                         






                                                  









                                                                     
                            





                                                          
                 



                                     
 
                            
 



                                           

 





                                                  
 

                                                                    
 
                                         






                                                                        













                                                                            
                                          





                                 
                                    


                                           









                                                                               
 
                                    
 
 
                       
 
                                         
 

                                   
 



                                             
 

                                                                           
                                                             
 
                                 



                                              



                                                


                                    
                                 
 
  











                                                                     
                                               




                                              
                                                                

                       
                                                 




















                                                                 








































                                                                      
                         
 
                       
                                               

                                              
                                
 
                       
 







                                   


                         
                                                     
 


                       
                    




                                                               






                                                                              




                                                                 
                                        
                                                                   

                                       
                                                             
 
                                                                     

 
                                                                             

                          
                           

 
                                                                             

                          
                           

 
  


                                                                             
                                                                           



                                                                            










                                                                      


                                                                     

                                       







                                                                 

  






                                                                        
                                      







                                                                 
  
 












                                                                            










                                                                        
      
 





                                                                          









                                                                  
                                                                                               
                                                       




                                                                               
                                                                   

                                          
                



                                  


                                    






                                                                     


                                  




                                
           
                          

                   

 


                                                    
 












                                                                             

                                          




                                     
 





                                                                           

                                                                            
 


                                
                         

                       
                                               
 





                                            
                                                        
                                                                        
 



                                                     
 
                                                    
                                                 
 


                                                         
 

  


                                                                          
   
                                                          
 
                                            
                                 
                                       
              
 
                                                










                                                                           
                                                
         
 




                                                                  
 
                                              
                                                                  

                 


  


                                                                          
   
                                                          
 
                           
                              
 






                                                                 
 
                                       
 
                 

 
                                                  
 




                                                                 
                                                       
         

 






                                                                        
                                   
 
                                                        

 

     
                                                         


 



                                                                               

      

      
                              

                                                                          
                 


                                
      
 
                        
                           

                       




                                             

                                                                
 
                                         

                         

 
                                         

                         

 


                                                  



                                                              
 







                                                       
 

                                                                            

 





                                            
                                                                          
 
                             
                                                    
                        

 
                                                                         
 





                                                                       
                               
                                                   
                        


  
                                                                       
   

                                                      
                              


  





                                                         
                                                    


                 
                                  












                                                        
                                                












                                                                  
                                               
   
                                                                           
 
                                        
                                         
 
                                    
                           


  

                                                     
                                                                            
 
                                        

                                         
                                   
                           

 



                                                         
                                                 



                                          

                                                                          
                            
                 





                                                                              
                                       
      

 











                                                                            
                 
 





                                                            


                                 
          



                                                                 


                                          


                                                             

                         


                                                




                                              





                                                        
                                                              
 

                                                                       

                                                                  
                         

                                                     



                                                 



                                                  




                                                                   
      

                                                                 

                                   

 
                      
                              
 
                                 

                     
                               
  




                                                         
          
                                                                            
 
                                   




                                                                   
                                                   




                                          


                                                   
 





                                                        






                                                                           





                                                                        
                                                                         

                            
                           
                           
                      
 







                                                                     
 










                                                                   


                                                                             
                                    
                 
 





                                                                
                                             

                                              
                         
                                                            
                                                                  
                                           
 
                  





                                                                       








                                                               
 












                                                                    
 






                                                                  

                    














                                                                
                                        










                                                        

                                                                    



                                                              
                                                   
 
                                    
                                                    
 
                                              
                             
 
                                                


  

                                                                     
   
                                                   
 
                                    
                                                    
 
                                              
                             
 
                                                


  















                                                                               
                                                                    

                                                                
                                 
 

                                                                        



                                                                
                                                          









                                                                      

                                                             







                                                 
                                                           






                                                          
                                                                 
   
          
                                                                               




                                                 
                                            
                                                                        
                                           









                                                                             














                                                                            
 
                                   


                                                                              

                                                         

                                      
         
 


                                  
                    
                                          





                                          
 
                                                      



                                       
 
                                                         




                                                                              
 
                                                                          
                              
                                                               
                          
                                           
                                                                             










                                                              
 













                                                                   
                                                                              
 
                                                 

                            
                      
 


                                      







                                                  
                                                                           






                                                  
                  




                                     
                        


                                 
                       





                                          


                                                      





                                                 
                                





                                              






                                               
                                                                           




                                                                    
                              
 

                       








                                                        
                            
                                


                    
                                  
                                        
 
                                



                                                    
    

                                                               




                                   
                                          
 
                                              
 

                               
                                                            



                                           


                                                                




                                                         

                                            
                                            

                                            


                                            

                                            



                                            
                                            
                                            
      
 
                                        
                        
                                          
 



                                               






                                                                        













                                                       
                             




                                                                      

                                                   
 
                                                                 
                                    
                                                                 
      
                                                              

                     
                     
                                                            
                                               
      
                  








                                                                        
                                                                       

                            
                      

                                     
                                         
                            


                                    
                                                              
                                        
                
                  

                                                               
                   
                                                
                                   
         
                                      
                                     



                                                    
                                   

 


                               

                                                                                          








                                                                                 
                                           




























                                                                            
                                     










                                                                     
                                     
 
   

                                                
                                                     








                                                                      


                                                            
 
                                                     




                                      
                                                    
                                            

                                                



                                                                           

                                                                       
                                                                  


                                                             
                                                                       

                            
                                           
                        




                                                                    
                                                                         

                                                                           
                                                                         




                                                                            
                                 

                                     



                                                        
 
                                                 

                           
                                                


                                                                              
                   
                                        
                                      
         





                                                                 
                                                       

                            

                                  




                                                                           
                                   
                                                                        





                                                    
                  
                                                       
                                        
 
                                     
 
                                            
                                           

                                






                                                                     
                            





                                             
                                  
                                       

                                    






                                                                
                                                      
      



                                                                   






                                                                      






















                                                                      
                                













                                                                    

                                   
 
                                








                                              
                                




                                                          














                                                                 
  

                                                                          
   
                                               
 
                                                       


                                   








                                                                                






                                                                              

                                                                            

 

                 
  




                                                           
                                                          


                             
                                 



                                                            
                                
                                              
                                                                           

                                              
                                                                           

                 

                             







                                                             
                                                            












                                                                              
                                                                      



                                 

                    




                                                               
                                                      
                                        

                                                    
                                                                               
                                
                      
                                                                               
         
                   

 
                                                                                





                                                               
  

                                                                   
                                                               

                                    
                                                                   
 
                                 
                            
                      


                                                 
                                               

                         
                                                  



                                                                               
 




                                               
 






                                   

                                                                             


                     
                                          
                                                                
                  

                                                     





                                                                        

                                                               
 
                                      
                                  
                                     



                                                                        
                                          




                                                                          
      
                                                                        
                                                                      
                                     






                                                                    

                                                                 
                         
         

                        

                                                                  
                         
         
 





                                                    

                                                                       
                        
                                                 
                                                               

                                                                  



                         

                                                              
                         
         


                 




                                                                   
 
                                              

                                           
 
                               

                         

                   
          
                                             
           

                                           
                                                    
                         

                                                       
                                                                         

                                                  

         
                                                 
                 
                                           
 
          
                                                                              
           
                                

                                                  

                                                  


          
                                                                           



                                                                      


                                     

                                             


  


                                                                             



                                                                           
                                                  


                                                                       
                                                              
                                           
                                                 

            

                                                                       
                                                                 
                                                                       
                                    



                                                                  
                                                            
 


                                    

























                                                                                









                                                                              
                                        

                                                                     
                                                                               


                                 


  
                                                                        

                                                                      


                                                         
                                                                     
                                                                     


                                                                              
                               

                                                                
                                    





                                                                   

                                                          

                                                       
 
                                 
                                        
                                        


                                           

            
                                                                               

                                
                                    
                                                                  
                                                                

                                                    
 

                                                                        
 
                                
                                                                             
 
                                                                
                                                                  

                                                              

                                    
 

                                                                    
 
                                                       

                                             
                                                                      





                                                                     
                                                                
                                
                                                                




                                                            

                                         
                                                         
                                                                 

                                                                          

                 


                                                                               

                                                                               
                   

                                                            



                                     
                                       
                                                

                                                               

                                                             
 













                                                                             

                                        
                                                                       
 


                                             


                                                               
                                                                              

                                            

                                                                  
                                                
                 





                                                                        


                                                                       








                                                                        
                  

                                                                          

                                                                              
                                       
                                        
 
                  
                                                                         




                                                                           

                                                                   

                                                        

                                                               
                 
 
                  
                                                               





                                                                         

                                                                              


                                                                   
                 

           


                                      
                                                                         







                                                                    
                                                    


                                                                             







                                                                               
                                                                            

                                                           











                                                                            

                                                                         
                                                                              
 
                                                                      

                                                                           

                                           





                                                                           
                                                 
                                                     









                                                                       
                                                                             
 
                                                                   
                                                                       
                                                           








                                                                              



                                                                     


                                                  

                                                                          
                                   
                                                          
                                                                           

                                             
                                                     
                                                                         

                                                                         
                    



                                                                          


                                                

                                                           

         


                       
                                                         
                                                                           
                         
 



                                                                
      
    






                                                                          
                  
                                                                      
                                                                  
 
                                       
                                   

              
                                                  
                                 



                                         
                               
                                         
 
                                                          
                                 
 

                                      
                                     






                       





                                                                        

                                                                         
   
                                                         
                                                                         
                                                      
 
                                                                      
                                  
                                
                           
                            
 

                           


                                                                                
                                                                             
                                         
           
                                                                    
                                                         
                            
 
                                          
 
     
                          
                                                                            
                                                  
 
                          
                                  
 




                                                    
                                                                   




                                                    
                                   


                                                         
                     



                                                                          
                                                                         

                                                     
                                      
                                                 
                                                                 
                                                                        
                                                   
                                         
 


                                                              
                                                               

                                              
                                                                            
                                           

                                                          
                                          
                                          
                 
         
 
                        



                                                                               
 
                                                                 




                                                                                
                                                                              



                                                    


                                                             
                                                   
                         
                                                                      
                                           





                                                                           
                                                                       
                 
              

                                          
                                      

                                                                         








                                                                             

         
                                                                     
                                                         


                              

             

                                             
                                  

               
                                            

                                                                         

                                          
                                                        
                                                         



                              

                                  
                        





                                                                         
                                                                              

                     
          

                                                                               

                                  
                                  
                                
                         
                        
                           

                           
 



                                                                                
                                         


                                                         
                            
 
                                                    
     
                                          
                                                                            
                                                         
                     
                                                              
                                  

         
                                                                             
                       
                                                              
                                  

         

                                   
                                                                   
 
                     


                                                      

                                                       
                                                                 

                                                                      
                                                        
 
                                           

                                                          

                                          

         
                        
                                                             

                                                                 

                                  
                                          
 
                                          
                        

             
                                                       
                                                        
                                                         
                          
                                  
 
                 





                                                                      
                                                          

                                

                                                  



                                                     

                                       





                                                   
                                                                        
                                                                             
                                                                        





                                                                          
         
                                                                        




                                                                      
         
                                  









                                                                           
                                                                       
 
                                              

                                
 
                                        
                                        


                                       

          
                                                       
                                                    
                                            
           
                                        
 

                                                      

                                    

                                                              
                                         
                                                    
                                                                         
                                      
         
 
                         
                                             
 

                                                                    



                                                      
                                                     

 


                               
                               

                                         

  
  









                                                                            
  












                                                                          
                                                    




                                                                         
                                       






                                                                                
                                                                         











                                                                               
                                                          

                                 
                                                      











                                                                                




                                                                       
                                                               
 

                                    

                                
                                                              
                                                     
                                    
                           




                                                                          
 
                                  



                                                   
                                     





                                                      


                                                 
                                                          
 
                                     



                                                      
                                                                          
                                                                             

                                                                             


                                                                             
                                                    
                         
                                                   
                 
                                   

                                                
                                                                            
                                                                   

                                                







                                                                            
         







                                                                         

                              








                                                                          



                                                                        
 
                                          






                                                                        

                                                           


                                



                                                          







                                                                               
                                                                 

                                                 

                                                                                











                                                                           
                                                               
 









                                                                       
                                                              











                                                                       
                                                               
 
                                             








                                                                          
                                                                 








                                                                          
                                                 



                                                     
 


                        


                                                
                                                       

 
 

      




                                          

                                                                    
   
                                                         
 
                            
                      
                   
 
                                     
 
                                  

                               

                                                          
                                        
                                        
         
 
                                   
 



                  

                                                      







                                                                   
                                            






                                                                  

                                        


  



                                                                        
                                                                        






                                                                 
                                            






                                                            









                                                                       








                                                                     
                                  

                        



                                                                          
 
                                                  
                                              






                                                                        
                               
                                                                      
                                                 




                                                                      


  










                                                                         







                                                                 
                                  






                                                                              
              


                                                                    
  


























































                                                                              







                                                                   

                                     
                                            

                           

                             
                            
                            
                                                  
                               
 
                 

                                         
      

 











                                                                  
 
                                         
 
                           


                     

                                                       
      
                               
                           


                                           

                                                               


                                                                             


                                 
                                         
 
                           


                     
                                                                            
                       


                                                


                                                           
      
 

                                                                            






                                 
                                     
   
                                                             
 




                                                                       
                                    
                        

                                            




                                
 
 




                                                           
          
                                                                    


                                                                       
                                                                   

                                     

                                                                  
                                              

                                              

                                                          

         





                                  
                                                       
 
                                        
                              

          

                                                             
           
                                                           
                                                        

                                 

         

                                    
                                              








                                                              
 





                                             
                                    
                      
                












                                     
 
                               
                                 
 
                                 
                            
                                     
 
                                                                 
                                                                      
                                                   
                    
                                                     
                                            

         



                                                          
 
                                      
                                      
 
                                                   
                                        
 
                                   

                                              



                                  
                                                                    





                                                                             


                                           
                                                         
                                                 
 



                                                         



                        
                                                                  
                                                                      




                                                       
 

                                                                           
                                                                    
           
                                                         

                       

                                                  
                           
                                                  
 





                                                                         
 


                                
                                                               






                                                                     
 
                                                  

                                                      

                                                  


                                    
                                                  
 





                                                                         



                           

                                                                      
 
                                                         
 


                                     

                                                                             


                                                                               
                                                                            




                                                                              
                                  
 
                                                                        

                                             
                                                        
                                                                               








                                                                   
                                                  
   
                                                       
                                                    






                                                        




                                                                            
                                                              




                                              
                                                           










                                                                      
    
                                                                         















                                                                   








                                                                                
                                   




                                                
                                                            



                                                     





                                                                             
                                       




                                                
                                                            



                                                     

                                                                 
 





                                                       
                                                                   

                                                       

                                                   


                                                            
                                              
                                                     

                                       

                  
                            
 
 

                                                              
 


                                     
                                                        
                                       

                       
 









                                                                              
                                                      

                                                                       
 
                                   
 








                                                                                
                     
                                                                        
 
                                                                 
 
                                           
 






                                                                               
                                                                   
 



                                                                              
 
                                                 
 







                                                                                  
                     

                                                                
 
                                                               
 
                                                         
 






                                                                            








                                                                         













































                                                                                

                                                              
 



                                             
 
                                   
 













                                                                     
 

                                      
            
                                                                  
 
                                                               
 

                                              
                                           
 
                                                                       
 

                        
                                                                 
 
                                                                 
 

                                











                                                                     
                                                      

                            
                                    
                      
                                                              



                                            
                            
 
                          
                            
                                      
                  
                                       

                                                     





                                                   

                       

                                                  
                    
                                       

                                                                         





                                   
                                                    
 
                                   
                            
                      







                                                                        
                            



                                                                           
                               
           
                                    


                                                    
                            
                  
                                       
 
                                            
                           


                                    
 
                    
                                       
                  

                                                                       
                   
                                                                    




                                               

                             




                                                       
                                                         
 

                                                                      
 



                                                                     










                                                                   
                          





                                                                     

                                








                                                              


                                                      

















                                                         
                                          







                                                     
                                          


                            
                         









                                                  



                                                    
                                      







                                                                  
                                                         
 
                                                      


                                                     

                                                                          
 
                            
 
                           











                                                   
                              


                                               
                           

 

                                                                     
 
                                                            
                            
                                                              
                      
 

                                                   




                                                              

                                                                          
                               

                                                           

                                                                  

                                        
                                                                    
                                                              
                               
                                                              

                               


                                                            
                                             
                                        
                                                  














                                                                              





                                                                    
 




                                                          
 
                   
                            



                                                                               

                                                                            
                                      

      




                                                                      
          




                                                                 


                                                                
                               


                                                                  

                                                           

                             
                            
                            
                                      
                  
                                          

                                                     
 
                          
                                                             
 

                                                  

                                        

                                                                         
         


                                                   

                              

                 













                                                                                    

                                      
















                                                                                                             

                                                                              
 

                                  
                   




                                                                       


                        
                                     


                                                                
 








                                                                           

                                                                               
 



                                                      


















                                                                               
                              
                   

                    
                               









                                                       










                                                                               
                              
                   

                              
                               


















                                                                         






                                    
                                                           
 
                                             

                              
 
                          




                                            
                                  




                                                           
                                                             




                                                   







                                                            




                                                                      



                                                        

                                                     
       
                                                   
 
                      

                                                              




                                                                      
                                                             


                                   
           



                                       
                           
                          






















                                                                               
                               

                   

                                                      
 




                                                                 

 

                                                  
                              
                   
 
                          






                                     



                                               
                                                         


                                    
                          
 
                      











                                                                          
                           
 
                                 

                               

                                                  
 







                                                                      
 
                   




                                                                  

                                                                       


                                     
                                       
 
                                     
                                             





                                                                
                                                      







                                    
                                
 


                                          




                                                             






                                                  
                               
 

                                                                         




                                 
                             




                                                                         
                                                                              


                                                                          
                                       
 
                                                                       

                    
                                              
                                  



                                              
                        
                                
         
                   
 





                                      
                                                               
                                  





                                   

                                    


                                                        
                                                              






                                               


                     
                                                                      






                                                                            
                                                      
 
                                


                                   
                              
 



                                              
                                                      

                 
                                


                                        
                              



















                                                                
                         
                        






















                                                                
                         
                        















                                                                           
                              
                                
                   
                          

                    
                               










                                               





                                                                   
                                           
                                             




                                                 

                                                                              

                                           
                                    
                                            
                                                                     
                      
 




                                    
                                                      
 
                                           
 
                               
                       
 
                                                   
                                                
                                                                      
                       
                                  
                                               
            
                                                                

                                  
                                                       
            
                                                                 


                               
                                                   

                            
                                                                         

         
                                                
                                                             
 
                            

 
                                                  
 
                                  
 


                                                                 
     

                                                                         







                                                                     
                                                               
                                           

                                  

                                         


                                 
                                    




                                                   

 

                                                              
                                              

 







                                                                  
                                                           
 
                                    

                            

                                            


                                            
                                                  
                                                 
                                  
 
                                   


                                                              


                                                            


                                                                        
                                                  
      



                                                                  
                                       






                                                             
                           
   
                            
 






















                                                            

                                                

 



                               
                                                                   

















                                                                     
                                                            

                                                
                                                                          
 
                                 
                            
                      
                    

                                     
                                                          



                              





                                                                    
                                             
                                                              
              

                                                               

         
                                                                           
                                              

                         
                                                                                








                                                                          
 

                   
                                        

  
                                                                   





                                                                     

                                                      
   
                                                                           
 
                                    
                           
 
                                            
                           






                                        
                          

                                                  
                          
 
                            
                  
                                              
 
                                  

                                             
                                                  
         
     
                
     
                                          
                   






                                                                    
                                       
 
                             
                      





                                                
                                          
                                       
 



















                                                              
                                                                         

                                          


                                                              

















                                                      










                                                                               
  
                                                                       
   
                                                                       
 
                            

                      




















                                                                              
 








                                                                  
                 





                                                                  








                                                                        
                                                         
 
                                                                  












                                                                  
                                  
 
                                  
 

                                 

                                 


                                                           
 
                                    

 

                                                                   

                                                           


                          
                                          
                                         



                                         
                                     
 


                                                                           


                                            
                                                         
 

                                



                                                 

                                                                        












                                                  
                                                    
                                                                      
 
                                         

                                                              
                               

                                                                       
                                      
 
                           


                                                                
                                                                       

                
                                   
                                            
                                 
 
                           




                                                                             
                                         
                                 
 


                                    
                                    
                                                    

                              
                                                           
                                             
 



                               


                                                         

                                                 
                                       
          
              


                                         
         
                                           
                                           
                                       

                                             
              




                                                  
                                                                 
 


                     

                                                        
                                
 


                                                                         
                                                                    


                                                     

                                                         


                                                




                       
           
                                        


                                                             









                                                  
                                                         
 


                            
                                                                     
                                                            
                                                                     
                                                            
                                                             
                                                         
                                                             
                                                         
                                                                   
                                                         
                                                             
                                                         
                                                                     
                                                         
                                                                   
                                                         
                                                                       
                                                         
                                                      

                                                         
                                                        
                                                         


                                                        



                     
                                                 








                                                           

                            



                                                 
                                                           
                                   







                                                             
                                              




                                                                  


                                     


                          
                                
                                              
                                                           
                                   
                                                         
                        
         

                                  

                                                              
 
                                               

                                                

                                                          
                                

                                                        
 
     



                                                



      




                                                
                                                         


















                                                      
                                                           



                               



                                                                       

                                                                            
 
                              
                             
                            
                      

                         
 
                            
                                   
                                                                                

                                          


                                                                             
                                                                 


                                                  
 
                        
                               
                                                                       
                                                               




                                                    
                                                                     

                                          

                                                         
                      
 

                             
                                    

                                                   
                                                                          
                                                           
                                                           


                                                            
 
                      
                             
                                                                               




                                                                       
                                         
                                    
                                                 
                                                 

                                                              
                                        
                                           
                                


                                               




                                                                      

                                                           

                                                  
                                                                  
                                                                     




                                                  
 

                              



                                                    
                                                                     
                                           


                                                         







                                                                          
                                                                 



                                        
                                      

                                                     
                

                                         

                                                                       

                                                             

                   
 
                               


                 
 
                         
 

                                                                              
 
                                               
                      
 
                                                                    
                               








                                                                        

         
                                                              
 
                                                            


                                                                       
                                                              


                                                                        
 
                                                        
            


                                                                  


                              





                                                                       
 
                                                               



                                                                
 
                                                                             



                                                                  
 
                                                                          
 
                                                                              
                                             
 


                                      
 
                                                             
                                                                           
 

                                                                      



                                                                       
 

                                                                
                                
                      
 



                                                                               
 

                                                                  
                                                         



                                                                           
                  
                                                                      
                              

                                
                        

                              
                                    
 
                               
                                                     
                               
 
                                                 
 
                                                       





                                                    


                                                   












                                                   

                                                                          





                                                                 
                                                                             










                                                                        


                                                        






                             







                                                   


                                                                 





                                                    
                                                              
                                           
 
                                                         
 
                                                           
                                                




                                  

                                                       
                                  



                                                 
                                                                


                                   













                                                                     

                                 








                                     



                                    

                                                


                                                 
                                                 






                                              



                                              



                  
  
                                                                   

                         

                                                                           
 
                                    


                                                                             
                               


                                                          
 
                                                        
                                                     

                                                            

                                          

         
                                      
                                


                                         


                                    
                               
                                       


                                
                                      



                                                            
                                              


                 
                                         

  



                                                                           




                                                                            
           
                                                                        
                                                                          


                                                                

                                                       

              

                             
                               
                                       
                                                               

                      
                                                 

                                 
                                                    
                                    
 
                                       
                                                                         

                                         
                                             
                                                                 









                                        
                              
 
                  
 




                                                                        
                                                                     



                                                             
                                                                




                                              
                                           
                                    
                                             




                                             
                                               










                                                
                                         





                                                                   
                           
  
                                                                          


                                                                            
                                                             
 
                              
                                            
              
 
                          
                                
 
                                         
                                   

                                                   
                                                                       
 
                                                              
                                                 
         
 
                        
 
                                                            
 
  














                                                                          
                     
   
                       

                                                                   
 
          

                                                                            
 
               
                                                         

                   
                             
 


                            
                      

                                                                   
                            

                                                         
          

                                                                             
 
                  



                                              
               
                                                           
                     

                              
          

                                                                             
 
               
                                                         



                   

                                                                   
 
          

                                                                             
 
                  
                      


                                         
                               


                                              
     
                    
      
               
                                                           
                     


                  
  


                                                                          
   
                                                         
                                                     
 
                                                             
                                                                       
 
                                                                   
                                                                              
 
                  
                                                         
 
                                                  
                                     

               
                                                               
                     
 
 






                                                                        
            
                                                       
                                                
 
                                                          
                                                                               





                                                                 
 



                                                                      
 
                        
 
                  
                                                              
                                                                            
 
                   
 
                                    


                                                       


                                       
                                                   
                                                                              
                                                          
 
                                                                  
                                                  














                                                          
 
                        
                                                                            

 
                        
 
  



















                                                                               
                                                               



                          

                                    









                                                                              
                                                               


                       




                                                                     
                                                                 




                                         



                                                                            





                                                        
                                                  
 




                                                                 
                                                                 
                                                                 













                       



                                                     





                                                 
























                                                                   
  

                                                                           
   

                                                                
 
                             
                               

                                                                              
                  
                                                      
                                                      
                            
 



















                                                              


                                                     
                                                                              
                                                

                                                                             
                                  
         
      
 
                                

                                                                  
                                       

         



                                                                         
          
                                                            
           
                                  
                                                   
 

                                                            

                  
                                           
                                                                             
                                                           
                                              
                                                       
                                                                     
                                                                                
                               
                                        


                                 
                                               
                                  
                                               
                                                                              
                               

                                      

                                                            


                       
                                                  
                                 
                                               
                                                              
                               

                                      
                                                                    
 

                       
                                                  
                                
                                               


                                                              
                               
                              
                                                                    

      

                       
                                                 
                                     
                                               

                                                                   
                               
                              
                                                                   




                                         
                                  


                                                                         

                                 
                                                                  

                                                               


         

                                      
                                  


                                                                   
                                 
 
                                                               

                                                               


         
                                    
                                           


                                                         

                                 


                                                               



                                
                          



                                                               
 
                                           

                                              

                      




                                                         
                                                    
                                 
                 
 
                                                      
                                                             
 

                                                                              




                                                                             
                                          
                                           
                                                
 

                                                       
                 
                                    
                                                             
                              
                                                       

                          
                                                   
                                                      
                                                          
 



                                                                  

                                      

                                                                 

                                         

                                                                      
                                                         


                                                                               
                                           
                         
                                            
                                                                    
                                              
                                                              


                                        
         


                                                                 
                       
                                  
                                                                      
 
                                               
         
      
                      
                                  
                                                                       
 
                                               

         
 
                                  
                                                                       
 
                                               

         
                  
                                         
                                                                   
 

                                       
 

                                                                         

                                                 

      
                                
                                  

                                        
                                                 
                              
                                                  
     
                                                  
      
                                             
         
 



























                                           
 
                  
      
                                            
                            
                          
      
 
 




                                                        

                                                                           

                                                                               







                                                               



                                                         
  
                                                                           

                                                                 
   
                                                            
 

                
                                   



                                                          
                                                            
                         
                                            
                                       

                   

 

                                                                
 
                                            
 
 



                                                                 
                                                            
 
                          

              
                                
                                                             
                            
                                                      

 















                                                                   

                                                          
                                                              



                                                                   


                                                                   


                                                                     

                                                                     



                                                                   
  


                                                                    
  

                              

                                                                 
 
                    
 
                                         
 


                                                                    
                                     


                                         
                                         

                                                                        







                                                                             

                               
                                          
                                                                                
                                        

         


                                                 

                                                                        


                                                   

                                                                          






                                            
                                                         
                            
                              
                              

                                       
 
                                           

 
                                                         
                                   
 
                          



                                                        
                                
                          
 
                 


















                                                                                
                      

                                                                      


                                                             
                                                                       
                                                                          


                                                        


                                                       


                       

                                                                     


                                                              
                                                                      
                                                                           


                                                        

                                                       


















                                                                            
                                                
 
                      
  

                                                                 



                                                                 




                               
                                                       










                                                                 

                                  
                         
                              
                                     
                                             

                                 
                             
                                    
                        
                               
                                            

                                 


                                   
 


                                


                                                          
 




                                                                            
                          
                                         



                                                                               
                                           
                          

                      

                                                                    




                                                                 
                      

                                                  
                                                                 
                      
                                 
                                            



                                
                                 




                                          




                                                           
                                                             

                                         
                                       


                              
                                                   

 












                                                          
                                                       

                                          

                                   




                                

                                                
 
                            
                                 

                       

 
                              


                                                                           
 
                                    






                                                                           



                                                  




                                          

                                     
                                
                            
 
      
 
                            


                                                                        
 

                                    



                               
                                                        



                                                                        


                       




                                            
                            
                               



                                         

                            
                 







                                                       


                        




                                                                
                                                               






                                                                 






                                                                 

                                    




                                                                       







                                                                       

                                  
         
 



                             





                                                                 


                                                         

                                  
 
                         
                                                      





                                                                       

                               
 
                                  
                              


                                          
                                   
                                          
                                        
                              
                                                              
                                                      



















                                                                               
                                                                                
                               

                                                                                










                                                                                
                                                   
                                                         

                                                                     
 
      


                                                                
                            
                                                     
                          
                                                                              
                               
                                                                              
                                                  
                                                        

                                                                        
      
      
 

                                                      
                 
                              
                              
                                       
                                           
                                 
                            
                               

                                                     
                                                     
      
                                   
                                              

         
                                    
 



                                                      
                 
                                                           

      



                                                                   












                                                                            



                                                              
 

                                                                   


                                                  
                                                     
 
                              




                                        
                

                                                                  


















                                                                                





                             


                                                                
 










                                               

                             
                                  
                            
                      
 
                                                 
                              





                                             
                                                    
                        
                                                    
                                                    
                                                    
      







                                                               
                                 
                 
 
                                       
                                       
 
                                      
 
                                     
                                         

                                  
                                                      


                               

















                                                               
                                      









                                                                            

                                                                              






                                                                             
                                                  




                          
 

                                                        







                                             



                          

 

                                                                            
 
                              
                                
                      

              
                                                                      

                         
                                                              

                         

                                 

                                  
                               
 

                                                                  


                                 

                                                              


                                 
                                                                       

















                                                                              
                                   



                                                               

                                                                            










                                                                              
                                    

                            



                                                      

                                                










                                            

                                                                          

                            
                                      


                      
                                                                    

                         
                                                                    


                         

                                                                    



                                  

                                                                 

                                 
 

                                                                    

                                 
 
                                                                           

         















                                                                            
                                  



                                                             

                                                                          










                                                                            
                                  
 
                         







                                                   
                                                                








                                              
                                                

                         
                                              

                         
                                                   
                                  

                                                 
         
                                              



                                                         
                                      
                                                       
                                                        
 
                  

    
                             


                                
                                                                          
                                                      
 
                                                         
                                                                    

 
                                                       
                                               
 
                            
              
 
                                                   
                                  

                                                   
         
                                
                                    
                                                        
 
                                                                         
                                                 

 
                                                       


                                                                               

                                             






                                       

                            
                                        

                              
                  
                                         

                                                         
 
                                        
 




                                                   


                                                    
                                         
 

                                   
                               
 
                              
                                                                          

                                           

                  
                          
                  


                                              
                                
 
                  
                                              
 
 








                                                                        

 

                                  
                                                                       

              
                            
 
          




                                                         

                                    

                                     
 
                                  
                                 
                          
 
                                                   

                                                   
                                    
                                                        







                                                                    
                            




                                                    
                                                 
         
 



                                                                              
                                                   

                                                 
                                                           
                                                        
     
                                    
                 

 



                                                       
      
 
                            
  
                                                         
   




                                                      
                                  
 
                                                

 

                                                        
 
                                  
 



                                                                
 

                 
 




                              
 





                                                            
 

                                                         
 


                                        
         
 




                                                       
 


                                                    

                                                                      
 
                                          
 




                                                                      
 


                                                                       


                                                                    
 



                                                
 
                                                 
         
 



                               

 
                                                                           
 






                                                           

 

                                                  
 
                       
 
                                          
                                  

                                                          
                            

                                                         

                                                            








                                                           
        
                                    


                                            

 











                                                                         



                                                  
                                                       

                          
                                                    


                                             







                                                                       


                               













                                                               
                            

                    


                                        







                                                       
 
                                          
                                  
                                           
                                    



                                            
                                  

                                            


                            


                                        









                                                                         

                 
                                  





























                                                                          
 
                          

                                                        
                                                               
 

                                                                            


                                   
                                                                
 
                                       
 
                            
                                                                     


                                            

                                         


                                        


                        

                                                                 
 
                                                



                                


                                                                    
 

                                                                              
                                                                          

                               


                                                                
      




                 
                                                                




                                                                         
                              
                                                                           
                                             
 
                                                                 

 
                                                                       
 
                                                


                                
                                    
 
                            
                                                                        
                                        
 
                                                                

 
                                                                       
 
                                                       
 










                                                                               
                                  
 
                                    
                              

                                 

                                                  
          

                            
         
                                        

                                                  
          

                                       

                                                      
          
      



                                                                             
                                                                            


                                          






                                                


                            
                                 









                                                                        
                                                              



                                                                      
                               




                                                                 
                                                            
 
                                                                         











                                                                      
                                                      











                                                              


                                                     



                                              
           
                                                              
 
                                             





                                                        
                                                                 
 
                                             

















                                                                       






















                                                                          


                                

                                            


          
                                                                          
 
                                                                    









                                                                
                



                                   
                            
                          
 

                                                              











                                       
/*
 *  kernel/sched.c
 *
 *  Kernel scheduler and related syscalls
 *
 *  Copyright (C) 1991-2002  Linus Torvalds
 *
 *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
 *		make semaphores SMP safe
 *  1998-11-19	Implemented schedule_timeout() and related stuff
 *		by Andrea Arcangeli
 *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
 *		hybrid priority-list and round-robin design with
 *		an array-switch method of distributing timeslices
 *		and per-CPU runqueues.  Cleanups and useful suggestions
 *		by Davide Libenzi, preemptible kernel bits by Robert Love.
 *  2003-09-03	Interactivity tuning by Con Kolivas.
 *  2004-04-02	Scheduler domains code by Nick Piggin
 *  2007-04-15  Work begun on replacing all interactivity tuning with a
 *              fair scheduling design by Con Kolivas.
 *  2007-05-05  Load balancing (smp-nice) and other improvements
 *              by Peter Williams
 *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
 *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
 *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
 *              Thomas Gleixner, Mike Kravetz
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/init.h>
#include <linux/uaccess.h>
#include <linux/highmem.h>
#include <linux/smp_lock.h>
#include <asm/mmu_context.h>
#include <linux/interrupt.h>
#include <linux/capability.h>
#include <linux/completion.h>
#include <linux/kernel_stat.h>
#include <linux/debug_locks.h>
#include <linux/security.h>
#include <linux/notifier.h>
#include <linux/profile.h>
#include <linux/freezer.h>
#include <linux/vmalloc.h>
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/pid_namespace.h>
#include <linux/smp.h>
#include <linux/threads.h>
#include <linux/timer.h>
#include <linux/rcupdate.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/percpu.h>
#include <linux/kthread.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/sysctl.h>
#include <linux/syscalls.h>
#include <linux/times.h>
#include <linux/tsacct_kern.h>
#include <linux/kprobes.h>
#include <linux/delayacct.h>
#include <linux/reciprocal_div.h>
#include <linux/unistd.h>
#include <linux/pagemap.h>
#include <linux/hrtimer.h>
#include <linux/tick.h>
#include <linux/bootmem.h>
#include <linux/debugfs.h>
#include <linux/ctype.h>
#include <linux/ftrace.h>
#include <trace/sched.h>

#include <asm/tlb.h>
#include <asm/irq_regs.h>

#include "sched_cpupri.h"

/*
 * Convert user-nice values [ -20 ... 0 ... 19 ]
 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
 * and back.
 */
#define NICE_TO_PRIO(nice)	(MAX_RT_PRIO + (nice) + 20)
#define PRIO_TO_NICE(prio)	((prio) - MAX_RT_PRIO - 20)
#define TASK_NICE(p)		PRIO_TO_NICE((p)->static_prio)

/*
 * 'User priority' is the nice value converted to something we
 * can work with better when scaling various scheduler parameters,
 * it's a [ 0 ... 39 ] range.
 */
#define USER_PRIO(p)		((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p)	USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO		(USER_PRIO(MAX_PRIO))

/*
 * Helpers for converting nanosecond timing to jiffy resolution
 */
#define NS_TO_JIFFIES(TIME)	((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))

#define NICE_0_LOAD		SCHED_LOAD_SCALE
#define NICE_0_SHIFT		SCHED_LOAD_SHIFT

/*
 * These are the 'tuning knobs' of the scheduler:
 *
 * default timeslice is 100 msecs (used only for SCHED_RR tasks).
 * Timeslices get refilled after they expire.
 */
#define DEF_TIMESLICE		(100 * HZ / 1000)

/*
 * single value that denotes runtime == period, ie unlimited time.
 */
#define RUNTIME_INF	((u64)~0ULL)

DEFINE_TRACE(sched_wait_task);
DEFINE_TRACE(sched_wakeup);
DEFINE_TRACE(sched_wakeup_new);
DEFINE_TRACE(sched_switch);
DEFINE_TRACE(sched_migrate_task);

#ifdef CONFIG_SMP
/*
 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
 * Since cpu_power is a 'constant', we can use a reciprocal divide.
 */
static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
{
	return reciprocal_divide(load, sg->reciprocal_cpu_power);
}

/*
 * Each time a sched group cpu_power is changed,
 * we must compute its reciprocal value
 */
static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
{
	sg->__cpu_power += val;
	sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
}
#endif

static inline int rt_policy(int policy)
{
	if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
		return 1;
	return 0;
}

static inline int task_has_rt_policy(struct task_struct *p)
{
	return rt_policy(p->policy);
}

/*
 * This is the priority-queue data structure of the RT scheduling class:
 */
struct rt_prio_array {
	DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
	struct list_head queue[MAX_RT_PRIO];
};

struct rt_bandwidth {
	/* nests inside the rq lock: */
	spinlock_t		rt_runtime_lock;
	ktime_t			rt_period;
	u64			rt_runtime;
	struct hrtimer		rt_period_timer;
};

static struct rt_bandwidth def_rt_bandwidth;

static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);

static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
{
	struct rt_bandwidth *rt_b =
		container_of(timer, struct rt_bandwidth, rt_period_timer);
	ktime_t now;
	int overrun;
	int idle = 0;

	for (;;) {
		now = hrtimer_cb_get_time(timer);
		overrun = hrtimer_forward(timer, now, rt_b->rt_period);

		if (!overrun)
			break;

		idle = do_sched_rt_period_timer(rt_b, overrun);
	}

	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
}

static
void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
{
	rt_b->rt_period = ns_to_ktime(period);
	rt_b->rt_runtime = runtime;

	spin_lock_init(&rt_b->rt_runtime_lock);

	hrtimer_init(&rt_b->rt_period_timer,
			CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rt_b->rt_period_timer.function = sched_rt_period_timer;
	rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
}

static inline int rt_bandwidth_enabled(void)
{
	return sysctl_sched_rt_runtime >= 0;
}

static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
{
	ktime_t now;

	if (rt_bandwidth_enabled() && rt_b->rt_runtime == RUNTIME_INF)
		return;

	if (hrtimer_active(&rt_b->rt_period_timer))
		return;

	spin_lock(&rt_b->rt_runtime_lock);
	for (;;) {
		if (hrtimer_active(&rt_b->rt_period_timer))
			break;

		now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
		hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
		hrtimer_start_expires(&rt_b->rt_period_timer,
				HRTIMER_MODE_ABS);
	}
	spin_unlock(&rt_b->rt_runtime_lock);
}

#ifdef CONFIG_RT_GROUP_SCHED
static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
{
	hrtimer_cancel(&rt_b->rt_period_timer);
}
#endif

/*
 * sched_domains_mutex serializes calls to arch_init_sched_domains,
 * detach_destroy_domains and partition_sched_domains.
 */
static DEFINE_MUTEX(sched_domains_mutex);

#ifdef CONFIG_GROUP_SCHED

#include <linux/cgroup.h>

struct cfs_rq;

static LIST_HEAD(task_groups);

/* task group related information */
struct task_group {
#ifdef CONFIG_CGROUP_SCHED
	struct cgroup_subsys_state css;
#endif

#ifdef CONFIG_FAIR_GROUP_SCHED
	/* schedulable entities of this group on each cpu */
	struct sched_entity **se;
	/* runqueue "owned" by this group on each cpu */
	struct cfs_rq **cfs_rq;
	unsigned long shares;
#endif

#ifdef CONFIG_RT_GROUP_SCHED
	struct sched_rt_entity **rt_se;
	struct rt_rq **rt_rq;

	struct rt_bandwidth rt_bandwidth;
#endif

	struct rcu_head rcu;
	struct list_head list;

	struct task_group *parent;
	struct list_head siblings;
	struct list_head children;
};

#ifdef CONFIG_USER_SCHED

/*
 * Root task group.
 * 	Every UID task group (including init_task_group aka UID-0) will
 * 	be a child to this group.
 */
struct task_group root_task_group;

#ifdef CONFIG_FAIR_GROUP_SCHED
/* Default task group's sched entity on each cpu */
static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
/* Default task group's cfs_rq on each cpu */
static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
#endif /* CONFIG_FAIR_GROUP_SCHED */

#ifdef CONFIG_RT_GROUP_SCHED
static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
#endif /* CONFIG_RT_GROUP_SCHED */
#else /* !CONFIG_USER_SCHED */
#define root_task_group init_task_group
#endif /* CONFIG_USER_SCHED */

/* task_group_lock serializes add/remove of task groups and also changes to
 * a task group's cpu shares.
 */
static DEFINE_SPINLOCK(task_group_lock);

#ifdef CONFIG_FAIR_GROUP_SCHED
#ifdef CONFIG_USER_SCHED
# define INIT_TASK_GROUP_LOAD	(2*NICE_0_LOAD)
#else /* !CONFIG_USER_SCHED */
# define INIT_TASK_GROUP_LOAD	NICE_0_LOAD
#endif /* CONFIG_USER_SCHED */

/*
 * A weight of 0 or 1 can cause arithmetics problems.
 * A weight of a cfs_rq is the sum of weights of which entities
 * are queued on this cfs_rq, so a weight of a entity should not be
 * too large, so as the shares value of a task group.
 * (The default weight is 1024 - so there's no practical
 *  limitation from this.)
 */
#define MIN_SHARES	2
#define MAX_SHARES	(1UL << 18)

static int init_task_group_load = INIT_TASK_GROUP_LOAD;
#endif

/* Default task group.
 *	Every task in system belong to this group at bootup.
 */
struct task_group init_task_group;

/* return group to which a task belongs */
static inline struct task_group *task_group(struct task_struct *p)
{
	struct task_group *tg;

#ifdef CONFIG_USER_SCHED
	tg = p->user->tg;
#elif defined(CONFIG_CGROUP_SCHED)
	tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
				struct task_group, css);
#else
	tg = &init_task_group;
#endif
	return tg;
}

/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
{
#ifdef CONFIG_FAIR_GROUP_SCHED
	p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
	p->se.parent = task_group(p)->se[cpu];
#endif

#ifdef CONFIG_RT_GROUP_SCHED
	p->rt.rt_rq  = task_group(p)->rt_rq[cpu];
	p->rt.parent = task_group(p)->rt_se[cpu];
#endif
}

#else

static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
static inline struct task_group *task_group(struct task_struct *p)
{
	return NULL;
}

#endif	/* CONFIG_GROUP_SCHED */

/* CFS-related fields in a runqueue */
struct cfs_rq {
	struct load_weight load;
	unsigned long nr_running;

	u64 exec_clock;
	u64 min_vruntime;

	struct rb_root tasks_timeline;
	struct rb_node *rb_leftmost;

	struct list_head tasks;
	struct list_head *balance_iterator;

	/*
	 * 'curr' points to currently running entity on this cfs_rq.
	 * It is set to NULL otherwise (i.e when none are currently running).
	 */
	struct sched_entity *curr, *next, *last;

	unsigned int nr_spread_over;

#ifdef CONFIG_FAIR_GROUP_SCHED
	struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */

	/*
	 * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
	 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
	 * (like users, containers etc.)
	 *
	 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
	 * list is used during load balance.
	 */
	struct list_head leaf_cfs_rq_list;
	struct task_group *tg;	/* group that "owns" this runqueue */

#ifdef CONFIG_SMP
	/*
	 * the part of load.weight contributed by tasks
	 */
	unsigned long task_weight;

	/*
	 *   h_load = weight * f(tg)
	 *
	 * Where f(tg) is the recursive weight fraction assigned to
	 * this group.
	 */
	unsigned long h_load;

	/*
	 * this cpu's part of tg->shares
	 */
	unsigned long shares;

	/*
	 * load.weight at the time we set shares
	 */
	unsigned long rq_weight;
#endif
#endif
};

/* Real-Time classes' related field in a runqueue: */
struct rt_rq {
	struct rt_prio_array active;
	unsigned long rt_nr_running;
#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
	int highest_prio; /* highest queued rt task prio */
#endif
#ifdef CONFIG_SMP
	unsigned long rt_nr_migratory;
	int overloaded;
#endif
	int rt_throttled;
	u64 rt_time;
	u64 rt_runtime;
	/* Nests inside the rq lock: */
	spinlock_t rt_runtime_lock;

#ifdef CONFIG_RT_GROUP_SCHED
	unsigned long rt_nr_boosted;

	struct rq *rq;
	struct list_head leaf_rt_rq_list;
	struct task_group *tg;
	struct sched_rt_entity *rt_se;
#endif
};

#ifdef CONFIG_SMP

/*
 * We add the notion of a root-domain which will be used to define per-domain
 * variables. Each exclusive cpuset essentially defines an island domain by
 * fully partitioning the member cpus from any other cpuset. Whenever a new
 * exclusive cpuset is created, we also create and attach a new root-domain
 * object.
 *
 */
struct root_domain {
	atomic_t refcount;
	cpumask_var_t span;
	cpumask_var_t online;

	/*
	 * The "RT overload" flag: it gets set if a CPU has more than
	 * one runnable RT task.
	 */
	cpumask_var_t rto_mask;
	atomic_t rto_count;
#ifdef CONFIG_SMP
	struct cpupri cpupri;
#endif
};

/*
 * By default the system creates a single root-domain with all cpus as
 * members (mimicking the global state we have today).
 */
static struct root_domain def_root_domain;

#endif

/*
 * This is the main, per-CPU runqueue data structure.
 *
 * Locking rule: those places that want to lock multiple runqueues
 * (such as the load balancing or the thread migration code), lock
 * acquire operations must be ordered by ascending &runqueue.
 */
struct rq {
	/* runqueue lock: */
	spinlock_t lock;

	/*
	 * nr_running and cpu_load should be in the same cacheline because
	 * remote CPUs use both these fields when doing load calculation.
	 */
	unsigned long nr_running;
	#define CPU_LOAD_IDX_MAX 5
	unsigned long cpu_load[CPU_LOAD_IDX_MAX];
	unsigned char idle_at_tick;
#ifdef CONFIG_NO_HZ
	unsigned long last_tick_seen;
	unsigned char in_nohz_recently;
#endif
	/* capture load from *all* tasks on this cpu: */
	struct load_weight load;
	unsigned long nr_load_updates;
	u64 nr_switches;

	struct cfs_rq cfs;
	struct rt_rq rt;

#ifdef CONFIG_FAIR_GROUP_SCHED
	/* list of leaf cfs_rq on this cpu: */
	struct list_head leaf_cfs_rq_list;
#endif
#ifdef CONFIG_RT_GROUP_SCHED
	struct list_head leaf_rt_rq_list;
#endif

	/*
	 * This is part of a global counter where only the total sum
	 * over all CPUs matters. A task can increase this counter on
	 * one CPU and if it got migrated afterwards it may decrease
	 * it on another CPU. Always updated under the runqueue lock:
	 */
	unsigned long nr_uninterruptible;

	struct task_struct *curr, *idle;
	unsigned long next_balance;
	struct mm_struct *prev_mm;

	u64 clock;

	atomic_t nr_iowait;

#ifdef CONFIG_SMP
	struct root_domain *rd;
	struct sched_domain *sd;

	/* For active balancing */
	int active_balance;
	int push_cpu;
	/* cpu of this runqueue: */
	int cpu;
	int online;

	unsigned long avg_load_per_task;

	struct task_struct *migration_thread;
	struct list_head migration_queue;
#endif

#ifdef CONFIG_SCHED_HRTICK
#ifdef CONFIG_SMP
	int hrtick_csd_pending;
	struct call_single_data hrtick_csd;
#endif
	struct hrtimer hrtick_timer;
#endif

#ifdef CONFIG_SCHEDSTATS
	/* latency stats */
	struct sched_info rq_sched_info;

	/* sys_sched_yield() stats */
	unsigned int yld_exp_empty;
	unsigned int yld_act_empty;
	unsigned int yld_both_empty;
	unsigned int yld_count;

	/* schedule() stats */
	unsigned int sched_switch;
	unsigned int sched_count;
	unsigned int sched_goidle;

	/* try_to_wake_up() stats */
	unsigned int ttwu_count;
	unsigned int ttwu_local;

	/* BKL stats */
	unsigned int bkl_count;
#endif
};

static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);

static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync)
{
	rq->curr->sched_class->check_preempt_curr(rq, p, sync);
}

static inline int cpu_of(struct rq *rq)
{
#ifdef CONFIG_SMP
	return rq->cpu;
#else
	return 0;
#endif
}

/*
 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
 * See detach_destroy_domains: synchronize_sched for details.
 *
 * The domain tree of any CPU may only be accessed from within
 * preempt-disabled sections.
 */
#define for_each_domain(cpu, __sd) \
	for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)

#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))
#define this_rq()		(&__get_cpu_var(runqueues))
#define task_rq(p)		cpu_rq(task_cpu(p))
#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)

static inline void update_rq_clock(struct rq *rq)
{
	rq->clock = sched_clock_cpu(cpu_of(rq));
}

/*
 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
 */
#ifdef CONFIG_SCHED_DEBUG
# define const_debug __read_mostly
#else
# define const_debug static const
#endif

/**
 * runqueue_is_locked
 *
 * Returns true if the current cpu runqueue is locked.
 * This interface allows printk to be called with the runqueue lock
 * held and know whether or not it is OK to wake up the klogd.
 */
int runqueue_is_locked(void)
{
	int cpu = get_cpu();
	struct rq *rq = cpu_rq(cpu);
	int ret;

	ret = spin_is_locked(&rq->lock);
	put_cpu();
	return ret;
}

/*
 * Debugging: various feature bits
 */

#define SCHED_FEAT(name, enabled)	\
	__SCHED_FEAT_##name ,

enum {
#include "sched_features.h"
};

#undef SCHED_FEAT

#define SCHED_FEAT(name, enabled)	\
	(1UL << __SCHED_FEAT_##name) * enabled |

const_debug unsigned int sysctl_sched_features =
#include "sched_features.h"
	0;

#undef SCHED_FEAT

#ifdef CONFIG_SCHED_DEBUG
#define SCHED_FEAT(name, enabled)	\
	#name ,

static __read_mostly char *sched_feat_names[] = {
#include "sched_features.h"
	NULL
};

#undef SCHED_FEAT

static int sched_feat_show(struct seq_file *m, void *v)
{
	int i;

	for (i = 0; sched_feat_names[i]; i++) {
		if (!(sysctl_sched_features & (1UL << i)))
			seq_puts(m, "NO_");
		seq_printf(m, "%s ", sched_feat_names[i]);
	}
	seq_puts(m, "\n");

	return 0;
}

static ssize_t
sched_feat_write(struct file *filp, const char __user *ubuf,
		size_t cnt, loff_t *ppos)
{
	char buf[64];
	char *cmp = buf;
	int neg = 0;
	int i;

	if (cnt > 63)
		cnt = 63;

	if (copy_from_user(&buf, ubuf, cnt))
		return -EFAULT;

	buf[cnt] = 0;

	if (strncmp(buf, "NO_", 3) == 0) {
		neg = 1;
		cmp += 3;
	}

	for (i = 0; sched_feat_names[i]; i++) {
		int len = strlen(sched_feat_names[i]);

		if (strncmp(cmp, sched_feat_names[i], len) == 0) {
			if (neg)
				sysctl_sched_features &= ~(1UL << i);
			else
				sysctl_sched_features |= (1UL << i);
			break;
		}
	}

	if (!sched_feat_names[i])
		return -EINVAL;

	filp->f_pos += cnt;

	return cnt;
}

static int sched_feat_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, sched_feat_show, NULL);
}

static struct file_operations sched_feat_fops = {
	.open		= sched_feat_open,
	.write		= sched_feat_write,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};

static __init int sched_init_debug(void)
{
	debugfs_create_file("sched_features", 0644, NULL, NULL,
			&sched_feat_fops);

	return 0;
}
late_initcall(sched_init_debug);

#endif

#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))

/*
 * Number of tasks to iterate in a single balance run.
 * Limited because this is done with IRQs disabled.
 */
const_debug unsigned int sysctl_sched_nr_migrate = 32;

/*
 * ratelimit for updating the group shares.
 * default: 0.25ms
 */
unsigned int sysctl_sched_shares_ratelimit = 250000;

/*
 * Inject some fuzzyness into changing the per-cpu group shares
 * this avoids remote rq-locks at the expense of fairness.
 * default: 4
 */
unsigned int sysctl_sched_shares_thresh = 4;

/*
 * period over which we measure -rt task cpu usage in us.
 * default: 1s
 */
unsigned int sysctl_sched_rt_period = 1000000;

static __read_mostly int scheduler_running;

/*
 * part of the period that we allow rt tasks to run in us.
 * default: 0.95s
 */
int sysctl_sched_rt_runtime = 950000;

static inline u64 global_rt_period(void)
{
	return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
}

static inline u64 global_rt_runtime(void)
{
	if (sysctl_sched_rt_runtime < 0)
		return RUNTIME_INF;

	return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
}

#ifndef prepare_arch_switch
# define prepare_arch_switch(next)	do { } while (0)
#endif
#ifndef finish_arch_switch
# define finish_arch_switch(prev)	do { } while (0)
#endif

static inline int task_current(struct rq *rq, struct task_struct *p)
{
	return rq->curr == p;
}

#ifndef __ARCH_WANT_UNLOCKED_CTXSW
static inline int task_running(struct rq *rq, struct task_struct *p)
{
	return task_current(rq, p);
}

static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
{
}

static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
{
#ifdef CONFIG_DEBUG_SPINLOCK
	/* this is a valid case when another task releases the spinlock */
	rq->lock.owner = current;
#endif
	/*
	 * If we are tracking spinlock dependencies then we have to
	 * fix up the runqueue lock - which gets 'carried over' from
	 * prev into current:
	 */
	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);

	spin_unlock_irq(&rq->lock);
}

#else /* __ARCH_WANT_UNLOCKED_CTXSW */
static inline int task_running(struct rq *rq, struct task_struct *p)
{
#ifdef CONFIG_SMP
	return p->oncpu;
#else
	return task_current(rq, p);
#endif
}

static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
{
#ifdef CONFIG_SMP
	/*
	 * We can optimise this out completely for !SMP, because the
	 * SMP rebalancing from interrupt is the only thing that cares
	 * here.
	 */
	next->oncpu = 1;
#endif
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	spin_unlock_irq(&rq->lock);
#else
	spin_unlock(&rq->lock);
#endif
}

static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
{
#ifdef CONFIG_SMP
	/*
	 * After ->oncpu is cleared, the task can be moved to a different CPU.
	 * We must ensure this doesn't happen until the switch is completely
	 * finished.
	 */
	smp_wmb();
	prev->oncpu = 0;
#endif
#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
	local_irq_enable();
#endif
}
#endif /* __ARCH_WANT_UNLOCKED_CTXSW */

/*
 * __task_rq_lock - lock the runqueue a given task resides on.
 * Must be called interrupts disabled.
 */
static inline struct rq *__task_rq_lock(struct task_struct *p)
	__acquires(rq->lock)
{
	for (;;) {
		struct rq *rq = task_rq(p);
		spin_lock(&rq->lock);
		if (likely(rq == task_rq(p)))
			return rq;
		spin_unlock(&rq->lock);
	}
}

/*
 * task_rq_lock - lock the runqueue a given task resides on and disable
 * interrupts. Note the ordering: we can safely lookup the task_rq without
 * explicitly disabling preemption.
 */
static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
	__acquires(rq->lock)
{
	struct rq *rq;

	for (;;) {
		local_irq_save(*flags);
		rq = task_rq(p);
		spin_lock(&rq->lock);
		if (likely(rq == task_rq(p)))
			return rq;
		spin_unlock_irqrestore(&rq->lock, *flags);
	}
}

void task_rq_unlock_wait(struct task_struct *p)
{
	struct rq *rq = task_rq(p);

	smp_mb(); /* spin-unlock-wait is not a full memory barrier */
	spin_unlock_wait(&rq->lock);
}

static void __task_rq_unlock(struct rq *rq)
	__releases(rq->lock)
{
	spin_unlock(&rq->lock);
}

static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
	__releases(rq->lock)
{
	spin_unlock_irqrestore(&rq->lock, *flags);
}

/*
 * this_rq_lock - lock this runqueue and disable interrupts.
 */
static struct rq *this_rq_lock(void)
	__acquires(rq->lock)
{
	struct rq *rq;

	local_irq_disable();
	rq = this_rq();
	spin_lock(&rq->lock);

	return rq;
}

#ifdef CONFIG_SCHED_HRTICK
/*
 * Use HR-timers to deliver accurate preemption points.
 *
 * Its all a bit involved since we cannot program an hrt while holding the
 * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
 * reschedule event.
 *
 * When we get rescheduled we reprogram the hrtick_timer outside of the
 * rq->lock.
 */

/*
 * Use hrtick when:
 *  - enabled by features
 *  - hrtimer is actually high res
 */
static inline int hrtick_enabled(struct rq *rq)
{
	if (!sched_feat(HRTICK))
		return 0;
	if (!cpu_active(cpu_of(rq)))
		return 0;
	return hrtimer_is_hres_active(&rq->hrtick_timer);
}

static void hrtick_clear(struct rq *rq)
{
	if (hrtimer_active(&rq->hrtick_timer))
		hrtimer_cancel(&rq->hrtick_timer);
}

/*
 * High-resolution timer tick.
 * Runs from hardirq context with interrupts disabled.
 */
static enum hrtimer_restart hrtick(struct hrtimer *timer)
{
	struct rq *rq = container_of(timer, struct rq, hrtick_timer);

	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());

	spin_lock(&rq->lock);
	update_rq_clock(rq);
	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
	spin_unlock(&rq->lock);

	return HRTIMER_NORESTART;
}

#ifdef CONFIG_SMP
/*
 * called from hardirq (IPI) context
 */
static void __hrtick_start(void *arg)
{
	struct rq *rq = arg;

	spin_lock(&rq->lock);
	hrtimer_restart(&rq->hrtick_timer);
	rq->hrtick_csd_pending = 0;
	spin_unlock(&rq->lock);
}

/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
static void hrtick_start(struct rq *rq, u64 delay)
{
	struct hrtimer *timer = &rq->hrtick_timer;
	ktime_t time = ktime_add_ns(timer->base->get_time(), delay);

	hrtimer_set_expires(timer, time);

	if (rq == this_rq()) {
		hrtimer_restart(timer);
	} else if (!rq->hrtick_csd_pending) {
		__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd);
		rq->hrtick_csd_pending = 1;
	}
}

static int
hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
{
	int cpu = (int)(long)hcpu;

	switch (action) {
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
		hrtick_clear(cpu_rq(cpu));
		return NOTIFY_OK;
	}

	return NOTIFY_DONE;
}

static __init void init_hrtick(void)
{
	hotcpu_notifier(hotplug_hrtick, 0);
}
#else
/*
 * Called to set the hrtick timer state.
 *
 * called with rq->lock held and irqs disabled
 */
static void hrtick_start(struct rq *rq, u64 delay)
{
	hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL);
}

static inline void init_hrtick(void)
{
}
#endif /* CONFIG_SMP */

static void init_rq_hrtick(struct rq *rq)
{
#ifdef CONFIG_SMP
	rq->hrtick_csd_pending = 0;

	rq->hrtick_csd.flags = 0;
	rq->hrtick_csd.func = __hrtick_start;
	rq->hrtick_csd.info = rq;
#endif

	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rq->hrtick_timer.function = hrtick;
	rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
}
#else	/* CONFIG_SCHED_HRTICK */
static inline void hrtick_clear(struct rq *rq)
{
}

static inline void init_rq_hrtick(struct rq *rq)
{
}

static inline void init_hrtick(void)
{
}
#endif	/* CONFIG_SCHED_HRTICK */

/*
 * resched_task - mark a task 'to be rescheduled now'.
 *
 * On UP this means the setting of the need_resched flag, on SMP it
 * might also involve a cross-CPU call to trigger the scheduler on
 * the target CPU.
 */
#ifdef CONFIG_SMP

#ifndef tsk_is_polling
#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
#endif

static void resched_task(struct task_struct *p)
{
	int cpu;

	assert_spin_locked(&task_rq(p)->lock);

	if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
		return;

	set_tsk_thread_flag(p, TIF_NEED_RESCHED);

	cpu = task_cpu(p);
	if (cpu == smp_processor_id())
		return;

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(p))
		smp_send_reschedule(cpu);
}

static void resched_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);
	unsigned long flags;

	if (!spin_trylock_irqsave(&rq->lock, flags))
		return;
	resched_task(cpu_curr(cpu));
	spin_unlock_irqrestore(&rq->lock, flags);
}

#ifdef CONFIG_NO_HZ
/*
 * When add_timer_on() enqueues a timer into the timer wheel of an
 * idle CPU then this timer might expire before the next timer event
 * which is scheduled to wake up that CPU. In case of a completely
 * idle system the next event might even be infinite time into the
 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
 * leaves the inner idle loop so the newly added timer is taken into
 * account when the CPU goes back to idle and evaluates the timer
 * wheel for the next timer event.
 */
void wake_up_idle_cpu(int cpu)
{
	struct rq *rq = cpu_rq(cpu);

	if (cpu == smp_processor_id())
		return;

	/*
	 * This is safe, as this function is called with the timer
	 * wheel base lock of (cpu) held. When the CPU is on the way
	 * to idle and has not yet set rq->curr to idle then it will
	 * be serialized on the timer wheel base lock and take the new
	 * timer into account automatically.
	 */
	if (rq->curr != rq->idle)
		return;

	/*
	 * We can set TIF_RESCHED on the idle task of the other CPU
	 * lockless. The worst case is that the other CPU runs the
	 * idle task through an additional NOOP schedule()
	 */
	set_tsk_thread_flag(rq->idle, TIF_NEED_RESCHED);

	/* NEED_RESCHED must be visible before we test polling */
	smp_mb();
	if (!tsk_is_polling(rq->idle))
		smp_send_reschedule(cpu);
}
#endif /* CONFIG_NO_HZ */

#else /* !CONFIG_SMP */
static void resched_task(struct task_struct *p)
{
	assert_spin_locked(&task_rq(p)->lock);
	set_tsk_need_resched(p);
}
#endif /* CONFIG_SMP */

#if BITS_PER_LONG == 32
# define WMULT_CONST	(~0UL)
#else
# define WMULT_CONST	(1UL << 32)
#endif

#define WMULT_SHIFT	32

/*
 * Shift right and round:
 */
#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))

/*
 * delta *= weight / lw
 */
static unsigned long
calc_delta_mine(unsigned long delta_exec, unsigned long weight,
		struct load_weight *lw)
{
	u64 tmp;

	if (!lw->inv_weight) {
		if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
			lw->inv_weight = 1;
		else
			lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
				/ (lw->weight+1);
	}

	tmp = (u64)delta_exec * weight;
	/*
	 * Check whether we'd overflow the 64-bit multiplication:
	 */
	if (unlikely(tmp > WMULT_CONST))
		tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
			WMULT_SHIFT/2);
	else
		tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);

	return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
}

static inline void update_load_add(struct load_weight *lw, unsigned long inc)
{
	lw->weight += inc;
	lw->inv_weight = 0;
}

static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
{
	lw->weight -= dec;
	lw->inv_weight = 0;
}

/*
 * To aid in avoiding the subversion of "niceness" due to uneven distribution
 * of tasks with abnormal "nice" values across CPUs the contribution that
 * each task makes to its run queue's load is weighted according to its
 * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
 * scaled version of the new time slice allocation that they receive on time
 * slice expiry etc.
 */

#define WEIGHT_IDLEPRIO		2
#define WMULT_IDLEPRIO		(1 << 31)

/*
 * Nice levels are multiplicative, with a gentle 10% change for every
 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
 * nice 1, it will get ~10% less CPU time than another CPU-bound task
 * that remained on nice 0.
 *
 * The "10% effect" is relative and cumulative: from _any_ nice level,
 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
 * If a task goes up by ~10% and another task goes down by ~10% then
 * the relative distance between them is ~25%.)
 */
static const int prio_to_weight[40] = {
 /* -20 */     88761,     71755,     56483,     46273,     36291,
 /* -15 */     29154,     23254,     18705,     14949,     11916,
 /* -10 */      9548,      7620,      6100,      4904,      3906,
 /*  -5 */      3121,      2501,      1991,      1586,      1277,
 /*   0 */      1024,       820,       655,       526,       423,
 /*   5 */       335,       272,       215,       172,       137,