eta name='generator' content='cgit v1.2.2'/>
aboutsummaryrefslogblamecommitdiffstats
path: root/mm/filemap.c
blob: e1979fdca8055e8cae04d081606fdf2f16ba8224 (plain) (tree)
1
2
3
4
5
6
7
8
9
10
11










                                                               
                         

                           
                          
                      
                             
                              
                      







                            
                              


                           
                         
                                                              
                             
                             

                     
  

                                                                   
                                                            
 
















                                                                      
                                                    
                                                                      

                                                             
  
             
                                                               

              
                    
                                                                     




                                                                       

                                                                        
  
                     
                                                   

                                                     
                  


                                            
                                                                      
  
                                 
                                                  


                                                                 
                                                                   

                                                                  
                                                                  
                                                                  
                                                               
                                                               

                                                                         

                                                                  


   
                                                                    
                                                                       
                                                          
   
                                                


                                                      







                                                                      
                                                          
 

                                                            
                                                                     
                           
                                                   

                                                      
                                  











                                                                         

 








                                                                               

                                                      
                                        
 
                                  
 
                                            
                                           
                                       
                                             
                                             


                               



                                      
                                    
 



                      
                                             
 
                            


                                                          
   
                                                                               

                                                      
                                                                
                                            
  


                                                                  
                                                                            
                                                                           


                                                                              

                                                                           



                                        
                                        

                                     











                                                                     
                                                                            







                                                          
                                                                         
                                           


                                                                            
                                        
 



                                              








                                                                        
   



                                                                        
  

                                                                    
   

                                                                             
 

                                                       


                            
 
                                  


                               













                                                                               
                                                     













                                                           


                                       
                                                                     
                                                


                                                                    







                                                    
                                                               




                                                         
                    

                               











                                                                          
         
                   
 
                                      
 





                                                                  




                                                                              


                                                               
                    

                               



                                                                       

                                                                   


                                           
         
                   
 
                                            
 
   
















                                                                               




                                    












                                                              
                                              






                                                                            

                                                                         



                                         






                                           
                                                                




                                        
                                                                           

                                                                            
                                                                              
                                               
 


                                     
                                        

                                                          
                                                                     

                         
 
                                                              
                         



                                        
                                                   
                                                                             
                                     
                                           
                                                                   
                                                             

                                             
                                                                              
                                                             
                                                             

                                                 
                                         
              
                                                     
    

                     
                                        

                                                                           
                                                               
 

                
                                                                 

                                         

                   
                                         
 
                  
                                          
 


                          
                                          






                                                                         
                            
         
                                   
 
                                  

      





















                                                                        
                                                    



                                                    
                                                                         



                                                                              










                                                                    
   
                                                                       

                                                  














                                                                      
                                     






                                                                              

                                                                               
   
                                   
 


                                                  



                                      


                                                    


                                          





                                              




                                         


                                                                            
   
                                   


                                                       
                                                                      



                                                                              
                                           



                                                              
                                                                               
 
                                        
 


                                                                 











                                                                  
                                                  
                         











                                                         


         




                                                

                                                                         
   
                                                                         
 
                     

                          





                                                                    

                                    
                                                 







                                                                              
                 












                                                                       
    

                          

                    

                             
   
                                                         

                                        





                                                                         
                                                                          


                          
       
                                              
                                                  





                                                         
                 
                                                 
         

                    



                                                       


                                            












                                                                             
                                              
 
                          



                                              


                                                    







                                                                              




                                                 
                 
         

                    




















                                                                        





                                    


                        
                                                                           

                                  
                                                   

                                    
 
                                                 





                                                                             
                                                    

                                             
                          


                                                                              
                           
                                 
                 




                                                      
                                              


                                                 
 
                                  

                                      
         
 
                          


                   














                                                                            





                                    


                        
                                                                             

                                  

                                                           
                                    
                              
 
                                                 







                                                                             
                          


                                                                              
                           
                              
                 
 



                                                      
                                              



                                                 




                                                                      
                                                                         



                                                 
                                  

                                      
         

                          
 
                                     
 







                                                             
                                                                         
                                                                     



                                                                            





                                    


                        

                                                             

                                  
                                                   

                                    
 
                                                 







                                                                             
                          

                                                                        
                           
                              
                 




                                                      
                                              




                                                 

                                      
         
 
                          
 

                                                   
 

                   
                                  
 




                                                                             
                                                                         







                                                                         
                                                                    

                                                          

                   
                                       



                                         
                                                                         
                                                                            




                                         

                                      

















                                                                            
                          

 
   
                                                   




                                     
                                                    
                                                                      


                                                                   
   

                                                                 
 
                                                        
                                            
                                               



                                                                   
                                 
                  
 
                                          

                                                         


                                                                                   

                                  
                                  
                             

                                      
                               

                                                     
                            
                                                          
                                                 





                                                                   
                                                           
                                                       
                                                                   
                 



                                                                               
                                                
                                                         


                                                                          




                                                                              
        

























                                                                               








                                                                        

                                                                        
                   
                                                                 
















                                                                                
                                     







                                                          


                                                 
 
                           
                                                                  












                                                        





                                                                      


                                                                           




                                                          
                                            
                 

                                          


                                                         


                                                            
                                                                          





                                                                 
                                                                    

                                                    



                                          












                                                                         



                                                      
                 
                                                            

                                                                   
                                                 




                                               



                              


                                          
 
                                                             
                            
 














                                                                          
                                          

                                                                      
                                     



















                                                                   






































                                                                                
   
                                                          


                                               
                                     
  



                                                   

                                                                  


                                          
                              
                     
                                     

                  


                                                                             


                                                                   
                            




                                              



                                                  

                                                                            



                                                                              
                                         
                                                     











                                                                                


                                                    
                 

         
                       

                                             













                                                                                
 
                                 

                                                          







                                                                         
                 

                                   



                      

                                     
                 




                                                                               


                                                                            
                                                             


                                                        
                
 









                                                                               
 
                                         
 


                                            



                             














                                                              

                          
 
                                         

                                                                    


                       

                                                        








                                                             


                           
                                                    

                                                          
                                      
                                     



















                                                                         

                                                                       

 
   
                                                            

                                                               
  
                                                                 





                                                                              
                                                                   

                  
                                         


                                                        
                                    
                          
                     
                    
 
                                                                              
                           
                                       
 
          

                                                          
                                              
                                                               
                  

                                                                   
                   
                                                                     
                           


                                                              
                                                                  

                                     
                                                      



                                            

                                                                
                                            
         








                                                 
          

                                                                           
           
                                          

                                       



                                                     
                                                                              
                                       
                                  
                                         
                                       

         
                         
                                     
 




                                                                  
                                              














                                                                    

                                    

                  





                                                                  
                             
                                                     




                                          


                                                  
                                
 
                                                                          
                                            
                               


                             


                                                                          
                                                       

                                  
                                        






                                                





                                                                          
                                   
    
                                      



                                    
                                                         
                                        
                                               
                                                   













                                                                      























                                                                               
                                                                    
                                              
                                                                     

                                           
 
                          



                                             
                                                            

                                                
                                                                       



                                                 
                                                                   

                                            





                                                 


                    
                                                                     
                                              
                                                                     


                                           




                          
                                                                    
                         
                            















                                         
                                    
         
    


                                 





                                                                  
                                                                            










                                                                            
                                                                     



                                                                                           

                                     


















                                                                                     
                                                                      












                                                                                        




                                                            
                                                                            






                                                                            
                                              
                                                                     

                                           
                                                                                      
 

                               
                                                          

                                                                           
                                    





                                                           
                                                                   




                               
                                   
                              




                             
                                                                            
                                                                                 








                                                                       
                                  


                                                                    
                                                                             




                                                                              
                             


                      
                                              
















                                                                       
                                                                    







                                                                              
                                       
 
                                                       
 

                                 

                                       
                                  


                                                 
                                                   
 

                                                                        
                                                                       
                   
                                                                      
                                 
 


                                                               



                                                   
                                          




                                         
                                     

         
                                
 









                                                                         
 
                                                            

                                                            
 
                                          



                                                         
                                                          






                                                                   
                                         

  

                                                 
                                                           





                                                                                         
                                                   



                               





















                                                                         
















                                                                           







                                                                       
                   










                                                   


                              




                                    





                                                                           
                                                                
                                                                       







                                                                          
 

                                                                              


                                   








                                                                      

                                  



                                                                             

                                                        
 
                                                                                  






                                                                               
                                                              



                                                                      






                                                                          
                                 
                 
















                                                                             
                          


                                                                          

                                                
                            
         
    



                                         



                                                                           

                                                                       

                   
                       
                          
                              
 


                                               

                                       

                                              
                 
                           
 
                                                           

                            

                                                                           





                                         
      
                                   

                    
                                           
 






                                                                      






                                                                             


                                  





                                                                        



                                                                      














                                                                            
                                                                             



                                                                


                                                     




                                                                                
                                         







                                                                            
                                            












                                                                               



                                                         



                                                    










                                                                        



                                                        
                                                      
 
                                  

                                     

         



                                           




















                                                                               

                                          
                                                         


                                                             




                                


                                                                          



                       










                                                                               
                                     


                         


                                     


                                                                   




                                                                             







                                                                          













                                                                               
 





                                                                             
                                                                                  














                                                                              



                                         

                                        










                                                                         

                                                                           

                                          
                                                    
                    


                                    
                                    
                                    
                                                                          
                                      
 
                                             

                            
                                                         
                                       

                                  
                                  



                                      









                                                                             


                                                                              
                                                                           
                                                                                
  














                                                                   
/*
 *	linux/mm/filemap.c
 *
 * Copyright (C) 1994-1999  Linus Torvalds
 */

/*
 * This file handles the generic file mmap semantics used by
 * most "normal" filesystems (but you don't /have/ to use this:
 * the NFS filesystem used to do this differently, for example)
 */
#include <linux/export.h>
#include <linux/compiler.h>
#include <linux/fs.h>
#include <linux/uaccess.h>
#include <linux/aio.h>
#include <linux/capability.h>
#include <linux/kernel_stat.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/hash.h>
#include <linux/writeback.h>
#include <linux/backing-dev.h>
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/security.h>
#include <linux/cpuset.h>
#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
#include <linux/memcontrol.h>
#include <linux/cleancache.h>
#include "internal.h"

/*
 * FIXME: remove all knowledge of the buffer layer from the core VM
 */
#include <linux/buffer_head.h> /* for try_to_free_buffers */

#include <asm/mman.h>

/*
 * Shared mappings implemented 30.11.1994. It's not fully working yet,
 * though.
 *
 * Shared mappings now work. 15.8.1995  Bruno.
 *
 * finished 'unifying' the page and buffer cache and SMP-threaded the
 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
 *
 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
 */

/*
 * Lock ordering:
 *
 *  ->i_mmap_mutex		(truncate_pagecache)
 *    ->private_lock		(__free_pte->__set_page_dirty_buffers)
 *      ->swap_lock		(exclusive_swap_page, others)
 *        ->mapping->tree_lock
 *
 *  ->i_mutex
 *    ->i_mmap_mutex		(truncate->unmap_mapping_range)
 *
 *  ->mmap_sem
 *    ->i_mmap_mutex
 *      ->page_table_lock or pte_lock	(various, mainly in memory.c)
 *        ->mapping->tree_lock	(arch-dependent flush_dcache_mmap_lock)
 *
 *  ->mmap_sem
 *    ->lock_page		(access_process_vm)
 *
 *  ->i_mutex			(generic_file_buffered_write)
 *    ->mmap_sem		(fault_in_pages_readable->do_page_fault)
 *
 *  bdi->wb.list_lock
 *    sb_lock			(fs/fs-writeback.c)
 *    ->mapping->tree_lock	(__sync_single_inode)
 *
 *  ->i_mmap_mutex
 *    ->anon_vma.lock		(vma_adjust)
 *
 *  ->anon_vma.lock
 *    ->page_table_lock or pte_lock	(anon_vma_prepare and various)
 *
 *  ->page_table_lock or pte_lock
 *    ->swap_lock		(try_to_unmap_one)
 *    ->private_lock		(try_to_unmap_one)
 *    ->tree_lock		(try_to_unmap_one)
 *    ->zone.lru_lock		(follow_page->mark_page_accessed)
 *    ->zone.lru_lock		(check_pte_range->isolate_lru_page)
 *    ->private_lock		(page_remove_rmap->set_page_dirty)
 *    ->tree_lock		(page_remove_rmap->set_page_dirty)
 *    bdi.wb->list_lock		(page_remove_rmap->set_page_dirty)
 *    ->inode->i_lock		(page_remove_rmap->set_page_dirty)
 *    bdi.wb->list_lock		(zap_pte_range->set_page_dirty)
 *    ->inode->i_lock		(zap_pte_range->set_page_dirty)
 *    ->private_lock		(zap_pte_range->__set_page_dirty_buffers)
 *
 * ->i_mmap_mutex
 *   ->tasklist_lock            (memory_failure, collect_procs_ao)
 */

/*
 * Delete a page from the page cache and free it. Caller has to make
 * sure the page is locked and that nobody else uses it - or that usage
 * is safe.  The caller must hold the mapping's tree_lock.
 */
void __delete_from_page_cache(struct page *page)
{
	struct address_space *mapping = page->mapping;

	/*
	 * if we're uptodate, flush out into the cleancache, otherwise
	 * invalidate any existing cleancache entries.  We can't leave
	 * stale data around in the cleancache once our page is gone
	 */
	if (PageUptodate(page) && PageMappedToDisk(page))
		cleancache_put_page(page);
	else
		cleancache_invalidate_page(mapping, page);

	radix_tree_delete(&mapping->page_tree, page->index);
	page->mapping = NULL;
	/* Leave page->index set: truncation lookup relies upon it */
	mapping->nrpages--;
	__dec_zone_page_state(page, NR_FILE_PAGES);
	if (PageSwapBacked(page))
		__dec_zone_page_state(page, NR_SHMEM);
	BUG_ON(page_mapped(page));

	/*
	 * Some filesystems seem to re-dirty the page even after
	 * the VM has canceled the dirty bit (eg ext3 journaling).
	 *
	 * Fix it up by doing a final dirty accounting check after
	 * having removed the page entirely.
	 */
	if (PageDirty(page) && mapping_cap_account_dirty(mapping)) {
		dec_zone_page_state(page, NR_FILE_DIRTY);
		dec_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE);
	}
}

/**
 * delete_from_page_cache - delete page from page cache
 * @page: the page which the kernel is trying to remove from page cache
 *
 * This must be called only on pages that have been verified to be in the page
 * cache and locked.  It will never put the page into the free list, the caller
 * has a reference on the page.
 */
void delete_from_page_cache(struct page *page)
{
	struct address_space *mapping = page->mapping;
	void (*freepage)(struct page *);

	BUG_ON(!PageLocked(page));

	freepage = mapping->a_ops->freepage;
	spin_lock_irq(&mapping->tree_lock);
	__delete_from_page_cache(page);
	spin_unlock_irq(&mapping->tree_lock);
	mem_cgroup_uncharge_cache_page(page);

	if (freepage)
		freepage(page);
	page_cache_release(page);
}
EXPORT_SYMBOL(delete_from_page_cache);

static int sleep_on_page(void *word)
{
	io_schedule();
	return 0;
}

static int sleep_on_page_killable(void *word)
{
	sleep_on_page(word);
	return fatal_signal_pending(current) ? -EINTR : 0;
}

/**
 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
 * @mapping:	address space structure to write
 * @start:	offset in bytes where the range starts
 * @end:	offset in bytes where the range ends (inclusive)
 * @sync_mode:	enable synchronous operation
 *
 * Start writeback against all of a mapping's dirty pages that lie
 * within the byte offsets <start, end> inclusive.
 *
 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
 * opposed to a regular memory cleansing writeback.  The difference between
 * these two operations is that if a dirty page/buffer is encountered, it must
 * be waited upon, and not just skipped over.
 */
int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
				loff_t end, int sync_mode)
{
	int ret;
	struct writeback_control wbc = {
		.sync_mode = sync_mode,
		.nr_to_write = LONG_MAX,
		.range_start = start,
		.range_end = end,
	};

	if (!mapping_cap_writeback_dirty(mapping))
		return 0;

	ret = do_writepages(mapping, &wbc);
	return ret;
}

static inline int __filemap_fdatawrite(struct address_space *mapping,
	int sync_mode)
{
	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
}

int filemap_fdatawrite(struct address_space *mapping)
{
	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
}
EXPORT_SYMBOL(filemap_fdatawrite);

int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
				loff_t end)
{
	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
}
EXPORT_SYMBOL(filemap_fdatawrite_range);

/**
 * filemap_flush - mostly a non-blocking flush
 * @mapping:	target address_space
 *
 * This is a mostly non-blocking flush.  Not suitable for data-integrity
 * purposes - I/O may not be started against all dirty pages.
 */
int filemap_flush(struct address_space *mapping)
{
	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
}
EXPORT_SYMBOL(filemap_flush);

/**
 * filemap_fdatawait_range - wait for writeback to complete
 * @mapping:		address space structure to wait for
 * @start_byte:		offset in bytes where the range starts
 * @end_byte:		offset in bytes where the range ends (inclusive)
 *
 * Walk the list of under-writeback pages of the given address space
 * in the given range and wait for all of them.
 */
int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
			    loff_t end_byte)
{
	pgoff_t index = start_byte >> PAGE_CACHE_SHIFT;
	pgoff_t end = end_byte >> PAGE_CACHE_SHIFT;
	struct pagevec pvec;
	int nr_pages;
	int ret = 0;

	if (end_byte < start_byte)
		return 0;

	pagevec_init(&pvec, 0);
	while ((index <= end) &&
			(nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
			PAGECACHE_TAG_WRITEBACK,
			min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
		unsigned i;

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			/* until radix tree lookup accepts end_index */
			if (page->index > end)
				continue;

			wait_on_page_writeback(page);
			if (TestClearPageError(page))
				ret = -EIO;
		}
		pagevec_release(&pvec);
		cond_resched();
	}

	/* Check for outstanding write errors */
	if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))
		ret = -ENOSPC;
	if (test_and_clear_bit(AS_EIO, &mapping->flags))
		ret = -EIO;

	return ret;
}
EXPORT_SYMBOL(filemap_fdatawait_range);

/**
 * filemap_fdatawait - wait for all under-writeback pages to complete
 * @mapping: address space structure to wait for
 *
 * Walk the list of under-writeback pages of the given address space
 * and wait for all of them.
 */
int filemap_fdatawait(struct address_space *mapping)
{
	loff_t i_size = i_size_read(mapping->host);

	if (i_size == 0)
		return 0;

	return filemap_fdatawait_range(mapping, 0, i_size - 1);
}
EXPORT_SYMBOL(filemap_fdatawait);

int filemap_write_and_wait(struct address_space *mapping)
{
	int err = 0;

	if (mapping->nrpages) {
		err = filemap_fdatawrite(mapping);
		/*
		 * Even if the above returned error, the pages may be
		 * written partially (e.g. -ENOSPC), so we wait for it.
		 * But the -EIO is special case, it may indicate the worst
		 * thing (e.g. bug) happened, so we avoid waiting for it.
		 */
		if (err != -EIO) {
			int err2 = filemap_fdatawait(mapping);
			if (!err)
				err = err2;
		}
	}
	return err;
}
EXPORT_SYMBOL(filemap_write_and_wait);

/**
 * filemap_write_and_wait_range - write out & wait on a file range
 * @mapping:	the address_space for the pages
 * @lstart:	offset in bytes where the range starts
 * @lend:	offset in bytes where the range ends (inclusive)
 *
 * Write out and wait upon file offsets lstart->lend, inclusive.
 *
 * Note that `lend' is inclusive (describes the last byte to be written) so
 * that this function can be used to write to the very end-of-file (end = -1).
 */
int filemap_write_and_wait_range(struct address_space *mapping,
				 loff_t lstart, loff_t lend)
{
	int err = 0;

	if (mapping->nrpages) {
		err = __filemap_fdatawrite_range(mapping, lstart, lend,
						 WB_SYNC_ALL);
		/* See comment of filemap_write_and_wait() */
		if (err != -EIO) {
			int err2 = filemap_fdatawait_range(mapping,
						lstart, lend);
			if (!err)
				err = err2;
		}
	}
	return err;
}
EXPORT_SYMBOL(filemap_write_and_wait_range);

/**
 * replace_page_cache_page - replace a pagecache page with a new one
 * @old:	page to be replaced
 * @new:	page to replace with
 * @gfp_mask:	allocation mode
 *
 * This function replaces a page in the pagecache with a new one.  On
 * success it acquires the pagecache reference for the new page and
 * drops it for the old page.  Both the old and new pages must be
 * locked.  This function does not add the new page to the LRU, the
 * caller must do that.