@ -590,9 +590,8 @@ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
if ( ! vma - > anon_vma | | ! page__anon_vma | |
vma - > anon_vma - > root ! = page__anon_vma - > root )
return - EFAULT ;
} else if ( page - > mapping & & ! ( vma - > vm_flags & VM_NONLINEAR ) ) {
if ( ! vma - > vm_file | |
vma - > vm_file - > f_mapping ! = page - > mapping )
} else if ( page - > mapping ) {
if ( ! vma - > vm_file | | vma - > vm_file - > f_mapping ! = page - > mapping )
return - EFAULT ;
} else
return - EFAULT ;
@ -1274,7 +1273,6 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
if ( pte_soft_dirty ( pteval ) )
swp_pte = pte_swp_mksoft_dirty ( swp_pte ) ;
set_pte_at ( mm , address , pte , swp_pte ) ;
BUG_ON ( pte_file ( * pte ) ) ;
} else if ( IS_ENABLED ( CONFIG_MIGRATION ) & &
( flags & TTU_MIGRATION ) ) {
/* Establish migration entry for a file page */
@ -1316,211 +1314,6 @@ out_mlock:
return ret ;
}
/*
* objrmap doesn ' t work for nonlinear VMAs because the assumption that
* offset - into - file correlates with offset - into - virtual - addresses does not hold .
* Consequently , given a particular page and its - > index , we cannot locate the
* ptes which are mapping that page without an exhaustive linear search .
*
* So what this code does is a mini " virtual scan " of each nonlinear VMA which
* maps the file to which the target page belongs . The - > vm_private_data field
* holds the current cursor into that scan . Successive searches will circulate
* around the vma ' s virtual address space .
*
* So as more replacement pressure is applied to the pages in a nonlinear VMA ,
* more scanning pressure is placed against them as well . Eventually pages
* will become fully unmapped and are eligible for eviction .
*
* For very sparsely populated VMAs this is a little inefficient - chances are
* there there won ' t be many ptes located within the scan cluster . In this case
* maybe we could scan further - to the end of the pte page , perhaps .
*
* Mlocked pages : check VM_LOCKED under mmap_sem held for read , if we can
* acquire it without blocking . If vma locked , mlock the pages in the cluster ,
* rather than unmapping them . If we encounter the " check_page " that vmscan is
* trying to unmap , return SWAP_MLOCK , else default SWAP_AGAIN .
*/
# define CLUSTER_SIZE min(32*PAGE_SIZE, PMD_SIZE)
# define CLUSTER_MASK (~(CLUSTER_SIZE - 1))
static int try_to_unmap_cluster ( unsigned long cursor , unsigned int * mapcount ,
struct vm_area_struct * vma , struct page * check_page )
{
struct mm_struct * mm = vma - > vm_mm ;
pmd_t * pmd ;
pte_t * pte ;
pte_t pteval ;
spinlock_t * ptl ;
struct page * page ;
unsigned long address ;
unsigned long mmun_start ; /* For mmu_notifiers */
unsigned long mmun_end ; /* For mmu_notifiers */
unsigned long end ;
int ret = SWAP_AGAIN ;
int locked_vma = 0 ;
address = ( vma - > vm_start + cursor ) & CLUSTER_MASK ;
end = address + CLUSTER_SIZE ;
if ( address < vma - > vm_start )
address = vma - > vm_start ;
if ( end > vma - > vm_end )
end = vma - > vm_end ;
pmd = mm_find_pmd ( mm , address ) ;
if ( ! pmd )
return ret ;
mmun_start = address ;
mmun_end = end ;
mmu_notifier_invalidate_range_start ( mm , mmun_start , mmun_end ) ;
/*
* If we can acquire the mmap_sem for read , and vma is VM_LOCKED ,
* keep the sem while scanning the cluster for mlocking pages .
*/
if ( down_read_trylock ( & vma - > vm_mm - > mmap_sem ) ) {
locked_vma = ( vma - > vm_flags & VM_LOCKED ) ;
if ( ! locked_vma )
up_read ( & vma - > vm_mm - > mmap_sem ) ; /* don't need it */
}
pte = pte_offset_map_lock ( mm , pmd , address , & ptl ) ;
/* Update high watermark before we lower rss */
update_hiwater_rss ( mm ) ;
for ( ; address < end ; pte + + , address + = PAGE_SIZE ) {
if ( ! pte_present ( * pte ) )
continue ;
page = vm_normal_page ( vma , address , * pte ) ;
BUG_ON ( ! page | | PageAnon ( page ) ) ;
if ( locked_vma ) {
if ( page = = check_page ) {
/* we know we have check_page locked */
mlock_vma_page ( page ) ;
ret = SWAP_MLOCK ;
} else if ( trylock_page ( page ) ) {
/*
* If we can lock the page , perform mlock .
* Otherwise leave the page alone , it will be
* eventually encountered again later .
*/
mlock_vma_page ( page ) ;
unlock_page ( page ) ;
}
continue ; /* don't unmap */
}
/*
* No need for _notify because we ' re within an
* mmu_notifier_invalidate_range_ { start | end } scope .
*/
if ( ptep_clear_flush_young ( vma , address , pte ) )
continue ;
/* Nuke the page table entry. */
flush_cache_page ( vma , address , pte_pfn ( * pte ) ) ;
pteval = ptep_clear_flush_notify ( vma , address , pte ) ;
/* If nonlinear, store the file page offset in the pte. */
if ( page - > index ! = linear_page_index ( vma , address ) ) {
pte_t ptfile = pgoff_to_pte ( page - > index ) ;
if ( pte_soft_dirty ( pteval ) )
ptfile = pte_file_mksoft_dirty ( ptfile ) ;
set_pte_at ( mm , address , pte , ptfile ) ;
}
/* Move the dirty bit to the physical page now the pte is gone. */
if ( pte_dirty ( pteval ) )
set_page_dirty ( page ) ;
page_remove_rmap ( page ) ;
page_cache_release ( page ) ;
dec_mm_counter ( mm , MM_FILEPAGES ) ;
( * mapcount ) - - ;
}
pte_unmap_unlock ( pte - 1 , ptl ) ;
mmu_notifier_invalidate_range_end ( mm , mmun_start , mmun_end ) ;
if ( locked_vma )
up_read ( & vma - > vm_mm - > mmap_sem ) ;
return ret ;
}
static int try_to_unmap_nonlinear ( struct page * page ,
struct address_space * mapping , void * arg )
{
struct vm_area_struct * vma ;
int ret = SWAP_AGAIN ;
unsigned long cursor ;
unsigned long max_nl_cursor = 0 ;
unsigned long max_nl_size = 0 ;
unsigned int mapcount ;
list_for_each_entry ( vma ,
& mapping - > i_mmap_nonlinear , shared . nonlinear ) {
cursor = ( unsigned long ) vma - > vm_private_data ;
if ( cursor > max_nl_cursor )
max_nl_cursor = cursor ;
cursor = vma - > vm_end - vma - > vm_start ;
if ( cursor > max_nl_size )
max_nl_size = cursor ;
}
if ( max_nl_size = = 0 ) { /* all nonlinears locked or reserved ? */
return SWAP_FAIL ;
}
/*
* We don ' t try to search for this page in the nonlinear vmas ,
* and page_referenced wouldn ' t have found it anyway . Instead
* just walk the nonlinear vmas trying to age and unmap some .
* The mapcount of the page we came in with is irrelevant ,
* but even so use it as a guide to how hard we should try ?
*/
mapcount = page_mapcount ( page ) ;
if ( ! mapcount )
return ret ;
cond_resched ( ) ;
max_nl_size = ( max_nl_size + CLUSTER_SIZE - 1 ) & CLUSTER_MASK ;
if ( max_nl_cursor = = 0 )
max_nl_cursor = CLUSTER_SIZE ;
do {
list_for_each_entry ( vma ,
& mapping - > i_mmap_nonlinear , shared . nonlinear ) {
cursor = ( unsigned long ) vma - > vm_private_data ;
while ( cursor < max_nl_cursor & &
cursor < vma - > vm_end - vma - > vm_start ) {
if ( try_to_unmap_cluster ( cursor , & mapcount ,
vma , page ) = = SWAP_MLOCK )
ret = SWAP_MLOCK ;
cursor + = CLUSTER_SIZE ;
vma - > vm_private_data = ( void * ) cursor ;
if ( ( int ) mapcount < = 0 )
return ret ;
}
vma - > vm_private_data = ( void * ) max_nl_cursor ;
}
cond_resched ( ) ;
max_nl_cursor + = CLUSTER_SIZE ;
} while ( max_nl_cursor < = max_nl_size ) ;
/*
* Don ' t loop forever ( perhaps all the remaining pages are
* in locked vmas ) . Reset cursor on all unreserved nonlinear
* vmas , now forgetting on which ones it had fallen behind .
*/
list_for_each_entry ( vma , & mapping - > i_mmap_nonlinear , shared . nonlinear )
vma - > vm_private_data = NULL ;
return ret ;
}
bool is_vma_temporary_stack ( struct vm_area_struct * vma )
{
int maybe_stack = vma - > vm_flags & ( VM_GROWSDOWN | VM_GROWSUP ) ;
@ -1566,7 +1359,6 @@ int try_to_unmap(struct page *page, enum ttu_flags flags)
. rmap_one = try_to_unmap_one ,
. arg = ( void * ) flags ,
. done = page_not_mapped ,
. file_nonlinear = try_to_unmap_nonlinear ,
. anon_lock = page_lock_anon_vma_read ,
} ;
@ -1612,12 +1404,6 @@ int try_to_munlock(struct page *page)
. rmap_one = try_to_unmap_one ,
. arg = ( void * ) TTU_MUNLOCK ,
. done = page_not_mapped ,
/*
* We don ' t bother to try to find the munlocked page in
* nonlinears . It ' s costly . Instead , later , page reclaim logic
* may call try_to_unmap ( ) and recover PG_mlocked lazily .
*/
. file_nonlinear = NULL ,
. anon_lock = page_lock_anon_vma_read ,
} ;
@ -1748,13 +1534,6 @@ static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc)
goto done ;
}
if ( ! rwc - > file_nonlinear )
goto done ;
if ( list_empty ( & mapping - > i_mmap_nonlinear ) )
goto done ;
ret = rwc - > file_nonlinear ( page , mapping , rwc - > arg ) ;
done :
i_mmap_unlock_read ( mapping ) ;
return ret ;
Kirill A. Shutemov
10 years ago
committed by