@ -10,6 +10,7 @@
# include <linux/list_lru.h>
# include <linux/slab.h>
# include <linux/mutex.h>
# include <linux/memcontrol.h>
# ifdef CONFIG_MEMCG_KMEM
static LIST_HEAD ( list_lrus ) ;
@ -38,16 +39,71 @@ static void list_lru_unregister(struct list_lru *lru)
}
# endif /* CONFIG_MEMCG_KMEM */
# ifdef CONFIG_MEMCG_KMEM
static inline bool list_lru_memcg_aware ( struct list_lru * lru )
{
return ! ! lru - > node [ 0 ] . memcg_lrus ;
}
static inline struct list_lru_one *
list_lru_from_memcg_idx ( struct list_lru_node * nlru , int idx )
{
/*
* The lock protects the array of per cgroup lists from relocation
* ( see memcg_update_list_lru_node ) .
*/
lockdep_assert_held ( & nlru - > lock ) ;
if ( nlru - > memcg_lrus & & idx > = 0 )
return nlru - > memcg_lrus - > lru [ idx ] ;
return & nlru - > lru ;
}
static inline struct list_lru_one *
list_lru_from_kmem ( struct list_lru_node * nlru , void * ptr )
{
struct mem_cgroup * memcg ;
if ( ! nlru - > memcg_lrus )
return & nlru - > lru ;
memcg = mem_cgroup_from_kmem ( ptr ) ;
if ( ! memcg )
return & nlru - > lru ;
return list_lru_from_memcg_idx ( nlru , memcg_cache_id ( memcg ) ) ;
}
# else
static inline bool list_lru_memcg_aware ( struct list_lru * lru )
{
return false ;
}
static inline struct list_lru_one *
list_lru_from_memcg_idx ( struct list_lru_node * nlru , int idx )
{
return & nlru - > lru ;
}
static inline struct list_lru_one *
list_lru_from_kmem ( struct list_lru_node * nlru , void * ptr )
{
return & nlru - > lru ;
}
# endif /* CONFIG_MEMCG_KMEM */
bool list_lru_add ( struct list_lru * lru , struct list_head * item )
{
int nid = page_to_nid ( virt_to_page ( item ) ) ;
struct list_lru_node * nlru = & lru - > node [ nid ] ;
struct list_lru_one * l ;
spin_lock ( & nlru - > lock ) ;
WARN_ON_ONCE ( nlru - > nr_items < 0 ) ;
l = list_lru_from_kmem ( nlru , item ) ;
WARN_ON_ONCE ( l - > nr_items < 0 ) ;
if ( list_empty ( item ) ) {
list_add_tail ( item , & nlru - > list ) ;
nlru - > nr_items + + ;
list_add_tail ( item , & l - > list ) ;
l - > nr_items + + ;
spin_unlock ( & nlru - > lock ) ;
return true ;
}
@ -60,12 +116,14 @@ bool list_lru_del(struct list_lru *lru, struct list_head *item)
{
int nid = page_to_nid ( virt_to_page ( item ) ) ;
struct list_lru_node * nlru = & lru - > node [ nid ] ;
struct list_lru_one * l ;
spin_lock ( & nlru - > lock ) ;
l = list_lru_from_kmem ( nlru , item ) ;
if ( ! list_empty ( item ) ) {
list_del_init ( item ) ;
n lru - > nr_items - - ;
WARN_ON_ONCE ( n lru - > nr_items < 0 ) ;
l - > nr_items - - ;
WARN_ON_ONCE ( l - > nr_items < 0 ) ;
spin_unlock ( & nlru - > lock ) ;
return true ;
}
@ -74,33 +132,58 @@ bool list_lru_del(struct list_lru *lru, struct list_head *item)
}
EXPORT_SYMBOL_GPL ( list_lru_del ) ;
unsigned long
list_lru_count_node ( struct list_lru * lru , int nid )
static unsigned long __list_lru_count_one ( struct list_lru * lru ,
int nid , int memcg_idx )
{
unsigned long count = 0 ;
struct list_lru_node * nlru = & lru - > node [ nid ] ;
struct list_lru_one * l ;
unsigned long count ;
spin_lock ( & nlru - > lock ) ;
WARN_ON_ONCE ( nlru - > nr_items < 0 ) ;
count + = nlru - > nr_items ;
l = list_lru_from_memcg_idx ( nlru , memcg_idx ) ;
WARN_ON_ONCE ( l - > nr_items < 0 ) ;
count = l - > nr_items ;
spin_unlock ( & nlru - > lock ) ;
return count ;
}
unsigned long list_lru_count_one ( struct list_lru * lru ,
int nid , struct mem_cgroup * memcg )
{
return __list_lru_count_one ( lru , nid , memcg_cache_id ( memcg ) ) ;
}
EXPORT_SYMBOL_GPL ( list_lru_count_one ) ;
unsigned long list_lru_count_node ( struct list_lru * lru , int nid )
{
long count = 0 ;
int memcg_idx ;
count + = __list_lru_count_one ( lru , nid , - 1 ) ;
if ( list_lru_memcg_aware ( lru ) ) {
for_each_memcg_cache_index ( memcg_idx )
count + = __list_lru_count_one ( lru , nid , memcg_idx ) ;
}
return count ;
}
EXPORT_SYMBOL_GPL ( list_lru_count_node ) ;
unsigned long
list_lru_walk_node ( struct list_lru * lru , int nid , list_lru_walk_cb isolate ,
void * cb_arg , unsigned long * nr_to_walk )
static unsigned long
__list_lru_walk_one ( struct list_lru * lru , int nid , int memcg_idx ,
list_lru_walk_cb isolate , void * cb_arg ,
unsigned long * nr_to_walk )
{
struct list_lru_node * nlru = & lru - > node [ nid ] ;
struct list_lru_node * nlru = & lru - > node [ nid ] ;
struct list_lru_one * l ;
struct list_head * item , * n ;
unsigned long isolated = 0 ;
spin_lock ( & nlru - > lock ) ;
l = list_lru_from_memcg_idx ( nlru , memcg_idx ) ;
restart :
list_for_each_safe ( item , n , & nlru - > list ) {
list_for_each_safe ( item , n , & l - > list ) {
enum lru_status ret ;
/*
@ -116,8 +199,8 @@ restart:
case LRU_REMOVED_RETRY :
assert_spin_locked ( & nlru - > lock ) ;
case LRU_REMOVED :
n lru - > nr_items - - ;
WARN_ON_ONCE ( n lru - > nr_items < 0 ) ;
l - > nr_items - - ;
WARN_ON_ONCE ( l - > nr_items < 0 ) ;
isolated + + ;
/*
* If the lru lock has been dropped , our list
@ -128,7 +211,7 @@ restart:
goto restart ;
break ;
case LRU_ROTATE :
list_move_tail ( item , & n lru - > list ) ;
list_move_tail ( item , & l - > list ) ;
break ;
case LRU_SKIP :
break ;
@ -147,36 +230,279 @@ restart:
spin_unlock ( & nlru - > lock ) ;
return isolated ;
}
unsigned long
list_lru_walk_one ( struct list_lru * lru , int nid , struct mem_cgroup * memcg ,
list_lru_walk_cb isolate , void * cb_arg ,
unsigned long * nr_to_walk )
{
return __list_lru_walk_one ( lru , nid , memcg_cache_id ( memcg ) ,
isolate , cb_arg , nr_to_walk ) ;
}
EXPORT_SYMBOL_GPL ( list_lru_walk_one ) ;
unsigned long list_lru_walk_node ( struct list_lru * lru , int nid ,
list_lru_walk_cb isolate , void * cb_arg ,
unsigned long * nr_to_walk )
{
long isolated = 0 ;
int memcg_idx ;
isolated + = __list_lru_walk_one ( lru , nid , - 1 , isolate , cb_arg ,
nr_to_walk ) ;
if ( * nr_to_walk > 0 & & list_lru_memcg_aware ( lru ) ) {
for_each_memcg_cache_index ( memcg_idx ) {
isolated + = __list_lru_walk_one ( lru , nid , memcg_idx ,
isolate , cb_arg , nr_to_walk ) ;
if ( * nr_to_walk < = 0 )
break ;
}
}
return isolated ;
}
EXPORT_SYMBOL_GPL ( list_lru_walk_node ) ;
int list_lru_init_key ( struct list_lru * lru , struct lock_class_key * key )
static void init_one_lru ( struct list_lru_one * l )
{
INIT_LIST_HEAD ( & l - > list ) ;
l - > nr_items = 0 ;
}
# ifdef CONFIG_MEMCG_KMEM
static void __memcg_destroy_list_lru_node ( struct list_lru_memcg * memcg_lrus ,
int begin , int end )
{
int i ;
for ( i = begin ; i < end ; i + + )
kfree ( memcg_lrus - > lru [ i ] ) ;
}
static int __memcg_init_list_lru_node ( struct list_lru_memcg * memcg_lrus ,
int begin , int end )
{
int i ;
for ( i = begin ; i < end ; i + + ) {
struct list_lru_one * l ;
l = kmalloc ( sizeof ( struct list_lru_one ) , GFP_KERNEL ) ;
if ( ! l )
goto fail ;
init_one_lru ( l ) ;
memcg_lrus - > lru [ i ] = l ;
}
return 0 ;
fail :
__memcg_destroy_list_lru_node ( memcg_lrus , begin , i - 1 ) ;
return - ENOMEM ;
}
static int memcg_init_list_lru_node ( struct list_lru_node * nlru )
{
int size = memcg_nr_cache_ids ;
nlru - > memcg_lrus = kmalloc ( size * sizeof ( void * ) , GFP_KERNEL ) ;
if ( ! nlru - > memcg_lrus )
return - ENOMEM ;
if ( __memcg_init_list_lru_node ( nlru - > memcg_lrus , 0 , size ) ) {
kfree ( nlru - > memcg_lrus ) ;
return - ENOMEM ;
}
return 0 ;
}
static void memcg_destroy_list_lru_node ( struct list_lru_node * nlru )
{
__memcg_destroy_list_lru_node ( nlru - > memcg_lrus , 0 , memcg_nr_cache_ids ) ;
kfree ( nlru - > memcg_lrus ) ;
}
static int memcg_update_list_lru_node ( struct list_lru_node * nlru ,
int old_size , int new_size )
{
struct list_lru_memcg * old , * new ;
BUG_ON ( old_size > new_size ) ;
old = nlru - > memcg_lrus ;
new = kmalloc ( new_size * sizeof ( void * ) , GFP_KERNEL ) ;
if ( ! new )
return - ENOMEM ;
if ( __memcg_init_list_lru_node ( new , old_size , new_size ) ) {
kfree ( new ) ;
return - ENOMEM ;
}
memcpy ( new , old , old_size * sizeof ( void * ) ) ;
/*
* The lock guarantees that we won ' t race with a reader
* ( see list_lru_from_memcg_idx ) .
*
* Since list_lru_ { add , del } may be called under an IRQ - safe lock ,
* we have to use IRQ - safe primitives here to avoid deadlock .
*/
spin_lock_irq ( & nlru - > lock ) ;
nlru - > memcg_lrus = new ;
spin_unlock_irq ( & nlru - > lock ) ;
kfree ( old ) ;
return 0 ;
}
static void memcg_cancel_update_list_lru_node ( struct list_lru_node * nlru ,
int old_size , int new_size )
{
/* do not bother shrinking the array back to the old size, because we
* cannot handle allocation failures here */
__memcg_destroy_list_lru_node ( nlru - > memcg_lrus , old_size , new_size ) ;
}
static int memcg_init_list_lru ( struct list_lru * lru , bool memcg_aware )
{
int i ;
for ( i = 0 ; i < nr_node_ids ; i + + ) {
if ( ! memcg_aware )
lru - > node [ i ] . memcg_lrus = NULL ;
else if ( memcg_init_list_lru_node ( & lru - > node [ i ] ) )
goto fail ;
}
return 0 ;
fail :
for ( i = i - 1 ; i > = 0 ; i - - )
memcg_destroy_list_lru_node ( & lru - > node [ i ] ) ;
return - ENOMEM ;
}
static void memcg_destroy_list_lru ( struct list_lru * lru )
{
int i ;
if ( ! list_lru_memcg_aware ( lru ) )
return ;
for ( i = 0 ; i < nr_node_ids ; i + + )
memcg_destroy_list_lru_node ( & lru - > node [ i ] ) ;
}
static int memcg_update_list_lru ( struct list_lru * lru ,
int old_size , int new_size )
{
int i ;
if ( ! list_lru_memcg_aware ( lru ) )
return 0 ;
for ( i = 0 ; i < nr_node_ids ; i + + ) {
if ( memcg_update_list_lru_node ( & lru - > node [ i ] ,
old_size , new_size ) )
goto fail ;
}
return 0 ;
fail :
for ( i = i - 1 ; i > = 0 ; i - - )
memcg_cancel_update_list_lru_node ( & lru - > node [ i ] ,
old_size , new_size ) ;
return - ENOMEM ;
}
static void memcg_cancel_update_list_lru ( struct list_lru * lru ,
int old_size , int new_size )
{
int i ;
if ( ! list_lru_memcg_aware ( lru ) )
return ;
for ( i = 0 ; i < nr_node_ids ; i + + )
memcg_cancel_update_list_lru_node ( & lru - > node [ i ] ,
old_size , new_size ) ;
}
int memcg_update_all_list_lrus ( int new_size )
{
int ret = 0 ;
struct list_lru * lru ;
int old_size = memcg_nr_cache_ids ;
mutex_lock ( & list_lrus_mutex ) ;
list_for_each_entry ( lru , & list_lrus , list ) {
ret = memcg_update_list_lru ( lru , old_size , new_size ) ;
if ( ret )
goto fail ;
}
out :
mutex_unlock ( & list_lrus_mutex ) ;
return ret ;
fail :
list_for_each_entry_continue_reverse ( lru , & list_lrus , list )
memcg_cancel_update_list_lru ( lru , old_size , new_size ) ;
goto out ;
}
# else
static int memcg_init_list_lru ( struct list_lru * lru , bool memcg_aware )
{
return 0 ;
}
static void memcg_destroy_list_lru ( struct list_lru * lru )
{
}
# endif /* CONFIG_MEMCG_KMEM */
int __list_lru_init ( struct list_lru * lru , bool memcg_aware ,
struct lock_class_key * key )
{
int i ;
size_t size = sizeof ( * lru - > node ) * nr_node_ids ;
int err = - ENOMEM ;
memcg_get_cache_ids ( ) ;
lru - > node = kzalloc ( size , GFP_KERNEL ) ;
if ( ! lru - > node )
return - ENOMEM ;
goto out ;
for ( i = 0 ; i < nr_node_ids ; i + + ) {
spin_lock_init ( & lru - > node [ i ] . lock ) ;
if ( key )
lockdep_set_class ( & lru - > node [ i ] . lock , key ) ;
INIT_LIST_HEAD ( & lru - > node [ i ] . list ) ;
lru - > node [ i ] . nr_items = 0 ;
init_one_lru ( & lru - > node [ i ] . lru ) ;
}
err = memcg_init_list_lru ( lru , memcg_aware ) ;
if ( err ) {
kfree ( lru - > node ) ;
goto out ;
}
list_lru_register ( lru ) ;
return 0 ;
out :
memcg_put_cache_ids ( ) ;
return err ;
}
EXPORT_SYMBOL_GPL ( list_lru_init_key ) ;
EXPORT_SYMBOL_GPL ( __ list_lru_init) ;
void list_lru_destroy ( struct list_lru * lru )
{
/* Already destroyed or not yet initialized? */
if ( ! lru - > node )
return ;
memcg_get_cache_ids ( ) ;
list_lru_unregister ( lru ) ;
memcg_destroy_list_lru ( lru ) ;
kfree ( lru - > node ) ;
lru - > node = NULL ;
memcg_put_cache_ids ( ) ;
}
EXPORT_SYMBOL_GPL ( list_lru_destroy ) ;
Vladimir Davydov
10 years ago
committed by