@ -1,5 +1,5 @@
/*
* Userspace Probes ( UProbes )
* User - space Probes ( UProbes )
*
* This program is free software ; you can redistribute it and / or modify
* it under the terms of the GNU General Public License as published by
@ -29,24 +29,26 @@
# include <linux/rmap.h> /* anon_vma_prepare */
# include <linux/mmu_notifier.h> /* set_pte_at_notify */
# include <linux/swap.h> /* try_to_free_swap */
# include <linux/uprobes.h>
static struct rb_root uprobes_tree = RB_ROOT ;
static DEFINE_SPINLOCK ( uprobes_treelock ) ; /* serialize rbtree access */
# define UPROBES_HASH_SZ 13
/* serialize (un)register */
static struct mutex uprobes_mutex [ UPROBES_HASH_SZ ] ;
# define uprobes_hash(v) (&uprobes_mutex[((unsigned long)(v)) %\
UPROBES_HASH_SZ ] )
# define uprobes_hash(v) (&uprobes_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ] )
/* serialize uprobe->pending_list */
static struct mutex uprobes_mmap_mutex [ UPROBES_HASH_SZ ] ;
# define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) %\
UPROBES_HASH_SZ ] )
# define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
/*
* uprobe_events allows us to skip the mmap_ uprobeif there are no uprobe
* uprobe_events allows us to skip the uprobe_mmap if there are no uprobe
* events active at this time . Probably a fine grained per inode count is
* better ?
*/
@ -58,9 +60,9 @@ static atomic_t uprobe_events = ATOMIC_INIT(0);
* vm_area_struct wasnt recommended .
*/
struct vma_info {
struct list_head probe_list ;
struct mm_struct * mm ;
loff_t vaddr ;
struct list_head probe_list ;
struct mm_struct * mm ;
loff_t vaddr ;
} ;
/*
@ -79,8 +81,7 @@ static bool valid_vma(struct vm_area_struct *vma, bool is_register)
if ( ! is_register )
return true ;
if ( ( vma - > vm_flags & ( VM_READ | VM_WRITE | VM_EXEC | VM_SHARED ) ) = =
( VM_READ | VM_EXEC ) )
if ( ( vma - > vm_flags & ( VM_READ | VM_WRITE | VM_EXEC | VM_SHARED ) ) = = ( VM_READ | VM_EXEC ) )
return true ;
return false ;
@ -92,6 +93,7 @@ static loff_t vma_address(struct vm_area_struct *vma, loff_t offset)
vaddr = vma - > vm_start + offset ;
vaddr - = vma - > vm_pgoff < < PAGE_SHIFT ;
return vaddr ;
}
@ -105,8 +107,7 @@ static loff_t vma_address(struct vm_area_struct *vma, loff_t offset)
*
* Returns 0 on success , - EFAULT on failure .
*/
static int __replace_page ( struct vm_area_struct * vma , struct page * page ,
struct page * kpage )
static int __replace_page ( struct vm_area_struct * vma , struct page * page , struct page * kpage )
{
struct mm_struct * mm = vma - > vm_mm ;
pgd_t * pgd ;
@ -163,7 +164,7 @@ out:
*/
bool __weak is_bkpt_insn ( uprobe_opcode_t * insn )
{
return ( * insn = = UPROBES_BKPT_INSN ) ;
return * insn = = UPROBES_BKPT_INSN ;
}
/*
@ -203,6 +204,7 @@ static int write_opcode(struct mm_struct *mm, struct uprobe *uprobe,
ret = get_user_pages ( NULL , mm , vaddr , 1 , 0 , 0 , & old_page , & vma ) ;
if ( ret < = 0 )
return ret ;
ret = - EINVAL ;
/*
@ -239,6 +241,7 @@ static int write_opcode(struct mm_struct *mm, struct uprobe *uprobe,
vaddr_new = kmap_atomic ( new_page ) ;
memcpy ( vaddr_new , vaddr_old , PAGE_SIZE ) ;
/* poke the new insn in, ASSUMES we don't cross page boundary */
vaddr & = ~ PAGE_MASK ;
BUG_ON ( vaddr + uprobe_opcode_sz > PAGE_SIZE ) ;
@ -260,7 +263,8 @@ unlock_out:
page_cache_release ( new_page ) ;
put_out :
put_page ( old_page ) ; /* we did a get_page in the beginning */
put_page ( old_page ) ;
return ret ;
}
@ -276,8 +280,7 @@ put_out:
* For mm @ mm , read the opcode at @ vaddr and store it in @ opcode .
* Return 0 ( success ) or a negative errno .
*/
static int read_opcode ( struct mm_struct * mm , unsigned long vaddr ,
uprobe_opcode_t * opcode )
static int read_opcode ( struct mm_struct * mm , unsigned long vaddr , uprobe_opcode_t * opcode )
{
struct page * page ;
void * vaddr_new ;
@ -293,15 +296,18 @@ static int read_opcode(struct mm_struct *mm, unsigned long vaddr,
memcpy ( opcode , vaddr_new + vaddr , uprobe_opcode_sz ) ;
kunmap_atomic ( vaddr_new ) ;
unlock_page ( page ) ;
put_page ( page ) ; /* we did a get_user_pages in the beginning */
put_page ( page ) ;
return 0 ;
}
static int is_bkpt_at_addr ( struct mm_struct * mm , unsigned long vaddr )
{
uprobe_opcode_t opcode ;
int result = read_opcode ( mm , vaddr , & opcode ) ;
int result ;
result = read_opcode ( mm , vaddr , & opcode ) ;
if ( result )
return result ;
@ -320,11 +326,11 @@ static int is_bkpt_at_addr(struct mm_struct *mm, unsigned long vaddr)
* For mm @ mm , store the breakpoint instruction at @ vaddr .
* Return 0 ( success ) or a negative errno .
*/
int __weak set_bkpt ( struct mm_struct * mm , struct uprobe * uprobe ,
unsigned long vaddr )
int __weak set_bkpt ( struct mm_struct * mm , struct uprobe * uprobe , unsigned long vaddr )
{
int result = is_bkpt_at_addr ( mm , vaddr ) ;
int result ;
result = is_bkpt_at_addr ( mm , vaddr ) ;
if ( result = = 1 )
return - EEXIST ;
@ -344,35 +350,35 @@ int __weak set_bkpt(struct mm_struct *mm, struct uprobe *uprobe,
* For mm @ mm , restore the original opcode ( opcode ) at @ vaddr .
* Return 0 ( success ) or a negative errno .
*/
int __weak set_orig_insn ( struct mm_struct * mm , struct uprobe * uprobe ,
unsigned long vaddr , bool verify )
int __weak
set_orig_insn ( struct mm_struct * mm , struct uprobe * uprobe , unsigned long vaddr , bool verify )
{
if ( verify ) {
int result = is_bkpt_at_addr ( mm , vaddr ) ;
int result ;
result = is_bkpt_at_addr ( mm , vaddr ) ;
if ( ! result )
return - EINVAL ;
if ( result ! = 1 )
return result ;
}
return write_opcode ( mm , uprobe , vaddr ,
* ( uprobe_opcode_t * ) uprobe - > insn ) ;
return write_opcode ( mm , uprobe , vaddr , * ( uprobe_opcode_t * ) uprobe - > insn ) ;
}
static int match_uprobe ( struct uprobe * l , struct uprobe * r )
{
if ( l - > inode < r - > inode )
return - 1 ;
if ( l - > inode > r - > inode )
return 1 ;
else {
if ( l - > offset < r - > offset )
return - 1 ;
if ( l - > offset > r - > offset )
return 1 ;
}
if ( l - > offset < r - > offset )
return - 1 ;
if ( l - > offset > r - > offset )
return 1 ;
return 0 ;
}
@ -391,6 +397,7 @@ static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
atomic_inc ( & uprobe - > ref ) ;
return uprobe ;
}
if ( match < 0 )
n = n - > rb_left ;
else
@ -411,6 +418,7 @@ static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
spin_lock_irqsave ( & uprobes_treelock , flags ) ;
uprobe = __find_uprobe ( inode , offset ) ;
spin_unlock_irqrestore ( & uprobes_treelock , flags ) ;
return uprobe ;
}
@ -436,16 +444,18 @@ static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
p = & parent - > rb_right ;
}
u = NULL ;
rb_link_node ( & uprobe - > rb_node , parent , p ) ;
rb_insert_color ( & uprobe - > rb_node , & uprobes_tree ) ;
/* get access + creation ref */
atomic_set ( & uprobe - > ref , 2 ) ;
return u ;
}
/*
* Acquires uprobes_treelock .
* Acquire uprobes_treelock .
* Matching uprobe already exists in rbtree ;
* increment ( access refcount ) and return the matching uprobe .
*
@ -460,6 +470,7 @@ static struct uprobe *insert_uprobe(struct uprobe *uprobe)
spin_lock_irqsave ( & uprobes_treelock , flags ) ;
u = __insert_uprobe ( uprobe ) ;
spin_unlock_irqrestore ( & uprobes_treelock , flags ) ;
return u ;
}
@ -490,19 +501,22 @@ static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
kfree ( uprobe ) ;
uprobe = cur_uprobe ;
iput ( inode ) ;
} else
} else {
atomic_inc ( & uprobe_events ) ;
}
return uprobe ;
}
/* Returns the previous consumer */
static struct uprobe_consumer * add_consumer ( struct uprobe * uprobe ,
struct uprobe_consumer * consumer )
static struct uprobe_consumer *
consumer_add ( struct uprobe * uprobe , struct uprobe_consumer * consumer )
{
down_write ( & uprobe - > consumer_rwsem ) ;
consumer - > next = uprobe - > consumers ;
uprobe - > consumers = consumer ;
up_write ( & uprobe - > consumer_rwsem ) ;
return consumer - > next ;
}
@ -511,8 +525,7 @@ static struct uprobe_consumer *add_consumer(struct uprobe *uprobe,
* Return true if the @ consumer is deleted successfully
* or return false .
*/
static bool del_consumer ( struct uprobe * uprobe ,
struct uprobe_consumer * consumer )
static bool consumer_del ( struct uprobe * uprobe , struct uprobe_consumer * consumer )
{
struct uprobe_consumer * * con ;
bool ret = false ;
@ -526,6 +539,7 @@ static bool del_consumer(struct uprobe *uprobe,
}
}
up_write ( & uprobe - > consumer_rwsem ) ;
return ret ;
}
@ -557,15 +571,15 @@ static int __copy_insn(struct address_space *mapping,
memcpy ( insn , vaddr + off1 , nbytes ) ;
kunmap_atomic ( vaddr ) ;
page_cache_release ( page ) ;
return 0 ;
}
static int copy_insn ( struct uprobe * uprobe , struct vm_area_struct * vma ,
unsigned long addr )
static int copy_insn ( struct uprobe * uprobe , struct vm_area_struct * vma , unsigned long addr )
{
struct address_space * mapping ;
int bytes ;
unsigned long nbytes ;
int bytes ;
addr & = ~ PAGE_MASK ;
nbytes = PAGE_SIZE - addr ;
@ -605,6 +619,7 @@ static int install_breakpoint(struct mm_struct *mm, struct uprobe *uprobe,
return - EEXIST ;
addr = ( unsigned long ) vaddr ;
if ( ! ( uprobe - > flags & UPROBES_COPY_INSN ) ) {
ret = copy_insn ( uprobe , vma , addr ) ;
if ( ret )
@ -613,7 +628,7 @@ static int install_breakpoint(struct mm_struct *mm, struct uprobe *uprobe,
if ( is_bkpt_insn ( ( uprobe_opcode_t * ) uprobe - > insn ) )
return - EEXIST ;
ret = analyze_insn ( mm , uprobe ) ;
ret = arch_uprobes_a nalyze_insn ( mm , uprobe ) ;
if ( ret )
return ret ;
@ -624,8 +639,7 @@ static int install_breakpoint(struct mm_struct *mm, struct uprobe *uprobe,
return ret ;
}
static void remove_breakpoint ( struct mm_struct * mm , struct uprobe * uprobe ,
loff_t vaddr )
static void remove_breakpoint ( struct mm_struct * mm , struct uprobe * uprobe , loff_t vaddr )
{
set_orig_insn ( mm , uprobe , ( unsigned long ) vaddr , true ) ;
}
@ -649,9 +663,11 @@ static struct vma_info *__find_next_vma_info(struct list_head *head,
struct prio_tree_iter iter ;
struct vm_area_struct * vma ;
struct vma_info * tmpvi ;
loff_t vaddr ;
unsigned long pgoff = offset > > PAGE_SHIFT ;
unsigned long pgoff ;
int existing_vma ;
loff_t vaddr ;
pgoff = offset > > PAGE_SHIFT ;
vma_prio_tree_foreach ( vma , & iter , & mapping - > i_mmap , pgoff , pgoff ) {
if ( ! valid_vma ( vma , is_register ) )
@ -659,6 +675,7 @@ static struct vma_info *__find_next_vma_info(struct list_head *head,
existing_vma = 0 ;
vaddr = vma_address ( vma , offset ) ;
list_for_each_entry ( tmpvi , head , probe_list ) {
if ( tmpvi - > mm = = vma - > vm_mm & & tmpvi - > vaddr = = vaddr ) {
existing_vma = 1 ;
@ -670,14 +687,15 @@ static struct vma_info *__find_next_vma_info(struct list_head *head,
* Another vma needs a probe to be installed . However skip
* installing the probe if the vma is about to be unlinked .
*/
if ( ! existing_vma & &
atomic_inc_not_zero ( & vma - > vm_mm - > mm_users ) ) {
if ( ! existing_vma & & atomic_inc_not_zero ( & vma - > vm_mm - > mm_users ) ) {
vi - > mm = vma - > vm_mm ;
vi - > vaddr = vaddr ;
list_add ( & vi - > probe_list , head ) ;
return vi ;
}
}
return NULL ;
}
@ -685,11 +703,12 @@ static struct vma_info *__find_next_vma_info(struct list_head *head,
* Iterate in the rmap prio tree and find a vma where a probe has not
* yet been inserted .
*/
static struct vma_info * find_next_vma_info ( struct list_head * head ,
loff_t offset , struct address_space * mapping ,
bool is_register )
static struct vma_info *
find_next_vma_info ( struct list_head * head , loff_t offset , struct address_space * mapping ,
bool is_register )
{
struct vma_info * vi , * retvi ;
vi = kzalloc ( sizeof ( struct vma_info ) , GFP_KERNEL ) ;
if ( ! vi )
return ERR_PTR ( - ENOMEM ) ;
@ -700,6 +719,7 @@ static struct vma_info *find_next_vma_info(struct list_head *head,
if ( ! retvi )
kfree ( vi ) ;
return retvi ;
}
@ -711,16 +731,23 @@ static int register_for_each_vma(struct uprobe *uprobe, bool is_register)
struct vma_info * vi , * tmpvi ;
struct mm_struct * mm ;
loff_t vaddr ;
int ret = 0 ;
int ret ;
mapping = uprobe - > inode - > i_mapping ;
INIT_LIST_HEAD ( & try_list ) ;
while ( ( vi = find_next_vma_info ( & try_list , uprobe - > offset ,
mapping , is_register ) ) ! = NULL ) {
ret = 0 ;
for ( ; ; ) {
vi = find_next_vma_info ( & try_list , uprobe - > offset , mapping , is_register ) ;
if ( ! vi )
break ;
if ( IS_ERR ( vi ) ) {
ret = PTR_ERR ( vi ) ;
break ;
}
mm = vi - > mm ;
down_read ( & mm - > mmap_sem ) ;
vma = find_vma ( mm , ( unsigned long ) vi - > vaddr ) ;
@ -755,19 +782,21 @@ static int register_for_each_vma(struct uprobe *uprobe, bool is_register)
break ;
}
}
list_for_each_entry_safe ( vi , tmpvi , & try_list , probe_list ) {
list_del ( & vi - > probe_list ) ;
kfree ( vi ) ;
}
return ret ;
}
static int __register_uprobe ( struct uprobe * uprobe )
static int __uprobe_ register ( struct uprobe * uprobe )
{
return register_for_each_vma ( uprobe , true ) ;
}
static void __unregister_uprobe ( struct uprobe * uprobe )
static void __uprobe_u nregister ( struct uprobe * uprobe )
{
if ( ! register_for_each_vma ( uprobe , false ) )
delete_uprobe ( uprobe ) ;
@ -776,15 +805,15 @@ static void __unregister_uprobe(struct uprobe *uprobe)
}
/*
* register_uprobe - register a probe
* uprobe_ register- register a probe
* @ inode : the file in which the probe has to be placed .
* @ offset : offset from the start of the file .
* @ consumer : information on howto handle the probe . .
*
* Apart from the access refcount , register_uprobe ( ) takes a creation
* Apart from the access refcount , uprobe_ register( ) takes a creation
* refcount ( thro alloc_uprobe ) if and only if this @ uprobe is getting
* inserted into the rbtree ( i . e first consumer for a @ inode : @ offset
* tuple ) . Creation refcount stops unregister_uprobe from freeing the
* tuple ) . Creation refcount stops uprobe_u nregister from freeing the
* @ uprobe even before the register operation is complete . Creation
* refcount is released when the last @ consumer for the @ uprobe
* unregisters .
@ -792,28 +821,29 @@ static void __unregister_uprobe(struct uprobe *uprobe)
* Return errno if it cannot successully install probes
* else return 0 ( success )
*/
int register_uprobe ( struct inode * inode , loff_t offset ,
struct uprobe_consumer * consumer )
int uprobe_register ( struct inode * inode , loff_t offset , struct uprobe_consumer * consumer )
{
struct uprobe * uprobe ;
int ret = - EINVAL ;
int ret ;
if ( ! inode | | ! consumer | | consumer - > next )
return ret ;
return - EINVAL ;
if ( offset > i_size_read ( inode ) )
return ret ;
return - EINVAL ;
ret = 0 ;
mutex_lock ( uprobes_hash ( inode ) ) ;
uprobe = alloc_uprobe ( inode , offset ) ;
if ( uprobe & & ! add_consumer ( uprobe , consumer ) ) {
ret = __register_uprobe ( uprobe ) ;
if ( uprobe & & ! consumer_add ( uprobe , consumer ) ) {
ret = __uprobe_register ( uprobe ) ;
if ( ret ) {
uprobe - > consumers = NULL ;
__unregister_uprobe ( uprobe ) ;
} else
__uprobe_u nregister ( uprobe ) ;
} else {
uprobe - > flags | = UPROBES_RUN_HANDLER ;
}
}
mutex_unlock ( uprobes_hash ( inode ) ) ;
@ -823,15 +853,14 @@ int register_uprobe(struct inode *inode, loff_t offset,
}
/*
* unregister_uprobe - unregister a already registered probe .
* uprobe_u nregister - unregister a already registered probe .
* @ inode : the file in which the probe has to be removed .
* @ offset : offset from the start of the file .
* @ consumer : identify which probe if multiple probes are colocated .
*/
void unregister_uprobe ( struct inode * inode , loff_t offset ,
struct uprobe_consumer * consumer )
void uprobe_unregister ( struct inode * inode , loff_t offset , struct uprobe_consumer * consumer )
{
struct uprobe * uprobe = NULL ;
struct uprobe * uprobe ;
if ( ! inode | | ! consumer )
return ;
@ -841,15 +870,14 @@ void unregister_uprobe(struct inode *inode, loff_t offset,
return ;
mutex_lock ( uprobes_hash ( inode ) ) ;
if ( ! del_consumer ( uprobe , consumer ) )
goto unreg_out ;
if ( ! uprobe - > consumers ) {
__unregister_uprobe ( uprobe ) ;
uprobe - > flags & = ~ UPROBES_RUN_HANDLER ;
if ( consumer_del ( uprobe , consumer ) ) {
if ( ! uprobe - > consumers ) {
__uprobe_unregister ( uprobe ) ;
uprobe - > flags & = ~ UPROBES_RUN_HANDLER ;
}
}
unreg_out :
mutex_unlock ( uprobes_hash ( inode ) ) ;
if ( uprobe )
put_uprobe ( uprobe ) ;
@ -870,6 +898,7 @@ static struct rb_node *find_least_offset_node(struct inode *inode)
while ( n ) {
uprobe = rb_entry ( n , struct uprobe , rb_node ) ;
match = match_uprobe ( & u , uprobe ) ;
if ( uprobe - > inode = = inode )
close_node = n ;
@ -881,6 +910,7 @@ static struct rb_node *find_least_offset_node(struct inode *inode)
else
n = n - > rb_right ;
}
return close_node ;
}
@ -890,11 +920,13 @@ static struct rb_node *find_least_offset_node(struct inode *inode)
static void build_probe_list ( struct inode * inode , struct list_head * head )
{
struct uprobe * uprobe ;
struct rb_node * n ;
unsigned long flags ;
struct rb_node * n ;
spin_lock_irqsave ( & uprobes_treelock , flags ) ;
n = find_least_offset_node ( inode ) ;
for ( ; n ; n = rb_next ( n ) ) {
uprobe = rb_entry ( n , struct uprobe , rb_node ) ;
if ( uprobe - > inode ! = inode )
@ -903,6 +935,7 @@ static void build_probe_list(struct inode *inode, struct list_head *head)
list_add ( & uprobe - > pending_list , head ) ;
atomic_inc ( & uprobe - > ref ) ;
}
spin_unlock_irqrestore ( & uprobes_treelock , flags ) ;
}
@ -912,42 +945,44 @@ static void build_probe_list(struct inode *inode, struct list_head *head)
*
* Return - ve no if we fail to insert probes and we cannot
* bail - out .
* Return 0 otherwise . i . e :
* Return 0 otherwise . i . e :
*
* - successful insertion of probes
* - ( or ) no possible probes to be inserted .
* - ( or ) insertion of probes failed but we can bail - out .
*/
int mmap_ uprobe( struct vm_area_struct * vma )
int uprobe_mmap ( struct vm_area_struct * vma )
{
struct list_head tmp_list ;
struct uprobe * uprobe , * u ;
struct inode * inode ;
int ret = 0 ;
int ret ;
if ( ! atomic_read ( & uprobe_events ) | | ! valid_vma ( vma , true ) )
return ret ; /* Bail-out */
return 0 ;
inode = vma - > vm_file - > f_mapping - > host ;
if ( ! inode )
return ret ;
return 0 ;
INIT_LIST_HEAD ( & tmp_list ) ;
mutex_lock ( uprobes_mmap_hash ( inode ) ) ;
build_probe_list ( inode , & tmp_list ) ;
ret = 0 ;
list_for_each_entry_safe ( uprobe , u , & tmp_list , pending_list ) {
loff_t vaddr ;
list_del ( & uprobe - > pending_list ) ;
if ( ! ret ) {
vaddr = vma_address ( vma , uprobe - > offset ) ;
if ( vaddr < vma - > vm_start | | vaddr > = vma - > vm_end ) {
put_uprobe ( uprobe ) ;
continue ;
if ( vaddr > = vma - > vm_start & & vaddr < vma - > vm_end ) {
ret = install_breakpoint ( vma - > vm_mm , uprobe , vma , vaddr ) ;
/* Ignore double add: */
if ( ret = = - EEXIST )
ret = 0 ;
}
ret = install_breakpoint ( vma - > vm_mm , uprobe , vma ,
vaddr ) ;
if ( ret = = - EEXIST )
ret = 0 ;
}
put_uprobe ( uprobe ) ;
}
Ingo Molnar
13 years ago