@ -2,6 +2,7 @@
* Optimized memory copy routines .
*
* Copyright ( C ) 2004 Randolph Chung < tausq @ debian . org >
* Copyright ( C ) 2013 Helge Deller < deller @ gmx . de >
*
* This program is free software ; you can redistribute it and / or modify
* it under the terms of the GNU General Public License as published by
@ -153,17 +154,21 @@ static inline void prefetch_dst(const void *addr)
# define prefetch_dst(addr) do { } while(0)
# endif
# define PA_MEMCPY_OK 0
# define PA_MEMCPY_LOAD_ERROR 1
# define PA_MEMCPY_STORE_ERROR 2
/* Copy from a not-aligned src to an aligned dst, using shifts. Handles 4 words
* per loop . This code is derived from glibc .
*/
static inline unsigned long copy_dstaligned ( unsigned long dst , unsigned long src , unsigned long len , unsigned long o_dst , unsigned long o_src , unsigned long o_len )
static inline unsigned long copy_dstaligned ( unsigned long dst ,
unsigned long src , unsigned long len )
{
/* gcc complains that a2 and a3 may be uninitialized, but actually
* they cannot be . Initialize a2 / a3 to shut gcc up .
*/
register unsigned int a0 , a1 , a2 = 0 , a3 = 0 ;
int sh_1 , sh_2 ;
struct exception_data * d ;
/* prefetch_src((const void *)src); */
@ -197,7 +202,7 @@ static inline unsigned long copy_dstaligned(unsigned long dst, unsigned long src
goto do2 ;
case 0 :
if ( len = = 0 )
return 0 ;
return PA_MEMCPY_OK ;
/* a3 = ((unsigned int *) src)[0];
a0 = ( ( unsigned int * ) src ) [ 1 ] ; */
ldw ( s_space , 0 , src , a3 , cda_ldw_exc ) ;
@ -256,42 +261,35 @@ do0:
preserve_branch ( handle_load_error ) ;
preserve_branch ( handle_store_error ) ;
return 0 ;
return PA_MEMCPY_OK ;
handle_load_error :
__asm__ __volatile__ ( " cda_ldw_exc: \n " ) ;
d = & __get_cpu_var ( exception_data ) ;
DPRINTF ( " cda_ldw_exc: o_len=%lu fault_addr=%lu o_src=%lu ret=%lu \n " ,
o_len , d - > fault_addr , o_src , o_len - d - > fault_addr + o_src ) ;
return o_len * 4 - d - > fault_addr + o_src ;
return PA_MEMCPY_LOAD_ERROR ;
handle_store_error :
__asm__ __volatile__ ( " cda_stw_exc: \n " ) ;
d = & __get_cpu_var ( exception_data ) ;
DPRINTF ( " cda_stw_exc: o_len=%lu fault_addr=%lu o_dst=%lu ret=%lu \n " ,
o_len , d - > fault_addr , o_dst , o_len - d - > fault_addr + o_dst ) ;
return o_len * 4 - d - > fault_addr + o_dst ;
return PA_MEMCPY_STORE_ERROR ;
}
/* Returns 0 for success, otherwise, returns number of bytes not transferred. */
static unsigned long pa_memcpy ( void * dstp , const void * srcp , unsigned long len )
/* Returns PA_MEMCPY_OK, PA_MEMCPY_LOAD_ERROR or PA_MEMCPY_STORE_ERROR.
* In case of an access fault the faulty address can be read from the per_cpu
* exception data struct . */
static unsigned long pa_memcpy_internal ( void * dstp , const void * srcp ,
unsigned long len )
{
register unsigned long src , dst , t1 , t2 , t3 ;
register unsigned char * pcs , * pcd ;
register unsigned int * pws , * pwd ;
register double * pds , * pdd ;
unsigned long ret = 0 ;
unsigned long o_dst , o_src , o_len ;
struct exception_data * d ;
unsigned long ret ;
src = ( unsigned long ) srcp ;
dst = ( unsigned long ) dstp ;
pcs = ( unsigned char * ) srcp ;
pcd = ( unsigned char * ) dstp ;
o_dst = dst ; o_src = src ; o_len = len ;
/* prefetch_src((const void *)srcp); */
if ( len < THRESHOLD )
@ -401,7 +399,7 @@ byte_copy:
len - - ;
}
return 0 ;
return PA_MEMCPY_OK ;
unaligned_copy :
/* possibly we are aligned on a word, but not on a double... */
@ -438,8 +436,7 @@ unaligned_copy:
src = ( unsigned long ) pcs ;
}
ret = copy_dstaligned ( dst , src , len / sizeof ( unsigned int ) ,
o_dst , o_src , o_len ) ;
ret = copy_dstaligned ( dst , src , len / sizeof ( unsigned int ) ) ;
if ( ret )
return ret ;
@ -454,17 +451,41 @@ unaligned_copy:
handle_load_error :
__asm__ __volatile__ ( " pmc_load_exc: \n " ) ;
d = & __get_cpu_var ( exception_data ) ;
DPRINTF ( " pmc_load_exc: o_len=%lu fault_addr=%lu o_src=%lu ret=%lu \n " ,
o_len , d - > fault_addr , o_src , o_len - d - > fault_addr + o_src ) ;
return o_len - d - > fault_addr + o_src ;
return PA_MEMCPY_LOAD_ERROR ;
handle_store_error :
__asm__ __volatile__ ( " pmc_store_exc: \n " ) ;
return PA_MEMCPY_STORE_ERROR ;
}
/* Returns 0 for success, otherwise, returns number of bytes not transferred. */
static unsigned long pa_memcpy ( void * dstp , const void * srcp , unsigned long len )
{
unsigned long ret , fault_addr , reference ;
struct exception_data * d ;
ret = pa_memcpy_internal ( dstp , srcp , len ) ;
if ( likely ( ret = = PA_MEMCPY_OK ) )
return 0 ;
/* if a load or store fault occured we can get the faulty addr */
d = & __get_cpu_var ( exception_data ) ;
DPRINTF ( " pmc_store_exc: o_len=%lu fault_addr=%lu o_dst=%lu ret=%lu \n " ,
o_len , d - > fault_addr , o_dst , o_len - d - > fault_addr + o_dst ) ;
return o_len - d - > fault_addr + o_dst ;
fault_addr = d - > fault_addr ;
/* error in load or store? */
if ( ret = = PA_MEMCPY_LOAD_ERROR )
reference = ( unsigned long ) srcp ;
else
reference = ( unsigned long ) dstp ;
DPRINTF ( " pa_memcpy: fault type = %lu, len=%lu fault_addr=%lu ref=%lu \n " ,
ret , len , fault_addr , reference ) ;
if ( fault_addr > = reference )
return len - ( fault_addr - reference ) ;
else
return len ;
}
# ifdef __KERNEL__