Back to home page

EIC code displayed by LXR
 
 

    

File indexing completed on 2026-10-05 09:36:32 

0001 /* -*- c -*-
0002    ----------------------------------------------------------------
0003 
0004    Notice that the following BSD-style license applies to this one
0005    file (valgrind.h) only.  The rest of Valgrind is licensed under the
0006    terms of the GNU General Public License, version 3, unless
0007    otherwise indicated.  See the COPYING file in the source
0008    distribution for details.
0009 
0010    ----------------------------------------------------------------
0011 
0012    This file is part of Valgrind, a dynamic binary instrumentation
0013    framework.
0014 
0015    Copyright (C) 2000-2017 Julian Seward.  All rights reserved.
0016 
0017    Redistribution and use in source and binary forms, with or without
0018    modification, are permitted provided that the following conditions
0019    are met:
0020 
0021    1. Redistributions of source code must retain the above copyright
0022       notice, this list of conditions and the following disclaimer.
0023 
0024    2. The origin of this software must not be misrepresented; you must 
0025       not claim that you wrote the original software.  If you use this 
0026       software in a product, an acknowledgment in the product 
0027       documentation would be appreciated but is not required.
0028 
0029    3. Altered source versions must be plainly marked as such, and must
0030       not be misrepresented as being the original software.
0031 
0032    4. The name of the author may not be used to endorse or promote 
0033       products derived from this software without specific prior written 
0034       permission.
0035 
0036    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
0037    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
0038    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
0039    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
0040    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
0041    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
0042    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
0043    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
0044    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
0045    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
0046    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
0047 
0048    ----------------------------------------------------------------
0049 
0050    Notice that the above BSD-style license applies to this one file
0051    (valgrind.h) only.  The entire rest of Valgrind is licensed under
0052    the terms of the GNU General Public License, version 3.  See the
0053    COPYING file in the source distribution for details.
0054 
0055    ---------------------------------------------------------------- 
0056 */
0057 
0058 
0059 /* This file is for inclusion into client (your!) code.
0060 
0061    You can use these macros to manipulate and query Valgrind's 
0062    execution inside your own programs.
0063 
0064    The resulting executables will still run without Valgrind, just a
0065    little bit more slowly than they otherwise would, but otherwise
0066    unchanged.  When not running on valgrind, each client request
0067    consumes very few (eg. 7) instructions, so the resulting performance
0068    loss is negligible unless you plan to execute client requests
0069    millions of times per second.  Nevertheless, if that is still a
0070    problem, you can compile with the NVALGRIND symbol defined (gcc
0071    -DNVALGRIND) so that client requests are not even compiled in.  */
0072 
0073 #ifndef __VALGRIND_H
0074 #define __VALGRIND_H
0075 
0076 
0077 /* ------------------------------------------------------------------ */
0078 /* VERSION NUMBER OF VALGRIND                                         */
0079 /* ------------------------------------------------------------------ */
0080 
0081 /* Specify Valgrind's version number, so that user code can
0082    conditionally compile based on our version number.  Note that these
0083    were introduced at version 3.6 and so do not exist in version 3.5
0084    or earlier.  The recommended way to use them to check for "version
0085    X.Y or later" is (eg)
0086 
0087 #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__)   \
0088     && (__VALGRIND_MAJOR__ > 3                                   \
0089         || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
0090 */
0091 #define __VALGRIND_MAJOR__    3
0092 #define __VALGRIND_MINOR__    27
0093 
0094 
0095 #include <stdarg.h>
0096 
0097 /* Nb: this file might be included in a file compiled with -ansi.  So
0098    we can't use C++ style "//" comments nor the "asm" keyword (instead
0099    use "__asm__"). */
0100 
0101 /* Derive some tags indicating what the target platform is.  Note
0102    that in this file we're using the compiler's CPP symbols for
0103    identifying architectures, which are different to the ones we use
0104    within the rest of Valgrind.  Note, __powerpc__ is active for both
0105    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
0106    latter (on Linux, that is).
0107 
0108    Misc note: how to find out what's predefined in gcc by default:
0109    gcc -Wp,-dM somefile.c
0110 */
0111 #undef PLAT_x86_darwin
0112 #undef PLAT_amd64_darwin
0113 #undef PLAT_x86_freebsd
0114 #undef PLAT_amd64_freebsd
0115 #undef PLAT_arm64_freebsd
0116 #undef PLAT_x86_win32
0117 #undef PLAT_amd64_win64
0118 #undef PLAT_x86_linux
0119 #undef PLAT_amd64_linux
0120 #undef PLAT_ppc32_linux
0121 #undef PLAT_ppc64be_linux
0122 #undef PLAT_ppc64le_linux
0123 #undef PLAT_arm_linux
0124 #undef PLAT_arm64_linux
0125 #undef PLAT_s390x_linux
0126 #undef PLAT_mips32_linux
0127 #undef PLAT_mips64_linux
0128 #undef PLAT_nanomips_linux
0129 #undef PLAT_riscv64_linux
0130 #undef PLAT_x86_solaris
0131 #undef PLAT_amd64_solaris
0132 
0133 
0134 #if defined(__APPLE__) && defined(__i386__)
0135 #  define PLAT_x86_darwin 1
0136 #elif defined(__APPLE__) && defined(__x86_64__)
0137 #  define PLAT_amd64_darwin 1
0138 #elif defined(__FreeBSD__) && defined(__i386__)
0139 #  define PLAT_x86_freebsd 1
0140 #elif defined(__FreeBSD__) && defined(__amd64__)
0141 #  define PLAT_amd64_freebsd 1
0142 #elif defined(__FreeBSD__) && defined(__aarch64__) && !defined(__arm__)
0143 #  define PLAT_arm64_freebsd 1
0144 #elif (defined(__MINGW32__) && defined(__i386__)) \
0145       || defined(__CYGWIN32__) \
0146       || (defined(_WIN32) && defined(_M_IX86))
0147 #  define PLAT_x86_win32 1
0148 #elif (defined(__MINGW32__) && defined(__x86_64__)) \
0149       || (defined(_WIN32) && defined(_M_X64))
0150 /* __MINGW32__ and _WIN32 are defined in 64 bit mode as well. */
0151 #  define PLAT_amd64_win64 1
0152 #elif defined(__linux__) && defined(__i386__)
0153 #  define PLAT_x86_linux 1
0154 #elif defined(__linux__) && defined(__x86_64__) && !defined(__ILP32__)
0155 #  define PLAT_amd64_linux 1
0156 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
0157 #  define PLAT_ppc32_linux 1
0158 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF != 2
0159 /* Big Endian uses ELF version 1 */
0160 #  define PLAT_ppc64be_linux 1
0161 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF == 2
0162 /* Little Endian uses ELF version 2 */
0163 #  define PLAT_ppc64le_linux 1
0164 #elif defined(__linux__) && defined(__arm__) && !defined(__aarch64__)
0165 #  define PLAT_arm_linux 1
0166 #elif defined(__linux__) && defined(__aarch64__) && !defined(__arm__)
0167 #  define PLAT_arm64_linux 1
0168 #elif defined(__linux__) && defined(__s390__) && defined(__s390x__)
0169 #  define PLAT_s390x_linux 1
0170 #elif defined(__linux__) && defined(__mips__) && (__mips==64)
0171 #  define PLAT_mips64_linux 1
0172 #elif defined(__linux__) && defined(__mips__) && (__mips==32)
0173 #  define PLAT_mips32_linux 1
0174 #elif defined(__linux__) && defined(__nanomips__)
0175 #  define PLAT_nanomips_linux 1
0176 #elif defined(__linux__) && defined(__riscv) && (__riscv_xlen == 64)
0177 #  define PLAT_riscv64_linux 1
0178 #elif defined(__sun) && defined(__i386__)
0179 #  define PLAT_x86_solaris 1
0180 #elif defined(__sun) && defined(__x86_64__)
0181 #  define PLAT_amd64_solaris 1
0182 #else
0183 /* If we're not compiling for our target platform, don't generate
0184    any inline asms.  */
0185 #  if !defined(NVALGRIND)
0186 #    define NVALGRIND 1
0187 #  endif
0188 #endif
0189 
0190 
0191 /* ------------------------------------------------------------------ */
0192 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
0193 /* in here of use to end-users -- skip to the next section.           */
0194 /* ------------------------------------------------------------------ */
0195 
0196 /*
0197  * VALGRIND_DO_CLIENT_REQUEST(): a statement that invokes a Valgrind client
0198  * request. Accepts both pointers and integers as arguments.
0199  *
0200  * VALGRIND_DO_CLIENT_REQUEST_STMT(): a statement that invokes a Valgrind
0201  * client request that does not return a value.
0202 
0203  * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind
0204  * client request and whose value equals the client request result.  Accepts
0205  * both pointers and integers as arguments.  Note that such calls are not
0206  * necessarily pure functions -- they may have side effects.
0207  */
0208 
0209 #define VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default,            \
0210                                    _zzq_request, _zzq_arg1, _zzq_arg2,  \
0211                                    _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
0212   do { (_zzq_rlval) = VALGRIND_DO_CLIENT_REQUEST_EXPR((_zzq_default),   \
0213                         (_zzq_request), (_zzq_arg1), (_zzq_arg2),       \
0214                         (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
0215 
0216 #define VALGRIND_DO_CLIENT_REQUEST_STMT(_zzq_request, _zzq_arg1,        \
0217                            _zzq_arg2,  _zzq_arg3, _zzq_arg4, _zzq_arg5) \
0218   do { (void) VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                        \
0219                     (_zzq_request), (_zzq_arg1), (_zzq_arg2),           \
0220                     (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
0221 
0222 #if defined(NVALGRIND)
0223 
0224 /* Define NVALGRIND to completely remove the Valgrind magic sequence
0225    from the compiled code (analogous to NDEBUG's effects on
0226    assert()) */
0227 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
0228         _zzq_default, _zzq_request,                               \
0229         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
0230       (_zzq_default)
0231 
0232 #else  /* ! NVALGRIND */
0233 
0234 /* The following defines the magic code sequences which the JITter
0235    spots and handles magically.  Don't look too closely at them as
0236    they will rot your brain.
0237 
0238    The assembly code sequences for all architectures is in this one
0239    file.  This is because this file must be stand-alone, and we don't
0240    want to have multiple files.
0241 
0242    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
0243    value gets put in the return slot, so that everything works when
0244    this is executed not under Valgrind.  Args are passed in a memory
0245    block, and so there's no intrinsic limit to the number that could
0246    be passed, but it's currently five.
0247    
0248    The macro args are: 
0249       _zzq_rlval    result lvalue
0250       _zzq_default  default value (result returned when running on real CPU)
0251       _zzq_request  request code
0252       _zzq_arg1..5  request params
0253 
0254    The other two macros are used to support function wrapping, and are
0255    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
0256    guest's NRADDR pseudo-register and whatever other information is
0257    needed to safely run the call original from the wrapper: on
0258    ppc64-linux, the R2 value at the divert point is also needed.  This
0259    information is abstracted into a user-visible type, OrigFn.
0260 
0261    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
0262    guest, but guarantees that the branch instruction will not be
0263    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
0264    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
0265    complete inline asm, since it needs to be combined with more magic
0266    inline asm stuff to be useful.
0267 */
0268 
0269 /* -------------- x86-{linux,darwin,solaris,freebsd} ------------- */
0270 
0271 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)  \
0272     ||  (defined(PLAT_x86_win32) && defined(__GNUC__)) \
0273     ||  defined(PLAT_x86_solaris) || defined(PLAT_x86_freebsd)
0274 
0275 typedef
0276    struct { 
0277       unsigned int nraddr; /* where's the code? */
0278    }
0279    OrigFn;
0280 
0281 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
0282                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
0283                      "roll $29, %%edi ; roll $19, %%edi\n\t"
0284 
0285 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
0286         _zzq_default, _zzq_request,                               \
0287         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
0288   __extension__                                                   \
0289   ({volatile unsigned int _zzq_args[6];                           \
0290     volatile unsigned int _zzq_result;                            \
0291     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
0292     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
0293     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
0294     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
0295     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
0296     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
0297     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0298                      /* %EDX = client_request ( %EAX ) */         \
0299                      "xchgl %%ebx,%%ebx"                          \
0300                      : "=d" (_zzq_result)                         \
0301                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
0302                      : "cc", "memory"                             \
0303                     );                                            \
0304     _zzq_result;                                                  \
0305   })
0306 
0307 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
0308   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
0309     volatile unsigned int __addr;                                 \
0310     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0311                      /* %EAX = guest_NRADDR */                    \
0312                      "xchgl %%ecx,%%ecx"                          \
0313                      : "=a" (__addr)                              \
0314                      :                                            \
0315                      : "cc", "memory"                             \
0316                     );                                            \
0317     _zzq_orig->nraddr = __addr;                                   \
0318   }
0319 
0320 #define VALGRIND_CALL_NOREDIR_EAX                                 \
0321                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0322                      /* call-noredir *%EAX */                     \
0323                      "xchgl %%edx,%%edx\n\t"
0324 
0325 #define VALGRIND_VEX_INJECT_IR()                                 \
0326  do {                                                            \
0327     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
0328                      "xchgl %%edi,%%edi\n\t"                     \
0329                      : : : "cc", "memory"                        \
0330                     );                                           \
0331  } while (0)
0332 
0333 #endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__)
0334           || PLAT_x86_solaris || PLAT_x86_freebsd */
0335 
0336 /* ------------------------- x86-Win32 ------------------------- */
0337 
0338 #if defined(PLAT_x86_win32) && !defined(__GNUC__)
0339 
0340 typedef
0341    struct { 
0342       unsigned int nraddr; /* where's the code? */
0343    }
0344    OrigFn;
0345 
0346 #if defined(_MSC_VER)
0347 
0348 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
0349                      __asm rol edi, 3  __asm rol edi, 13          \
0350                      __asm rol edi, 29 __asm rol edi, 19
0351 
0352 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
0353         _zzq_default, _zzq_request,                               \
0354         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
0355     valgrind_do_client_request_expr((uintptr_t)(_zzq_default),    \
0356         (uintptr_t)(_zzq_request), (uintptr_t)(_zzq_arg1),        \
0357         (uintptr_t)(_zzq_arg2), (uintptr_t)(_zzq_arg3),           \
0358         (uintptr_t)(_zzq_arg4), (uintptr_t)(_zzq_arg5))
0359 
0360 static __inline uintptr_t
0361 valgrind_do_client_request_expr(uintptr_t _zzq_default, uintptr_t _zzq_request,
0362                                 uintptr_t _zzq_arg1, uintptr_t _zzq_arg2,
0363                                 uintptr_t _zzq_arg3, uintptr_t _zzq_arg4,
0364                                 uintptr_t _zzq_arg5)
0365 {
0366     volatile uintptr_t _zzq_args[6];
0367     volatile unsigned int _zzq_result;
0368     _zzq_args[0] = (uintptr_t)(_zzq_request);
0369     _zzq_args[1] = (uintptr_t)(_zzq_arg1);
0370     _zzq_args[2] = (uintptr_t)(_zzq_arg2);
0371     _zzq_args[3] = (uintptr_t)(_zzq_arg3);
0372     _zzq_args[4] = (uintptr_t)(_zzq_arg4);
0373     _zzq_args[5] = (uintptr_t)(_zzq_arg5);
0374     __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default
0375             __SPECIAL_INSTRUCTION_PREAMBLE
0376             /* %EDX = client_request ( %EAX ) */
0377             __asm xchg ebx,ebx
0378             __asm mov _zzq_result, edx
0379     }
0380     return _zzq_result;
0381 }
0382 
0383 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
0384   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
0385     volatile unsigned int __addr;                                 \
0386     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                        \
0387             /* %EAX = guest_NRADDR */                             \
0388             __asm xchg ecx,ecx                                    \
0389             __asm mov __addr, eax                                 \
0390     }                                                             \
0391     _zzq_orig->nraddr = __addr;                                   \
0392   }
0393 
0394 #define VALGRIND_CALL_NOREDIR_EAX ERROR
0395 
0396 #define VALGRIND_VEX_INJECT_IR()                                 \
0397  do {                                                            \
0398     __asm { __SPECIAL_INSTRUCTION_PREAMBLE                       \
0399             __asm xchg edi,edi                                   \
0400     }                                                            \
0401  } while (0)
0402 
0403 #else
0404 #error Unsupported compiler.
0405 #endif
0406 
0407 #endif /* PLAT_x86_win32 */
0408 
0409 /* -------------- amd64-{linux,darwin,solaris,freebsd} ------------ */
0410 
0411 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin) \
0412     ||  defined(PLAT_amd64_solaris) \
0413     ||  defined(PLAT_amd64_freebsd) \
0414     ||  (defined(PLAT_amd64_win64) && defined(__GNUC__))
0415 
0416 typedef
0417    struct { 
0418       unsigned long int nraddr; /* where's the code? */
0419    }
0420    OrigFn;
0421 
0422 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
0423                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
0424                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
0425 
0426 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
0427         _zzq_default, _zzq_request,                               \
0428         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
0429     __extension__                                                 \
0430     ({ volatile unsigned long int _zzq_args[6];                   \
0431     volatile unsigned long int _zzq_result;                       \
0432     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
0433     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
0434     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
0435     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
0436     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
0437     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
0438     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0439                      /* %RDX = client_request ( %RAX ) */         \
0440                      "xchgq %%rbx,%%rbx"                          \
0441                      : "=d" (_zzq_result)                         \
0442                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
0443                      : "cc", "memory"                             \
0444                     );                                            \
0445     _zzq_result;                                                  \
0446     })
0447 
0448 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
0449   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
0450     volatile unsigned long int __addr;                            \
0451     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0452                      /* %RAX = guest_NRADDR */                    \
0453                      "xchgq %%rcx,%%rcx"                          \
0454                      : "=a" (__addr)                              \
0455                      :                                            \
0456                      : "cc", "memory"                             \
0457                     );                                            \
0458     _zzq_orig->nraddr = __addr;                                   \
0459   }
0460 
0461 #define VALGRIND_CALL_NOREDIR_RAX                                 \
0462                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0463                      /* call-noredir *%RAX */                     \
0464                      "xchgq %%rdx,%%rdx\n\t"
0465 
0466 #define VALGRIND_VEX_INJECT_IR()                                 \
0467  do {                                                            \
0468     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
0469                      "xchgq %%rdi,%%rdi\n\t"                     \
0470                      : : : "cc", "memory"                        \
0471                     );                                           \
0472  } while (0)
0473 
0474 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin
0475           || PLAT_amd64_solaris || PLAT_amd64_freebsd */
0476 
0477 /* ------------------------- amd64-Win64 ------------------------- */
0478 
0479 #if defined(PLAT_amd64_win64) && !defined(__GNUC__)
0480 
0481 #error Unsupported compiler.
0482 
0483 #endif /* PLAT_amd64_win64 */
0484 
0485 /* ------------------------ ppc32-linux ------------------------ */
0486 
0487 #if defined(PLAT_ppc32_linux)
0488 
0489 typedef
0490    struct { 
0491       unsigned int nraddr; /* where's the code? */
0492    }
0493    OrigFn;
0494 
0495 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
0496                     "rlwinm 0,0,3,0,31  ; rlwinm 0,0,13,0,31\n\t" \
0497                     "rlwinm 0,0,29,0,31 ; rlwinm 0,0,19,0,31\n\t"
0498 
0499 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
0500         _zzq_default, _zzq_request,                               \
0501         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
0502                                                                   \
0503     __extension__                                                 \
0504   ({         unsigned int  _zzq_args[6];                          \
0505              unsigned int  _zzq_result;                           \
0506              unsigned int* _zzq_ptr;                              \
0507     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
0508     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
0509     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
0510     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
0511     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
0512     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
0513     _zzq_ptr = _zzq_args;                                         \
0514     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
0515                      "mr 4,%2\n\t" /*ptr*/                        \
0516                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0517                      /* %R3 = client_request ( %R4 ) */           \
0518                      "or 1,1,1\n\t"                               \
0519                      "mr %0,3"     /*result*/                     \
0520                      : "=b" (_zzq_result)                         \
0521                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
0522                      : "cc", "memory", "r3", "r4");               \
0523     _zzq_result;                                                  \
0524     })
0525 
0526 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
0527   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
0528     unsigned int __addr;                                          \
0529     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0530                      /* %R3 = guest_NRADDR */                     \
0531                      "or 2,2,2\n\t"                               \
0532                      "mr %0,3"                                    \
0533                      : "=b" (__addr)                              \
0534                      :                                            \
0535                      : "cc", "memory", "r3"                       \
0536                     );                                            \
0537     _zzq_orig->nraddr = __addr;                                   \
0538   }
0539 
0540 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
0541                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0542                      /* branch-and-link-to-noredir *%R11 */       \
0543                      "or 3,3,3\n\t"
0544 
0545 #define VALGRIND_VEX_INJECT_IR()                                 \
0546  do {                                                            \
0547     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
0548                      "or 5,5,5\n\t"                              \
0549                     );                                           \
0550  } while (0)
0551 
0552 #endif /* PLAT_ppc32_linux */
0553 
0554 /* ------------------------ ppc64-linux ------------------------ */
0555 
0556 #if defined(PLAT_ppc64be_linux)
0557 
0558 typedef
0559    struct { 
0560       unsigned long int nraddr; /* where's the code? */
0561       unsigned long int r2;  /* what tocptr do we need? */
0562    }
0563    OrigFn;
0564 
0565 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
0566                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
0567                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
0568 
0569 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
0570         _zzq_default, _zzq_request,                               \
0571         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
0572                                                                   \
0573   __extension__                                                   \
0574   ({         unsigned long int  _zzq_args[6];                     \
0575              unsigned long int  _zzq_result;                      \
0576              unsigned long int* _zzq_ptr;                         \
0577     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
0578     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
0579     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
0580     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
0581     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
0582     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
0583     _zzq_ptr = _zzq_args;                                         \
0584     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
0585                      "mr 4,%2\n\t" /*ptr*/                        \
0586                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0587                      /* %R3 = client_request ( %R4 ) */           \
0588                      "or 1,1,1\n\t"                               \
0589                      "mr %0,3"     /*result*/                     \
0590                      : "=b" (_zzq_result)                         \
0591                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
0592                      : "cc", "memory", "r3", "r4");               \
0593     _zzq_result;                                                  \
0594   })
0595 
0596 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
0597   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
0598     unsigned long int __addr;                                     \
0599     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0600                      /* %R3 = guest_NRADDR */                     \
0601                      "or 2,2,2\n\t"                               \
0602                      "mr %0,3"                                    \
0603                      : "=b" (__addr)                              \
0604                      :                                            \
0605                      : "cc", "memory", "r3"                       \
0606                     );                                            \
0607     _zzq_orig->nraddr = __addr;                                   \
0608     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0609                      /* %R3 = guest_NRADDR_GPR2 */                \
0610                      "or 4,4,4\n\t"                               \
0611                      "mr %0,3"                                    \
0612                      : "=b" (__addr)                              \
0613                      :                                            \
0614                      : "cc", "memory", "r3"                       \
0615                     );                                            \
0616     _zzq_orig->r2 = __addr;                                       \
0617   }
0618 
0619 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
0620                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0621                      /* branch-and-link-to-noredir *%R11 */       \
0622                      "or 3,3,3\n\t"
0623 
0624 #define VALGRIND_VEX_INJECT_IR()                                 \
0625  do {                                                            \
0626     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
0627                      "or 5,5,5\n\t"                              \
0628                     );                                           \
0629  } while (0)
0630 
0631 #endif /* PLAT_ppc64be_linux */
0632 
0633 #if defined(PLAT_ppc64le_linux)
0634 
0635 typedef
0636    struct {
0637       unsigned long int nraddr; /* where's the code? */
0638       unsigned long int r2;     /* what tocptr do we need? */
0639    }
0640    OrigFn;
0641 
0642 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
0643                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
0644                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
0645 
0646 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
0647         _zzq_default, _zzq_request,                               \
0648         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
0649                                                                   \
0650   __extension__                                                   \
0651   ({         unsigned long int  _zzq_args[6];                     \
0652              unsigned long int  _zzq_result;                      \
0653              unsigned long int* _zzq_ptr;                         \
0654     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
0655     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
0656     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
0657     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
0658     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
0659     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
0660     _zzq_ptr = _zzq_args;                                         \
0661     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
0662                      "mr 4,%2\n\t" /*ptr*/                        \
0663                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0664                      /* %R3 = client_request ( %R4 ) */           \
0665                      "or 1,1,1\n\t"                               \
0666                      "mr %0,3"     /*result*/                     \
0667                      : "=b" (_zzq_result)                         \
0668                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
0669                      : "cc", "memory", "r3", "r4");               \
0670     _zzq_result;                                                  \
0671   })
0672 
0673 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
0674   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
0675     unsigned long int __addr;                                     \
0676     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0677                      /* %R3 = guest_NRADDR */                     \
0678                      "or 2,2,2\n\t"                               \
0679                      "mr %0,3"                                    \
0680                      : "=b" (__addr)                              \
0681                      :                                            \
0682                      : "cc", "memory", "r3"                       \
0683                     );                                            \
0684     _zzq_orig->nraddr = __addr;                                   \
0685     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0686                      /* %R3 = guest_NRADDR_GPR2 */                \
0687                      "or 4,4,4\n\t"                               \
0688                      "mr %0,3"                                    \
0689                      : "=b" (__addr)                              \
0690                      :                                            \
0691                      : "cc", "memory", "r3"                       \
0692                     );                                            \
0693     _zzq_orig->r2 = __addr;                                       \
0694   }
0695 
0696 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                   \
0697                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0698                      /* branch-and-link-to-noredir *%R12 */       \
0699                      "or 3,3,3\n\t"
0700 
0701 #define VALGRIND_VEX_INJECT_IR()                                 \
0702  do {                                                            \
0703     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
0704                      "or 5,5,5\n\t"                              \
0705                     );                                           \
0706  } while (0)
0707 
0708 #endif /* PLAT_ppc64le_linux */
0709 
0710 /* ------------------------- arm-linux ------------------------- */
0711 
0712 #if defined(PLAT_arm_linux)
0713 
0714 typedef
0715    struct { 
0716       unsigned int nraddr; /* where's the code? */
0717    }
0718    OrigFn;
0719 
0720 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
0721             "mov r12, r12, ror #3  ; mov r12, r12, ror #13 \n\t"  \
0722             "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
0723 
0724 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
0725         _zzq_default, _zzq_request,                               \
0726         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
0727                                                                   \
0728   __extension__                                                   \
0729   ({volatile unsigned int  _zzq_args[6];                          \
0730     volatile unsigned int  _zzq_result;                           \
0731     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
0732     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
0733     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
0734     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
0735     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
0736     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
0737     __asm__ volatile("mov r3, %1\n\t" /*default*/                 \
0738                      "mov r4, %2\n\t" /*ptr*/                     \
0739                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0740                      /* R3 = client_request ( R4 ) */             \
0741                      "orr r10, r10, r10\n\t"                      \
0742                      "mov %0, r3"     /*result*/                  \
0743                      : "=r" (_zzq_result)                         \
0744                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
0745                      : "cc","memory", "r3", "r4");                \
0746     _zzq_result;                                                  \
0747   })
0748 
0749 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
0750   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
0751     unsigned int __addr;                                          \
0752     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0753                      /* R3 = guest_NRADDR */                      \
0754                      "orr r11, r11, r11\n\t"                      \
0755                      "mov %0, r3"                                 \
0756                      : "=r" (__addr)                              \
0757                      :                                            \
0758                      : "cc", "memory", "r3"                       \
0759                     );                                            \
0760     _zzq_orig->nraddr = __addr;                                   \
0761   }
0762 
0763 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                    \
0764                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0765                      /* branch-and-link-to-noredir *%R4 */        \
0766                      "orr r12, r12, r12\n\t"
0767 
0768 #define VALGRIND_VEX_INJECT_IR()                                 \
0769  do {                                                            \
0770     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
0771                      "orr r9, r9, r9\n\t"                        \
0772                      : : : "cc", "memory"                        \
0773                     );                                           \
0774  } while (0)
0775 
0776 #endif /* PLAT_arm_linux */
0777 
0778 /* -------------------- arm64-{linux,freebsd} -------------------- */
0779 
0780 #if defined(PLAT_arm64_linux) || defined(PLAT_arm64_freebsd)
0781 
0782 typedef
0783    struct { 
0784       unsigned long int nraddr; /* where's the code? */
0785    }
0786    OrigFn;
0787 
0788 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
0789             "ror x12, x12, #3  ;  ror x12, x12, #13 \n\t"         \
0790             "ror x12, x12, #51 ;  ror x12, x12, #61 \n\t"
0791 
0792 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
0793         _zzq_default, _zzq_request,                               \
0794         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
0795                                                                   \
0796   __extension__                                                   \
0797   ({volatile unsigned long int  _zzq_args[6];                     \
0798     volatile unsigned long int  _zzq_result;                      \
0799     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
0800     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
0801     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
0802     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
0803     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
0804     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
0805     __asm__ volatile("mov x3, %1\n\t" /*default*/                 \
0806                      "mov x4, %2\n\t" /*ptr*/                     \
0807                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0808                      /* X3 = client_request ( X4 ) */             \
0809                      "orr x10, x10, x10\n\t"                      \
0810                      "mov %0, x3"     /*result*/                  \
0811                      : "=r" (_zzq_result)                         \
0812                      : "r" ((unsigned long int)(_zzq_default)),   \
0813                        "r" (&_zzq_args[0])                        \
0814                      : "cc","memory", "x3", "x4");                \
0815     _zzq_result;                                                  \
0816   })
0817 
0818 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
0819   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
0820     unsigned long int __addr;                                     \
0821     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0822                      /* X3 = guest_NRADDR */                      \
0823                      "orr x11, x11, x11\n\t"                      \
0824                      "mov %0, x3"                                 \
0825                      : "=r" (__addr)                              \
0826                      :                                            \
0827                      : "cc", "memory", "x3"                       \
0828                     );                                            \
0829     _zzq_orig->nraddr = __addr;                                   \
0830   }
0831 
0832 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                    \
0833                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0834                      /* branch-and-link-to-noredir X8 */          \
0835                      "orr x12, x12, x12\n\t"
0836 
0837 #define VALGRIND_VEX_INJECT_IR()                                 \
0838  do {                                                            \
0839     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
0840                      "orr x9, x9, x9\n\t"                        \
0841                      : : : "cc", "memory"                        \
0842                     );                                           \
0843  } while (0)
0844 
0845 #endif /* PLAT_arm64_linux || PLAT_arm64_freebsd */
0846 
0847 /* ------------------------ s390x-linux ------------------------ */
0848 
0849 #if defined(PLAT_s390x_linux)
0850 
0851 typedef
0852   struct {
0853      unsigned long int nraddr; /* where's the code? */
0854   }
0855   OrigFn;
0856 
0857 /* __SPECIAL_INSTRUCTION_PREAMBLE will be used to identify Valgrind specific
0858  * code. This detection is implemented in platform specific toIR.c
0859  * (e.g. VEX/priv/guest_s390_decoder.c).
0860  */
0861 #define __SPECIAL_INSTRUCTION_PREAMBLE                           \
0862                      "lr 15,15\n\t"                              \
0863                      "lr 1,1\n\t"                                \
0864                      "lr 2,2\n\t"                                \
0865                      "lr 3,3\n\t"
0866 
0867 #define __CLIENT_REQUEST_CODE "lr 2,2\n\t"
0868 #define __GET_NR_CONTEXT_CODE "lr 3,3\n\t"
0869 #define __CALL_NO_REDIR_CODE  "lr 4,4\n\t"
0870 #define __VEX_INJECT_IR_CODE  "lr 5,5\n\t"
0871 
0872 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                         \
0873        _zzq_default, _zzq_request,                               \
0874        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
0875   __extension__                                                  \
0876  ({volatile unsigned long int _zzq_args[6];                      \
0877    volatile unsigned long int _zzq_result;                       \
0878    _zzq_args[0] = (unsigned long int)(_zzq_request);             \
0879    _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
0880    _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
0881    _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
0882    _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
0883    _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
0884    __asm__ volatile(/* r2 = args */                              \
0885                     "lgr 2,%1\n\t"                               \
0886                     /* r3 = default */                           \
0887                     "lgr 3,%2\n\t"                               \
0888                     __SPECIAL_INSTRUCTION_PREAMBLE               \
0889                     __CLIENT_REQUEST_CODE                        \
0890                     /* results = r3 */                           \
0891                     "lgr %0, 3\n\t"                              \
0892                     : "=d" (_zzq_result)                         \
0893                     : "a" (&_zzq_args[0]),                       \
0894                       "0" ((unsigned long int)_zzq_default)      \
0895                     : "cc", "2", "3", "memory"                   \
0896                    );                                            \
0897    _zzq_result;                                                  \
0898  })
0899 
0900 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                      \
0901  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
0902    volatile unsigned long int __addr;                            \
0903    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0904                     __GET_NR_CONTEXT_CODE                        \
0905                     "lgr %0, 3\n\t"                              \
0906                     : "=a" (__addr)                              \
0907                     :                                            \
0908                     : "cc", "3", "memory"                        \
0909                    );                                            \
0910    _zzq_orig->nraddr = __addr;                                   \
0911  }
0912 
0913 #define VALGRIND_CALL_NOREDIR_R1                                 \
0914                     __SPECIAL_INSTRUCTION_PREAMBLE               \
0915                     __CALL_NO_REDIR_CODE
0916 
0917 #define VALGRIND_VEX_INJECT_IR()                                 \
0918  do {                                                            \
0919     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
0920                      __VEX_INJECT_IR_CODE);                      \
0921  } while (0)
0922 
0923 #endif /* PLAT_s390x_linux */
0924 
0925 /* ------------------------- mips32-linux ---------------- */
0926 
0927 #if defined(PLAT_mips32_linux)
0928 
0929 typedef
0930    struct { 
0931       unsigned int nraddr; /* where's the code? */
0932    }
0933    OrigFn;
0934 
0935 /* .word  0x342
0936  * .word  0x742
0937  * .word  0xC2
0938  * .word  0x4C2*/
0939 #define __SPECIAL_INSTRUCTION_PREAMBLE          \
0940                      "srl $0, $0, 13\n\t"       \
0941                      "srl $0, $0, 29\n\t"       \
0942                      "srl $0, $0, 3\n\t"        \
0943                      "srl $0, $0, 19\n\t"
0944                     
0945 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
0946        _zzq_default, _zzq_request,                                \
0947        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
0948   __extension__                                                   \
0949   ({ volatile unsigned int _zzq_args[6];                          \
0950     volatile unsigned int _zzq_result;                            \
0951     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
0952     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
0953     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
0954     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
0955     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
0956     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
0957         __asm__ volatile("move $11, %1\n\t" /*default*/           \
0958                      "move $12, %2\n\t" /*ptr*/                   \
0959                      __SPECIAL_INSTRUCTION_PREAMBLE               \
0960                      /* T3 = client_request ( T4 ) */             \
0961                      "or $13, $13, $13\n\t"                       \
0962                      "move %0, $11\n\t"     /*result*/            \
0963                      : "=r" (_zzq_result)                         \
0964                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
0965                      : "$11", "$12", "memory");                   \
0966     _zzq_result;                                                  \
0967   })
0968 
0969 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
0970   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
0971     volatile unsigned int __addr;                                 \
0972     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
0973                      /* %t9 = guest_NRADDR */                     \
0974                      "or $14, $14, $14\n\t"                       \
0975                      "move %0, $11"     /*result*/                \
0976                      : "=r" (__addr)                              \
0977                      :                                            \
0978                      : "$11"                                      \
0979                     );                                            \
0980     _zzq_orig->nraddr = __addr;                                   \
0981   }
0982 
0983 #define VALGRIND_CALL_NOREDIR_T9                                 \
0984                      __SPECIAL_INSTRUCTION_PREAMBLE              \
0985                      /* call-noredir *%t9 */                     \
0986                      "or $15, $15, $15\n\t"
0987 
0988 #define VALGRIND_VEX_INJECT_IR()                                 \
0989  do {                                                            \
0990     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
0991                      "or $11, $11, $11\n\t"                      \
0992                     );                                           \
0993  } while (0)
0994 
0995 
0996 #endif /* PLAT_mips32_linux */
0997 
0998 /* ------------------------- mips64-linux ---------------- */
0999 
1000 #if defined(PLAT_mips64_linux)
1001 
1002 typedef
1003    struct {
1004       unsigned long nraddr; /* where's the code? */
1005    }
1006    OrigFn;
1007 
1008 /* dsll $0,$0, 3
1009  * dsll $0,$0, 13
1010  * dsll $0,$0, 29
1011  * dsll $0,$0, 19*/
1012 #define __SPECIAL_INSTRUCTION_PREAMBLE                              \
1013                      "dsll $0,$0, 3 ; dsll $0,$0,13\n\t"            \
1014                      "dsll $0,$0,29 ; dsll $0,$0,19\n\t"
1015 
1016 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                            \
1017        _zzq_default, _zzq_request,                                  \
1018        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)       \
1019   __extension__                                                     \
1020   ({ volatile unsigned long int _zzq_args[6];                       \
1021     volatile unsigned long int _zzq_result;                         \
1022     _zzq_args[0] = (unsigned long int)(_zzq_request);               \
1023     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                  \
1024     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                  \
1025     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                  \
1026     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                  \
1027     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                  \
1028         __asm__ volatile("move $11, %1\n\t" /*default*/             \
1029                          "move $12, %2\n\t" /*ptr*/                 \
1030                          __SPECIAL_INSTRUCTION_PREAMBLE             \
1031                          /* $11 = client_request ( $12 ) */         \
1032                          "or $13, $13, $13\n\t"                     \
1033                          "move %0, $11\n\t"     /*result*/          \
1034                          : "=r" (_zzq_result)                       \
1035                          : "r" (_zzq_default), "r" (&_zzq_args[0])  \
1036                          : "$11", "$12", "memory");                 \
1037     _zzq_result;                                                    \
1038   })
1039 
1040 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                         \
1041   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                     \
1042     volatile unsigned long int __addr;                              \
1043     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
1044                      /* $11 = guest_NRADDR */                       \
1045                      "or $14, $14, $14\n\t"                         \
1046                      "move %0, $11"     /*result*/                  \
1047                      : "=r" (__addr)                                \
1048                      :                                              \
1049                      : "$11");                                      \
1050     _zzq_orig->nraddr = __addr;                                     \
1051   }
1052 
1053 #define VALGRIND_CALL_NOREDIR_T9                                    \
1054                      __SPECIAL_INSTRUCTION_PREAMBLE                 \
1055                      /* call-noredir $25 */                         \
1056                      "or $15, $15, $15\n\t"
1057 
1058 #define VALGRIND_VEX_INJECT_IR()                                    \
1059  do {                                                               \
1060     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
1061                      "or $11, $11, $11\n\t"                         \
1062                     );                                              \
1063  } while (0)
1064 
1065 #endif /* PLAT_mips64_linux */
1066 
1067 #if defined(PLAT_nanomips_linux)
1068 
1069 typedef
1070    struct {
1071       unsigned int nraddr; /* where's the code? */
1072    }
1073    OrigFn;
1074 /*
1075    8000 c04d  srl  zero, zero, 13
1076    8000 c05d  srl  zero, zero, 29
1077    8000 c043  srl  zero, zero,  3
1078    8000 c053  srl  zero, zero, 19
1079 */
1080 
1081 #define __SPECIAL_INSTRUCTION_PREAMBLE "srl[32] $zero, $zero, 13 \n\t" \
1082                                        "srl[32] $zero, $zero, 29 \n\t" \
1083                                        "srl[32] $zero, $zero, 3  \n\t" \
1084                                        "srl[32] $zero, $zero, 19 \n\t"
1085 
1086 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
1087        _zzq_default, _zzq_request,                                \
1088        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
1089   __extension__                                                   \
1090   ({ volatile unsigned int _zzq_args[6];                          \
1091     volatile unsigned int _zzq_result;                            \
1092     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
1093     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
1094     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
1095     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
1096     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
1097     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
1098     __asm__ volatile("move $a7, %1\n\t" /* default */             \
1099                      "move $t0, %2\n\t" /* ptr */                 \
1100                      __SPECIAL_INSTRUCTION_PREAMBLE               \
1101                      /* $a7 = client_request( $t0 ) */            \
1102                      "or[32] $t0, $t0, $t0\n\t"                   \
1103                      "move %0, $a7\n\t"     /* result */          \
1104                      : "=r" (_zzq_result)                         \
1105                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
1106                      : "$a7", "$t0", "memory");                   \
1107     _zzq_result;                                                  \
1108   })
1109 
1110 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                         \
1111   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                     \
1112     volatile unsigned long int __addr;                              \
1113     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
1114                      /* $a7 = guest_NRADDR */                       \
1115                      "or[32] $t1, $t1, $t1\n\t"                     \
1116                      "move %0, $a7"     /*result*/                  \
1117                      : "=r" (__addr)                                \
1118                      :                                              \
1119                      : "$a7");                                      \
1120     _zzq_orig->nraddr = __addr;                                     \
1121   }
1122 
1123 #define VALGRIND_CALL_NOREDIR_T9                                    \
1124                      __SPECIAL_INSTRUCTION_PREAMBLE                 \
1125                      /* call-noredir $25 */                         \
1126                      "or[32] $t2, $t2, $t2\n\t"
1127 
1128 #define VALGRIND_VEX_INJECT_IR()                                    \
1129  do {                                                               \
1130     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
1131                      "or[32] $t3, $t3, $t3\n\t"                     \
1132                     );                                              \
1133  } while (0)
1134 
1135 #endif
1136 
1137 /* ----------------------- riscv64-linux ------------------------ */
1138 
1139 #if defined(PLAT_riscv64_linux)
1140 
1141 typedef
1142    struct {
1143       unsigned long int nraddr; /* where's the code? */
1144    }
1145    OrigFn;
1146 
1147 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
1148             ".option push\n\t"                                    \
1149             ".option norvc\n\t"                                   \
1150             "srli zero, zero, 3\n\t"                              \
1151             "srli zero, zero, 13\n\t"                             \
1152             "srli zero, zero, 51\n\t"                             \
1153             "srli zero, zero, 61\n\t"
1154 
1155 #define __SPECIAL_INSTRUCTION_POSTAMBLE                           \
1156             ".option pop\n\t"                                     \
1157 
1158 #define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
1159         _zzq_default, _zzq_request,                               \
1160         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
1161                                                                   \
1162   __extension__                                                   \
1163   ({volatile unsigned long int  _zzq_args[6];                     \
1164     volatile unsigned long int  _zzq_result;                      \
1165     _zzq_args[0] = (unsigned long int)(_zzq_request);             \
1166     _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
1167     _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
1168     _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
1169     _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
1170     _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
1171     __asm__ volatile("mv a3, %1\n\t" /*default*/                  \
1172                      "mv a4, %2\n\t" /*ptr*/                      \
1173                      __SPECIAL_INSTRUCTION_PREAMBLE               \
1174                      /* a3 = client_request ( a4 ) */             \
1175                      "or a0, a0, a0\n\t"                          \
1176                      __SPECIAL_INSTRUCTION_POSTAMBLE              \
1177                      "mv %0, a3"     /*result*/                   \
1178                      : "=r" (_zzq_result)                         \
1179                      : "r" ((unsigned long int)(_zzq_default)),   \
1180                        "r" (&_zzq_args[0])                        \
1181                      : "memory", "a3", "a4");                     \
1182     _zzq_result;                                                  \
1183   })
1184 
1185 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
1186   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
1187     unsigned long int __addr;                                     \
1188     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
1189                      /* a3 = guest_NRADDR */                      \
1190                      "or a1, a1, a1\n\t"                          \
1191                      __SPECIAL_INSTRUCTION_POSTAMBLE              \
1192                      "mv %0, a3"                                  \
1193                      : "=r" (__addr)                              \
1194                      :                                            \
1195                      : "memory", "a3"                             \
1196                     );                                            \
1197     _zzq_orig->nraddr = __addr;                                   \
1198   }
1199 
1200 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                    \
1201                      __SPECIAL_INSTRUCTION_PREAMBLE               \
1202                      /* branch-and-link-to-noredir t0 */          \
1203                      "or a2, a2, a2\n\t"                          \
1204                      __SPECIAL_INSTRUCTION_POSTAMBLE
1205 
1206 #define VALGRIND_VEX_INJECT_IR()                                 \
1207  do {                                                            \
1208     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
1209                      "or a3, a3, a3\n\t"                         \
1210                      __SPECIAL_INSTRUCTION_POSTAMBLE             \
1211                      : : : "memory"                              \
1212                     );                                           \
1213  } while (0)
1214 
1215 #endif /* PLAT_riscv64_linux */
1216 
1217 /* Insert assembly code for other platforms here... */
1218 
1219 #endif /* NVALGRIND */
1220 
1221 
1222 /* ------------------------------------------------------------------ */
1223 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
1224 /* ugly.  It's the least-worst tradeoff I can think of.               */
1225 /* ------------------------------------------------------------------ */
1226 
1227 /* This section defines magic (a.k.a appalling-hack) macros for doing
1228    guaranteed-no-redirection macros, so as to get from function
1229    wrappers to the functions they are wrapping.  The whole point is to
1230    construct standard call sequences, but to do the call itself with a
1231    special no-redirect call pseudo-instruction that the JIT
1232    understands and handles specially.  This section is long and
1233    repetitious, and I can't see a way to make it shorter.
1234 
1235    The naming scheme is as follows:
1236 
1237       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
1238 
1239    'W' stands for "word" and 'v' for "void".  Hence there are
1240    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
1241    and for each, the possibility of returning a word-typed result, or
1242    no result.
1243 */
1244 
1245 /* Use these to write the name of your wrapper.  NOTE: duplicates
1246    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h.  NOTE also: inserts
1247    the default behaviour equivalance class tag "0000" into the name.
1248    See pub_tool_redir.h for details -- normally you don't need to
1249    think about this, though. */
1250 
1251 /* Use an extra level of macroisation so as to ensure the soname/fnname
1252    args are fully macro-expanded before pasting them together. */
1253 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
1254 
1255 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
1256    VG_CONCAT4(_vgw00000ZU_,soname,_,fnname)
1257 
1258 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
1259    VG_CONCAT4(_vgw00000ZZ_,soname,_,fnname)
1260 
1261 /* Use this macro from within a wrapper function to collect the
1262    context (address and possibly other info) of the original function.
1263    Once you have that you can then use it in one of the CALL_FN_
1264    macros.  The type of the argument _lval is OrigFn. */
1265 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
1266 
1267 /* Also provide end-user facilities for function replacement, rather
1268    than wrapping.  A replacement function differs from a wrapper in
1269    that it has no way to get hold of the original function being
1270    called, and hence no way to call onwards to it.  In a replacement
1271    function, VALGRIND_GET_ORIG_FN always returns zero. */
1272 
1273 #define I_REPLACE_SONAME_FNNAME_ZU(soname,fnname)                 \
1274    VG_CONCAT4(_vgr00000ZU_,soname,_,fnname)
1275 
1276 #define I_REPLACE_SONAME_FNNAME_ZZ(soname,fnname)                 \
1277    VG_CONCAT4(_vgr00000ZZ_,soname,_,fnname)
1278 
1279 /* Derivatives of the main macros below, for calling functions
1280    returning void. */
1281 
1282 #define CALL_FN_v_v(fnptr)                                        \
1283    do { volatile unsigned long _junk;                             \
1284         CALL_FN_W_v(_junk,fnptr); } while (0)
1285 
1286 #define CALL_FN_v_W(fnptr, arg1)                                  \
1287    do { volatile unsigned long _junk;                             \
1288         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
1289 
1290 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
1291    do { volatile unsigned long _junk;                             \
1292         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
1293 
1294 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
1295    do { volatile unsigned long _junk;                             \
1296         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
1297 
1298 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
1299    do { volatile unsigned long _junk;                             \
1300         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
1301 
1302 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
1303    do { volatile unsigned long _junk;                             \
1304         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
1305 
1306 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
1307    do { volatile unsigned long _junk;                             \
1308         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
1309 
1310 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
1311    do { volatile unsigned long _junk;                             \
1312         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
1313 
1314 /* -------------- x86-{linux,darwin,solaris,freebsd} ------------- */
1315 
1316 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin) \
1317     ||  defined(PLAT_x86_solaris)  || defined(PLAT_x86_freebsd)
1318 
1319 /* These regs are trashed by the hidden call.  No need to mention eax
1320    as gcc can already see that, plus causes gcc to bomb. */
1321 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
1322 
1323 /* Macros to save and align the stack before making a function
1324    call and restore it afterwards as gcc may not keep the stack
1325    pointer aligned if it doesn't realise calls are being made
1326    to other functions. */
1327 
1328 #define VALGRIND_ALIGN_STACK               \
1329       "movl %%esp,%%edi\n\t"               \
1330       "andl $0xfffffff0,%%esp\n\t"
1331 #define VALGRIND_RESTORE_STACK             \
1332       "movl %%edi,%%esp\n\t"
1333 
1334 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
1335    long) == 4. */
1336 
1337 #define CALL_FN_W_v(lval, orig)                                   \
1338    do {                                                           \
1339       volatile OrigFn        _orig = (orig);                      \
1340       volatile unsigned long _argvec[1];                          \
1341       volatile unsigned long _res;                                \
1342       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1343       __asm__ volatile(                                           \
1344          VALGRIND_ALIGN_STACK                                     \
1345          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1346          VALGRIND_CALL_NOREDIR_EAX                                \
1347          VALGRIND_RESTORE_STACK                                   \
1348          : /*out*/   "=a" (_res)                                  \
1349          : /*in*/    "a" (&_argvec[0])                            \
1350          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1351       );                                                          \
1352       lval = (__typeof__(lval)) _res;                             \
1353    } while (0)
1354 
1355 #define CALL_FN_W_W(lval, orig, arg1)                             \
1356    do {                                                           \
1357       volatile OrigFn        _orig = (orig);                      \
1358       volatile unsigned long _argvec[2];                          \
1359       volatile unsigned long _res;                                \
1360       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1361       _argvec[1] = (unsigned long)(arg1);                         \
1362       __asm__ volatile(                                           \
1363          VALGRIND_ALIGN_STACK                                     \
1364          "subl $12, %%esp\n\t"                                    \
1365          "pushl 4(%%eax)\n\t"                                     \
1366          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1367          VALGRIND_CALL_NOREDIR_EAX                                \
1368          VALGRIND_RESTORE_STACK                                   \
1369          : /*out*/   "=a" (_res)                                  \
1370          : /*in*/    "a" (&_argvec[0])                            \
1371          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1372       );                                                          \
1373       lval = (__typeof__(lval)) _res;                             \
1374    } while (0)
1375 
1376 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1377    do {                                                           \
1378       volatile OrigFn        _orig = (orig);                      \
1379       volatile unsigned long _argvec[3];                          \
1380       volatile unsigned long _res;                                \
1381       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1382       _argvec[1] = (unsigned long)(arg1);                         \
1383       _argvec[2] = (unsigned long)(arg2);                         \
1384       __asm__ volatile(                                           \
1385          VALGRIND_ALIGN_STACK                                     \
1386          "subl $8, %%esp\n\t"                                     \
1387          "pushl 8(%%eax)\n\t"                                     \
1388          "pushl 4(%%eax)\n\t"                                     \
1389          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1390          VALGRIND_CALL_NOREDIR_EAX                                \
1391          VALGRIND_RESTORE_STACK                                   \
1392          : /*out*/   "=a" (_res)                                  \
1393          : /*in*/    "a" (&_argvec[0])                            \
1394          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1395       );                                                          \
1396       lval = (__typeof__(lval)) _res;                             \
1397    } while (0)
1398 
1399 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1400    do {                                                           \
1401       volatile OrigFn        _orig = (orig);                      \
1402       volatile unsigned long _argvec[4];                          \
1403       volatile unsigned long _res;                                \
1404       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1405       _argvec[1] = (unsigned long)(arg1);                         \
1406       _argvec[2] = (unsigned long)(arg2);                         \
1407       _argvec[3] = (unsigned long)(arg3);                         \
1408       __asm__ volatile(                                           \
1409          VALGRIND_ALIGN_STACK                                     \
1410          "subl $4, %%esp\n\t"                                     \
1411          "pushl 12(%%eax)\n\t"                                    \
1412          "pushl 8(%%eax)\n\t"                                     \
1413          "pushl 4(%%eax)\n\t"                                     \
1414          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1415          VALGRIND_CALL_NOREDIR_EAX                                \
1416          VALGRIND_RESTORE_STACK                                   \
1417          : /*out*/   "=a" (_res)                                  \
1418          : /*in*/    "a" (&_argvec[0])                            \
1419          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1420       );                                                          \
1421       lval = (__typeof__(lval)) _res;                             \
1422    } while (0)
1423 
1424 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1425    do {                                                           \
1426       volatile OrigFn        _orig = (orig);                      \
1427       volatile unsigned long _argvec[5];                          \
1428       volatile unsigned long _res;                                \
1429       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1430       _argvec[1] = (unsigned long)(arg1);                         \
1431       _argvec[2] = (unsigned long)(arg2);                         \
1432       _argvec[3] = (unsigned long)(arg3);                         \
1433       _argvec[4] = (unsigned long)(arg4);                         \
1434       __asm__ volatile(                                           \
1435          VALGRIND_ALIGN_STACK                                     \
1436          "pushl 16(%%eax)\n\t"                                    \
1437          "pushl 12(%%eax)\n\t"                                    \
1438          "pushl 8(%%eax)\n\t"                                     \
1439          "pushl 4(%%eax)\n\t"                                     \
1440          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1441          VALGRIND_CALL_NOREDIR_EAX                                \
1442          VALGRIND_RESTORE_STACK                                   \
1443          : /*out*/   "=a" (_res)                                  \
1444          : /*in*/    "a" (&_argvec[0])                            \
1445          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1446       );                                                          \
1447       lval = (__typeof__(lval)) _res;                             \
1448    } while (0)
1449 
1450 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1451    do {                                                           \
1452       volatile OrigFn        _orig = (orig);                      \
1453       volatile unsigned long _argvec[6];                          \
1454       volatile unsigned long _res;                                \
1455       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1456       _argvec[1] = (unsigned long)(arg1);                         \
1457       _argvec[2] = (unsigned long)(arg2);                         \
1458       _argvec[3] = (unsigned long)(arg3);                         \
1459       _argvec[4] = (unsigned long)(arg4);                         \
1460       _argvec[5] = (unsigned long)(arg5);                         \
1461       __asm__ volatile(                                           \
1462          VALGRIND_ALIGN_STACK                                     \
1463          "subl $12, %%esp\n\t"                                    \
1464          "pushl 20(%%eax)\n\t"                                    \
1465          "pushl 16(%%eax)\n\t"                                    \
1466          "pushl 12(%%eax)\n\t"                                    \
1467          "pushl 8(%%eax)\n\t"                                     \
1468          "pushl 4(%%eax)\n\t"                                     \
1469          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1470          VALGRIND_CALL_NOREDIR_EAX                                \
1471          VALGRIND_RESTORE_STACK                                   \
1472          : /*out*/   "=a" (_res)                                  \
1473          : /*in*/    "a" (&_argvec[0])                            \
1474          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1475       );                                                          \
1476       lval = (__typeof__(lval)) _res;                             \
1477    } while (0)
1478 
1479 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1480    do {                                                           \
1481       volatile OrigFn        _orig = (orig);                      \
1482       volatile unsigned long _argvec[7];                          \
1483       volatile unsigned long _res;                                \
1484       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1485       _argvec[1] = (unsigned long)(arg1);                         \
1486       _argvec[2] = (unsigned long)(arg2);                         \
1487       _argvec[3] = (unsigned long)(arg3);                         \
1488       _argvec[4] = (unsigned long)(arg4);                         \
1489       _argvec[5] = (unsigned long)(arg5);                         \
1490       _argvec[6] = (unsigned long)(arg6);                         \
1491       __asm__ volatile(                                           \
1492          VALGRIND_ALIGN_STACK                                     \
1493          "subl $8, %%esp\n\t"                                     \
1494          "pushl 24(%%eax)\n\t"                                    \
1495          "pushl 20(%%eax)\n\t"                                    \
1496          "pushl 16(%%eax)\n\t"                                    \
1497          "pushl 12(%%eax)\n\t"                                    \
1498          "pushl 8(%%eax)\n\t"                                     \
1499          "pushl 4(%%eax)\n\t"                                     \
1500          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1501          VALGRIND_CALL_NOREDIR_EAX                                \
1502          VALGRIND_RESTORE_STACK                                   \
1503          : /*out*/   "=a" (_res)                                  \
1504          : /*in*/    "a" (&_argvec[0])                            \
1505          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1506       );                                                          \
1507       lval = (__typeof__(lval)) _res;                             \
1508    } while (0)
1509 
1510 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1511                                  arg7)                            \
1512    do {                                                           \
1513       volatile OrigFn        _orig = (orig);                      \
1514       volatile unsigned long _argvec[8];                          \
1515       volatile unsigned long _res;                                \
1516       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1517       _argvec[1] = (unsigned long)(arg1);                         \
1518       _argvec[2] = (unsigned long)(arg2);                         \
1519       _argvec[3] = (unsigned long)(arg3);                         \
1520       _argvec[4] = (unsigned long)(arg4);                         \
1521       _argvec[5] = (unsigned long)(arg5);                         \
1522       _argvec[6] = (unsigned long)(arg6);                         \
1523       _argvec[7] = (unsigned long)(arg7);                         \
1524       __asm__ volatile(                                           \
1525          VALGRIND_ALIGN_STACK                                     \
1526          "subl $4, %%esp\n\t"                                     \
1527          "pushl 28(%%eax)\n\t"                                    \
1528          "pushl 24(%%eax)\n\t"                                    \
1529          "pushl 20(%%eax)\n\t"                                    \
1530          "pushl 16(%%eax)\n\t"                                    \
1531          "pushl 12(%%eax)\n\t"                                    \
1532          "pushl 8(%%eax)\n\t"                                     \
1533          "pushl 4(%%eax)\n\t"                                     \
1534          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1535          VALGRIND_CALL_NOREDIR_EAX                                \
1536          VALGRIND_RESTORE_STACK                                   \
1537          : /*out*/   "=a" (_res)                                  \
1538          : /*in*/    "a" (&_argvec[0])                            \
1539          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1540       );                                                          \
1541       lval = (__typeof__(lval)) _res;                             \
1542    } while (0)
1543 
1544 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1545                                  arg7,arg8)                       \
1546    do {                                                           \
1547       volatile OrigFn        _orig = (orig);                      \
1548       volatile unsigned long _argvec[9];                          \
1549       volatile unsigned long _res;                                \
1550       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1551       _argvec[1] = (unsigned long)(arg1);                         \
1552       _argvec[2] = (unsigned long)(arg2);                         \
1553       _argvec[3] = (unsigned long)(arg3);                         \
1554       _argvec[4] = (unsigned long)(arg4);                         \
1555       _argvec[5] = (unsigned long)(arg5);                         \
1556       _argvec[6] = (unsigned long)(arg6);                         \
1557       _argvec[7] = (unsigned long)(arg7);                         \
1558       _argvec[8] = (unsigned long)(arg8);                         \
1559       __asm__ volatile(                                           \
1560          VALGRIND_ALIGN_STACK                                     \
1561          "pushl 32(%%eax)\n\t"                                    \
1562          "pushl 28(%%eax)\n\t"                                    \
1563          "pushl 24(%%eax)\n\t"                                    \
1564          "pushl 20(%%eax)\n\t"                                    \
1565          "pushl 16(%%eax)\n\t"                                    \
1566          "pushl 12(%%eax)\n\t"                                    \
1567          "pushl 8(%%eax)\n\t"                                     \
1568          "pushl 4(%%eax)\n\t"                                     \
1569          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1570          VALGRIND_CALL_NOREDIR_EAX                                \
1571          VALGRIND_RESTORE_STACK                                   \
1572          : /*out*/   "=a" (_res)                                  \
1573          : /*in*/    "a" (&_argvec[0])                            \
1574          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1575       );                                                          \
1576       lval = (__typeof__(lval)) _res;                             \
1577    } while (0)
1578 
1579 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1580                                  arg7,arg8,arg9)                  \
1581    do {                                                           \
1582       volatile OrigFn        _orig = (orig);                      \
1583       volatile unsigned long _argvec[10];                         \
1584       volatile unsigned long _res;                                \
1585       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1586       _argvec[1] = (unsigned long)(arg1);                         \
1587       _argvec[2] = (unsigned long)(arg2);                         \
1588       _argvec[3] = (unsigned long)(arg3);                         \
1589       _argvec[4] = (unsigned long)(arg4);                         \
1590       _argvec[5] = (unsigned long)(arg5);                         \
1591       _argvec[6] = (unsigned long)(arg6);                         \
1592       _argvec[7] = (unsigned long)(arg7);                         \
1593       _argvec[8] = (unsigned long)(arg8);                         \
1594       _argvec[9] = (unsigned long)(arg9);                         \
1595       __asm__ volatile(                                           \
1596          VALGRIND_ALIGN_STACK                                     \
1597          "subl $12, %%esp\n\t"                                    \
1598          "pushl 36(%%eax)\n\t"                                    \
1599          "pushl 32(%%eax)\n\t"                                    \
1600          "pushl 28(%%eax)\n\t"                                    \
1601          "pushl 24(%%eax)\n\t"                                    \
1602          "pushl 20(%%eax)\n\t"                                    \
1603          "pushl 16(%%eax)\n\t"                                    \
1604          "pushl 12(%%eax)\n\t"                                    \
1605          "pushl 8(%%eax)\n\t"                                     \
1606          "pushl 4(%%eax)\n\t"                                     \
1607          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1608          VALGRIND_CALL_NOREDIR_EAX                                \
1609          VALGRIND_RESTORE_STACK                                   \
1610          : /*out*/   "=a" (_res)                                  \
1611          : /*in*/    "a" (&_argvec[0])                            \
1612          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1613       );                                                          \
1614       lval = (__typeof__(lval)) _res;                             \
1615    } while (0)
1616 
1617 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1618                                   arg7,arg8,arg9,arg10)           \
1619    do {                                                           \
1620       volatile OrigFn        _orig = (orig);                      \
1621       volatile unsigned long _argvec[11];                         \
1622       volatile unsigned long _res;                                \
1623       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1624       _argvec[1] = (unsigned long)(arg1);                         \
1625       _argvec[2] = (unsigned long)(arg2);                         \
1626       _argvec[3] = (unsigned long)(arg3);                         \
1627       _argvec[4] = (unsigned long)(arg4);                         \
1628       _argvec[5] = (unsigned long)(arg5);                         \
1629       _argvec[6] = (unsigned long)(arg6);                         \
1630       _argvec[7] = (unsigned long)(arg7);                         \
1631       _argvec[8] = (unsigned long)(arg8);                         \
1632       _argvec[9] = (unsigned long)(arg9);                         \
1633       _argvec[10] = (unsigned long)(arg10);                       \
1634       __asm__ volatile(                                           \
1635          VALGRIND_ALIGN_STACK                                     \
1636          "subl $8, %%esp\n\t"                                     \
1637          "pushl 40(%%eax)\n\t"                                    \
1638          "pushl 36(%%eax)\n\t"                                    \
1639          "pushl 32(%%eax)\n\t"                                    \
1640          "pushl 28(%%eax)\n\t"                                    \
1641          "pushl 24(%%eax)\n\t"                                    \
1642          "pushl 20(%%eax)\n\t"                                    \
1643          "pushl 16(%%eax)\n\t"                                    \
1644          "pushl 12(%%eax)\n\t"                                    \
1645          "pushl 8(%%eax)\n\t"                                     \
1646          "pushl 4(%%eax)\n\t"                                     \
1647          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1648          VALGRIND_CALL_NOREDIR_EAX                                \
1649          VALGRIND_RESTORE_STACK                                   \
1650          : /*out*/   "=a" (_res)                                  \
1651          : /*in*/    "a" (&_argvec[0])                            \
1652          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1653       );                                                          \
1654       lval = (__typeof__(lval)) _res;                             \
1655    } while (0)
1656 
1657 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1658                                   arg6,arg7,arg8,arg9,arg10,      \
1659                                   arg11)                          \
1660    do {                                                           \
1661       volatile OrigFn        _orig = (orig);                      \
1662       volatile unsigned long _argvec[12];                         \
1663       volatile unsigned long _res;                                \
1664       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1665       _argvec[1] = (unsigned long)(arg1);                         \
1666       _argvec[2] = (unsigned long)(arg2);                         \
1667       _argvec[3] = (unsigned long)(arg3);                         \
1668       _argvec[4] = (unsigned long)(arg4);                         \
1669       _argvec[5] = (unsigned long)(arg5);                         \
1670       _argvec[6] = (unsigned long)(arg6);                         \
1671       _argvec[7] = (unsigned long)(arg7);                         \
1672       _argvec[8] = (unsigned long)(arg8);                         \
1673       _argvec[9] = (unsigned long)(arg9);                         \
1674       _argvec[10] = (unsigned long)(arg10);                       \
1675       _argvec[11] = (unsigned long)(arg11);                       \
1676       __asm__ volatile(                                           \
1677          VALGRIND_ALIGN_STACK                                     \
1678          "subl $4, %%esp\n\t"                                     \
1679          "pushl 44(%%eax)\n\t"                                    \
1680          "pushl 40(%%eax)\n\t"                                    \
1681          "pushl 36(%%eax)\n\t"                                    \
1682          "pushl 32(%%eax)\n\t"                                    \
1683          "pushl 28(%%eax)\n\t"                                    \
1684          "pushl 24(%%eax)\n\t"                                    \
1685          "pushl 20(%%eax)\n\t"                                    \
1686          "pushl 16(%%eax)\n\t"                                    \
1687          "pushl 12(%%eax)\n\t"                                    \
1688          "pushl 8(%%eax)\n\t"                                     \
1689          "pushl 4(%%eax)\n\t"                                     \
1690          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1691          VALGRIND_CALL_NOREDIR_EAX                                \
1692          VALGRIND_RESTORE_STACK                                   \
1693          : /*out*/   "=a" (_res)                                  \
1694          : /*in*/    "a" (&_argvec[0])                            \
1695          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1696       );                                                          \
1697       lval = (__typeof__(lval)) _res;                             \
1698    } while (0)
1699 
1700 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1701                                   arg6,arg7,arg8,arg9,arg10,      \
1702                                   arg11,arg12)                    \
1703    do {                                                           \
1704       volatile OrigFn        _orig = (orig);                      \
1705       volatile unsigned long _argvec[13];                         \
1706       volatile unsigned long _res;                                \
1707       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1708       _argvec[1] = (unsigned long)(arg1);                         \
1709       _argvec[2] = (unsigned long)(arg2);                         \
1710       _argvec[3] = (unsigned long)(arg3);                         \
1711       _argvec[4] = (unsigned long)(arg4);                         \
1712       _argvec[5] = (unsigned long)(arg5);                         \
1713       _argvec[6] = (unsigned long)(arg6);                         \
1714       _argvec[7] = (unsigned long)(arg7);                         \
1715       _argvec[8] = (unsigned long)(arg8);                         \
1716       _argvec[9] = (unsigned long)(arg9);                         \
1717       _argvec[10] = (unsigned long)(arg10);                       \
1718       _argvec[11] = (unsigned long)(arg11);                       \
1719       _argvec[12] = (unsigned long)(arg12);                       \
1720       __asm__ volatile(                                           \
1721          VALGRIND_ALIGN_STACK                                     \
1722          "pushl 48(%%eax)\n\t"                                    \
1723          "pushl 44(%%eax)\n\t"                                    \
1724          "pushl 40(%%eax)\n\t"                                    \
1725          "pushl 36(%%eax)\n\t"                                    \
1726          "pushl 32(%%eax)\n\t"                                    \
1727          "pushl 28(%%eax)\n\t"                                    \
1728          "pushl 24(%%eax)\n\t"                                    \
1729          "pushl 20(%%eax)\n\t"                                    \
1730          "pushl 16(%%eax)\n\t"                                    \
1731          "pushl 12(%%eax)\n\t"                                    \
1732          "pushl 8(%%eax)\n\t"                                     \
1733          "pushl 4(%%eax)\n\t"                                     \
1734          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1735          VALGRIND_CALL_NOREDIR_EAX                                \
1736          VALGRIND_RESTORE_STACK                                   \
1737          : /*out*/   "=a" (_res)                                  \
1738          : /*in*/    "a" (&_argvec[0])                            \
1739          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1740       );                                                          \
1741       lval = (__typeof__(lval)) _res;                             \
1742    } while (0)
1743 
1744 #endif /* PLAT_x86_linux || PLAT_x86_darwin
1745           || PLAT_x86_solaris || PLAT_x86_freebsd */
1746 
1747 /* ------------- amd64-{linux,darwin,solaris,freebsd} ------------ */
1748 
1749 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin) \
1750     ||  defined(PLAT_amd64_solaris)  ||  defined(PLAT_amd64_freebsd)
1751 
1752 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1753 
1754 /* These regs are trashed by the hidden call. */
1755 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1756                             "rdi", "r8", "r9", "r10", "r11"
1757 
1758 /* This is all pretty complex.  It's so as to make stack unwinding
1759    work reliably.  See bug 243270.  The basic problem is the sub and
1760    add of 128 of %rsp in all of the following macros.  If gcc believes
1761    the CFA is in %rsp, then unwinding may fail, because what's at the
1762    CFA is not what gcc "expected" when it constructs the CFIs for the
1763    places where the macros are instantiated.
1764 
1765    But we can't just add a CFI annotation to increase the CFA offset
1766    by 128, to match the sub of 128 from %rsp, because we don't know
1767    whether gcc has chosen %rsp as the CFA at that point, or whether it
1768    has chosen some other register (eg, %rbp).  In the latter case,
1769    adding a CFI annotation to change the CFA offset is simply wrong.
1770 
1771    So the solution is to get hold of the CFA using
1772    __builtin_dwarf_cfa(), put it in a known register, and add a
1773    CFI annotation to say what the register is.  We choose %rbp for
1774    this (perhaps perversely), because:
1775 
1776    (1) %rbp is already subject to unwinding.  If a new register was
1777        chosen then the unwinder would have to unwind it in all stack
1778        traces, which is expensive, and
1779 
1780    (2) %rbp is already subject to precise exception updates in the
1781        JIT.  If a new register was chosen, we'd have to have precise
1782        exceptions for it too, which reduces performance of the
1783        generated code.
1784 
1785    However .. one extra complication.  We can't just whack the result
1786    of __builtin_dwarf_cfa() into %rbp and then add %rbp to the
1787    list of trashed registers at the end of the inline assembly
1788    fragments; gcc won't allow %rbp to appear in that list.  Hence
1789    instead we need to stash %rbp in %r15 for the duration of the asm,
1790    and say that %r15 is trashed instead.  gcc seems happy to go with
1791    that.
1792 
1793    Oh .. and this all needs to be conditionalised so that it is
1794    unchanged from before this commit, when compiled with older gccs
1795    that don't support __builtin_dwarf_cfa.  Furthermore, since
1796    this header file is freestanding, it has to be independent of
1797    config.h, and so the following conditionalisation cannot depend on
1798    configure time checks.
1799 
1800    Although it's not clear from
1801    'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)',
1802    this expression excludes Darwin.
1803    .cfi directives in Darwin assembly appear to be completely
1804    different and I haven't investigated how they work.
1805 
1806    For even more entertainment value, note we have to use the
1807    completely undocumented __builtin_dwarf_cfa(), which appears to
1808    really compute the CFA, whereas __builtin_frame_address(0) claims
1809    to but actually doesn't.  See
1810    https://bugs.kde.org/show_bug.cgi?id=243270#c47
1811 */
1812 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
1813 #  define __FRAME_POINTER                                         \
1814       ,"r"(__builtin_dwarf_cfa())
1815 #  define VALGRIND_CFI_PROLOGUE                                   \
1816       "movq %%rbp, %%r15\n\t"                                     \
1817       "movq %2, %%rbp\n\t"                                        \
1818       ".cfi_remember_state\n\t"                                   \
1819       ".cfi_def_cfa rbp, 0\n\t"
1820 #  define VALGRIND_CFI_EPILOGUE                                   \
1821       "movq %%r15, %%rbp\n\t"                                     \
1822       ".cfi_restore_state\n\t"
1823 #else
1824 #  define __FRAME_POINTER
1825 #  define VALGRIND_CFI_PROLOGUE
1826 #  define VALGRIND_CFI_EPILOGUE
1827 #endif
1828 
1829 /* Macros to save and align the stack before making a function
1830    call and restore it afterwards as gcc may not keep the stack
1831    pointer aligned if it doesn't realise calls are being made
1832    to other functions. */
1833 
1834 #define VALGRIND_ALIGN_STACK               \
1835       "movq %%rsp,%%r14\n\t"               \
1836       "andq $0xfffffffffffffff0,%%rsp\n\t"
1837 #define VALGRIND_RESTORE_STACK             \
1838       "movq %%r14,%%rsp\n\t"
1839 
1840 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1841    long) == 8. */
1842 
1843 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1844    macros.  In order not to trash the stack redzone, we need to drop
1845    %rsp by 128 before the hidden call, and restore afterwards.  The
1846    nastyness is that it is only by luck that the stack still appears
1847    to be unwindable during the hidden call - since then the behaviour
1848    of any routine using this macro does not match what the CFI data
1849    says.  Sigh.
1850 
1851    Why is this important?  Imagine that a wrapper has a stack
1852    allocated local, and passes to the hidden call, a pointer to it.
1853    Because gcc does not know about the hidden call, it may allocate
1854    that local in the redzone.  Unfortunately the hidden call may then
1855    trash it before it comes to use it.  So we must step clear of the
1856    redzone, for the duration of the hidden call, to make it safe.
1857 
1858    Probably the same problem afflicts the other redzone-style ABIs too
1859    (ppc64-linux); but for those, the stack is
1860    self describing (none of this CFI nonsense) so at least messing
1861    with the stack pointer doesn't give a danger of non-unwindable
1862    stack. */
1863 
1864 #define CALL_FN_W_v(lval, orig)                                        \
1865    do {                                                                \
1866       volatile OrigFn        _orig = (orig);                           \
1867       volatile unsigned long _argvec[1];                               \
1868       volatile unsigned long _res;                                     \
1869       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1870       __asm__ volatile(                                                \
1871          VALGRIND_CFI_PROLOGUE                                         \
1872          VALGRIND_ALIGN_STACK                                          \
1873          "subq $128,%%rsp\n\t"                                         \
1874          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1875          VALGRIND_CALL_NOREDIR_RAX                                     \
1876          VALGRIND_RESTORE_STACK                                        \
1877          VALGRIND_CFI_EPILOGUE                                         \
1878          : /*out*/   "=a" (_res)                                       \
1879          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1880          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1881       );                                                               \
1882       lval = (__typeof__(lval)) _res;                                  \
1883    } while (0)
1884 
1885 #define CALL_FN_W_W(lval, orig, arg1)                                  \
1886    do {                                                                \
1887       volatile OrigFn        _orig = (orig);                           \
1888       volatile unsigned long _argvec[2];                               \
1889       volatile unsigned long _res;                                     \
1890       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1891       _argvec[1] = (unsigned long)(arg1);                              \
1892       __asm__ volatile(                                                \
1893          VALGRIND_CFI_PROLOGUE                                         \
1894          VALGRIND_ALIGN_STACK                                          \
1895          "subq $128,%%rsp\n\t"                                         \
1896          "movq 8(%%rax), %%rdi\n\t"                                    \
1897          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1898          VALGRIND_CALL_NOREDIR_RAX                                     \
1899          VALGRIND_RESTORE_STACK                                        \
1900          VALGRIND_CFI_EPILOGUE                                         \
1901          : /*out*/   "=a" (_res)                                       \
1902          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1903          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1904       );                                                               \
1905       lval = (__typeof__(lval)) _res;                                  \
1906    } while (0)
1907 
1908 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                            \
1909    do {                                                                \
1910       volatile OrigFn        _orig = (orig);                           \
1911       volatile unsigned long _argvec[3];                               \
1912       volatile unsigned long _res;                                     \
1913       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1914       _argvec[1] = (unsigned long)(arg1);                              \
1915       _argvec[2] = (unsigned long)(arg2);                              \
1916       __asm__ volatile(                                                \
1917          VALGRIND_CFI_PROLOGUE                                         \
1918          VALGRIND_ALIGN_STACK                                          \
1919          "subq $128,%%rsp\n\t"                                         \
1920          "movq 16(%%rax), %%rsi\n\t"                                   \
1921          "movq 8(%%rax), %%rdi\n\t"                                    \
1922          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1923          VALGRIND_CALL_NOREDIR_RAX                                     \
1924          VALGRIND_RESTORE_STACK                                        \
1925          VALGRIND_CFI_EPILOGUE                                         \
1926          : /*out*/   "=a" (_res)                                       \
1927          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1928          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1929       );                                                               \
1930       lval = (__typeof__(lval)) _res;                                  \
1931    } while (0)
1932 
1933 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                      \
1934    do {                                                                \
1935       volatile OrigFn        _orig = (orig);                           \
1936       volatile unsigned long _argvec[4];                               \
1937       volatile unsigned long _res;                                     \
1938       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1939       _argvec[1] = (unsigned long)(arg1);                              \
1940       _argvec[2] = (unsigned long)(arg2);                              \
1941       _argvec[3] = (unsigned long)(arg3);                              \
1942       __asm__ volatile(                                                \
1943          VALGRIND_CFI_PROLOGUE                                         \
1944          VALGRIND_ALIGN_STACK                                          \
1945          "subq $128,%%rsp\n\t"                                         \
1946          "movq 24(%%rax), %%rdx\n\t"                                   \
1947          "movq 16(%%rax), %%rsi\n\t"                                   \
1948          "movq 8(%%rax), %%rdi\n\t"                                    \
1949          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1950          VALGRIND_CALL_NOREDIR_RAX                                     \
1951          VALGRIND_RESTORE_STACK                                        \
1952          VALGRIND_CFI_EPILOGUE                                         \
1953          : /*out*/   "=a" (_res)                                       \
1954          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1955          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1956       );                                                               \
1957       lval = (__typeof__(lval)) _res;                                  \
1958    } while (0)
1959 
1960 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)                \
1961    do {                                                                \
1962       volatile OrigFn        _orig = (orig);                           \
1963       volatile unsigned long _argvec[5];                               \
1964       volatile unsigned long _res;                                     \
1965       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1966       _argvec[1] = (unsigned long)(arg1);                              \
1967       _argvec[2] = (unsigned long)(arg2);                              \
1968       _argvec[3] = (unsigned long)(arg3);                              \
1969       _argvec[4] = (unsigned long)(arg4);                              \
1970       __asm__ volatile(                                                \
1971          VALGRIND_CFI_PROLOGUE                                         \
1972          VALGRIND_ALIGN_STACK                                          \
1973          "subq $128,%%rsp\n\t"                                         \
1974          "movq 32(%%rax), %%rcx\n\t"                                   \
1975          "movq 24(%%rax), %%rdx\n\t"                                   \
1976          "movq 16(%%rax), %%rsi\n\t"                                   \
1977          "movq 8(%%rax), %%rdi\n\t"                                    \
1978          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1979          VALGRIND_CALL_NOREDIR_RAX                                     \
1980          VALGRIND_RESTORE_STACK                                        \
1981          VALGRIND_CFI_EPILOGUE                                         \
1982          : /*out*/   "=a" (_res)                                       \
1983          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1984          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1985       );                                                               \
1986       lval = (__typeof__(lval)) _res;                                  \
1987    } while (0)
1988 
1989 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)             \
1990    do {                                                                \
1991       volatile OrigFn        _orig = (orig);                           \
1992       volatile unsigned long _argvec[6];                               \
1993       volatile unsigned long _res;                                     \
1994       _argvec[0] = (unsigned long)_orig.nraddr;                        \
1995       _argvec[1] = (unsigned long)(arg1);                              \
1996       _argvec[2] = (unsigned long)(arg2);                              \
1997       _argvec[3] = (unsigned long)(arg3);                              \
1998       _argvec[4] = (unsigned long)(arg4);                              \
1999       _argvec[5] = (unsigned long)(arg5);                              \
2000       __asm__ volatile(                                                \
2001          VALGRIND_CFI_PROLOGUE                                         \
2002          VALGRIND_ALIGN_STACK                                          \
2003          "subq $128,%%rsp\n\t"                                         \
2004          "movq 40(%%rax), %%r8\n\t"                                    \
2005          "movq 32(%%rax), %%rcx\n\t"                                   \
2006          "movq 24(%%rax), %%rdx\n\t"                                   \
2007          "movq 16(%%rax), %%rsi\n\t"                                   \
2008          "movq 8(%%rax), %%rdi\n\t"                                    \
2009          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2010          VALGRIND_CALL_NOREDIR_RAX                                     \
2011          VALGRIND_RESTORE_STACK                                        \
2012          VALGRIND_CFI_EPILOGUE                                         \
2013          : /*out*/   "=a" (_res)                                       \
2014          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2015          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2016       );                                                               \
2017       lval = (__typeof__(lval)) _res;                                  \
2018    } while (0)
2019 
2020 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)        \
2021    do {                                                                \
2022       volatile OrigFn        _orig = (orig);                           \
2023       volatile unsigned long _argvec[7];                               \
2024       volatile unsigned long _res;                                     \
2025       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2026       _argvec[1] = (unsigned long)(arg1);                              \
2027       _argvec[2] = (unsigned long)(arg2);                              \
2028       _argvec[3] = (unsigned long)(arg3);                              \
2029       _argvec[4] = (unsigned long)(arg4);                              \
2030       _argvec[5] = (unsigned long)(arg5);                              \
2031       _argvec[6] = (unsigned long)(arg6);                              \
2032       __asm__ volatile(                                                \
2033          VALGRIND_CFI_PROLOGUE                                         \
2034          VALGRIND_ALIGN_STACK                                          \
2035          "subq $128,%%rsp\n\t"                                         \
2036          "movq 48(%%rax), %%r9\n\t"                                    \
2037          "movq 40(%%rax), %%r8\n\t"                                    \
2038          "movq 32(%%rax), %%rcx\n\t"                                   \
2039          "movq 24(%%rax), %%rdx\n\t"                                   \
2040          "movq 16(%%rax), %%rsi\n\t"                                   \
2041          "movq 8(%%rax), %%rdi\n\t"                                    \
2042          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2043          VALGRIND_CALL_NOREDIR_RAX                                     \
2044          VALGRIND_RESTORE_STACK                                        \
2045          VALGRIND_CFI_EPILOGUE                                         \
2046          : /*out*/   "=a" (_res)                                       \
2047          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2048          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2049       );                                                               \
2050       lval = (__typeof__(lval)) _res;                                  \
2051    } while (0)
2052 
2053 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
2054                                  arg7)                                 \
2055    do {                                                                \
2056       volatile OrigFn        _orig = (orig);                           \
2057       volatile unsigned long _argvec[8];                               \
2058       volatile unsigned long _res;                                     \
2059       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2060       _argvec[1] = (unsigned long)(arg1);                              \
2061       _argvec[2] = (unsigned long)(arg2);                              \
2062       _argvec[3] = (unsigned long)(arg3);                              \
2063       _argvec[4] = (unsigned long)(arg4);                              \
2064       _argvec[5] = (unsigned long)(arg5);                              \
2065       _argvec[6] = (unsigned long)(arg6);                              \
2066       _argvec[7] = (unsigned long)(arg7);                              \
2067       __asm__ volatile(                                                \
2068          VALGRIND_CFI_PROLOGUE                                         \
2069          VALGRIND_ALIGN_STACK                                          \
2070          "subq $136,%%rsp\n\t"                                         \
2071          "pushq 56(%%rax)\n\t"                                         \
2072          "movq 48(%%rax), %%r9\n\t"                                    \
2073          "movq 40(%%rax), %%r8\n\t"                                    \
2074          "movq 32(%%rax), %%rcx\n\t"                                   \
2075          "movq 24(%%rax), %%rdx\n\t"                                   \
2076          "movq 16(%%rax), %%rsi\n\t"                                   \
2077          "movq 8(%%rax), %%rdi\n\t"                                    \
2078          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2079          VALGRIND_CALL_NOREDIR_RAX                                     \
2080          VALGRIND_RESTORE_STACK                                        \
2081          VALGRIND_CFI_EPILOGUE                                         \
2082          : /*out*/   "=a" (_res)                                       \
2083          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2084          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2085       );                                                               \
2086       lval = (__typeof__(lval)) _res;                                  \
2087    } while (0)
2088 
2089 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
2090                                  arg7,arg8)                            \
2091    do {                                                                \
2092       volatile OrigFn        _orig = (orig);                           \
2093       volatile unsigned long _argvec[9];                               \
2094       volatile unsigned long _res;                                     \
2095       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2096       _argvec[1] = (unsigned long)(arg1);                              \
2097       _argvec[2] = (unsigned long)(arg2);                              \
2098       _argvec[3] = (unsigned long)(arg3);                              \
2099       _argvec[4] = (unsigned long)(arg4);                              \
2100       _argvec[5] = (unsigned long)(arg5);                              \
2101       _argvec[6] = (unsigned long)(arg6);                              \
2102       _argvec[7] = (unsigned long)(arg7);                              \
2103       _argvec[8] = (unsigned long)(arg8);                              \
2104       __asm__ volatile(                                                \
2105          VALGRIND_CFI_PROLOGUE                                         \
2106          VALGRIND_ALIGN_STACK                                          \
2107          "subq $128,%%rsp\n\t"                                         \
2108          "pushq 64(%%rax)\n\t"                                         \
2109          "pushq 56(%%rax)\n\t"                                         \
2110          "movq 48(%%rax), %%r9\n\t"                                    \
2111          "movq 40(%%rax), %%r8\n\t"                                    \
2112          "movq 32(%%rax), %%rcx\n\t"                                   \
2113          "movq 24(%%rax), %%rdx\n\t"                                   \
2114          "movq 16(%%rax), %%rsi\n\t"                                   \
2115          "movq 8(%%rax), %%rdi\n\t"                                    \
2116          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2117          VALGRIND_CALL_NOREDIR_RAX                                     \
2118          VALGRIND_RESTORE_STACK                                        \
2119          VALGRIND_CFI_EPILOGUE                                         \
2120          : /*out*/   "=a" (_res)                                       \
2121          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2122          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2123       );                                                               \
2124       lval = (__typeof__(lval)) _res;                                  \
2125    } while (0)
2126 
2127 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
2128                                  arg7,arg8,arg9)                       \
2129    do {                                                                \
2130       volatile OrigFn        _orig = (orig);                           \
2131       volatile unsigned long _argvec[10];                              \
2132       volatile unsigned long _res;                                     \
2133       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2134       _argvec[1] = (unsigned long)(arg1);                              \
2135       _argvec[2] = (unsigned long)(arg2);                              \
2136       _argvec[3] = (unsigned long)(arg3);                              \
2137       _argvec[4] = (unsigned long)(arg4);                              \
2138       _argvec[5] = (unsigned long)(arg5);                              \
2139       _argvec[6] = (unsigned long)(arg6);                              \
2140       _argvec[7] = (unsigned long)(arg7);                              \
2141       _argvec[8] = (unsigned long)(arg8);                              \
2142       _argvec[9] = (unsigned long)(arg9);                              \
2143       __asm__ volatile(                                                \
2144          VALGRIND_CFI_PROLOGUE                                         \
2145          VALGRIND_ALIGN_STACK                                          \
2146          "subq $136,%%rsp\n\t"                                         \
2147          "pushq 72(%%rax)\n\t"                                         \
2148          "pushq 64(%%rax)\n\t"                                         \
2149          "pushq 56(%%rax)\n\t"                                         \
2150          "movq 48(%%rax), %%r9\n\t"                                    \
2151          "movq 40(%%rax), %%r8\n\t"                                    \
2152          "movq 32(%%rax), %%rcx\n\t"                                   \
2153          "movq 24(%%rax), %%rdx\n\t"                                   \
2154          "movq 16(%%rax), %%rsi\n\t"                                   \
2155          "movq 8(%%rax), %%rdi\n\t"                                    \
2156          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2157          VALGRIND_CALL_NOREDIR_RAX                                     \
2158          VALGRIND_RESTORE_STACK                                        \
2159          VALGRIND_CFI_EPILOGUE                                         \
2160          : /*out*/   "=a" (_res)                                       \
2161          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2162          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2163       );                                                               \
2164       lval = (__typeof__(lval)) _res;                                  \
2165    } while (0)
2166 
2167 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
2168                                   arg7,arg8,arg9,arg10)                \
2169    do {                                                                \
2170       volatile OrigFn        _orig = (orig);                           \
2171       volatile unsigned long _argvec[11];                              \
2172       volatile unsigned long _res;                                     \
2173       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2174       _argvec[1] = (unsigned long)(arg1);                              \
2175       _argvec[2] = (unsigned long)(arg2);                              \
2176       _argvec[3] = (unsigned long)(arg3);                              \
2177       _argvec[4] = (unsigned long)(arg4);                              \
2178       _argvec[5] = (unsigned long)(arg5);                              \
2179       _argvec[6] = (unsigned long)(arg6);                              \
2180       _argvec[7] = (unsigned long)(arg7);                              \
2181       _argvec[8] = (unsigned long)(arg8);                              \
2182       _argvec[9] = (unsigned long)(arg9);                              \
2183       _argvec[10] = (unsigned long)(arg10);                            \
2184       __asm__ volatile(                                                \
2185          VALGRIND_CFI_PROLOGUE                                         \
2186          VALGRIND_ALIGN_STACK                                          \
2187          "subq $128,%%rsp\n\t"                                         \
2188          "pushq 80(%%rax)\n\t"                                         \
2189          "pushq 72(%%rax)\n\t"                                         \
2190          "pushq 64(%%rax)\n\t"                                         \
2191          "pushq 56(%%rax)\n\t"                                         \
2192          "movq 48(%%rax), %%r9\n\t"                                    \
2193          "movq 40(%%rax), %%r8\n\t"                                    \
2194          "movq 32(%%rax), %%rcx\n\t"                                   \
2195          "movq 24(%%rax), %%rdx\n\t"                                   \
2196          "movq 16(%%rax), %%rsi\n\t"                                   \
2197          "movq 8(%%rax), %%rdi\n\t"                                    \
2198          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2199          VALGRIND_CALL_NOREDIR_RAX                                     \
2200          VALGRIND_RESTORE_STACK                                        \
2201          VALGRIND_CFI_EPILOGUE                                         \
2202          : /*out*/   "=a" (_res)                                       \
2203          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2204          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2205       );                                                               \
2206       lval = (__typeof__(lval)) _res;                                  \
2207    } while (0)
2208 
2209 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
2210                                   arg7,arg8,arg9,arg10,arg11)          \
2211    do {                                                                \
2212       volatile OrigFn        _orig = (orig);                           \
2213       volatile unsigned long _argvec[12];                              \
2214       volatile unsigned long _res;                                     \
2215       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2216       _argvec[1] = (unsigned long)(arg1);                              \
2217       _argvec[2] = (unsigned long)(arg2);                              \
2218       _argvec[3] = (unsigned long)(arg3);                              \
2219       _argvec[4] = (unsigned long)(arg4);                              \
2220       _argvec[5] = (unsigned long)(arg5);                              \
2221       _argvec[6] = (unsigned long)(arg6);                              \
2222       _argvec[7] = (unsigned long)(arg7);                              \
2223       _argvec[8] = (unsigned long)(arg8);                              \
2224       _argvec[9] = (unsigned long)(arg9);                              \
2225       _argvec[10] = (unsigned long)(arg10);                            \
2226       _argvec[11] = (unsigned long)(arg11);                            \
2227       __asm__ volatile(                                                \
2228          VALGRIND_CFI_PROLOGUE                                         \
2229          VALGRIND_ALIGN_STACK                                          \
2230          "subq $136,%%rsp\n\t"                                         \
2231          "pushq 88(%%rax)\n\t"                                         \
2232          "pushq 80(%%rax)\n\t"                                         \
2233          "pushq 72(%%rax)\n\t"                                         \
2234          "pushq 64(%%rax)\n\t"                                         \
2235          "pushq 56(%%rax)\n\t"                                         \
2236          "movq 48(%%rax), %%r9\n\t"                                    \
2237          "movq 40(%%rax), %%r8\n\t"                                    \
2238          "movq 32(%%rax), %%rcx\n\t"                                   \
2239          "movq 24(%%rax), %%rdx\n\t"                                   \
2240          "movq 16(%%rax), %%rsi\n\t"                                   \
2241          "movq 8(%%rax), %%rdi\n\t"                                    \
2242          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2243          VALGRIND_CALL_NOREDIR_RAX                                     \
2244          VALGRIND_RESTORE_STACK                                        \
2245          VALGRIND_CFI_EPILOGUE                                         \
2246          : /*out*/   "=a" (_res)                                       \
2247          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2248          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2249       );                                                               \
2250       lval = (__typeof__(lval)) _res;                                  \
2251    } while (0)
2252 
2253 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
2254                                 arg7,arg8,arg9,arg10,arg11,arg12)      \
2255    do {                                                                \
2256       volatile OrigFn        _orig = (orig);                           \
2257       volatile unsigned long _argvec[13];                              \
2258       volatile unsigned long _res;                                     \
2259       _argvec[0] = (unsigned long)_orig.nraddr;                        \
2260       _argvec[1] = (unsigned long)(arg1);                              \
2261       _argvec[2] = (unsigned long)(arg2);                              \
2262       _argvec[3] = (unsigned long)(arg3);                              \
2263       _argvec[4] = (unsigned long)(arg4);                              \
2264       _argvec[5] = (unsigned long)(arg5);                              \
2265       _argvec[6] = (unsigned long)(arg6);                              \
2266       _argvec[7] = (unsigned long)(arg7);                              \
2267       _argvec[8] = (unsigned long)(arg8);                              \
2268       _argvec[9] = (unsigned long)(arg9);                              \
2269       _argvec[10] = (unsigned long)(arg10);                            \
2270       _argvec[11] = (unsigned long)(arg11);                            \
2271       _argvec[12] = (unsigned long)(arg12);                            \
2272       __asm__ volatile(                                                \
2273          VALGRIND_CFI_PROLOGUE                                         \
2274          VALGRIND_ALIGN_STACK                                          \
2275          "subq $128,%%rsp\n\t"                                         \
2276          "pushq 96(%%rax)\n\t"                                         \
2277          "pushq 88(%%rax)\n\t"                                         \
2278          "pushq 80(%%rax)\n\t"                                         \
2279          "pushq 72(%%rax)\n\t"                                         \
2280          "pushq 64(%%rax)\n\t"                                         \
2281          "pushq 56(%%rax)\n\t"                                         \
2282          "movq 48(%%rax), %%r9\n\t"                                    \
2283          "movq 40(%%rax), %%r8\n\t"                                    \
2284          "movq 32(%%rax), %%rcx\n\t"                                   \
2285          "movq 24(%%rax), %%rdx\n\t"                                   \
2286          "movq 16(%%rax), %%rsi\n\t"                                   \
2287          "movq 8(%%rax), %%rdi\n\t"                                    \
2288          "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2289          VALGRIND_CALL_NOREDIR_RAX                                     \
2290          VALGRIND_RESTORE_STACK                                        \
2291          VALGRIND_CFI_EPILOGUE                                         \
2292          : /*out*/   "=a" (_res)                                       \
2293          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2294          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2295       );                                                               \
2296       lval = (__typeof__(lval)) _res;                                  \
2297    } while (0)
2298 
2299 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin
2300           || PLAT_amd64_solaris || PLAT_amd64_freebsd */
2301 
2302 /* ------------------------ ppc32-linux ------------------------ */
2303 
2304 #if defined(PLAT_ppc32_linux)
2305 
2306 /* This is useful for finding out about the on-stack stuff:
2307 
2308    extern int f9  ( int,int,int,int,int,int,int,int,int );
2309    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
2310    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
2311    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
2312 
2313    int g9 ( void ) {
2314       return f9(11,22,33,44,55,66,77,88,99);
2315    }
2316    int g10 ( void ) {
2317       return f10(11,22,33,44,55,66,77,88,99,110);
2318    }
2319    int g11 ( void ) {
2320       return f11(11,22,33,44,55,66,77,88,99,110,121);
2321    }
2322    int g12 ( void ) {
2323       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
2324    }
2325 */
2326 
2327 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2328 
2329 /* These regs are trashed by the hidden call. */
2330 #define __CALLER_SAVED_REGS                                       \
2331    "lr", "ctr", "xer",                                            \
2332    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2333    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2334    "r11", "r12", "r13"
2335 
2336 /* Macros to save and align the stack before making a function
2337    call and restore it afterwards as gcc may not keep the stack
2338    pointer aligned if it doesn't realise calls are being made
2339    to other functions. */
2340 
2341 #define VALGRIND_ALIGN_STACK               \
2342       "mr 28,1\n\t"                        \
2343       "rlwinm 1,1,0,0,27\n\t"
2344 #define VALGRIND_RESTORE_STACK             \
2345       "mr 1,28\n\t"
2346 
2347 /* These CALL_FN_ macros assume that on ppc32-linux, 
2348    sizeof(unsigned long) == 4. */
2349 
2350 #define CALL_FN_W_v(lval, orig)                                   \
2351    do {                                                           \
2352       volatile OrigFn        _orig = (orig);                      \
2353       volatile unsigned long _argvec[1];                          \
2354       volatile unsigned long _res;                                \
2355       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2356       __asm__ volatile(                                           \
2357          VALGRIND_ALIGN_STACK                                     \
2358          "mr 11,%1\n\t"                                           \
2359          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2360          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2361          VALGRIND_RESTORE_STACK                                   \
2362          "mr %0,3"                                                \
2363          : /*out*/   "=r" (_res)                                  \
2364          : /*in*/    "r" (&_argvec[0])                            \
2365          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2366       );                                                          \
2367       lval = (__typeof__(lval)) _res;                             \
2368    } while (0)
2369 
2370 #define CALL_FN_W_W(lval, orig, arg1)                             \
2371    do {                                                           \
2372       volatile OrigFn        _orig = (orig);                      \
2373       volatile unsigned long _argvec[2];                          \
2374       volatile unsigned long _res;                                \
2375       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2376       _argvec[1] = (unsigned long)arg1;                           \
2377       __asm__ volatile(                                           \
2378          VALGRIND_ALIGN_STACK                                     \
2379          "mr 11,%1\n\t"                                           \
2380          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2381          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2382          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2383          VALGRIND_RESTORE_STACK                                   \
2384          "mr %0,3"                                                \
2385          : /*out*/   "=r" (_res)                                  \
2386          : /*in*/    "r" (&_argvec[0])                            \
2387          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2388       );                                                          \
2389       lval = (__typeof__(lval)) _res;                             \
2390    } while (0)
2391 
2392 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2393    do {                                                           \
2394       volatile OrigFn        _orig = (orig);                      \
2395       volatile unsigned long _argvec[3];                          \
2396       volatile unsigned long _res;                                \
2397       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2398       _argvec[1] = (unsigned long)arg1;                           \
2399       _argvec[2] = (unsigned long)arg2;                           \
2400       __asm__ volatile(                                           \
2401          VALGRIND_ALIGN_STACK                                     \
2402          "mr 11,%1\n\t"                                           \
2403          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2404          "lwz 4,8(11)\n\t"                                        \
2405          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2406          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2407          VALGRIND_RESTORE_STACK                                   \
2408          "mr %0,3"                                                \
2409          : /*out*/   "=r" (_res)                                  \
2410          : /*in*/    "r" (&_argvec[0])                            \
2411          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2412       );                                                          \
2413       lval = (__typeof__(lval)) _res;                             \
2414    } while (0)
2415 
2416 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2417    do {                                                           \
2418       volatile OrigFn        _orig = (orig);                      \
2419       volatile unsigned long _argvec[4];                          \
2420       volatile unsigned long _res;                                \
2421       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2422       _argvec[1] = (unsigned long)arg1;                           \
2423       _argvec[2] = (unsigned long)arg2;                           \
2424       _argvec[3] = (unsigned long)arg3;                           \
2425       __asm__ volatile(                                           \
2426          VALGRIND_ALIGN_STACK                                     \
2427          "mr 11,%1\n\t"                                           \
2428          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2429          "lwz 4,8(11)\n\t"                                        \
2430          "lwz 5,12(11)\n\t"                                       \
2431          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2432          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2433          VALGRIND_RESTORE_STACK                                   \
2434          "mr %0,3"                                                \
2435          : /*out*/   "=r" (_res)                                  \
2436          : /*in*/    "r" (&_argvec[0])                            \
2437          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2438       );                                                          \
2439       lval = (__typeof__(lval)) _res;                             \
2440    } while (0)
2441 
2442 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2443    do {                                                           \
2444       volatile OrigFn        _orig = (orig);                      \
2445       volatile unsigned long _argvec[5];                          \
2446       volatile unsigned long _res;                                \
2447       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2448       _argvec[1] = (unsigned long)arg1;                           \
2449       _argvec[2] = (unsigned long)arg2;                           \
2450       _argvec[3] = (unsigned long)arg3;                           \
2451       _argvec[4] = (unsigned long)arg4;                           \
2452       __asm__ volatile(                                           \
2453          VALGRIND_ALIGN_STACK                                     \
2454          "mr 11,%1\n\t"                                           \
2455          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2456          "lwz 4,8(11)\n\t"                                        \
2457          "lwz 5,12(11)\n\t"                                       \
2458          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2459          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2460          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2461          VALGRIND_RESTORE_STACK                                   \
2462          "mr %0,3"                                                \
2463          : /*out*/   "=r" (_res)                                  \
2464          : /*in*/    "r" (&_argvec[0])                            \
2465          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2466       );                                                          \
2467       lval = (__typeof__(lval)) _res;                             \
2468    } while (0)
2469 
2470 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2471    do {                                                           \
2472       volatile OrigFn        _orig = (orig);                      \
2473       volatile unsigned long _argvec[6];                          \
2474       volatile unsigned long _res;                                \
2475       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2476       _argvec[1] = (unsigned long)arg1;                           \
2477       _argvec[2] = (unsigned long)arg2;                           \
2478       _argvec[3] = (unsigned long)arg3;                           \
2479       _argvec[4] = (unsigned long)arg4;                           \
2480       _argvec[5] = (unsigned long)arg5;                           \
2481       __asm__ volatile(                                           \
2482          VALGRIND_ALIGN_STACK                                     \
2483          "mr 11,%1\n\t"                                           \
2484          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2485          "lwz 4,8(11)\n\t"                                        \
2486          "lwz 5,12(11)\n\t"                                       \
2487          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2488          "lwz 7,20(11)\n\t"                                       \
2489          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2490          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2491          VALGRIND_RESTORE_STACK                                   \
2492          "mr %0,3"                                                \
2493          : /*out*/   "=r" (_res)                                  \
2494          : /*in*/    "r" (&_argvec[0])                            \
2495          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2496       );                                                          \
2497       lval = (__typeof__(lval)) _res;                             \
2498    } while (0)
2499 
2500 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2501    do {                                                           \
2502       volatile OrigFn        _orig = (orig);                      \
2503       volatile unsigned long _argvec[7];                          \
2504       volatile unsigned long _res;                                \
2505       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2506       _argvec[1] = (unsigned long)arg1;                           \
2507       _argvec[2] = (unsigned long)arg2;                           \
2508       _argvec[3] = (unsigned long)arg3;                           \
2509       _argvec[4] = (unsigned long)arg4;                           \
2510       _argvec[5] = (unsigned long)arg5;                           \
2511       _argvec[6] = (unsigned long)arg6;                           \
2512       __asm__ volatile(                                           \
2513          VALGRIND_ALIGN_STACK                                     \
2514          "mr 11,%1\n\t"                                           \
2515          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2516          "lwz 4,8(11)\n\t"                                        \
2517          "lwz 5,12(11)\n\t"                                       \
2518          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2519          "lwz 7,20(11)\n\t"                                       \
2520          "lwz 8,24(11)\n\t"                                       \
2521          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2522          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2523          VALGRIND_RESTORE_STACK                                   \
2524          "mr %0,3"                                                \
2525          : /*out*/   "=r" (_res)                                  \
2526          : /*in*/    "r" (&_argvec[0])                            \
2527          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2528       );                                                          \
2529       lval = (__typeof__(lval)) _res;                             \
2530    } while (0)
2531 
2532 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2533                                  arg7)                            \
2534    do {                                                           \
2535       volatile OrigFn        _orig = (orig);                      \
2536       volatile unsigned long _argvec[8];                          \
2537       volatile unsigned long _res;                                \
2538       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2539       _argvec[1] = (unsigned long)arg1;                           \
2540       _argvec[2] = (unsigned long)arg2;                           \
2541       _argvec[3] = (unsigned long)arg3;                           \
2542       _argvec[4] = (unsigned long)arg4;                           \
2543       _argvec[5] = (unsigned long)arg5;                           \
2544       _argvec[6] = (unsigned long)arg6;                           \
2545       _argvec[7] = (unsigned long)arg7;                           \
2546       __asm__ volatile(                                           \
2547          VALGRIND_ALIGN_STACK                                     \
2548          "mr 11,%1\n\t"                                           \
2549          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2550          "lwz 4,8(11)\n\t"                                        \
2551          "lwz 5,12(11)\n\t"                                       \
2552          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2553          "lwz 7,20(11)\n\t"                                       \
2554          "lwz 8,24(11)\n\t"                                       \
2555          "lwz 9,28(11)\n\t"                                       \
2556          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2557          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2558          VALGRIND_RESTORE_STACK                                   \
2559          "mr %0,3"                                                \
2560          : /*out*/   "=r" (_res)                                  \
2561          : /*in*/    "r" (&_argvec[0])                            \
2562          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2563       );                                                          \
2564       lval = (__typeof__(lval)) _res;                             \
2565    } while (0)
2566 
2567 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2568                                  arg7,arg8)                       \
2569    do {                                                           \
2570       volatile OrigFn        _orig = (orig);                      \
2571       volatile unsigned long _argvec[9];                          \
2572       volatile unsigned long _res;                                \
2573       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2574       _argvec[1] = (unsigned long)arg1;                           \
2575       _argvec[2] = (unsigned long)arg2;                           \
2576       _argvec[3] = (unsigned long)arg3;                           \
2577       _argvec[4] = (unsigned long)arg4;                           \
2578       _argvec[5] = (unsigned long)arg5;                           \
2579       _argvec[6] = (unsigned long)arg6;                           \
2580       _argvec[7] = (unsigned long)arg7;                           \
2581       _argvec[8] = (unsigned long)arg8;                           \
2582       __asm__ volatile(                                           \
2583          VALGRIND_ALIGN_STACK                                     \
2584          "mr 11,%1\n\t"                                           \
2585          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2586          "lwz 4,8(11)\n\t"                                        \
2587          "lwz 5,12(11)\n\t"                                       \
2588          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2589          "lwz 7,20(11)\n\t"                                       \
2590          "lwz 8,24(11)\n\t"                                       \
2591          "lwz 9,28(11)\n\t"                                       \
2592          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2593          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2594          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2595          VALGRIND_RESTORE_STACK                                   \
2596          "mr %0,3"                                                \
2597          : /*out*/   "=r" (_res)                                  \
2598          : /*in*/    "r" (&_argvec[0])                            \
2599          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2600       );                                                          \
2601       lval = (__typeof__(lval)) _res;                             \
2602    } while (0)
2603 
2604 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2605                                  arg7,arg8,arg9)                  \
2606    do {                                                           \
2607       volatile OrigFn        _orig = (orig);                      \
2608       volatile unsigned long _argvec[10];                         \
2609       volatile unsigned long _res;                                \
2610       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2611       _argvec[1] = (unsigned long)arg1;                           \
2612       _argvec[2] = (unsigned long)arg2;                           \
2613       _argvec[3] = (unsigned long)arg3;                           \
2614       _argvec[4] = (unsigned long)arg4;                           \
2615       _argvec[5] = (unsigned long)arg5;                           \
2616       _argvec[6] = (unsigned long)arg6;                           \
2617       _argvec[7] = (unsigned long)arg7;                           \
2618       _argvec[8] = (unsigned long)arg8;                           \
2619       _argvec[9] = (unsigned long)arg9;                           \
2620       __asm__ volatile(                                           \
2621          VALGRIND_ALIGN_STACK                                     \
2622          "mr 11,%1\n\t"                                           \
2623          "addi 1,1,-16\n\t"                                       \
2624          /* arg9 */                                               \
2625          "lwz 3,36(11)\n\t"                                       \
2626          "stw 3,8(1)\n\t"                                         \
2627          /* args1-8 */                                            \
2628          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2629          "lwz 4,8(11)\n\t"                                        \
2630          "lwz 5,12(11)\n\t"                                       \
2631          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2632          "lwz 7,20(11)\n\t"                                       \
2633          "lwz 8,24(11)\n\t"                                       \
2634          "lwz 9,28(11)\n\t"                                       \
2635          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2636          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2637          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2638          VALGRIND_RESTORE_STACK                                   \
2639          "mr %0,3"                                                \
2640          : /*out*/   "=r" (_res)                                  \
2641          : /*in*/    "r" (&_argvec[0])                            \
2642          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2643       );                                                          \
2644       lval = (__typeof__(lval)) _res;                             \
2645    } while (0)
2646 
2647 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2648                                   arg7,arg8,arg9,arg10)           \
2649    do {                                                           \
2650       volatile OrigFn        _orig = (orig);                      \
2651       volatile unsigned long _argvec[11];                         \
2652       volatile unsigned long _res;                                \
2653       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2654       _argvec[1] = (unsigned long)arg1;                           \
2655       _argvec[2] = (unsigned long)arg2;                           \
2656       _argvec[3] = (unsigned long)arg3;                           \
2657       _argvec[4] = (unsigned long)arg4;                           \
2658       _argvec[5] = (unsigned long)arg5;                           \
2659       _argvec[6] = (unsigned long)arg6;                           \
2660       _argvec[7] = (unsigned long)arg7;                           \
2661       _argvec[8] = (unsigned long)arg8;                           \
2662       _argvec[9] = (unsigned long)arg9;                           \
2663       _argvec[10] = (unsigned long)arg10;                         \
2664       __asm__ volatile(                                           \
2665          VALGRIND_ALIGN_STACK                                     \
2666          "mr 11,%1\n\t"                                           \
2667          "addi 1,1,-16\n\t"                                       \
2668          /* arg10 */                                              \
2669          "lwz 3,40(11)\n\t"                                       \
2670          "stw 3,12(1)\n\t"                                        \
2671          /* arg9 */                                               \
2672          "lwz 3,36(11)\n\t"                                       \
2673          "stw 3,8(1)\n\t"                                         \
2674          /* args1-8 */                                            \
2675          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2676          "lwz 4,8(11)\n\t"                                        \
2677          "lwz 5,12(11)\n\t"                                       \
2678          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2679          "lwz 7,20(11)\n\t"                                       \
2680          "lwz 8,24(11)\n\t"                                       \
2681          "lwz 9,28(11)\n\t"                                       \
2682          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2683          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2684          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2685          VALGRIND_RESTORE_STACK                                   \
2686          "mr %0,3"                                                \
2687          : /*out*/   "=r" (_res)                                  \
2688          : /*in*/    "r" (&_argvec[0])                            \
2689          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2690       );                                                          \
2691       lval = (__typeof__(lval)) _res;                             \
2692    } while (0)
2693 
2694 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2695                                   arg7,arg8,arg9,arg10,arg11)     \
2696    do {                                                           \
2697       volatile OrigFn        _orig = (orig);                      \
2698       volatile unsigned long _argvec[12];                         \
2699       volatile unsigned long _res;                                \
2700       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2701       _argvec[1] = (unsigned long)arg1;                           \
2702       _argvec[2] = (unsigned long)arg2;                           \
2703       _argvec[3] = (unsigned long)arg3;                           \
2704       _argvec[4] = (unsigned long)arg4;                           \
2705       _argvec[5] = (unsigned long)arg5;                           \
2706       _argvec[6] = (unsigned long)arg6;                           \
2707       _argvec[7] = (unsigned long)arg7;                           \
2708       _argvec[8] = (unsigned long)arg8;                           \
2709       _argvec[9] = (unsigned long)arg9;                           \
2710       _argvec[10] = (unsigned long)arg10;                         \
2711       _argvec[11] = (unsigned long)arg11;                         \
2712       __asm__ volatile(                                           \
2713          VALGRIND_ALIGN_STACK                                     \
2714          "mr 11,%1\n\t"                                           \
2715          "addi 1,1,-32\n\t"                                       \
2716          /* arg11 */                                              \
2717          "lwz 3,44(11)\n\t"                                       \
2718          "stw 3,16(1)\n\t"                                        \
2719          /* arg10 */                                              \
2720          "lwz 3,40(11)\n\t"                                       \
2721          "stw 3,12(1)\n\t"                                        \
2722          /* arg9 */                                               \
2723          "lwz 3,36(11)\n\t"                                       \
2724          "stw 3,8(1)\n\t"                                         \
2725          /* args1-8 */                                            \
2726          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2727          "lwz 4,8(11)\n\t"                                        \
2728          "lwz 5,12(11)\n\t"                                       \
2729          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2730          "lwz 7,20(11)\n\t"                                       \
2731          "lwz 8,24(11)\n\t"                                       \
2732          "lwz 9,28(11)\n\t"                                       \
2733          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2734          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2735          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2736          VALGRIND_RESTORE_STACK                                   \
2737          "mr %0,3"                                                \
2738          : /*out*/   "=r" (_res)                                  \
2739          : /*in*/    "r" (&_argvec[0])                            \
2740          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2741       );                                                          \
2742       lval = (__typeof__(lval)) _res;                             \
2743    } while (0)
2744 
2745 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2746                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2747    do {                                                           \
2748       volatile OrigFn        _orig = (orig);                      \
2749       volatile unsigned long _argvec[13];                         \
2750       volatile unsigned long _res;                                \
2751       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2752       _argvec[1] = (unsigned long)arg1;                           \
2753       _argvec[2] = (unsigned long)arg2;                           \
2754       _argvec[3] = (unsigned long)arg3;                           \
2755       _argvec[4] = (unsigned long)arg4;                           \
2756       _argvec[5] = (unsigned long)arg5;                           \
2757       _argvec[6] = (unsigned long)arg6;                           \
2758       _argvec[7] = (unsigned long)arg7;                           \
2759       _argvec[8] = (unsigned long)arg8;                           \
2760       _argvec[9] = (unsigned long)arg9;                           \
2761       _argvec[10] = (unsigned long)arg10;                         \
2762       _argvec[11] = (unsigned long)arg11;                         \
2763       _argvec[12] = (unsigned long)arg12;                         \
2764       __asm__ volatile(                                           \
2765          VALGRIND_ALIGN_STACK                                     \
2766          "mr 11,%1\n\t"                                           \
2767          "addi 1,1,-32\n\t"                                       \
2768          /* arg12 */                                              \
2769          "lwz 3,48(11)\n\t"                                       \
2770          "stw 3,20(1)\n\t"                                        \
2771          /* arg11 */                                              \
2772          "lwz 3,44(11)\n\t"                                       \
2773          "stw 3,16(1)\n\t"                                        \
2774          /* arg10 */                                              \
2775          "lwz 3,40(11)\n\t"                                       \
2776          "stw 3,12(1)\n\t"                                        \
2777          /* arg9 */                                               \
2778          "lwz 3,36(11)\n\t"                                       \
2779          "stw 3,8(1)\n\t"                                         \
2780          /* args1-8 */                                            \
2781          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2782          "lwz 4,8(11)\n\t"                                        \
2783          "lwz 5,12(11)\n\t"                                       \
2784          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2785          "lwz 7,20(11)\n\t"                                       \
2786          "lwz 8,24(11)\n\t"                                       \
2787          "lwz 9,28(11)\n\t"                                       \
2788          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2789          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2790          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2791          VALGRIND_RESTORE_STACK                                   \
2792          "mr %0,3"                                                \
2793          : /*out*/   "=r" (_res)                                  \
2794          : /*in*/    "r" (&_argvec[0])                            \
2795          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2796       );                                                          \
2797       lval = (__typeof__(lval)) _res;                             \
2798    } while (0)
2799 
2800 #endif /* PLAT_ppc32_linux */
2801 
2802 /* ------------------------ ppc64-linux ------------------------ */
2803 
2804 #if defined(PLAT_ppc64be_linux)
2805 
2806 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2807 
2808 /* These regs are trashed by the hidden call. */
2809 #define __CALLER_SAVED_REGS                                       \
2810    "lr", "ctr", "xer",                                            \
2811    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2812    "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",         \
2813    "r11", "r12", "r13"
2814 
2815 /* Macros to save and align the stack before making a function
2816    call and restore it afterwards as gcc may not keep the stack
2817    pointer aligned if it doesn't realise calls are being made
2818    to other functions. */
2819 
2820 #define VALGRIND_ALIGN_STACK               \
2821       "mr 28,1\n\t"                        \
2822       "rldicr 1,1,0,59\n\t"
2823 #define VALGRIND_RESTORE_STACK             \
2824       "mr 1,28\n\t"
2825 
2826 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
2827    long) == 8. */
2828 
2829 #define CALL_FN_W_v(lval, orig)                                   \
2830    do {                                                           \
2831       volatile OrigFn        _orig = (orig);                      \
2832       volatile unsigned long _argvec[3+0];                        \
2833       volatile unsigned long _res;                                \
2834       /* _argvec[0] holds current r2 across the call */           \
2835       _argvec[1] = (unsigned long)_orig.r2;                       \
2836       _argvec[2] = (unsigned long)_orig.nraddr;                   \
2837       __asm__ volatile(                                           \
2838          VALGRIND_ALIGN_STACK                                     \
2839          "mr 11,%1\n\t"                                           \
2840          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2841          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2842          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2843          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2844          "mr 11,%1\n\t"                                           \
2845          "mr %0,3\n\t"                                            \
2846          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2847          VALGRIND_RESTORE_STACK                                   \
2848          : /*out*/   "=r" (_res)                                  \
2849          : /*in*/    "r" (&_argvec[2])                            \
2850          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2851       );                                                          \
2852       lval = (__typeof__(lval)) _res;                             \
2853    } while (0)
2854 
2855 #define CALL_FN_W_W(lval, orig, arg1)                             \
2856    do {                                                           \
2857       volatile OrigFn        _orig = (orig);                      \
2858       volatile unsigned long _argvec[3+1];                        \
2859       volatile unsigned long _res;                                \
2860       /* _argvec[0] holds current r2 across the call */           \
2861       _argvec[1]   = (unsigned long)_orig.r2;                     \
2862       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2863       _argvec[2+1] = (unsigned long)arg1;                         \
2864       __asm__ volatile(                                           \
2865          VALGRIND_ALIGN_STACK                                     \
2866          "mr 11,%1\n\t"                                           \
2867          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2868          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2869          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2870          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2871          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2872          "mr 11,%1\n\t"                                           \
2873          "mr %0,3\n\t"                                            \
2874          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2875          VALGRIND_RESTORE_STACK                                   \
2876          : /*out*/   "=r" (_res)                                  \
2877          : /*in*/    "r" (&_argvec[2])                            \
2878          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2879       );                                                          \
2880       lval = (__typeof__(lval)) _res;                             \
2881    } while (0)
2882 
2883 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2884    do {                                                           \
2885       volatile OrigFn        _orig = (orig);                      \
2886       volatile unsigned long _argvec[3+2];                        \
2887       volatile unsigned long _res;                                \
2888       /* _argvec[0] holds current r2 across the call */           \
2889       _argvec[1]   = (unsigned long)_orig.r2;                     \
2890       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2891       _argvec[2+1] = (unsigned long)arg1;                         \
2892       _argvec[2+2] = (unsigned long)arg2;                         \
2893       __asm__ volatile(                                           \
2894          VALGRIND_ALIGN_STACK                                     \
2895          "mr 11,%1\n\t"                                           \
2896          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2897          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2898          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2899          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2900          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2901          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2902          "mr 11,%1\n\t"                                           \
2903          "mr %0,3\n\t"                                            \
2904          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2905          VALGRIND_RESTORE_STACK                                   \
2906          : /*out*/   "=r" (_res)                                  \
2907          : /*in*/    "r" (&_argvec[2])                            \
2908          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2909       );                                                          \
2910       lval = (__typeof__(lval)) _res;                             \
2911    } while (0)
2912 
2913 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2914    do {                                                           \
2915       volatile OrigFn        _orig = (orig);                      \
2916       volatile unsigned long _argvec[3+3];                        \
2917       volatile unsigned long _res;                                \
2918       /* _argvec[0] holds current r2 across the call */           \
2919       _argvec[1]   = (unsigned long)_orig.r2;                     \
2920       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2921       _argvec[2+1] = (unsigned long)arg1;                         \
2922       _argvec[2+2] = (unsigned long)arg2;                         \
2923       _argvec[2+3] = (unsigned long)arg3;                         \
2924       __asm__ volatile(                                           \
2925          VALGRIND_ALIGN_STACK                                     \
2926          "mr 11,%1\n\t"                                           \
2927          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2928          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2929          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2930          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2931          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2932          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2933          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2934          "mr 11,%1\n\t"                                           \
2935          "mr %0,3\n\t"                                            \
2936          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2937          VALGRIND_RESTORE_STACK                                   \
2938          : /*out*/   "=r" (_res)                                  \
2939          : /*in*/    "r" (&_argvec[2])                            \
2940          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2941       );                                                          \
2942       lval = (__typeof__(lval)) _res;                             \
2943    } while (0)
2944 
2945 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2946    do {                                                           \
2947       volatile OrigFn        _orig = (orig);                      \
2948       volatile unsigned long _argvec[3+4];                        \
2949       volatile unsigned long _res;                                \
2950       /* _argvec[0] holds current r2 across the call */           \
2951       _argvec[1]   = (unsigned long)_orig.r2;                     \
2952       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2953       _argvec[2+1] = (unsigned long)arg1;                         \
2954       _argvec[2+2] = (unsigned long)arg2;                         \
2955       _argvec[2+3] = (unsigned long)arg3;                         \
2956       _argvec[2+4] = (unsigned long)arg4;                         \
2957       __asm__ volatile(                                           \
2958          VALGRIND_ALIGN_STACK                                     \
2959          "mr 11,%1\n\t"                                           \
2960          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2961          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2962          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2963          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2964          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2965          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2966          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2967          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2968          "mr 11,%1\n\t"                                           \
2969          "mr %0,3\n\t"                                            \
2970          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2971          VALGRIND_RESTORE_STACK                                   \
2972          : /*out*/   "=r" (_res)                                  \
2973          : /*in*/    "r" (&_argvec[2])                            \
2974          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2975       );                                                          \
2976       lval = (__typeof__(lval)) _res;                             \
2977    } while (0)
2978 
2979 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2980    do {                                                           \
2981       volatile OrigFn        _orig = (orig);                      \
2982       volatile unsigned long _argvec[3+5];                        \
2983       volatile unsigned long _res;                                \
2984       /* _argvec[0] holds current r2 across the call */           \
2985       _argvec[1]   = (unsigned long)_orig.r2;                     \
2986       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2987       _argvec[2+1] = (unsigned long)arg1;                         \
2988       _argvec[2+2] = (unsigned long)arg2;                         \
2989       _argvec[2+3] = (unsigned long)arg3;                         \
2990       _argvec[2+4] = (unsigned long)arg4;                         \
2991       _argvec[2+5] = (unsigned long)arg5;                         \
2992       __asm__ volatile(                                           \
2993          VALGRIND_ALIGN_STACK                                     \
2994          "mr 11,%1\n\t"                                           \
2995          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2996          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2997          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2998          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2999          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3000          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3001          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3002          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3003          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3004          "mr 11,%1\n\t"                                           \
3005          "mr %0,3\n\t"                                            \
3006          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3007          VALGRIND_RESTORE_STACK                                   \
3008          : /*out*/   "=r" (_res)                                  \
3009          : /*in*/    "r" (&_argvec[2])                            \
3010          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3011       );                                                          \
3012       lval = (__typeof__(lval)) _res;                             \
3013    } while (0)
3014 
3015 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3016    do {                                                           \
3017       volatile OrigFn        _orig = (orig);                      \
3018       volatile unsigned long _argvec[3+6];                        \
3019       volatile unsigned long _res;                                \
3020       /* _argvec[0] holds current r2 across the call */           \
3021       _argvec[1]   = (unsigned long)_orig.r2;                     \
3022       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3023       _argvec[2+1] = (unsigned long)arg1;                         \
3024       _argvec[2+2] = (unsigned long)arg2;                         \
3025       _argvec[2+3] = (unsigned long)arg3;                         \
3026       _argvec[2+4] = (unsigned long)arg4;                         \
3027       _argvec[2+5] = (unsigned long)arg5;                         \
3028       _argvec[2+6] = (unsigned long)arg6;                         \
3029       __asm__ volatile(                                           \
3030          VALGRIND_ALIGN_STACK                                     \
3031          "mr 11,%1\n\t"                                           \
3032          "std 2,-16(11)\n\t"  /* save tocptr */                   \
3033          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3034          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3035          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3036          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3037          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3038          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3039          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3040          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3041          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3042          "mr 11,%1\n\t"                                           \
3043          "mr %0,3\n\t"                                            \
3044          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3045          VALGRIND_RESTORE_STACK                                   \
3046          : /*out*/   "=r" (_res)                                  \
3047          : /*in*/    "r" (&_argvec[2])                            \
3048          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3049       );                                                          \
3050       lval = (__typeof__(lval)) _res;                             \
3051    } while (0)
3052 
3053 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3054                                  arg7)                            \
3055    do {                                                           \
3056       volatile OrigFn        _orig = (orig);                      \
3057       volatile unsigned long _argvec[3+7];                        \
3058       volatile unsigned long _res;                                \
3059       /* _argvec[0] holds current r2 across the call */           \
3060       _argvec[1]   = (unsigned long)_orig.r2;                     \
3061       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3062       _argvec[2+1] = (unsigned long)arg1;                         \
3063       _argvec[2+2] = (unsigned long)arg2;                         \
3064       _argvec[2+3] = (unsigned long)arg3;                         \
3065       _argvec[2+4] = (unsigned long)arg4;                         \
3066       _argvec[2+5] = (unsigned long)arg5;                         \
3067       _argvec[2+6] = (unsigned long)arg6;                         \
3068       _argvec[2+7] = (unsigned long)arg7;                         \
3069       __asm__ volatile(                                           \
3070          VALGRIND_ALIGN_STACK                                     \
3071          "mr 11,%1\n\t"                                           \
3072          "std 2,-16(11)\n\t"  /* save tocptr */                   \
3073          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3074          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3075          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3076          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3077          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3078          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3079          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3080          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3081          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3082          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3083          "mr 11,%1\n\t"                                           \
3084          "mr %0,3\n\t"                                            \
3085          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3086          VALGRIND_RESTORE_STACK                                   \
3087          : /*out*/   "=r" (_res)                                  \
3088          : /*in*/    "r" (&_argvec[2])                            \
3089          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3090       );                                                          \
3091       lval = (__typeof__(lval)) _res;                             \
3092    } while (0)
3093 
3094 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3095                                  arg7,arg8)                       \
3096    do {                                                           \
3097       volatile OrigFn        _orig = (orig);                      \
3098       volatile unsigned long _argvec[3+8];                        \
3099       volatile unsigned long _res;                                \
3100       /* _argvec[0] holds current r2 across the call */           \
3101       _argvec[1]   = (unsigned long)_orig.r2;                     \
3102       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3103       _argvec[2+1] = (unsigned long)arg1;                         \
3104       _argvec[2+2] = (unsigned long)arg2;                         \
3105       _argvec[2+3] = (unsigned long)arg3;                         \
3106       _argvec[2+4] = (unsigned long)arg4;                         \
3107       _argvec[2+5] = (unsigned long)arg5;                         \
3108       _argvec[2+6] = (unsigned long)arg6;                         \
3109       _argvec[2+7] = (unsigned long)arg7;                         \
3110       _argvec[2+8] = (unsigned long)arg8;                         \
3111       __asm__ volatile(                                           \
3112          VALGRIND_ALIGN_STACK                                     \
3113          "mr 11,%1\n\t"                                           \
3114          "std 2,-16(11)\n\t"  /* save tocptr */                   \
3115          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3116          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3117          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3118          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3119          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3120          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3121          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3122          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3123          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3124          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3125          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3126          "mr 11,%1\n\t"                                           \
3127          "mr %0,3\n\t"                                            \
3128          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3129          VALGRIND_RESTORE_STACK                                   \
3130          : /*out*/   "=r" (_res)                                  \
3131          : /*in*/    "r" (&_argvec[2])                            \
3132          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3133       );                                                          \
3134       lval = (__typeof__(lval)) _res;                             \
3135    } while (0)
3136 
3137 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3138                                  arg7,arg8,arg9)                  \
3139    do {                                                           \
3140       volatile OrigFn        _orig = (orig);                      \
3141       volatile unsigned long _argvec[3+9];                        \
3142       volatile unsigned long _res;                                \
3143       /* _argvec[0] holds current r2 across the call */           \
3144       _argvec[1]   = (unsigned long)_orig.r2;                     \
3145       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3146       _argvec[2+1] = (unsigned long)arg1;                         \
3147       _argvec[2+2] = (unsigned long)arg2;                         \
3148       _argvec[2+3] = (unsigned long)arg3;                         \
3149       _argvec[2+4] = (unsigned long)arg4;                         \
3150       _argvec[2+5] = (unsigned long)arg5;                         \
3151       _argvec[2+6] = (unsigned long)arg6;                         \
3152       _argvec[2+7] = (unsigned long)arg7;                         \
3153       _argvec[2+8] = (unsigned long)arg8;                         \
3154       _argvec[2+9] = (unsigned long)arg9;                         \
3155       __asm__ volatile(                                           \
3156          VALGRIND_ALIGN_STACK                                     \
3157          "mr 11,%1\n\t"                                           \
3158          "std 2,-16(11)\n\t"  /* save tocptr */                   \
3159          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3160          "addi 1,1,-128\n\t"  /* expand stack frame */            \
3161          /* arg9 */                                               \
3162          "ld  3,72(11)\n\t"                                       \
3163          "std 3,112(1)\n\t"                                       \
3164          /* args1-8 */                                            \
3165          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3166          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3167          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3168          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3169          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3170          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3171          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3172          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3173          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3174          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3175          "mr 11,%1\n\t"                                           \
3176          "mr %0,3\n\t"                                            \
3177          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3178          VALGRIND_RESTORE_STACK                                   \
3179          : /*out*/   "=r" (_res)                                  \
3180          : /*in*/    "r" (&_argvec[2])                            \
3181          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3182       );                                                          \
3183       lval = (__typeof__(lval)) _res;                             \
3184    } while (0)
3185 
3186 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3187                                   arg7,arg8,arg9,arg10)           \
3188    do {                                                           \
3189       volatile OrigFn        _orig = (orig);                      \
3190       volatile unsigned long _argvec[3+10];                       \
3191       volatile unsigned long _res;                                \
3192       /* _argvec[0] holds current r2 across the call */           \
3193       _argvec[1]   = (unsigned long)_orig.r2;                     \
3194       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3195       _argvec[2+1] = (unsigned long)arg1;                         \
3196       _argvec[2+2] = (unsigned long)arg2;                         \
3197       _argvec[2+3] = (unsigned long)arg3;                         \
3198       _argvec[2+4] = (unsigned long)arg4;                         \
3199       _argvec[2+5] = (unsigned long)arg5;                         \
3200       _argvec[2+6] = (unsigned long)arg6;                         \
3201       _argvec[2+7] = (unsigned long)arg7;                         \
3202       _argvec[2+8] = (unsigned long)arg8;                         \
3203       _argvec[2+9] = (unsigned long)arg9;                         \
3204       _argvec[2+10] = (unsigned long)arg10;                       \
3205       __asm__ volatile(                                           \
3206          VALGRIND_ALIGN_STACK                                     \
3207          "mr 11,%1\n\t"                                           \
3208          "std 2,-16(11)\n\t"  /* save tocptr */                   \
3209          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3210          "addi 1,1,-128\n\t"  /* expand stack frame */            \
3211          /* arg10 */                                              \
3212          "ld  3,80(11)\n\t"                                       \
3213          "std 3,120(1)\n\t"                                       \
3214          /* arg9 */                                               \
3215          "ld  3,72(11)\n\t"                                       \
3216          "std 3,112(1)\n\t"                                       \
3217          /* args1-8 */                                            \
3218          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3219          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3220          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3221          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3222          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3223          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3224          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3225          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3226          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3227          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3228          "mr 11,%1\n\t"                                           \
3229          "mr %0,3\n\t"                                            \
3230          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3231          VALGRIND_RESTORE_STACK                                   \
3232          : /*out*/   "=r" (_res)                                  \
3233          : /*in*/    "r" (&_argvec[2])                            \
3234          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3235       );                                                          \
3236       lval = (__typeof__(lval)) _res;                             \
3237    } while (0)
3238 
3239 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3240                                   arg7,arg8,arg9,arg10,arg11)     \
3241    do {                                                           \
3242       volatile OrigFn        _orig = (orig);                      \
3243       volatile unsigned long _argvec[3+11];                       \
3244       volatile unsigned long _res;                                \
3245       /* _argvec[0] holds current r2 across the call */           \
3246       _argvec[1]   = (unsigned long)_orig.r2;                     \
3247       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3248       _argvec[2+1] = (unsigned long)arg1;                         \
3249       _argvec[2+2] = (unsigned long)arg2;                         \
3250       _argvec[2+3] = (unsigned long)arg3;                         \
3251       _argvec[2+4] = (unsigned long)arg4;                         \
3252       _argvec[2+5] = (unsigned long)arg5;                         \
3253       _argvec[2+6] = (unsigned long)arg6;                         \
3254       _argvec[2+7] = (unsigned long)arg7;                         \
3255       _argvec[2+8] = (unsigned long)arg8;                         \
3256       _argvec[2+9] = (unsigned long)arg9;                         \
3257       _argvec[2+10] = (unsigned long)arg10;                       \
3258       _argvec[2+11] = (unsigned long)arg11;                       \
3259       __asm__ volatile(                                           \
3260          VALGRIND_ALIGN_STACK                                     \
3261          "mr 11,%1\n\t"                                           \
3262          "std 2,-16(11)\n\t"  /* save tocptr */                   \
3263          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3264          "addi 1,1,-144\n\t"  /* expand stack frame */            \
3265          /* arg11 */                                              \
3266          "ld  3,88(11)\n\t"                                       \
3267          "std 3,128(1)\n\t"                                       \
3268          /* arg10 */                                              \
3269          "ld  3,80(11)\n\t"                                       \
3270          "std 3,120(1)\n\t"                                       \
3271          /* arg9 */                                               \
3272          "ld  3,72(11)\n\t"                                       \
3273          "std 3,112(1)\n\t"                                       \
3274          /* args1-8 */                                            \
3275          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3276          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3277          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3278          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3279          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3280          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3281          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3282          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3283          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3284          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3285          "mr 11,%1\n\t"                                           \
3286          "mr %0,3\n\t"                                            \
3287          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3288          VALGRIND_RESTORE_STACK                                   \
3289          : /*out*/   "=r" (_res)                                  \
3290          : /*in*/    "r" (&_argvec[2])                            \
3291          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3292       );                                                          \
3293       lval = (__typeof__(lval)) _res;                             \
3294    } while (0)
3295 
3296 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3297                                 arg7,arg8,arg9,arg10,arg11,arg12) \
3298    do {                                                           \
3299       volatile OrigFn        _orig = (orig);                      \
3300       volatile unsigned long _argvec[3+12];                       \
3301       volatile unsigned long _res;                                \
3302       /* _argvec[0] holds current r2 across the call */           \
3303       _argvec[1]   = (unsigned long)_orig.r2;                     \
3304       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3305       _argvec[2+1] = (unsigned long)arg1;                         \
3306       _argvec[2+2] = (unsigned long)arg2;                         \
3307       _argvec[2+3] = (unsigned long)arg3;                         \
3308       _argvec[2+4] = (unsigned long)arg4;                         \
3309       _argvec[2+5] = (unsigned long)arg5;                         \
3310       _argvec[2+6] = (unsigned long)arg6;                         \
3311       _argvec[2+7] = (unsigned long)arg7;                         \
3312       _argvec[2+8] = (unsigned long)arg8;                         \
3313       _argvec[2+9] = (unsigned long)arg9;                         \
3314       _argvec[2+10] = (unsigned long)arg10;                       \
3315       _argvec[2+11] = (unsigned long)arg11;                       \
3316       _argvec[2+12] = (unsigned long)arg12;                       \
3317       __asm__ volatile(                                           \
3318          VALGRIND_ALIGN_STACK                                     \
3319          "mr 11,%1\n\t"                                           \
3320          "std 2,-16(11)\n\t"  /* save tocptr */                   \
3321          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3322          "addi 1,1,-144\n\t"  /* expand stack frame */            \
3323          /* arg12 */                                              \
3324          "ld  3,96(11)\n\t"                                       \
3325          "std 3,136(1)\n\t"                                       \
3326          /* arg11 */                                              \
3327          "ld  3,88(11)\n\t"                                       \
3328          "std 3,128(1)\n\t"                                       \
3329          /* arg10 */                                              \
3330          "ld  3,80(11)\n\t"                                       \
3331          "std 3,120(1)\n\t"                                       \
3332          /* arg9 */                                               \
3333          "ld  3,72(11)\n\t"                                       \
3334          "std 3,112(1)\n\t"                                       \
3335          /* args1-8 */                                            \
3336          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3337          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3338          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3339          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3340          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3341          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3342          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3343          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3344          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3345          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3346          "mr 11,%1\n\t"                                           \
3347          "mr %0,3\n\t"                                            \
3348          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3349          VALGRIND_RESTORE_STACK                                   \
3350          : /*out*/   "=r" (_res)                                  \
3351          : /*in*/    "r" (&_argvec[2])                            \
3352          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3353       );                                                          \
3354       lval = (__typeof__(lval)) _res;                             \
3355    } while (0)
3356 
3357 #endif /* PLAT_ppc64be_linux */
3358 
3359 /* ------------------------- ppc64le-linux ----------------------- */
3360 #if defined(PLAT_ppc64le_linux)
3361 
3362 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3363 
3364 /* These regs are trashed by the hidden call. */
3365 #define __CALLER_SAVED_REGS                                       \
3366    "lr", "ctr", "xer",                                            \
3367    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3368    "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",         \
3369    "r11", "r12", "r13"
3370 
3371 /* Macros to save and align the stack before making a function
3372    call and restore it afterwards as gcc may not keep the stack
3373    pointer aligned if it doesn't realise calls are being made
3374    to other functions. */
3375 
3376 #define VALGRIND_ALIGN_STACK               \
3377       "mr 28,1\n\t"                        \
3378       "rldicr 1,1,0,59\n\t"
3379 #define VALGRIND_RESTORE_STACK             \
3380       "mr 1,28\n\t"
3381 
3382 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
3383    long) == 8. */
3384 
3385 #define CALL_FN_W_v(lval, orig)                                   \
3386    do {                                                           \
3387       volatile OrigFn        _orig = (orig);                      \
3388       volatile unsigned long _argvec[3+0];                        \
3389       volatile unsigned long _res;                                \
3390       /* _argvec[0] holds current r2 across the call */           \
3391       _argvec[1] = (unsigned long)_orig.r2;                       \
3392       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3393       __asm__ volatile(                                           \
3394          VALGRIND_ALIGN_STACK                                     \
3395          "mr 12,%1\n\t"                                           \
3396          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3397          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3398          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3399          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3400          "mr 12,%1\n\t"                                           \
3401          "mr %0,3\n\t"                                            \
3402          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3403          VALGRIND_RESTORE_STACK                                   \
3404          : /*out*/   "=r" (_res)                                  \
3405          : /*in*/    "r" (&_argvec[2])                            \
3406          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3407       );                                                          \
3408       lval = (__typeof__(lval)) _res;                             \
3409    } while (0)
3410 
3411 #define CALL_FN_W_W(lval, orig, arg1)                             \
3412    do {                                                           \
3413       volatile OrigFn        _orig = (orig);                      \
3414       volatile unsigned long _argvec[3+1];                        \
3415       volatile unsigned long _res;                                \
3416       /* _argvec[0] holds current r2 across the call */           \
3417       _argvec[1]   = (unsigned long)_orig.r2;                     \
3418       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3419       _argvec[2+1] = (unsigned long)arg1;                         \
3420       __asm__ volatile(                                           \
3421          VALGRIND_ALIGN_STACK                                     \
3422          "mr 12,%1\n\t"                                           \
3423          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3424          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3425          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3426          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3427          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3428          "mr 12,%1\n\t"                                           \
3429          "mr %0,3\n\t"                                            \
3430          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3431          VALGRIND_RESTORE_STACK                                   \
3432          : /*out*/   "=r" (_res)                                  \
3433          : /*in*/    "r" (&_argvec[2])                            \
3434          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3435       );                                                          \
3436       lval = (__typeof__(lval)) _res;                             \
3437    } while (0)
3438 
3439 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3440    do {                                                           \
3441       volatile OrigFn        _orig = (orig);                      \
3442       volatile unsigned long _argvec[3+2];                        \
3443       volatile unsigned long _res;                                \
3444       /* _argvec[0] holds current r2 across the call */           \
3445       _argvec[1]   = (unsigned long)_orig.r2;                     \
3446       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3447       _argvec[2+1] = (unsigned long)arg1;                         \
3448       _argvec[2+2] = (unsigned long)arg2;                         \
3449       __asm__ volatile(                                           \
3450          VALGRIND_ALIGN_STACK                                     \
3451          "mr 12,%1\n\t"                                           \
3452          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3453          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3454          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3455          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3456          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3457          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3458          "mr 12,%1\n\t"                                           \
3459          "mr %0,3\n\t"                                            \
3460          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3461          VALGRIND_RESTORE_STACK                                   \
3462          : /*out*/   "=r" (_res)                                  \
3463          : /*in*/    "r" (&_argvec[2])                            \
3464          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3465       );                                                          \
3466       lval = (__typeof__(lval)) _res;                             \
3467    } while (0)
3468 
3469 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3470    do {                                                           \
3471       volatile OrigFn        _orig = (orig);                      \
3472       volatile unsigned long _argvec[3+3];                        \
3473       volatile unsigned long _res;                                \
3474       /* _argvec[0] holds current r2 across the call */           \
3475       _argvec[1]   = (unsigned long)_orig.r2;                     \
3476       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3477       _argvec[2+1] = (unsigned long)arg1;                         \
3478       _argvec[2+2] = (unsigned long)arg2;                         \
3479       _argvec[2+3] = (unsigned long)arg3;                         \
3480       __asm__ volatile(                                           \
3481          VALGRIND_ALIGN_STACK                                     \
3482          "mr 12,%1\n\t"                                           \
3483          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3484          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3485          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3486          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3487          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3488          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3489          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3490          "mr 12,%1\n\t"                                           \
3491          "mr %0,3\n\t"                                            \
3492          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3493          VALGRIND_RESTORE_STACK                                   \
3494          : /*out*/   "=r" (_res)                                  \
3495          : /*in*/    "r" (&_argvec[2])                            \
3496          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3497       );                                                          \
3498       lval = (__typeof__(lval)) _res;                             \
3499    } while (0)
3500 
3501 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3502    do {                                                           \
3503       volatile OrigFn        _orig = (orig);                      \
3504       volatile unsigned long _argvec[3+4];                        \
3505       volatile unsigned long _res;                                \
3506       /* _argvec[0] holds current r2 across the call */           \
3507       _argvec[1]   = (unsigned long)_orig.r2;                     \
3508       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3509       _argvec[2+1] = (unsigned long)arg1;                         \
3510       _argvec[2+2] = (unsigned long)arg2;                         \
3511       _argvec[2+3] = (unsigned long)arg3;                         \
3512       _argvec[2+4] = (unsigned long)arg4;                         \
3513       __asm__ volatile(                                           \
3514          VALGRIND_ALIGN_STACK                                     \
3515          "mr 12,%1\n\t"                                           \
3516          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3517          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3518          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3519          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3520          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3521          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3522          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3523          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3524          "mr 12,%1\n\t"                                           \
3525          "mr %0,3\n\t"                                            \
3526          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3527          VALGRIND_RESTORE_STACK                                   \
3528          : /*out*/   "=r" (_res)                                  \
3529          : /*in*/    "r" (&_argvec[2])                            \
3530          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3531       );                                                          \
3532       lval = (__typeof__(lval)) _res;                             \
3533    } while (0)
3534 
3535 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3536    do {                                                           \
3537       volatile OrigFn        _orig = (orig);                      \
3538       volatile unsigned long _argvec[3+5];                        \
3539       volatile unsigned long _res;                                \
3540       /* _argvec[0] holds current r2 across the call */           \
3541       _argvec[1]   = (unsigned long)_orig.r2;                     \
3542       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3543       _argvec[2+1] = (unsigned long)arg1;                         \
3544       _argvec[2+2] = (unsigned long)arg2;                         \
3545       _argvec[2+3] = (unsigned long)arg3;                         \
3546       _argvec[2+4] = (unsigned long)arg4;                         \
3547       _argvec[2+5] = (unsigned long)arg5;                         \
3548       __asm__ volatile(                                           \
3549          VALGRIND_ALIGN_STACK                                     \
3550          "mr 12,%1\n\t"                                           \
3551          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3552          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3553          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3554          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3555          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3556          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3557          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3558          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3559          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3560          "mr 12,%1\n\t"                                           \
3561          "mr %0,3\n\t"                                            \
3562          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3563          VALGRIND_RESTORE_STACK                                   \
3564          : /*out*/   "=r" (_res)                                  \
3565          : /*in*/    "r" (&_argvec[2])                            \
3566          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3567       );                                                          \
3568       lval = (__typeof__(lval)) _res;                             \
3569    } while (0)
3570 
3571 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3572    do {                                                           \
3573       volatile OrigFn        _orig = (orig);                      \
3574       volatile unsigned long _argvec[3+6];                        \
3575       volatile unsigned long _res;                                \
3576       /* _argvec[0] holds current r2 across the call */           \
3577       _argvec[1]   = (unsigned long)_orig.r2;                     \
3578       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3579       _argvec[2+1] = (unsigned long)arg1;                         \
3580       _argvec[2+2] = (unsigned long)arg2;                         \
3581       _argvec[2+3] = (unsigned long)arg3;                         \
3582       _argvec[2+4] = (unsigned long)arg4;                         \
3583       _argvec[2+5] = (unsigned long)arg5;                         \
3584       _argvec[2+6] = (unsigned long)arg6;                         \
3585       __asm__ volatile(                                           \
3586          VALGRIND_ALIGN_STACK                                     \
3587          "mr 12,%1\n\t"                                           \
3588          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3589          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3590          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3591          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3592          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3593          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3594          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3595          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3596          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3597          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3598          "mr 12,%1\n\t"                                           \
3599          "mr %0,3\n\t"                                            \
3600          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3601          VALGRIND_RESTORE_STACK                                   \
3602          : /*out*/   "=r" (_res)                                  \
3603          : /*in*/    "r" (&_argvec[2])                            \
3604          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3605       );                                                          \
3606       lval = (__typeof__(lval)) _res;                             \
3607    } while (0)
3608 
3609 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3610                                  arg7)                            \
3611    do {                                                           \
3612       volatile OrigFn        _orig = (orig);                      \
3613       volatile unsigned long _argvec[3+7];                        \
3614       volatile unsigned long _res;                                \
3615       /* _argvec[0] holds current r2 across the call */           \
3616       _argvec[1]   = (unsigned long)_orig.r2;                     \
3617       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3618       _argvec[2+1] = (unsigned long)arg1;                         \
3619       _argvec[2+2] = (unsigned long)arg2;                         \
3620       _argvec[2+3] = (unsigned long)arg3;                         \
3621       _argvec[2+4] = (unsigned long)arg4;                         \
3622       _argvec[2+5] = (unsigned long)arg5;                         \
3623       _argvec[2+6] = (unsigned long)arg6;                         \
3624       _argvec[2+7] = (unsigned long)arg7;                         \
3625       __asm__ volatile(                                           \
3626          VALGRIND_ALIGN_STACK                                     \
3627          "mr 12,%1\n\t"                                           \
3628          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3629          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3630          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3631          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3632          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3633          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3634          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3635          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3636          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3637          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3638          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3639          "mr 12,%1\n\t"                                           \
3640          "mr %0,3\n\t"                                            \
3641          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3642          VALGRIND_RESTORE_STACK                                   \
3643          : /*out*/   "=r" (_res)                                  \
3644          : /*in*/    "r" (&_argvec[2])                            \
3645          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3646       );                                                          \
3647       lval = (__typeof__(lval)) _res;                             \
3648    } while (0)
3649 
3650 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3651                                  arg7,arg8)                       \
3652    do {                                                           \
3653       volatile OrigFn        _orig = (orig);                      \
3654       volatile unsigned long _argvec[3+8];                        \
3655       volatile unsigned long _res;                                \
3656       /* _argvec[0] holds current r2 across the call */           \
3657       _argvec[1]   = (unsigned long)_orig.r2;                     \
3658       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3659       _argvec[2+1] = (unsigned long)arg1;                         \
3660       _argvec[2+2] = (unsigned long)arg2;                         \
3661       _argvec[2+3] = (unsigned long)arg3;                         \
3662       _argvec[2+4] = (unsigned long)arg4;                         \
3663       _argvec[2+5] = (unsigned long)arg5;                         \
3664       _argvec[2+6] = (unsigned long)arg6;                         \
3665       _argvec[2+7] = (unsigned long)arg7;                         \
3666       _argvec[2+8] = (unsigned long)arg8;                         \
3667       __asm__ volatile(                                           \
3668          VALGRIND_ALIGN_STACK                                     \
3669          "mr 12,%1\n\t"                                           \
3670          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3671          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3672          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3673          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3674          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3675          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3676          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3677          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3678          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3679          "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
3680          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3681          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3682          "mr 12,%1\n\t"                                           \
3683          "mr %0,3\n\t"                                            \
3684          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3685          VALGRIND_RESTORE_STACK                                   \
3686          : /*out*/   "=r" (_res)                                  \
3687          : /*in*/    "r" (&_argvec[2])                            \
3688          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3689       );                                                          \
3690       lval = (__typeof__(lval)) _res;                             \
3691    } while (0)
3692 
3693 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3694                                  arg7,arg8,arg9)                  \
3695    do {                                                           \
3696       volatile OrigFn        _orig = (orig);                      \
3697       volatile unsigned long _argvec[3+9];                        \
3698       volatile unsigned long _res;                                \
3699       /* _argvec[0] holds current r2 across the call */           \
3700       _argvec[1]   = (unsigned long)_orig.r2;                     \
3701       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3702       _argvec[2+1] = (unsigned long)arg1;                         \
3703       _argvec[2+2] = (unsigned long)arg2;                         \
3704       _argvec[2+3] = (unsigned long)arg3;                         \
3705       _argvec[2+4] = (unsigned long)arg4;                         \
3706       _argvec[2+5] = (unsigned long)arg5;                         \
3707       _argvec[2+6] = (unsigned long)arg6;                         \
3708       _argvec[2+7] = (unsigned long)arg7;                         \
3709       _argvec[2+8] = (unsigned long)arg8;                         \
3710       _argvec[2+9] = (unsigned long)arg9;                         \
3711       __asm__ volatile(                                           \
3712          VALGRIND_ALIGN_STACK                                     \
3713          "mr 12,%1\n\t"                                           \
3714          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3715          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3716          "addi 1,1,-128\n\t"  /* expand stack frame */            \
3717          /* arg9 */                                               \
3718          "ld  3,72(12)\n\t"                                       \
3719          "std 3,96(1)\n\t"                                        \
3720          /* args1-8 */                                            \
3721          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3722          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3723          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3724          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3725          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3726          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3727          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3728          "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
3729          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3730          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3731          "mr 12,%1\n\t"                                           \
3732          "mr %0,3\n\t"                                            \
3733          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3734          VALGRIND_RESTORE_STACK                                   \
3735          : /*out*/   "=r" (_res)                                  \
3736          : /*in*/    "r" (&_argvec[2])                            \
3737          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3738       );                                                          \
3739       lval = (__typeof__(lval)) _res;                             \
3740    } while (0)
3741 
3742 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3743                                   arg7,arg8,arg9,arg10)           \
3744    do {                                                           \
3745       volatile OrigFn        _orig = (orig);                      \
3746       volatile unsigned long _argvec[3+10];                       \
3747       volatile unsigned long _res;                                \
3748       /* _argvec[0] holds current r2 across the call */           \
3749       _argvec[1]   = (unsigned long)_orig.r2;                     \
3750       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3751       _argvec[2+1] = (unsigned long)arg1;                         \
3752       _argvec[2+2] = (unsigned long)arg2;                         \
3753       _argvec[2+3] = (unsigned long)arg3;                         \
3754       _argvec[2+4] = (unsigned long)arg4;                         \
3755       _argvec[2+5] = (unsigned long)arg5;                         \
3756       _argvec[2+6] = (unsigned long)arg6;                         \
3757       _argvec[2+7] = (unsigned long)arg7;                         \
3758       _argvec[2+8] = (unsigned long)arg8;                         \
3759       _argvec[2+9] = (unsigned long)arg9;                         \
3760       _argvec[2+10] = (unsigned long)arg10;                       \
3761       __asm__ volatile(                                           \
3762          VALGRIND_ALIGN_STACK                                     \
3763          "mr 12,%1\n\t"                                           \
3764          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3765          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3766          "addi 1,1,-128\n\t"  /* expand stack frame */            \
3767          /* arg10 */                                              \
3768          "ld  3,80(12)\n\t"                                       \
3769          "std 3,104(1)\n\t"                                       \
3770          /* arg9 */                                               \
3771          "ld  3,72(12)\n\t"                                       \
3772          "std 3,96(1)\n\t"                                        \
3773          /* args1-8 */                                            \
3774          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3775          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3776          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3777          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3778          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3779          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3780          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3781          "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
3782          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3783          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3784          "mr 12,%1\n\t"                                           \
3785          "mr %0,3\n\t"                                            \
3786          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3787          VALGRIND_RESTORE_STACK                                   \
3788          : /*out*/   "=r" (_res)                                  \
3789          : /*in*/    "r" (&_argvec[2])                            \
3790          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3791       );                                                          \
3792       lval = (__typeof__(lval)) _res;                             \
3793    } while (0)
3794 
3795 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3796                                   arg7,arg8,arg9,arg10,arg11)     \
3797    do {                                                           \
3798       volatile OrigFn        _orig = (orig);                      \
3799       volatile unsigned long _argvec[3+11];                       \
3800       volatile unsigned long _res;                                \
3801       /* _argvec[0] holds current r2 across the call */           \
3802       _argvec[1]   = (unsigned long)_orig.r2;                     \
3803       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3804       _argvec[2+1] = (unsigned long)arg1;                         \
3805       _argvec[2+2] = (unsigned long)arg2;                         \
3806       _argvec[2+3] = (unsigned long)arg3;                         \
3807       _argvec[2+4] = (unsigned long)arg4;                         \
3808       _argvec[2+5] = (unsigned long)arg5;                         \
3809       _argvec[2+6] = (unsigned long)arg6;                         \
3810       _argvec[2+7] = (unsigned long)arg7;                         \
3811       _argvec[2+8] = (unsigned long)arg8;                         \
3812       _argvec[2+9] = (unsigned long)arg9;                         \
3813       _argvec[2+10] = (unsigned long)arg10;                       \
3814       _argvec[2+11] = (unsigned long)arg11;                       \
3815       __asm__ volatile(                                           \
3816          VALGRIND_ALIGN_STACK                                     \
3817          "mr 12,%1\n\t"                                           \
3818          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3819          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3820          "addi 1,1,-144\n\t"  /* expand stack frame */            \
3821          /* arg11 */                                              \
3822          "ld  3,88(12)\n\t"                                       \
3823          "std 3,112(1)\n\t"                                       \
3824          /* arg10 */                                              \
3825          "ld  3,80(12)\n\t"                                       \
3826          "std 3,104(1)\n\t"                                       \
3827          /* arg9 */                                               \
3828          "ld  3,72(12)\n\t"                                       \
3829          "std 3,96(1)\n\t"                                        \
3830          /* args1-8 */                                            \
3831          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3832          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3833          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3834          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3835          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3836          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3837          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3838          "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
3839          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3840          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3841          "mr 12,%1\n\t"                                           \
3842          "mr %0,3\n\t"                                            \
3843          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3844          VALGRIND_RESTORE_STACK                                   \
3845          : /*out*/   "=r" (_res)                                  \
3846          : /*in*/    "r" (&_argvec[2])                            \
3847          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3848       );                                                          \
3849       lval = (__typeof__(lval)) _res;                             \
3850    } while (0)
3851 
3852 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3853                                 arg7,arg8,arg9,arg10,arg11,arg12) \
3854    do {                                                           \
3855       volatile OrigFn        _orig = (orig);                      \
3856       volatile unsigned long _argvec[3+12];                       \
3857       volatile unsigned long _res;                                \
3858       /* _argvec[0] holds current r2 across the call */           \
3859       _argvec[1]   = (unsigned long)_orig.r2;                     \
3860       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3861       _argvec[2+1] = (unsigned long)arg1;                         \
3862       _argvec[2+2] = (unsigned long)arg2;                         \
3863       _argvec[2+3] = (unsigned long)arg3;                         \
3864       _argvec[2+4] = (unsigned long)arg4;                         \
3865       _argvec[2+5] = (unsigned long)arg5;                         \
3866       _argvec[2+6] = (unsigned long)arg6;                         \
3867       _argvec[2+7] = (unsigned long)arg7;                         \
3868       _argvec[2+8] = (unsigned long)arg8;                         \
3869       _argvec[2+9] = (unsigned long)arg9;                         \
3870       _argvec[2+10] = (unsigned long)arg10;                       \
3871       _argvec[2+11] = (unsigned long)arg11;                       \
3872       _argvec[2+12] = (unsigned long)arg12;                       \
3873       __asm__ volatile(                                           \
3874          VALGRIND_ALIGN_STACK                                     \
3875          "mr 12,%1\n\t"                                           \
3876          "std 2,-16(12)\n\t"  /* save tocptr */                   \
3877          "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3878          "addi 1,1,-144\n\t"  /* expand stack frame */            \
3879          /* arg12 */                                              \
3880          "ld  3,96(12)\n\t"                                       \
3881          "std 3,120(1)\n\t"                                       \
3882          /* arg11 */                                              \
3883          "ld  3,88(12)\n\t"                                       \
3884          "std 3,112(1)\n\t"                                       \
3885          /* arg10 */                                              \
3886          "ld  3,80(12)\n\t"                                       \
3887          "std 3,104(1)\n\t"                                       \
3888          /* arg9 */                                               \
3889          "ld  3,72(12)\n\t"                                       \
3890          "std 3,96(1)\n\t"                                        \
3891          /* args1-8 */                                            \
3892          "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3893          "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3894          "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3895          "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3896          "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3897          "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3898          "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3899          "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
3900          "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3901          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3902          "mr 12,%1\n\t"                                           \
3903          "mr %0,3\n\t"                                            \
3904          "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3905          VALGRIND_RESTORE_STACK                                   \
3906          : /*out*/   "=r" (_res)                                  \
3907          : /*in*/    "r" (&_argvec[2])                            \
3908          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3909       );                                                          \
3910       lval = (__typeof__(lval)) _res;                             \
3911    } while (0)
3912 
3913 #endif /* PLAT_ppc64le_linux */
3914 
3915 /* ------------------------- arm-linux ------------------------- */
3916 
3917 #if defined(PLAT_arm_linux)
3918 
3919 /* These regs are trashed by the hidden call. */
3920 #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4", "r12", "r14"
3921 
3922 /* Macros to save and align the stack before making a function
3923    call and restore it afterwards as gcc may not keep the stack
3924    pointer aligned if it doesn't realise calls are being made
3925    to other functions. */
3926 
3927 /* This is a bit tricky.  We store the original stack pointer in r10
3928    as it is callee-saves.  gcc doesn't allow the use of r11 for some
3929    reason.  Also, we can't directly "bic" the stack pointer in thumb
3930    mode since r13 isn't an allowed register number in that context.
3931    So use r4 as a temporary, since that is about to get trashed
3932    anyway, just after each use of this macro.  Side effect is we need
3933    to be very careful about any future changes, since
3934    VALGRIND_ALIGN_STACK simply assumes r4 is usable. */
3935 #define VALGRIND_ALIGN_STACK               \
3936       "mov r10, sp\n\t"                    \
3937       "mov r4,  sp\n\t"                    \
3938       "bic r4,  r4, #7\n\t"                \
3939       "mov sp,  r4\n\t"
3940 #define VALGRIND_RESTORE_STACK             \
3941       "mov sp,  r10\n\t"
3942 
3943 /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned
3944    long) == 4. */
3945 
3946 #define CALL_FN_W_v(lval, orig)                                   \
3947    do {                                                           \
3948       volatile OrigFn        _orig = (orig);                      \
3949       volatile unsigned long _argvec[1];                          \
3950       volatile unsigned long _res;                                \
3951       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3952       __asm__ volatile(                                           \
3953          VALGRIND_ALIGN_STACK                                     \
3954          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3955          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3956          VALGRIND_RESTORE_STACK                                   \
3957          "mov %0, r0\n"                                           \
3958          : /*out*/   "=r" (_res)                                  \
3959          : /*in*/    "0" (&_argvec[0])                            \
3960          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3961       );                                                          \
3962       lval = (__typeof__(lval)) _res;                             \
3963    } while (0)
3964 
3965 #define CALL_FN_W_W(lval, orig, arg1)                             \
3966    do {                                                           \
3967       volatile OrigFn        _orig = (orig);                      \
3968       volatile unsigned long _argvec[2];                          \
3969       volatile unsigned long _res;                                \
3970       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3971       _argvec[1] = (unsigned long)(arg1);                         \
3972       __asm__ volatile(                                           \
3973          VALGRIND_ALIGN_STACK                                     \
3974          "ldr r0, [%1, #4] \n\t"                                  \
3975          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3976          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3977          VALGRIND_RESTORE_STACK                                   \
3978          "mov %0, r0\n"                                           \
3979          : /*out*/   "=r" (_res)                                  \
3980          : /*in*/    "0" (&_argvec[0])                            \
3981          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3982       );                                                          \
3983       lval = (__typeof__(lval)) _res;                             \
3984    } while (0)
3985 
3986 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3987    do {                                                           \
3988       volatile OrigFn        _orig = (orig);                      \
3989       volatile unsigned long _argvec[3];                          \
3990       volatile unsigned long _res;                                \
3991       _argvec[0] = (unsigned long)_orig.nraddr;                   \
3992       _argvec[1] = (unsigned long)(arg1);                         \
3993       _argvec[2] = (unsigned long)(arg2);                         \
3994       __asm__ volatile(                                           \
3995          VALGRIND_ALIGN_STACK                                     \
3996          "ldr r0, [%1, #4] \n\t"                                  \
3997          "ldr r1, [%1, #8] \n\t"                                  \
3998          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3999          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4000          VALGRIND_RESTORE_STACK                                   \
4001          "mov %0, r0\n"                                           \
4002          : /*out*/   "=r" (_res)                                  \
4003          : /*in*/    "0" (&_argvec[0])                            \
4004          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4005       );                                                          \
4006       lval = (__typeof__(lval)) _res;                             \
4007    } while (0)
4008 
4009 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
4010    do {                                                           \
4011       volatile OrigFn        _orig = (orig);                      \
4012       volatile unsigned long _argvec[4];                          \
4013       volatile unsigned long _res;                                \
4014       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4015       _argvec[1] = (unsigned long)(arg1);                         \
4016       _argvec[2] = (unsigned long)(arg2);                         \
4017       _argvec[3] = (unsigned long)(arg3);                         \
4018       __asm__ volatile(                                           \
4019          VALGRIND_ALIGN_STACK                                     \
4020          "ldr r0, [%1, #4] \n\t"                                  \
4021          "ldr r1, [%1, #8] \n\t"                                  \
4022          "ldr r2, [%1, #12] \n\t"                                 \
4023          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4024          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4025          VALGRIND_RESTORE_STACK                                   \
4026          "mov %0, r0\n"                                           \
4027          : /*out*/   "=r" (_res)                                  \
4028          : /*in*/    "0" (&_argvec[0])                            \
4029          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4030       );                                                          \
4031       lval = (__typeof__(lval)) _res;                             \
4032    } while (0)
4033 
4034 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
4035    do {                                                           \
4036       volatile OrigFn        _orig = (orig);                      \
4037       volatile unsigned long _argvec[5];                          \
4038       volatile unsigned long _res;                                \
4039       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4040       _argvec[1] = (unsigned long)(arg1);                         \
4041       _argvec[2] = (unsigned long)(arg2);                         \
4042       _argvec[3] = (unsigned long)(arg3);                         \
4043       _argvec[4] = (unsigned long)(arg4);                         \
4044       __asm__ volatile(                                           \
4045          VALGRIND_ALIGN_STACK                                     \
4046          "ldr r0, [%1, #4] \n\t"                                  \
4047          "ldr r1, [%1, #8] \n\t"                                  \
4048          "ldr r2, [%1, #12] \n\t"                                 \
4049          "ldr r3, [%1, #16] \n\t"                                 \
4050          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4051          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4052          VALGRIND_RESTORE_STACK                                   \
4053          "mov %0, r0"                                             \
4054          : /*out*/   "=r" (_res)                                  \
4055          : /*in*/    "0" (&_argvec[0])                            \
4056          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4057       );                                                          \
4058       lval = (__typeof__(lval)) _res;                             \
4059    } while (0)
4060 
4061 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
4062    do {                                                           \
4063       volatile OrigFn        _orig = (orig);                      \
4064       volatile unsigned long _argvec[6];                          \
4065       volatile unsigned long _res;                                \
4066       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4067       _argvec[1] = (unsigned long)(arg1);                         \
4068       _argvec[2] = (unsigned long)(arg2);                         \
4069       _argvec[3] = (unsigned long)(arg3);                         \
4070       _argvec[4] = (unsigned long)(arg4);                         \
4071       _argvec[5] = (unsigned long)(arg5);                         \
4072       __asm__ volatile(                                           \
4073          VALGRIND_ALIGN_STACK                                     \
4074          "sub sp, sp, #4 \n\t"                                    \
4075          "ldr r0, [%1, #20] \n\t"                                 \
4076          "push {r0} \n\t"                                         \
4077          "ldr r0, [%1, #4] \n\t"                                  \
4078          "ldr r1, [%1, #8] \n\t"                                  \
4079          "ldr r2, [%1, #12] \n\t"                                 \
4080          "ldr r3, [%1, #16] \n\t"                                 \
4081          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4082          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4083          VALGRIND_RESTORE_STACK                                   \
4084          "mov %0, r0"                                             \
4085          : /*out*/   "=r" (_res)                                  \
4086          : /*in*/    "0" (&_argvec[0])                            \
4087          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4088       );                                                          \
4089       lval = (__typeof__(lval)) _res;                             \
4090    } while (0)
4091 
4092 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
4093    do {                                                           \
4094       volatile OrigFn        _orig = (orig);                      \
4095       volatile unsigned long _argvec[7];                          \
4096       volatile unsigned long _res;                                \
4097       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4098       _argvec[1] = (unsigned long)(arg1);                         \
4099       _argvec[2] = (unsigned long)(arg2);                         \
4100       _argvec[3] = (unsigned long)(arg3);                         \
4101       _argvec[4] = (unsigned long)(arg4);                         \
4102       _argvec[5] = (unsigned long)(arg5);                         \
4103       _argvec[6] = (unsigned long)(arg6);                         \
4104       __asm__ volatile(                                           \
4105          VALGRIND_ALIGN_STACK                                     \
4106          "ldr r0, [%1, #20] \n\t"                                 \
4107          "ldr r1, [%1, #24] \n\t"                                 \
4108          "push {r0, r1} \n\t"                                     \
4109          "ldr r0, [%1, #4] \n\t"                                  \
4110          "ldr r1, [%1, #8] \n\t"                                  \
4111          "ldr r2, [%1, #12] \n\t"                                 \
4112          "ldr r3, [%1, #16] \n\t"                                 \
4113          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4114          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4115          VALGRIND_RESTORE_STACK                                   \
4116          "mov %0, r0"                                             \
4117          : /*out*/   "=r" (_res)                                  \
4118          : /*in*/    "0" (&_argvec[0])                            \
4119          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4120       );                                                          \
4121       lval = (__typeof__(lval)) _res;                             \
4122    } while (0)
4123 
4124 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4125                                  arg7)                            \
4126    do {                                                           \
4127       volatile OrigFn        _orig = (orig);                      \
4128       volatile unsigned long _argvec[8];                          \
4129       volatile unsigned long _res;                                \
4130       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4131       _argvec[1] = (unsigned long)(arg1);                         \
4132       _argvec[2] = (unsigned long)(arg2);                         \
4133       _argvec[3] = (unsigned long)(arg3);                         \
4134       _argvec[4] = (unsigned long)(arg4);                         \
4135       _argvec[5] = (unsigned long)(arg5);                         \
4136       _argvec[6] = (unsigned long)(arg6);                         \
4137       _argvec[7] = (unsigned long)(arg7);                         \
4138       __asm__ volatile(                                           \
4139          VALGRIND_ALIGN_STACK                                     \
4140          "sub sp, sp, #4 \n\t"                                    \
4141          "ldr r0, [%1, #20] \n\t"                                 \
4142          "ldr r1, [%1, #24] \n\t"                                 \
4143          "ldr r2, [%1, #28] \n\t"                                 \
4144          "push {r0, r1, r2} \n\t"                                 \
4145          "ldr r0, [%1, #4] \n\t"                                  \
4146          "ldr r1, [%1, #8] \n\t"                                  \
4147          "ldr r2, [%1, #12] \n\t"                                 \
4148          "ldr r3, [%1, #16] \n\t"                                 \
4149          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4150          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4151          VALGRIND_RESTORE_STACK                                   \
4152          "mov %0, r0"                                             \
4153          : /*out*/   "=r" (_res)                                  \
4154          : /*in*/    "0" (&_argvec[0])                            \
4155          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4156       );                                                          \
4157       lval = (__typeof__(lval)) _res;                             \
4158    } while (0)
4159 
4160 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4161                                  arg7,arg8)                       \
4162    do {                                                           \
4163       volatile OrigFn        _orig = (orig);                      \
4164       volatile unsigned long _argvec[9];                          \
4165       volatile unsigned long _res;                                \
4166       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4167       _argvec[1] = (unsigned long)(arg1);                         \
4168       _argvec[2] = (unsigned long)(arg2);                         \
4169       _argvec[3] = (unsigned long)(arg3);                         \
4170       _argvec[4] = (unsigned long)(arg4);                         \
4171       _argvec[5] = (unsigned long)(arg5);                         \
4172       _argvec[6] = (unsigned long)(arg6);                         \
4173       _argvec[7] = (unsigned long)(arg7);                         \
4174       _argvec[8] = (unsigned long)(arg8);                         \
4175       __asm__ volatile(                                           \
4176          VALGRIND_ALIGN_STACK                                     \
4177          "ldr r0, [%1, #20] \n\t"                                 \
4178          "ldr r1, [%1, #24] \n\t"                                 \
4179          "ldr r2, [%1, #28] \n\t"                                 \
4180          "ldr r3, [%1, #32] \n\t"                                 \
4181          "push {r0, r1, r2, r3} \n\t"                             \
4182          "ldr r0, [%1, #4] \n\t"                                  \
4183          "ldr r1, [%1, #8] \n\t"                                  \
4184          "ldr r2, [%1, #12] \n\t"                                 \
4185          "ldr r3, [%1, #16] \n\t"                                 \
4186          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4187          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4188          VALGRIND_RESTORE_STACK                                   \
4189          "mov %0, r0"                                             \
4190          : /*out*/   "=r" (_res)                                  \
4191          : /*in*/    "0" (&_argvec[0])                            \
4192          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4193       );                                                          \
4194       lval = (__typeof__(lval)) _res;                             \
4195    } while (0)
4196 
4197 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4198                                  arg7,arg8,arg9)                  \
4199    do {                                                           \
4200       volatile OrigFn        _orig = (orig);                      \
4201       volatile unsigned long _argvec[10];                         \
4202       volatile unsigned long _res;                                \
4203       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4204       _argvec[1] = (unsigned long)(arg1);                         \
4205       _argvec[2] = (unsigned long)(arg2);                         \
4206       _argvec[3] = (unsigned long)(arg3);                         \
4207       _argvec[4] = (unsigned long)(arg4);                         \
4208       _argvec[5] = (unsigned long)(arg5);                         \
4209       _argvec[6] = (unsigned long)(arg6);                         \
4210       _argvec[7] = (unsigned long)(arg7);                         \
4211       _argvec[8] = (unsigned long)(arg8);                         \
4212       _argvec[9] = (unsigned long)(arg9);                         \
4213       __asm__ volatile(                                           \
4214          VALGRIND_ALIGN_STACK                                     \
4215          "sub sp, sp, #4 \n\t"                                    \
4216          "ldr r0, [%1, #20] \n\t"                                 \
4217          "ldr r1, [%1, #24] \n\t"                                 \
4218          "ldr r2, [%1, #28] \n\t"                                 \
4219          "ldr r3, [%1, #32] \n\t"                                 \
4220          "ldr r4, [%1, #36] \n\t"                                 \
4221          "push {r0, r1, r2, r3, r4} \n\t"                         \
4222          "ldr r0, [%1, #4] \n\t"                                  \
4223          "ldr r1, [%1, #8] \n\t"                                  \
4224          "ldr r2, [%1, #12] \n\t"                                 \
4225          "ldr r3, [%1, #16] \n\t"                                 \
4226          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4227          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4228          VALGRIND_RESTORE_STACK                                   \
4229          "mov %0, r0"                                             \
4230          : /*out*/   "=r" (_res)                                  \
4231          : /*in*/    "0" (&_argvec[0])                            \
4232          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4233       );                                                          \
4234       lval = (__typeof__(lval)) _res;                             \
4235    } while (0)
4236 
4237 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4238                                   arg7,arg8,arg9,arg10)           \
4239    do {                                                           \
4240       volatile OrigFn        _orig = (orig);                      \
4241       volatile unsigned long _argvec[11];                         \
4242       volatile unsigned long _res;                                \
4243       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4244       _argvec[1] = (unsigned long)(arg1);                         \
4245       _argvec[2] = (unsigned long)(arg2);                         \
4246       _argvec[3] = (unsigned long)(arg3);                         \
4247       _argvec[4] = (unsigned long)(arg4);                         \
4248       _argvec[5] = (unsigned long)(arg5);                         \
4249       _argvec[6] = (unsigned long)(arg6);                         \
4250       _argvec[7] = (unsigned long)(arg7);                         \
4251       _argvec[8] = (unsigned long)(arg8);                         \
4252       _argvec[9] = (unsigned long)(arg9);                         \
4253       _argvec[10] = (unsigned long)(arg10);                       \
4254       __asm__ volatile(                                           \
4255          VALGRIND_ALIGN_STACK                                     \
4256          "ldr r0, [%1, #40] \n\t"                                 \
4257          "push {r0} \n\t"                                         \
4258          "ldr r0, [%1, #20] \n\t"                                 \
4259          "ldr r1, [%1, #24] \n\t"                                 \
4260          "ldr r2, [%1, #28] \n\t"                                 \
4261          "ldr r3, [%1, #32] \n\t"                                 \
4262          "ldr r4, [%1, #36] \n\t"                                 \
4263          "push {r0, r1, r2, r3, r4} \n\t"                         \
4264          "ldr r0, [%1, #4] \n\t"                                  \
4265          "ldr r1, [%1, #8] \n\t"                                  \
4266          "ldr r2, [%1, #12] \n\t"                                 \
4267          "ldr r3, [%1, #16] \n\t"                                 \
4268          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4269          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4270          VALGRIND_RESTORE_STACK                                   \
4271          "mov %0, r0"                                             \
4272          : /*out*/   "=r" (_res)                                  \
4273          : /*in*/    "0" (&_argvec[0])                            \
4274          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4275       );                                                          \
4276       lval = (__typeof__(lval)) _res;                             \
4277    } while (0)
4278 
4279 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
4280                                   arg6,arg7,arg8,arg9,arg10,      \
4281                                   arg11)                          \
4282    do {                                                           \
4283       volatile OrigFn        _orig = (orig);                      \
4284       volatile unsigned long _argvec[12];                         \
4285       volatile unsigned long _res;                                \
4286       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4287       _argvec[1] = (unsigned long)(arg1);                         \
4288       _argvec[2] = (unsigned long)(arg2);                         \
4289       _argvec[3] = (unsigned long)(arg3);                         \
4290       _argvec[4] = (unsigned long)(arg4);                         \
4291       _argvec[5] = (unsigned long)(arg5);                         \
4292       _argvec[6] = (unsigned long)(arg6);                         \
4293       _argvec[7] = (unsigned long)(arg7);                         \
4294       _argvec[8] = (unsigned long)(arg8);                         \
4295       _argvec[9] = (unsigned long)(arg9);                         \
4296       _argvec[10] = (unsigned long)(arg10);                       \
4297       _argvec[11] = (unsigned long)(arg11);                       \
4298       __asm__ volatile(                                           \
4299          VALGRIND_ALIGN_STACK                                     \
4300          "sub sp, sp, #4 \n\t"                                    \
4301          "ldr r0, [%1, #40] \n\t"                                 \
4302          "ldr r1, [%1, #44] \n\t"                                 \
4303          "push {r0, r1} \n\t"                                     \
4304          "ldr r0, [%1, #20] \n\t"                                 \
4305          "ldr r1, [%1, #24] \n\t"                                 \
4306          "ldr r2, [%1, #28] \n\t"                                 \
4307          "ldr r3, [%1, #32] \n\t"                                 \
4308          "ldr r4, [%1, #36] \n\t"                                 \
4309          "push {r0, r1, r2, r3, r4} \n\t"                         \
4310          "ldr r0, [%1, #4] \n\t"                                  \
4311          "ldr r1, [%1, #8] \n\t"                                  \
4312          "ldr r2, [%1, #12] \n\t"                                 \
4313          "ldr r3, [%1, #16] \n\t"                                 \
4314          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4315          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4316          VALGRIND_RESTORE_STACK                                   \
4317          "mov %0, r0"                                             \
4318          : /*out*/   "=r" (_res)                                  \
4319          : /*in*/    "0" (&_argvec[0])                            \
4320          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4321       );                                                          \
4322       lval = (__typeof__(lval)) _res;                             \
4323    } while (0)
4324 
4325 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
4326                                   arg6,arg7,arg8,arg9,arg10,      \
4327                                   arg11,arg12)                    \
4328    do {                                                           \
4329       volatile OrigFn        _orig = (orig);                      \
4330       volatile unsigned long _argvec[13];                         \
4331       volatile unsigned long _res;                                \
4332       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4333       _argvec[1] = (unsigned long)(arg1);                         \
4334       _argvec[2] = (unsigned long)(arg2);                         \
4335       _argvec[3] = (unsigned long)(arg3);                         \
4336       _argvec[4] = (unsigned long)(arg4);                         \
4337       _argvec[5] = (unsigned long)(arg5);                         \
4338       _argvec[6] = (unsigned long)(arg6);                         \
4339       _argvec[7] = (unsigned long)(arg7);                         \
4340       _argvec[8] = (unsigned long)(arg8);                         \
4341       _argvec[9] = (unsigned long)(arg9);                         \
4342       _argvec[10] = (unsigned long)(arg10);                       \
4343       _argvec[11] = (unsigned long)(arg11);                       \
4344       _argvec[12] = (unsigned long)(arg12);                       \
4345       __asm__ volatile(                                           \
4346          VALGRIND_ALIGN_STACK                                     \
4347          "ldr r0, [%1, #40] \n\t"                                 \
4348          "ldr r1, [%1, #44] \n\t"                                 \
4349          "ldr r2, [%1, #48] \n\t"                                 \
4350          "push {r0, r1, r2} \n\t"                                 \
4351          "ldr r0, [%1, #20] \n\t"                                 \
4352          "ldr r1, [%1, #24] \n\t"                                 \
4353          "ldr r2, [%1, #28] \n\t"                                 \
4354          "ldr r3, [%1, #32] \n\t"                                 \
4355          "ldr r4, [%1, #36] \n\t"                                 \
4356          "push {r0, r1, r2, r3, r4} \n\t"                         \
4357          "ldr r0, [%1, #4] \n\t"                                  \
4358          "ldr r1, [%1, #8] \n\t"                                  \
4359          "ldr r2, [%1, #12] \n\t"                                 \
4360          "ldr r3, [%1, #16] \n\t"                                 \
4361          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4362          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4363          VALGRIND_RESTORE_STACK                                   \
4364          "mov %0, r0"                                             \
4365          : /*out*/   "=r" (_res)                                  \
4366          : /*in*/    "0" (&_argvec[0])                            \
4367          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4368       );                                                          \
4369       lval = (__typeof__(lval)) _res;                             \
4370    } while (0)
4371 
4372 #endif /* PLAT_arm_linux */
4373 
4374 /* ------------------- arm64-{linux,freebsd} -------------------- */
4375 
4376 #if defined(PLAT_arm64_linux) || defined(PLAT_arm64_freebsd)
4377 
4378 /* These regs are trashed by the hidden call. */
4379 #define __CALLER_SAVED_REGS \
4380      "x0", "x1", "x2", "x3","x4", "x5", "x6", "x7", "x8", "x9",   \
4381      "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17",      \
4382      "x18", "x19", "x20", "x30",                                  \
4383      "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",  \
4384      "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",      \
4385      "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",      \
4386      "v26", "v27", "v28", "v29", "v30", "v31"
4387 
4388 /* x21 is callee-saved, so we can use it to save and restore SP around
4389    the hidden call. */
4390 #define VALGRIND_ALIGN_STACK               \
4391       "mov x21, sp\n\t"                    \
4392       "bic sp, x21, #15\n\t"
4393 #define VALGRIND_RESTORE_STACK             \
4394       "mov sp,  x21\n\t"
4395 
4396 /* These CALL_FN_ macros assume that on arm64-linux,
4397    sizeof(unsigned long) == 8. */
4398 
4399 #define CALL_FN_W_v(lval, orig)                                   \
4400    do {                                                           \
4401       volatile OrigFn        _orig = (orig);                      \
4402       volatile unsigned long _argvec[1];                          \
4403       volatile unsigned long _res;                                \
4404       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4405       __asm__ volatile(                                           \
4406          VALGRIND_ALIGN_STACK                                     \
4407          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4408          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4409          VALGRIND_RESTORE_STACK                                   \
4410          "mov %0, x0\n"                                           \
4411          : /*out*/   "=r" (_res)                                  \
4412          : /*in*/    "0" (&_argvec[0])                            \
4413          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4414       );                                                          \
4415       lval = (__typeof__(lval)) _res;                             \
4416    } while (0)
4417 
4418 #define CALL_FN_W_W(lval, orig, arg1)                             \
4419    do {                                                           \
4420       volatile OrigFn        _orig = (orig);                      \
4421       volatile unsigned long _argvec[2];                          \
4422       volatile unsigned long _res;                                \
4423       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4424       _argvec[1] = (unsigned long)(arg1);                         \
4425       __asm__ volatile(                                           \
4426          VALGRIND_ALIGN_STACK                                     \
4427          "ldr x0, [%1, #8] \n\t"                                  \
4428          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4429          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4430          VALGRIND_RESTORE_STACK                                   \
4431          "mov %0, x0\n"                                           \
4432          : /*out*/   "=r" (_res)                                  \
4433          : /*in*/    "0" (&_argvec[0])                            \
4434          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4435       );                                                          \
4436       lval = (__typeof__(lval)) _res;                             \
4437    } while (0)
4438 
4439 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
4440    do {                                                           \
4441       volatile OrigFn        _orig = (orig);                      \
4442       volatile unsigned long _argvec[3];                          \
4443       volatile unsigned long _res;                                \
4444       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4445       _argvec[1] = (unsigned long)(arg1);                         \
4446       _argvec[2] = (unsigned long)(arg2);                         \
4447       __asm__ volatile(                                           \
4448          VALGRIND_ALIGN_STACK                                     \
4449          "ldr x0, [%1, #8] \n\t"                                  \
4450          "ldr x1, [%1, #16] \n\t"                                 \
4451          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4452          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4453          VALGRIND_RESTORE_STACK                                   \
4454          "mov %0, x0\n"                                           \
4455          : /*out*/   "=r" (_res)                                  \
4456          : /*in*/    "0" (&_argvec[0])                            \
4457          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4458       );                                                          \
4459       lval = (__typeof__(lval)) _res;                             \
4460    } while (0)
4461 
4462 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
4463    do {                                                           \
4464       volatile OrigFn        _orig = (orig);                      \
4465       volatile unsigned long _argvec[4];                          \
4466       volatile unsigned long _res;                                \
4467       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4468       _argvec[1] = (unsigned long)(arg1);                         \
4469       _argvec[2] = (unsigned long)(arg2);                         \
4470       _argvec[3] = (unsigned long)(arg3);                         \
4471       __asm__ volatile(                                           \
4472          VALGRIND_ALIGN_STACK                                     \
4473          "ldr x0, [%1, #8] \n\t"                                  \
4474          "ldr x1, [%1, #16] \n\t"                                 \
4475          "ldr x2, [%1, #24] \n\t"                                 \
4476          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4477          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4478          VALGRIND_RESTORE_STACK                                   \
4479          "mov %0, x0\n"                                           \
4480          : /*out*/   "=r" (_res)                                  \
4481          : /*in*/    "0" (&_argvec[0])                            \
4482          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4483       );                                                          \
4484       lval = (__typeof__(lval)) _res;                             \
4485    } while (0)
4486 
4487 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
4488    do {                                                           \
4489       volatile OrigFn        _orig = (orig);                      \
4490       volatile unsigned long _argvec[5];                          \
4491       volatile unsigned long _res;                                \
4492       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4493       _argvec[1] = (unsigned long)(arg1);                         \
4494       _argvec[2] = (unsigned long)(arg2);                         \
4495       _argvec[3] = (unsigned long)(arg3);                         \
4496       _argvec[4] = (unsigned long)(arg4);                         \
4497       __asm__ volatile(                                           \
4498          VALGRIND_ALIGN_STACK                                     \
4499          "ldr x0, [%1, #8] \n\t"                                  \
4500          "ldr x1, [%1, #16] \n\t"                                 \
4501          "ldr x2, [%1, #24] \n\t"                                 \
4502          "ldr x3, [%1, #32] \n\t"                                 \
4503          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4504          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4505          VALGRIND_RESTORE_STACK                                   \
4506          "mov %0, x0"                                             \
4507          : /*out*/   "=r" (_res)                                  \
4508          : /*in*/    "0" (&_argvec[0])                            \
4509          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4510       );                                                          \
4511       lval = (__typeof__(lval)) _res;                             \
4512    } while (0)
4513 
4514 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
4515    do {                                                           \
4516       volatile OrigFn        _orig = (orig);                      \
4517       volatile unsigned long _argvec[6];                          \
4518       volatile unsigned long _res;                                \
4519       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4520       _argvec[1] = (unsigned long)(arg1);                         \
4521       _argvec[2] = (unsigned long)(arg2);                         \
4522       _argvec[3] = (unsigned long)(arg3);                         \
4523       _argvec[4] = (unsigned long)(arg4);                         \
4524       _argvec[5] = (unsigned long)(arg5);                         \
4525       __asm__ volatile(                                           \
4526          VALGRIND_ALIGN_STACK                                     \
4527          "ldr x0, [%1, #8] \n\t"                                  \
4528          "ldr x1, [%1, #16] \n\t"                                 \
4529          "ldr x2, [%1, #24] \n\t"                                 \
4530          "ldr x3, [%1, #32] \n\t"                                 \
4531          "ldr x4, [%1, #40] \n\t"                                 \
4532          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4533          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4534          VALGRIND_RESTORE_STACK                                   \
4535          "mov %0, x0"                                             \
4536          : /*out*/   "=r" (_res)                                  \
4537          : /*in*/    "0" (&_argvec[0])                            \
4538          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4539       );                                                          \
4540       lval = (__typeof__(lval)) _res;                             \
4541    } while (0)
4542 
4543 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
4544    do {                                                           \
4545       volatile OrigFn        _orig = (orig);                      \
4546       volatile unsigned long _argvec[7];                          \
4547       volatile unsigned long _res;                                \
4548       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4549       _argvec[1] = (unsigned long)(arg1);                         \
4550       _argvec[2] = (unsigned long)(arg2);                         \
4551       _argvec[3] = (unsigned long)(arg3);                         \
4552       _argvec[4] = (unsigned long)(arg4);                         \
4553       _argvec[5] = (unsigned long)(arg5);                         \
4554       _argvec[6] = (unsigned long)(arg6);                         \
4555       __asm__ volatile(                                           \
4556          VALGRIND_ALIGN_STACK                                     \
4557          "ldr x0, [%1, #8] \n\t"                                  \
4558          "ldr x1, [%1, #16] \n\t"                                 \
4559          "ldr x2, [%1, #24] \n\t"                                 \
4560          "ldr x3, [%1, #32] \n\t"                                 \
4561          "ldr x4, [%1, #40] \n\t"                                 \
4562          "ldr x5, [%1, #48] \n\t"                                 \
4563          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4564          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4565          VALGRIND_RESTORE_STACK                                   \
4566          "mov %0, x0"                                             \
4567          : /*out*/   "=r" (_res)                                  \
4568          : /*in*/    "0" (&_argvec[0])                            \
4569          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4570       );                                                          \
4571       lval = (__typeof__(lval)) _res;                             \
4572    } while (0)
4573 
4574 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4575                                  arg7)                            \
4576    do {                                                           \
4577       volatile OrigFn        _orig = (orig);                      \
4578       volatile unsigned long _argvec[8];                          \
4579       volatile unsigned long _res;                                \
4580       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4581       _argvec[1] = (unsigned long)(arg1);                         \
4582       _argvec[2] = (unsigned long)(arg2);                         \
4583       _argvec[3] = (unsigned long)(arg3);                         \
4584       _argvec[4] = (unsigned long)(arg4);                         \
4585       _argvec[5] = (unsigned long)(arg5);                         \
4586       _argvec[6] = (unsigned long)(arg6);                         \
4587       _argvec[7] = (unsigned long)(arg7);                         \
4588       __asm__ volatile(                                           \
4589          VALGRIND_ALIGN_STACK                                     \
4590          "ldr x0, [%1, #8] \n\t"                                  \
4591          "ldr x1, [%1, #16] \n\t"                                 \
4592          "ldr x2, [%1, #24] \n\t"                                 \
4593          "ldr x3, [%1, #32] \n\t"                                 \
4594          "ldr x4, [%1, #40] \n\t"                                 \
4595          "ldr x5, [%1, #48] \n\t"                                 \
4596          "ldr x6, [%1, #56] \n\t"                                 \
4597          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4598          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4599          VALGRIND_RESTORE_STACK                                   \
4600          "mov %0, x0"                                             \
4601          : /*out*/   "=r" (_res)                                  \
4602          : /*in*/    "0" (&_argvec[0])                            \
4603          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4604       );                                                          \
4605       lval = (__typeof__(lval)) _res;                             \
4606    } while (0)
4607 
4608 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4609                                  arg7,arg8)                       \
4610    do {                                                           \
4611       volatile OrigFn        _orig = (orig);                      \
4612       volatile unsigned long _argvec[9];                          \
4613       volatile unsigned long _res;                                \
4614       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4615       _argvec[1] = (unsigned long)(arg1);                         \
4616       _argvec[2] = (unsigned long)(arg2);                         \
4617       _argvec[3] = (unsigned long)(arg3);                         \
4618       _argvec[4] = (unsigned long)(arg4);                         \
4619       _argvec[5] = (unsigned long)(arg5);                         \
4620       _argvec[6] = (unsigned long)(arg6);                         \
4621       _argvec[7] = (unsigned long)(arg7);                         \
4622       _argvec[8] = (unsigned long)(arg8);                         \
4623       __asm__ volatile(                                           \
4624          VALGRIND_ALIGN_STACK                                     \
4625          "ldr x0, [%1, #8] \n\t"                                  \
4626          "ldr x1, [%1, #16] \n\t"                                 \
4627          "ldr x2, [%1, #24] \n\t"                                 \
4628          "ldr x3, [%1, #32] \n\t"                                 \
4629          "ldr x4, [%1, #40] \n\t"                                 \
4630          "ldr x5, [%1, #48] \n\t"                                 \
4631          "ldr x6, [%1, #56] \n\t"                                 \
4632          "ldr x7, [%1, #64] \n\t"                                 \
4633          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4634          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4635          VALGRIND_RESTORE_STACK                                   \
4636          "mov %0, x0"                                             \
4637          : /*out*/   "=r" (_res)                                  \
4638          : /*in*/    "0" (&_argvec[0])                            \
4639          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4640       );                                                          \
4641       lval = (__typeof__(lval)) _res;                             \
4642    } while (0)
4643 
4644 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4645                                  arg7,arg8,arg9)                  \
4646    do {                                                           \
4647       volatile OrigFn        _orig = (orig);                      \
4648       volatile unsigned long _argvec[10];                         \
4649       volatile unsigned long _res;                                \
4650       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4651       _argvec[1] = (unsigned long)(arg1);                         \
4652       _argvec[2] = (unsigned long)(arg2);                         \
4653       _argvec[3] = (unsigned long)(arg3);                         \
4654       _argvec[4] = (unsigned long)(arg4);                         \
4655       _argvec[5] = (unsigned long)(arg5);                         \
4656       _argvec[6] = (unsigned long)(arg6);                         \
4657       _argvec[7] = (unsigned long)(arg7);                         \
4658       _argvec[8] = (unsigned long)(arg8);                         \
4659       _argvec[9] = (unsigned long)(arg9);                         \
4660       __asm__ volatile(                                           \
4661          VALGRIND_ALIGN_STACK                                     \
4662          "sub sp, sp, #0x20 \n\t"                                 \
4663          "ldr x0, [%1, #8] \n\t"                                  \
4664          "ldr x1, [%1, #16] \n\t"                                 \
4665          "ldr x2, [%1, #24] \n\t"                                 \
4666          "ldr x3, [%1, #32] \n\t"                                 \
4667          "ldr x4, [%1, #40] \n\t"                                 \
4668          "ldr x5, [%1, #48] \n\t"                                 \
4669          "ldr x6, [%1, #56] \n\t"                                 \
4670          "ldr x7, [%1, #64] \n\t"                                 \
4671          "ldr x8, [%1, #72] \n\t"                                 \
4672          "str x8, [sp, #0]  \n\t"                                 \
4673          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4674          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4675          VALGRIND_RESTORE_STACK                                   \
4676          "mov %0, x0"                                             \
4677          : /*out*/   "=r" (_res)                                  \
4678          : /*in*/    "0" (&_argvec[0])                            \
4679          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4680       );                                                          \
4681       lval = (__typeof__(lval)) _res;                             \
4682    } while (0)
4683 
4684 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4685                                   arg7,arg8,arg9,arg10)           \
4686    do {                                                           \
4687       volatile OrigFn        _orig = (orig);                      \
4688       volatile unsigned long _argvec[11];                         \
4689       volatile unsigned long _res;                                \
4690       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4691       _argvec[1] = (unsigned long)(arg1);                         \
4692       _argvec[2] = (unsigned long)(arg2);                         \
4693       _argvec[3] = (unsigned long)(arg3);                         \
4694       _argvec[4] = (unsigned long)(arg4);                         \
4695       _argvec[5] = (unsigned long)(arg5);                         \
4696       _argvec[6] = (unsigned long)(arg6);                         \
4697       _argvec[7] = (unsigned long)(arg7);                         \
4698       _argvec[8] = (unsigned long)(arg8);                         \
4699       _argvec[9] = (unsigned long)(arg9);                         \
4700       _argvec[10] = (unsigned long)(arg10);                       \
4701       __asm__ volatile(                                           \
4702          VALGRIND_ALIGN_STACK                                     \
4703          "sub sp, sp, #0x20 \n\t"                                 \
4704          "ldr x0, [%1, #8] \n\t"                                  \
4705          "ldr x1, [%1, #16] \n\t"                                 \
4706          "ldr x2, [%1, #24] \n\t"                                 \
4707          "ldr x3, [%1, #32] \n\t"                                 \
4708          "ldr x4, [%1, #40] \n\t"                                 \
4709          "ldr x5, [%1, #48] \n\t"                                 \
4710          "ldr x6, [%1, #56] \n\t"                                 \
4711          "ldr x7, [%1, #64] \n\t"                                 \
4712          "ldr x8, [%1, #72] \n\t"                                 \
4713          "str x8, [sp, #0]  \n\t"                                 \
4714          "ldr x8, [%1, #80] \n\t"                                 \
4715          "str x8, [sp, #8]  \n\t"                                 \
4716          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4717          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4718          VALGRIND_RESTORE_STACK                                   \
4719          "mov %0, x0"                                             \
4720          : /*out*/   "=r" (_res)                                  \
4721          : /*in*/    "0" (&_argvec[0])                            \
4722          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4723       );                                                          \
4724       lval = (__typeof__(lval)) _res;                             \
4725    } while (0)
4726 
4727 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4728                                   arg7,arg8,arg9,arg10,arg11)     \
4729    do {                                                           \
4730       volatile OrigFn        _orig = (orig);                      \
4731       volatile unsigned long _argvec[12];                         \
4732       volatile unsigned long _res;                                \
4733       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4734       _argvec[1] = (unsigned long)(arg1);                         \
4735       _argvec[2] = (unsigned long)(arg2);                         \
4736       _argvec[3] = (unsigned long)(arg3);                         \
4737       _argvec[4] = (unsigned long)(arg4);                         \
4738       _argvec[5] = (unsigned long)(arg5);                         \
4739       _argvec[6] = (unsigned long)(arg6);                         \
4740       _argvec[7] = (unsigned long)(arg7);                         \
4741       _argvec[8] = (unsigned long)(arg8);                         \
4742       _argvec[9] = (unsigned long)(arg9);                         \
4743       _argvec[10] = (unsigned long)(arg10);                       \
4744       _argvec[11] = (unsigned long)(arg11);                       \
4745       __asm__ volatile(                                           \
4746          VALGRIND_ALIGN_STACK                                     \
4747          "sub sp, sp, #0x30 \n\t"                                 \
4748          "ldr x0, [%1, #8] \n\t"                                  \
4749          "ldr x1, [%1, #16] \n\t"                                 \
4750          "ldr x2, [%1, #24] \n\t"                                 \
4751          "ldr x3, [%1, #32] \n\t"                                 \
4752          "ldr x4, [%1, #40] \n\t"                                 \
4753          "ldr x5, [%1, #48] \n\t"                                 \
4754          "ldr x6, [%1, #56] \n\t"                                 \
4755          "ldr x7, [%1, #64] \n\t"                                 \
4756          "ldr x8, [%1, #72] \n\t"                                 \
4757          "str x8, [sp, #0]  \n\t"                                 \
4758          "ldr x8, [%1, #80] \n\t"                                 \
4759          "str x8, [sp, #8]  \n\t"                                 \
4760          "ldr x8, [%1, #88] \n\t"                                 \
4761          "str x8, [sp, #16] \n\t"                                 \
4762          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4763          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4764          VALGRIND_RESTORE_STACK                                   \
4765          "mov %0, x0"                                             \
4766          : /*out*/   "=r" (_res)                                  \
4767          : /*in*/    "0" (&_argvec[0])                            \
4768          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4769       );                                                          \
4770       lval = (__typeof__(lval)) _res;                             \
4771    } while (0)
4772 
4773 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4774                                   arg7,arg8,arg9,arg10,arg11,     \
4775                                   arg12)                          \
4776    do {                                                           \
4777       volatile OrigFn        _orig = (orig);                      \
4778       volatile unsigned long _argvec[13];                         \
4779       volatile unsigned long _res;                                \
4780       _argvec[0] = (unsigned long)_orig.nraddr;                   \
4781       _argvec[1] = (unsigned long)(arg1);                         \
4782       _argvec[2] = (unsigned long)(arg2);                         \
4783       _argvec[3] = (unsigned long)(arg3);                         \
4784       _argvec[4] = (unsigned long)(arg4);                         \
4785       _argvec[5] = (unsigned long)(arg5);                         \
4786       _argvec[6] = (unsigned long)(arg6);                         \
4787       _argvec[7] = (unsigned long)(arg7);                         \
4788       _argvec[8] = (unsigned long)(arg8);                         \
4789       _argvec[9] = (unsigned long)(arg9);                         \
4790       _argvec[10] = (unsigned long)(arg10);                       \
4791       _argvec[11] = (unsigned long)(arg11);                       \
4792       _argvec[12] = (unsigned long)(arg12);                       \
4793       __asm__ volatile(                                           \
4794          VALGRIND_ALIGN_STACK                                     \
4795          "sub sp, sp, #0x30 \n\t"                                 \
4796          "ldr x0, [%1, #8] \n\t"                                  \
4797          "ldr x1, [%1, #16] \n\t"                                 \
4798          "ldr x2, [%1, #24] \n\t"                                 \
4799          "ldr x3, [%1, #32] \n\t"                                 \
4800          "ldr x4, [%1, #40] \n\t"                                 \
4801          "ldr x5, [%1, #48] \n\t"                                 \
4802          "ldr x6, [%1, #56] \n\t"                                 \
4803          "ldr x7, [%1, #64] \n\t"                                 \
4804          "ldr x8, [%1, #72] \n\t"                                 \
4805          "str x8, [sp, #0]  \n\t"                                 \
4806          "ldr x8, [%1, #80] \n\t"                                 \
4807          "str x8, [sp, #8]  \n\t"                                 \
4808          "ldr x8, [%1, #88] \n\t"                                 \
4809          "str x8, [sp, #16] \n\t"                                 \
4810          "ldr x8, [%1, #96] \n\t"                                 \
4811          "str x8, [sp, #24] \n\t"                                 \
4812          "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4813          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4814          VALGRIND_RESTORE_STACK                                   \
4815          "mov %0, x0"                                             \
4816          : /*out*/   "=r" (_res)                                  \
4817          : /*in*/    "0" (&_argvec[0])                            \
4818          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4819       );                                                          \
4820       lval = (__typeof__(lval)) _res;                             \
4821    } while (0)
4822 
4823 #endif /* PLAT_arm64_linux || PLAT_arm64_freebsd */
4824 
4825 /* ------------------------- s390x-linux ------------------------- */
4826 
4827 #if defined(PLAT_s390x_linux)
4828 
4829 /* Similar workaround as amd64 (see above), but we use r11 as frame
4830    pointer and save the old r11 in r7. r11 might be used for
4831    argvec, therefore we copy argvec in r1 since r1 is clobbered
4832    after the call anyway.  */
4833 #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
4834 #  define __FRAME_POINTER                                         \
4835       ,"d"(__builtin_dwarf_cfa())
4836 #  define VALGRIND_CFI_PROLOGUE                                   \
4837       ".cfi_remember_state\n\t"                                   \
4838       "lgr 1,%1\n\t" /* copy the argvec pointer in r1 */          \
4839       "lgr 7,11\n\t"                                              \
4840       "lgr 11,%2\n\t"                                             \
4841       ".cfi_def_cfa 11, 0\n\t"
4842 #  define VALGRIND_CFI_EPILOGUE                                   \
4843       "lgr 11, 7\n\t"                                             \
4844       ".cfi_restore_state\n\t"
4845 #else
4846 #  define __FRAME_POINTER
4847 #  define VALGRIND_CFI_PROLOGUE                                   \
4848       "lgr 1,%1\n\t"
4849 #  define VALGRIND_CFI_EPILOGUE
4850 #endif
4851 
4852 /* Nb: On s390 the stack pointer is properly aligned *at all times*
4853    according to the s390 GCC maintainer. (The ABI specification is not
4854    precise in this regard.) Therefore, VALGRIND_ALIGN_STACK and
4855    VALGRIND_RESTORE_STACK are not defined here. */
4856 
4857 /* These regs are trashed by the hidden call. Note that we overwrite
4858    r14 in s390_irgen_noredir (VEX/priv/guest_s390_irgen.c) to give the
4859    function a proper return address. All others are ABI defined call
4860    clobbers. */
4861 #if defined(__VX__) || defined(__S390_VX__)
4862 #define __CALLER_SAVED_REGS "0", "1", "2", "3", "4", "5", "14",   \
4863       "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",             \
4864       "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",       \
4865       "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",     \
4866       "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
4867 #else
4868 #define __CALLER_SAVED_REGS "0", "1", "2", "3", "4", "5", "14",   \
4869       "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7"
4870 #endif
4871 
4872 /* Nb: Although r11 is modified in the asm snippets below (inside 
4873    VALGRIND_CFI_PROLOGUE) it is not listed in the clobber section, for
4874    two reasons:
4875    (1) r11 is restored in VALGRIND_CFI_EPILOGUE, so effectively it is not
4876        modified
4877    (2) GCC will complain that r11 cannot appear inside a clobber section,
4878        when compiled with -O -fno-omit-frame-pointer
4879  */
4880 
4881 #define CALL_FN_W_v(lval, orig)                                  \
4882    do {                                                          \
4883       volatile OrigFn        _orig = (orig);                     \
4884       volatile unsigned long  _argvec[1];                        \
4885       volatile unsigned long _res;                               \
4886       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4887       __asm__ volatile(                                          \
4888          VALGRIND_CFI_PROLOGUE                                   \
4889          "aghi 15,-160\n\t"                                      \
4890          "lg 1, 0(1)\n\t"  /* target->r1 */                      \
4891          VALGRIND_CALL_NOREDIR_R1                                \
4892          "aghi 15,160\n\t"                                       \
4893          VALGRIND_CFI_EPILOGUE                                   \
4894          "lgr %0, 2\n\t"                                         \
4895          : /*out*/   "=d" (_res)                                 \
4896          : /*in*/    "d" (&_argvec[0]) __FRAME_POINTER           \
4897          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4898       );                                                         \
4899       lval = (__typeof__(lval)) _res;                            \
4900    } while (0)
4901 
4902 /* The call abi has the arguments in r2-r6 and stack */
4903 #define CALL_FN_W_W(lval, orig, arg1)                            \
4904    do {                                                          \
4905       volatile OrigFn        _orig = (orig);                     \
4906       volatile unsigned long _argvec[2];                         \
4907       volatile unsigned long _res;                               \
4908       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4909       _argvec[1] = (unsigned long)arg1;                          \
4910       __asm__ volatile(                                          \
4911          VALGRIND_CFI_PROLOGUE                                   \
4912          "aghi 15,-160\n\t"                                      \
4913          "lg 2, 8(1)\n\t"                                        \
4914          "lg 1, 0(1)\n\t"                                        \
4915          VALGRIND_CALL_NOREDIR_R1                                \
4916          "aghi 15,160\n\t"                                       \
4917          VALGRIND_CFI_EPILOGUE                                   \
4918          "lgr %0, 2\n\t"                                         \
4919          : /*out*/   "=d" (_res)                                 \
4920          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4921          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4922       );                                                         \
4923       lval = (__typeof__(lval)) _res;                            \
4924    } while (0)
4925 
4926 #define CALL_FN_W_WW(lval, orig, arg1, arg2)                     \
4927    do {                                                          \
4928       volatile OrigFn        _orig = (orig);                     \
4929       volatile unsigned long _argvec[3];                         \
4930       volatile unsigned long _res;                               \
4931       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4932       _argvec[1] = (unsigned long)arg1;                          \
4933       _argvec[2] = (unsigned long)arg2;                          \
4934       __asm__ volatile(                                          \
4935          VALGRIND_CFI_PROLOGUE                                   \
4936          "aghi 15,-160\n\t"                                      \
4937          "lg 2, 8(1)\n\t"                                        \
4938          "lg 3,16(1)\n\t"                                        \
4939          "lg 1, 0(1)\n\t"                                        \
4940          VALGRIND_CALL_NOREDIR_R1                                \
4941          "aghi 15,160\n\t"                                       \
4942          VALGRIND_CFI_EPILOGUE                                   \
4943          "lgr %0, 2\n\t"                                         \
4944          : /*out*/   "=d" (_res)                                 \
4945          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4946          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4947       );                                                         \
4948       lval = (__typeof__(lval)) _res;                            \
4949    } while (0)
4950 
4951 #define CALL_FN_W_WWW(lval, orig, arg1, arg2, arg3)              \
4952    do {                                                          \
4953       volatile OrigFn        _orig = (orig);                     \
4954       volatile unsigned long _argvec[4];                         \
4955       volatile unsigned long _res;                               \
4956       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4957       _argvec[1] = (unsigned long)arg1;                          \
4958       _argvec[2] = (unsigned long)arg2;                          \
4959       _argvec[3] = (unsigned long)arg3;                          \
4960       __asm__ volatile(                                          \
4961          VALGRIND_CFI_PROLOGUE                                   \
4962          "aghi 15,-160\n\t"                                      \
4963          "lg 2, 8(1)\n\t"                                        \
4964          "lg 3,16(1)\n\t"                                        \
4965          "lg 4,24(1)\n\t"                                        \
4966          "lg 1, 0(1)\n\t"                                        \
4967          VALGRIND_CALL_NOREDIR_R1                                \
4968          "aghi 15,160\n\t"                                       \
4969          VALGRIND_CFI_EPILOGUE                                   \
4970          "lgr %0, 2\n\t"                                         \
4971          : /*out*/   "=d" (_res)                                 \
4972          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4973          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4974       );                                                         \
4975       lval = (__typeof__(lval)) _res;                            \
4976    } while (0)
4977 
4978 #define CALL_FN_W_WWWW(lval, orig, arg1, arg2, arg3, arg4)       \
4979    do {                                                          \
4980       volatile OrigFn        _orig = (orig);                     \
4981       volatile unsigned long _argvec[5];                         \
4982       volatile unsigned long _res;                               \
4983       _argvec[0] = (unsigned long)_orig.nraddr;                  \
4984       _argvec[1] = (unsigned long)arg1;                          \
4985       _argvec[2] = (unsigned long)arg2;                          \
4986       _argvec[3] = (unsigned long)arg3;                          \
4987       _argvec[4] = (unsigned long)arg4;                          \
4988       __asm__ volatile(                                          \
4989          VALGRIND_CFI_PROLOGUE                                   \
4990          "aghi 15,-160\n\t"                                      \
4991          "lg 2, 8(1)\n\t"                                        \
4992          "lg 3,16(1)\n\t"                                        \
4993          "lg 4,24(1)\n\t"                                        \
4994          "lg 5,32(1)\n\t"                                        \
4995          "lg 1, 0(1)\n\t"                                        \
4996          VALGRIND_CALL_NOREDIR_R1                                \
4997          "aghi 15,160\n\t"                                       \
4998          VALGRIND_CFI_EPILOGUE                                   \
4999          "lgr %0, 2\n\t"                                         \
5000          : /*out*/   "=d" (_res)                                 \
5001          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5002          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
5003       );                                                         \
5004       lval = (__typeof__(lval)) _res;                            \
5005    } while (0)
5006 
5007 #define CALL_FN_W_5W(lval, orig, arg1, arg2, arg3, arg4, arg5)   \
5008    do {                                                          \
5009       volatile OrigFn        _orig = (orig);                     \
5010       volatile unsigned long _argvec[6];                         \
5011       volatile unsigned long _res;                               \
5012       _argvec[0] = (unsigned long)_orig.nraddr;                  \
5013       _argvec[1] = (unsigned long)arg1;                          \
5014       _argvec[2] = (unsigned long)arg2;                          \
5015       _argvec[3] = (unsigned long)arg3;                          \
5016       _argvec[4] = (unsigned long)arg4;                          \
5017       _argvec[5] = (unsigned long)arg5;                          \
5018       __asm__ volatile(                                          \
5019          VALGRIND_CFI_PROLOGUE                                   \
5020          "aghi 15,-160\n\t"                                      \
5021          "lg 2, 8(1)\n\t"                                        \
5022          "lg 3,16(1)\n\t"                                        \
5023          "lg 4,24(1)\n\t"                                        \
5024          "lg 5,32(1)\n\t"                                        \
5025          "lg 6,40(1)\n\t"                                        \
5026          "lg 1, 0(1)\n\t"                                        \
5027          VALGRIND_CALL_NOREDIR_R1                                \
5028          "aghi 15,160\n\t"                                       \
5029          VALGRIND_CFI_EPILOGUE                                   \
5030          "lgr %0, 2\n\t"                                         \
5031          : /*out*/   "=d" (_res)                                 \
5032          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5033          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5034       );                                                         \
5035       lval = (__typeof__(lval)) _res;                            \
5036    } while (0)
5037 
5038 #define CALL_FN_W_6W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
5039                      arg6)                                       \
5040    do {                                                          \
5041       volatile OrigFn        _orig = (orig);                     \
5042       volatile unsigned long _argvec[7];                         \
5043       volatile unsigned long _res;                               \
5044       _argvec[0] = (unsigned long)_orig.nraddr;                  \
5045       _argvec[1] = (unsigned long)arg1;                          \
5046       _argvec[2] = (unsigned long)arg2;                          \
5047       _argvec[3] = (unsigned long)arg3;                          \
5048       _argvec[4] = (unsigned long)arg4;                          \
5049       _argvec[5] = (unsigned long)arg5;                          \
5050       _argvec[6] = (unsigned long)arg6;                          \
5051       __asm__ volatile(                                          \
5052          VALGRIND_CFI_PROLOGUE                                   \
5053          "aghi 15,-168\n\t"                                      \
5054          "lg 2, 8(1)\n\t"                                        \
5055          "lg 3,16(1)\n\t"                                        \
5056          "lg 4,24(1)\n\t"                                        \
5057          "lg 5,32(1)\n\t"                                        \
5058          "lg 6,40(1)\n\t"                                        \
5059          "mvc 160(8,15), 48(1)\n\t"                              \
5060          "lg 1, 0(1)\n\t"                                        \
5061          VALGRIND_CALL_NOREDIR_R1                                \
5062          "aghi 15,168\n\t"                                       \
5063          VALGRIND_CFI_EPILOGUE                                   \
5064          "lgr %0, 2\n\t"                                         \
5065          : /*out*/   "=d" (_res)                                 \
5066          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5067          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5068       );                                                         \
5069       lval = (__typeof__(lval)) _res;                            \
5070    } while (0)
5071 
5072 #define CALL_FN_W_7W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
5073                      arg6, arg7)                                 \
5074    do {                                                          \
5075       volatile OrigFn        _orig = (orig);                     \
5076       volatile unsigned long _argvec[8];                         \
5077       volatile unsigned long _res;                               \
5078       _argvec[0] = (unsigned long)_orig.nraddr;                  \
5079       _argvec[1] = (unsigned long)arg1;                          \
5080       _argvec[2] = (unsigned long)arg2;                          \
5081       _argvec[3] = (unsigned long)arg3;                          \
5082       _argvec[4] = (unsigned long)arg4;                          \
5083       _argvec[5] = (unsigned long)arg5;                          \
5084       _argvec[6] = (unsigned long)arg6;                          \
5085       _argvec[7] = (unsigned long)arg7;                          \
5086       __asm__ volatile(                                          \
5087          VALGRIND_CFI_PROLOGUE                                   \
5088          "aghi 15,-176\n\t"                                      \
5089          "lg 2, 8(1)\n\t"                                        \
5090          "lg 3,16(1)\n\t"                                        \
5091          "lg 4,24(1)\n\t"                                        \
5092          "lg 5,32(1)\n\t"                                        \
5093          "lg 6,40(1)\n\t"                                        \
5094          "mvc 160(8,15), 48(1)\n\t"                              \
5095          "mvc 168(8,15), 56(1)\n\t"                              \
5096          "lg 1, 0(1)\n\t"                                        \
5097          VALGRIND_CALL_NOREDIR_R1                                \
5098          "aghi 15,176\n\t"                                       \
5099          VALGRIND_CFI_EPILOGUE                                   \
5100          "lgr %0, 2\n\t"                                         \
5101          : /*out*/   "=d" (_res)                                 \
5102          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5103          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5104       );                                                         \
5105       lval = (__typeof__(lval)) _res;                            \
5106    } while (0)
5107 
5108 #define CALL_FN_W_8W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
5109                      arg6, arg7 ,arg8)                           \
5110    do {                                                          \
5111       volatile OrigFn        _orig = (orig);                     \
5112       volatile unsigned long _argvec[9];                         \
5113       volatile unsigned long _res;                               \
5114       _argvec[0] = (unsigned long)_orig.nraddr;                  \
5115       _argvec[1] = (unsigned long)arg1;                          \
5116       _argvec[2] = (unsigned long)arg2;                          \
5117       _argvec[3] = (unsigned long)arg3;                          \
5118       _argvec[4] = (unsigned long)arg4;                          \
5119       _argvec[5] = (unsigned long)arg5;                          \
5120       _argvec[6] = (unsigned long)arg6;                          \
5121       _argvec[7] = (unsigned long)arg7;                          \
5122       _argvec[8] = (unsigned long)arg8;                          \
5123       __asm__ volatile(                                          \
5124          VALGRIND_CFI_PROLOGUE                                   \
5125          "aghi 15,-184\n\t"                                      \
5126          "lg 2, 8(1)\n\t"                                        \
5127          "lg 3,16(1)\n\t"                                        \
5128          "lg 4,24(1)\n\t"                                        \
5129          "lg 5,32(1)\n\t"                                        \
5130          "lg 6,40(1)\n\t"                                        \
5131          "mvc 160(8,15), 48(1)\n\t"                              \
5132          "mvc 168(8,15), 56(1)\n\t"                              \
5133          "mvc 176(8,15), 64(1)\n\t"                              \
5134          "lg 1, 0(1)\n\t"                                        \
5135          VALGRIND_CALL_NOREDIR_R1                                \
5136          "aghi 15,184\n\t"                                       \
5137          VALGRIND_CFI_EPILOGUE                                   \
5138          "lgr %0, 2\n\t"                                         \
5139          : /*out*/   "=d" (_res)                                 \
5140          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5141          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5142       );                                                         \
5143       lval = (__typeof__(lval)) _res;                            \
5144    } while (0)
5145 
5146 #define CALL_FN_W_9W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
5147                      arg6, arg7 ,arg8, arg9)                     \
5148    do {                                                          \
5149       volatile OrigFn        _orig = (orig);                     \
5150       volatile unsigned long _argvec[10];                        \
5151       volatile unsigned long _res;                               \
5152       _argvec[0] = (unsigned long)_orig.nraddr;                  \
5153       _argvec[1] = (unsigned long)arg1;                          \
5154       _argvec[2] = (unsigned long)arg2;                          \
5155       _argvec[3] = (unsigned long)arg3;                          \
5156       _argvec[4] = (unsigned long)arg4;                          \
5157       _argvec[5] = (unsigned long)arg5;                          \
5158       _argvec[6] = (unsigned long)arg6;                          \
5159       _argvec[7] = (unsigned long)arg7;                          \
5160       _argvec[8] = (unsigned long)arg8;                          \
5161       _argvec[9] = (unsigned long)arg9;                          \
5162       __asm__ volatile(                                          \
5163          VALGRIND_CFI_PROLOGUE                                   \
5164          "aghi 15,-192\n\t"                                      \
5165          "lg 2, 8(1)\n\t"                                        \
5166          "lg 3,16(1)\n\t"                                        \
5167          "lg 4,24(1)\n\t"                                        \
5168          "lg 5,32(1)\n\t"                                        \
5169          "lg 6,40(1)\n\t"                                        \
5170          "mvc 160(8,15), 48(1)\n\t"                              \
5171          "mvc 168(8,15), 56(1)\n\t"                              \
5172          "mvc 176(8,15), 64(1)\n\t"                              \
5173          "mvc 184(8,15), 72(1)\n\t"                              \
5174          "lg 1, 0(1)\n\t"                                        \
5175          VALGRIND_CALL_NOREDIR_R1                                \
5176          "aghi 15,192\n\t"                                       \
5177          VALGRIND_CFI_EPILOGUE                                   \
5178          "lgr %0, 2\n\t"                                         \
5179          : /*out*/   "=d" (_res)                                 \
5180          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5181          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5182       );                                                         \
5183       lval = (__typeof__(lval)) _res;                            \
5184    } while (0)
5185 
5186 #define CALL_FN_W_10W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
5187                      arg6, arg7 ,arg8, arg9, arg10)              \
5188    do {                                                          \
5189       volatile OrigFn        _orig = (orig);                     \
5190       volatile unsigned long _argvec[11];                        \
5191       volatile unsigned long _res;                               \
5192       _argvec[0] = (unsigned long)_orig.nraddr;                  \
5193       _argvec[1] = (unsigned long)arg1;                          \
5194       _argvec[2] = (unsigned long)arg2;                          \
5195       _argvec[3] = (unsigned long)arg3;                          \
5196       _argvec[4] = (unsigned long)arg4;                          \
5197       _argvec[5] = (unsigned long)arg5;                          \
5198       _argvec[6] = (unsigned long)arg6;                          \
5199       _argvec[7] = (unsigned long)arg7;                          \
5200       _argvec[8] = (unsigned long)arg8;                          \
5201       _argvec[9] = (unsigned long)arg9;                          \
5202       _argvec[10] = (unsigned long)arg10;                        \
5203       __asm__ volatile(                                          \
5204          VALGRIND_CFI_PROLOGUE                                   \
5205          "aghi 15,-200\n\t"                                      \
5206          "lg 2, 8(1)\n\t"                                        \
5207          "lg 3,16(1)\n\t"                                        \
5208          "lg 4,24(1)\n\t"                                        \
5209          "lg 5,32(1)\n\t"                                        \
5210          "lg 6,40(1)\n\t"                                        \
5211          "mvc 160(8,15), 48(1)\n\t"                              \
5212          "mvc 168(8,15), 56(1)\n\t"                              \
5213          "mvc 176(8,15), 64(1)\n\t"                              \
5214          "mvc 184(8,15), 72(1)\n\t"                              \
5215          "mvc 192(8,15), 80(1)\n\t"                              \
5216          "lg 1, 0(1)\n\t"                                        \
5217          VALGRIND_CALL_NOREDIR_R1                                \
5218          "aghi 15,200\n\t"                                       \
5219          VALGRIND_CFI_EPILOGUE                                   \
5220          "lgr %0, 2\n\t"                                         \
5221          : /*out*/   "=d" (_res)                                 \
5222          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5223          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5224       );                                                         \
5225       lval = (__typeof__(lval)) _res;                            \
5226    } while (0)
5227 
5228 #define CALL_FN_W_11W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
5229                      arg6, arg7 ,arg8, arg9, arg10, arg11)       \
5230    do {                                                          \
5231       volatile OrigFn        _orig = (orig);                     \
5232       volatile unsigned long _argvec[12];                        \
5233       volatile unsigned long _res;                               \
5234       _argvec[0] = (unsigned long)_orig.nraddr;                  \
5235       _argvec[1] = (unsigned long)arg1;                          \
5236       _argvec[2] = (unsigned long)arg2;                          \
5237       _argvec[3] = (unsigned long)arg3;                          \
5238       _argvec[4] = (unsigned long)arg4;                          \
5239       _argvec[5] = (unsigned long)arg5;                          \
5240       _argvec[6] = (unsigned long)arg6;                          \
5241       _argvec[7] = (unsigned long)arg7;                          \
5242       _argvec[8] = (unsigned long)arg8;                          \
5243       _argvec[9] = (unsigned long)arg9;                          \
5244       _argvec[10] = (unsigned long)arg10;                        \
5245       _argvec[11] = (unsigned long)arg11;                        \
5246       __asm__ volatile(                                          \
5247          VALGRIND_CFI_PROLOGUE                                   \
5248          "aghi 15,-208\n\t"                                      \
5249          "lg 2, 8(1)\n\t"                                        \
5250          "lg 3,16(1)\n\t"                                        \
5251          "lg 4,24(1)\n\t"                                        \
5252          "lg 5,32(1)\n\t"                                        \
5253          "lg 6,40(1)\n\t"                                        \
5254          "mvc 160(8,15), 48(1)\n\t"                              \
5255          "mvc 168(8,15), 56(1)\n\t"                              \
5256          "mvc 176(8,15), 64(1)\n\t"                              \
5257          "mvc 184(8,15), 72(1)\n\t"                              \
5258          "mvc 192(8,15), 80(1)\n\t"                              \
5259          "mvc 200(8,15), 88(1)\n\t"                              \
5260          "lg 1, 0(1)\n\t"                                        \
5261          VALGRIND_CALL_NOREDIR_R1                                \
5262          "aghi 15,208\n\t"                                       \
5263          VALGRIND_CFI_EPILOGUE                                   \
5264          "lgr %0, 2\n\t"                                         \
5265          : /*out*/   "=d" (_res)                                 \
5266          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5267          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5268       );                                                         \
5269       lval = (__typeof__(lval)) _res;                            \
5270    } while (0)
5271 
5272 #define CALL_FN_W_12W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
5273                      arg6, arg7 ,arg8, arg9, arg10, arg11, arg12)\
5274    do {                                                          \
5275       volatile OrigFn        _orig = (orig);                     \
5276       volatile unsigned long _argvec[13];                        \
5277       volatile unsigned long _res;                               \
5278       _argvec[0] = (unsigned long)_orig.nraddr;                  \
5279       _argvec[1] = (unsigned long)arg1;                          \
5280       _argvec[2] = (unsigned long)arg2;                          \
5281       _argvec[3] = (unsigned long)arg3;                          \
5282       _argvec[4] = (unsigned long)arg4;                          \
5283       _argvec[5] = (unsigned long)arg5;                          \
5284       _argvec[6] = (unsigned long)arg6;                          \
5285       _argvec[7] = (unsigned long)arg7;                          \
5286       _argvec[8] = (unsigned long)arg8;                          \
5287       _argvec[9] = (unsigned long)arg9;                          \
5288       _argvec[10] = (unsigned long)arg10;                        \
5289       _argvec[11] = (unsigned long)arg11;                        \
5290       _argvec[12] = (unsigned long)arg12;                        \
5291       __asm__ volatile(                                          \
5292          VALGRIND_CFI_PROLOGUE                                   \
5293          "aghi 15,-216\n\t"                                      \
5294          "lg 2, 8(1)\n\t"                                        \
5295          "lg 3,16(1)\n\t"                                        \
5296          "lg 4,24(1)\n\t"                                        \
5297          "lg 5,32(1)\n\t"                                        \
5298          "lg 6,40(1)\n\t"                                        \
5299          "mvc 160(8,15), 48(1)\n\t"                              \
5300          "mvc 168(8,15), 56(1)\n\t"                              \
5301          "mvc 176(8,15), 64(1)\n\t"                              \
5302          "mvc 184(8,15), 72(1)\n\t"                              \
5303          "mvc 192(8,15), 80(1)\n\t"                              \
5304          "mvc 200(8,15), 88(1)\n\t"                              \
5305          "mvc 208(8,15), 96(1)\n\t"                              \
5306          "lg 1, 0(1)\n\t"                                        \
5307          VALGRIND_CALL_NOREDIR_R1                                \
5308          "aghi 15,216\n\t"                                       \
5309          VALGRIND_CFI_EPILOGUE                                   \
5310          "lgr %0, 2\n\t"                                         \
5311          : /*out*/   "=d" (_res)                                 \
5312          : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5313          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5314       );                                                         \
5315       lval = (__typeof__(lval)) _res;                            \
5316    } while (0)
5317 
5318 
5319 #endif /* PLAT_s390x_linux */
5320 
5321 /* ------------------------- mips32-linux ----------------------- */
5322  
5323 #if defined(PLAT_mips32_linux)
5324 
5325 /* These regs are trashed by the hidden call. */
5326 #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6",       \
5327 "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \
5328 "$25", "$31"
5329 
5330 /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned
5331    long) == 4. */
5332 
5333 #define CALL_FN_W_v(lval, orig)                                   \
5334    do {                                                           \
5335       volatile OrigFn        _orig = (orig);                      \
5336       volatile unsigned long _argvec[1];                          \
5337       volatile unsigned long _res;                                \
5338       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5339       __asm__ volatile(                                           \
5340          "subu $29, $29, 8 \n\t"                                  \
5341          "sw $28, 0($29) \n\t"                                    \
5342          "sw $31, 4($29) \n\t"                                    \
5343          "subu $29, $29, 16 \n\t"                                 \
5344          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5345          VALGRIND_CALL_NOREDIR_T9                                 \
5346          "addu $29, $29, 16\n\t"                                  \
5347          "lw $28, 0($29) \n\t"                                    \
5348          "lw $31, 4($29) \n\t"                                    \
5349          "addu $29, $29, 8 \n\t"                                  \
5350          "move %0, $2\n"                                          \
5351          : /*out*/   "=r" (_res)                                  \
5352          : /*in*/    "0" (&_argvec[0])                            \
5353          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5354       );                                                          \
5355       lval = (__typeof__(lval)) _res;                             \
5356    } while (0)
5357 
5358 #define CALL_FN_W_W(lval, orig, arg1)                             \
5359    do {                                                           \
5360       volatile OrigFn        _orig = (orig);                      \
5361      volatile unsigned long _argvec[2];                           \
5362       volatile unsigned long _res;                                \
5363       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5364       _argvec[1] = (unsigned long)(arg1);                         \
5365       __asm__ volatile(                                           \
5366          "subu $29, $29, 8 \n\t"                                  \
5367          "sw $28, 0($29) \n\t"                                    \
5368          "sw $31, 4($29) \n\t"                                    \
5369          "subu $29, $29, 16 \n\t"                                 \
5370          "lw $4, 4(%1) \n\t"   /* arg1*/                          \
5371          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5372          VALGRIND_CALL_NOREDIR_T9                                 \
5373          "addu $29, $29, 16 \n\t"                                 \
5374          "lw $28, 0($29) \n\t"                                    \
5375          "lw $31, 4($29) \n\t"                                    \
5376          "addu $29, $29, 8 \n\t"                                  \
5377          "move %0, $2\n"                                          \
5378          : /*out*/   "=r" (_res)                                  \
5379          : /*in*/    "0" (&_argvec[0])                            \
5380          : /*trash*/ "memory",  __CALLER_SAVED_REGS               \
5381       );                                                          \
5382       lval = (__typeof__(lval)) _res;                             \
5383    } while (0)
5384 
5385 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
5386    do {                                                           \
5387       volatile OrigFn        _orig = (orig);                      \
5388       volatile unsigned long _argvec[3];                          \
5389       volatile unsigned long _res;                                \
5390       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5391       _argvec[1] = (unsigned long)(arg1);                         \
5392       _argvec[2] = (unsigned long)(arg2);                         \
5393       __asm__ volatile(                                           \
5394          "subu $29, $29, 8 \n\t"                                  \
5395          "sw $28, 0($29) \n\t"                                    \
5396          "sw $31, 4($29) \n\t"                                    \
5397          "subu $29, $29, 16 \n\t"                                 \
5398          "lw $4, 4(%1) \n\t"                                      \
5399          "lw $5, 8(%1) \n\t"                                      \
5400          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5401          VALGRIND_CALL_NOREDIR_T9                                 \
5402          "addu $29, $29, 16 \n\t"                                 \
5403          "lw $28, 0($29) \n\t"                                    \
5404          "lw $31, 4($29) \n\t"                                    \
5405          "addu $29, $29, 8 \n\t"                                  \
5406          "move %0, $2\n"                                          \
5407          : /*out*/   "=r" (_res)                                  \
5408          : /*in*/    "0" (&_argvec[0])                            \
5409          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5410       );                                                          \
5411       lval = (__typeof__(lval)) _res;                             \
5412    } while (0)
5413 
5414 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
5415    do {                                                           \
5416       volatile OrigFn        _orig = (orig);                      \
5417       volatile unsigned long _argvec[4];                          \
5418       volatile unsigned long _res;                                \
5419       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5420       _argvec[1] = (unsigned long)(arg1);                         \
5421       _argvec[2] = (unsigned long)(arg2);                         \
5422       _argvec[3] = (unsigned long)(arg3);                         \
5423       __asm__ volatile(                                           \
5424          "subu $29, $29, 8 \n\t"                                  \
5425          "sw $28, 0($29) \n\t"                                    \
5426          "sw $31, 4($29) \n\t"                                    \
5427          "subu $29, $29, 16 \n\t"                                 \
5428          "lw $4, 4(%1) \n\t"                                      \
5429          "lw $5, 8(%1) \n\t"                                      \
5430          "lw $6, 12(%1) \n\t"                                     \
5431          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5432          VALGRIND_CALL_NOREDIR_T9                                 \
5433          "addu $29, $29, 16 \n\t"                                 \
5434          "lw $28, 0($29) \n\t"                                    \
5435          "lw $31, 4($29) \n\t"                                    \
5436          "addu $29, $29, 8 \n\t"                                  \
5437          "move %0, $2\n"                                          \
5438          : /*out*/   "=r" (_res)                                  \
5439          : /*in*/    "0" (&_argvec[0])                            \
5440          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5441       );                                                          \
5442       lval = (__typeof__(lval)) _res;                             \
5443    } while (0)
5444 
5445 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
5446    do {                                                           \
5447       volatile OrigFn        _orig = (orig);                      \
5448       volatile unsigned long _argvec[5];                          \
5449       volatile unsigned long _res;                                \
5450       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5451       _argvec[1] = (unsigned long)(arg1);                         \
5452       _argvec[2] = (unsigned long)(arg2);                         \
5453       _argvec[3] = (unsigned long)(arg3);                         \
5454       _argvec[4] = (unsigned long)(arg4);                         \
5455       __asm__ volatile(                                           \
5456          "subu $29, $29, 8 \n\t"                                  \
5457          "sw $28, 0($29) \n\t"                                    \
5458          "sw $31, 4($29) \n\t"                                    \
5459          "subu $29, $29, 16 \n\t"                                 \
5460          "lw $4, 4(%1) \n\t"                                      \
5461          "lw $5, 8(%1) \n\t"                                      \
5462          "lw $6, 12(%1) \n\t"                                     \
5463          "lw $7, 16(%1) \n\t"                                     \
5464          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5465          VALGRIND_CALL_NOREDIR_T9                                 \
5466          "addu $29, $29, 16 \n\t"                                 \
5467          "lw $28, 0($29) \n\t"                                    \
5468          "lw $31, 4($29) \n\t"                                    \
5469          "addu $29, $29, 8 \n\t"                                  \
5470          "move %0, $2\n"                                          \
5471          : /*out*/   "=r" (_res)                                  \
5472          : /*in*/    "0" (&_argvec[0])                            \
5473          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5474       );                                                          \
5475       lval = (__typeof__(lval)) _res;                             \
5476    } while (0)
5477 
5478 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
5479    do {                                                           \
5480       volatile OrigFn        _orig = (orig);                      \
5481       volatile unsigned long _argvec[6];                          \
5482       volatile unsigned long _res;                                \
5483       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5484       _argvec[1] = (unsigned long)(arg1);                         \
5485       _argvec[2] = (unsigned long)(arg2);                         \
5486       _argvec[3] = (unsigned long)(arg3);                         \
5487       _argvec[4] = (unsigned long)(arg4);                         \
5488       _argvec[5] = (unsigned long)(arg5);                         \
5489       __asm__ volatile(                                           \
5490          "subu $29, $29, 8 \n\t"                                  \
5491          "sw $28, 0($29) \n\t"                                    \
5492          "sw $31, 4($29) \n\t"                                    \
5493          "lw $4, 20(%1) \n\t"                                     \
5494          "subu $29, $29, 24\n\t"                                  \
5495          "sw $4, 16($29) \n\t"                                    \
5496          "lw $4, 4(%1) \n\t"                                      \
5497          "lw $5, 8(%1) \n\t"                                      \
5498          "lw $6, 12(%1) \n\t"                                     \
5499          "lw $7, 16(%1) \n\t"                                     \
5500          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5501          VALGRIND_CALL_NOREDIR_T9                                 \
5502          "addu $29, $29, 24 \n\t"                                 \
5503          "lw $28, 0($29) \n\t"                                    \
5504          "lw $31, 4($29) \n\t"                                    \
5505          "addu $29, $29, 8 \n\t"                                  \
5506          "move %0, $2\n"                                          \
5507          : /*out*/   "=r" (_res)                                  \
5508          : /*in*/    "0" (&_argvec[0])                            \
5509          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5510       );                                                          \
5511       lval = (__typeof__(lval)) _res;                             \
5512    } while (0)
5513 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
5514    do {                                                           \
5515       volatile OrigFn        _orig = (orig);                      \
5516       volatile unsigned long _argvec[7];                          \
5517       volatile unsigned long _res;                                \
5518       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5519       _argvec[1] = (unsigned long)(arg1);                         \
5520       _argvec[2] = (unsigned long)(arg2);                         \
5521       _argvec[3] = (unsigned long)(arg3);                         \
5522       _argvec[4] = (unsigned long)(arg4);                         \
5523       _argvec[5] = (unsigned long)(arg5);                         \
5524       _argvec[6] = (unsigned long)(arg6);                         \
5525       __asm__ volatile(                                           \
5526          "subu $29, $29, 8 \n\t"                                  \
5527          "sw $28, 0($29) \n\t"                                    \
5528          "sw $31, 4($29) \n\t"                                    \
5529          "lw $4, 20(%1) \n\t"                                     \
5530          "subu $29, $29, 32\n\t"                                  \
5531          "sw $4, 16($29) \n\t"                                    \
5532          "lw $4, 24(%1) \n\t"                                     \
5533          "nop\n\t"                                                \
5534          "sw $4, 20($29) \n\t"                                    \
5535          "lw $4, 4(%1) \n\t"                                      \
5536          "lw $5, 8(%1) \n\t"                                      \
5537          "lw $6, 12(%1) \n\t"                                     \
5538          "lw $7, 16(%1) \n\t"                                     \
5539          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5540          VALGRIND_CALL_NOREDIR_T9                                 \
5541          "addu $29, $29, 32 \n\t"                                 \
5542          "lw $28, 0($29) \n\t"                                    \
5543          "lw $31, 4($29) \n\t"                                    \
5544          "addu $29, $29, 8 \n\t"                                  \
5545          "move %0, $2\n"                                          \
5546          : /*out*/   "=r" (_res)                                  \
5547          : /*in*/    "0" (&_argvec[0])                            \
5548          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5549       );                                                          \
5550       lval = (__typeof__(lval)) _res;                             \
5551    } while (0)
5552 
5553 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5554                                  arg7)                            \
5555    do {                                                           \
5556       volatile OrigFn        _orig = (orig);                      \
5557       volatile unsigned long _argvec[8];                          \
5558       volatile unsigned long _res;                                \
5559       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5560       _argvec[1] = (unsigned long)(arg1);                         \
5561       _argvec[2] = (unsigned long)(arg2);                         \
5562       _argvec[3] = (unsigned long)(arg3);                         \
5563       _argvec[4] = (unsigned long)(arg4);                         \
5564       _argvec[5] = (unsigned long)(arg5);                         \
5565       _argvec[6] = (unsigned long)(arg6);                         \
5566       _argvec[7] = (unsigned long)(arg7);                         \
5567       __asm__ volatile(                                           \
5568          "subu $29, $29, 8 \n\t"                                  \
5569          "sw $28, 0($29) \n\t"                                    \
5570          "sw $31, 4($29) \n\t"                                    \
5571          "lw $4, 20(%1) \n\t"                                     \
5572          "subu $29, $29, 32\n\t"                                  \
5573          "sw $4, 16($29) \n\t"                                    \
5574          "lw $4, 24(%1) \n\t"                                     \
5575          "sw $4, 20($29) \n\t"                                    \
5576          "lw $4, 28(%1) \n\t"                                     \
5577          "sw $4, 24($29) \n\t"                                    \
5578          "lw $4, 4(%1) \n\t"                                      \
5579          "lw $5, 8(%1) \n\t"                                      \
5580          "lw $6, 12(%1) \n\t"                                     \
5581          "lw $7, 16(%1) \n\t"                                     \
5582          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5583          VALGRIND_CALL_NOREDIR_T9                                 \
5584          "addu $29, $29, 32 \n\t"                                 \
5585          "lw $28, 0($29) \n\t"                                    \
5586          "lw $31, 4($29) \n\t"                                    \
5587          "addu $29, $29, 8 \n\t"                                  \
5588          "move %0, $2\n"                                          \
5589          : /*out*/   "=r" (_res)                                  \
5590          : /*in*/    "0" (&_argvec[0])                            \
5591          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5592       );                                                          \
5593       lval = (__typeof__(lval)) _res;                             \
5594    } while (0)
5595 
5596 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5597                                  arg7,arg8)                       \
5598    do {                                                           \
5599       volatile OrigFn        _orig = (orig);                      \
5600       volatile unsigned long _argvec[9];                          \
5601       volatile unsigned long _res;                                \
5602       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5603       _argvec[1] = (unsigned long)(arg1);                         \
5604       _argvec[2] = (unsigned long)(arg2);                         \
5605       _argvec[3] = (unsigned long)(arg3);                         \
5606       _argvec[4] = (unsigned long)(arg4);                         \
5607       _argvec[5] = (unsigned long)(arg5);                         \
5608       _argvec[6] = (unsigned long)(arg6);                         \
5609       _argvec[7] = (unsigned long)(arg7);                         \
5610       _argvec[8] = (unsigned long)(arg8);                         \
5611       __asm__ volatile(                                           \
5612          "subu $29, $29, 8 \n\t"                                  \
5613          "sw $28, 0($29) \n\t"                                    \
5614          "sw $31, 4($29) \n\t"                                    \
5615          "lw $4, 20(%1) \n\t"                                     \
5616          "subu $29, $29, 40\n\t"                                  \
5617          "sw $4, 16($29) \n\t"                                    \
5618          "lw $4, 24(%1) \n\t"                                     \
5619          "sw $4, 20($29) \n\t"                                    \
5620          "lw $4, 28(%1) \n\t"                                     \
5621          "sw $4, 24($29) \n\t"                                    \
5622          "lw $4, 32(%1) \n\t"                                     \
5623          "sw $4, 28($29) \n\t"                                    \
5624          "lw $4, 4(%1) \n\t"                                      \
5625          "lw $5, 8(%1) \n\t"                                      \
5626          "lw $6, 12(%1) \n\t"                                     \
5627          "lw $7, 16(%1) \n\t"                                     \
5628          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5629          VALGRIND_CALL_NOREDIR_T9                                 \
5630          "addu $29, $29, 40 \n\t"                                 \
5631          "lw $28, 0($29) \n\t"                                    \
5632          "lw $31, 4($29) \n\t"                                    \
5633          "addu $29, $29, 8 \n\t"                                  \
5634          "move %0, $2\n"                                          \
5635          : /*out*/   "=r" (_res)                                  \
5636          : /*in*/    "0" (&_argvec[0])                            \
5637          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5638       );                                                          \
5639       lval = (__typeof__(lval)) _res;                             \
5640    } while (0)
5641 
5642 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5643                                  arg7,arg8,arg9)                  \
5644    do {                                                           \
5645       volatile OrigFn        _orig = (orig);                      \
5646       volatile unsigned long _argvec[10];                         \
5647       volatile unsigned long _res;                                \
5648       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5649       _argvec[1] = (unsigned long)(arg1);                         \
5650       _argvec[2] = (unsigned long)(arg2);                         \
5651       _argvec[3] = (unsigned long)(arg3);                         \
5652       _argvec[4] = (unsigned long)(arg4);                         \
5653       _argvec[5] = (unsigned long)(arg5);                         \
5654       _argvec[6] = (unsigned long)(arg6);                         \
5655       _argvec[7] = (unsigned long)(arg7);                         \
5656       _argvec[8] = (unsigned long)(arg8);                         \
5657       _argvec[9] = (unsigned long)(arg9);                         \
5658       __asm__ volatile(                                           \
5659          "subu $29, $29, 8 \n\t"                                  \
5660          "sw $28, 0($29) \n\t"                                    \
5661          "sw $31, 4($29) \n\t"                                    \
5662          "lw $4, 20(%1) \n\t"                                     \
5663          "subu $29, $29, 40\n\t"                                  \
5664          "sw $4, 16($29) \n\t"                                    \
5665          "lw $4, 24(%1) \n\t"                                     \
5666          "sw $4, 20($29) \n\t"                                    \
5667          "lw $4, 28(%1) \n\t"                                     \
5668          "sw $4, 24($29) \n\t"                                    \
5669          "lw $4, 32(%1) \n\t"                                     \
5670          "sw $4, 28($29) \n\t"                                    \
5671          "lw $4, 36(%1) \n\t"                                     \
5672          "sw $4, 32($29) \n\t"                                    \
5673          "lw $4, 4(%1) \n\t"                                      \
5674          "lw $5, 8(%1) \n\t"                                      \
5675          "lw $6, 12(%1) \n\t"                                     \
5676          "lw $7, 16(%1) \n\t"                                     \
5677          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5678          VALGRIND_CALL_NOREDIR_T9                                 \
5679          "addu $29, $29, 40 \n\t"                                 \
5680          "lw $28, 0($29) \n\t"                                    \
5681          "lw $31, 4($29) \n\t"                                    \
5682          "addu $29, $29, 8 \n\t"                                  \
5683          "move %0, $2\n"                                          \
5684          : /*out*/   "=r" (_res)                                  \
5685          : /*in*/    "0" (&_argvec[0])                            \
5686          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5687       );                                                          \
5688       lval = (__typeof__(lval)) _res;                             \
5689    } while (0)
5690 
5691 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
5692                                   arg7,arg8,arg9,arg10)           \
5693    do {                                                           \
5694       volatile OrigFn        _orig = (orig);                      \
5695       volatile unsigned long _argvec[11];                         \
5696       volatile unsigned long _res;                                \
5697       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5698       _argvec[1] = (unsigned long)(arg1);                         \
5699       _argvec[2] = (unsigned long)(arg2);                         \
5700       _argvec[3] = (unsigned long)(arg3);                         \
5701       _argvec[4] = (unsigned long)(arg4);                         \
5702       _argvec[5] = (unsigned long)(arg5);                         \
5703       _argvec[6] = (unsigned long)(arg6);                         \
5704       _argvec[7] = (unsigned long)(arg7);                         \
5705       _argvec[8] = (unsigned long)(arg8);                         \
5706       _argvec[9] = (unsigned long)(arg9);                         \
5707       _argvec[10] = (unsigned long)(arg10);                       \
5708       __asm__ volatile(                                           \
5709          "subu $29, $29, 8 \n\t"                                  \
5710          "sw $28, 0($29) \n\t"                                    \
5711          "sw $31, 4($29) \n\t"                                    \
5712          "lw $4, 20(%1) \n\t"                                     \
5713          "subu $29, $29, 48\n\t"                                  \
5714          "sw $4, 16($29) \n\t"                                    \
5715          "lw $4, 24(%1) \n\t"                                     \
5716          "sw $4, 20($29) \n\t"                                    \
5717          "lw $4, 28(%1) \n\t"                                     \
5718          "sw $4, 24($29) \n\t"                                    \
5719          "lw $4, 32(%1) \n\t"                                     \
5720          "sw $4, 28($29) \n\t"                                    \
5721          "lw $4, 36(%1) \n\t"                                     \
5722          "sw $4, 32($29) \n\t"                                    \
5723          "lw $4, 40(%1) \n\t"                                     \
5724          "sw $4, 36($29) \n\t"                                    \
5725          "lw $4, 4(%1) \n\t"                                      \
5726          "lw $5, 8(%1) \n\t"                                      \
5727          "lw $6, 12(%1) \n\t"                                     \
5728          "lw $7, 16(%1) \n\t"                                     \
5729          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5730          VALGRIND_CALL_NOREDIR_T9                                 \
5731          "addu $29, $29, 48 \n\t"                                 \
5732          "lw $28, 0($29) \n\t"                                    \
5733          "lw $31, 4($29) \n\t"                                    \
5734          "addu $29, $29, 8 \n\t"                                  \
5735          "move %0, $2\n"                                          \
5736          : /*out*/   "=r" (_res)                                  \
5737          : /*in*/    "0" (&_argvec[0])                            \
5738          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5739       );                                                          \
5740       lval = (__typeof__(lval)) _res;                             \
5741    } while (0)
5742 
5743 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
5744                                   arg6,arg7,arg8,arg9,arg10,      \
5745                                   arg11)                          \
5746    do {                                                           \
5747       volatile OrigFn        _orig = (orig);                      \
5748       volatile unsigned long _argvec[12];                         \
5749       volatile unsigned long _res;                                \
5750       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5751       _argvec[1] = (unsigned long)(arg1);                         \
5752       _argvec[2] = (unsigned long)(arg2);                         \
5753       _argvec[3] = (unsigned long)(arg3);                         \
5754       _argvec[4] = (unsigned long)(arg4);                         \
5755       _argvec[5] = (unsigned long)(arg5);                         \
5756       _argvec[6] = (unsigned long)(arg6);                         \
5757       _argvec[7] = (unsigned long)(arg7);                         \
5758       _argvec[8] = (unsigned long)(arg8);                         \
5759       _argvec[9] = (unsigned long)(arg9);                         \
5760       _argvec[10] = (unsigned long)(arg10);                       \
5761       _argvec[11] = (unsigned long)(arg11);                       \
5762       __asm__ volatile(                                           \
5763          "subu $29, $29, 8 \n\t"                                  \
5764          "sw $28, 0($29) \n\t"                                    \
5765          "sw $31, 4($29) \n\t"                                    \
5766          "lw $4, 20(%1) \n\t"                                     \
5767          "subu $29, $29, 48\n\t"                                  \
5768          "sw $4, 16($29) \n\t"                                    \
5769          "lw $4, 24(%1) \n\t"                                     \
5770          "sw $4, 20($29) \n\t"                                    \
5771          "lw $4, 28(%1) \n\t"                                     \
5772          "sw $4, 24($29) \n\t"                                    \
5773          "lw $4, 32(%1) \n\t"                                     \
5774          "sw $4, 28($29) \n\t"                                    \
5775          "lw $4, 36(%1) \n\t"                                     \
5776          "sw $4, 32($29) \n\t"                                    \
5777          "lw $4, 40(%1) \n\t"                                     \
5778          "sw $4, 36($29) \n\t"                                    \
5779          "lw $4, 44(%1) \n\t"                                     \
5780          "sw $4, 40($29) \n\t"                                    \
5781          "lw $4, 4(%1) \n\t"                                      \
5782          "lw $5, 8(%1) \n\t"                                      \
5783          "lw $6, 12(%1) \n\t"                                     \
5784          "lw $7, 16(%1) \n\t"                                     \
5785          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5786          VALGRIND_CALL_NOREDIR_T9                                 \
5787          "addu $29, $29, 48 \n\t"                                 \
5788          "lw $28, 0($29) \n\t"                                    \
5789          "lw $31, 4($29) \n\t"                                    \
5790          "addu $29, $29, 8 \n\t"                                  \
5791          "move %0, $2\n"                                          \
5792          : /*out*/   "=r" (_res)                                  \
5793          : /*in*/    "0" (&_argvec[0])                            \
5794          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5795       );                                                          \
5796       lval = (__typeof__(lval)) _res;                             \
5797    } while (0)
5798 
5799 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
5800                                   arg6,arg7,arg8,arg9,arg10,      \
5801                                   arg11,arg12)                    \
5802    do {                                                           \
5803       volatile OrigFn        _orig = (orig);                      \
5804       volatile unsigned long _argvec[13];                         \
5805       volatile unsigned long _res;                                \
5806       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5807       _argvec[1] = (unsigned long)(arg1);                         \
5808       _argvec[2] = (unsigned long)(arg2);                         \
5809       _argvec[3] = (unsigned long)(arg3);                         \
5810       _argvec[4] = (unsigned long)(arg4);                         \
5811       _argvec[5] = (unsigned long)(arg5);                         \
5812       _argvec[6] = (unsigned long)(arg6);                         \
5813       _argvec[7] = (unsigned long)(arg7);                         \
5814       _argvec[8] = (unsigned long)(arg8);                         \
5815       _argvec[9] = (unsigned long)(arg9);                         \
5816       _argvec[10] = (unsigned long)(arg10);                       \
5817       _argvec[11] = (unsigned long)(arg11);                       \
5818       _argvec[12] = (unsigned long)(arg12);                       \
5819       __asm__ volatile(                                           \
5820          "subu $29, $29, 8 \n\t"                                  \
5821          "sw $28, 0($29) \n\t"                                    \
5822          "sw $31, 4($29) \n\t"                                    \
5823          "lw $4, 20(%1) \n\t"                                     \
5824          "subu $29, $29, 56\n\t"                                  \
5825          "sw $4, 16($29) \n\t"                                    \
5826          "lw $4, 24(%1) \n\t"                                     \
5827          "sw $4, 20($29) \n\t"                                    \
5828          "lw $4, 28(%1) \n\t"                                     \
5829          "sw $4, 24($29) \n\t"                                    \
5830          "lw $4, 32(%1) \n\t"                                     \
5831          "sw $4, 28($29) \n\t"                                    \
5832          "lw $4, 36(%1) \n\t"                                     \
5833          "sw $4, 32($29) \n\t"                                    \
5834          "lw $4, 40(%1) \n\t"                                     \
5835          "sw $4, 36($29) \n\t"                                    \
5836          "lw $4, 44(%1) \n\t"                                     \
5837          "sw $4, 40($29) \n\t"                                    \
5838          "lw $4, 48(%1) \n\t"                                     \
5839          "sw $4, 44($29) \n\t"                                    \
5840          "lw $4, 4(%1) \n\t"                                      \
5841          "lw $5, 8(%1) \n\t"                                      \
5842          "lw $6, 12(%1) \n\t"                                     \
5843          "lw $7, 16(%1) \n\t"                                     \
5844          "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5845          VALGRIND_CALL_NOREDIR_T9                                 \
5846          "addu $29, $29, 56 \n\t"                                 \
5847          "lw $28, 0($29) \n\t"                                    \
5848          "lw $31, 4($29) \n\t"                                    \
5849          "addu $29, $29, 8 \n\t"                                  \
5850          "move %0, $2\n"                                          \
5851          : /*out*/   "=r" (_res)                                  \
5852          : /*in*/    "r" (&_argvec[0])                            \
5853          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5854       );                                                          \
5855       lval = (__typeof__(lval)) _res;                             \
5856    } while (0)
5857 
5858 #endif /* PLAT_mips32_linux */
5859 
5860 /* ------------------------- nanomips-linux -------------------- */
5861 
5862 #if defined(PLAT_nanomips_linux)
5863 
5864 /* These regs are trashed by the hidden call. */
5865 #define __CALLER_SAVED_REGS "$t4", "$t5", "$a0", "$a1", "$a2",     \
5866 "$a3", "$a4", "$a5", "$a6", "$a7", "$t0", "$t1", "$t2", "$t3",     \
5867 "$t8","$t9", "$at"
5868 
5869 /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned
5870    long) == 4. */
5871 
5872 #define CALL_FN_W_v(lval, orig)                                   \
5873    do {                                                           \
5874       volatile OrigFn        _orig = (orig);                      \
5875       volatile unsigned long _argvec[1];                          \
5876       volatile unsigned long _res;                                \
5877       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5878       __asm__ volatile(                                           \
5879          "lw $t9, 0(%1)\n\t"                                      \
5880          VALGRIND_CALL_NOREDIR_T9                                 \
5881          "move %0, $a0\n"                                         \
5882          : /*out*/   "=r" (_res)                                  \
5883          : /*in*/    "r" (&_argvec[0])                            \
5884          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5885       );                                                          \
5886       lval = (__typeof__(lval)) _res;                             \
5887    } while (0)
5888 
5889 #define CALL_FN_W_W(lval, orig, arg1)                             \
5890    do {                                                           \
5891       volatile OrigFn        _orig = (orig);                      \
5892       volatile unsigned long _argvec[2];                          \
5893       volatile unsigned long _res;                                \
5894       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5895       _argvec[1] = (unsigned long)(arg1);                         \
5896       __asm__ volatile(                                           \
5897          "lw $t9, 0(%1)\n\t"                                      \
5898          "lw $a0, 4(%1)\n\t"                                      \
5899          VALGRIND_CALL_NOREDIR_T9                                 \
5900          "move %0, $a0\n"                                         \
5901          : /*out*/   "=r" (_res)                                  \
5902          : /*in*/    "r" (&_argvec[0])                            \
5903          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5904       );                                                          \
5905       lval = (__typeof__(lval)) _res;                             \
5906    } while (0)
5907 
5908 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
5909    do {                                                           \
5910       volatile OrigFn        _orig = (orig);                      \
5911       volatile unsigned long _argvec[3];                          \
5912       volatile unsigned long _res;                                \
5913       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5914       _argvec[1] = (unsigned long)(arg1);                         \
5915       _argvec[2] = (unsigned long)(arg2);                         \
5916       __asm__ volatile(                                           \
5917          "lw $t9, 0(%1)\n\t"                                      \
5918          "lw $a0, 4(%1)\n\t"                                      \
5919          "lw $a1, 8(%1)\n\t"                                      \
5920          VALGRIND_CALL_NOREDIR_T9                                 \
5921          "move %0, $a0\n"                                         \
5922          : /*out*/   "=r" (_res)                                  \
5923          : /*in*/    "r" (&_argvec[0])                            \
5924          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5925       );                                                          \
5926       lval = (__typeof__(lval)) _res;                             \
5927    } while (0)
5928 
5929 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
5930    do {                                                           \
5931       volatile OrigFn        _orig = (orig);                      \
5932       volatile unsigned long _argvec[4];                          \
5933       volatile unsigned long _res;                                \
5934       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5935       _argvec[1] = (unsigned long)(arg1);                         \
5936       _argvec[2] = (unsigned long)(arg2);                         \
5937       _argvec[3] = (unsigned long)(arg3);                         \
5938       __asm__ volatile(                                           \
5939          "lw $t9, 0(%1)\n\t"                                      \
5940          "lw $a0, 4(%1)\n\t"                                      \
5941          "lw $a1, 8(%1)\n\t"                                      \
5942          "lw $a2,12(%1)\n\t"                                      \
5943          VALGRIND_CALL_NOREDIR_T9                                 \
5944          "move %0, $a0\n"                                         \
5945          : /*out*/   "=r" (_res)                                  \
5946          : /*in*/    "r" (&_argvec[0])                            \
5947          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5948       );                                                          \
5949       lval = (__typeof__(lval)) _res;                             \
5950    } while (0)
5951 
5952 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
5953    do {                                                           \
5954       volatile OrigFn        _orig = (orig);                      \
5955       volatile unsigned long _argvec[5];                          \
5956       volatile unsigned long _res;                                \
5957       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5958       _argvec[1] = (unsigned long)(arg1);                         \
5959       _argvec[2] = (unsigned long)(arg2);                         \
5960       _argvec[3] = (unsigned long)(arg3);                         \
5961       _argvec[4] = (unsigned long)(arg4);                         \
5962       __asm__ volatile(                                           \
5963          "lw $t9, 0(%1)\n\t"                                      \
5964          "lw $a0, 4(%1)\n\t"                                      \
5965          "lw $a1, 8(%1)\n\t"                                      \
5966          "lw $a2,12(%1)\n\t"                                      \
5967          "lw $a3,16(%1)\n\t"                                      \
5968          VALGRIND_CALL_NOREDIR_T9                                 \
5969          "move %0, $a0\n"                                         \
5970          : /*out*/   "=r" (_res)                                  \
5971          : /*in*/    "r" (&_argvec[0])                            \
5972          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5973       );                                                          \
5974       lval = (__typeof__(lval)) _res;                             \
5975    } while (0)
5976 
5977 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
5978    do {                                                           \
5979       volatile OrigFn        _orig = (orig);                      \
5980       volatile unsigned long _argvec[6];                          \
5981       volatile unsigned long _res;                                \
5982       _argvec[0] = (unsigned long)_orig.nraddr;                   \
5983       _argvec[1] = (unsigned long)(arg1);                         \
5984       _argvec[2] = (unsigned long)(arg2);                         \
5985       _argvec[3] = (unsigned long)(arg3);                         \
5986       _argvec[4] = (unsigned long)(arg4);                         \
5987       _argvec[5] = (unsigned long)(arg5);                         \
5988       __asm__ volatile(                                           \
5989          "lw $t9, 0(%1)\n\t"                                      \
5990          "lw $a0, 4(%1)\n\t"                                      \
5991          "lw $a1, 8(%1)\n\t"                                      \
5992          "lw $a2,12(%1)\n\t"                                      \
5993          "lw $a3,16(%1)\n\t"                                      \
5994          "lw $a4,20(%1)\n\t"                                      \
5995          VALGRIND_CALL_NOREDIR_T9                                 \
5996          "move %0, $a0\n"                                         \
5997          : /*out*/   "=r" (_res)                                  \
5998          : /*in*/    "r" (&_argvec[0])                            \
5999          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6000       );                                                          \
6001       lval = (__typeof__(lval)) _res;                             \
6002    } while (0)
6003 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
6004    do {                                                           \
6005       volatile OrigFn        _orig = (orig);                      \
6006       volatile unsigned long _argvec[7];                          \
6007       volatile unsigned long _res;                                \
6008       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6009       _argvec[1] = (unsigned long)(arg1);                         \
6010       _argvec[2] = (unsigned long)(arg2);                         \
6011       _argvec[3] = (unsigned long)(arg3);                         \
6012       _argvec[4] = (unsigned long)(arg4);                         \
6013       _argvec[5] = (unsigned long)(arg5);                         \
6014       _argvec[6] = (unsigned long)(arg6);                         \
6015       __asm__ volatile(                                           \
6016          "lw $t9, 0(%1)\n\t"                                      \
6017          "lw $a0, 4(%1)\n\t"                                      \
6018          "lw $a1, 8(%1)\n\t"                                      \
6019          "lw $a2,12(%1)\n\t"                                      \
6020          "lw $a3,16(%1)\n\t"                                      \
6021          "lw $a4,20(%1)\n\t"                                      \
6022          "lw $a5,24(%1)\n\t"                                      \
6023          VALGRIND_CALL_NOREDIR_T9                                 \
6024          "move %0, $a0\n"                                         \
6025          : /*out*/   "=r" (_res)                                  \
6026          : /*in*/    "r" (&_argvec[0])                            \
6027          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6028       );                                                          \
6029       lval = (__typeof__(lval)) _res;                             \
6030    } while (0)
6031 
6032 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
6033                                  arg7)                            \
6034    do {                                                           \
6035       volatile OrigFn        _orig = (orig);                      \
6036       volatile unsigned long _argvec[8];                          \
6037       volatile unsigned long _res;                                \
6038       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6039       _argvec[1] = (unsigned long)(arg1);                         \
6040       _argvec[2] = (unsigned long)(arg2);                         \
6041       _argvec[3] = (unsigned long)(arg3);                         \
6042       _argvec[4] = (unsigned long)(arg4);                         \
6043       _argvec[5] = (unsigned long)(arg5);                         \
6044       _argvec[6] = (unsigned long)(arg6);                         \
6045       _argvec[7] = (unsigned long)(arg7);                         \
6046       __asm__ volatile(                                           \
6047          "lw $t9, 0(%1)\n\t"                                      \
6048          "lw $a0, 4(%1)\n\t"                                      \
6049          "lw $a1, 8(%1)\n\t"                                      \
6050          "lw $a2,12(%1)\n\t"                                      \
6051          "lw $a3,16(%1)\n\t"                                      \
6052          "lw $a4,20(%1)\n\t"                                      \
6053          "lw $a5,24(%1)\n\t"                                      \
6054          "lw $a6,28(%1)\n\t"                                      \
6055          VALGRIND_CALL_NOREDIR_T9                                 \
6056          "move %0, $a0\n"                                         \
6057          : /*out*/   "=r" (_res)                                  \
6058          : /*in*/    "r" (&_argvec[0])                            \
6059          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6060       );                                                          \
6061       lval = (__typeof__(lval)) _res;                             \
6062    } while (0)
6063 
6064 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
6065                                  arg7,arg8)                       \
6066    do {                                                           \
6067       volatile OrigFn        _orig = (orig);                      \
6068       volatile unsigned long _argvec[9];                          \
6069       volatile unsigned long _res;                                \
6070       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6071       _argvec[1] = (unsigned long)(arg1);                         \
6072       _argvec[2] = (unsigned long)(arg2);                         \
6073       _argvec[3] = (unsigned long)(arg3);                         \
6074       _argvec[4] = (unsigned long)(arg4);                         \
6075       _argvec[5] = (unsigned long)(arg5);                         \
6076       _argvec[6] = (unsigned long)(arg6);                         \
6077       _argvec[7] = (unsigned long)(arg7);                         \
6078       _argvec[8] = (unsigned long)(arg8);                         \
6079       __asm__ volatile(                                           \
6080          "lw $t9, 0(%1)\n\t"                                      \
6081          "lw $a0, 4(%1)\n\t"                                      \
6082          "lw $a1, 8(%1)\n\t"                                      \
6083          "lw $a2,12(%1)\n\t"                                      \
6084          "lw $a3,16(%1)\n\t"                                      \
6085          "lw $a4,20(%1)\n\t"                                      \
6086          "lw $a5,24(%1)\n\t"                                      \
6087          "lw $a6,28(%1)\n\t"                                      \
6088          "lw $a7,32(%1)\n\t"                                      \
6089          VALGRIND_CALL_NOREDIR_T9                                 \
6090          "move %0, $a0\n"                                         \
6091          : /*out*/   "=r" (_res)                                  \
6092          : /*in*/    "r" (&_argvec[0])                            \
6093          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6094       );                                                          \
6095       lval = (__typeof__(lval)) _res;                             \
6096    } while (0)
6097 
6098 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
6099                                  arg7,arg8,arg9)                  \
6100    do {                                                           \
6101       volatile OrigFn        _orig = (orig);                      \
6102       volatile unsigned long _argvec[10];                         \
6103       volatile unsigned long _res;                                \
6104       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6105       _argvec[1] = (unsigned long)(arg1);                         \
6106       _argvec[2] = (unsigned long)(arg2);                         \
6107       _argvec[3] = (unsigned long)(arg3);                         \
6108       _argvec[4] = (unsigned long)(arg4);                         \
6109       _argvec[5] = (unsigned long)(arg5);                         \
6110       _argvec[6] = (unsigned long)(arg6);                         \
6111       _argvec[7] = (unsigned long)(arg7);                         \
6112       _argvec[8] = (unsigned long)(arg8);                         \
6113       _argvec[9] = (unsigned long)(arg9);                         \
6114       __asm__ volatile(                                           \
6115          "addiu $sp, $sp, -16  \n\t"                              \
6116          "lw $t9,36(%1)        \n\t"                              \
6117          "sw $t9, 0($sp)       \n\t"                              \
6118          "lw $t9, 0(%1)        \n\t"                              \
6119          "lw $a0, 4(%1)        \n\t"                              \
6120          "lw $a1, 8(%1)        \n\t"                              \
6121          "lw $a2,12(%1)        \n\t"                              \
6122          "lw $a3,16(%1)        \n\t"                              \
6123          "lw $a4,20(%1)        \n\t"                              \
6124          "lw $a5,24(%1)        \n\t"                              \
6125          "lw $a6,28(%1)        \n\t"                              \
6126          "lw $a7,32(%1)        \n\t"                              \
6127          VALGRIND_CALL_NOREDIR_T9                                 \
6128          "move %0, $a0         \n\t"                              \
6129          "addiu $sp, $sp, 16   \n\t"                              \
6130          : /*out*/   "=r" (_res)                                  \
6131          : /*in*/    "r" (&_argvec[0])                            \
6132          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6133       );                                                          \
6134       lval = (__typeof__(lval)) _res;                             \
6135    } while (0)
6136 
6137 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
6138                                   arg7,arg8,arg9,arg10)           \
6139    do {                                                           \
6140       volatile OrigFn        _orig = (orig);                      \
6141       volatile unsigned long _argvec[11];                         \
6142       volatile unsigned long _res;                                \
6143       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6144       _argvec[1] = (unsigned long)(arg1);                         \
6145       _argvec[2] = (unsigned long)(arg2);                         \
6146       _argvec[3] = (unsigned long)(arg3);                         \
6147       _argvec[4] = (unsigned long)(arg4);                         \
6148       _argvec[5] = (unsigned long)(arg5);                         \
6149       _argvec[6] = (unsigned long)(arg6);                         \
6150       _argvec[7] = (unsigned long)(arg7);                         \
6151       _argvec[8] = (unsigned long)(arg8);                         \
6152       _argvec[9] = (unsigned long)(arg9);                         \
6153       _argvec[10] = (unsigned long)(arg10);                       \
6154       __asm__ volatile(                                           \
6155          "addiu $sp, $sp, -16  \n\t"                              \
6156          "lw $t9,36(%1)        \n\t"                              \
6157          "sw $t9, 0($sp)       \n\t"                              \
6158          "lw $t9,40(%1)        \n\t"                              \
6159          "sw $t9, 4($sp)       \n\t"                              \
6160          "lw $t9, 0(%1)        \n\t"                              \
6161          "lw $a0, 4(%1)        \n\t"                              \
6162          "lw $a1, 8(%1)        \n\t"                              \
6163          "lw $a2,12(%1)        \n\t"                              \
6164          "lw $a3,16(%1)        \n\t"                              \
6165          "lw $a4,20(%1)        \n\t"                              \
6166          "lw $a5,24(%1)        \n\t"                              \
6167          "lw $a6,28(%1)        \n\t"                              \
6168          "lw $a7,32(%1)        \n\t"                              \
6169          VALGRIND_CALL_NOREDIR_T9                                 \
6170          "move %0, $a0         \n\t"                              \
6171          "addiu $sp, $sp, 16   \n\t"                              \
6172          : /*out*/   "=r" (_res)                                  \
6173          : /*in*/    "r" (&_argvec[0])                            \
6174          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6175       );                                                          \
6176       lval = (__typeof__(lval)) _res;                             \
6177    } while (0)
6178 
6179 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
6180                                   arg6,arg7,arg8,arg9,arg10,      \
6181                                   arg11)                          \
6182    do {                                                           \
6183       volatile OrigFn        _orig = (orig);                      \
6184       volatile unsigned long _argvec[12];                         \
6185       volatile unsigned long _res;                                \
6186       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6187       _argvec[1] = (unsigned long)(arg1);                         \
6188       _argvec[2] = (unsigned long)(arg2);                         \
6189       _argvec[3] = (unsigned long)(arg3);                         \
6190       _argvec[4] = (unsigned long)(arg4);                         \
6191       _argvec[5] = (unsigned long)(arg5);                         \
6192       _argvec[6] = (unsigned long)(arg6);                         \
6193       _argvec[7] = (unsigned long)(arg7);                         \
6194       _argvec[8] = (unsigned long)(arg8);                         \
6195       _argvec[9] = (unsigned long)(arg9);                         \
6196       _argvec[10] = (unsigned long)(arg10);                       \
6197       _argvec[11] = (unsigned long)(arg11);                       \
6198       __asm__ volatile(                                           \
6199          "addiu $sp, $sp, -16  \n\t"                              \
6200          "lw $t9,36(%1)        \n\t"                              \
6201          "sw $t9, 0($sp)       \n\t"                              \
6202          "lw $t9,40(%1)        \n\t"                              \
6203          "sw $t9, 4($sp)       \n\t"                              \
6204          "lw $t9,44(%1)        \n\t"                              \
6205          "sw $t9, 8($sp)       \n\t"                              \
6206          "lw $t9, 0(%1)        \n\t"                              \
6207          "lw $a0, 4(%1)        \n\t"                              \
6208          "lw $a1, 8(%1)        \n\t"                              \
6209          "lw $a2,12(%1)        \n\t"                              \
6210          "lw $a3,16(%1)        \n\t"                              \
6211          "lw $a4,20(%1)        \n\t"                              \
6212          "lw $a5,24(%1)        \n\t"                              \
6213          "lw $a6,28(%1)        \n\t"                              \
6214          "lw $a7,32(%1)        \n\t"                              \
6215          VALGRIND_CALL_NOREDIR_T9                                 \
6216          "move %0, $a0         \n\t"                              \
6217          "addiu $sp, $sp, 16   \n\t"                              \
6218          : /*out*/   "=r" (_res)                                  \
6219          : /*in*/    "r" (&_argvec[0])                            \
6220          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6221       );                                                          \
6222       lval = (__typeof__(lval)) _res;                             \
6223    } while (0)
6224 
6225 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
6226                                   arg6,arg7,arg8,arg9,arg10,      \
6227                                   arg11,arg12)                    \
6228    do {                                                           \
6229       volatile OrigFn        _orig = (orig);                      \
6230       volatile unsigned long _argvec[13];                         \
6231       volatile unsigned long _res;                                \
6232       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6233       _argvec[1] = (unsigned long)(arg1);                         \
6234       _argvec[2] = (unsigned long)(arg2);                         \
6235       _argvec[3] = (unsigned long)(arg3);                         \
6236       _argvec[4] = (unsigned long)(arg4);                         \
6237       _argvec[5] = (unsigned long)(arg5);                         \
6238       _argvec[6] = (unsigned long)(arg6);                         \
6239       _argvec[7] = (unsigned long)(arg7);                         \
6240       _argvec[8] = (unsigned long)(arg8);                         \
6241       _argvec[9] = (unsigned long)(arg9);                         \
6242       _argvec[10] = (unsigned long)(arg10);                       \
6243       _argvec[11] = (unsigned long)(arg11);                       \
6244       _argvec[12] = (unsigned long)(arg12);                       \
6245       __asm__ volatile(                                           \
6246          "addiu $sp, $sp, -16  \n\t"                              \
6247          "lw $t9,36(%1)        \n\t"                              \
6248          "sw $t9, 0($sp)       \n\t"                              \
6249          "lw $t9,40(%1)        \n\t"                              \
6250          "sw $t9, 4($sp)       \n\t"                              \
6251          "lw $t9,44(%1)        \n\t"                              \
6252          "sw $t9, 8($sp)       \n\t"                              \
6253          "lw $t9,48(%1)        \n\t"                              \
6254          "sw $t9,12($sp)       \n\t"                              \
6255          "lw $t9, 0(%1)        \n\t"                              \
6256          "lw $a0, 4(%1)        \n\t"                              \
6257          "lw $a1, 8(%1)        \n\t"                              \
6258          "lw $a2,12(%1)        \n\t"                              \
6259          "lw $a3,16(%1)        \n\t"                              \
6260          "lw $a4,20(%1)        \n\t"                              \
6261          "lw $a5,24(%1)        \n\t"                              \
6262          "lw $a6,28(%1)        \n\t"                              \
6263          "lw $a7,32(%1)        \n\t"                              \
6264          VALGRIND_CALL_NOREDIR_T9                                 \
6265          "move %0, $a0         \n\t"                              \
6266          "addiu $sp, $sp, 16   \n\t"                              \
6267          : /*out*/   "=r" (_res)                                  \
6268          : /*in*/    "r" (&_argvec[0])                            \
6269          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6270       );                                                          \
6271       lval = (__typeof__(lval)) _res;                             \
6272    } while (0)
6273 
6274 #endif /* PLAT_nanomips_linux */
6275 
6276 /* ------------------------- mips64-linux ------------------------- */
6277 
6278 #if defined(PLAT_mips64_linux)
6279 
6280 /* These regs are trashed by the hidden call. */
6281 #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6",       \
6282 "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \
6283 "$25", "$31"
6284 
6285 /* These CALL_FN_ macros assume that on mips64-linux,
6286    sizeof(long long) == 8. */
6287 
6288 #define MIPS64_LONG2REG_CAST(x) ((long long)(long)x)
6289 
6290 #define CALL_FN_W_v(lval, orig)                                   \
6291    do {                                                           \
6292       volatile OrigFn        _orig = (orig);                      \
6293       volatile unsigned long long _argvec[1];                     \
6294       volatile unsigned long long _res;                           \
6295       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6296       __asm__ volatile(                                           \
6297          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6298          VALGRIND_CALL_NOREDIR_T9                                 \
6299          "move %0, $2\n"                                          \
6300          : /*out*/   "=r" (_res)                                  \
6301          : /*in*/    "0" (&_argvec[0])                            \
6302          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6303       );                                                          \
6304       lval = (__typeof__(lval)) (long)_res;                       \
6305    } while (0)
6306 
6307 #define CALL_FN_W_W(lval, orig, arg1)                             \
6308    do {                                                           \
6309       volatile OrigFn        _orig = (orig);                      \
6310       volatile unsigned long long _argvec[2];                     \
6311       volatile unsigned long long  _res;                          \
6312       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6313       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6314       __asm__ volatile(                                           \
6315          "ld $4, 8(%1)\n\t"   /* arg1*/                           \
6316          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6317          VALGRIND_CALL_NOREDIR_T9                                 \
6318          "move %0, $2\n"                                          \
6319          : /*out*/   "=r" (_res)                                  \
6320          : /*in*/    "r" (&_argvec[0])                            \
6321          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6322       );                                                          \
6323       lval = (__typeof__(lval)) (long)_res;                       \
6324    } while (0)
6325 
6326 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
6327    do {                                                           \
6328       volatile OrigFn        _orig = (orig);                      \
6329       volatile unsigned long long _argvec[3];                     \
6330       volatile unsigned long long _res;                           \
6331       _argvec[0] = _orig.nraddr;                                  \
6332       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6333       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6334       __asm__ volatile(                                           \
6335          "ld $4, 8(%1)\n\t"                                       \
6336          "ld $5, 16(%1)\n\t"                                      \
6337          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6338          VALGRIND_CALL_NOREDIR_T9                                 \
6339          "move %0, $2\n"                                          \
6340          : /*out*/   "=r" (_res)                                  \
6341          : /*in*/    "r" (&_argvec[0])                            \
6342          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6343       );                                                          \
6344       lval = (__typeof__(lval)) (long)_res;                       \
6345    } while (0)
6346 
6347 
6348 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
6349    do {                                                           \
6350       volatile OrigFn        _orig = (orig);                      \
6351       volatile unsigned long long _argvec[4];                     \
6352       volatile unsigned long long _res;                           \
6353       _argvec[0] = _orig.nraddr;                                  \
6354       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6355       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6356       _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6357       __asm__ volatile(                                           \
6358          "ld $4, 8(%1)\n\t"                                       \
6359          "ld $5, 16(%1)\n\t"                                      \
6360          "ld $6, 24(%1)\n\t"                                      \
6361          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6362          VALGRIND_CALL_NOREDIR_T9                                 \
6363          "move %0, $2\n"                                          \
6364          : /*out*/   "=r" (_res)                                  \
6365          : /*in*/    "r" (&_argvec[0])                            \
6366          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6367       );                                                          \
6368       lval = (__typeof__(lval)) (long)_res;                       \
6369    } while (0)
6370 
6371 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
6372    do {                                                           \
6373       volatile OrigFn        _orig = (orig);                      \
6374       volatile unsigned long long _argvec[5];                     \
6375       volatile unsigned long long _res;                           \
6376       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6377       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6378       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6379       _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6380       _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6381       __asm__ volatile(                                           \
6382          "ld $4, 8(%1)\n\t"                                       \
6383          "ld $5, 16(%1)\n\t"                                      \
6384          "ld $6, 24(%1)\n\t"                                      \
6385          "ld $7, 32(%1)\n\t"                                      \
6386          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6387          VALGRIND_CALL_NOREDIR_T9                                 \
6388          "move %0, $2\n"                                          \
6389          : /*out*/   "=r" (_res)                                  \
6390          : /*in*/    "r" (&_argvec[0])                            \
6391          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6392       );                                                          \
6393       lval = (__typeof__(lval)) (long)_res;                       \
6394    } while (0)
6395 
6396 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
6397    do {                                                           \
6398       volatile OrigFn        _orig = (orig);                      \
6399       volatile unsigned long long _argvec[6];                     \
6400       volatile unsigned long long _res;                           \
6401       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6402       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6403       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6404       _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6405       _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6406       _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
6407       __asm__ volatile(                                           \
6408          "ld $4, 8(%1)\n\t"                                       \
6409          "ld $5, 16(%1)\n\t"                                      \
6410          "ld $6, 24(%1)\n\t"                                      \
6411          "ld $7, 32(%1)\n\t"                                      \
6412          "ld $8, 40(%1)\n\t"                                      \
6413          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6414          VALGRIND_CALL_NOREDIR_T9                                 \
6415          "move %0, $2\n"                                          \
6416          : /*out*/   "=r" (_res)                                  \
6417          : /*in*/    "r" (&_argvec[0])                            \
6418          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6419       );                                                          \
6420       lval = (__typeof__(lval)) (long)_res;                       \
6421    } while (0)
6422 
6423 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
6424    do {                                                           \
6425       volatile OrigFn        _orig = (orig);                      \
6426       volatile unsigned long long _argvec[7];                     \
6427       volatile unsigned long long _res;                           \
6428       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6429       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6430       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6431       _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6432       _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6433       _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
6434       _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
6435       __asm__ volatile(                                           \
6436          "ld $4, 8(%1)\n\t"                                       \
6437          "ld $5, 16(%1)\n\t"                                      \
6438          "ld $6, 24(%1)\n\t"                                      \
6439          "ld $7, 32(%1)\n\t"                                      \
6440          "ld $8, 40(%1)\n\t"                                      \
6441          "ld $9, 48(%1)\n\t"                                      \
6442          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6443          VALGRIND_CALL_NOREDIR_T9                                 \
6444          "move %0, $2\n"                                          \
6445          : /*out*/   "=r" (_res)                                  \
6446          : /*in*/    "r" (&_argvec[0])                            \
6447          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6448       );                                                          \
6449       lval = (__typeof__(lval)) (long)_res;                       \
6450    } while (0)
6451 
6452 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
6453                                  arg7)                            \
6454    do {                                                           \
6455       volatile OrigFn        _orig = (orig);                      \
6456       volatile unsigned long long _argvec[8];                     \
6457       volatile unsigned long long _res;                           \
6458       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6459       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6460       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6461       _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6462       _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6463       _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
6464       _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
6465       _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
6466       __asm__ volatile(                                           \
6467          "ld $4, 8(%1)\n\t"                                       \
6468          "ld $5, 16(%1)\n\t"                                      \
6469          "ld $6, 24(%1)\n\t"                                      \
6470          "ld $7, 32(%1)\n\t"                                      \
6471          "ld $8, 40(%1)\n\t"                                      \
6472          "ld $9, 48(%1)\n\t"                                      \
6473          "ld $10, 56(%1)\n\t"                                     \
6474          "ld $25, 0(%1) \n\t"  /* target->t9 */                   \
6475          VALGRIND_CALL_NOREDIR_T9                                 \
6476          "move %0, $2\n"                                          \
6477          : /*out*/   "=r" (_res)                                  \
6478          : /*in*/    "r" (&_argvec[0])                            \
6479          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6480       );                                                          \
6481       lval = (__typeof__(lval)) (long)_res;                       \
6482    } while (0)
6483 
6484 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
6485                                  arg7,arg8)                       \
6486    do {                                                           \
6487       volatile OrigFn        _orig = (orig);                      \
6488       volatile unsigned long long _argvec[9];                     \
6489       volatile unsigned long long _res;                           \
6490       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6491       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6492       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6493       _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6494       _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6495       _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
6496       _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
6497       _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
6498       _argvec[8] = MIPS64_LONG2REG_CAST(arg8);                    \
6499       __asm__ volatile(                                           \
6500          "ld $4, 8(%1)\n\t"                                       \
6501          "ld $5, 16(%1)\n\t"                                      \
6502          "ld $6, 24(%1)\n\t"                                      \
6503          "ld $7, 32(%1)\n\t"                                      \
6504          "ld $8, 40(%1)\n\t"                                      \
6505          "ld $9, 48(%1)\n\t"                                      \
6506          "ld $10, 56(%1)\n\t"                                     \
6507          "ld $11, 64(%1)\n\t"                                     \
6508          "ld $25, 0(%1) \n\t"  /* target->t9 */                   \
6509          VALGRIND_CALL_NOREDIR_T9                                 \
6510          "move %0, $2\n"                                          \
6511          : /*out*/   "=r" (_res)                                  \
6512          : /*in*/    "r" (&_argvec[0])                            \
6513          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6514       );                                                          \
6515       lval = (__typeof__(lval)) (long)_res;                       \
6516    } while (0)
6517 
6518 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
6519                                  arg7,arg8,arg9)                  \
6520    do {                                                           \
6521       volatile OrigFn        _orig = (orig);                      \
6522       volatile unsigned long long _argvec[10];                    \
6523       volatile unsigned long long _res;                           \
6524       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6525       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6526       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6527       _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6528       _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6529       _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
6530       _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
6531       _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
6532       _argvec[8] = MIPS64_LONG2REG_CAST(arg8);                    \
6533       _argvec[9] = MIPS64_LONG2REG_CAST(arg9);                    \
6534       __asm__ volatile(                                           \
6535          "dsubu $29, $29, 8\n\t"                                  \
6536          "ld $4, 72(%1)\n\t"                                      \
6537          "sd $4, 0($29)\n\t"                                      \
6538          "ld $4, 8(%1)\n\t"                                       \
6539          "ld $5, 16(%1)\n\t"                                      \
6540          "ld $6, 24(%1)\n\t"                                      \
6541          "ld $7, 32(%1)\n\t"                                      \
6542          "ld $8, 40(%1)\n\t"                                      \
6543          "ld $9, 48(%1)\n\t"                                      \
6544          "ld $10, 56(%1)\n\t"                                     \
6545          "ld $11, 64(%1)\n\t"                                     \
6546          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6547          VALGRIND_CALL_NOREDIR_T9                                 \
6548          "daddu $29, $29, 8\n\t"                                  \
6549          "move %0, $2\n"                                          \
6550          : /*out*/   "=r" (_res)                                  \
6551          : /*in*/    "r" (&_argvec[0])                            \
6552          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6553       );                                                          \
6554       lval = (__typeof__(lval)) (long)_res;                       \
6555    } while (0)
6556 
6557 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
6558                                   arg7,arg8,arg9,arg10)           \
6559    do {                                                           \
6560       volatile OrigFn        _orig = (orig);                      \
6561       volatile unsigned long long _argvec[11];                    \
6562       volatile unsigned long long _res;                           \
6563       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6564       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6565       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6566       _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6567       _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6568       _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
6569       _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
6570       _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
6571       _argvec[8] = MIPS64_LONG2REG_CAST(arg8);                    \
6572       _argvec[9] = MIPS64_LONG2REG_CAST(arg9);                    \
6573       _argvec[10] = MIPS64_LONG2REG_CAST(arg10);                  \
6574       __asm__ volatile(                                           \
6575          "dsubu $29, $29, 16\n\t"                                 \
6576          "ld $4, 72(%1)\n\t"                                      \
6577          "sd $4, 0($29)\n\t"                                      \
6578          "ld $4, 80(%1)\n\t"                                      \
6579          "sd $4, 8($29)\n\t"                                      \
6580          "ld $4, 8(%1)\n\t"                                       \
6581          "ld $5, 16(%1)\n\t"                                      \
6582          "ld $6, 24(%1)\n\t"                                      \
6583          "ld $7, 32(%1)\n\t"                                      \
6584          "ld $8, 40(%1)\n\t"                                      \
6585          "ld $9, 48(%1)\n\t"                                      \
6586          "ld $10, 56(%1)\n\t"                                     \
6587          "ld $11, 64(%1)\n\t"                                     \
6588          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6589          VALGRIND_CALL_NOREDIR_T9                                 \
6590          "daddu $29, $29, 16\n\t"                                 \
6591          "move %0, $2\n"                                          \
6592          : /*out*/   "=r" (_res)                                  \
6593          : /*in*/    "r" (&_argvec[0])                            \
6594          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6595       );                                                          \
6596       lval = (__typeof__(lval)) (long)_res;                       \
6597    } while (0)
6598 
6599 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
6600                                   arg6,arg7,arg8,arg9,arg10,      \
6601                                   arg11)                          \
6602    do {                                                           \
6603       volatile OrigFn        _orig = (orig);                      \
6604       volatile unsigned long long _argvec[12];                    \
6605       volatile unsigned long long _res;                           \
6606       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6607       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6608       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6609       _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6610       _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6611       _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
6612       _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
6613       _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
6614       _argvec[8] = MIPS64_LONG2REG_CAST(arg8);                    \
6615       _argvec[9] = MIPS64_LONG2REG_CAST(arg9);                    \
6616       _argvec[10] = MIPS64_LONG2REG_CAST(arg10);                  \
6617       _argvec[11] = MIPS64_LONG2REG_CAST(arg11);                  \
6618       __asm__ volatile(                                           \
6619          "dsubu $29, $29, 24\n\t"                                 \
6620          "ld $4, 72(%1)\n\t"                                      \
6621          "sd $4, 0($29)\n\t"                                      \
6622          "ld $4, 80(%1)\n\t"                                      \
6623          "sd $4, 8($29)\n\t"                                      \
6624          "ld $4, 88(%1)\n\t"                                      \
6625          "sd $4, 16($29)\n\t"                                     \
6626          "ld $4, 8(%1)\n\t"                                       \
6627          "ld $5, 16(%1)\n\t"                                      \
6628          "ld $6, 24(%1)\n\t"                                      \
6629          "ld $7, 32(%1)\n\t"                                      \
6630          "ld $8, 40(%1)\n\t"                                      \
6631          "ld $9, 48(%1)\n\t"                                      \
6632          "ld $10, 56(%1)\n\t"                                     \
6633          "ld $11, 64(%1)\n\t"                                     \
6634          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6635          VALGRIND_CALL_NOREDIR_T9                                 \
6636          "daddu $29, $29, 24\n\t"                                 \
6637          "move %0, $2\n"                                          \
6638          : /*out*/   "=r" (_res)                                  \
6639          : /*in*/    "r" (&_argvec[0])                            \
6640          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6641       );                                                          \
6642       lval = (__typeof__(lval)) (long)_res;                       \
6643    } while (0)
6644 
6645 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
6646                                   arg6,arg7,arg8,arg9,arg10,      \
6647                                   arg11,arg12)                    \
6648    do {                                                           \
6649       volatile OrigFn        _orig = (orig);                      \
6650       volatile unsigned long long _argvec[13];                    \
6651       volatile unsigned long long _res;                           \
6652       _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6653       _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6654       _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6655       _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6656       _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6657       _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
6658       _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
6659       _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
6660       _argvec[8] = MIPS64_LONG2REG_CAST(arg8);                    \
6661       _argvec[9] = MIPS64_LONG2REG_CAST(arg9);                    \
6662       _argvec[10] = MIPS64_LONG2REG_CAST(arg10);                  \
6663       _argvec[11] = MIPS64_LONG2REG_CAST(arg11);                  \
6664       _argvec[12] = MIPS64_LONG2REG_CAST(arg12);                  \
6665       __asm__ volatile(                                           \
6666          "dsubu $29, $29, 32\n\t"                                 \
6667          "ld $4, 72(%1)\n\t"                                      \
6668          "sd $4, 0($29)\n\t"                                      \
6669          "ld $4, 80(%1)\n\t"                                      \
6670          "sd $4, 8($29)\n\t"                                      \
6671          "ld $4, 88(%1)\n\t"                                      \
6672          "sd $4, 16($29)\n\t"                                     \
6673          "ld $4, 96(%1)\n\t"                                      \
6674          "sd $4, 24($29)\n\t"                                     \
6675          "ld $4, 8(%1)\n\t"                                       \
6676          "ld $5, 16(%1)\n\t"                                      \
6677          "ld $6, 24(%1)\n\t"                                      \
6678          "ld $7, 32(%1)\n\t"                                      \
6679          "ld $8, 40(%1)\n\t"                                      \
6680          "ld $9, 48(%1)\n\t"                                      \
6681          "ld $10, 56(%1)\n\t"                                     \
6682          "ld $11, 64(%1)\n\t"                                     \
6683          "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6684          VALGRIND_CALL_NOREDIR_T9                                 \
6685          "daddu $29, $29, 32\n\t"                                 \
6686          "move %0, $2\n"                                          \
6687          : /*out*/   "=r" (_res)                                  \
6688          : /*in*/    "r" (&_argvec[0])                            \
6689          : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6690       );                                                          \
6691       lval = (__typeof__(lval)) (long)_res;                       \
6692    } while (0)
6693 
6694 #endif /* PLAT_mips64_linux */
6695 
6696 /* ----------------------- riscv64-linux ----------------------- */
6697 
6698 #if defined(PLAT_riscv64_linux)
6699 
6700 /* These regs are trashed by the hidden call. */
6701 #define __CALLER_SAVED_REGS \
6702      "ra",                                                        \
6703      "t0", "t1", "t2", "t3", "t4", "t5", "t6",                    \
6704      "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",              \
6705      "ft0", "ft1", "ft2", "ft3", "ft4", "ft5", "ft6", "ft7",      \
6706      "ft8", "ft9", "ft10", "ft11",                                \
6707      "fa0", "fa1", "fa2", "fa3", "fa4", "fa5", "fa6", "fa7"
6708 
6709 /* s11 is callee-saved, so we can use it to save and restore sp around
6710    the hidden call. */
6711 #define VALGRIND_ALIGN_STACK               \
6712       "mv s11, sp\n\t"                     \
6713       "andi sp, sp, 0xfffffffffffffff0\n\t"
6714 #define VALGRIND_RESTORE_STACK             \
6715       "mv sp, s11\n\t"
6716 
6717 /* These CALL_FN_ macros assume that on riscv64-linux,
6718    sizeof(unsigned long) == 8. */
6719 
6720 #define CALL_FN_W_v(lval, orig)                                   \
6721    do {                                                           \
6722       volatile OrigFn        _orig = (orig);                      \
6723       volatile unsigned long _argvec[1];                          \
6724       volatile unsigned long _res;                                \
6725       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6726       __asm__ volatile(                                           \
6727          VALGRIND_ALIGN_STACK                                     \
6728          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
6729          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
6730          VALGRIND_RESTORE_STACK                                   \
6731          "mv %0, a0\n"                                            \
6732          : /*out*/   "=r" (_res)                                  \
6733          : /*in*/    "0" (&_argvec[0])                            \
6734          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
6735       );                                                          \
6736       lval = (__typeof__(lval)) _res;                             \
6737    } while (0)
6738 
6739 #define CALL_FN_W_W(lval, orig, arg1)                             \
6740    do {                                                           \
6741       volatile OrigFn        _orig = (orig);                      \
6742       volatile unsigned long _argvec[2];                          \
6743       volatile unsigned long _res;                                \
6744       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6745       _argvec[1] = (unsigned long)(arg1);                         \
6746       __asm__ volatile(                                           \
6747          VALGRIND_ALIGN_STACK                                     \
6748          "ld a0, 8(%1) \n\t"                                      \
6749          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
6750          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
6751          VALGRIND_RESTORE_STACK                                   \
6752          "mv %0, a0\n"                                            \
6753          : /*out*/   "=r" (_res)                                  \
6754          : /*in*/    "0" (&_argvec[0])                            \
6755          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
6756       );                                                          \
6757       lval = (__typeof__(lval)) _res;                             \
6758    } while (0)
6759 
6760 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
6761    do {                                                           \
6762       volatile OrigFn        _orig = (orig);                      \
6763       volatile unsigned long _argvec[3];                          \
6764       volatile unsigned long _res;                                \
6765       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6766       _argvec[1] = (unsigned long)(arg1);                         \
6767       _argvec[2] = (unsigned long)(arg2);                         \
6768       __asm__ volatile(                                           \
6769          VALGRIND_ALIGN_STACK                                     \
6770          "ld a0, 8(%1) \n\t"                                      \
6771          "ld a1, 16(%1) \n\t"                                     \
6772          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
6773          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
6774          VALGRIND_RESTORE_STACK                                   \
6775          "mv %0, a0\n"                                            \
6776          : /*out*/   "=r" (_res)                                  \
6777          : /*in*/    "0" (&_argvec[0])                            \
6778          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
6779       );                                                          \
6780       lval = (__typeof__(lval)) _res;                             \
6781    } while (0)
6782 
6783 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
6784    do {                                                           \
6785       volatile OrigFn        _orig = (orig);                      \
6786       volatile unsigned long _argvec[4];                          \
6787       volatile unsigned long _res;                                \
6788       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6789       _argvec[1] = (unsigned long)(arg1);                         \
6790       _argvec[2] = (unsigned long)(arg2);                         \
6791       _argvec[3] = (unsigned long)(arg3);                         \
6792       __asm__ volatile(                                           \
6793          VALGRIND_ALIGN_STACK                                     \
6794          "ld a0, 8(%1) \n\t"                                      \
6795          "ld a1, 16(%1) \n\t"                                     \
6796          "ld a2, 24(%1) \n\t"                                     \
6797          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
6798          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
6799          VALGRIND_RESTORE_STACK                                   \
6800          "mv %0, a0\n"                                            \
6801          : /*out*/   "=r" (_res)                                  \
6802          : /*in*/    "0" (&_argvec[0])                            \
6803          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
6804       );                                                          \
6805       lval = (__typeof__(lval)) _res;                             \
6806    } while (0)
6807 
6808 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
6809    do {                                                           \
6810       volatile OrigFn        _orig = (orig);                      \
6811       volatile unsigned long _argvec[5];                          \
6812       volatile unsigned long _res;                                \
6813       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6814       _argvec[1] = (unsigned long)(arg1);                         \
6815       _argvec[2] = (unsigned long)(arg2);                         \
6816       _argvec[3] = (unsigned long)(arg3);                         \
6817       _argvec[4] = (unsigned long)(arg4);                         \
6818       __asm__ volatile(                                           \
6819          VALGRIND_ALIGN_STACK                                     \
6820          "ld a0, 8(%1) \n\t"                                      \
6821          "ld a1, 16(%1) \n\t"                                     \
6822          "ld a2, 24(%1) \n\t"                                     \
6823          "ld a3, 32(%1) \n\t"                                     \
6824          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
6825          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
6826          VALGRIND_RESTORE_STACK                                   \
6827          "mv %0, a0"                                              \
6828          : /*out*/   "=r" (_res)                                  \
6829          : /*in*/    "0" (&_argvec[0])                            \
6830          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
6831       );                                                          \
6832       lval = (__typeof__(lval)) _res;                             \
6833    } while (0)
6834 
6835 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
6836    do {                                                           \
6837       volatile OrigFn        _orig = (orig);                      \
6838       volatile unsigned long _argvec[6];                          \
6839       volatile unsigned long _res;                                \
6840       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6841       _argvec[1] = (unsigned long)(arg1);                         \
6842       _argvec[2] = (unsigned long)(arg2);                         \
6843       _argvec[3] = (unsigned long)(arg3);                         \
6844       _argvec[4] = (unsigned long)(arg4);                         \
6845       _argvec[5] = (unsigned long)(arg5);                         \
6846       __asm__ volatile(                                           \
6847          VALGRIND_ALIGN_STACK                                     \
6848          "ld a0, 8(%1) \n\t"                                      \
6849          "ld a1, 16(%1) \n\t"                                     \
6850          "ld a2, 24(%1) \n\t"                                     \
6851          "ld a3, 32(%1) \n\t"                                     \
6852          "ld a4, 40(%1) \n\t"                                     \
6853          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
6854          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
6855          VALGRIND_RESTORE_STACK                                   \
6856          "mv %0, a0"                                              \
6857          : /*out*/   "=r" (_res)                                  \
6858          : /*in*/    "0" (&_argvec[0])                            \
6859          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
6860       );                                                          \
6861       lval = (__typeof__(lval)) _res;                             \
6862    } while (0)
6863 
6864 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
6865    do {                                                           \
6866       volatile OrigFn        _orig = (orig);                      \
6867       volatile unsigned long _argvec[7];                          \
6868       volatile unsigned long _res;                                \
6869       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6870       _argvec[1] = (unsigned long)(arg1);                         \
6871       _argvec[2] = (unsigned long)(arg2);                         \
6872       _argvec[3] = (unsigned long)(arg3);                         \
6873       _argvec[4] = (unsigned long)(arg4);                         \
6874       _argvec[5] = (unsigned long)(arg5);                         \
6875       _argvec[6] = (unsigned long)(arg6);                         \
6876       __asm__ volatile(                                           \
6877          VALGRIND_ALIGN_STACK                                     \
6878          "ld a0, 8(%1) \n\t"                                      \
6879          "ld a1, 16(%1) \n\t"                                     \
6880          "ld a2, 24(%1) \n\t"                                     \
6881          "ld a3, 32(%1) \n\t"                                     \
6882          "ld a4, 40(%1) \n\t"                                     \
6883          "ld a5, 48(%1) \n\t"                                     \
6884          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
6885          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
6886          VALGRIND_RESTORE_STACK                                   \
6887          "mv %0, a0"                                              \
6888          : /*out*/   "=r" (_res)                                  \
6889          : /*in*/    "0" (&_argvec[0])                            \
6890          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
6891       );                                                          \
6892       lval = (__typeof__(lval)) _res;                             \
6893    } while (0)
6894 
6895 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
6896                                  arg7)                            \
6897    do {                                                           \
6898       volatile OrigFn        _orig = (orig);                      \
6899       volatile unsigned long _argvec[8];                          \
6900       volatile unsigned long _res;                                \
6901       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6902       _argvec[1] = (unsigned long)(arg1);                         \
6903       _argvec[2] = (unsigned long)(arg2);                         \
6904       _argvec[3] = (unsigned long)(arg3);                         \
6905       _argvec[4] = (unsigned long)(arg4);                         \
6906       _argvec[5] = (unsigned long)(arg5);                         \
6907       _argvec[6] = (unsigned long)(arg6);                         \
6908       _argvec[7] = (unsigned long)(arg7);                         \
6909       __asm__ volatile(                                           \
6910          VALGRIND_ALIGN_STACK                                     \
6911          "ld a0, 8(%1) \n\t"                                      \
6912          "ld a1, 16(%1) \n\t"                                     \
6913          "ld a2, 24(%1) \n\t"                                     \
6914          "ld a3, 32(%1) \n\t"                                     \
6915          "ld a4, 40(%1) \n\t"                                     \
6916          "ld a5, 48(%1) \n\t"                                     \
6917          "ld a6, 56(%1) \n\t"                                     \
6918          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
6919          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
6920          VALGRIND_RESTORE_STACK                                   \
6921          "mv %0, a0"                                              \
6922          : /*out*/   "=r" (_res)                                  \
6923          : /*in*/    "0" (&_argvec[0])                            \
6924          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
6925       );                                                          \
6926       lval = (__typeof__(lval)) _res;                             \
6927    } while (0)
6928 
6929 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
6930                                  arg7,arg8)                       \
6931    do {                                                           \
6932       volatile OrigFn        _orig = (orig);                      \
6933       volatile unsigned long _argvec[9];                          \
6934       volatile unsigned long _res;                                \
6935       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6936       _argvec[1] = (unsigned long)(arg1);                         \
6937       _argvec[2] = (unsigned long)(arg2);                         \
6938       _argvec[3] = (unsigned long)(arg3);                         \
6939       _argvec[4] = (unsigned long)(arg4);                         \
6940       _argvec[5] = (unsigned long)(arg5);                         \
6941       _argvec[6] = (unsigned long)(arg6);                         \
6942       _argvec[7] = (unsigned long)(arg7);                         \
6943       _argvec[8] = (unsigned long)(arg8);                         \
6944       __asm__ volatile(                                           \
6945          VALGRIND_ALIGN_STACK                                     \
6946          "ld a0, 8(%1) \n\t"                                      \
6947          "ld a1, 16(%1) \n\t"                                     \
6948          "ld a2, 24(%1) \n\t"                                     \
6949          "ld a3, 32(%1) \n\t"                                     \
6950          "ld a4, 40(%1) \n\t"                                     \
6951          "ld a5, 48(%1) \n\t"                                     \
6952          "ld a6, 56(%1) \n\t"                                     \
6953          "ld a7, 64(%1) \n\t"                                     \
6954          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
6955          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
6956          VALGRIND_RESTORE_STACK                                   \
6957          "mv %0, a0"                                              \
6958          : /*out*/   "=r" (_res)                                  \
6959          : /*in*/    "0" (&_argvec[0])                            \
6960          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
6961       );                                                          \
6962       lval = (__typeof__(lval)) _res;                             \
6963    } while (0)
6964 
6965 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
6966                                  arg7,arg8,arg9)                  \
6967    do {                                                           \
6968       volatile OrigFn        _orig = (orig);                      \
6969       volatile unsigned long _argvec[10];                         \
6970       volatile unsigned long _res;                                \
6971       _argvec[0] = (unsigned long)_orig.nraddr;                   \
6972       _argvec[1] = (unsigned long)(arg1);                         \
6973       _argvec[2] = (unsigned long)(arg2);                         \
6974       _argvec[3] = (unsigned long)(arg3);                         \
6975       _argvec[4] = (unsigned long)(arg4);                         \
6976       _argvec[5] = (unsigned long)(arg5);                         \
6977       _argvec[6] = (unsigned long)(arg6);                         \
6978       _argvec[7] = (unsigned long)(arg7);                         \
6979       _argvec[8] = (unsigned long)(arg8);                         \
6980       _argvec[9] = (unsigned long)(arg9);                         \
6981       __asm__ volatile(                                           \
6982          VALGRIND_ALIGN_STACK                                     \
6983          "addi sp, sp, -16 \n\t"                                  \
6984          "ld a0, 8(%1) \n\t"                                      \
6985          "ld a1, 16(%1) \n\t"                                     \
6986          "ld a2, 24(%1) \n\t"                                     \
6987          "ld a3, 32(%1) \n\t"                                     \
6988          "ld a4, 40(%1) \n\t"                                     \
6989          "ld a5, 48(%1) \n\t"                                     \
6990          "ld a6, 56(%1) \n\t"                                     \
6991          "ld a7, 64(%1) \n\t"                                     \
6992          "ld t0, 72(%1) \n\t"                                     \
6993          "sd t0, 0(sp)  \n\t"                                     \
6994          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
6995          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
6996          VALGRIND_RESTORE_STACK                                   \
6997          "mv %0, a0"                                              \
6998          : /*out*/   "=r" (_res)                                  \
6999          : /*in*/    "0" (&_argvec[0])                            \
7000          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
7001       );                                                          \
7002       lval = (__typeof__(lval)) _res;                             \
7003    } while (0)
7004 
7005 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
7006                                   arg7,arg8,arg9,arg10)           \
7007    do {                                                           \
7008       volatile OrigFn        _orig = (orig);                      \
7009       volatile unsigned long _argvec[11];                         \
7010       volatile unsigned long _res;                                \
7011       _argvec[0] = (unsigned long)_orig.nraddr;                   \
7012       _argvec[1] = (unsigned long)(arg1);                         \
7013       _argvec[2] = (unsigned long)(arg2);                         \
7014       _argvec[3] = (unsigned long)(arg3);                         \
7015       _argvec[4] = (unsigned long)(arg4);                         \
7016       _argvec[5] = (unsigned long)(arg5);                         \
7017       _argvec[6] = (unsigned long)(arg6);                         \
7018       _argvec[7] = (unsigned long)(arg7);                         \
7019       _argvec[8] = (unsigned long)(arg8);                         \
7020       _argvec[9] = (unsigned long)(arg9);                         \
7021       _argvec[10] = (unsigned long)(arg10);                       \
7022       __asm__ volatile(                                           \
7023          VALGRIND_ALIGN_STACK                                     \
7024          "addi sp, sp, -16 \n\t"                                  \
7025          "ld a0, 8(%1) \n\t"                                      \
7026          "ld a1, 16(%1) \n\t"                                     \
7027          "ld a2, 24(%1) \n\t"                                     \
7028          "ld a3, 32(%1) \n\t"                                     \
7029          "ld a4, 40(%1) \n\t"                                     \
7030          "ld a5, 48(%1) \n\t"                                     \
7031          "ld a6, 56(%1) \n\t"                                     \
7032          "ld a7, 64(%1) \n\t"                                     \
7033          "ld t0, 72(%1) \n\t"                                     \
7034          "sd t0, 0(sp)  \n\t"                                     \
7035          "ld t0, 80(%1) \n\t"                                     \
7036          "sd t0, 8(sp)  \n\t"                                     \
7037          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
7038          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
7039          VALGRIND_RESTORE_STACK                                   \
7040          "mv %0, a0"                                              \
7041          : /*out*/   "=r" (_res)                                  \
7042          : /*in*/    "0" (&_argvec[0])                            \
7043          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
7044       );                                                          \
7045       lval = (__typeof__(lval)) _res;                             \
7046    } while (0)
7047 
7048 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
7049                                   arg7,arg8,arg9,arg10,arg11)     \
7050    do {                                                           \
7051       volatile OrigFn        _orig = (orig);                      \
7052       volatile unsigned long _argvec[12];                         \
7053       volatile unsigned long _res;                                \
7054       _argvec[0] = (unsigned long)_orig.nraddr;                   \
7055       _argvec[1] = (unsigned long)(arg1);                         \
7056       _argvec[2] = (unsigned long)(arg2);                         \
7057       _argvec[3] = (unsigned long)(arg3);                         \
7058       _argvec[4] = (unsigned long)(arg4);                         \
7059       _argvec[5] = (unsigned long)(arg5);                         \
7060       _argvec[6] = (unsigned long)(arg6);                         \
7061       _argvec[7] = (unsigned long)(arg7);                         \
7062       _argvec[8] = (unsigned long)(arg8);                         \
7063       _argvec[9] = (unsigned long)(arg9);                         \
7064       _argvec[10] = (unsigned long)(arg10);                       \
7065       _argvec[11] = (unsigned long)(arg11);                       \
7066       __asm__ volatile(                                           \
7067          VALGRIND_ALIGN_STACK                                     \
7068          "addi sp, sp, -32 \n\t"                                  \
7069          "ld a0, 8(%1) \n\t"                                      \
7070          "ld a1, 16(%1) \n\t"                                     \
7071          "ld a2, 24(%1) \n\t"                                     \
7072          "ld a3, 32(%1) \n\t"                                     \
7073          "ld a4, 40(%1) \n\t"                                     \
7074          "ld a5, 48(%1) \n\t"                                     \
7075          "ld a6, 56(%1) \n\t"                                     \
7076          "ld a7, 64(%1) \n\t"                                     \
7077          "ld t0, 72(%1) \n\t"                                     \
7078          "sd t0, 0(sp)  \n\t"                                     \
7079          "ld t0, 80(%1) \n\t"                                     \
7080          "sd t0, 8(sp)  \n\t"                                     \
7081          "ld t0, 88(%1) \n\t"                                     \
7082          "sd t0, 16(sp)  \n\t"                                    \
7083          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
7084          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
7085          VALGRIND_RESTORE_STACK                                   \
7086          "mv %0, a0"                                              \
7087          : /*out*/   "=r" (_res)                                  \
7088          : /*in*/    "0" (&_argvec[0])                            \
7089          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
7090       );                                                          \
7091       lval = (__typeof__(lval)) _res;                             \
7092    } while (0)
7093 
7094 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
7095                                   arg7,arg8,arg9,arg10,arg11,     \
7096                                   arg12)                          \
7097    do {                                                           \
7098       volatile OrigFn        _orig = (orig);                      \
7099       volatile unsigned long _argvec[13];                         \
7100       volatile unsigned long _res;                                \
7101       _argvec[0] = (unsigned long)_orig.nraddr;                   \
7102       _argvec[1] = (unsigned long)(arg1);                         \
7103       _argvec[2] = (unsigned long)(arg2);                         \
7104       _argvec[3] = (unsigned long)(arg3);                         \
7105       _argvec[4] = (unsigned long)(arg4);                         \
7106       _argvec[5] = (unsigned long)(arg5);                         \
7107       _argvec[6] = (unsigned long)(arg6);                         \
7108       _argvec[7] = (unsigned long)(arg7);                         \
7109       _argvec[8] = (unsigned long)(arg8);                         \
7110       _argvec[9] = (unsigned long)(arg9);                         \
7111       _argvec[10] = (unsigned long)(arg10);                       \
7112       _argvec[11] = (unsigned long)(arg11);                       \
7113       _argvec[12] = (unsigned long)(arg12);                       \
7114       __asm__ volatile(                                           \
7115          VALGRIND_ALIGN_STACK                                     \
7116          "addi sp, sp, -32 \n\t"                                  \
7117          "ld a0, 8(%1) \n\t"                                      \
7118          "ld a1, 16(%1) \n\t"                                     \
7119          "ld a2, 24(%1) \n\t"                                     \
7120          "ld a3, 32(%1) \n\t"                                     \
7121          "ld a4, 40(%1) \n\t"                                     \
7122          "ld a5, 48(%1) \n\t"                                     \
7123          "ld a6, 56(%1) \n\t"                                     \
7124          "ld a7, 64(%1) \n\t"                                     \
7125          "ld t0, 72(%1) \n\t"                                     \
7126          "sd t0, 0(sp)  \n\t"                                     \
7127          "ld t0, 80(%1) \n\t"                                     \
7128          "sd t0, 8(sp)  \n\t"                                     \
7129          "ld t0, 88(%1) \n\t"                                     \
7130          "sd t0, 16(sp)  \n\t"                                    \
7131          "ld t0, 96(%1) \n\t"                                     \
7132          "sd t0, 24(sp)  \n\t"                                    \
7133          "ld t0, 0(%1) \n\t"  /* target->t0 */                    \
7134          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_T0                   \
7135          VALGRIND_RESTORE_STACK                                   \
7136          "mv %0, a0"                                              \
7137          : /*out*/   "=r" (_res)                                  \
7138          : /*in*/    "0" (&_argvec[0])                            \
7139          : /*trash*/ "memory", __CALLER_SAVED_REGS, "s11"         \
7140       );                                                          \
7141       lval = (__typeof__(lval)) _res;                             \
7142    } while (0)
7143 
7144 #endif /* PLAT_riscv64_linux */
7145 
7146 /* ------------------------------------------------------------------ */
7147 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
7148 /*                                                                    */
7149 /* ------------------------------------------------------------------ */
7150 
7151 /* Some request codes.  There are many more of these, but most are not
7152    exposed to end-user view.  These are the public ones, all of the
7153    form 0x1000 + small_number.
7154 
7155    Core ones are in the range 0x00000000--0x0000ffff.  The non-public
7156    ones start at 0x2000.
7157 */
7158 
7159 /* These macros are used by tools -- they must be public, but don't
7160    embed them into other programs. */
7161 #define VG_USERREQ_TOOL_BASE(a,b) \
7162    ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
7163 #define VG_IS_TOOL_USERREQ(a, b, v) \
7164    (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
7165 
7166 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !! 
7167    This enum comprises an ABI exported by Valgrind to programs
7168    which use client requests.  DO NOT CHANGE THE NUMERIC VALUES OF THESE
7169    ENTRIES, NOR DELETE ANY -- add new ones at the end of the most
7170    relevant group. */
7171 typedef
7172    enum { VG_USERREQ__RUNNING_ON_VALGRIND      = 0x1001,
7173           VG_USERREQ__DISCARD_TRANSLATIONS     = 0x1002,
7174           VG_USERREQ__VALGRIND_REPLACES_MALLOC = 0x1003,
7175           VG_USERREQ__VALGRIND_GET_TOOLNAME    = 0x1004,
7176 
7177           /* These allow any function to be called from the simulated
7178              CPU but run on the real CPU.  Nb: the first arg passed to
7179              the function is always the ThreadId of the running
7180              thread!  So CLIENT_CALL0 actually requires a 1 arg
7181              function, etc. */
7182           VG_USERREQ__CLIENT_CALL0 = 0x1101,
7183           VG_USERREQ__CLIENT_CALL1 = 0x1102,
7184           VG_USERREQ__CLIENT_CALL2 = 0x1103,
7185           VG_USERREQ__CLIENT_CALL3 = 0x1104,
7186 
7187           /* Can be useful in regression testing suites -- eg. can
7188              send Valgrind's output to /dev/null and still count
7189              errors. */
7190           VG_USERREQ__COUNT_ERRORS = 0x1201,
7191 
7192           /* Allows the client program and/or gdbserver to execute a monitor
7193              command. */
7194           VG_USERREQ__GDB_MONITOR_COMMAND = 0x1202,
7195 
7196           /* Allows the client program to change a dynamic command line
7197              option.  */
7198           VG_USERREQ__CLO_CHANGE = 0x1203,
7199 
7200           /* These are useful and can be interpreted by any tool that
7201              tracks malloc() et al, by using vg_replace_malloc.c. */
7202           VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
7203           VG_USERREQ__RESIZEINPLACE_BLOCK = 0x130b,
7204           VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
7205           /* Memory pool support. */
7206           VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
7207           VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
7208           VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
7209           VG_USERREQ__MEMPOOL_FREE     = 0x1306,
7210           VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
7211           VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
7212           VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
7213           VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
7214 
7215           /* Allow printfs to valgrind log. */
7216           /* The first two pass the va_list argument by value, which
7217              assumes it is the same size as or smaller than a UWord,
7218              which generally isn't the case.  Hence are deprecated.
7219              The second two pass the vargs by reference and so are
7220              immune to this problem. */
7221           /* both :: char* fmt, va_list vargs (DEPRECATED) */
7222           VG_USERREQ__PRINTF           = 0x1401,
7223           VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
7224           /* both :: char* fmt, va_list* vargs */
7225           VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403,
7226           VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404,
7227 
7228           /* Stack support. */
7229           VG_USERREQ__STACK_REGISTER   = 0x1501,
7230           VG_USERREQ__STACK_DEREGISTER = 0x1502,
7231           VG_USERREQ__STACK_CHANGE     = 0x1503,
7232 
7233           /* Wine support */
7234           VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601,
7235 
7236           /* Querying of debug info. */
7237           VG_USERREQ__MAP_IP_TO_SRCLOC = 0x1701,
7238 
7239           /* Disable/enable error reporting level.  Takes a single
7240              Word arg which is the delta to this thread's error
7241              disablement indicator.  Hence 1 disables or further
7242              disables errors, and -1 moves back towards enablement.
7243              Other values are not allowed. */
7244           VG_USERREQ__CHANGE_ERR_DISABLEMENT = 0x1801,
7245 
7246           /* Some requests used for Valgrind internal, such as
7247              self-test or self-hosting. */
7248           /* Initialise IR injection */
7249           VG_USERREQ__VEX_INIT_FOR_IRI = 0x1901,
7250           /* Used by Inner Valgrind to inform Outer Valgrind where to
7251              find the list of inner guest threads */
7252           VG_USERREQ__INNER_THREADS    = 0x1902
7253    } Vg_ClientRequest;
7254 
7255 #if !defined(__GNUC__)
7256 #  define __extension__ /* */
7257 #endif
7258 
7259 
7260 /* Returns the number of Valgrinds this code is running under.  That
7261    is, 0 if running natively, 1 if running under Valgrind, 2 if
7262    running under Valgrind which is running under another Valgrind,
7263    etc. */
7264 #define RUNNING_ON_VALGRIND                                           \
7265     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */,         \
7266                                     VG_USERREQ__RUNNING_ON_VALGRIND,  \
7267                                     0, 0, 0, 0, 0)                    \
7268 
7269 
7270 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
7271    _qzz_len - 1].  Useful if you are debugging a JITter or some such,
7272    since it provides a way to make sure valgrind will retranslate the
7273    invalidated area.  Returns no value. */
7274 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)              \
7275     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DISCARD_TRANSLATIONS,  \
7276                                     _qzz_addr, _qzz_len, 0, 0, 0)
7277 
7278 /* Returns 1 if the tool replaces malloc (e.g., memcheck). Returns 0
7279    if the tool does not replace malloc (e.g., cachegrind and callgrind)
7280    or if the executable is not running under VALGRIND. */
7281 #define VALGRIND_REPLACES_MALLOC                                          \
7282     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */,             \
7283                                     VG_USERREQ__VALGRIND_REPLACES_MALLOC, \
7284                                     0, 0, 0, 0, 0)
7285 
7286 /* Get the running tool name as a string.
7287    Returns the required length (including terminating nul) for the input
7288    zzbuf. Returns 0 and the contents of zzbuf are not modified
7289    if not running under Valgrind.
7290    zzbuf may be NULL, which can be used to query the required buffer
7291    length before making a real request for the tool name.
7292    zzbuflen should be the length of zzbuf.
7293    The returned string in zzbuf will be nul terminated. If zzbuf is too
7294    small to contain the tool name then the name will be truncated */
7295 #define VALGRIND_GET_TOOLNAME(zzbuf,zzbuflen)                          \
7296     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                       \
7297                                     VG_USERREQ__VALGRIND_GET_TOOLNAME, \
7298                                     (char*)(zzbuf),                    \
7299                                     (zzbuflen),                        \
7300                                     0, 0, 0)
7301 
7302 #define VALGRIND_INNER_THREADS(_qzz_addr)                               \
7303    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__INNER_THREADS,           \
7304                                    _qzz_addr, 0, 0, 0, 0)
7305 
7306 
7307 /* These requests are for getting Valgrind itself to print something.
7308    Possibly with a backtrace.  This is a really ugly hack.  The return value
7309    is the number of characters printed, excluding the "**<pid>** " part at the
7310    start and the backtrace (if present). */
7311 
7312 #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER)
7313 /* Modern GCC will optimize the static routine out if unused,
7314    and unused attribute will shut down warnings about it.  */
7315 static int VALGRIND_PRINTF(const char *format, ...)
7316    __attribute__((format(__printf__, 1, 2), __unused__));
7317 #endif
7318 static int
7319 #if defined(_MSC_VER)
7320 __inline
7321 #endif
7322 VALGRIND_PRINTF(const char *format, ...)
7323 {
7324 #if defined(NVALGRIND)
7325    (void)format;
7326    return 0;
7327 #else /* NVALGRIND */
7328 #if defined(_MSC_VER) || defined(__MINGW64__)
7329    uintptr_t _qzz_res;
7330 #else
7331    unsigned long _qzz_res;
7332 #endif
7333    va_list vargs;
7334    va_start(vargs, format);
7335 #if defined(_MSC_VER) || defined(__MINGW64__)
7336    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
7337                               VG_USERREQ__PRINTF_VALIST_BY_REF,
7338                               (uintptr_t)format,
7339                               (uintptr_t)&vargs,
7340                               0, 0, 0);
7341 #else
7342    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
7343                               VG_USERREQ__PRINTF_VALIST_BY_REF,
7344                               (unsigned long)format,
7345                               (unsigned long)&vargs, 
7346                               0, 0, 0);
7347 #endif
7348    va_end(vargs);
7349    return (int)_qzz_res;
7350 #endif /* NVALGRIND */
7351 }
7352 
7353 #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER)
7354 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
7355    __attribute__((format(__printf__, 1, 2), __unused__));
7356 #endif
7357 static int
7358 #if defined(_MSC_VER)
7359 __inline
7360 #endif
7361 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
7362 {
7363 #if defined(NVALGRIND)
7364    (void)format;
7365    return 0;
7366 #else /* NVALGRIND */
7367 #if defined(_MSC_VER) || defined(__MINGW64__)
7368    uintptr_t _qzz_res;
7369 #else
7370    unsigned long _qzz_res;
7371 #endif
7372    va_list vargs;
7373    va_start(vargs, format);
7374 #if defined(_MSC_VER) || defined(__MINGW64__)
7375    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
7376                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
7377                               (uintptr_t)format,
7378                               (uintptr_t)&vargs,
7379                               0, 0, 0);
7380 #else
7381    _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
7382                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
7383                               (unsigned long)format,
7384                               (unsigned long)&vargs, 
7385                               0, 0, 0);
7386 #endif
7387    va_end(vargs);
7388    return (int)_qzz_res;
7389 #endif /* NVALGRIND */
7390 }
7391 
7392 
7393 /* These requests allow control to move from the simulated CPU to the
7394    real CPU, calling an arbitrary function.
7395    
7396    Note that the current ThreadId is inserted as the first argument.
7397    So this call:
7398 
7399      VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
7400 
7401    requires f to have this signature:
7402 
7403      Word f(Word tid, Word arg1, Word arg2)
7404 
7405    where "Word" is a word-sized type.
7406 
7407    Note that these client requests are not entirely reliable.  For example,
7408    if you call a function with them that subsequently calls printf(),
7409    there's a high chance Valgrind will crash.  Generally, your prospects of
7410    these working are made higher if the called function does not refer to
7411    any global variables, and does not refer to any libc or other functions
7412    (printf et al).  Any kind of entanglement with libc or dynamic linking is
7413    likely to have a bad outcome, for tricky reasons which we've grappled
7414    with a lot in the past.
7415 */
7416 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
7417     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,       \
7418                                     VG_USERREQ__CLIENT_CALL0,     \
7419                                     _qyy_fn,                      \
7420                                     0, 0, 0, 0)
7421 
7422 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)                    \
7423     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,            \
7424                                     VG_USERREQ__CLIENT_CALL1,          \
7425                                     _qyy_fn,                           \
7426                                     _qyy_arg1, 0, 0, 0)
7427 
7428 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)         \
7429     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,            \
7430                                     VG_USERREQ__CLIENT_CALL2,          \
7431                                     _qyy_fn,                           \
7432                                     _qyy_arg1, _qyy_arg2, 0, 0)
7433 
7434 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
7435     VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,             \
7436                                     VG_USERREQ__CLIENT_CALL3,           \
7437                                     _qyy_fn,                            \
7438                                     _qyy_arg1, _qyy_arg2,               \
7439                                     _qyy_arg3, 0)
7440 
7441 
7442 /* Counts the number of errors that have been recorded by a tool.  Nb:
7443    the tool must record the errors with VG_(maybe_record_error)() or
7444    VG_(unique_error)() for them to be counted. */
7445 #define VALGRIND_COUNT_ERRORS                                     \
7446     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(                    \
7447                                0 /* default return */,            \
7448                                VG_USERREQ__COUNT_ERRORS,          \
7449                                0, 0, 0, 0, 0)
7450 
7451 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
7452    when heap blocks are allocated in order to give accurate results.  This
7453    happens automatically for the standard allocator functions such as
7454    malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
7455    delete[], etc.
7456 
7457    But if your program uses a custom allocator, this doesn't automatically
7458    happen, and Valgrind will not do as well.  For example, if you allocate
7459    superblocks with mmap() and then allocates chunks of the superblocks, all
7460    Valgrind's observations will be at the mmap() level and it won't know that
7461    the chunks should be considered separate entities.  In Memcheck's case,
7462    that means you probably won't get heap block overrun detection (because
7463    there won't be redzones marked as unaddressable) and you definitely won't
7464    get any leak detection.
7465 
7466    The following client requests allow a custom allocator to be annotated so
7467    that it can be handled accurately by Valgrind.
7468 
7469    VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
7470    by a malloc()-like function.  For Memcheck (an illustrative case), this
7471    does two things:
7472 
7473    - It records that the block has been allocated.  This means any addresses
7474      within the block mentioned in error messages will be
7475      identified as belonging to the block.  It also means that if the block
7476      isn't freed it will be detected by the leak checker.
7477 
7478    - It marks the block as being addressable and undefined (if 'is_zeroed' is
7479      not set), or addressable and defined (if 'is_zeroed' is set).  This
7480      controls how accesses to the block by the program are handled.
7481    
7482    'addr' is the start of the usable block (ie. after any
7483    redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
7484    can apply redzones -- these are blocks of padding at the start and end of
7485    each block.  Adding redzones is recommended as it makes it much more likely
7486    Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
7487    zeroed (or filled with another predictable value), as is the case for
7488    calloc().
7489    
7490    VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
7491    heap block -- that will be used by the client program -- is allocated.
7492    It's best to put it at the outermost level of the allocator if possible;
7493    for example, if you have a function my_alloc() which calls
7494    internal_alloc(), and the client request is put inside internal_alloc(),
7495    stack traces relating to the heap block will contain entries for both
7496    my_alloc() and internal_alloc(), which is probably not what you want.
7497 
7498    For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
7499    custom blocks from within a heap block, B, that has been allocated with
7500    malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
7501    -- the custom blocks will take precedence.
7502 
7503    VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
7504    Memcheck, it does two things:
7505 
7506    - It records that the block has been deallocated.  This assumes that the
7507      block was annotated as having been allocated via
7508      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
7509 
7510    - It marks the block as being unaddressable.
7511 
7512    VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
7513    heap block is deallocated.
7514 
7515    VALGRIND_RESIZEINPLACE_BLOCK informs a tool about reallocation. For
7516    Memcheck, it does four things:
7517 
7518    - It records that the size of a block has been changed.  This assumes that
7519      the block was annotated as having been allocated via
7520      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
7521 
7522    - If the block shrunk, it marks the freed memory as being unaddressable.
7523 
7524    - If the block grew, it marks the new area as undefined and defines a red
7525      zone past the end of the new block.
7526 
7527    - The V-bits of the overlap between the old and the new block are preserved.
7528 
7529    VALGRIND_RESIZEINPLACE_BLOCK should be put after allocation of the new block
7530    and before deallocation of the old block.
7531 
7532    In many cases, these three client requests will not be enough to get your
7533    allocator working well with Memcheck.  More specifically, if your allocator
7534    writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
7535    will be necessary to mark the memory as addressable just before the zeroing
7536    occurs, otherwise you'll get a lot of invalid write errors.  For example,
7537    you'll need to do this if your allocator recycles freed blocks, but it
7538    zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
7539    Alternatively, if your allocator reuses freed blocks for allocator-internal
7540    data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
7541 
7542    Really, what's happening is a blurring of the lines between the client
7543    program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
7544    memory should be considered unaddressable to the client program, but the
7545    allocator knows more than the rest of the client program and so may be able
7546    to safely access it.  Extra client requests are necessary for Valgrind to
7547    understand the distinction between the allocator and the rest of the
7548    program.
7549 
7550    Ignored if addr == 0.
7551 */
7552 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)          \
7553     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MALLOCLIKE_BLOCK,       \
7554                                     addr, sizeB, rzB, is_zeroed, 0)
7555 
7556 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
7557    Ignored if addr == 0.
7558 */
7559 #define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB)     \
7560     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__RESIZEINPLACE_BLOCK,    \
7561                                     addr, oldSizeB, newSizeB, rzB, 0)
7562 
7563 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
7564    Ignored if addr == 0.
7565 */
7566 #define VALGRIND_FREELIKE_BLOCK(addr, rzB)                              \
7567     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__FREELIKE_BLOCK,         \
7568                                     addr, rzB, 0, 0, 0)
7569 
7570 /* Create a memory pool. */
7571 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
7572     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL,   \
7573                                     pool, rzB, is_zeroed, 0, 0)
7574 
7575 /* Create a memory pool with some flags specifying extended behaviour.
7576    When flags is zero, the behaviour is identical to VALGRIND_CREATE_MEMPOOL.
7577    
7578    The flag VALGRIND_MEMPOOL_METAPOOL specifies that the pieces of memory 
7579    associated with the pool using VALGRIND_MEMPOOL_ALLOC  will be used
7580    by the application as superblocks to dole out MALLOC_LIKE blocks using
7581    VALGRIND_MALLOCLIKE_BLOCK. In other words, a meta pool is a "2 levels"
7582    pool : first level is the blocks described by VALGRIND_MEMPOOL_ALLOC.
7583    The second level blocks are described using VALGRIND_MALLOCLIKE_BLOCK.
7584    Note that the association between the pool and the second level blocks
7585    is implicit : second level blocks will be located inside first level
7586    blocks. It is necessary to use the VALGRIND_MEMPOOL_METAPOOL flag
7587    for such 2 levels pools, as otherwise valgrind will detect overlapping
7588    memory blocks, and will abort execution (e.g. during leak search).
7589 
7590    Such a meta pool can also be marked as an 'auto free' pool using the flag
7591    VALGRIND_MEMPOOL_AUTO_FREE, which must be OR-ed together with the
7592    VALGRIND_MEMPOOL_METAPOOL. For an 'auto free' pool, VALGRIND_MEMPOOL_FREE
7593    will automatically free the second level blocks that are contained
7594    inside the first level block freed with VALGRIND_MEMPOOL_FREE.
7595    In other words, calling VALGRIND_MEMPOOL_FREE will cause implicit calls
7596    to VALGRIND_FREELIKE_BLOCK for all the second level blocks included
7597    in the first level block.
7598    Note: it is an error to use the VALGRIND_MEMPOOL_AUTO_FREE flag
7599    without the VALGRIND_MEMPOOL_METAPOOL flag.
7600 */
7601 #define VALGRIND_MEMPOOL_AUTO_FREE  1
7602 #define VALGRIND_MEMPOOL_METAPOOL   2
7603 #define VALGRIND_CREATE_MEMPOOL_EXT(pool, rzB, is_zeroed, flags)        \
7604    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL,          \
7605                                    pool, rzB, is_zeroed, flags, 0)
7606 
7607 /* Destroy a memory pool. */
7608 #define VALGRIND_DESTROY_MEMPOOL(pool)                            \
7609     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DESTROY_MEMPOOL,  \
7610                                     pool, 0, 0, 0, 0)
7611 
7612 /* Associate a piece of memory with a memory pool. */
7613 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
7614     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_ALLOC,    \
7615                                     pool, addr, size, 0, 0)
7616 
7617 /* Disassociate a piece of memory from a memory pool. */
7618 #define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
7619     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_FREE,     \
7620                                     pool, addr, 0, 0, 0)
7621 
7622 /* Disassociate any pieces outside a particular range. */
7623 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
7624     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_TRIM,     \
7625                                     pool, addr, size, 0, 0)
7626 
7627 /* Resize and/or move a piece associated with a memory pool. */
7628 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
7629     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MOVE_MEMPOOL,     \
7630                                     poolA, poolB, 0, 0, 0)
7631 
7632 /* Resize and/or move a piece associated with a memory pool. */
7633 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
7634     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_CHANGE,   \
7635                                     pool, addrA, addrB, size, 0)
7636 
7637 /* Return 1 if a mempool exists, else 0. */
7638 #define VALGRIND_MEMPOOL_EXISTS(pool)                             \
7639     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
7640                                VG_USERREQ__MEMPOOL_EXISTS,        \
7641                                pool, 0, 0, 0, 0)
7642 
7643 /* Mark a piece of memory as being a stack. Returns a stack id.
7644    start is the lowest addressable stack byte, end is the highest
7645    addressable stack byte. */
7646 #define VALGRIND_STACK_REGISTER(start, end)                       \
7647     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
7648                                VG_USERREQ__STACK_REGISTER,        \
7649                                start, end, 0, 0, 0)
7650 
7651 /* Unmark the piece of memory associated with a stack id as being a
7652    stack. */
7653 #define VALGRIND_STACK_DEREGISTER(id)                             \
7654     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_DEREGISTER, \
7655                                     id, 0, 0, 0, 0)
7656 
7657 /* Change the start and end address of the stack id.
7658    start is the new lowest addressable stack byte, end is the new highest
7659    addressable stack byte. */
7660 #define VALGRIND_STACK_CHANGE(id, start, end)                     \
7661     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_CHANGE,     \
7662                                     id, start, end, 0, 0)
7663 
7664 /* Load PDB debug info for Wine PE image_map. */
7665 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)     \
7666     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__LOAD_PDB_DEBUGINFO, \
7667                                     fd, ptr, total_size, delta, 0)
7668 
7669 /* Map a code address to a source file name and line number.  buf64
7670    must point to a 64-byte buffer in the caller's address space.  The
7671    result will be dumped in there and is guaranteed to be zero
7672    terminated.  If no info is found, the first byte is set to zero. */
7673 #define VALGRIND_MAP_IP_TO_SRCLOC(addr, buf64)                    \
7674     (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
7675                                VG_USERREQ__MAP_IP_TO_SRCLOC,      \
7676                                addr, buf64, 0, 0, 0)
7677 
7678 /* Disable error reporting for this thread.  Behaves in a stack like
7679    way, so you can safely call this multiple times provided that
7680    VALGRIND_ENABLE_ERROR_REPORTING is called the same number of times
7681    to re-enable reporting.  The first call of this macro disables
7682    reporting.  Subsequent calls have no effect except to increase the
7683    number of VALGRIND_ENABLE_ERROR_REPORTING calls needed to re-enable
7684    reporting.  Child threads do not inherit this setting from their
7685    parents -- they are always created with reporting enabled. */
7686 #define VALGRIND_DISABLE_ERROR_REPORTING                                \
7687     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \
7688                                     1, 0, 0, 0, 0)
7689 
7690 /* Re-enable error reporting, as per comments on
7691    VALGRIND_DISABLE_ERROR_REPORTING. */
7692 #define VALGRIND_ENABLE_ERROR_REPORTING                                 \
7693     VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \
7694                                     -1, 0, 0, 0, 0)
7695 
7696 /* Execute a monitor command from the client program.
7697    If a connection is opened with GDB, the output will be sent
7698    according to the output mode set for vgdb.
7699    If no connection is opened, output will go to the log output.
7700    Returns 1 if command not recognised, 0 otherwise. */
7701 #define VALGRIND_MONITOR_COMMAND(command)                               \
7702    VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__GDB_MONITOR_COMMAND, \
7703                                    command, 0, 0, 0, 0)
7704 
7705 
7706 /* Change the value of a dynamic command line option.
7707    Note that unknown or not dynamically changeable options
7708    will cause a warning message to be output.  */
7709 #define VALGRIND_CLO_CHANGE(option)                           \
7710    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CLO_CHANGE, \
7711                                    option, 0, 0, 0, 0)
7712 
7713 
7714 #undef PLAT_x86_darwin
7715 #undef PLAT_amd64_darwin
7716 #undef PLAT_x86_freebsd
7717 #undef PLAT_amd64_freebsd
7718 #undef PLAT_arm64_freebsd
7719 #undef PLAT_x86_win32
7720 #undef PLAT_amd64_win64
7721 #undef PLAT_x86_linux
7722 #undef PLAT_amd64_linux
7723 #undef PLAT_ppc32_linux
7724 #undef PLAT_ppc64be_linux
7725 #undef PLAT_ppc64le_linux
7726 #undef PLAT_arm_linux
7727 #undef PLAT_s390x_linux
7728 #undef PLAT_mips32_linux
7729 #undef PLAT_mips64_linux
7730 #undef PLAT_nanomips_linux
7731 #undef PLAT_riscv64_linux
7732 #undef PLAT_x86_solaris
7733 #undef PLAT_amd64_solaris
7734 
7735 #endif   /* __VALGRIND_H */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.3.7 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!