include/flint/longlong_asm_gcc.h

0001 /*
0002     Copyright 1991-1994, 1996, 1997, 1999-2005, 2007-2009, 2011-2020 Free
0003     Software Foundation, Inc.
0004
0005     Copyright (C) 2023, 2024 Albin Ahlbäck
0006
0007     This file is part of FLINT.
0008
0009     Contains code from GNU MP Library.
0010
0011     FLINT is free software: you can redistribute it and/or modify it under
0012     the terms of the GNU Lesser General Public License (LGPL) as published
0013     by the Free Software Foundation; either version 3 of the License, or
0014     (at your option) any later version.  See <https://www.gnu.org/licenses/>.
0015 */
0016
0017 #ifndef LONGLONG_ASM_H
0018 #define LONGLONG_ASM_H
0019
0020 /* Machine specific operations */
0021 #if defined (__amd64__) || (FLINT_BITS == 32 && (defined (__i386__) || defined (__i486__)))
0022
0023 # if FLINT_BITS == 64 && defined (__amd64__)
0024 #  define _ASM_ADD "addq"
0025 #  define _ASM_ADC "adcq"
0026 #  define _ASM_SUB "subq"
0027 #  define _ASM_SBB "sbbq"
0028 #  define _ASM_MUL "mulq"
0029 #  define _ASM_IMUL "imulq"
0030 #  define _ASM_PRE "q"
0031 #  define _ASM_RME "rme"
0032 # else
0033 #  define _ASM_ADD "addl"
0034 #  define _ASM_ADC "adcl"
0035 #  define _ASM_SUB "subl"
0036 #  define _ASM_SBB "sbbl"
0037 #  define _ASM_MUL "mull"
0038 #  define _ASM_IMUL "imull"
0039 #  define _ASM_PRE "k"
0040 #  define _ASM_RME "g"
0041 # endif
0042
0043 # define add_ssaaaa(s1, s0, a1, a0, b1, b0) \
0044   __asm__(_ASM_ADD " %5,%" _ASM_PRE "1\n" \
0045      "\t" _ASM_ADC " %3,%" _ASM_PRE "0" \
0046     : "=r" (s1), "=&r" (s0) \
0047     : "0" ((ulong)(a1)), _ASM_RME ((ulong)(b1)), \
0048       "%1" ((ulong)(a0)), _ASM_RME ((ulong)(b0)))
0049
0050 # define add_sssaaaaaa(s2, s1, s0, a2, a1, a0, b2, b1, b0) \
0051   __asm__(_ASM_ADD " %8,%" _ASM_PRE "2\n" \
0052      "\t" _ASM_ADC " %6,%" _ASM_PRE "1\n" \
0053      "\t" _ASM_ADC " %4,%" _ASM_PRE "0" \
0054     : "=r" (s2), "=&r" (s1), "=&r" (s0) \
0055     : "0" ((ulong)(a2)), _ASM_RME ((ulong)(b2)), \
0056       "1" ((ulong)(a1)), _ASM_RME ((ulong)(b1)), \
0057       "2" ((ulong)(a0)), _ASM_RME ((ulong)(b0)))
0058
0059 # define add_ssssaaaaaaaa(s3, s2, s1, s0, a3, a2, a1, a0, b3, b2, b1, b0) \
0060   __asm__(_ASM_ADD " %11,%" _ASM_PRE "3\n" \
0061      "\t" _ASM_ADC " %9,%" _ASM_PRE "2\n" \
0062      "\t" _ASM_ADC " %7,%" _ASM_PRE "1\n" \
0063      "\t" _ASM_ADC " %5,%" _ASM_PRE "0" \
0064     : "=r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0065     : "0" ((ulong)(a3)), _ASM_RME ((ulong)(b3)), \
0066       "1" ((ulong)(a2)), _ASM_RME ((ulong)(b2)), \
0067       "2" ((ulong)(a1)), _ASM_RME ((ulong)(b1)), \
0068       "3" ((ulong)(a0)), _ASM_RME ((ulong)(b0)))
0069
0070 # define add_sssssaaaaaaaaaa(s4, s3, s2, s1, s0, a4, a3, a2, a1, a0, b4, b3, b2, b1, b0) \
0071   __asm__(_ASM_ADD " %14,%" _ASM_PRE "4\n" \
0072      "\t" _ASM_ADC " %12,%" _ASM_PRE "3\n" \
0073      "\t" _ASM_ADC " %10,%" _ASM_PRE "2\n" \
0074      "\t" _ASM_ADC " %8,%" _ASM_PRE "1\n" \
0075      "\t" _ASM_ADC " %6,%" _ASM_PRE "0" \
0076     : "=r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0077     : "0" ((ulong)(a4)), _ASM_RME ((ulong)(b4)), \
0078       "1" ((ulong)(a3)), _ASM_RME ((ulong)(b3)), \
0079       "2" ((ulong)(a2)), _ASM_RME ((ulong)(b2)), \
0080       "3" ((ulong)(a1)), _ASM_RME ((ulong)(b1)), \
0081       "4" ((ulong)(a0)), _ASM_RME ((ulong)(b0)))
0082
0083 # define sub_ddmmss(d1, d0, m1, m0, s1, s0) \
0084   __asm__(_ASM_SUB " %5,%" _ASM_PRE "1\n" \
0085      "\t" _ASM_SBB " %3,%" _ASM_PRE "0" \
0086     : "=r" (d1), "=&r" (d0) \
0087     : "0" ((ulong)(m1)), _ASM_RME ((ulong)(s1)), \
0088       "1" ((ulong)(m0)), _ASM_RME ((ulong)(s0)))
0089
0090 # define sub_dddmmmsss(d2, d1, d0, m2, m1, m0, s2, s1, s0) \
0091   __asm__(_ASM_SUB " %8,%" _ASM_PRE "2\n" \
0092      "\t" _ASM_SBB " %6,%" _ASM_PRE "1\n" \
0093      "\t" _ASM_SBB " %4,%" _ASM_PRE "0" \
0094     : "=r" (d2), "=&r" (d1), "=&r" (d0) \
0095     : "0" ((ulong)(m2)), _ASM_RME ((ulong)(s2)), \
0096       "1" ((ulong)(m1)), _ASM_RME ((ulong)(s1)), \
0097       "2" ((ulong)(m0)), _ASM_RME ((ulong)(s0)))
0098
0099 # define sub_ddddmmmmssss(d3, d2, d1, d0, m3, m2, m1, m0, s3, s2, s1, s0) \
0100   __asm__(_ASM_SUB " %11,%" _ASM_PRE "3\n" \
0101      "\t" _ASM_SBB " %9,%" _ASM_PRE "2\n" \
0102      "\t" _ASM_SBB " %7,%" _ASM_PRE "1\n" \
0103      "\t" _ASM_SBB " %5,%" _ASM_PRE "0" \
0104     : "=r" (d3), "=&r" (d2), "=&r" (d1), "=&r" (d0) \
0105     : "0" ((ulong)(m3)), _ASM_RME ((ulong)(s3)), \
0106       "1" ((ulong)(m2)), _ASM_RME ((ulong)(s2)), \
0107       "2" ((ulong)(m1)), _ASM_RME ((ulong)(s1)), \
0108       "3" ((ulong)(m0)), _ASM_RME ((ulong)(s0)))
0109
0110 # define sub_dddddmmmmmsssss(d4, d3, d2, d1, d0, m4, m3, m2, m1, m0, s4, s3, s2, s1, s0) \
0111   __asm__(_ASM_SUB " %14,%" _ASM_PRE "4\n" \
0112      "\t" _ASM_SBB " %12,%" _ASM_PRE "3\n" \
0113      "\t" _ASM_SBB " %10,%" _ASM_PRE "2\n" \
0114      "\t" _ASM_SBB " %8,%" _ASM_PRE "1\n" \
0115      "\t" _ASM_SBB " %6,%" _ASM_PRE "0" \
0116     : "=r" (d4), "=&r" (d3), "=&r" (d2), "=&r" (d1), "=&r" (d0) \
0117     : "0" ((ulong)(m4)), _ASM_RME ((ulong)(s4)), \
0118       "1" ((ulong)(m3)), _ASM_RME ((ulong)(s3)), \
0119       "2" ((ulong)(m2)), _ASM_RME ((ulong)(s2)), \
0120       "3" ((ulong)(m1)), _ASM_RME ((ulong)(s1)), \
0121       "4" ((ulong)(m0)), _ASM_RME ((ulong)(s0)))
0122
0123 /* x86 does not have enough registers */
0124 # if FLINT_BITS == 64 && defined (__amd64__)
0125
0126 # define add_ssssssaaaaaaaaaaaa(s5, s4, s3, s2, s1, s0, a5, a4, a3, a2, a1, a0, b5, b4, b3, b2, b1, b0) \
0127   __asm__(_ASM_ADD " %17,%" _ASM_PRE "5\n" \
0128      "\t" _ASM_ADC " %15,%" _ASM_PRE "4\n" \
0129      "\t" _ASM_ADC " %13,%" _ASM_PRE "3\n" \
0130      "\t" _ASM_ADC " %11,%" _ASM_PRE "2\n" \
0131      "\t" _ASM_ADC " %9,%" _ASM_PRE "1\n" \
0132      "\t" _ASM_ADC " %7,%" _ASM_PRE "0" \
0133     : "=r" (s5), "=&r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0134     : "0" ((ulong)(a5)), _ASM_RME ((ulong)(b5)), \
0135       "1" ((ulong)(a4)), _ASM_RME ((ulong)(b4)), \
0136       "2" ((ulong)(a3)), _ASM_RME ((ulong)(b3)), \
0137       "3" ((ulong)(a2)), _ASM_RME ((ulong)(b2)), \
0138       "4" ((ulong)(a1)), _ASM_RME ((ulong)(b1)), \
0139       "5" ((ulong)(a0)), _ASM_RME ((ulong)(b0)))
0140
0141 # define add_sssssssaaaaaaaaaaaaaa(s6, s5, s4, s3, s2, s1, s0, a6, a5, a4, a3, a2, a1, a0, b6, b5, b4, b3, b2, b1, b0) \
0142   __asm__(_ASM_ADD " %20,%" _ASM_PRE "6\n" \
0143      "\t" _ASM_ADC " %18,%" _ASM_PRE "5\n" \
0144      "\t" _ASM_ADC " %16,%" _ASM_PRE "4\n" \
0145      "\t" _ASM_ADC " %14,%" _ASM_PRE "3\n" \
0146      "\t" _ASM_ADC " %12,%" _ASM_PRE "2\n" \
0147      "\t" _ASM_ADC " %10,%" _ASM_PRE "1\n" \
0148      "\t" _ASM_ADC " %8,%" _ASM_PRE "0" \
0149     : "=r" (s6), "=&r" (s5), "=&r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0150     : "0" ((ulong)(a6)), _ASM_RME ((ulong)(b6)), \
0151       "1" ((ulong)(a5)), _ASM_RME ((ulong)(b5)), \
0152       "2" ((ulong)(a4)), _ASM_RME ((ulong)(b4)), \
0153       "3" ((ulong)(a3)), _ASM_RME ((ulong)(b3)), \
0154       "4" ((ulong)(a2)), _ASM_RME ((ulong)(b2)), \
0155       "5" ((ulong)(a1)), _ASM_RME ((ulong)(b1)), \
0156       "6" ((ulong)(a0)), _ASM_RME ((ulong)(b0)))
0157
0158 # define add_ssssssssaaaaaaaaaaaaaaaa(s7, s6, s5, s4, s3, s2, s1, s0, a7, a6, a5, a4, a3, a2, a1, a0, b7, b6, b5, b4, b3, b2, b1, b0) \
0159   __asm__(_ASM_ADD " %23,%" _ASM_PRE "7\n" \
0160      "\t" _ASM_ADC " %21,%" _ASM_PRE "6\n" \
0161      "\t" _ASM_ADC " %19,%" _ASM_PRE "5\n" \
0162      "\t" _ASM_ADC " %17,%" _ASM_PRE "4\n" \
0163      "\t" _ASM_ADC " %15,%" _ASM_PRE "3\n" \
0164      "\t" _ASM_ADC " %13,%" _ASM_PRE "2\n" \
0165      "\t" _ASM_ADC " %11,%" _ASM_PRE "1\n" \
0166      "\t" _ASM_ADC " %9,%" _ASM_PRE "0" \
0167     : "=r" (s7), "=&r" (s6), "=&r" (s5), "=&r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0168     : "0" ((ulong)(a7)), _ASM_RME ((ulong)(b7)), \
0169       "1" ((ulong)(a6)), _ASM_RME ((ulong)(b6)), \
0170       "2" ((ulong)(a5)), _ASM_RME ((ulong)(b5)), \
0171       "3" ((ulong)(a4)), _ASM_RME ((ulong)(b4)), \
0172       "4" ((ulong)(a3)), _ASM_RME ((ulong)(b3)), \
0173       "5" ((ulong)(a2)), _ASM_RME ((ulong)(b2)), \
0174       "6" ((ulong)(a1)), _ASM_RME ((ulong)(b1)), \
0175       "7" ((ulong)(a0)), _ASM_RME ((ulong)(b0)))
0176
0177 # define sub_ddddddmmmmmmssssss(d5, d4, d3, d2, d1, d0, m5, m4, m3, m2, m1, m0, s5, s4, s3, s2, s1, s0) \
0178   __asm__(_ASM_SUB " %17,%" _ASM_PRE "5\n" \
0179      "\t" _ASM_SBB " %15,%" _ASM_PRE "4\n" \
0180      "\t" _ASM_SBB " %13,%" _ASM_PRE "3\n" \
0181      "\t" _ASM_SBB " %11,%" _ASM_PRE "2\n" \
0182      "\t" _ASM_SBB " %9,%" _ASM_PRE "1\n" \
0183      "\t" _ASM_SBB " %7,%" _ASM_PRE "0" \
0184     : "=r" (d5), "=&r" (d4), "=&r" (d3), "=&r" (d2), "=&r" (d1), "=&r" (d0) \
0185     : "0" ((ulong)(m5)), _ASM_RME ((ulong)(s5)), \
0186       "1" ((ulong)(m4)), _ASM_RME ((ulong)(s4)), \
0187       "2" ((ulong)(m3)), _ASM_RME ((ulong)(s3)), \
0188       "3" ((ulong)(m2)), _ASM_RME ((ulong)(s2)), \
0189       "4" ((ulong)(m1)), _ASM_RME ((ulong)(s1)), \
0190       "5" ((ulong)(m0)), _ASM_RME ((ulong)(s0)))
0191
0192 # define sub_dddddddmmmmmmmsssssss(d6, d5, d4, d3, d2, d1, d0, m6, m5, m4, m3, m2, m1, m0, s6, s5, s4, s3, s2, s1, s0) \
0193   __asm__(_ASM_SUB " %20,%" _ASM_PRE "6\n" \
0194      "\t" _ASM_SBB " %18,%" _ASM_PRE "5\n" \
0195      "\t" _ASM_SBB " %16,%" _ASM_PRE "4\n" \
0196      "\t" _ASM_SBB " %14,%" _ASM_PRE "3\n" \
0197      "\t" _ASM_SBB " %12,%" _ASM_PRE "2\n" \
0198      "\t" _ASM_SBB " %10,%" _ASM_PRE "1\n" \
0199      "\t" _ASM_SBB " %8,%" _ASM_PRE "0" \
0200     : "=r" (d6), "=&r" (d5), "=&r" (d4), "=&r" (d3), "=&r" (d2), "=&r" (d1), "=&r" (d0) \
0201     : "0" ((ulong)(m6)), _ASM_RME ((ulong)(s6)), \
0202       "1" ((ulong)(m5)), _ASM_RME ((ulong)(s5)), \
0203       "2" ((ulong)(m4)), _ASM_RME ((ulong)(s4)), \
0204       "3" ((ulong)(m3)), _ASM_RME ((ulong)(s3)), \
0205       "4" ((ulong)(m2)), _ASM_RME ((ulong)(s2)), \
0206       "5" ((ulong)(m1)), _ASM_RME ((ulong)(s1)), \
0207       "6" ((ulong)(m0)), _ASM_RME ((ulong)(s0)))
0208
0209 # define sub_ddddddddmmmmmmmmssssssss(d7, d6, d5, d4, d3, d2, d1, d0, m7, m6, m5, m4, m3, m2, m1, m0, s7, s6, s5, s4, s3, s2, s1, s0) \
0210   __asm__(_ASM_SUB " %23,%" _ASM_PRE "7\n" \
0211      "\t" _ASM_SBB " %21,%" _ASM_PRE "6\n" \
0212      "\t" _ASM_SBB " %19,%" _ASM_PRE "5\n" \
0213      "\t" _ASM_SBB " %17,%" _ASM_PRE "4\n" \
0214      "\t" _ASM_SBB " %15,%" _ASM_PRE "3\n" \
0215      "\t" _ASM_SBB " %13,%" _ASM_PRE "2\n" \
0216      "\t" _ASM_SBB " %11,%" _ASM_PRE "1\n" \
0217      "\t" _ASM_SBB " %9,%" _ASM_PRE "0" \
0218     : "=r" (d7), "=&r" (d6), "=&r" (d5), "=&r" (d4), "=&r" (d3), "=&r" (d2), "=&r" (d1), "=&r" (d0) \
0219     : "0" ((ulong)(m7)), _ASM_RME ((ulong)(s7)), \
0220       "1" ((ulong)(m6)), _ASM_RME ((ulong)(s6)), \
0221       "2" ((ulong)(m5)), _ASM_RME ((ulong)(s5)), \
0222       "3" ((ulong)(m4)), _ASM_RME ((ulong)(s4)), \
0223       "4" ((ulong)(m3)), _ASM_RME ((ulong)(s3)), \
0224       "5" ((ulong)(m2)), _ASM_RME ((ulong)(s2)), \
0225       "6" ((ulong)(m1)), _ASM_RME ((ulong)(s1)), \
0226       "7" ((ulong)(m0)), _ASM_RME ((ulong)(s0)))
0227
0228 #endif
0229
0230 # if defined(__BMI2__) && defined(__amd64__)
0231 #  define umul_ppmm(w1, w0, u, v) \
0232   __asm__("mulx\t%3, %q0, %q1" \
0233     : "=r" (w0), "=r" (w1) \
0234     : "%d" ((ulong)(u)), "rm" ((ulong)(v)))
0235 # else
0236 #  define umul_ppmm(w1, w0, u, v) \
0237   __asm__(_ASM_MUL " %3" \
0238     : "=a" (w0), "=d" (w1) \
0239     : "%0" ((ulong)(u)), "rm" ((ulong)(v)))
0240 #endif
0241
0242 # define smul_ppmm(w1, w0, u, v) \
0243   __asm__(_ASM_IMUL " %3" \
0244     : "=a" (w0), "=d" (w1) \
0245     : "%0" ((ulong)(u)), "rm" ((ulong)(v)))
0246
0247 #elif (FLINT_BITS == 64 && defined(__aarch64__)) || (FLINT_BITS == 32 && defined(__arm__))
0248
0249 # define add_ssaaaa(s1, s0, a1, a0, b1, b0) \
0250   __asm__("adds %1,%3,%5\n" \
0251         "\tadc %0,%2,%4" \
0252     : "=r" (s1), "=&r" (s0) \
0253     : "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0254       "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0255     : "cc")
0256
0257 # define sub_ddmmss(s1, s0, a1, a0, b1, b0) \
0258   __asm__("subs %1,%3,%5\n" \
0259         "\tsbc  %0,%2,%4" \
0260     : "=r" (s1), "=&r" (s0) \
0261     : "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0262       "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0263     : "cc")
0264
0265 # define add_sssaaaaaa(s2, s1, s0, a2, a1, a0, b2, b1, b0) \
0266   __asm__("adds %2,%5,%8\n" \
0267         "\tadcs %1,%4,%7\n" \
0268         "\tadc %0,%3,%6" \
0269     : "=r" (s2), "=&r" (s1), "=&r" (s0) \
0270     : "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0271       "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0272     : "cc")
0273
0274 # define sub_dddmmmsss(s2, s1, s0, a2, a1, a0, b2, b1, b0) \
0275   __asm__("subs %2,%5,%8\n" \
0276         "\tsbcs %1,%4,%7\n" \
0277         "\tsbc  %0,%3,%6" \
0278     : "=r" (s2), "=&r" (s1), "=&r" (s0) \
0279     : "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0280       "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0281     : "cc")
0282
0283 /* ARM (32-bit) only has 16 registers, and has problems with inline assembly
0284  * with too many registers.  See issue #2131. */
0285 # if FLINT_BITS == 64 && defined(__aarch64__)
0286 # define add_ssssaaaaaaaa(s3, s2, s1, s0, a3, a2, a1, a0, b3, b2, b1, b0) \
0287   __asm__("adds %3,%7,%11\n" \
0288         "\tadcs %2,%6,%10\n" \
0289         "\tadcs %1,%5,%9\n" \
0290         "\tadc  %0,%4,%8" \
0291     : "=r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0292     : "r" ((ulong)(a3)), "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0293       "r" ((ulong)(b3)), "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0294     : "cc")
0295
0296 # define sub_ddddmmmmssss(s3, s2, s1, s0, a3, a2, a1, a0, b3, b2, b1, b0) \
0297   __asm__ ("subs %3,%7,%11\n\tsbcs %2,%6,%10\n\tsbcs %1,%5,%9\n\tsbc %0,%4,%8"\
0298        : "=r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0299        : "r" ((ulong)(a3)), "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0300          "r" ((ulong)(b3)), "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0301        : "cc")
0302
0303 # define add_sssssaaaaaaaaaa(s4, s3, s2, s1, s0, a4, a3, a2, a1, a0, b4, b3, b2, b1, b0) \
0304   __asm__ ("adds %4,%9,%14\n\tadcs %3,%8,%13\n\tadcs %2,%7,%12\n\tadcs %1,%6,%11\n\tadc %0,%5,%10"\
0305        : "=r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0306        : "r" ((ulong)(a4)), "r" ((ulong)(a3)), "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0307          "r" ((ulong)(b4)), "r" ((ulong)(b3)), "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0308        : "cc")
0309
0310 # define sub_dddddmmmmmsssss(s4, s3, s2, s1, s0, a4, a3, a2, a1, a0, b4, b3, b2, b1, b0) \
0311   __asm__ ("subs %4,%9,%14\n\tsbcs %3,%8,%13\n\tsbcs %2,%7,%12\n\tsbcs %1,%6,%11\n\tsbc %0,%5,%10"\
0312        : "=r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0313        : "r" ((ulong)(a4)), "r" ((ulong)(a3)), "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0314          "r" ((ulong)(b4)), "r" ((ulong)(b3)), "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0315        : "cc")
0316
0317 # define add_ssssssaaaaaaaaaaaa(s5, s4, s3, s2, s1, s0, a5, a4, a3, a2, a1, a0, b5, b4, b3, b2, b1, b0) \
0318   __asm__ ("adds %5,%11,%17\n\tadcs %4,%10,%16\n\tadcs %3,%9,%15\n\tadcs %2,%8,%14\n\tadcs %1,%7,%13\n\tadc %0,%6,%12"\
0319        : "=r" (s5), "=&r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0320        : "r" ((ulong)(a5)), "r" ((ulong)(a4)), "r" ((ulong)(a3)), "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0321          "r" ((ulong)(b5)), "r" ((ulong)(b4)), "r" ((ulong)(b3)), "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0322        : "cc")
0323
0324 # define sub_ddddddmmmmmmssssss(s5, s4, s3, s2, s1, s0, a5, a4, a3, a2, a1, a0, b5, b4, b3, b2, b1, b0) \
0325   __asm__ ("subs %5,%11,%17\n\tsbcs %4,%10,%16\n\tsbcs %3,%9,%15\n\tsbcs %2,%8,%14\n\tsbcs %1,%7,%13\n\tsbc %0,%6,%12"\
0326        : "=r" (s5), "=&r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0327        : "r" ((ulong)(a5)), "r" ((ulong)(a4)), "r" ((ulong)(a3)), "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0328          "r" ((ulong)(b5)), "r" ((ulong)(b4)), "r" ((ulong)(b3)), "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0329        : "cc")
0330
0331 # define add_sssssssaaaaaaaaaaaaaa(s6, s5, s4, s3, s2, s1, s0, a6, a5, a4, a3, a2, a1, a0, b6, b5, b4, b3, b2, b1, b0) \
0332   __asm__ ("adds %6,%13,%20\n\tadcs %5,%12,%19\n\tadcs %4,%11,%18\n\tadcs %3,%10,%17\n\tadcs %2,%9,%16\n\tadcs %1,%8,%15\n\tadc %0,%7,%14"\
0333        : "=r" (s6), "=&r" (s5), "=&r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0334        : "r" ((ulong)(a6)), "r" ((ulong)(a5)), "r" ((ulong)(a4)), "r" ((ulong)(a3)), "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0335          "r" ((ulong)(b6)), "r" ((ulong)(b5)), "r" ((ulong)(b4)), "r" ((ulong)(b3)), "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0336        : "cc")
0337
0338 # define sub_dddddddmmmmmmmsssssss(s6, s5, s4, s3, s2, s1, s0, a6, a5, a4, a3, a2, a1, a0, b6, b5, b4, b3, b2, b1, b0) \
0339   __asm__ ("subs %6,%13,%20\n\tsbcs %5,%12,%19\n\tsbcs %4,%11,%18\n\tsbcs %3,%10,%17\n\tsbcs %2,%9,%16\n\tsbcs %1,%8,%15\n\tsbc %0,%7,%14"\
0340        : "=r" (s6), "=&r" (s5), "=&r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0341        : "r" ((ulong)(a6)), "r" ((ulong)(a5)), "r" ((ulong)(a4)), "r" ((ulong)(a3)), "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0342          "r" ((ulong)(b6)), "r" ((ulong)(b5)), "r" ((ulong)(b4)), "r" ((ulong)(b3)), "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0343        : "cc")
0344
0345 # define add_ssssssssaaaaaaaaaaaaaaaa(s7, s6, s5, s4, s3, s2, s1, s0, a7, a6, a5, a4, a3, a2, a1, a0, b7, b6, b5, b4, b3, b2, b1, b0) \
0346   __asm__ ("adds %7,%15,%23\n\tadcs %6,%14,%22\n\tadcs %5,%13,%21\n\tadcs %4,%12,%20\n\tadcs %3,%11,%19\n\tadcs %2,%10,%18\n\tadcs %1,%9,%17\n\tadc %0,%8,%16"\
0347        : "=r" (s7), "=&r" (s6), "=&r" (s5), "=&r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0348        : "r" ((ulong)(a7)), "r" ((ulong)(a6)), "r" ((ulong)(a5)), "r" ((ulong)(a4)), "r" ((ulong)(a3)), "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0349          "r" ((ulong)(b7)), "r" ((ulong)(b6)), "r" ((ulong)(b5)), "r" ((ulong)(b4)), "r" ((ulong)(b3)), "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0350        : "cc")
0351
0352 # define sub_ddddddddmmmmmmmmssssssss(s7, s6, s5, s4, s3, s2, s1, s0, a7, a6, a5, a4, a3, a2, a1, a0, b7, b6, b5, b4, b3, b2, b1, b0) \
0353   __asm__ ("subs %7,%15,%23\n\tsbcs %6,%14,%22\n\tsbcs %5,%13,%21\n\tsbcs %4,%12,%20\n\tsbcs %3,%11,%19\n\tsbcs %2,%10,%18\n\tsbcs %1,%9,%17\n\tsbc %0,%8,%16"\
0354        : "=r" (s7), "=&r" (s6), "=&r" (s5), "=&r" (s4), "=&r" (s3), "=&r" (s2), "=&r" (s1), "=&r" (s0) \
0355        : "r" ((ulong)(a7)), "r" ((ulong)(a6)), "r" ((ulong)(a5)), "r" ((ulong)(a4)), "r" ((ulong)(a3)), "r" ((ulong)(a2)), "r" ((ulong)(a1)), "r" ((ulong)(a0)), \
0356          "r" ((ulong)(b7)), "r" ((ulong)(b6)), "r" ((ulong)(b5)), "r" ((ulong)(b4)), "r" ((ulong)(b3)), "r" ((ulong)(b2)), "r" ((ulong)(b1)), "rI" ((ulong)(b0)) \
0357        : "cc")
0358 # endif
0359
0360 # if defined(__arm__)
0361 #  define umul_ppmm(xh, xl, a, b) \
0362   __asm__("umull %0,%1,%2,%3" \
0363     : "=&r" (xl), "=&r" (xh) \
0364     : "r" (a), "r" (b))
0365
0366 #  define smul_ppmm(xh, xl, a, b) \
0367   __asm__("smull %0,%1,%2,%3" \
0368     : "=&r" (xl), "=&r" (xh) \
0369     : "r" (a), "r" (b))
0370 # else
0371 #  define umul_ppmm(xh, xl, a, b) \
0372   __asm__("mul %0,%2,%3\n" \
0373       "umulh %1,%2,%3" \
0374     : "=&r" (xl), "=&r" (xh) \
0375     : "r" (a), "r" (b))
0376
0377 #  define smul_ppmm(xh, xl, a, b) \
0378   __asm__("mul %0,%2,%3\n" \
0379       "smulh %1,%2,%3" \
0380     : "=&r" (xl), "=&r" (xh) \
0381     : "r" (a), "r" (b))
0382 # endif
0383
0384 #endif
0385
0386 #endif