include/flint/mpn_extras.h

0001 /*
0002     Copyright (C) 2010 Fredrik Johansson
0003
0004     2x2 mul code taken from MPFR 2.3.0
0005     (Copyright (C) 1991-2007 Free Software Foundation, Inc.)
0006
0007     This file is part of FLINT.
0008
0009     FLINT is free software: you can redistribute it and/or modify it under
0010     the terms of the GNU Lesser General Public License (LGPL) as published
0011     by the Free Software Foundation; either version 3 of the License, or
0012     (at your option) any later version.  See <https://www.gnu.org/licenses/>.
0013 */
0014
0015 #ifndef MPN_EXTRAS_H
0016 #define MPN_EXTRAS_H
0017
0018 #ifdef MPN_EXTRAS_INLINES_C
0019 #define MPN_EXTRAS_INLINE
0020 #else
0021 #define MPN_EXTRAS_INLINE static inline
0022 #endif
0023
0024 #include <gmp.h>
0025 #include "longlong.h"
0026 #include "ulong_extras.h"
0027
0028 #ifdef __cplusplus
0029 extern "C" {
0030 #endif
0031
0032 /* mpn macros ****************************************************************/
0033
0034 FLINT_FORCE_INLINE
0035 void flint_mpn_zero(mp_ptr xp, mp_size_t n)
0036 {
0037     mp_size_t ix;
0038     for (ix = 0; ix < n; ix++)
0039         xp[ix] = UWORD(0);
0040 }
0041
0042 FLINT_FORCE_INLINE
0043 void flint_mpn_copyi(mp_ptr xp, mp_srcptr yp, mp_size_t n)
0044 {
0045     mp_size_t ix;
0046     for (ix = 0; ix < n; ix++)
0047         xp[ix] = yp[ix];
0048 }
0049
0050 FLINT_FORCE_INLINE
0051 void flint_mpn_copyd(mp_ptr xp, mp_srcptr yp, mp_size_t n)
0052 {
0053     mp_size_t ix;
0054     for (ix = n - 1; ix >= 0; ix--)
0055         xp[ix] = yp[ix];
0056 }
0057
0058 FLINT_FORCE_INLINE
0059 void flint_mpn_store(mp_ptr xp, mp_size_t n, mp_limb_t y)
0060 {
0061     mp_size_t ix;
0062     for (ix = 0; ix < n; ix++)
0063         xp[ix] = y;
0064 }
0065
0066 MPN_EXTRAS_INLINE
0067 int flint_mpn_zero_p(mp_srcptr x, mp_size_t xsize)
0068 {
0069     slong i;
0070     for (i = 0; i < xsize; i++)
0071     {
0072         if (x[i])
0073             return 0;
0074     }
0075     return 1;
0076 }
0077
0078 FLINT_FORCE_INLINE
0079 int flint_mpn_equal_p(mp_srcptr x, mp_srcptr y, mp_size_t xsize)
0080 {
0081     slong i;
0082     for (i = 0; i < xsize; i++)
0083     {
0084         if (x[i] != y[i])
0085             return 0;
0086     }
0087     return 1;
0088 }
0089
0090 FLINT_FORCE_INLINE void
0091 flint_mpn_negmod_n(mp_ptr res, mp_srcptr x, mp_srcptr m, mp_size_t n)
0092 {
0093     if (flint_mpn_zero_p(x, n))
0094         flint_mpn_zero(res, n);
0095     else
0096         mpn_sub_n(res, m, x, n);
0097 }
0098
0099 FLINT_FORCE_INLINE void
0100 flint_mpn_addmod_n(mp_ptr res, mp_srcptr x, mp_srcptr y, mp_srcptr m, mp_size_t n)
0101 {
0102     mp_limb_t cy;
0103     cy = mpn_add_n(res, x, y, n);
0104     if (cy || mpn_cmp(res, m, n) >= 0)
0105         mpn_sub_n(res, res, m, n);
0106 }
0107
0108 FLINT_FORCE_INLINE void
0109 flint_mpn_submod_n(mp_ptr res, mp_srcptr x, mp_srcptr y, mp_srcptr m, mp_size_t n)
0110 {
0111     int cmp = (mpn_cmp(x, y, n) < 0);
0112     mpn_sub_n(res, x, y, n);
0113     if (cmp)
0114         mpn_add_n(res, res, m, n);
0115 }
0116
0117 /* assumes yn <= n and y < m */
0118 FLINT_FORCE_INLINE void
0119 flint_mpn_addmod_n_m(mp_ptr res, mp_srcptr x, mp_srcptr y, mp_size_t yn, mp_srcptr m, mp_size_t n)
0120 {
0121     mp_limb_t cy;
0122     cy = mpn_add(res, x, n, y, yn);
0123     if (cy || mpn_cmp(res, m, n) >= 0)
0124         mpn_sub_n(res, res, m, n);
0125 }
0126
0127 /* assumes yn <= n and y < m */
0128 FLINT_FORCE_INLINE void
0129 flint_mpn_submod_n_m(mp_ptr res, mp_srcptr x, mp_srcptr y, mp_size_t yn, mp_srcptr m, mp_size_t n)
0130 {
0131     int cmp = (flint_mpn_zero_p(x + yn, n - yn) && mpn_cmp(x, y, yn) < 0);
0132     mpn_sub(res, x, n, y, yn);
0133     if (cmp)
0134         mpn_add_n(res, res, m, n);
0135 }
0136
0137 FLINT_FORCE_INLINE void
0138 flint_mpn_negmod_2(mp_ptr res, mp_srcptr x, mp_srcptr m)
0139 {
0140     if (x[0] == 0 && x[1] == 0)
0141         res[1] = res[0] = 0;
0142     else
0143         sub_ddmmss(res[1], res[0], m[1], m[0], x[1], x[0]);
0144 }
0145
0146 FLINT_FORCE_INLINE void
0147 flint_mpn_addmod_2(mp_ptr res, mp_srcptr x, mp_srcptr y, mp_srcptr m)
0148 {
0149     mp_limb_t cy;
0150     mp_limb_t m1 = m[1], m0 = m[0];
0151     add_sssaaaaaa(cy, res[1], res[0], 0, x[1], x[0], 0, y[1], y[0]);
0152     if (cy || (res[1] > m1 || (res[1] == m1 && res[0] >= m0)))
0153         sub_ddmmss(res[1], res[0], res[1], res[0], m1, m0);
0154 }
0155
0156 /* assumes msb of m is zero */
0157 FLINT_FORCE_INLINE void
0158 _flint_mpn_addmod_2(mp_ptr res, mp_srcptr x, mp_srcptr y, mp_srcptr m)
0159 {
0160     mp_limb_t m1 = m[1], m0 = m[0];
0161     add_ssaaaa(res[1], res[0], x[1], x[0], y[1], y[0]);
0162     if (res[1] > m1 || (res[1] == m1 && res[0] >= m0))
0163         sub_ddmmss(res[1], res[0], res[1], res[0], m1, m0);
0164 }
0165
0166 FLINT_FORCE_INLINE void
0167 flint_mpn_submod_2(mp_ptr res, mp_srcptr x, mp_srcptr y, mp_srcptr m)
0168 {
0169     int cmp;
0170     mp_limb_t m1 = m[1], m0 = m[0];
0171     mp_limb_t x1 = x[1], x0 = x[0];
0172     mp_limb_t y1 = y[1], y0 = y[0];
0173     cmp = (x1 < y1) || (x1 == y1 && x0 < y0);
0174     sub_ddmmss(res[1], res[0], x1, x0, y1, y0);
0175     if (cmp)
0176         add_ssaaaa(res[1], res[0], res[1], res[0], m1, m0);
0177 }
0178
0179 FLINT_FORCE_INLINE int
0180 flint_mpn_signed_sub_n(mp_ptr res, mp_srcptr x, mp_srcptr y, mp_size_t n)
0181 {
0182     if (mpn_cmp(x, y, n) >= 0)
0183     {
0184         mpn_sub_n(res, x, y, n);
0185         return 0;
0186     }
0187     else
0188     {
0189         mpn_sub_n(res, y, x, n);
0190         return 1;
0191     }
0192 }
0193
0194 /* add without carry in or carry out */
0195 #define NN_ADD_2(r, u, v) add_ssaaaa((r)[1], (r)[0], (u)[1], (u)[0], (v)[1], (v)[0])
0196 #define NN_ADD_3(r, u, v) add_sssaaaaaa((r)[2], (r)[1], (r)[0], (u)[2], (u)[1], (u)[0], (v)[2], (v)[1], (v)[0])
0197 #define NN_ADD_4(r, u, v) add_ssssaaaaaaaa((r)[3], (r)[2], (r)[1], (r)[0], (u)[3], (u)[2], (u)[1], (u)[0], (v)[3], (v)[2], (v)[1], (v)[0])
0198 #define NN_ADD_5(r, u, v) add_sssssaaaaaaaaaa((r)[4], (r)[3], (r)[2], (r)[1], (r)[0], (u)[4], (u)[3], (u)[2], (u)[1], (u)[0], (v)[4], (v)[3], (v)[2], (v)[1], (v)[0])
0199 #define NN_ADD_6(r, u, v) add_ssssssaaaaaaaaaaaa((r)[5], (r)[4], (r)[3], (r)[2], (r)[1], (r)[0], (u)[5], (u)[4], (u)[3], (u)[2], (u)[1], (u)[0], (v)[5], (v)[4], (v)[3], (v)[2], (v)[1], (v)[0])
0200 #define NN_ADD_7(r, u, v) add_sssssssaaaaaaaaaaaaaa((r)[6], (r)[5], (r)[4], (r)[3], (r)[2], (r)[1], (r)[0], (u)[6], (u)[5], (u)[4], (u)[3], (u)[2], (u)[1], (u)[0], (v)[6], (v)[5], (v)[4], (v)[3], (v)[2], (v)[1], (v)[0])
0201 #define NN_ADD_8(r, u, v) add_ssssssssaaaaaaaaaaaaaaaa((r)[7], (r)[6], (r)[5], (r)[4], (r)[3], (r)[2], (r)[1], (r)[0], (u)[7], (u)[6], (u)[5], (u)[4], (u)[3], (u)[2], (u)[1], (u)[0], (v)[7], (v)[6], (v)[5], (v)[4], (v)[3], (v)[2], (v)[1], (v)[0])
0202
0203 #define NN_SUB_2(r, u, v) sub_ddmmss((r)[1], (r)[0], (u)[1], (u)[0], (v)[1], (v)[0])
0204 #define NN_SUB_3(r, u, v) sub_dddmmmsss((r)[2], (r)[1], (r)[0], (u)[2], (u)[1], (u)[0], (v)[2], (v)[1], (v)[0])
0205 #define NN_SUB_4(r, u, v) sub_ddddmmmmssss((r)[3], (r)[2], (r)[1], (r)[0], (u)[3], (u)[2], (u)[1], (u)[0], (v)[3], (v)[2], (v)[1], (v)[0])
0206 #define NN_SUB_5(r, u, v) sub_dddddmmmmmsssss((r)[4], (r)[3], (r)[2], (r)[1], (r)[0], (u)[4], (u)[3], (u)[2], (u)[1], (u)[0], (v)[4], (v)[3], (v)[2], (v)[1], (v)[0])
0207 #define NN_SUB_6(r, u, v) sub_ddddddmmmmmmssssss((r)[5], (r)[4], (r)[3], (r)[2], (r)[1], (r)[0], (u)[5], (u)[4], (u)[3], (u)[2], (u)[1], (u)[0], (v)[5], (v)[4], (v)[3], (v)[2], (v)[1], (v)[0])
0208 #define NN_SUB_7(r, u, v) sub_dddddddmmmmmmmsssssss((r)[6], (r)[5], (r)[4], (r)[3], (r)[2], (r)[1], (r)[0], (u)[6], (u)[5], (u)[4], (u)[3], (u)[2], (u)[1], (u)[0], (v)[6], (v)[5], (v)[4], (v)[3], (v)[2], (v)[1], (v)[0])
0209 #define NN_SUB_8(r, u, v) sub_ddddddddmmmmmmmmssssssss((r)[7], (r)[6], (r)[5], (r)[4], (r)[3], (r)[2], (r)[1], (r)[0], (u)[7], (u)[6], (u)[5], (u)[4], (u)[3], (u)[2], (u)[1], (u)[0], (v)[7], (v)[6], (v)[5], (v)[4], (v)[3], (v)[2], (v)[1], (v)[0])
0210
0211 #define DEF_SIGNED_SUB(n) \
0212 FLINT_FORCE_INLINE int \
0213 flint_mpn_signed_sub_ ## n(mp_ptr res, mp_srcptr x, mp_srcptr y) \
0214 { \
0215     if (mpn_cmp(x, y, n) >= 0) \
0216     { \
0217         NN_SUB_ ## n(res, x, y); \
0218         return 0; \
0219     } \
0220     else \
0221     { \
0222         NN_SUB_ ## n(res, y, x); \
0223         return 1; \
0224     } \
0225 }
0226
0227 DEF_SIGNED_SUB(2)
0228 DEF_SIGNED_SUB(3)
0229 DEF_SIGNED_SUB(4)
0230 DEF_SIGNED_SUB(5)
0231 DEF_SIGNED_SUB(6)
0232 DEF_SIGNED_SUB(7)
0233 DEF_SIGNED_SUB(8)
0234
0235 FLINT_FORCE_INLINE void
0236 flint_mpn_signed_div2(mp_ptr res, mp_srcptr x, mp_size_t n)
0237 {
0238     mp_limb_t s = x[n - 1] & (UWORD(1) << (FLINT_BITS - 1));
0239     mpn_rshift(res, x, n, 1);
0240     res[n - 1] |= s;
0241 }
0242
0243 void flint_mpn_mulmod_preinvn_2(mp_ptr r,
0244         mp_srcptr a, mp_srcptr b,
0245         mp_srcptr d, mp_srcptr dinv, ulong norm);
0246
0247 char * flint_mpn_get_str(char * res, int base, mp_srcptr x, mp_size_t xn, int negative);
0248
0249
0250 #define MPN_NORM(a, an)                         \
0251     do {                                        \
0252         while ((an) != 0 && (a)[(an) - 1] == 0) \
0253            (an)--;                              \
0254     } while (0)
0255
0256 #define MPN_SWAP(a, an, b, bn) \
0257   do { \
0258     FLINT_SWAP(mp_ptr, a, b); \
0259     FLINT_SWAP(mp_size_t, an, bn); \
0260   } while (0)
0261
0262 #define BITS_TO_LIMBS(b) (((b) + GMP_NUMB_BITS - 1) / GMP_NUMB_BITS)
0263
0264 /* mpn macros for few-limb arithmetic ****************************************/
0265
0266 #define MPN_LEFT_SHIFT_HI(hi, lo, n) \
0267     ((n) > 0 ? (((hi) << (n)) | ((lo) >> (GMP_LIMB_BITS - (n)))) : (hi))
0268
0269 #define MPN_RIGHT_SHIFT_LOW(hi, lo, n) \
0270     ((n) > 0 ? (((lo) >> (n)) | ((hi) << (GMP_LIMB_BITS - (n)))) : (lo))
0271
0272 #define FLINT_MPN_MUL_2X1(r2, r1, r0, a1, a0, b0)           \
0273     do {                                                    \
0274         mp_limb_t t1;                                       \
0275         umul_ppmm(r1, r0, a0, b0);                          \
0276         umul_ppmm(r2, t1, a1, b0);                          \
0277         add_ssaaaa(r2, r1, r2, r1, 0, t1);                  \
0278     } while (0)
0279
0280 #define FLINT_MPN_MUL_2X2(r3, r2, r1, r0, a1, a0, b1, b0)   \
0281     do {                                                                  \
0282         mp_limb_t __v1, __v2, __u1, __u2;                                 \
0283         mp_limb_t __r3, __r2, __r1, __r0;                                 \
0284         mp_limb_t __a1 = (a1), __a0 = (a0), __b1 = (b1), __b0 = (b0);     \
0285         umul_ppmm(__r1, __r0, __a0, __b0);                                \
0286         umul_ppmm(__r3, __r2, __a1, __b1);                                \
0287         umul_ppmm(__v2, __v1, __a0, __b1);                                \
0288         add_sssaaaaaa(__r3, __r2, __r1, __r3, __r2, __r1, 0, __v2, __v1); \
0289         umul_ppmm(__u2, __u1, __a1, __b0);                                \
0290         add_sssaaaaaa(__r3, __r2, __r1, __r3, __r2, __r1, 0, __u2, __u1); \
0291         (r0) = __r0; (r1) = __r1; (r2) = __r2; (r3) = __r3;               \
0292     } while (0)
0293
0294 /* Low two words of 2x2 product */
0295 #define FLINT_MPN_MULLOW_2X2(r1, r0, a1, a0, b1, b0) \
0296     do { \
0297         umul_ppmm(r1, r0, a0, b0); \
0298         (r1) += (a0) * (b1) + (a1) * (b0); \
0299     } while (0)
0300
0301 /* Low three words of 2x2 product */
0302 #define FLINT_MPN_MUL_3P2X2(r2, r1, r0, a1, a0, b1, b0)                \
0303     do {                                                               \
0304         mp_limb_t __t1, __t2, __u1, __u2;                              \
0305         mp_limb_t __r2, __r1, __r0;                                    \
0306         mp_limb_t __a1 = (a1), __a0 = (a0), __b1 = (b1), __b0 = (b0);  \
0307         umul_ppmm(__r1, __r0, __a0, __b0);                             \
0308         __r2 = __a1 * __b1;                                            \
0309         umul_ppmm(__t2, __t1, __a0, __b1);                             \
0310         add_ssaaaa(__r2, __r1, __r2, __r1, __t2, __t1);                \
0311         umul_ppmm(__u2, __u1, __a1, __b0);                             \
0312         add_ssaaaa(__r2, __r1, __r2, __r1, __u2, __u1);                \
0313         (r0) = __r0; (r1) = __r1; (r2) = __r2;                         \
0314     } while (0)
0315
0316 #define FLINT_MPN_SQR_2X2(r3, r2, r1, r0, a1, a0)   \
0317     do {                                                                     \
0318         mp_limb_t __u1, __u2, __u3;                                          \
0319         mp_limb_t __r3, __r2, __r1, __r0;                                    \
0320         mp_limb_t __a1 = (a1), __a0 = (a0);                                  \
0321         umul_ppmm(__u2, __u1, __a0, __a1);                                   \
0322         add_sssaaaaaa(__u3, __u2, __u1, 0, __u2, __u1, 0, __u2, __u1);       \
0323         umul_ppmm(__r1, __r0, __a0, __a0);                                   \
0324         umul_ppmm(__r3, __r2, __a1, __a1);                                   \
0325         add_sssaaaaaa(__r3, __r2, __r1, __r3, __r2, __r1, __u3, __u2, __u1); \
0326         (r0) = __r0; (r1) = __r1; (r2) = __r2; (r3) = __r3;                  \
0327     } while (0)
0328
0329
0330 /* {s0,s1,s2} = u[0]v[n-1] + u[1]v[n-2] + ... */
0331 /* Assumes n >= 2 */
0332 #define NN_DOTREV_S3_1X1(s2, s1, s0, u, v, n) \
0333     do { \
0334         mp_limb_t __dt0, __dt1, __ds0, __ds1, __ds2; \
0335         slong __i; \
0336         FLINT_ASSERT((n) >= 2); \
0337         umul_ppmm(__ds1, __ds0, (u)[0], (v)[(n) - 1]); \
0338         umul_ppmm(__dt1, __dt0, (u)[1], (v)[(n) - 2]); \
0339         add_sssaaaaaa(__ds2, __ds1, __ds0, 0, __ds1, __ds0, 0, __dt1, __dt0); \
0340         for (__i = 2; __i < (n); __i++) \
0341         { \
0342             umul_ppmm(__dt1, __dt0, (u)[__i], (v)[(n) - 1 - __i]); \
0343             add_sssaaaaaa(__ds2, __ds1, __ds0, __ds2, __ds1, __ds0, 0, __dt1, __dt0); \
0344         } \
0345         (s0) = __ds0; (s1) = __ds1; (s2) = __ds2; \
0346     } while (0) \
0347
0348 /* Like NN_DOTREV_S3_1X1 but summing only over the high parts of the products. */
0349 #define NN_DOTREV_S3_1X1_HIGH(s2, s1, u, v, n) \
0350     do { \
0351         mp_limb_t __dt1, __ds1, __ds2; \
0352         slong __i; \
0353         FLINT_ASSERT((n) >= 2); \
0354         __ds1 = n_mulhi((u)[0], (v)[(n) - 1]); \
0355         __dt1 = n_mulhi((u)[1], (v)[(n) - 2]); \
0356         add_ssaaaa(__ds2, __ds1, 0, __ds1, 0, __dt1); \
0357         for (__i = 2; __i < (n); __i++) \
0358         { \
0359             __dt1 = n_mulhi((u)[__i], (v)[(n) - 1 - __i]); \
0360             add_ssaaaa(__ds2, __ds1, __ds2, __ds1, 0, __dt1); \
0361         } \
0362         (s1) = __ds1; (s2) = __ds2; \
0363     } while (0) \
0364
0365 /* {r0,r1,r2} = {s0,s1,s2} + u[0]v[n-1] + u[1]v[n-2] + ... */
0366 /* Assumes n >= 1. May have s2 != 0, but the final sum is assumed to fit in 3 limbs. */
0367 #define NN_DOTREV_S3_A3_1X1(r2, r1, r0, s2, s1, s0, u, v, n) \
0368     do { \
0369         mp_limb_t __dt0, __dt1, __ds0, __ds1, __ds2; \
0370         slong __i; \
0371         FLINT_ASSERT((n) >= 1); \
0372         __ds0 = (s0); __ds1 = (s1); __ds2 = (s2); \
0373         for (__i = 0; __i < (n); __i++) \
0374         { \
0375             umul_ppmm(__dt1, __dt0, (u)[__i], (v)[(n) - 1 - __i]); \
0376             add_sssaaaaaa(__ds2, __ds1, __ds0, __ds2, __ds1, __ds0, 0, __dt1, __dt0); \
0377         } \
0378         (r0) = __ds0; (r1) = __ds1; (r2) = __ds2; \
0379     } while (0) \
0380
0381 #define NN_MUL_1X1 umul_ppmm
0382
0383 /* {r0,r1} = {s0,s1} + x * y, with no carry-out. */
0384 #define NN_ADDMUL_S2_A2_1X1(r1, r0, s1, s0, x, y) \
0385     do { \
0386         mp_limb_t __dt0, __dt1; \
0387         umul_ppmm(__dt1, __dt0, (x), (y)); \
0388         add_ssaaaa(r1, r0, s1, s0, __dt1, __dt0); \
0389     } while (0); \
0390
0391 #define flint_mpn_divrem21_preinv(q, a_hi, a_lo, dinv) \
0392    do { \
0393       mp_limb_t __q2, __q3; \
0394       umul_ppmm((q), __q2, (a_hi), (dinv)); \
0395       __q3 = n_mulhi((a_lo), (dinv)); \
0396       add_ssaaaa((q), __q2, (q), __q2, 0, __q3); \
0397       add_ssaaaa((q), __q2, (q), __q2, (a_hi), (a_lo)); \
0398    } while (0)
0399
0400 /* addition ******************************************************************/
0401
0402 #if FLINT_HAVE_ASSEMBLY_x86_64_adx
0403 /* Simultaneously adds two n-limbed integers onto result and returns carry. */
0404 /* NOTE: Requires n >= 4 */
0405 # define FLINT_HAVE_NATIVE_mpn_2add_n_inplace 1
0406 mp_limb_t flint_mpn_2add_n_inplace(mp_ptr, mp_srcptr, mp_srcptr, mp_size_t);
0407 #endif
0408
0409 #if FLINT_HAVE_NATIVE_mpn_add_nc
0410 # define mpn_add_nc __gmpn_add_nc
0411 mp_limb_t mpn_add_nc(mp_ptr, mp_srcptr, mp_srcptr, mp_size_t, mp_limb_t);
0412 #else
0413 FLINT_FORCE_INLINE mp_limb_t
0414 mpn_add_nc(mp_ptr rp, mp_srcptr up, mp_srcptr vp, mp_size_t n, mp_limb_t ci)
0415 {
0416     mp_limb_t co;
0417     co = mpn_add_n(rp, up, vp, n);
0418     co += mpn_add_1(rp, rp, n, ci);
0419     return co;
0420 }
0421 #endif
0422
0423 #if FLINT_HAVE_NATIVE_mpn_sub_nc
0424 # define mpn_sub_nc __gmpn_sub_nc
0425 mp_limb_t mpn_sub_nc(mp_ptr, mp_srcptr, mp_srcptr, mp_size_t, mp_limb_t);
0426 #else
0427 FLINT_FORCE_INLINE mp_limb_t
0428 mpn_sub_nc(mp_ptr rp, mp_srcptr up, mp_srcptr vp, mp_size_t n, mp_limb_t ci)
0429 {
0430     mp_limb_t co;
0431     co = mpn_sub_n(rp, up, vp, n);
0432     co += mpn_sub_1(rp, rp, n, ci);
0433     return co;
0434 }
0435 #endif
0436
0437 #if FLINT_HAVE_NATIVE_mpn_add_n_sub_n
0438 /* mpn_add_n_sub_n basically only exists for IA64 and certain PowerPC and s390
0439  * systems. We will assume that a native one does not exist. */
0440 # undef FLINT_HAVE_NATIVE_mpn_add_n_sub_n
0441 # define FLINT_HAVE_NATIVE_mpn_add_n_sub_n 0
0442 #endif
0443
0444 mp_limb_t flint_mpn_sumdiff_n(mp_ptr s, mp_ptr d, mp_srcptr x, mp_srcptr y, mp_size_t n);
0445
0446 /* shifting ******************************************************************/
0447
0448 #if FLINT_HAVE_NATIVE_mpn_addlsh1_n
0449 # define mpn_addlsh1_n __gmpn_addlsh1_n
0450 mp_limb_t mpn_addlsh1_n(mp_ptr, mp_srcptr, mp_srcptr, mp_size_t);
0451 #endif
0452
0453 #if FLINT_HAVE_NATIVE_mpn_addlsh1_n_ip1
0454 # define mpn_addlsh1_n_ip1 __gmpn_addlsh1_n_ip1
0455 mp_limb_t mpn_addlsh1_n_ip1(mp_ptr, mp_srcptr, mp_srcptr, mp_size_t);
0456 #elif FLINT_HAVE_NATIVE_mpn_addlsh1_n
0457 # define mpn_addlsh1_n_ip1(a,b,n) mpn_addlsh1_n(a,a,b,n)
0458 # define FLINT_HAVE_NATIVE_mpn_addlsh1_n_ip1 2
0459 #endif
0460
0461 #if FLINT_HAVE_NATIVE_mpn_rsh1add_n
0462 # define mpn_rsh1add_n __gmpn_rsh1add_n
0463 mp_limb_t mpn_rsh1add_n(mp_ptr, mp_srcptr, mp_srcptr, mp_size_t);
0464 #endif
0465
0466 #if FLINT_HAVE_NATIVE_mpn_rsh1sub_n
0467 # define mpn_rsh1sub_n __gmpn_rsh1sub_n
0468 mp_limb_t mpn_rsh1sub_n(mp_ptr, mp_srcptr, mp_srcptr, mp_size_t);
0469 #endif
0470
0471 /* multiplication (general) **************************************************/
0472
0473 #if FLINT_HAVE_ASSEMBLY_x86_64_adx
0474 # define FLINT_MPN_MUL_FUNC_TAB_WIDTH 17
0475 # define FLINT_MPN_SQR_FUNC_TAB_WIDTH 14
0476
0477 # define FLINT_HAVE_MUL_FUNC(n, m) ((n) <= 16)
0478 # define FLINT_HAVE_MUL_N_FUNC(n) ((n) <= 16)
0479 # define FLINT_HAVE_SQR_FUNC(n) ((n) <= FLINT_MPN_SQR_FUNC_TAB_WIDTH)
0480
0481 # define FLINT_MPN_MUL_HARD(rp, xp, xn, yp, yn) (flint_mpn_mul_func_tab[xn][yn](rp, xp, yp))
0482 # define FLINT_MPN_MUL_N_HARD(rp, xp, yp, n) (flint_mpn_mul_n_func_tab[n](rp, xp, yp))
0483 # define FLINT_MPN_SQR_HARD(rp, xp, n) (flint_mpn_sqr_func_tab[n](rp, xp))
0484 #elif FLINT_HAVE_ASSEMBLY_armv8
0485 # define FLINT_MPN_MUL_FUNC_N_TAB_WIDTH 15
0486 # define FLINT_MPN_SQR_FUNC_TAB_WIDTH 9
0487
0488 # define FLINT_HAVE_MUL_FUNC(n, m) FLINT_HAVE_MUL_N_FUNC(n)
0489 # define FLINT_HAVE_MUL_N_FUNC(n) ((n) <= FLINT_MPN_MUL_FUNC_N_TAB_WIDTH)
0490 # define FLINT_HAVE_SQR_FUNC(n) ((n) <= FLINT_MPN_SQR_FUNC_TAB_WIDTH)
0491
0492 # define FLINT_MPN_MUL_HARD(rp, xp, xn, yp, yn) (flint_mpn_mul_func_n_tab[xn](rp, xp, yp, yn))
0493 # define FLINT_MPN_MUL_N_HARD(rp, xp, yp, n) (flint_mpn_mul_func_n_tab[n](rp, xp, yp, n))
0494 # define FLINT_MPN_SQR_HARD(rp, xp, n) (flint_mpn_sqr_func_tab[n](rp, xp))
0495
0496 # define FLINT_HAVE_NATIVE_mpn_mul_2 1
0497 mp_limb_t flint_mpn_mul_2(mp_ptr, mp_srcptr, mp_size_t, mp_srcptr);
0498 #else
0499 # define FLINT_MPN_MUL_FUNC_TAB_WIDTH 8
0500 # define FLINT_MPN_SQR_FUNC_TAB_WIDTH 0
0501
0502 # define FLINT_HAVE_MUL_FUNC(n, m) ((n) <= 7 || ((n) <= 14 && (m) == 1))
0503 # define FLINT_HAVE_MUL_N_FUNC(n) ((n) <= 7)
0504 # define FLINT_HAVE_SQR_FUNC(n) (0)
0505
0506 # define FLINT_MPN_MUL_HARD(rp, xp, xn, yp, yn) (flint_mpn_mul_func_tab[xn][yn](rp, xp, yp))
0507 # define FLINT_MPN_MUL_N_HARD(rp, xp, yp, n) (flint_mpn_mul_n_func_tab[n](rp, xp, yp))
0508 # define FLINT_MPN_SQR_HARD(rp, xp, n) (flint_mpn_sqr_func_tab[n](rp, xp))
0509 #endif
0510
0511 #define FLINT_MUL_USE_FUNC_TAB 1
0512
0513 typedef mp_limb_t (* flint_mpn_mul_func_t)(mp_ptr, mp_srcptr, mp_srcptr);
0514 typedef mp_limb_t (* flint_mpn_mul_func_n_t)(mp_ptr, mp_srcptr, mp_srcptr, mp_size_t);
0515 typedef mp_limb_t (* flint_mpn_sqr_func_t)(mp_ptr, mp_srcptr);
0516
0517 #ifdef FLINT_MPN_MUL_FUNC_N_TAB_WIDTH
0518 FLINT_DLL extern const flint_mpn_mul_func_n_t flint_mpn_mul_func_n_tab[];
0519 #else
0520 FLINT_DLL extern const flint_mpn_mul_func_t flint_mpn_mul_func_tab[][FLINT_MPN_MUL_FUNC_TAB_WIDTH];
0521 FLINT_DLL extern const flint_mpn_mul_func_t flint_mpn_mul_n_func_tab[];
0522 #endif
0523
0524 FLINT_DLL extern const flint_mpn_sqr_func_t flint_mpn_sqr_func_tab[];
0525
0526 void flint_mpn_mul_toom22(mp_ptr pp, mp_srcptr ap, mp_size_t an, mp_srcptr bp, mp_size_t bn, mp_ptr scratch);
0527 void flint_mpn_mul_toom32(mp_ptr pp, mp_srcptr ap, mp_size_t an, mp_srcptr bp, mp_size_t bn, mp_ptr scratch);
0528
0529 mp_limb_t _flint_mpn_mul(mp_ptr r, mp_srcptr x, mp_size_t xn, mp_srcptr y, mp_size_t yn);
0530 void _flint_mpn_mul_n(mp_ptr r, mp_srcptr x, mp_srcptr y, mp_size_t n);
0531 mp_limb_t _flint_mpn_sqr(mp_ptr r, mp_srcptr x, mp_size_t n);
0532
0533 MPN_EXTRAS_INLINE mp_limb_t
0534 flint_mpn_mul(mp_ptr r, mp_srcptr x, mp_size_t xn, mp_srcptr y, mp_size_t yn)
0535 {
0536     FLINT_ASSERT(xn >= yn);
0537     FLINT_ASSERT(yn >= 1);
0538     FLINT_ASSERT(r != x);
0539     FLINT_ASSERT(r != y);
0540
0541     if (FLINT_MUL_USE_FUNC_TAB && FLINT_HAVE_MUL_FUNC(xn, yn))
0542         return FLINT_MPN_MUL_HARD(r, x, xn, y, yn);
0543     else
0544         return _flint_mpn_mul(r, x, xn, y, yn);
0545 }
0546
0547 MPN_EXTRAS_INLINE void
0548 flint_mpn_mul_n(mp_ptr r, mp_srcptr x, mp_srcptr y, mp_size_t n)
0549 {
0550     FLINT_ASSERT(n >= 1);
0551     FLINT_ASSERT(r != x);
0552     FLINT_ASSERT(r != y);
0553
0554     if (FLINT_MUL_USE_FUNC_TAB && FLINT_HAVE_MUL_N_FUNC(n))
0555         FLINT_MPN_MUL_N_HARD(r, x, y, n);
0556     else
0557         _flint_mpn_mul_n(r, x, y, n);
0558 }
0559
0560 MPN_EXTRAS_INLINE mp_limb_t
0561 flint_mpn_sqr(mp_ptr r, mp_srcptr x, mp_size_t n)
0562 {
0563     FLINT_ASSERT(n >= 1);
0564
0565     if (FLINT_MUL_USE_FUNC_TAB && FLINT_HAVE_SQR_FUNC(n))
0566     {
0567         /* NOTE: Aliasing allowed */
0568         return FLINT_MPN_SQR_HARD(r, x, n);
0569     }
0570     else
0571     {
0572         FLINT_ASSERT(r != x);
0573         return _flint_mpn_sqr(r, x, n);
0574     }
0575 }
0576
0577 /* Like flint_mpn_mul but allow operands in either order, completely
0578    inline some small products, and also check for squaring. */
0579 #define FLINT_MPN_MUL_WITH_SPECIAL_CASES(_z, _x, _xn, _y, _yn) \
0580     if ((_xn) == (_yn)) \
0581     { \
0582         if ((_xn) == 1) \
0583         { \
0584             umul_ppmm((_z)[1], (_z)[0], (_x)[0], (_y)[0]); \
0585         } \
0586         else if ((_xn) == 2) \
0587         { \
0588             mp_limb_t __tt_x1, __tt_x0, __tt_y1, __tt_y0; \
0589             __tt_x0 = (_x)[0]; \
0590             __tt_x1 = (_x)[1]; \
0591             __tt_y0 = (_y)[0]; \
0592             __tt_y1 = (_y)[1]; \
0593             FLINT_MPN_MUL_2X2((_z)[3], (_z)[2], (_z)[1], (_z)[0], __tt_x1, __tt_x0, __tt_y1, __tt_y0); \
0594         } \
0595         else if ((_x) == (_y)) \
0596             flint_mpn_sqr((_z), (_x), (_xn)); \
0597         else \
0598             flint_mpn_mul_n((_z), (_x), (_y), (_xn)); \
0599     } \
0600     else if ((_xn) > (_yn)) \
0601     { \
0602         flint_mpn_mul((_z), (_x), (_xn), (_y), (_yn)); \
0603     } \
0604     else \
0605     { \
0606         flint_mpn_mul((_z), (_y), (_yn), (_x), (_xn)); \
0607     }
0608
0609 /* High and low multiplication *******************************************************/
0610
0611 #define FLINT_HAVE_MULLOW_FUNC(n) ((n) <= FLINT_MPN_MULLOW_FUNC_TAB_WIDTH)
0612 #define FLINT_HAVE_MULHIGH_FUNC(n) ((n) <= FLINT_MPN_MULHIGH_FUNC_TAB_WIDTH)
0613 #define FLINT_HAVE_SQRHIGH_FUNC(n) ((n) <= FLINT_MPN_SQRHIGH_FUNC_TAB_WIDTH)
0614 #define FLINT_HAVE_MULHIGH_NORMALISED_FUNC(n) ((n) <= FLINT_MPN_MULHIGH_NORMALISED_FUNC_TAB_WIDTH)
0615 #define FLINT_HAVE_SQRHIGH_NORMALISED_FUNC(n) ((n) <= FLINT_MPN_SQRHIGH_NORMALISED_FUNC_TAB_WIDTH)
0616
0617 typedef struct { mp_limb_t m1; mp_limb_t m2; } mp_limb_pair_t;
0618 typedef mp_limb_pair_t (* flint_mpn_sqrhigh_normalised_func_t)(mp_ptr, mp_srcptr);
0619 typedef mp_limb_pair_t (* flint_mpn_mulhigh_normalised_func_t)(mp_ptr, mp_srcptr, mp_srcptr);
0620
0621 FLINT_DLL extern const flint_mpn_mul_func_t flint_mpn_mullow_func_tab[];
0622 FLINT_DLL extern const flint_mpn_mul_func_t flint_mpn_mulhigh_func_tab[];
0623 FLINT_DLL extern const flint_mpn_sqr_func_t flint_mpn_sqrhigh_func_tab[];
0624 FLINT_DLL extern const flint_mpn_mulhigh_normalised_func_t flint_mpn_mulhigh_normalised_func_tab[];
0625 FLINT_DLL extern const flint_mpn_sqrhigh_normalised_func_t flint_mpn_sqrhigh_normalised_func_tab[];
0626
0627 #if FLINT_HAVE_ASSEMBLY_x86_64_adx
0628 # define FLINT_MPN_MULLOW_FUNC_TAB_WIDTH 8
0629 # define FLINT_MPN_MULHIGH_FUNC_TAB_WIDTH 13
0630 /* n with best effective cycles/limb (and current largest assembly case) -- used by mulhigh_recursive */
0631 # define FLINT_MPN_MULHIGH_BEST_TAB_N 9
0632 # define FLINT_MPN_SQRHIGH_FUNC_TAB_WIDTH 8
0633 # define FLINT_MPN_MULHIGH_NORMALISED_FUNC_TAB_WIDTH 9
0634 # define FLINT_MPN_SQRHIGH_NORMALISED_FUNC_TAB_WIDTH 8
0635
0636 # define FLINT_HAVE_NATIVE_mpn_mullow_basecase 1
0637 /* NOTE: This function only works for n >= 6 */
0638 # define FLINT_HAVE_NATIVE_mpn_mulhigh_basecase 1
0639 /* NOTE: This function only works for n >= 6 */
0640 # define FLINT_HAVE_NATIVE_mpn_sqrhigh_basecase 1
0641
0642 #elif FLINT_HAVE_ASSEMBLY_armv8
0643 # define FLINT_MPN_MULLOW_FUNC_TAB_WIDTH 0
0644 # define FLINT_MPN_MULHIGH_FUNC_TAB_WIDTH 8
0645 # define FLINT_MPN_MULHIGH_BEST_TAB_N 8
0646 # define FLINT_MPN_SQRHIGH_FUNC_TAB_WIDTH 8
0647 # define FLINT_MPN_MULHIGH_NORMALISED_FUNC_TAB_WIDTH 0
0648 # define FLINT_MPN_SQRHIGH_NORMALISED_FUNC_TAB_WIDTH 0
0649
0650 /* NOTE: This function only works for n > 8 */
0651 # define FLINT_HAVE_NATIVE_mpn_mulhigh_basecase 1
0652
0653 #else
0654 /* TODO: generic hardcoded mullows */
0655 # define FLINT_MPN_MULLOW_FUNC_TAB_WIDTH 0
0656 # define FLINT_MPN_MULHIGH_FUNC_TAB_WIDTH 16
0657 # define FLINT_MPN_MULHIGH_BEST_TAB_N 16
0658 # define FLINT_MPN_SQRHIGH_FUNC_TAB_WIDTH 2
0659 # define FLINT_MPN_MULHIGH_NORMALISED_FUNC_TAB_WIDTH 0
0660 # define FLINT_MPN_SQRHIGH_NORMALISED_FUNC_TAB_WIDTH 0
0661
0662 #endif
0663
0664 /* FIXME: this tuning is for x86_64_adx with fft_small */
0665 /* FIXME: we currently assume that the same parameters are optimal for both mulhigh and mullow */
0666 #define FLINT_MPN_MULLOW_MULDERS_CUTOFF 50
0667 #define FLINT_MPN_MULHIGH_MULDERS_CUTOFF 40
0668 #define FLINT_MPN_MULHIGH_MUL_CUTOFF 2000
0669 #define FLINT_MPN_MULHIGH_K_TAB_SIZE 2048
0670
0671 FLINT_DLL extern const signed short flint_mpn_mulhigh_k_tab[FLINT_MPN_MULHIGH_K_TAB_SIZE];
0672
0673 mp_limb_t flint_mpn_mullow_basecase(mp_ptr res, mp_srcptr u, mp_srcptr v, mp_size_t n);
0674 void _flint_mpn_mullow_n_mulders_recursive(mp_ptr rp, mp_srcptr np, mp_srcptr mp, mp_size_t n);
0675 mp_limb_t _flint_mpn_mullow_n_mulders(mp_ptr res, mp_srcptr u, mp_srcptr v, mp_size_t n);
0676 mp_limb_t _flint_mpn_mullow_n_mul(mp_ptr res, mp_srcptr u, mp_srcptr v, mp_size_t n);
0677 mp_limb_t _flint_mpn_mullow_n(mp_ptr res, mp_srcptr u, mp_srcptr v, mp_size_t n);
0678
0679 mp_limb_t _flint_mpn_mulhigh_basecase(mp_ptr res, mp_srcptr u, mp_srcptr v, mp_size_t n);
0680 void _flint_mpn_mulhigh_n_mulders_recursive(mp_ptr rp, mp_srcptr np, mp_srcptr mp, mp_size_t n);
0681 mp_limb_t _flint_mpn_mulhigh_n_naive(mp_ptr rp, mp_srcptr up, mp_srcptr vp, mp_size_t n);
0682 mp_limb_t _flint_mpn_mulhigh_n_recursive(mp_ptr r, mp_srcptr x, mp_srcptr y, mp_size_t n);
0683 mp_limb_t _flint_mpn_mulhigh_n_mulders(mp_ptr res, mp_srcptr u, mp_srcptr v, mp_size_t n);
0684 mp_limb_t _flint_mpn_mulhigh_n_mul(mp_ptr res, mp_srcptr u, mp_srcptr v, mp_size_t n);
0685 mp_limb_t _flint_mpn_mulhigh_n(mp_ptr res, mp_srcptr u, mp_srcptr v, mp_size_t n);
0686
0687 #if FLINT_HAVE_ASSEMBLY_x86_64_adx
0688 MPN_EXTRAS_INLINE
0689 mp_limb_t _flint_mpn_mulhigh_n_basecase2(mp_ptr rp, mp_srcptr xp, mp_srcptr yp, mp_size_t n)
0690 {
0691     if (n <= 22)
0692         return _flint_mpn_mulhigh_n_recursive(rp, xp, yp, n);
0693     else
0694         return _flint_mpn_mulhigh_basecase(rp, xp, yp, n);
0695 }
0696 #else
0697 #define _flint_mpn_mulhigh_n_basecase2 _flint_mpn_mulhigh_basecase
0698 #endif
0699
0700 MPN_EXTRAS_INLINE
0701 mp_limb_t flint_mpn_mullow_n(mp_ptr rp, mp_srcptr xp, mp_srcptr yp, mp_size_t n)
0702 {
0703     FLINT_ASSERT(n >= 1);
0704     FLINT_ASSERT(rp != xp);
0705
0706     if (FLINT_HAVE_MULLOW_FUNC(n))
0707         return flint_mpn_mullow_func_tab[n](rp, xp, yp);
0708     else
0709         return _flint_mpn_mullow_n(rp, xp, yp, n);
0710 }
0711
0712 MPN_EXTRAS_INLINE
0713 mp_limb_t flint_mpn_mulhigh_n(mp_ptr rp, mp_srcptr xp, mp_srcptr yp, mp_size_t n)
0714 {
0715     FLINT_ASSERT(n >= 1);
0716
0717     if (FLINT_HAVE_MULHIGH_FUNC(n)) /* NOTE: Aliasing allowed here */
0718         return flint_mpn_mulhigh_func_tab[n](rp, xp, yp);
0719     else
0720         return _flint_mpn_mulhigh_n(rp, xp, yp, n);
0721 }
0722
0723 /* We just want the high or low n limbs, but rp has 2n limbs available
0724    which can be used for scratch space or for doing a full multiply
0725    without temporary allocations. TODO: exploit this in the Mulders range
0726    by calling Mulders directly. */
0727 MPN_EXTRAS_INLINE
0728 void flint_mpn_mul_or_mullow_n(mp_ptr rp, mp_srcptr xp, mp_srcptr yp, mp_size_t n)
0729 {
0730     FLINT_ASSERT(n >= 1);
0731
0732     if (FLINT_HAVE_MULLOW_FUNC(n))
0733         rp[n] = flint_mpn_mullow_func_tab[n](rp, xp, yp);
0734     else if (n < FLINT_MPN_MULHIGH_MUL_CUTOFF)
0735         rp[n] = _flint_mpn_mullow_n(rp, xp, yp, n);
0736     else
0737         flint_mpn_mul_n(rp, xp, yp, n);
0738 }
0739
0740 MPN_EXTRAS_INLINE
0741 void flint_mpn_mul_or_mulhigh_n(mp_ptr rp, mp_srcptr xp, mp_srcptr yp, mp_size_t n)
0742 {
0743     FLINT_ASSERT(n >= 1);
0744
0745     if (FLINT_HAVE_MULHIGH_FUNC(n))
0746         rp[n - 1] = flint_mpn_mulhigh_func_tab[n](rp + n, xp, yp);
0747     else if (n < FLINT_MPN_MULHIGH_MUL_CUTOFF)
0748         rp[n - 1] = _flint_mpn_mulhigh_n(rp + n, xp, yp, n);
0749     else
0750         flint_mpn_mul_n(rp, xp, yp, n);
0751 }
0752
0753 #define FLINT_MPN_SQRHIGH_MULDERS_CUTOFF 90
0754 #define FLINT_MPN_SQRHIGH_SQR_CUTOFF 2000
0755 #define FLINT_MPN_SQRHIGH_K_TAB_SIZE 2048
0756
0757 #if FLINT_HAVE_ASSEMBLY_x86_64_adx
0758 mp_limb_t _flint_mpn_sqrhigh_basecase_even(mp_ptr, mp_srcptr, mp_size_t);
0759 mp_limb_t _flint_mpn_sqrhigh_basecase_odd(mp_ptr, mp_srcptr, mp_size_t);
0760
0761 MPN_EXTRAS_INLINE mp_limb_t _flint_mpn_sqrhigh_basecase(mp_ptr rp, mp_srcptr xp, mp_size_t n)
0762 {
0763     FLINT_ASSERT(n >= 1);
0764     FLINT_ASSERT(rp != xp);
0765
0766     if (n & 1)
0767         return _flint_mpn_sqrhigh_basecase_odd(rp, xp, n >> 1);
0768     else
0769         return _flint_mpn_sqrhigh_basecase_even(rp, xp, n >> 1);
0770 }
0771
0772 #else
0773 /* todo */
0774 MPN_EXTRAS_INLINE mp_limb_t _flint_mpn_sqrhigh_basecase(mp_ptr res, mp_srcptr u, mp_size_t n)
0775 {
0776     return _flint_mpn_mulhigh_basecase(res, u, u, n);
0777 }
0778 #endif
0779
0780 void _flint_mpn_sqrhigh_mulders_recursive(mp_ptr rp, mp_srcptr np, mp_size_t n);
0781 mp_limb_t _flint_mpn_sqrhigh_mulders(mp_ptr res, mp_srcptr u, mp_size_t n);
0782 mp_limb_t _flint_mpn_sqrhigh_sqr(mp_ptr res, mp_srcptr u, mp_size_t n);
0783 mp_limb_t _flint_mpn_sqrhigh(mp_ptr res, mp_srcptr u, mp_size_t n);
0784
0785 MPN_EXTRAS_INLINE
0786 mp_limb_t flint_mpn_sqrhigh(mp_ptr rp, mp_srcptr xp, mp_size_t n)
0787 {
0788     FLINT_ASSERT(n >= 1);
0789
0790     if (FLINT_HAVE_SQRHIGH_FUNC(n)) /* NOTE: Aliasing allowed here */
0791         return flint_mpn_sqrhigh_func_tab[n](rp, xp);
0792     else
0793         return _flint_mpn_sqrhigh(rp, xp, n);
0794 }
0795
0796 mp_limb_pair_t _flint_mpn_mulhigh_normalised(mp_ptr rp, mp_srcptr xp, mp_srcptr yp, mp_size_t n);
0797
0798 MPN_EXTRAS_INLINE
0799 mp_limb_pair_t flint_mpn_mulhigh_normalised(mp_ptr rp, mp_srcptr xp, mp_srcptr yp, mp_size_t n)
0800 {
0801     FLINT_ASSERT(n >= 1);
0802
0803     if (FLINT_HAVE_MULHIGH_NORMALISED_FUNC(n))
0804         return flint_mpn_mulhigh_normalised_func_tab[n](rp, xp, yp);
0805     else
0806         return _flint_mpn_mulhigh_normalised(rp, xp, yp, n);
0807 }
0808
0809 mp_limb_pair_t _flint_mpn_sqrhigh_normalised(mp_ptr rp, mp_srcptr xp, mp_size_t n);
0810
0811 MPN_EXTRAS_INLINE
0812 mp_limb_pair_t flint_mpn_sqrhigh_normalised(mp_ptr rp, mp_srcptr xp, mp_size_t n)
0813 {
0814     FLINT_ASSERT(n >= 1);
0815
0816     if (FLINT_HAVE_SQRHIGH_NORMALISED_FUNC(n))
0817         return flint_mpn_sqrhigh_normalised_func_tab[n](rp, xp);
0818     else
0819         return _flint_mpn_sqrhigh_normalised(rp, xp, n);
0820 }
0821
0822 /* division ******************************************************************/
0823
0824 #if FLINT_HAVE_NATIVE_mpn_modexact_1_odd
0825 # define mpn_modexact_1_odd __gmpn_modexact_1_odd
0826 mp_limb_t mpn_modexact_1_odd(mp_srcptr, mp_size_t, mp_limb_t);
0827 #endif
0828
0829 #if FLINT_HAVE_NATIVE_mpn_invert_limb
0830 # define mpn_invert_limb __gmpn_invert_limb
0831 mp_limb_t mpn_invert_limb(mp_limb_t);
0832 #endif
0833
0834 mp_limb_t flint_mpn_preinv1(mp_limb_t d, mp_limb_t d2);
0835 void flint_mpn_preinvn(mp_ptr dinv, mp_srcptr d, mp_size_t n);
0836
0837 #if defined(mpn_modexact_1_odd)
0838 MPN_EXTRAS_INLINE
0839 int flint_mpn_divisible_1_odd(mp_srcptr x, mp_size_t xsize, mp_limb_t d)
0840 {
0841     return mpn_modexact_1_odd(x, xsize, d) == 0;
0842 }
0843 #else
0844 # include "gmpcompat.h"
0845 MPN_EXTRAS_INLINE
0846 int flint_mpn_divisible_1_odd(mp_srcptr x, mp_size_t xsize, mp_limb_t d)
0847 {
0848     __mpz_struct s;
0849     s._mp_size = xsize;
0850     s._mp_d = (mp_ptr) x;
0851     return flint_mpz_divisible_ui_p(&s, d);
0852 }
0853 #endif
0854
0855 FLINT_FORCE_INLINE
0856 void mpn_tdiv_q(mp_ptr qp, mp_srcptr np, mp_size_t nn, mp_srcptr dp, mp_size_t dn)
0857 {
0858     mp_ptr _scratch;
0859     TMP_INIT;
0860     TMP_START;
0861     _scratch = (mp_ptr) TMP_ALLOC(dn * sizeof(mp_limb_t));
0862     mpn_tdiv_qr(qp, _scratch, 0, np, nn, dp, dn);
0863     TMP_END;
0864 }
0865
0866 int flint_mpn_divides(mp_ptr q, mp_srcptr ap, mp_size_t an, mp_srcptr bp, mp_size_t bn, mp_ptr scr);
0867
0868 void flint_mpn_mod_preinvn(mp_ptr r, mp_srcptr a, mp_size_t m, mp_srcptr d, mp_size_t n, mp_srcptr dinv);
0869
0870 mp_limb_t flint_mpn_divrem_preinv1(mp_ptr q, mp_ptr a, mp_size_t m, mp_srcptr b, mp_size_t n, mp_limb_t dinv);
0871 mp_limb_t flint_mpn_divrem_preinvn(mp_ptr q, mp_ptr r, mp_srcptr a, mp_size_t m, mp_srcptr d, mp_size_t n, mp_srcptr dinv);
0872
0873 mp_limb_t flint_mpn_divrem_1_preinv(mp_ptr qp, mp_srcptr up, mp_size_t n, mp_limb_t d, mp_limb_t dinv, unsigned int norm);
0874 mp_limb_t flint_mpn_divrem_2_1_preinv_norm(mp_ptr qp, mp_srcptr up, mp_limb_t d, mp_limb_t dinv);
0875 mp_limb_t flint_mpn_divrem_2_1_preinv_unnorm(mp_ptr qp, mp_srcptr up, mp_limb_t d, mp_limb_t dinv, unsigned int norm);
0876 mp_limb_t flint_mpn_divrem_3_1_preinv_norm(mp_ptr qp, mp_srcptr up, mp_limb_t d, mp_limb_t dinv);
0877 mp_limb_t flint_mpn_divrem_3_1_preinv_unnorm(mp_ptr qp, mp_srcptr up, mp_limb_t d, mp_limb_t dinv, unsigned int norm);
0878
0879 /* composed arithmetic *******************************************************/
0880
0881 mp_size_t flint_mpn_fmms1(mp_ptr y, mp_limb_t a1, mp_srcptr x1, mp_limb_t a2, mp_srcptr x2, mp_size_t n);
0882
0883 /* debug *********************************************************************/
0884
0885 void flint_mpn_debug(mp_srcptr x, mp_size_t xsize);
0886
0887 /* factorisation *************************************************************/
0888
0889 mp_size_t flint_mpn_remove_2exp(mp_ptr x, mp_size_t xsize, flint_bitcnt_t * bits);
0890
0891 mp_size_t flint_mpn_remove_power_ascending(mp_ptr x, mp_size_t xsize, mp_ptr p, mp_size_t psize, ulong * exp);
0892
0893 int flint_mpn_factor_trial(mp_srcptr x, mp_size_t xsize, slong start, slong stop);
0894
0895 int flint_mpn_factor_trial_tree(slong * factors, mp_srcptr x, mp_size_t xsize, slong num_primes);
0896
0897 /* greatest common divisor ***************************************************/
0898
0899 mp_size_t flint_mpn_gcd_full2(mp_ptr gp, mp_srcptr ap, mp_size_t an, mp_srcptr bp, mp_size_t bn, mp_ptr scr);
0900 mp_size_t flint_mpn_gcd_full(mp_ptr gp, mp_srcptr ap, mp_size_t an, mp_srcptr bp, mp_size_t bn);
0901
0902 /* modular arithmetic ********************************************************/
0903
0904 void flint_mpn_mulmod_preinv1(mp_ptr r, mp_srcptr a, mp_srcptr b, mp_size_t n, mp_srcptr d, mp_limb_t dinv, ulong norm);
0905 void flint_mpn_mulmod_preinvn(mp_ptr r, mp_srcptr a, mp_srcptr b, mp_size_t n, mp_srcptr d, mp_srcptr dinv, ulong norm);
0906
0907 #define MPN_MULMOD_PRECOND_NONE 0
0908 #define MPN_MULMOD_PRECOND_SHOUP 1
0909 #define MPN_MULMOD_PRECOND_MATRIX 2
0910
0911 int flint_mpn_mulmod_want_precond(mp_size_t n, slong num, ulong norm);
0912
0913 void flint_mpn_mulmod_precond_matrix_precompute(mp_ptr apre, mp_srcptr a, mp_size_t n, mp_srcptr d, mp_srcptr dinv, ulong norm);
0914 mp_size_t flint_mpn_mulmod_precond_matrix_alloc(mp_size_t n);
0915 void flint_mpn_mulmod_precond_matrix(mp_ptr rp, mp_srcptr apre, mp_srcptr b, mp_size_t n, mp_srcptr d, mp_srcptr dinv, ulong norm);
0916
0917 void flint_mpn_mulmod_precond_shoup_precompute(mp_ptr apre, mp_srcptr a, mp_size_t n, mp_srcptr dnormed, mp_srcptr dinv, ulong norm);
0918 void flint_mpn_mulmod_precond_shoup(mp_ptr res, mp_srcptr a, mp_srcptr apre, mp_srcptr b, mp_size_t n, mp_srcptr d, ulong norm);
0919
0920 void flint_mpn_fmmamod_preinvn(mp_ptr r, mp_srcptr a, mp_srcptr b, mp_srcptr e, mp_srcptr f, mp_size_t n, mp_srcptr d, mp_srcptr dinv, ulong norm);
0921 void flint_mpn_fmmamod_preinvn_2(mp_ptr r, mp_srcptr a, mp_srcptr b, mp_srcptr e, mp_srcptr f, mp_srcptr d, mp_srcptr dinv, ulong norm);
0922 void flint_mpn_fmmamod_precond_matrix(mp_ptr rp, mp_srcptr apre1, mp_srcptr b1, mp_srcptr apre2, mp_srcptr b2, mp_size_t n, mp_srcptr d, mp_srcptr dinv, ulong norm);
0923
0924 int flint_mpn_mulmod_2expp1_basecase(mp_ptr xp, mp_srcptr yp, mp_srcptr zp, int c, flint_bitcnt_t b, mp_ptr tp);
0925
0926 /* miscellaneous *************************************************************/
0927
0928 double flint_mpn_get_d(mp_srcptr ptr, mp_size_t size, mp_size_t sign, long exp);
0929
0930 /* random ********************************************************************/
0931
0932 void flint_mpn_rrandom(mp_ptr rp, flint_rand_t state, mp_size_t n);
0933 void flint_mpn_rrandomb(mp_ptr rp, flint_rand_t state, flint_bitcnt_t n);
0934 void flint_mpn_urandomb(mp_ptr rp, flint_rand_t state, flint_bitcnt_t n);
0935 void flint_mpn_urandomm(mp_ptr rp, flint_rand_t state, mp_srcptr xp, mp_size_t xn);
0936
0937 /******************************************************************************
0938     Divisions where the quotient is expected to be small. All function do:
0939         input: n > d > 0
0940         output: q = n/d, r = n%d
0941     for various small sizes of n and d.
0942     Not in a function because compiler refuses to inline eudiv_qrrnndd.
0943     Each macro takes a prefix t for its local vars.
0944 ******************************************************************************/
0945
0946 #define eudiv_qrnd(q, r, n, d, t)           \
0947 do {                                        \
0948     mp_limb_t t##q, t##a = n, t##b = d;     \
0949                                             \
0950     FLINT_ASSERT(t##a > t##b);              \
0951     FLINT_ASSERT(t##b > 0);                 \
0952                                             \
0953     t##a -= t##b;                           \
0954     for (t##q = 1; t##q < 5; t##q++)        \
0955     {                                       \
0956         if (t##a < t##b)                    \
0957             goto t##quotient_found;         \
0958         t##a -= t##b;                       \
0959     }                                       \
0960     t##q += t##a / t##b;                    \
0961     t##a = t##a % t##b;                     \
0962                                             \
0963 t##quotient_found:                          \
0964                                             \
0965     q = t##q;                               \
0966     r = t##a;                               \
0967 } while (0)
0968
0969
0970 #define eudiv_qqrnnd(q1, q0, r0, n1, n0, d0, t)         \
0971 do {                                                    \
0972     mp_limb_t t##a1 = n1, t##a0 = n0, t##b0 = d0;       \
0973     mp_limb_t t##q1, t##q0, t##r0, t##u;                \
0974                                                         \
0975     FLINT_ASSERT(t##a1 > 0 || t##a0 > t##b0);           \
0976                                                         \
0977     udiv_qrnnd(t##q1, t##u, 0, t##a1, t##b0);           \
0978     udiv_qrnnd(t##q0, t##r0, t##u, t##a0, t##b0);       \
0979                                                         \
0980     q1 = t##q1;                                         \
0981     q0 = t##q0;                                         \
0982     r0 = t##r0;                                         \
0983 } while (0)
0984
0985 /* d must be normalized, i.e. d1 != 0 */
0986 #define eudiv_qrrnndd(q0, r1, r0, n1, n0, d1, d0, t)                        \
0987 do {                                                                        \
0988     int t##i;                                                               \
0989     mp_limb_t t##a1 = n1, t##a0 = n0, t##b1 = d1, t##b0 = d0;               \
0990     mp_limb_t t##r1, t##r0, t##u2, t##u1, t##u0, t##q, t##v1, t##v0;        \
0991                                                                             \
0992     FLINT_ASSERT(t##a1 != 0);                                               \
0993     FLINT_ASSERT(t##b1 != 0);                                               \
0994     FLINT_ASSERT(t##b1 < t##a1 || (t##b1 == t##a1 && t##b0 < t##a0));       \
0995                                                                             \
0996     t##q = 1;                                                               \
0997                                                                             \
0998     sub_ddmmss(t##r1,t##r0, t##a1,t##a0, t##b1,t##b0);                      \
0999                                                                             \
1000 t##subtract:                                                                \
1001                                                                             \
1002     for (t##i = 2; t##i <= 4; t##i++)                                       \
1003     {                                                                       \
1004         sub_dddmmmsss(t##u2,t##u1,t##u0, 0,t##r1,t##r0, 0,t##b1,t##b0);     \
1005         if (t##u2 != 0)                                                     \
1006             goto t##quotient_found;                                         \
1007         t##q += 1;                                                          \
1008         t##r0 = t##u0;                                                      \
1009         t##r1 = t##u1;                                                      \
1010     }                                                                       \
1011                                                                             \
1012     if (t##r1 != 0)                                                         \
1013     {                                                                       \
1014         int t##ncnt, t##dcnt;                                               \
1015         mp_limb_t t##qq = 0;                                                \
1016                                                                             \
1017         t##ncnt = flint_clz(t##r1);                                \
1018         t##dcnt = flint_clz(t##b1);                                \
1019         t##dcnt -= t##ncnt;                                                 \
1020         if (t##dcnt <= 0)                                                   \
1021             goto t##subtract;                                               \
1022                                                                             \
1023         t##v1 = (t##b1 << t##dcnt) | (t##b0 >> (FLINT_BITS - t##dcnt));     \
1024         t##v0 = t##b0 << t##dcnt;                                           \
1025                                                                             \
1026         do {                                                                \
1027             sub_dddmmmsss(t##u2,t##u1,t##u0, 0,t##r1,t##r0, 0,t##v1,t##v0); \
1028             t##qq = 2*t##qq + 1 + t##u2;                                    \
1029             t##r1 = t##u2 ? t##r1 : t##u1;                                  \
1030             t##r0 = t##u2 ? t##r0 : t##u0;                                  \
1031             t##v0 = (t##v1 << (FLINT_BITS - 1)) | (t##v0 >> 1);             \
1032             t##v1 = t##v1 >> 1;                                             \
1033             t##dcnt--;                                                      \
1034         } while (t##dcnt >= 0);                                             \
1035                                                                             \
1036         t##q += t##qq;                                                      \
1037     }                                                                       \
1038                                                                             \
1039 t##quotient_found:                                                          \
1040                                                                             \
1041     FLINT_ASSERT(t##r1 < t##b1 || (t##r1 == t##b1 && t##r0 < t##b0));       \
1042                                                                             \
1043     q0 = t##q;                                                              \
1044     r0 = t##r0;                                                             \
1045     r1 = t##r1;                                                             \
1046 } while (0)
1047
1048
1049 #ifdef __cplusplus
1050 }
1051 #endif
1052
1053 #endif