You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1112 lines
57 KiB

  1. /*
  2. Public domain by Andrew M. <liquidsun@gmail.com>
  3. See: https://github.com/floodyberry/curve25519-donna
  4. SSE2 curve25519 implementation
  5. */
  6. #include <emmintrin.h>
  7. typedef __m128i xmmi;
  8. typedef union packedelem8_t {
  9. unsigned char u[16];
  10. xmmi v;
  11. } packedelem8;
  12. typedef union packedelem32_t {
  13. uint32_t u[4];
  14. xmmi v;
  15. } packedelem32;
  16. typedef union packedelem64_t {
  17. uint64_t u[2];
  18. xmmi v;
  19. } packedelem64;
  20. /* 10 elements + an extra 2 to fit in 3 xmm registers */
  21. typedef uint32_t bignum25519[12];
  22. typedef packedelem32 packed32bignum25519[5];
  23. typedef packedelem64 packed64bignum25519[10];
  24. static const packedelem32 bot32bitmask = {{0xffffffff, 0x00000000, 0xffffffff, 0x00000000}};
  25. static const packedelem32 top32bitmask = {{0x00000000, 0xffffffff, 0x00000000, 0xffffffff}};
  26. static const packedelem32 top64bitmask = {{0x00000000, 0x00000000, 0xffffffff, 0xffffffff}};
  27. static const packedelem32 bot64bitmask = {{0xffffffff, 0xffffffff, 0x00000000, 0x00000000}};
  28. /* reduction masks */
  29. static const packedelem64 packedmask26 = {{0x03ffffff, 0x03ffffff}};
  30. static const packedelem64 packedmask25 = {{0x01ffffff, 0x01ffffff}};
  31. static const packedelem32 packedmask2625 = {{0x3ffffff,0,0x1ffffff,0}};
  32. static const packedelem32 packedmask26262626 = {{0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff}};
  33. static const packedelem32 packedmask25252525 = {{0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff}};
  34. /* multipliers */
  35. static const packedelem64 packednineteen = {{19, 19}};
  36. static const packedelem64 packednineteenone = {{19, 1}};
  37. static const packedelem64 packedthirtyeight = {{38, 38}};
  38. static const packedelem64 packed3819 = {{19*2,19}};
  39. static const packedelem64 packed9638 = {{19*4,19*2}};
  40. /* 121666,121665 */
  41. static const packedelem64 packed121666121665 = {{121666, 121665}};
  42. /* 2*(2^255 - 19) = 0 mod p */
  43. static const packedelem32 packed2p0 = {{0x7ffffda,0x3fffffe,0x7fffffe,0x3fffffe}};
  44. static const packedelem32 packed2p1 = {{0x7fffffe,0x3fffffe,0x7fffffe,0x3fffffe}};
  45. static const packedelem32 packed2p2 = {{0x7fffffe,0x3fffffe,0x0000000,0x0000000}};
  46. static const packedelem32 packed32packed2p0 = {{0x7ffffda,0x7ffffda,0x3fffffe,0x3fffffe}};
  47. static const packedelem32 packed32packed2p1 = {{0x7fffffe,0x7fffffe,0x3fffffe,0x3fffffe}};
  48. /* 4*(2^255 - 19) = 0 mod p */
  49. static const packedelem32 packed4p0 = {{0xfffffb4,0x7fffffc,0xffffffc,0x7fffffc}};
  50. static const packedelem32 packed4p1 = {{0xffffffc,0x7fffffc,0xffffffc,0x7fffffc}};
  51. static const packedelem32 packed4p2 = {{0xffffffc,0x7fffffc,0x0000000,0x0000000}};
  52. static const packedelem32 packed32packed4p0 = {{0xfffffb4,0xfffffb4,0x7fffffc,0x7fffffc}};
  53. static const packedelem32 packed32packed4p1 = {{0xffffffc,0xffffffc,0x7fffffc,0x7fffffc}};
  54. /* out = in */
  55. DONNA_INLINE static void
  56. curve25519_copy(bignum25519 out, const bignum25519 in) {
  57. xmmi x0,x1,x2;
  58. x0 = _mm_load_si128((xmmi*)in + 0);
  59. x1 = _mm_load_si128((xmmi*)in + 1);
  60. x2 = _mm_load_si128((xmmi*)in + 2);
  61. _mm_store_si128((xmmi*)out + 0, x0);
  62. _mm_store_si128((xmmi*)out + 1, x1);
  63. _mm_store_si128((xmmi*)out + 2, x2);
  64. }
  65. /* out = a + b */
  66. DONNA_INLINE static void
  67. curve25519_add(bignum25519 out, const bignum25519 a, const bignum25519 b) {
  68. xmmi a0,a1,a2,b0,b1,b2;
  69. a0 = _mm_load_si128((xmmi*)a + 0);
  70. a1 = _mm_load_si128((xmmi*)a + 1);
  71. a2 = _mm_load_si128((xmmi*)a + 2);
  72. b0 = _mm_load_si128((xmmi*)b + 0);
  73. b1 = _mm_load_si128((xmmi*)b + 1);
  74. b2 = _mm_load_si128((xmmi*)b + 2);
  75. a0 = _mm_add_epi32(a0, b0);
  76. a1 = _mm_add_epi32(a1, b1);
  77. a2 = _mm_add_epi32(a2, b2);
  78. _mm_store_si128((xmmi*)out + 0, a0);
  79. _mm_store_si128((xmmi*)out + 1, a1);
  80. _mm_store_si128((xmmi*)out + 2, a2);
  81. }
  82. #define curve25519_add_after_basic curve25519_add_reduce
  83. DONNA_INLINE static void
  84. curve25519_add_reduce(bignum25519 out, const bignum25519 a, const bignum25519 b) {
  85. xmmi a0,a1,a2,b0,b1,b2;
  86. xmmi c1,c2,c3;
  87. xmmi r0,r1,r2,r3,r4,r5;
  88. a0 = _mm_load_si128((xmmi*)a + 0);
  89. a1 = _mm_load_si128((xmmi*)a + 1);
  90. a2 = _mm_load_si128((xmmi*)a + 2);
  91. b0 = _mm_load_si128((xmmi*)b + 0);
  92. b1 = _mm_load_si128((xmmi*)b + 1);
  93. b2 = _mm_load_si128((xmmi*)b + 2);
  94. a0 = _mm_add_epi32(a0, b0);
  95. a1 = _mm_add_epi32(a1, b1);
  96. a2 = _mm_add_epi32(a2, b2);
  97. r0 = _mm_and_si128(_mm_unpacklo_epi64(a0, a1), bot32bitmask.v);
  98. r1 = _mm_srli_epi64(_mm_unpacklo_epi64(a0, a1), 32);
  99. r2 = _mm_and_si128(_mm_unpackhi_epi64(a0, a1), bot32bitmask.v);
  100. r3 = _mm_srli_epi64(_mm_unpackhi_epi64(a0, a1), 32);
  101. r4 = _mm_and_si128(_mm_unpacklo_epi64(_mm_setzero_si128(), a2), bot32bitmask.v);
  102. r5 = _mm_srli_epi64(_mm_unpacklo_epi64(_mm_setzero_si128(), a2), 32);
  103. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  104. c1 = _mm_srli_epi64(r1, 25); c2 = _mm_srli_epi64(r3, 25); r1 = _mm_and_si128(r1, packedmask25.v); r3 = _mm_and_si128(r3, packedmask25.v); r2 = _mm_add_epi64(r2, c1); r4 = _mm_add_epi64(r4, c2); c3 = _mm_slli_si128(c2, 8);
  105. c1 = _mm_srli_epi64(r4, 26); r4 = _mm_and_si128(r4, packedmask26.v); r5 = _mm_add_epi64(r5, c1);
  106. c1 = _mm_srli_epi64(r5, 25); r5 = _mm_and_si128(r5, packedmask25.v); r0 = _mm_add_epi64(r0, _mm_unpackhi_epi64(_mm_mul_epu32(c1, packednineteen.v), c3));
  107. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  108. _mm_store_si128((xmmi*)out + 0, _mm_unpacklo_epi64(_mm_unpacklo_epi32(r0, r1), _mm_unpacklo_epi32(r2, r3)));
  109. _mm_store_si128((xmmi*)out + 1, _mm_unpacklo_epi64(_mm_unpackhi_epi32(r0, r1), _mm_unpackhi_epi32(r2, r3)));
  110. _mm_store_si128((xmmi*)out + 2, _mm_unpackhi_epi32(r4, r5));
  111. }
  112. DONNA_INLINE static void
  113. curve25519_sub(bignum25519 out, const bignum25519 a, const bignum25519 b) {
  114. xmmi a0,a1,a2,b0,b1,b2;
  115. xmmi c1,c2;
  116. xmmi r0,r1;
  117. a0 = _mm_load_si128((xmmi*)a + 0);
  118. a1 = _mm_load_si128((xmmi*)a + 1);
  119. a2 = _mm_load_si128((xmmi*)a + 2);
  120. a0 = _mm_add_epi32(a0, packed2p0.v);
  121. a1 = _mm_add_epi32(a1, packed2p1.v);
  122. a2 = _mm_add_epi32(a2, packed2p2.v);
  123. b0 = _mm_load_si128((xmmi*)b + 0);
  124. b1 = _mm_load_si128((xmmi*)b + 1);
  125. b2 = _mm_load_si128((xmmi*)b + 2);
  126. a0 = _mm_sub_epi32(a0, b0);
  127. a1 = _mm_sub_epi32(a1, b1);
  128. a2 = _mm_sub_epi32(a2, b2);
  129. r0 = _mm_and_si128(_mm_shuffle_epi32(a0, _MM_SHUFFLE(2,2,0,0)), bot32bitmask.v);
  130. r1 = _mm_and_si128(_mm_shuffle_epi32(a0, _MM_SHUFFLE(3,3,1,1)), bot32bitmask.v);
  131. c1 = _mm_srli_epi32(r0, 26);
  132. c2 = _mm_srli_epi32(r1, 25);
  133. r0 = _mm_and_si128(r0, packedmask26.v);
  134. r1 = _mm_and_si128(r1, packedmask25.v);
  135. r0 = _mm_add_epi32(r0, _mm_slli_si128(c2, 8));
  136. r1 = _mm_add_epi32(r1, c1);
  137. a0 = _mm_unpacklo_epi64(_mm_unpacklo_epi32(r0, r1), _mm_unpackhi_epi32(r0, r1));
  138. a1 = _mm_add_epi32(a1, _mm_srli_si128(c2, 8));
  139. _mm_store_si128((xmmi*)out + 0, a0);
  140. _mm_store_si128((xmmi*)out + 1, a1);
  141. _mm_store_si128((xmmi*)out + 2, a2);
  142. }
  143. DONNA_INLINE static void
  144. curve25519_sub_after_basic(bignum25519 out, const bignum25519 a, const bignum25519 b) {
  145. xmmi a0,a1,a2,b0,b1,b2;
  146. xmmi c1,c2,c3;
  147. xmmi r0,r1,r2,r3,r4,r5;
  148. a0 = _mm_load_si128((xmmi*)a + 0);
  149. a1 = _mm_load_si128((xmmi*)a + 1);
  150. a2 = _mm_load_si128((xmmi*)a + 2);
  151. a0 = _mm_add_epi32(a0, packed4p0.v);
  152. a1 = _mm_add_epi32(a1, packed4p1.v);
  153. a2 = _mm_add_epi32(a2, packed4p2.v);
  154. b0 = _mm_load_si128((xmmi*)b + 0);
  155. b1 = _mm_load_si128((xmmi*)b + 1);
  156. b2 = _mm_load_si128((xmmi*)b + 2);
  157. a0 = _mm_sub_epi32(a0, b0);
  158. a1 = _mm_sub_epi32(a1, b1);
  159. a2 = _mm_sub_epi32(a2, b2);
  160. r0 = _mm_and_si128(_mm_unpacklo_epi64(a0, a1), bot32bitmask.v);
  161. r1 = _mm_srli_epi64(_mm_unpacklo_epi64(a0, a1), 32);
  162. r2 = _mm_and_si128(_mm_unpackhi_epi64(a0, a1), bot32bitmask.v);
  163. r3 = _mm_srli_epi64(_mm_unpackhi_epi64(a0, a1), 32);
  164. r4 = _mm_and_si128(_mm_unpacklo_epi64(_mm_setzero_si128(), a2), bot32bitmask.v);
  165. r5 = _mm_srli_epi64(_mm_unpacklo_epi64(_mm_setzero_si128(), a2), 32);
  166. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  167. c1 = _mm_srli_epi64(r1, 25); c2 = _mm_srli_epi64(r3, 25); r1 = _mm_and_si128(r1, packedmask25.v); r3 = _mm_and_si128(r3, packedmask25.v); r2 = _mm_add_epi64(r2, c1); r4 = _mm_add_epi64(r4, c2); c3 = _mm_slli_si128(c2, 8);
  168. c1 = _mm_srli_epi64(r4, 26); r4 = _mm_and_si128(r4, packedmask26.v); r5 = _mm_add_epi64(r5, c1);
  169. c1 = _mm_srli_epi64(r5, 25); r5 = _mm_and_si128(r5, packedmask25.v); r0 = _mm_add_epi64(r0, _mm_unpackhi_epi64(_mm_mul_epu32(c1, packednineteen.v), c3));
  170. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  171. _mm_store_si128((xmmi*)out + 0, _mm_unpacklo_epi64(_mm_unpacklo_epi32(r0, r1), _mm_unpacklo_epi32(r2, r3)));
  172. _mm_store_si128((xmmi*)out + 1, _mm_unpacklo_epi64(_mm_unpackhi_epi32(r0, r1), _mm_unpackhi_epi32(r2, r3)));
  173. _mm_store_si128((xmmi*)out + 2, _mm_unpackhi_epi32(r4, r5));
  174. }
  175. DONNA_INLINE static void
  176. curve25519_sub_reduce(bignum25519 out, const bignum25519 a, const bignum25519 b) {
  177. xmmi a0,a1,a2,b0,b1,b2;
  178. xmmi c1,c2,c3;
  179. xmmi r0,r1,r2,r3,r4,r5;
  180. a0 = _mm_load_si128((xmmi*)a + 0);
  181. a1 = _mm_load_si128((xmmi*)a + 1);
  182. a2 = _mm_load_si128((xmmi*)a + 2);
  183. a0 = _mm_add_epi32(a0, packed2p0.v);
  184. a1 = _mm_add_epi32(a1, packed2p1.v);
  185. a2 = _mm_add_epi32(a2, packed2p2.v);
  186. b0 = _mm_load_si128((xmmi*)b + 0);
  187. b1 = _mm_load_si128((xmmi*)b + 1);
  188. b2 = _mm_load_si128((xmmi*)b + 2);
  189. a0 = _mm_sub_epi32(a0, b0);
  190. a1 = _mm_sub_epi32(a1, b1);
  191. a2 = _mm_sub_epi32(a2, b2);
  192. r0 = _mm_and_si128(_mm_unpacklo_epi64(a0, a1), bot32bitmask.v);
  193. r1 = _mm_srli_epi64(_mm_unpacklo_epi64(a0, a1), 32);
  194. r2 = _mm_and_si128(_mm_unpackhi_epi64(a0, a1), bot32bitmask.v);
  195. r3 = _mm_srli_epi64(_mm_unpackhi_epi64(a0, a1), 32);
  196. r4 = _mm_and_si128(_mm_unpacklo_epi64(_mm_setzero_si128(), a2), bot32bitmask.v);
  197. r5 = _mm_srli_epi64(_mm_unpacklo_epi64(_mm_setzero_si128(), a2), 32);
  198. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  199. c1 = _mm_srli_epi64(r1, 25); c2 = _mm_srli_epi64(r3, 25); r1 = _mm_and_si128(r1, packedmask25.v); r3 = _mm_and_si128(r3, packedmask25.v); r2 = _mm_add_epi64(r2, c1); r4 = _mm_add_epi64(r4, c2); c3 = _mm_slli_si128(c2, 8);
  200. c1 = _mm_srli_epi64(r4, 26); r4 = _mm_and_si128(r4, packedmask26.v); r5 = _mm_add_epi64(r5, c1);
  201. c1 = _mm_srli_epi64(r5, 25); r5 = _mm_and_si128(r5, packedmask25.v); r0 = _mm_add_epi64(r0, _mm_unpackhi_epi64(_mm_mul_epu32(c1, packednineteen.v), c3));
  202. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  203. _mm_store_si128((xmmi*)out + 0, _mm_unpacklo_epi64(_mm_unpacklo_epi32(r0, r1), _mm_unpacklo_epi32(r2, r3)));
  204. _mm_store_si128((xmmi*)out + 1, _mm_unpacklo_epi64(_mm_unpackhi_epi32(r0, r1), _mm_unpackhi_epi32(r2, r3)));
  205. _mm_store_si128((xmmi*)out + 2, _mm_unpackhi_epi32(r4, r5));
  206. }
  207. DONNA_INLINE static void
  208. curve25519_neg(bignum25519 out, const bignum25519 b) {
  209. xmmi a0,a1,a2,b0,b1,b2;
  210. xmmi c1,c2,c3;
  211. xmmi r0,r1,r2,r3,r4,r5;
  212. a0 = packed2p0.v;
  213. a1 = packed2p1.v;
  214. a2 = packed2p2.v;
  215. b0 = _mm_load_si128((xmmi*)b + 0);
  216. b1 = _mm_load_si128((xmmi*)b + 1);
  217. b2 = _mm_load_si128((xmmi*)b + 2);
  218. a0 = _mm_sub_epi32(a0, b0);
  219. a1 = _mm_sub_epi32(a1, b1);
  220. a2 = _mm_sub_epi32(a2, b2);
  221. r0 = _mm_and_si128(_mm_unpacklo_epi64(a0, a1), bot32bitmask.v);
  222. r1 = _mm_srli_epi64(_mm_unpacklo_epi64(a0, a1), 32);
  223. r2 = _mm_and_si128(_mm_unpackhi_epi64(a0, a1), bot32bitmask.v);
  224. r3 = _mm_srli_epi64(_mm_unpackhi_epi64(a0, a1), 32);
  225. r4 = _mm_and_si128(_mm_unpacklo_epi64(_mm_setzero_si128(), a2), bot32bitmask.v);
  226. r5 = _mm_srli_epi64(_mm_unpacklo_epi64(_mm_setzero_si128(), a2), 32);
  227. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  228. c1 = _mm_srli_epi64(r1, 25); c2 = _mm_srli_epi64(r3, 25); r1 = _mm_and_si128(r1, packedmask25.v); r3 = _mm_and_si128(r3, packedmask25.v); r2 = _mm_add_epi64(r2, c1); r4 = _mm_add_epi64(r4, c2); c3 = _mm_slli_si128(c2, 8);
  229. c1 = _mm_srli_epi64(r4, 26); r4 = _mm_and_si128(r4, packedmask26.v); r5 = _mm_add_epi64(r5, c1);
  230. c1 = _mm_srli_epi64(r5, 25); r5 = _mm_and_si128(r5, packedmask25.v); r0 = _mm_add_epi64(r0, _mm_unpackhi_epi64(_mm_mul_epu32(c1, packednineteen.v), c3));
  231. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  232. _mm_store_si128((xmmi*)out + 0, _mm_unpacklo_epi64(_mm_unpacklo_epi32(r0, r1), _mm_unpacklo_epi32(r2, r3)));
  233. _mm_store_si128((xmmi*)out + 1, _mm_unpacklo_epi64(_mm_unpackhi_epi32(r0, r1), _mm_unpackhi_epi32(r2, r3)));
  234. _mm_store_si128((xmmi*)out + 2, _mm_unpackhi_epi32(r4, r5));
  235. }
  236. /* Multiply two numbers: out = in2 * in */
  237. static void
  238. curve25519_mul(bignum25519 out, const bignum25519 r, const bignum25519 s) {
  239. xmmi m01,m23,m45,m67,m89;
  240. xmmi m0123,m4567;
  241. xmmi s0123,s4567;
  242. xmmi s01,s23,s45,s67,s89;
  243. xmmi s12,s34,s56,s78,s9;
  244. xmmi r0,r2,r4,r6,r8;
  245. xmmi r1,r3,r5,r7,r9;
  246. xmmi r119,r219,r319,r419,r519,r619,r719,r819,r919;
  247. xmmi c1,c2,c3;
  248. s0123 = _mm_load_si128((xmmi*)s + 0);
  249. s01 = _mm_shuffle_epi32(s0123,_MM_SHUFFLE(3,1,2,0));
  250. s12 = _mm_shuffle_epi32(s0123, _MM_SHUFFLE(2,2,1,1));
  251. s23 = _mm_shuffle_epi32(s0123,_MM_SHUFFLE(3,3,2,2));
  252. s4567 = _mm_load_si128((xmmi*)s + 1);
  253. s34 = _mm_unpacklo_epi64(_mm_srli_si128(s0123,12),s4567);
  254. s45 = _mm_shuffle_epi32(s4567,_MM_SHUFFLE(3,1,2,0));
  255. s56 = _mm_shuffle_epi32(s4567, _MM_SHUFFLE(2,2,1,1));
  256. s67 = _mm_shuffle_epi32(s4567,_MM_SHUFFLE(3,3,2,2));
  257. s89 = _mm_load_si128((xmmi*)s + 2);
  258. s78 = _mm_unpacklo_epi64(_mm_srli_si128(s4567,12),s89);
  259. s89 = _mm_shuffle_epi32(s89,_MM_SHUFFLE(3,1,2,0));
  260. s9 = _mm_shuffle_epi32(s89, _MM_SHUFFLE(3,3,2,2));
  261. r0 = _mm_load_si128((xmmi*)r + 0);
  262. r1 = _mm_shuffle_epi32(r0, _MM_SHUFFLE(1,1,1,1));
  263. r1 = _mm_add_epi64(r1, _mm_and_si128(r1, top64bitmask.v));
  264. r2 = _mm_shuffle_epi32(r0, _MM_SHUFFLE(2,2,2,2));
  265. r3 = _mm_shuffle_epi32(r0, _MM_SHUFFLE(3,3,3,3));
  266. r3 = _mm_add_epi64(r3, _mm_and_si128(r3, top64bitmask.v));
  267. r0 = _mm_shuffle_epi32(r0, _MM_SHUFFLE(0,0,0,0));
  268. r4 = _mm_load_si128((xmmi*)r + 1);
  269. r5 = _mm_shuffle_epi32(r4, _MM_SHUFFLE(1,1,1,1));
  270. r5 = _mm_add_epi64(r5, _mm_and_si128(r5, top64bitmask.v));
  271. r6 = _mm_shuffle_epi32(r4, _MM_SHUFFLE(2,2,2,2));
  272. r7 = _mm_shuffle_epi32(r4, _MM_SHUFFLE(3,3,3,3));
  273. r7 = _mm_add_epi64(r7, _mm_and_si128(r7, top64bitmask.v));
  274. r4 = _mm_shuffle_epi32(r4, _MM_SHUFFLE(0,0,0,0));
  275. r8 = _mm_load_si128((xmmi*)r + 2);
  276. r9 = _mm_shuffle_epi32(r8, _MM_SHUFFLE(3,1,3,1));
  277. r9 = _mm_add_epi64(r9, _mm_and_si128(r9, top64bitmask.v));
  278. r8 = _mm_shuffle_epi32(r8, _MM_SHUFFLE(3,0,3,0));
  279. m01 = _mm_mul_epu32(r1,s01);
  280. m23 = _mm_mul_epu32(r1,s23);
  281. m45 = _mm_mul_epu32(r1,s45);
  282. m67 = _mm_mul_epu32(r1,s67);
  283. m23 = _mm_add_epi64(m23,_mm_mul_epu32(r3,s01));
  284. m45 = _mm_add_epi64(m45,_mm_mul_epu32(r3,s23));
  285. m67 = _mm_add_epi64(m67,_mm_mul_epu32(r3,s45));
  286. m89 = _mm_mul_epu32(r1,s89);
  287. m45 = _mm_add_epi64(m45,_mm_mul_epu32(r5,s01));
  288. m67 = _mm_add_epi64(m67,_mm_mul_epu32(r5,s23));
  289. m89 = _mm_add_epi64(m89,_mm_mul_epu32(r3,s67));
  290. m67 = _mm_add_epi64(m67,_mm_mul_epu32(r7,s01));
  291. m89 = _mm_add_epi64(m89,_mm_mul_epu32(r5,s45));
  292. m89 = _mm_add_epi64(m89,_mm_mul_epu32(r7,s23));
  293. m89 = _mm_add_epi64(m89,_mm_mul_epu32(r9,s01));
  294. /* shift up */
  295. m89 = _mm_unpackhi_epi64(m67,_mm_slli_si128(m89,8));
  296. m67 = _mm_unpackhi_epi64(m45,_mm_slli_si128(m67,8));
  297. m45 = _mm_unpackhi_epi64(m23,_mm_slli_si128(m45,8));
  298. m23 = _mm_unpackhi_epi64(m01,_mm_slli_si128(m23,8));
  299. m01 = _mm_unpackhi_epi64(_mm_setzero_si128(),_mm_slli_si128(m01,8));
  300. m01 = _mm_add_epi64(m01,_mm_mul_epu32(r0,s01));
  301. m23 = _mm_add_epi64(m23,_mm_mul_epu32(r0,s23));
  302. m45 = _mm_add_epi64(m45,_mm_mul_epu32(r0,s45));
  303. m67 = _mm_add_epi64(m67,_mm_mul_epu32(r0,s67));
  304. m23 = _mm_add_epi64(m23,_mm_mul_epu32(r2,s01));
  305. m45 = _mm_add_epi64(m45,_mm_mul_epu32(r2,s23));
  306. m67 = _mm_add_epi64(m67,_mm_mul_epu32(r4,s23));
  307. m89 = _mm_add_epi64(m89,_mm_mul_epu32(r0,s89));
  308. m45 = _mm_add_epi64(m45,_mm_mul_epu32(r4,s01));
  309. m67 = _mm_add_epi64(m67,_mm_mul_epu32(r2,s45));
  310. m89 = _mm_add_epi64(m89,_mm_mul_epu32(r2,s67));
  311. m67 = _mm_add_epi64(m67,_mm_mul_epu32(r6,s01));
  312. m89 = _mm_add_epi64(m89,_mm_mul_epu32(r4,s45));
  313. m89 = _mm_add_epi64(m89,_mm_mul_epu32(r6,s23));
  314. m89 = _mm_add_epi64(m89,_mm_mul_epu32(r8,s01));
  315. r219 = _mm_mul_epu32(r2, packednineteen.v);
  316. r419 = _mm_mul_epu32(r4, packednineteen.v);
  317. r619 = _mm_mul_epu32(r6, packednineteen.v);
  318. r819 = _mm_mul_epu32(r8, packednineteen.v);
  319. r119 = _mm_shuffle_epi32(r1,_MM_SHUFFLE(0,0,2,2)); r119 = _mm_mul_epu32(r119, packednineteen.v);
  320. r319 = _mm_shuffle_epi32(r3,_MM_SHUFFLE(0,0,2,2)); r319 = _mm_mul_epu32(r319, packednineteen.v);
  321. r519 = _mm_shuffle_epi32(r5,_MM_SHUFFLE(0,0,2,2)); r519 = _mm_mul_epu32(r519, packednineteen.v);
  322. r719 = _mm_shuffle_epi32(r7,_MM_SHUFFLE(0,0,2,2)); r719 = _mm_mul_epu32(r719, packednineteen.v);
  323. r919 = _mm_shuffle_epi32(r9,_MM_SHUFFLE(0,0,2,2)); r919 = _mm_mul_epu32(r919, packednineteen.v);
  324. m01 = _mm_add_epi64(m01,_mm_mul_epu32(r919,s12));
  325. m23 = _mm_add_epi64(m23,_mm_mul_epu32(r919,s34));
  326. m45 = _mm_add_epi64(m45,_mm_mul_epu32(r919,s56));
  327. m67 = _mm_add_epi64(m67,_mm_mul_epu32(r919,s78));
  328. m01 = _mm_add_epi64(m01,_mm_mul_epu32(r719,s34));
  329. m23 = _mm_add_epi64(m23,_mm_mul_epu32(r719,s56));
  330. m45 = _mm_add_epi64(m45,_mm_mul_epu32(r719,s78));
  331. m67 = _mm_add_epi64(m67,_mm_mul_epu32(r719,s9));
  332. m01 = _mm_add_epi64(m01,_mm_mul_epu32(r519,s56));
  333. m23 = _mm_add_epi64(m23,_mm_mul_epu32(r519,s78));
  334. m45 = _mm_add_epi64(m45,_mm_mul_epu32(r519,s9));
  335. m67 = _mm_add_epi64(m67,_mm_mul_epu32(r819,s89));
  336. m01 = _mm_add_epi64(m01,_mm_mul_epu32(r319,s78));
  337. m23 = _mm_add_epi64(m23,_mm_mul_epu32(r319,s9));
  338. m45 = _mm_add_epi64(m45,_mm_mul_epu32(r619,s89));
  339. m89 = _mm_add_epi64(m89,_mm_mul_epu32(r919,s9));
  340. m01 = _mm_add_epi64(m01,_mm_mul_epu32(r819,s23));
  341. m23 = _mm_add_epi64(m23,_mm_mul_epu32(r819,s45));
  342. m45 = _mm_add_epi64(m45,_mm_mul_epu32(r819,s67));
  343. m01 = _mm_add_epi64(m01,_mm_mul_epu32(r619,s45));
  344. m23 = _mm_add_epi64(m23,_mm_mul_epu32(r619,s67));
  345. m01 = _mm_add_epi64(m01,_mm_mul_epu32(r419,s67));
  346. m23 = _mm_add_epi64(m23,_mm_mul_epu32(r419,s89));
  347. m01 = _mm_add_epi64(m01,_mm_mul_epu32(r219,s89));
  348. m01 = _mm_add_epi64(m01,_mm_mul_epu32(r119,s9));
  349. r0 = _mm_unpacklo_epi64(m01, m45);
  350. r1 = _mm_unpackhi_epi64(m01, m45);
  351. r2 = _mm_unpacklo_epi64(m23, m67);
  352. r3 = _mm_unpackhi_epi64(m23, m67);
  353. r4 = _mm_unpacklo_epi64(m89, m89);
  354. r5 = _mm_unpackhi_epi64(m89, m89);
  355. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  356. c1 = _mm_srli_epi64(r1, 25); c2 = _mm_srli_epi64(r3, 25); r1 = _mm_and_si128(r1, packedmask25.v); r3 = _mm_and_si128(r3, packedmask25.v); r2 = _mm_add_epi64(r2, c1); r4 = _mm_add_epi64(r4, c2); c3 = _mm_slli_si128(c2, 8);
  357. c1 = _mm_srli_epi64(r4, 26); r4 = _mm_and_si128(r4, packedmask26.v); r5 = _mm_add_epi64(r5, c1);
  358. c1 = _mm_srli_epi64(r5, 25); r5 = _mm_and_si128(r5, packedmask25.v); r0 = _mm_add_epi64(r0, _mm_unpackhi_epi64(_mm_mul_epu32(c1, packednineteen.v), c3));
  359. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  360. m0123 = _mm_unpacklo_epi32(r0, r1);
  361. m4567 = _mm_unpackhi_epi32(r0, r1);
  362. m0123 = _mm_unpacklo_epi64(m0123, _mm_unpacklo_epi32(r2, r3));
  363. m4567 = _mm_unpacklo_epi64(m4567, _mm_unpackhi_epi32(r2, r3));
  364. m89 = _mm_unpackhi_epi32(r4, r5);
  365. _mm_store_si128((xmmi*)out + 0, m0123);
  366. _mm_store_si128((xmmi*)out + 1, m4567);
  367. _mm_store_si128((xmmi*)out + 2, m89);
  368. }
  369. DONNA_NOINLINE static void
  370. curve25519_mul_noinline(bignum25519 out, const bignum25519 r, const bignum25519 s) {
  371. curve25519_mul(out, r, s);
  372. }
  373. #define curve25519_square(r, n) curve25519_square_times(r, n, 1)
  374. static void
  375. curve25519_square_times(bignum25519 r, const bignum25519 in, int count) {
  376. xmmi m01,m23,m45,m67,m89;
  377. xmmi r0,r1,r2,r3,r4,r5,r6,r7,r8,r9;
  378. xmmi r0a,r1a,r2a,r3a,r7a,r9a;
  379. xmmi r0123,r4567;
  380. xmmi r01,r23,r45,r67,r6x,r89,r8x;
  381. xmmi r12,r34,r56,r78,r9x;
  382. xmmi r5619;
  383. xmmi c1,c2,c3;
  384. r0123 = _mm_load_si128((xmmi*)in + 0);
  385. r01 = _mm_shuffle_epi32(r0123,_MM_SHUFFLE(3,1,2,0));
  386. r23 = _mm_shuffle_epi32(r0123,_MM_SHUFFLE(3,3,2,2));
  387. r4567 = _mm_load_si128((xmmi*)in + 1);
  388. r45 = _mm_shuffle_epi32(r4567,_MM_SHUFFLE(3,1,2,0));
  389. r67 = _mm_shuffle_epi32(r4567,_MM_SHUFFLE(3,3,2,2));
  390. r89 = _mm_load_si128((xmmi*)in + 2);
  391. r89 = _mm_shuffle_epi32(r89,_MM_SHUFFLE(3,1,2,0));
  392. do {
  393. r12 = _mm_unpackhi_epi64(r01, _mm_slli_si128(r23, 8));
  394. r0 = _mm_shuffle_epi32(r01, _MM_SHUFFLE(0,0,0,0));
  395. r0 = _mm_add_epi64(r0, _mm_and_si128(r0, top64bitmask.v));
  396. r0a = _mm_shuffle_epi32(r0,_MM_SHUFFLE(3,2,1,2));
  397. r1 = _mm_shuffle_epi32(r01, _MM_SHUFFLE(2,2,2,2));
  398. r2 = _mm_shuffle_epi32(r23, _MM_SHUFFLE(0,0,0,0));
  399. r2 = _mm_add_epi64(r2, _mm_and_si128(r2, top64bitmask.v));
  400. r2a = _mm_shuffle_epi32(r2,_MM_SHUFFLE(3,2,1,2));
  401. r3 = _mm_shuffle_epi32(r23, _MM_SHUFFLE(2,2,2,2));
  402. r34 = _mm_unpackhi_epi64(r23, _mm_slli_si128(r45, 8));
  403. r4 = _mm_shuffle_epi32(r45, _MM_SHUFFLE(0,0,0,0));
  404. r4 = _mm_add_epi64(r4, _mm_and_si128(r4, top64bitmask.v));
  405. r56 = _mm_unpackhi_epi64(r45, _mm_slli_si128(r67, 8));
  406. r5619 = _mm_mul_epu32(r56, packednineteen.v);
  407. r5 = _mm_shuffle_epi32(r5619, _MM_SHUFFLE(1,1,1,0));
  408. r6 = _mm_shuffle_epi32(r5619, _MM_SHUFFLE(3,2,3,2));
  409. r78 = _mm_unpackhi_epi64(r67, _mm_slli_si128(r89, 8));
  410. r6x = _mm_unpacklo_epi64(r67, _mm_setzero_si128());
  411. r7 = _mm_shuffle_epi32(r67, _MM_SHUFFLE(2,2,2,2));
  412. r7 = _mm_mul_epu32(r7, packed3819.v);
  413. r7a = _mm_shuffle_epi32(r7, _MM_SHUFFLE(3,3,3,2));
  414. r8x = _mm_unpacklo_epi64(r89, _mm_setzero_si128());
  415. r8 = _mm_shuffle_epi32(r89, _MM_SHUFFLE(0,0,0,0));
  416. r8 = _mm_mul_epu32(r8, packednineteen.v);
  417. r9 = _mm_shuffle_epi32(r89, _MM_SHUFFLE(2,2,2,2));
  418. r9x = _mm_slli_epi32(_mm_shuffle_epi32(r89, _MM_SHUFFLE(3,3,3,2)), 1);
  419. r9 = _mm_mul_epu32(r9, packed3819.v);
  420. r9a = _mm_shuffle_epi32(r9, _MM_SHUFFLE(2,2,2,2));
  421. m01 = _mm_mul_epu32(r01, r0);
  422. m23 = _mm_mul_epu32(r23, r0a);
  423. m45 = _mm_mul_epu32(r45, r0a);
  424. m45 = _mm_add_epi64(m45, _mm_mul_epu32(r23, r2));
  425. r23 = _mm_slli_epi32(r23, 1);
  426. m67 = _mm_mul_epu32(r67, r0a);
  427. m67 = _mm_add_epi64(m67, _mm_mul_epu32(r45, r2a));
  428. m89 = _mm_mul_epu32(r89, r0a);
  429. m89 = _mm_add_epi64(m89, _mm_mul_epu32(r67, r2a));
  430. r67 = _mm_slli_epi32(r67, 1);
  431. m89 = _mm_add_epi64(m89, _mm_mul_epu32(r45, r4));
  432. r45 = _mm_slli_epi32(r45, 1);
  433. r1 = _mm_slli_epi32(r1, 1);
  434. r3 = _mm_slli_epi32(r3, 1);
  435. r1a = _mm_add_epi64(r1, _mm_and_si128(r1, bot64bitmask.v));
  436. r3a = _mm_add_epi64(r3, _mm_and_si128(r3, bot64bitmask.v));
  437. m23 = _mm_add_epi64(m23, _mm_mul_epu32(r12, r1));
  438. m45 = _mm_add_epi64(m45, _mm_mul_epu32(r34, r1a));
  439. m67 = _mm_add_epi64(m67, _mm_mul_epu32(r56, r1a));
  440. m67 = _mm_add_epi64(m67, _mm_mul_epu32(r34, r3));
  441. r34 = _mm_slli_epi32(r34, 1);
  442. m89 = _mm_add_epi64(m89, _mm_mul_epu32(r78, r1a));
  443. r78 = _mm_slli_epi32(r78, 1);
  444. m89 = _mm_add_epi64(m89, _mm_mul_epu32(r56, r3a));
  445. r56 = _mm_slli_epi32(r56, 1);
  446. m01 = _mm_add_epi64(m01, _mm_mul_epu32(_mm_slli_epi32(r12, 1), r9));
  447. m01 = _mm_add_epi64(m01, _mm_mul_epu32(r34, r7));
  448. m23 = _mm_add_epi64(m23, _mm_mul_epu32(r34, r9));
  449. m01 = _mm_add_epi64(m01, _mm_mul_epu32(r56, r5));
  450. m23 = _mm_add_epi64(m23, _mm_mul_epu32(r56, r7));
  451. m45 = _mm_add_epi64(m45, _mm_mul_epu32(r56, r9));
  452. m01 = _mm_add_epi64(m01, _mm_mul_epu32(r23, r8));
  453. m01 = _mm_add_epi64(m01, _mm_mul_epu32(r45, r6));
  454. m23 = _mm_add_epi64(m23, _mm_mul_epu32(r45, r8));
  455. m23 = _mm_add_epi64(m23, _mm_mul_epu32(r6x, r6));
  456. m45 = _mm_add_epi64(m45, _mm_mul_epu32(r78, r7a));
  457. m67 = _mm_add_epi64(m67, _mm_mul_epu32(r78, r9));
  458. m45 = _mm_add_epi64(m45, _mm_mul_epu32(r67, r8));
  459. m67 = _mm_add_epi64(m67, _mm_mul_epu32(r8x, r8));
  460. m89 = _mm_add_epi64(m89, _mm_mul_epu32(r9x, r9a));
  461. r0 = _mm_unpacklo_epi64(m01, m45);
  462. r1 = _mm_unpackhi_epi64(m01, m45);
  463. r2 = _mm_unpacklo_epi64(m23, m67);
  464. r3 = _mm_unpackhi_epi64(m23, m67);
  465. r4 = _mm_unpacklo_epi64(m89, m89);
  466. r5 = _mm_unpackhi_epi64(m89, m89);
  467. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  468. c1 = _mm_srli_epi64(r1, 25); c2 = _mm_srli_epi64(r3, 25); r1 = _mm_and_si128(r1, packedmask25.v); r3 = _mm_and_si128(r3, packedmask25.v); r2 = _mm_add_epi64(r2, c1); r4 = _mm_add_epi64(r4, c2); c3 = _mm_slli_si128(c2, 8);
  469. c1 = _mm_srli_epi64(r4, 26); r4 = _mm_and_si128(r4, packedmask26.v); r5 = _mm_add_epi64(r5, c1);
  470. c1 = _mm_srli_epi64(r5, 25); r5 = _mm_and_si128(r5, packedmask25.v); r0 = _mm_add_epi64(r0, _mm_unpackhi_epi64(_mm_mul_epu32(c1, packednineteen.v), c3));
  471. c1 = _mm_srli_epi64(r0, 26); c2 = _mm_srli_epi64(r2, 26); r0 = _mm_and_si128(r0, packedmask26.v); r2 = _mm_and_si128(r2, packedmask26.v); r1 = _mm_add_epi64(r1, c1); r3 = _mm_add_epi64(r3, c2);
  472. r01 = _mm_unpacklo_epi64(r0, r1);
  473. r45 = _mm_unpackhi_epi64(r0, r1);
  474. r23 = _mm_unpacklo_epi64(r2, r3);
  475. r67 = _mm_unpackhi_epi64(r2, r3);
  476. r89 = _mm_unpackhi_epi64(r4, r5);
  477. } while (--count);
  478. r0123 = _mm_shuffle_epi32(r23, _MM_SHUFFLE(2,0,3,3));
  479. r4567 = _mm_shuffle_epi32(r67, _MM_SHUFFLE(2,0,3,3));
  480. r0123 = _mm_or_si128(r0123, _mm_shuffle_epi32(r01, _MM_SHUFFLE(3,3,2,0)));
  481. r4567 = _mm_or_si128(r4567, _mm_shuffle_epi32(r45, _MM_SHUFFLE(3,3,2,0)));
  482. r89 = _mm_shuffle_epi32(r89, _MM_SHUFFLE(3,3,2,0));
  483. _mm_store_si128((xmmi*)r + 0, r0123);
  484. _mm_store_si128((xmmi*)r + 1, r4567);
  485. _mm_store_si128((xmmi*)r + 2, r89);
  486. }
  487. DONNA_INLINE static void
  488. curve25519_tangle32(packedelem32 *out, const bignum25519 x, const bignum25519 z) {
  489. xmmi x0,x1,x2,z0,z1,z2;
  490. x0 = _mm_load_si128((xmmi *)(x + 0));
  491. x1 = _mm_load_si128((xmmi *)(x + 4));
  492. x2 = _mm_load_si128((xmmi *)(x + 8));
  493. z0 = _mm_load_si128((xmmi *)(z + 0));
  494. z1 = _mm_load_si128((xmmi *)(z + 4));
  495. z2 = _mm_load_si128((xmmi *)(z + 8));
  496. out[0].v = _mm_unpacklo_epi32(x0, z0);
  497. out[1].v = _mm_unpackhi_epi32(x0, z0);
  498. out[2].v = _mm_unpacklo_epi32(x1, z1);
  499. out[3].v = _mm_unpackhi_epi32(x1, z1);
  500. out[4].v = _mm_unpacklo_epi32(x2, z2);
  501. }
  502. DONNA_INLINE static void
  503. curve25519_untangle32(bignum25519 x, bignum25519 z, const packedelem32 *in) {
  504. xmmi t0,t1,t2,t3,t4,zero;
  505. t0 = _mm_shuffle_epi32(in[0].v, _MM_SHUFFLE(3,1,2,0));
  506. t1 = _mm_shuffle_epi32(in[1].v, _MM_SHUFFLE(3,1,2,0));
  507. t2 = _mm_shuffle_epi32(in[2].v, _MM_SHUFFLE(3,1,2,0));
  508. t3 = _mm_shuffle_epi32(in[3].v, _MM_SHUFFLE(3,1,2,0));
  509. t4 = _mm_shuffle_epi32(in[4].v, _MM_SHUFFLE(3,1,2,0));
  510. zero = _mm_setzero_si128();
  511. _mm_store_si128((xmmi *)x + 0, _mm_unpacklo_epi64(t0, t1));
  512. _mm_store_si128((xmmi *)x + 1, _mm_unpacklo_epi64(t2, t3));
  513. _mm_store_si128((xmmi *)x + 2, _mm_unpacklo_epi64(t4, zero));
  514. _mm_store_si128((xmmi *)z + 0, _mm_unpackhi_epi64(t0, t1));
  515. _mm_store_si128((xmmi *)z + 1, _mm_unpackhi_epi64(t2, t3));
  516. _mm_store_si128((xmmi *)z + 2, _mm_unpackhi_epi64(t4, zero));
  517. }
  518. DONNA_INLINE static void
  519. curve25519_add_reduce_packed32(packedelem32 *out, const packedelem32 *r, const packedelem32 *s) {
  520. xmmi r0,r1,r2,r3,r4;
  521. xmmi s0,s1,s2,s3,s4,s5;
  522. xmmi c1,c2;
  523. r0 = _mm_add_epi32(r[0].v, s[0].v);
  524. r1 = _mm_add_epi32(r[1].v, s[1].v);
  525. r2 = _mm_add_epi32(r[2].v, s[2].v);
  526. r3 = _mm_add_epi32(r[3].v, s[3].v);
  527. r4 = _mm_add_epi32(r[4].v, s[4].v);
  528. s0 = _mm_unpacklo_epi64(r0, r2); /* 00 44 */
  529. s1 = _mm_unpackhi_epi64(r0, r2); /* 11 55 */
  530. s2 = _mm_unpacklo_epi64(r1, r3); /* 22 66 */
  531. s3 = _mm_unpackhi_epi64(r1, r3); /* 33 77 */
  532. s4 = _mm_unpacklo_epi64(_mm_setzero_si128(), r4); /* 00 88 */
  533. s5 = _mm_unpackhi_epi64(_mm_setzero_si128(), r4); /* 00 99 */
  534. c1 = _mm_srli_epi32(s0, 26); c2 = _mm_srli_epi32(s2, 26); s0 = _mm_and_si128(s0, packedmask26262626.v); s2 = _mm_and_si128(s2, packedmask26262626.v); s1 = _mm_add_epi32(s1, c1); s3 = _mm_add_epi32(s3, c2);
  535. c1 = _mm_srli_epi32(s1, 25); c2 = _mm_srli_epi32(s3, 25); s1 = _mm_and_si128(s1, packedmask25252525.v); s3 = _mm_and_si128(s3, packedmask25252525.v); s2 = _mm_add_epi32(s2, c1); s4 = _mm_add_epi32(s4, _mm_unpackhi_epi64(_mm_setzero_si128(), c2)); s0 = _mm_add_epi32(s0, _mm_unpacklo_epi64(_mm_setzero_si128(), c2));
  536. c1 = _mm_srli_epi32(s2, 26); c2 = _mm_srli_epi32(s4, 26); s2 = _mm_and_si128(s2, packedmask26262626.v); s4 = _mm_and_si128(s4, packedmask26262626.v); s3 = _mm_add_epi32(s3, c1); s5 = _mm_add_epi32(s5, c2);
  537. c1 = _mm_srli_epi32(s3, 25); c2 = _mm_srli_epi32(s5, 25); s3 = _mm_and_si128(s3, packedmask25252525.v); s5 = _mm_and_si128(s5, packedmask25252525.v); s4 = _mm_add_epi32(s4, c1); s0 = _mm_add_epi32(s0, _mm_or_si128(_mm_slli_si128(c1, 8), _mm_srli_si128(_mm_add_epi32(_mm_add_epi32(_mm_slli_epi32(c2, 4), _mm_slli_epi32(c2, 1)), c2), 8)));
  538. c1 = _mm_srli_epi32(s0, 26); c2 = _mm_srli_epi32(s2, 26); s0 = _mm_and_si128(s0, packedmask26262626.v); s2 = _mm_and_si128(s2, packedmask26262626.v); s1 = _mm_add_epi32(s1, c1); s3 = _mm_add_epi32(s3, c2);
  539. out[0].v = _mm_unpacklo_epi64(s0, s1); /* 00 11 */
  540. out[1].v = _mm_unpacklo_epi64(s2, s3); /* 22 33 */
  541. out[2].v = _mm_unpackhi_epi64(s0, s1); /* 44 55 */
  542. out[3].v = _mm_unpackhi_epi64(s2, s3); /* 66 77 */
  543. out[4].v = _mm_unpackhi_epi64(s4, s5); /* 88 99 */
  544. }
  545. DONNA_INLINE static void
  546. curve25519_add_packed32(packedelem32 *out, const packedelem32 *r, const packedelem32 *s) {
  547. out[0].v = _mm_add_epi32(r[0].v, s[0].v);
  548. out[1].v = _mm_add_epi32(r[1].v, s[1].v);
  549. out[2].v = _mm_add_epi32(r[2].v, s[2].v);
  550. out[3].v = _mm_add_epi32(r[3].v, s[3].v);
  551. out[4].v = _mm_add_epi32(r[4].v, s[4].v);
  552. }
  553. DONNA_INLINE static void
  554. curve25519_sub_packed32(packedelem32 *out, const packedelem32 *r, const packedelem32 *s) {
  555. xmmi r0,r1,r2,r3,r4;
  556. xmmi s0,s1,s2,s3;
  557. xmmi c1,c2;
  558. r0 = _mm_add_epi32(r[0].v, packed32packed2p0.v);
  559. r1 = _mm_add_epi32(r[1].v, packed32packed2p1.v);
  560. r2 = _mm_add_epi32(r[2].v, packed32packed2p1.v);
  561. r3 = _mm_add_epi32(r[3].v, packed32packed2p1.v);
  562. r4 = _mm_add_epi32(r[4].v, packed32packed2p1.v);
  563. r0 = _mm_sub_epi32(r0, s[0].v); /* 00 11 */
  564. r1 = _mm_sub_epi32(r1, s[1].v); /* 22 33 */
  565. r2 = _mm_sub_epi32(r2, s[2].v); /* 44 55 */
  566. r3 = _mm_sub_epi32(r3, s[3].v); /* 66 77 */
  567. r4 = _mm_sub_epi32(r4, s[4].v); /* 88 99 */
  568. s0 = _mm_unpacklo_epi64(r0, r2); /* 00 44 */
  569. s1 = _mm_unpackhi_epi64(r0, r2); /* 11 55 */
  570. s2 = _mm_unpacklo_epi64(r1, r3); /* 22 66 */
  571. s3 = _mm_unpackhi_epi64(r1, r3); /* 33 77 */
  572. c1 = _mm_srli_epi32(s0, 26); c2 = _mm_srli_epi32(s2, 26); s0 = _mm_and_si128(s0, packedmask26262626.v); s2 = _mm_and_si128(s2, packedmask26262626.v); s1 = _mm_add_epi32(s1, c1); s3 = _mm_add_epi32(s3, c2);
  573. c1 = _mm_srli_epi32(s1, 25); c2 = _mm_srli_epi32(s3, 25); s1 = _mm_and_si128(s1, packedmask25252525.v); s3 = _mm_and_si128(s3, packedmask25252525.v); s2 = _mm_add_epi32(s2, c1); r4 = _mm_add_epi32(r4, _mm_srli_si128(c2, 8)); s0 = _mm_add_epi32(s0, _mm_slli_si128(c2, 8));
  574. out[0].v = _mm_unpacklo_epi64(s0, s1); /* 00 11 */
  575. out[1].v = _mm_unpacklo_epi64(s2, s3); /* 22 33 */
  576. out[2].v = _mm_unpackhi_epi64(s0, s1); /* 44 55 */
  577. out[3].v = _mm_unpackhi_epi64(s2, s3); /* 66 77 */
  578. out[4].v = r4;
  579. }
  580. DONNA_INLINE static void
  581. curve25519_sub_after_basic_packed32(packedelem32 *out, const packedelem32 *r, const packedelem32 *s) {
  582. xmmi r0,r1,r2,r3,r4;
  583. xmmi s0,s1,s2,s3,s4,s5;
  584. xmmi c1,c2;
  585. r0 = _mm_add_epi32(r[0].v, packed32packed4p0.v);
  586. r1 = _mm_add_epi32(r[1].v, packed32packed4p1.v);
  587. r2 = _mm_add_epi32(r[2].v, packed32packed4p1.v);
  588. r3 = _mm_add_epi32(r[3].v, packed32packed4p1.v);
  589. r4 = _mm_add_epi32(r[4].v, packed32packed4p1.v);
  590. r0 = _mm_sub_epi32(r0, s[0].v); /* 00 11 */
  591. r1 = _mm_sub_epi32(r1, s[1].v); /* 22 33 */
  592. r2 = _mm_sub_epi32(r2, s[2].v); /* 44 55 */
  593. r3 = _mm_sub_epi32(r3, s[3].v); /* 66 77 */
  594. r4 = _mm_sub_epi32(r4, s[4].v); /* 88 99 */
  595. s0 = _mm_unpacklo_epi64(r0, r2); /* 00 44 */
  596. s1 = _mm_unpackhi_epi64(r0, r2); /* 11 55 */
  597. s2 = _mm_unpacklo_epi64(r1, r3); /* 22 66 */
  598. s3 = _mm_unpackhi_epi64(r1, r3); /* 33 77 */
  599. s4 = _mm_unpacklo_epi64(_mm_setzero_si128(), r4); /* 00 88 */
  600. s5 = _mm_unpackhi_epi64(_mm_setzero_si128(), r4); /* 00 99 */
  601. c1 = _mm_srli_epi32(s0, 26); c2 = _mm_srli_epi32(s2, 26); s0 = _mm_and_si128(s0, packedmask26262626.v); s2 = _mm_and_si128(s2, packedmask26262626.v); s1 = _mm_add_epi32(s1, c1); s3 = _mm_add_epi32(s3, c2);
  602. c1 = _mm_srli_epi32(s1, 25); c2 = _mm_srli_epi32(s3, 25); s1 = _mm_and_si128(s1, packedmask25252525.v); s3 = _mm_and_si128(s3, packedmask25252525.v); s2 = _mm_add_epi32(s2, c1); s4 = _mm_add_epi32(s4, _mm_unpackhi_epi64(_mm_setzero_si128(), c2)); s0 = _mm_add_epi32(s0, _mm_unpacklo_epi64(_mm_setzero_si128(), c2));
  603. c1 = _mm_srli_epi32(s2, 26); c2 = _mm_srli_epi32(s4, 26); s2 = _mm_and_si128(s2, packedmask26262626.v); s4 = _mm_and_si128(s4, packedmask26262626.v); s3 = _mm_add_epi32(s3, c1); s5 = _mm_add_epi32(s5, c2);
  604. c1 = _mm_srli_epi32(s3, 25); c2 = _mm_srli_epi32(s5, 25); s3 = _mm_and_si128(s3, packedmask25252525.v); s5 = _mm_and_si128(s5, packedmask25252525.v); s4 = _mm_add_epi32(s4, c1); s0 = _mm_add_epi32(s0, _mm_or_si128(_mm_slli_si128(c1, 8), _mm_srli_si128(_mm_add_epi32(_mm_add_epi32(_mm_slli_epi32(c2, 4), _mm_slli_epi32(c2, 1)), c2), 8)));
  605. c1 = _mm_srli_epi32(s0, 26); c2 = _mm_srli_epi32(s2, 26); s0 = _mm_and_si128(s0, packedmask26262626.v); s2 = _mm_and_si128(s2, packedmask26262626.v); s1 = _mm_add_epi32(s1, c1); s3 = _mm_add_epi32(s3, c2);
  606. out[0].v = _mm_unpacklo_epi64(s0, s1); /* 00 11 */
  607. out[1].v = _mm_unpacklo_epi64(s2, s3); /* 22 33 */
  608. out[2].v = _mm_unpackhi_epi64(s0, s1); /* 44 55 */
  609. out[3].v = _mm_unpackhi_epi64(s2, s3); /* 66 77 */
  610. out[4].v = _mm_unpackhi_epi64(s4, s5); /* 88 99 */
  611. }
  612. DONNA_INLINE static void
  613. curve25519_tangle64_from32(packedelem64 *a, packedelem64 *b, const packedelem32 *c, const packedelem32 *d) {
  614. xmmi c0,c1,c2,c3,c4,c5,t;
  615. xmmi d0,d1,d2,d3,d4,d5;
  616. xmmi t0,t1,t2,t3,t4,zero;
  617. t0 = _mm_shuffle_epi32(c[0].v, _MM_SHUFFLE(3,1,2,0));
  618. t1 = _mm_shuffle_epi32(c[1].v, _MM_SHUFFLE(3,1,2,0));
  619. t2 = _mm_shuffle_epi32(d[0].v, _MM_SHUFFLE(3,1,2,0));
  620. t3 = _mm_shuffle_epi32(d[1].v, _MM_SHUFFLE(3,1,2,0));
  621. c0 = _mm_unpacklo_epi64(t0, t1);
  622. c3 = _mm_unpackhi_epi64(t0, t1);
  623. d0 = _mm_unpacklo_epi64(t2, t3);
  624. d3 = _mm_unpackhi_epi64(t2, t3);
  625. t = _mm_unpacklo_epi64(c0, d0); a[0].v = t; a[1].v = _mm_srli_epi64(t, 32);
  626. t = _mm_unpackhi_epi64(c0, d0); a[2].v = t; a[3].v = _mm_srli_epi64(t, 32);
  627. t = _mm_unpacklo_epi64(c3, d3); b[0].v = t; b[1].v = _mm_srli_epi64(t, 32);
  628. t = _mm_unpackhi_epi64(c3, d3); b[2].v = t; b[3].v = _mm_srli_epi64(t, 32);
  629. t0 = _mm_shuffle_epi32(c[2].v, _MM_SHUFFLE(3,1,2,0));
  630. t1 = _mm_shuffle_epi32(c[3].v, _MM_SHUFFLE(3,1,2,0));
  631. t2 = _mm_shuffle_epi32(d[2].v, _MM_SHUFFLE(3,1,2,0));
  632. t3 = _mm_shuffle_epi32(d[3].v, _MM_SHUFFLE(3,1,2,0));
  633. c1 = _mm_unpacklo_epi64(t0, t1);
  634. c4 = _mm_unpackhi_epi64(t0, t1);
  635. d1 = _mm_unpacklo_epi64(t2, t3);
  636. d4 = _mm_unpackhi_epi64(t2, t3);
  637. t = _mm_unpacklo_epi64(c1, d1); a[4].v = t; a[5].v = _mm_srli_epi64(t, 32);
  638. t = _mm_unpackhi_epi64(c1, d1); a[6].v = t; a[7].v = _mm_srli_epi64(t, 32);
  639. t = _mm_unpacklo_epi64(c4, d4); b[4].v = t; b[5].v = _mm_srli_epi64(t, 32);
  640. t = _mm_unpackhi_epi64(c4, d4); b[6].v = t; b[7].v = _mm_srli_epi64(t, 32);
  641. t4 = _mm_shuffle_epi32(c[4].v, _MM_SHUFFLE(3,1,2,0));
  642. zero = _mm_setzero_si128();
  643. c2 = _mm_unpacklo_epi64(t4, zero);
  644. c5 = _mm_unpackhi_epi64(t4, zero);
  645. t4 = _mm_shuffle_epi32(d[4].v, _MM_SHUFFLE(3,1,2,0));
  646. d2 = _mm_unpacklo_epi64(t4, zero);
  647. d5 = _mm_unpackhi_epi64(t4, zero);
  648. t = _mm_unpacklo_epi64(c2, d2); a[8].v = t; a[9].v = _mm_srli_epi64(t, 32);
  649. t = _mm_unpacklo_epi64(c5, d5); b[8].v = t; b[9].v = _mm_srli_epi64(t, 32);
  650. }
  651. DONNA_INLINE static void
  652. curve25519_tangle64(packedelem64 *out, const bignum25519 x, const bignum25519 z) {
  653. xmmi x0,x1,x2,z0,z1,z2,t;
  654. x0 = _mm_load_si128((xmmi *)x + 0);
  655. x1 = _mm_load_si128((xmmi *)x + 1);
  656. x2 = _mm_load_si128((xmmi *)x + 2);
  657. z0 = _mm_load_si128((xmmi *)z + 0);
  658. z1 = _mm_load_si128((xmmi *)z + 1);
  659. z2 = _mm_load_si128((xmmi *)z + 2);
  660. t = _mm_unpacklo_epi64(x0, z0); out[0].v = t; out[1].v = _mm_srli_epi64(t, 32);
  661. t = _mm_unpackhi_epi64(x0, z0); out[2].v = t; out[3].v = _mm_srli_epi64(t, 32);
  662. t = _mm_unpacklo_epi64(x1, z1); out[4].v = t; out[5].v = _mm_srli_epi64(t, 32);
  663. t = _mm_unpackhi_epi64(x1, z1); out[6].v = t; out[7].v = _mm_srli_epi64(t, 32);
  664. t = _mm_unpacklo_epi64(x2, z2); out[8].v = t; out[9].v = _mm_srli_epi64(t, 32);
  665. }
  666. DONNA_INLINE static void
  667. curve25519_tangleone64(packedelem64 *out, const bignum25519 x) {
  668. xmmi x0,x1,x2;
  669. x0 = _mm_load_si128((xmmi *)(x + 0));
  670. x1 = _mm_load_si128((xmmi *)(x + 4));
  671. x2 = _mm_load_si128((xmmi *)(x + 8));
  672. out[0].v = _mm_shuffle_epi32(x0, _MM_SHUFFLE(0,0,0,0));
  673. out[1].v = _mm_shuffle_epi32(x0, _MM_SHUFFLE(1,1,1,1));
  674. out[2].v = _mm_shuffle_epi32(x0, _MM_SHUFFLE(2,2,2,2));
  675. out[3].v = _mm_shuffle_epi32(x0, _MM_SHUFFLE(3,3,3,3));
  676. out[4].v = _mm_shuffle_epi32(x1, _MM_SHUFFLE(0,0,0,0));
  677. out[5].v = _mm_shuffle_epi32(x1, _MM_SHUFFLE(1,1,1,1));
  678. out[6].v = _mm_shuffle_epi32(x1, _MM_SHUFFLE(2,2,2,2));
  679. out[7].v = _mm_shuffle_epi32(x1, _MM_SHUFFLE(3,3,3,3));
  680. out[8].v = _mm_shuffle_epi32(x2, _MM_SHUFFLE(0,0,0,0));
  681. out[9].v = _mm_shuffle_epi32(x2, _MM_SHUFFLE(1,1,1,1));
  682. }
  683. DONNA_INLINE static void
  684. curve25519_swap64(packedelem64 *out) {
  685. out[0].v = _mm_shuffle_epi32(out[0].v, _MM_SHUFFLE(1,0,3,2));
  686. out[1].v = _mm_shuffle_epi32(out[1].v, _MM_SHUFFLE(1,0,3,2));
  687. out[2].v = _mm_shuffle_epi32(out[2].v, _MM_SHUFFLE(1,0,3,2));
  688. out[3].v = _mm_shuffle_epi32(out[3].v, _MM_SHUFFLE(1,0,3,2));
  689. out[4].v = _mm_shuffle_epi32(out[4].v, _MM_SHUFFLE(1,0,3,2));
  690. out[5].v = _mm_shuffle_epi32(out[5].v, _MM_SHUFFLE(1,0,3,2));
  691. out[6].v = _mm_shuffle_epi32(out[6].v, _MM_SHUFFLE(1,0,3,2));
  692. out[7].v = _mm_shuffle_epi32(out[7].v, _MM_SHUFFLE(1,0,3,2));
  693. out[8].v = _mm_shuffle_epi32(out[8].v, _MM_SHUFFLE(1,0,3,2));
  694. out[9].v = _mm_shuffle_epi32(out[9].v, _MM_SHUFFLE(1,0,3,2));
  695. }
  696. DONNA_INLINE static void
  697. curve25519_untangle64(bignum25519 x, bignum25519 z, const packedelem64 *in) {
  698. _mm_store_si128((xmmi *)(x + 0), _mm_unpacklo_epi64(_mm_unpacklo_epi32(in[0].v, in[1].v), _mm_unpacklo_epi32(in[2].v, in[3].v)));
  699. _mm_store_si128((xmmi *)(x + 4), _mm_unpacklo_epi64(_mm_unpacklo_epi32(in[4].v, in[5].v), _mm_unpacklo_epi32(in[6].v, in[7].v)));
  700. _mm_store_si128((xmmi *)(x + 8), _mm_unpacklo_epi32(in[8].v, in[9].v) );
  701. _mm_store_si128((xmmi *)(z + 0), _mm_unpacklo_epi64(_mm_unpackhi_epi32(in[0].v, in[1].v), _mm_unpackhi_epi32(in[2].v, in[3].v)));
  702. _mm_store_si128((xmmi *)(z + 4), _mm_unpacklo_epi64(_mm_unpackhi_epi32(in[4].v, in[5].v), _mm_unpackhi_epi32(in[6].v, in[7].v)));
  703. _mm_store_si128((xmmi *)(z + 8), _mm_unpackhi_epi32(in[8].v, in[9].v) );
  704. }
  705. DONNA_INLINE static void
  706. curve25519_mul_packed64(packedelem64 *out, const packedelem64 *r, const packedelem64 *s) {
  707. xmmi r1,r2,r3,r4,r5,r6,r7,r8,r9;
  708. xmmi r1_2,r3_2,r5_2,r7_2,r9_2;
  709. xmmi c1,c2;
  710. out[0].v = _mm_mul_epu32(r[0].v, s[0].v);
  711. out[1].v = _mm_add_epi64(_mm_mul_epu32(r[0].v, s[1].v), _mm_mul_epu32(r[1].v, s[0].v));
  712. r1_2 = _mm_slli_epi32(r[1].v, 1);
  713. out[2].v = _mm_add_epi64(_mm_mul_epu32(r[0].v, s[2].v), _mm_add_epi64(_mm_mul_epu32(r1_2 , s[1].v), _mm_mul_epu32(r[2].v, s[0].v)));
  714. out[3].v = _mm_add_epi64(_mm_mul_epu32(r[0].v, s[3].v), _mm_add_epi64(_mm_mul_epu32(r[1].v, s[2].v), _mm_add_epi64(_mm_mul_epu32(r[2].v, s[1].v), _mm_mul_epu32(r[3].v, s[0].v))));
  715. r3_2 = _mm_slli_epi32(r[3].v, 1);
  716. out[4].v = _mm_add_epi64(_mm_mul_epu32(r[0].v, s[4].v), _mm_add_epi64(_mm_mul_epu32(r1_2 , s[3].v), _mm_add_epi64(_mm_mul_epu32(r[2].v, s[2].v), _mm_add_epi64(_mm_mul_epu32(r3_2 , s[1].v), _mm_mul_epu32(r[4].v, s[0].v)))));
  717. out[5].v = _mm_add_epi64(_mm_mul_epu32(r[0].v, s[5].v), _mm_add_epi64(_mm_mul_epu32(r[1].v, s[4].v), _mm_add_epi64(_mm_mul_epu32(r[2].v, s[3].v), _mm_add_epi64(_mm_mul_epu32(r[3].v, s[2].v), _mm_add_epi64(_mm_mul_epu32(r[4].v, s[1].v), _mm_mul_epu32(r[5].v, s[0].v))))));
  718. r5_2 = _mm_slli_epi32(r[5].v, 1);
  719. out[6].v = _mm_add_epi64(_mm_mul_epu32(r[0].v, s[6].v), _mm_add_epi64(_mm_mul_epu32(r1_2 , s[5].v), _mm_add_epi64(_mm_mul_epu32(r[2].v, s[4].v), _mm_add_epi64(_mm_mul_epu32(r3_2 , s[3].v), _mm_add_epi64(_mm_mul_epu32(r[4].v, s[2].v), _mm_add_epi64(_mm_mul_epu32(r5_2 , s[1].v), _mm_mul_epu32(r[6].v, s[0].v)))))));
  720. out[7].v = _mm_add_epi64(_mm_mul_epu32(r[0].v, s[7].v), _mm_add_epi64(_mm_mul_epu32(r[1].v, s[6].v), _mm_add_epi64(_mm_mul_epu32(r[2].v, s[5].v), _mm_add_epi64(_mm_mul_epu32(r[3].v, s[4].v), _mm_add_epi64(_mm_mul_epu32(r[4].v, s[3].v), _mm_add_epi64(_mm_mul_epu32(r[5].v, s[2].v), _mm_add_epi64(_mm_mul_epu32(r[6].v, s[1].v), _mm_mul_epu32(r[7].v , s[0].v))))))));
  721. r7_2 = _mm_slli_epi32(r[7].v, 1);
  722. out[8].v = _mm_add_epi64(_mm_mul_epu32(r[0].v, s[8].v), _mm_add_epi64(_mm_mul_epu32(r1_2 , s[7].v), _mm_add_epi64(_mm_mul_epu32(r[2].v, s[6].v), _mm_add_epi64(_mm_mul_epu32(r3_2 , s[5].v), _mm_add_epi64(_mm_mul_epu32(r[4].v, s[4].v), _mm_add_epi64(_mm_mul_epu32(r5_2 , s[3].v), _mm_add_epi64(_mm_mul_epu32(r[6].v, s[2].v), _mm_add_epi64(_mm_mul_epu32(r7_2 , s[1].v), _mm_mul_epu32(r[8].v, s[0].v)))))))));
  723. out[9].v = _mm_add_epi64(_mm_mul_epu32(r[0].v, s[9].v), _mm_add_epi64(_mm_mul_epu32(r[1].v, s[8].v), _mm_add_epi64(_mm_mul_epu32(r[2].v, s[7].v), _mm_add_epi64(_mm_mul_epu32(r[3].v, s[6].v), _mm_add_epi64(_mm_mul_epu32(r[4].v, s[5].v), _mm_add_epi64(_mm_mul_epu32(r[5].v, s[4].v), _mm_add_epi64(_mm_mul_epu32(r[6].v, s[3].v), _mm_add_epi64(_mm_mul_epu32(r[7].v, s[2].v), _mm_add_epi64(_mm_mul_epu32(r[8].v, s[1].v), _mm_mul_epu32(r[9].v, s[0].v))))))))));
  724. r1 = _mm_mul_epu32(r[1].v, packednineteen.v);
  725. r2 = _mm_mul_epu32(r[2].v, packednineteen.v);
  726. r1_2 = _mm_slli_epi32(r1, 1);
  727. r3 = _mm_mul_epu32(r[3].v, packednineteen.v);
  728. r4 = _mm_mul_epu32(r[4].v, packednineteen.v);
  729. r3_2 = _mm_slli_epi32(r3, 1);
  730. r5 = _mm_mul_epu32(r[5].v, packednineteen.v);
  731. r6 = _mm_mul_epu32(r[6].v, packednineteen.v);
  732. r5_2 = _mm_slli_epi32(r5, 1);
  733. r7 = _mm_mul_epu32(r[7].v, packednineteen.v);
  734. r8 = _mm_mul_epu32(r[8].v, packednineteen.v);
  735. r7_2 = _mm_slli_epi32(r7, 1);
  736. r9 = _mm_mul_epu32(r[9].v, packednineteen.v);
  737. r9_2 = _mm_slli_epi32(r9, 1);
  738. out[0].v = _mm_add_epi64(out[0].v, _mm_add_epi64(_mm_mul_epu32(r9_2, s[1].v), _mm_add_epi64(_mm_mul_epu32(r8, s[2].v), _mm_add_epi64(_mm_mul_epu32(r7_2, s[3].v), _mm_add_epi64(_mm_mul_epu32(r6, s[4].v), _mm_add_epi64(_mm_mul_epu32(r5_2, s[5].v), _mm_add_epi64(_mm_mul_epu32(r4, s[6].v), _mm_add_epi64(_mm_mul_epu32(r3_2, s[7].v), _mm_add_epi64(_mm_mul_epu32(r2, s[8].v), _mm_mul_epu32(r1_2, s[9].v))))))))));
  739. out[1].v = _mm_add_epi64(out[1].v, _mm_add_epi64(_mm_mul_epu32(r9 , s[2].v), _mm_add_epi64(_mm_mul_epu32(r8, s[3].v), _mm_add_epi64(_mm_mul_epu32(r7 , s[4].v), _mm_add_epi64(_mm_mul_epu32(r6, s[5].v), _mm_add_epi64(_mm_mul_epu32(r5 , s[6].v), _mm_add_epi64(_mm_mul_epu32(r4, s[7].v), _mm_add_epi64(_mm_mul_epu32(r3 , s[8].v), _mm_mul_epu32(r2, s[9].v)))))))));
  740. out[2].v = _mm_add_epi64(out[2].v, _mm_add_epi64(_mm_mul_epu32(r9_2, s[3].v), _mm_add_epi64(_mm_mul_epu32(r8, s[4].v), _mm_add_epi64(_mm_mul_epu32(r7_2, s[5].v), _mm_add_epi64(_mm_mul_epu32(r6, s[6].v), _mm_add_epi64(_mm_mul_epu32(r5_2, s[7].v), _mm_add_epi64(_mm_mul_epu32(r4, s[8].v), _mm_mul_epu32(r3_2, s[9].v))))))));
  741. out[3].v = _mm_add_epi64(out[3].v, _mm_add_epi64(_mm_mul_epu32(r9 , s[4].v), _mm_add_epi64(_mm_mul_epu32(r8, s[5].v), _mm_add_epi64(_mm_mul_epu32(r7 , s[6].v), _mm_add_epi64(_mm_mul_epu32(r6, s[7].v), _mm_add_epi64(_mm_mul_epu32(r5 , s[8].v), _mm_mul_epu32(r4, s[9].v)))))));
  742. out[4].v = _mm_add_epi64(out[4].v, _mm_add_epi64(_mm_mul_epu32(r9_2, s[5].v), _mm_add_epi64(_mm_mul_epu32(r8, s[6].v), _mm_add_epi64(_mm_mul_epu32(r7_2, s[7].v), _mm_add_epi64(_mm_mul_epu32(r6, s[8].v), _mm_mul_epu32(r5_2, s[9].v))))));
  743. out[5].v = _mm_add_epi64(out[5].v, _mm_add_epi64(_mm_mul_epu32(r9 , s[6].v), _mm_add_epi64(_mm_mul_epu32(r8, s[7].v), _mm_add_epi64(_mm_mul_epu32(r7 , s[8].v), _mm_mul_epu32(r6, s[9].v)))));
  744. out[6].v = _mm_add_epi64(out[6].v, _mm_add_epi64(_mm_mul_epu32(r9_2, s[7].v), _mm_add_epi64(_mm_mul_epu32(r8, s[8].v), _mm_mul_epu32(r7_2, s[9].v))));
  745. out[7].v = _mm_add_epi64(out[7].v, _mm_add_epi64(_mm_mul_epu32(r9 , s[8].v), _mm_mul_epu32(r8, s[9].v)));
  746. out[8].v = _mm_add_epi64(out[8].v, _mm_mul_epu32(r9_2, s[9].v));
  747. c1 = _mm_srli_epi64(out[0].v, 26); c2 = _mm_srli_epi64(out[4].v, 26); out[0].v = _mm_and_si128(out[0].v, packedmask26.v); out[4].v = _mm_and_si128(out[4].v, packedmask26.v); out[1].v = _mm_add_epi64(out[1].v, c1); out[5].v = _mm_add_epi64(out[5].v, c2);
  748. c1 = _mm_srli_epi64(out[1].v, 25); c2 = _mm_srli_epi64(out[5].v, 25); out[1].v = _mm_and_si128(out[1].v, packedmask25.v); out[5].v = _mm_and_si128(out[5].v, packedmask25.v); out[2].v = _mm_add_epi64(out[2].v, c1); out[6].v = _mm_add_epi64(out[6].v, c2);
  749. c1 = _mm_srli_epi64(out[2].v, 26); c2 = _mm_srli_epi64(out[6].v, 26); out[2].v = _mm_and_si128(out[2].v, packedmask26.v); out[6].v = _mm_and_si128(out[6].v, packedmask26.v); out[3].v = _mm_add_epi64(out[3].v, c1); out[7].v = _mm_add_epi64(out[7].v, c2);
  750. c1 = _mm_srli_epi64(out[3].v, 25); c2 = _mm_srli_epi64(out[7].v, 25); out[3].v = _mm_and_si128(out[3].v, packedmask25.v); out[7].v = _mm_and_si128(out[7].v, packedmask25.v); out[4].v = _mm_add_epi64(out[4].v, c1); out[8].v = _mm_add_epi64(out[8].v, c2);
  751. c2 = _mm_srli_epi64(out[8].v, 26); out[8].v = _mm_and_si128(out[8].v, packedmask26.v); out[9].v = _mm_add_epi64(out[9].v, c2);
  752. c2 = _mm_srli_epi64(out[9].v, 25); out[9].v = _mm_and_si128(out[9].v, packedmask25.v); out[0].v = _mm_add_epi64(out[0].v, _mm_mul_epu32(c2, packednineteen.v));
  753. c1 = _mm_srli_epi64(out[0].v, 26); c2 = _mm_srli_epi64(out[4].v, 26); out[0].v = _mm_and_si128(out[0].v, packedmask26.v); out[4].v = _mm_and_si128(out[4].v, packedmask26.v); out[1].v = _mm_add_epi64(out[1].v, c1); out[5].v = _mm_add_epi64(out[5].v, c2);
  754. }
  755. DONNA_INLINE static void
  756. curve25519_square_packed64(packedelem64 *out, const packedelem64 *r) {
  757. xmmi r0,r1,r2,r3;
  758. xmmi r1_2,r3_2,r4_2,r5_2,r6_2,r7_2;
  759. xmmi d5,d6,d7,d8,d9;
  760. xmmi c1,c2;
  761. r0 = r[0].v;
  762. r1 = r[1].v;
  763. r2 = r[2].v;
  764. r3 = r[3].v;
  765. out[0].v = _mm_mul_epu32(r0, r0);
  766. r0 = _mm_slli_epi32(r0, 1);
  767. out[1].v = _mm_mul_epu32(r0, r1);
  768. r1_2 = _mm_slli_epi32(r1, 1);
  769. out[2].v = _mm_add_epi64(_mm_mul_epu32(r0, r2 ), _mm_mul_epu32(r1, r1_2));
  770. r1 = r1_2;
  771. out[3].v = _mm_add_epi64(_mm_mul_epu32(r0, r3 ), _mm_mul_epu32(r1, r2 ));
  772. r3_2 = _mm_slli_epi32(r3, 1);
  773. out[4].v = _mm_add_epi64(_mm_mul_epu32(r0, r[4].v), _mm_add_epi64(_mm_mul_epu32(r1, r3_2 ), _mm_mul_epu32(r2, r2)));
  774. r2 = _mm_slli_epi32(r2, 1);
  775. out[5].v = _mm_add_epi64(_mm_mul_epu32(r0, r[5].v), _mm_add_epi64(_mm_mul_epu32(r1, r[4].v), _mm_mul_epu32(r2, r3)));
  776. r5_2 = _mm_slli_epi32(r[5].v, 1);
  777. out[6].v = _mm_add_epi64(_mm_mul_epu32(r0, r[6].v), _mm_add_epi64(_mm_mul_epu32(r1, r5_2 ), _mm_add_epi64(_mm_mul_epu32(r2, r[4].v), _mm_mul_epu32(r3, r3_2 ))));
  778. r3 = r3_2;
  779. out[7].v = _mm_add_epi64(_mm_mul_epu32(r0, r[7].v), _mm_add_epi64(_mm_mul_epu32(r1, r[6].v), _mm_add_epi64(_mm_mul_epu32(r2, r[5].v), _mm_mul_epu32(r3, r[4].v))));
  780. r7_2 = _mm_slli_epi32(r[7].v, 1);
  781. out[8].v = _mm_add_epi64(_mm_mul_epu32(r0, r[8].v), _mm_add_epi64(_mm_mul_epu32(r1, r7_2 ), _mm_add_epi64(_mm_mul_epu32(r2, r[6].v), _mm_add_epi64(_mm_mul_epu32(r3, r5_2 ), _mm_mul_epu32(r[4].v, r[4].v)))));
  782. out[9].v = _mm_add_epi64(_mm_mul_epu32(r0, r[9].v), _mm_add_epi64(_mm_mul_epu32(r1, r[8].v), _mm_add_epi64(_mm_mul_epu32(r2, r[7].v), _mm_add_epi64(_mm_mul_epu32(r3, r[6].v), _mm_mul_epu32(r[4].v, r5_2 )))));
  783. d5 = _mm_mul_epu32(r[5].v, packedthirtyeight.v);
  784. d6 = _mm_mul_epu32(r[6].v, packednineteen.v);
  785. d7 = _mm_mul_epu32(r[7].v, packedthirtyeight.v);
  786. d8 = _mm_mul_epu32(r[8].v, packednineteen.v);
  787. d9 = _mm_mul_epu32(r[9].v, packedthirtyeight.v);
  788. r4_2 = _mm_slli_epi32(r[4].v, 1);
  789. r6_2 = _mm_slli_epi32(r[6].v, 1);
  790. out[0].v = _mm_add_epi64(out[0].v, _mm_add_epi64(_mm_mul_epu32(d9, r1 ), _mm_add_epi64(_mm_mul_epu32(d8, r2 ), _mm_add_epi64(_mm_mul_epu32(d7, r3 ), _mm_add_epi64(_mm_mul_epu32(d6, r4_2), _mm_mul_epu32(d5, r[5].v))))));
  791. out[1].v = _mm_add_epi64(out[1].v, _mm_add_epi64(_mm_mul_epu32(d9, _mm_srli_epi32(r2, 1)), _mm_add_epi64(_mm_mul_epu32(d8, r3 ), _mm_add_epi64(_mm_mul_epu32(d7, r[4].v), _mm_mul_epu32(d6, r5_2 )))));
  792. out[2].v = _mm_add_epi64(out[2].v, _mm_add_epi64(_mm_mul_epu32(d9, r3 ), _mm_add_epi64(_mm_mul_epu32(d8, r4_2), _mm_add_epi64(_mm_mul_epu32(d7, r5_2 ), _mm_mul_epu32(d6, r[6].v)))));
  793. out[3].v = _mm_add_epi64(out[3].v, _mm_add_epi64(_mm_mul_epu32(d9, r[4].v ), _mm_add_epi64(_mm_mul_epu32(d8, r5_2), _mm_mul_epu32(d7, r[6].v))));
  794. out[4].v = _mm_add_epi64(out[4].v, _mm_add_epi64(_mm_mul_epu32(d9, r5_2 ), _mm_add_epi64(_mm_mul_epu32(d8, r6_2), _mm_mul_epu32(d7, r[7].v))));
  795. out[5].v = _mm_add_epi64(out[5].v, _mm_add_epi64(_mm_mul_epu32(d9, r[6].v ), _mm_mul_epu32(d8, r7_2 )));
  796. out[6].v = _mm_add_epi64(out[6].v, _mm_add_epi64(_mm_mul_epu32(d9, r7_2 ), _mm_mul_epu32(d8, r[8].v)));
  797. out[7].v = _mm_add_epi64(out[7].v, _mm_mul_epu32(d9, r[8].v));
  798. out[8].v = _mm_add_epi64(out[8].v, _mm_mul_epu32(d9, r[9].v));
  799. c1 = _mm_srli_epi64(out[0].v, 26); c2 = _mm_srli_epi64(out[4].v, 26); out[0].v = _mm_and_si128(out[0].v, packedmask26.v); out[4].v = _mm_and_si128(out[4].v, packedmask26.v); out[1].v = _mm_add_epi64(out[1].v, c1); out[5].v = _mm_add_epi64(out[5].v, c2);
  800. c1 = _mm_srli_epi64(out[1].v, 25); c2 = _mm_srli_epi64(out[5].v, 25); out[1].v = _mm_and_si128(out[1].v, packedmask25.v); out[5].v = _mm_and_si128(out[5].v, packedmask25.v); out[2].v = _mm_add_epi64(out[2].v, c1); out[6].v = _mm_add_epi64(out[6].v, c2);
  801. c1 = _mm_srli_epi64(out[2].v, 26); c2 = _mm_srli_epi64(out[6].v, 26); out[2].v = _mm_and_si128(out[2].v, packedmask26.v); out[6].v = _mm_and_si128(out[6].v, packedmask26.v); out[3].v = _mm_add_epi64(out[3].v, c1); out[7].v = _mm_add_epi64(out[7].v, c2);
  802. c1 = _mm_srli_epi64(out[3].v, 25); c2 = _mm_srli_epi64(out[7].v, 25); out[3].v = _mm_and_si128(out[3].v, packedmask25.v); out[7].v = _mm_and_si128(out[7].v, packedmask25.v); out[4].v = _mm_add_epi64(out[4].v, c1); out[8].v = _mm_add_epi64(out[8].v, c2);
  803. c2 = _mm_srli_epi64(out[8].v, 26); out[8].v = _mm_and_si128(out[8].v, packedmask26.v); out[9].v = _mm_add_epi64(out[9].v, c2);
  804. c2 = _mm_srli_epi64(out[9].v, 25); out[9].v = _mm_and_si128(out[9].v, packedmask25.v); out[0].v = _mm_add_epi64(out[0].v, _mm_mul_epu32(c2, packednineteen.v));
  805. c1 = _mm_srli_epi64(out[0].v, 26); c2 = _mm_srli_epi64(out[4].v, 26); out[0].v = _mm_and_si128(out[0].v, packedmask26.v); out[4].v = _mm_and_si128(out[4].v, packedmask26.v); out[1].v = _mm_add_epi64(out[1].v, c1); out[5].v = _mm_add_epi64(out[5].v, c2);
  806. }
  807. /* Take a little-endian, 32-byte number and expand it into polynomial form */
  808. static void
  809. curve25519_expand(bignum25519 out, const unsigned char in[32]) {
  810. uint32_t x0,x1,x2,x3,x4,x5,x6,x7;
  811. x0 = *(uint32_t *)(in + 0);
  812. x1 = *(uint32_t *)(in + 4);
  813. x2 = *(uint32_t *)(in + 8);
  814. x3 = *(uint32_t *)(in + 12);
  815. x4 = *(uint32_t *)(in + 16);
  816. x5 = *(uint32_t *)(in + 20);
  817. x6 = *(uint32_t *)(in + 24);
  818. x7 = *(uint32_t *)(in + 28);
  819. out[0] = ( x0 ) & 0x3ffffff;
  820. out[1] = ((((uint64_t)x1 << 32) | x0) >> 26) & 0x1ffffff;
  821. out[2] = ((((uint64_t)x2 << 32) | x1) >> 19) & 0x3ffffff;
  822. out[3] = ((((uint64_t)x3 << 32) | x2) >> 13) & 0x1ffffff;
  823. out[4] = (( x3) >> 6) & 0x3ffffff;
  824. out[5] = ( x4 ) & 0x1ffffff;
  825. out[6] = ((((uint64_t)x5 << 32) | x4) >> 25) & 0x3ffffff;
  826. out[7] = ((((uint64_t)x6 << 32) | x5) >> 19) & 0x1ffffff;
  827. out[8] = ((((uint64_t)x7 << 32) | x6) >> 12) & 0x3ffffff;
  828. out[9] = (( x7) >> 6) & 0x1ffffff;
  829. out[10] = 0;
  830. out[11] = 0;
  831. }
  832. /* Take a fully reduced polynomial form number and contract it into a
  833. * little-endian, 32-byte array
  834. */
  835. static void
  836. curve25519_contract(unsigned char out[32], const bignum25519 in) {
  837. bignum25519 ALIGN(16) f;
  838. curve25519_copy(f, in);
  839. #define carry_pass() \
  840. f[1] += f[0] >> 26; f[0] &= 0x3ffffff; \
  841. f[2] += f[1] >> 25; f[1] &= 0x1ffffff; \
  842. f[3] += f[2] >> 26; f[2] &= 0x3ffffff; \
  843. f[4] += f[3] >> 25; f[3] &= 0x1ffffff; \
  844. f[5] += f[4] >> 26; f[4] &= 0x3ffffff; \
  845. f[6] += f[5] >> 25; f[5] &= 0x1ffffff; \
  846. f[7] += f[6] >> 26; f[6] &= 0x3ffffff; \
  847. f[8] += f[7] >> 25; f[7] &= 0x1ffffff; \
  848. f[9] += f[8] >> 26; f[8] &= 0x3ffffff;
  849. #define carry_pass_full() \
  850. carry_pass() \
  851. f[0] += 19 * (f[9] >> 25); f[9] &= 0x1ffffff;
  852. #define carry_pass_final() \
  853. carry_pass() \
  854. f[9] &= 0x1ffffff;
  855. carry_pass_full()
  856. carry_pass_full()
  857. /* now t is between 0 and 2^255-1, properly carried. */
  858. /* case 1: between 0 and 2^255-20. case 2: between 2^255-19 and 2^255-1. */
  859. f[0] += 19;
  860. carry_pass_full()
  861. /* now between 19 and 2^255-1 in both cases, and offset by 19. */
  862. f[0] += (1 << 26) - 19;
  863. f[1] += (1 << 25) - 1;
  864. f[2] += (1 << 26) - 1;
  865. f[3] += (1 << 25) - 1;
  866. f[4] += (1 << 26) - 1;
  867. f[5] += (1 << 25) - 1;
  868. f[6] += (1 << 26) - 1;
  869. f[7] += (1 << 25) - 1;
  870. f[8] += (1 << 26) - 1;
  871. f[9] += (1 << 25) - 1;
  872. /* now between 2^255 and 2^256-20, and offset by 2^255. */
  873. carry_pass_final()
  874. #undef carry_pass
  875. #undef carry_full
  876. #undef carry_final
  877. f[1] <<= 2;
  878. f[2] <<= 3;
  879. f[3] <<= 5;
  880. f[4] <<= 6;
  881. f[6] <<= 1;
  882. f[7] <<= 3;
  883. f[8] <<= 4;
  884. f[9] <<= 6;
  885. #define F(i, s) \
  886. out[s+0] |= (unsigned char )(f[i] & 0xff); \
  887. out[s+1] = (unsigned char )((f[i] >> 8) & 0xff); \
  888. out[s+2] = (unsigned char )((f[i] >> 16) & 0xff); \
  889. out[s+3] = (unsigned char )((f[i] >> 24) & 0xff);
  890. out[0] = 0;
  891. out[16] = 0;
  892. F(0,0);
  893. F(1,3);
  894. F(2,6);
  895. F(3,9);
  896. F(4,12);
  897. F(5,16);
  898. F(6,19);
  899. F(7,22);
  900. F(8,25);
  901. F(9,28);
  902. #undef F
  903. }
  904. /* if (iswap) swap(a, b) */
  905. DONNA_INLINE static void
  906. curve25519_swap_conditional(bignum25519 a, bignum25519 b, uint32_t iswap) {
  907. const uint32_t swap = (uint32_t)(-(int32_t)iswap);
  908. xmmi a0,a1,a2,b0,b1,b2,x0,x1,x2;
  909. xmmi mask = _mm_cvtsi32_si128(swap);
  910. mask = _mm_shuffle_epi32(mask, 0);
  911. a0 = _mm_load_si128((xmmi *)a + 0);
  912. a1 = _mm_load_si128((xmmi *)a + 1);
  913. b0 = _mm_load_si128((xmmi *)b + 0);
  914. b1 = _mm_load_si128((xmmi *)b + 1);
  915. b0 = _mm_xor_si128(a0, b0);
  916. b1 = _mm_xor_si128(a1, b1);
  917. x0 = _mm_and_si128(b0, mask);
  918. x1 = _mm_and_si128(b1, mask);
  919. x0 = _mm_xor_si128(x0, a0);
  920. x1 = _mm_xor_si128(x1, a1);
  921. a0 = _mm_xor_si128(x0, b0);
  922. a1 = _mm_xor_si128(x1, b1);
  923. _mm_store_si128((xmmi *)a + 0, x0);
  924. _mm_store_si128((xmmi *)a + 1, x1);
  925. _mm_store_si128((xmmi *)b + 0, a0);
  926. _mm_store_si128((xmmi *)b + 1, a1);
  927. a2 = _mm_load_si128((xmmi *)a + 2);
  928. b2 = _mm_load_si128((xmmi *)b + 2);
  929. b2 = _mm_xor_si128(a2, b2);
  930. x2 = _mm_and_si128(b2, mask);
  931. x2 = _mm_xor_si128(x2, a2);
  932. a2 = _mm_xor_si128(x2, b2);
  933. _mm_store_si128((xmmi *)b + 2, a2);
  934. _mm_store_si128((xmmi *)a + 2, x2);
  935. }
  936. /* out = (flag) ? out : in */
  937. DONNA_INLINE static void
  938. curve25519_move_conditional_bytes(uint8_t out[96], const uint8_t in[96], uint32_t flag) {
  939. xmmi a0,a1,a2,a3,a4,a5,b0,b1,b2,b3,b4,b5;
  940. const uint32_t nb = flag - 1;
  941. xmmi masknb = _mm_shuffle_epi32(_mm_cvtsi32_si128(nb),0);
  942. a0 = _mm_load_si128((xmmi *)in + 0);
  943. a1 = _mm_load_si128((xmmi *)in + 1);
  944. a2 = _mm_load_si128((xmmi *)in + 2);
  945. b0 = _mm_load_si128((xmmi *)out + 0);
  946. b1 = _mm_load_si128((xmmi *)out + 1);
  947. b2 = _mm_load_si128((xmmi *)out + 2);
  948. a0 = _mm_andnot_si128(masknb, a0);
  949. a1 = _mm_andnot_si128(masknb, a1);
  950. a2 = _mm_andnot_si128(masknb, a2);
  951. b0 = _mm_and_si128(masknb, b0);
  952. b1 = _mm_and_si128(masknb, b1);
  953. b2 = _mm_and_si128(masknb, b2);
  954. a0 = _mm_or_si128(a0, b0);
  955. a1 = _mm_or_si128(a1, b1);
  956. a2 = _mm_or_si128(a2, b2);
  957. _mm_store_si128((xmmi*)out + 0, a0);
  958. _mm_store_si128((xmmi*)out + 1, a1);
  959. _mm_store_si128((xmmi*)out + 2, a2);
  960. a3 = _mm_load_si128((xmmi *)in + 3);
  961. a4 = _mm_load_si128((xmmi *)in + 4);
  962. a5 = _mm_load_si128((xmmi *)in + 5);
  963. b3 = _mm_load_si128((xmmi *)out + 3);
  964. b4 = _mm_load_si128((xmmi *)out + 4);
  965. b5 = _mm_load_si128((xmmi *)out + 5);
  966. a3 = _mm_andnot_si128(masknb, a3);
  967. a4 = _mm_andnot_si128(masknb, a4);
  968. a5 = _mm_andnot_si128(masknb, a5);
  969. b3 = _mm_and_si128(masknb, b3);
  970. b4 = _mm_and_si128(masknb, b4);
  971. b5 = _mm_and_si128(masknb, b5);
  972. a3 = _mm_or_si128(a3, b3);
  973. a4 = _mm_or_si128(a4, b4);
  974. a5 = _mm_or_si128(a5, b5);
  975. _mm_store_si128((xmmi*)out + 3, a3);
  976. _mm_store_si128((xmmi*)out + 4, a4);
  977. _mm_store_si128((xmmi*)out + 5, a5);
  978. }