You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

351 lines
13 KiB

  1. #if defined(ED25519_GCC_64BIT_X86_CHOOSE)
  2. #define HAVE_GE25519_SCALARMULT_BASE_CHOOSE_NIELS
  3. DONNA_NOINLINE static void
  4. ge25519_scalarmult_base_choose_niels(ge25519_niels *t, const uint8_t table[256][96], uint32_t pos, signed char b) {
  5. int64_t breg = (int64_t)b;
  6. uint64_t sign = (uint64_t)breg >> 63;
  7. uint64_t mask = ~(sign - 1);
  8. uint64_t u = (breg + mask) ^ mask;
  9. __asm__ __volatile__ (
  10. /* ysubx+xaddy+t2d */
  11. "movq %0, %%rax ;\n"
  12. "movd %%rax, %%xmm14 ;\n"
  13. "pshufd $0x00, %%xmm14, %%xmm14 ;\n"
  14. "pxor %%xmm0, %%xmm0 ;\n"
  15. "pxor %%xmm1, %%xmm1 ;\n"
  16. "pxor %%xmm2, %%xmm2 ;\n"
  17. "pxor %%xmm3, %%xmm3 ;\n"
  18. "pxor %%xmm4, %%xmm4 ;\n"
  19. "pxor %%xmm5, %%xmm5 ;\n"
  20. /* 0 */
  21. "movq $0, %%rax ;\n"
  22. "movd %%rax, %%xmm15 ;\n"
  23. "pshufd $0x00, %%xmm15, %%xmm15 ;\n"
  24. "pcmpeqd %%xmm14, %%xmm15 ;\n"
  25. "movq $1, %%rax ;\n"
  26. "movd %%rax, %%xmm6 ;\n"
  27. "pxor %%xmm7, %%xmm7 ;\n"
  28. "pand %%xmm15, %%xmm6 ;\n"
  29. "pand %%xmm15, %%xmm7 ;\n"
  30. "por %%xmm6, %%xmm0 ;\n"
  31. "por %%xmm7, %%xmm1 ;\n"
  32. "por %%xmm6, %%xmm2 ;\n"
  33. "por %%xmm7, %%xmm3 ;\n"
  34. /* 1 */
  35. "movq $1, %%rax ;\n"
  36. "movd %%rax, %%xmm15 ;\n"
  37. "pshufd $0x00, %%xmm15, %%xmm15 ;\n"
  38. "pcmpeqd %%xmm14, %%xmm15 ;\n"
  39. "movdqa 0(%1), %%xmm6 ;\n"
  40. "movdqa 16(%1), %%xmm7 ;\n"
  41. "movdqa 32(%1), %%xmm8 ;\n"
  42. "movdqa 48(%1), %%xmm9 ;\n"
  43. "movdqa 64(%1), %%xmm10 ;\n"
  44. "movdqa 80(%1), %%xmm11 ;\n"
  45. "pand %%xmm15, %%xmm6 ;\n"
  46. "pand %%xmm15, %%xmm7 ;\n"
  47. "pand %%xmm15, %%xmm8 ;\n"
  48. "pand %%xmm15, %%xmm9 ;\n"
  49. "pand %%xmm15, %%xmm10 ;\n"
  50. "pand %%xmm15, %%xmm11 ;\n"
  51. "por %%xmm6, %%xmm0 ;\n"
  52. "por %%xmm7, %%xmm1 ;\n"
  53. "por %%xmm8, %%xmm2 ;\n"
  54. "por %%xmm9, %%xmm3 ;\n"
  55. "por %%xmm10, %%xmm4 ;\n"
  56. "por %%xmm11, %%xmm5 ;\n"
  57. /* 2 */
  58. "movq $2, %%rax ;\n"
  59. "movd %%rax, %%xmm15 ;\n"
  60. "pshufd $0x00, %%xmm15, %%xmm15 ;\n"
  61. "pcmpeqd %%xmm14, %%xmm15 ;\n"
  62. "movdqa 96(%1), %%xmm6 ;\n"
  63. "movdqa 112(%1), %%xmm7 ;\n"
  64. "movdqa 128(%1), %%xmm8 ;\n"
  65. "movdqa 144(%1), %%xmm9 ;\n"
  66. "movdqa 160(%1), %%xmm10 ;\n"
  67. "movdqa 176(%1), %%xmm11 ;\n"
  68. "pand %%xmm15, %%xmm6 ;\n"
  69. "pand %%xmm15, %%xmm7 ;\n"
  70. "pand %%xmm15, %%xmm8 ;\n"
  71. "pand %%xmm15, %%xmm9 ;\n"
  72. "pand %%xmm15, %%xmm10 ;\n"
  73. "pand %%xmm15, %%xmm11 ;\n"
  74. "por %%xmm6, %%xmm0 ;\n"
  75. "por %%xmm7, %%xmm1 ;\n"
  76. "por %%xmm8, %%xmm2 ;\n"
  77. "por %%xmm9, %%xmm3 ;\n"
  78. "por %%xmm10, %%xmm4 ;\n"
  79. "por %%xmm11, %%xmm5 ;\n"
  80. /* 3 */
  81. "movq $3, %%rax ;\n"
  82. "movd %%rax, %%xmm15 ;\n"
  83. "pshufd $0x00, %%xmm15, %%xmm15 ;\n"
  84. "pcmpeqd %%xmm14, %%xmm15 ;\n"
  85. "movdqa 192(%1), %%xmm6 ;\n"
  86. "movdqa 208(%1), %%xmm7 ;\n"
  87. "movdqa 224(%1), %%xmm8 ;\n"
  88. "movdqa 240(%1), %%xmm9 ;\n"
  89. "movdqa 256(%1), %%xmm10 ;\n"
  90. "movdqa 272(%1), %%xmm11 ;\n"
  91. "pand %%xmm15, %%xmm6 ;\n"
  92. "pand %%xmm15, %%xmm7 ;\n"
  93. "pand %%xmm15, %%xmm8 ;\n"
  94. "pand %%xmm15, %%xmm9 ;\n"
  95. "pand %%xmm15, %%xmm10 ;\n"
  96. "pand %%xmm15, %%xmm11 ;\n"
  97. "por %%xmm6, %%xmm0 ;\n"
  98. "por %%xmm7, %%xmm1 ;\n"
  99. "por %%xmm8, %%xmm2 ;\n"
  100. "por %%xmm9, %%xmm3 ;\n"
  101. "por %%xmm10, %%xmm4 ;\n"
  102. "por %%xmm11, %%xmm5 ;\n"
  103. /* 4 */
  104. "movq $4, %%rax ;\n"
  105. "movd %%rax, %%xmm15 ;\n"
  106. "pshufd $0x00, %%xmm15, %%xmm15 ;\n"
  107. "pcmpeqd %%xmm14, %%xmm15 ;\n"
  108. "movdqa 288(%1), %%xmm6 ;\n"
  109. "movdqa 304(%1), %%xmm7 ;\n"
  110. "movdqa 320(%1), %%xmm8 ;\n"
  111. "movdqa 336(%1), %%xmm9 ;\n"
  112. "movdqa 352(%1), %%xmm10 ;\n"
  113. "movdqa 368(%1), %%xmm11 ;\n"
  114. "pand %%xmm15, %%xmm6 ;\n"
  115. "pand %%xmm15, %%xmm7 ;\n"
  116. "pand %%xmm15, %%xmm8 ;\n"
  117. "pand %%xmm15, %%xmm9 ;\n"
  118. "pand %%xmm15, %%xmm10 ;\n"
  119. "pand %%xmm15, %%xmm11 ;\n"
  120. "por %%xmm6, %%xmm0 ;\n"
  121. "por %%xmm7, %%xmm1 ;\n"
  122. "por %%xmm8, %%xmm2 ;\n"
  123. "por %%xmm9, %%xmm3 ;\n"
  124. "por %%xmm10, %%xmm4 ;\n"
  125. "por %%xmm11, %%xmm5 ;\n"
  126. /* 5 */
  127. "movq $5, %%rax ;\n"
  128. "movd %%rax, %%xmm15 ;\n"
  129. "pshufd $0x00, %%xmm15, %%xmm15 ;\n"
  130. "pcmpeqd %%xmm14, %%xmm15 ;\n"
  131. "movdqa 384(%1), %%xmm6 ;\n"
  132. "movdqa 400(%1), %%xmm7 ;\n"
  133. "movdqa 416(%1), %%xmm8 ;\n"
  134. "movdqa 432(%1), %%xmm9 ;\n"
  135. "movdqa 448(%1), %%xmm10 ;\n"
  136. "movdqa 464(%1), %%xmm11 ;\n"
  137. "pand %%xmm15, %%xmm6 ;\n"
  138. "pand %%xmm15, %%xmm7 ;\n"
  139. "pand %%xmm15, %%xmm8 ;\n"
  140. "pand %%xmm15, %%xmm9 ;\n"
  141. "pand %%xmm15, %%xmm10 ;\n"
  142. "pand %%xmm15, %%xmm11 ;\n"
  143. "por %%xmm6, %%xmm0 ;\n"
  144. "por %%xmm7, %%xmm1 ;\n"
  145. "por %%xmm8, %%xmm2 ;\n"
  146. "por %%xmm9, %%xmm3 ;\n"
  147. "por %%xmm10, %%xmm4 ;\n"
  148. "por %%xmm11, %%xmm5 ;\n"
  149. /* 6 */
  150. "movq $6, %%rax ;\n"
  151. "movd %%rax, %%xmm15 ;\n"
  152. "pshufd $0x00, %%xmm15, %%xmm15 ;\n"
  153. "pcmpeqd %%xmm14, %%xmm15 ;\n"
  154. "movdqa 480(%1), %%xmm6 ;\n"
  155. "movdqa 496(%1), %%xmm7 ;\n"
  156. "movdqa 512(%1), %%xmm8 ;\n"
  157. "movdqa 528(%1), %%xmm9 ;\n"
  158. "movdqa 544(%1), %%xmm10 ;\n"
  159. "movdqa 560(%1), %%xmm11 ;\n"
  160. "pand %%xmm15, %%xmm6 ;\n"
  161. "pand %%xmm15, %%xmm7 ;\n"
  162. "pand %%xmm15, %%xmm8 ;\n"
  163. "pand %%xmm15, %%xmm9 ;\n"
  164. "pand %%xmm15, %%xmm10 ;\n"
  165. "pand %%xmm15, %%xmm11 ;\n"
  166. "por %%xmm6, %%xmm0 ;\n"
  167. "por %%xmm7, %%xmm1 ;\n"
  168. "por %%xmm8, %%xmm2 ;\n"
  169. "por %%xmm9, %%xmm3 ;\n"
  170. "por %%xmm10, %%xmm4 ;\n"
  171. "por %%xmm11, %%xmm5 ;\n"
  172. /* 7 */
  173. "movq $7, %%rax ;\n"
  174. "movd %%rax, %%xmm15 ;\n"
  175. "pshufd $0x00, %%xmm15, %%xmm15 ;\n"
  176. "pcmpeqd %%xmm14, %%xmm15 ;\n"
  177. "movdqa 576(%1), %%xmm6 ;\n"
  178. "movdqa 592(%1), %%xmm7 ;\n"
  179. "movdqa 608(%1), %%xmm8 ;\n"
  180. "movdqa 624(%1), %%xmm9 ;\n"
  181. "movdqa 640(%1), %%xmm10 ;\n"
  182. "movdqa 656(%1), %%xmm11 ;\n"
  183. "pand %%xmm15, %%xmm6 ;\n"
  184. "pand %%xmm15, %%xmm7 ;\n"
  185. "pand %%xmm15, %%xmm8 ;\n"
  186. "pand %%xmm15, %%xmm9 ;\n"
  187. "pand %%xmm15, %%xmm10 ;\n"
  188. "pand %%xmm15, %%xmm11 ;\n"
  189. "por %%xmm6, %%xmm0 ;\n"
  190. "por %%xmm7, %%xmm1 ;\n"
  191. "por %%xmm8, %%xmm2 ;\n"
  192. "por %%xmm9, %%xmm3 ;\n"
  193. "por %%xmm10, %%xmm4 ;\n"
  194. "por %%xmm11, %%xmm5 ;\n"
  195. /* 8 */
  196. "movq $8, %%rax ;\n"
  197. "movd %%rax, %%xmm15 ;\n"
  198. "pshufd $0x00, %%xmm15, %%xmm15 ;\n"
  199. "pcmpeqd %%xmm14, %%xmm15 ;\n"
  200. "movdqa 672(%1), %%xmm6 ;\n"
  201. "movdqa 688(%1), %%xmm7 ;\n"
  202. "movdqa 704(%1), %%xmm8 ;\n"
  203. "movdqa 720(%1), %%xmm9 ;\n"
  204. "movdqa 736(%1), %%xmm10 ;\n"
  205. "movdqa 752(%1), %%xmm11 ;\n"
  206. "pand %%xmm15, %%xmm6 ;\n"
  207. "pand %%xmm15, %%xmm7 ;\n"
  208. "pand %%xmm15, %%xmm8 ;\n"
  209. "pand %%xmm15, %%xmm9 ;\n"
  210. "pand %%xmm15, %%xmm10 ;\n"
  211. "pand %%xmm15, %%xmm11 ;\n"
  212. "por %%xmm6, %%xmm0 ;\n"
  213. "por %%xmm7, %%xmm1 ;\n"
  214. "por %%xmm8, %%xmm2 ;\n"
  215. "por %%xmm9, %%xmm3 ;\n"
  216. "por %%xmm10, %%xmm4 ;\n"
  217. "por %%xmm11, %%xmm5 ;\n"
  218. /* conditionally swap ysubx and xaddy */
  219. "movq %3, %%rax ;\n"
  220. "xorq $1, %%rax ;\n"
  221. "movd %%rax, %%xmm14 ;\n"
  222. "pxor %%xmm15, %%xmm15 ;\n"
  223. "pshufd $0x00, %%xmm14, %%xmm14 ;\n"
  224. "pxor %%xmm0, %%xmm2 ;\n"
  225. "pxor %%xmm1, %%xmm3 ;\n"
  226. "pcmpeqd %%xmm14, %%xmm15 ;\n"
  227. "movdqa %%xmm2, %%xmm6 ;\n"
  228. "movdqa %%xmm3, %%xmm7 ;\n"
  229. "pand %%xmm15, %%xmm6 ;\n"
  230. "pand %%xmm15, %%xmm7 ;\n"
  231. "pxor %%xmm6, %%xmm0 ;\n"
  232. "pxor %%xmm7, %%xmm1 ;\n"
  233. "pxor %%xmm0, %%xmm2 ;\n"
  234. "pxor %%xmm1, %%xmm3 ;\n"
  235. /* store ysubx */
  236. "movq $0x7ffffffffffff, %%rax ;\n"
  237. "movd %%xmm0, %%rcx ;\n"
  238. "movd %%xmm0, %%r8 ;\n"
  239. "movd %%xmm1, %%rsi ;\n"
  240. "pshufd $0xee, %%xmm0, %%xmm0 ;\n"
  241. "pshufd $0xee, %%xmm1, %%xmm1 ;\n"
  242. "movd %%xmm0, %%rdx ;\n"
  243. "movd %%xmm1, %%rdi ;\n"
  244. "shrdq $51, %%rdx, %%r8 ;\n"
  245. "shrdq $38, %%rsi, %%rdx ;\n"
  246. "shrdq $25, %%rdi, %%rsi ;\n"
  247. "shrq $12, %%rdi ;\n"
  248. "andq %%rax, %%rcx ;\n"
  249. "andq %%rax, %%r8 ;\n"
  250. "andq %%rax, %%rdx ;\n"
  251. "andq %%rax, %%rsi ;\n"
  252. "andq %%rax, %%rdi ;\n"
  253. "movq %%rcx, 0(%2) ;\n"
  254. "movq %%r8, 8(%2) ;\n"
  255. "movq %%rdx, 16(%2) ;\n"
  256. "movq %%rsi, 24(%2) ;\n"
  257. "movq %%rdi, 32(%2) ;\n"
  258. /* store xaddy */
  259. "movq $0x7ffffffffffff, %%rax ;\n"
  260. "movd %%xmm2, %%rcx ;\n"
  261. "movd %%xmm2, %%r8 ;\n"
  262. "movd %%xmm3, %%rsi ;\n"
  263. "pshufd $0xee, %%xmm2, %%xmm2 ;\n"
  264. "pshufd $0xee, %%xmm3, %%xmm3 ;\n"
  265. "movd %%xmm2, %%rdx ;\n"
  266. "movd %%xmm3, %%rdi ;\n"
  267. "shrdq $51, %%rdx, %%r8 ;\n"
  268. "shrdq $38, %%rsi, %%rdx ;\n"
  269. "shrdq $25, %%rdi, %%rsi ;\n"
  270. "shrq $12, %%rdi ;\n"
  271. "andq %%rax, %%rcx ;\n"
  272. "andq %%rax, %%r8 ;\n"
  273. "andq %%rax, %%rdx ;\n"
  274. "andq %%rax, %%rsi ;\n"
  275. "andq %%rax, %%rdi ;\n"
  276. "movq %%rcx, 40(%2) ;\n"
  277. "movq %%r8, 48(%2) ;\n"
  278. "movq %%rdx, 56(%2) ;\n"
  279. "movq %%rsi, 64(%2) ;\n"
  280. "movq %%rdi, 72(%2) ;\n"
  281. /* extract t2d */
  282. "movq $0x7ffffffffffff, %%rax ;\n"
  283. "movd %%xmm4, %%rcx ;\n"
  284. "movd %%xmm4, %%r8 ;\n"
  285. "movd %%xmm5, %%rsi ;\n"
  286. "pshufd $0xee, %%xmm4, %%xmm4 ;\n"
  287. "pshufd $0xee, %%xmm5, %%xmm5 ;\n"
  288. "movd %%xmm4, %%rdx ;\n"
  289. "movd %%xmm5, %%rdi ;\n"
  290. "shrdq $51, %%rdx, %%r8 ;\n"
  291. "shrdq $38, %%rsi, %%rdx ;\n"
  292. "shrdq $25, %%rdi, %%rsi ;\n"
  293. "shrq $12, %%rdi ;\n"
  294. "andq %%rax, %%rcx ;\n"
  295. "andq %%rax, %%r8 ;\n"
  296. "andq %%rax, %%rdx ;\n"
  297. "andq %%rax, %%rsi ;\n"
  298. "andq %%rax, %%rdi ;\n"
  299. /* conditionally negate t2d */
  300. "movq %3, %%rax ;\n"
  301. "movq $0xfffffffffffda, %%r9 ;\n"
  302. "movq $0xffffffffffffe, %%r10 ;\n"
  303. "movq %%r10, %%r11 ;\n"
  304. "movq %%r10, %%r12 ;\n"
  305. "movq %%r10, %%r13 ;\n"
  306. "subq %%rcx, %%r9 ;\n"
  307. "subq %%r8, %%r10 ;\n"
  308. "subq %%rdx, %%r11 ;\n"
  309. "subq %%rsi, %%r12 ;\n"
  310. "subq %%rdi, %%r13 ;\n"
  311. "cmpq $1, %%rax ;\n"
  312. "cmove %%r9, %%rcx ;\n"
  313. "cmove %%r10, %%r8 ;\n"
  314. "cmove %%r11, %%rdx ;\n"
  315. "cmove %%r12, %%rsi ;\n"
  316. "cmove %%r13, %%rdi ;\n"
  317. /* store t2d */
  318. "movq %%rcx, 80(%2) ;\n"
  319. "movq %%r8, 88(%2) ;\n"
  320. "movq %%rdx, 96(%2) ;\n"
  321. "movq %%rsi, 104(%2) ;\n"
  322. "movq %%rdi, 112(%2) ;\n"
  323. :
  324. : "m"(u), "r"(&table[pos * 8]), "r"(t), "m"(sign) /* %0 = u, %1 = table, %2 = t, %3 = sign */
  325. :
  326. "%rax", "%rcx", "%rdx", "%rdi", "%rsi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13",
  327. "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm14", "%xmm14",
  328. "cc", "memory"
  329. );
  330. }
  331. #endif /* defined(ED25519_GCC_64BIT_X86_CHOOSE) */