jdmrgext-avx2.asm 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593
  1. ;
  2. ; jdmrgext.asm - merged upsampling/color conversion (64-bit AVX2)
  3. ;
  4. ; Copyright 2009, 2012 Pierre Ossman <ossman@cendio.se> for Cendio AB
  5. ; Copyright (C) 2009, 2012, 2016, D. R. Commander.
  6. ; Copyright (C) 2015, Intel Corporation.
  7. ;
  8. ; Based on the x86 SIMD extension for IJG JPEG library
  9. ; Copyright (C) 1999-2006, MIYASAKA Masaru.
  10. ; For conditions of distribution and use, see copyright notice in jsimdext.inc
  11. ;
  12. ; This file should be assembled with NASM (Netwide Assembler),
  13. ; can *not* be assembled with Microsoft's MASM or any compatible
  14. ; assembler (including Borland's Turbo Assembler).
  15. ; NASM is available from http://nasm.sourceforge.net/ or
  16. ; http://sourceforge.net/project/showfiles.php?group_id=6208
  17. %include "jcolsamp.inc"
  18. ; --------------------------------------------------------------------------
  19. ;
  20. ; Upsample and color convert for the case of 2:1 horizontal and 1:1 vertical.
  21. ;
  22. ; GLOBAL(void)
  23. ; jsimd_h2v1_merged_upsample_avx2(JDIMENSION output_width,
  24. ; JSAMPIMAGE input_buf,
  25. ; JDIMENSION in_row_group_ctr,
  26. ; JSAMPARRAY output_buf);
  27. ;
  28. ; r10d = JDIMENSION output_width
  29. ; r11 = JSAMPIMAGE input_buf
  30. ; r12d = JDIMENSION in_row_group_ctr
  31. ; r13 = JSAMPARRAY output_buf
  32. %define wk(i) rbp - (WK_NUM - (i)) * SIZEOF_YMMWORD ; ymmword wk[WK_NUM]
  33. %define WK_NUM 3
  34. align 32
  35. GLOBAL_FUNCTION(jsimd_h2v1_merged_upsample_avx2)
  36. EXTN(jsimd_h2v1_merged_upsample_avx2):
  37. push rbp
  38. mov rax, rsp ; rax = original rbp
  39. sub rsp, byte 4
  40. and rsp, byte (-SIZEOF_YMMWORD) ; align to 256 bits
  41. mov [rsp], rax
  42. mov rbp, rsp ; rbp = aligned rbp
  43. lea rsp, [wk(0)]
  44. collect_args 4
  45. push rbx
  46. mov ecx, r10d ; col
  47. test rcx, rcx
  48. jz near .return
  49. push rcx
  50. mov rdi, r11
  51. mov ecx, r12d
  52. mov rsi, JSAMPARRAY [rdi+0*SIZEOF_JSAMPARRAY]
  53. mov rbx, JSAMPARRAY [rdi+1*SIZEOF_JSAMPARRAY]
  54. mov rdx, JSAMPARRAY [rdi+2*SIZEOF_JSAMPARRAY]
  55. mov rdi, r13
  56. mov rsi, JSAMPROW [rsi+rcx*SIZEOF_JSAMPROW] ; inptr0
  57. mov rbx, JSAMPROW [rbx+rcx*SIZEOF_JSAMPROW] ; inptr1
  58. mov rdx, JSAMPROW [rdx+rcx*SIZEOF_JSAMPROW] ; inptr2
  59. mov rdi, JSAMPROW [rdi] ; outptr
  60. pop rcx ; col
  61. .columnloop:
  62. vmovdqu ymm6, YMMWORD [rbx] ; ymm6=Cb(0123456789ABCDEFGHIJKLMNOPQRSTUV)
  63. vmovdqu ymm7, YMMWORD [rdx] ; ymm7=Cr(0123456789ABCDEFGHIJKLMNOPQRSTUV)
  64. vpxor ymm1, ymm1, ymm1 ; ymm1=(all 0's)
  65. vpcmpeqw ymm3, ymm3, ymm3
  66. vpsllw ymm3, ymm3, 7 ; ymm3={0xFF80 0xFF80 0xFF80 0xFF80 ..}
  67. vpermq ymm6, ymm6, 0xd8 ; ymm6=Cb(01234567GHIJKLMN89ABCDEFOPQRSTUV)
  68. vpermq ymm7, ymm7, 0xd8 ; ymm7=Cr(01234567GHIJKLMN89ABCDEFOPQRSTUV)
  69. vpunpcklbw ymm4, ymm6, ymm1 ; ymm4=Cb(0123456789ABCDEF)=CbL
  70. vpunpckhbw ymm6, ymm6, ymm1 ; ymm6=Cb(GHIJKLMNOPQRSTUV)=CbH
  71. vpunpcklbw ymm0, ymm7, ymm1 ; ymm0=Cr(0123456789ABCDEF)=CrL
  72. vpunpckhbw ymm7, ymm7, ymm1 ; ymm7=Cr(GHIJKLMNOPQRSTUV)=CrH
  73. vpaddw ymm5, ymm6, ymm3
  74. vpaddw ymm2, ymm4, ymm3
  75. vpaddw ymm1, ymm7, ymm3
  76. vpaddw ymm3, ymm0, ymm3
  77. ; (Original)
  78. ; R = Y + 1.40200 * Cr
  79. ; G = Y - 0.34414 * Cb - 0.71414 * Cr
  80. ; B = Y + 1.77200 * Cb
  81. ;
  82. ; (This implementation)
  83. ; R = Y + 0.40200 * Cr + Cr
  84. ; G = Y - 0.34414 * Cb + 0.28586 * Cr - Cr
  85. ; B = Y - 0.22800 * Cb + Cb + Cb
  86. vpaddw ymm6, ymm5, ymm5 ; ymm6=2*CbH
  87. vpaddw ymm4, ymm2, ymm2 ; ymm4=2*CbL
  88. vpaddw ymm7, ymm1, ymm1 ; ymm7=2*CrH
  89. vpaddw ymm0, ymm3, ymm3 ; ymm0=2*CrL
  90. vpmulhw ymm6, ymm6, [rel PW_MF0228] ; ymm6=(2*CbH * -FIX(0.22800))
  91. vpmulhw ymm4, ymm4, [rel PW_MF0228] ; ymm4=(2*CbL * -FIX(0.22800))
  92. vpmulhw ymm7, ymm7, [rel PW_F0402] ; ymm7=(2*CrH * FIX(0.40200))
  93. vpmulhw ymm0, ymm0, [rel PW_F0402] ; ymm0=(2*CrL * FIX(0.40200))
  94. vpaddw ymm6, ymm6, [rel PW_ONE]
  95. vpaddw ymm4, ymm4, [rel PW_ONE]
  96. vpsraw ymm6, ymm6, 1 ; ymm6=(CbH * -FIX(0.22800))
  97. vpsraw ymm4, ymm4, 1 ; ymm4=(CbL * -FIX(0.22800))
  98. vpaddw ymm7, ymm7, [rel PW_ONE]
  99. vpaddw ymm0, ymm0, [rel PW_ONE]
  100. vpsraw ymm7, ymm7, 1 ; ymm7=(CrH * FIX(0.40200))
  101. vpsraw ymm0, ymm0, 1 ; ymm0=(CrL * FIX(0.40200))
  102. vpaddw ymm6, ymm6, ymm5
  103. vpaddw ymm4, ymm4, ymm2
  104. vpaddw ymm6, ymm6, ymm5 ; ymm6=(CbH * FIX(1.77200))=(B-Y)H
  105. vpaddw ymm4, ymm4, ymm2 ; ymm4=(CbL * FIX(1.77200))=(B-Y)L
  106. vpaddw ymm7, ymm7, ymm1 ; ymm7=(CrH * FIX(1.40200))=(R-Y)H
  107. vpaddw ymm0, ymm0, ymm3 ; ymm0=(CrL * FIX(1.40200))=(R-Y)L
  108. vmovdqa YMMWORD [wk(0)], ymm6 ; wk(0)=(B-Y)H
  109. vmovdqa YMMWORD [wk(1)], ymm7 ; wk(1)=(R-Y)H
  110. vpunpckhwd ymm6, ymm5, ymm1
  111. vpunpcklwd ymm5, ymm5, ymm1
  112. vpmaddwd ymm5, ymm5, [rel PW_MF0344_F0285]
  113. vpmaddwd ymm6, ymm6, [rel PW_MF0344_F0285]
  114. vpunpckhwd ymm7, ymm2, ymm3
  115. vpunpcklwd ymm2, ymm2, ymm3
  116. vpmaddwd ymm2, ymm2, [rel PW_MF0344_F0285]
  117. vpmaddwd ymm7, ymm7, [rel PW_MF0344_F0285]
  118. vpaddd ymm5, ymm5, [rel PD_ONEHALF]
  119. vpaddd ymm6, ymm6, [rel PD_ONEHALF]
  120. vpsrad ymm5, ymm5, SCALEBITS
  121. vpsrad ymm6, ymm6, SCALEBITS
  122. vpaddd ymm2, ymm2, [rel PD_ONEHALF]
  123. vpaddd ymm7, ymm7, [rel PD_ONEHALF]
  124. vpsrad ymm2, ymm2, SCALEBITS
  125. vpsrad ymm7, ymm7, SCALEBITS
  126. vpackssdw ymm5, ymm5, ymm6 ; ymm5=CbH*-FIX(0.344)+CrH*FIX(0.285)
  127. vpackssdw ymm2, ymm2, ymm7 ; ymm2=CbL*-FIX(0.344)+CrL*FIX(0.285)
  128. vpsubw ymm5, ymm5, ymm1 ; ymm5=CbH*-FIX(0.344)+CrH*-FIX(0.714)=(G-Y)H
  129. vpsubw ymm2, ymm2, ymm3 ; ymm2=CbL*-FIX(0.344)+CrL*-FIX(0.714)=(G-Y)L
  130. vmovdqa YMMWORD [wk(2)], ymm5 ; wk(2)=(G-Y)H
  131. mov al, 2 ; Yctr
  132. jmp short .Yloop_1st
  133. .Yloop_2nd:
  134. vmovdqa ymm0, YMMWORD [wk(1)] ; ymm0=(R-Y)H
  135. vmovdqa ymm2, YMMWORD [wk(2)] ; ymm2=(G-Y)H
  136. vmovdqa ymm4, YMMWORD [wk(0)] ; ymm4=(B-Y)H
  137. .Yloop_1st:
  138. vmovdqu ymm7, YMMWORD [rsi] ; ymm7=Y(0123456789ABCDEFGHIJKLMNOPQRSTUV)
  139. vpcmpeqw ymm6, ymm6, ymm6
  140. vpsrlw ymm6, ymm6, BYTE_BIT ; ymm6={0xFF 0x00 0xFF 0x00 ..}
  141. vpand ymm6, ymm6, ymm7 ; ymm6=Y(02468ACEGIKMOQSU)=YE
  142. vpsrlw ymm7, ymm7, BYTE_BIT ; ymm7=Y(13579BDFHJLNPRTV)=YO
  143. vmovdqa ymm1, ymm0 ; ymm1=ymm0=(R-Y)(L/H)
  144. vmovdqa ymm3, ymm2 ; ymm3=ymm2=(G-Y)(L/H)
  145. vmovdqa ymm5, ymm4 ; ymm5=ymm4=(B-Y)(L/H)
  146. vpaddw ymm0, ymm0, ymm6 ; ymm0=((R-Y)+YE)=RE=R(02468ACEGIKMOQSU)
  147. vpaddw ymm1, ymm1, ymm7 ; ymm1=((R-Y)+YO)=RO=R(13579BDFHJLNPRTV)
  148. vpackuswb ymm0, ymm0, ymm0 ; ymm0=R(02468ACE********GIKMOQSU********)
  149. vpackuswb ymm1, ymm1, ymm1 ; ymm1=R(13579BDF********HJLNPRTV********)
  150. vpaddw ymm2, ymm2, ymm6 ; ymm2=((G-Y)+YE)=GE=G(02468ACEGIKMOQSU)
  151. vpaddw ymm3, ymm3, ymm7 ; ymm3=((G-Y)+YO)=GO=G(13579BDFHJLNPRTV)
  152. vpackuswb ymm2, ymm2, ymm2 ; ymm2=G(02468ACE********GIKMOQSU********)
  153. vpackuswb ymm3, ymm3, ymm3 ; ymm3=G(13579BDF********HJLNPRTV********)
  154. vpaddw ymm4, ymm4, ymm6 ; ymm4=((B-Y)+YE)=BE=B(02468ACEGIKMOQSU)
  155. vpaddw ymm5, ymm5, ymm7 ; ymm5=((B-Y)+YO)=BO=B(13579BDFHJLNPRTV)
  156. vpackuswb ymm4, ymm4, ymm4 ; ymm4=B(02468ACE********GIKMOQSU********)
  157. vpackuswb ymm5, ymm5, ymm5 ; ymm5=B(13579BDF********HJLNPRTV********)
  158. %if RGB_PIXELSIZE == 3 ; ---------------
  159. ; ymmA=(00 02 04 06 08 0A 0C 0E ** 0G 0I 0K 0M 0O 0Q 0S 0U **)
  160. ; ymmB=(01 03 05 07 09 0B 0D 0F ** 0H 0J 0L 0N 0P 0R 0T 0V **)
  161. ; ymmC=(10 12 14 16 18 1A 1C 1E ** 1G 1I 1K 1M 1O 1Q 1S 1U **)
  162. ; ymmD=(11 13 15 17 19 1B 1D 1F ** 1H 1J 1L 1N 1P 1R 1T 1V **)
  163. ; ymmE=(20 22 24 26 28 2A 2C 2E ** 2G 2I 2K 2M 2O 2Q 2S 2U **)
  164. ; ymmF=(21 23 25 27 29 2B 2D 2F ** 2H 2J 2L 2N 2P 2R 2T 2V **)
  165. ; ymmG=(** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **)
  166. ; ymmH=(** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **)
  167. vpunpcklbw ymmA, ymmA, ymmC ; ymmA=(00 10 02 12 04 14 06 16 08 18 0A 1A 0C 1C 0E 1E
  168. ; 0G 1G 0I 1I 0K 1K 0M 1M 0O 1O 0Q 1Q 0S 1S 0U 1U)
  169. vpunpcklbw ymmE, ymmE, ymmB ; ymmE=(20 01 22 03 24 05 26 07 28 09 2A 0B 2C 0D 2E 0F
  170. ; 2G 0H 2I 0J 2K 0L 2M 0N 2O 0P 2Q 0R 2S 0T 2U 0V)
  171. vpunpcklbw ymmD, ymmD, ymmF ; ymmD=(11 21 13 23 15 25 17 27 19 29 1B 2B 1D 2D 1F 2F
  172. ; 1H 2H 1J 2J 1L 2L 1N 2N 1P 2P 1R 2R 1T 2T 1V 2V)
  173. vpsrldq ymmH, ymmA, 2 ; ymmH=(02 12 04 14 06 16 08 18 0A 1A 0C 1C 0E 1E 0G 1G
  174. ; 0I 1I 0K 1K 0M 1M 0O 1O 0Q 1Q 0S 1S 0U 1U -- --)
  175. vpunpckhwd ymmG, ymmA, ymmE ; ymmG=(08 18 28 09 0A 1A 2A 0B 0C 1C 2C 0D 0E 1E 2E 0F
  176. ; 0O 1O 2O 0P 0Q 1Q 2Q 0R 0S 1S 2S 0T 0U 1U 2U 0V)
  177. vpunpcklwd ymmA, ymmA, ymmE ; ymmA=(00 10 20 01 02 12 22 03 04 14 24 05 06 16 26 07
  178. ; 0G 1G 2G 0H 0I 1I 2I 0J 0K 1K 2K 0L 0M 1M 2M 0N)
  179. vpsrldq ymmE, ymmE, 2 ; ymmE=(22 03 24 05 26 07 28 09 2A 0B 2C 0D 2E 0F 2G 0H
  180. ; 2I 0J 2K 0L 2M 0N 2O 0P 2Q 0R 2S 0T 2U 0V -- --)
  181. vpsrldq ymmB, ymmD, 2 ; ymmB=(13 23 15 25 17 27 19 29 1B 2B 1D 2D 1F 2F 1H 2H
  182. ; 1J 2J 1L 2L 1N 2N 1P 2P 1R 2R 1T 2T 1V 2V -- --)
  183. vpunpckhwd ymmC, ymmD, ymmH ; ymmC=(19 29 0A 1A 1B 2B 0C 1C 1D 2D 0E 1E 1F 2F 0G 1G
  184. ; 1P 2P 0Q 1Q 1R 2R 0S 1S 1T 2T 0U 1U 1V 2V -- --)
  185. vpunpcklwd ymmD, ymmD, ymmH ; ymmD=(11 21 02 12 13 23 04 14 15 25 06 16 17 27 08 18
  186. ; 1H 2H 0I 1I 1J 2J 0K 1K 1L 2L 0M 1M 1N 2N 0O 1O)
  187. vpunpckhwd ymmF, ymmE, ymmB ; ymmF=(2A 0B 1B 2B 2C 0D 1D 2D 2E 0F 1F 2F 2G 0H 1H 2H
  188. ; 2Q 0R 1R 2R 2S 0T 1T 2T 2U 0V 1V 2V -- -- -- --)
  189. vpunpcklwd ymmE, ymmE, ymmB ; ymmE=(22 03 13 23 24 05 15 25 26 07 17 27 28 09 19 29
  190. ; 2I 0J 1J 2J 2K 0L 1L 2L 2M 0N 1N 2N 2O 0P 1P 2P)
  191. vpshufd ymmH, ymmA, 0x4E ; ymmH=(04 14 24 05 06 16 26 07 00 10 20 01 02 12 22 03
  192. ; 0K 1K 2K 0L 0M 1M 2M 0N 0G 1G 2G 0H 0I 1I 2I 0J)
  193. vpunpckldq ymmA, ymmA, ymmD ; ymmA=(00 10 20 01 11 21 02 12 02 12 22 03 13 23 04 14
  194. ; 0G 1G 2G 0H 1H 2H 0I 1I 0I 1I 2I 0J 1J 2J 0K 1K)
  195. vpunpckhdq ymmD, ymmD, ymmE ; ymmD=(15 25 06 16 26 07 17 27 17 27 08 18 28 09 19 29
  196. ; 1L 2L 0M 1M 2M 0N 1N 2N 1N 2N 0O 1O 2O 0P 1P 2P)
  197. vpunpckldq ymmE, ymmE, ymmH ; ymmE=(22 03 13 23 04 14 24 05 24 05 15 25 06 16 26 07
  198. ; 2I 0J 1J 2J 0K 1K 2K 0L 2K 0L 1L 2L 0M 1M 2M 0N)
  199. vpshufd ymmH, ymmG, 0x4E ; ymmH=(0C 1C 2C 0D 0E 1E 2E 0F 08 18 28 09 0A 1A 2A 0B
  200. ; 0S 1S 2S 0T 0U 1U 2U 0V 0O 1O 2O 0P 0Q 1Q 2Q 0R)
  201. vpunpckldq ymmG, ymmG, ymmC ; ymmG=(08 18 28 09 19 29 0A 1A 0A 1A 2A 0B 1B 2B 0C 1C
  202. ; 0O 1O 2O 0P 1P 2P 0Q 1Q 0Q 1Q 2Q 0R 1R 2R 0S 1S)
  203. vpunpckhdq ymmC, ymmC, ymmF ; ymmC=(1D 2D 0E 1E 2E 0F 1F 2F 1F 2F 0G 1G 2G 0H 1H 2H
  204. ; 1T 2T 0U 1U 2U 0V 1V 2V 1V 2V -- -- -- -- -- --)
  205. vpunpckldq ymmF, ymmF, ymmH ; ymmF=(2A 0B 1B 2B 0C 1C 2C 0D 2C 0D 1D 2D 0E 1E 2E 0F
  206. ; 2Q 0R 1R 2R 0S 1S 2S 0T 2S 0T 1T 2T 0U 1U 2U 0V)
  207. vpunpcklqdq ymmH, ymmA, ymmE ; ymmH=(00 10 20 01 11 21 02 12 22 03 13 23 04 14 24 05
  208. ; 0G 1G 2G 0H 1H 2H 0I 1I 2I 0J 1J 2J 0K 1K 2K 0L)
  209. vpunpcklqdq ymmG, ymmD, ymmG ; ymmG=(15 25 06 16 26 07 17 27 08 18 28 09 19 29 0A 1A
  210. ; 1L 2L 0M 1M 2M 0N 1N 2N 0O 1O 2O 0P 1P 2P 0Q 1Q)
  211. vpunpcklqdq ymmC, ymmF, ymmC ; ymmC=(2A 0B 1B 2B 0C 1C 2C 0D 1D 2D 0E 1E 2E 0F 1F 2F
  212. ; 2Q 0R 1R 2R 0S 1S 2S 0T 1T 2T 0U 1U 2U 0V 1V 2V)
  213. vperm2i128 ymmA, ymmH, ymmG, 0x20 ; ymmA=(00 10 20 01 11 21 02 12 22 03 13 23 04 14 24 05
  214. ; 15 25 06 16 26 07 17 27 08 18 28 09 19 29 0A 1A)
  215. vperm2i128 ymmD, ymmC, ymmH, 0x30 ; ymmD=(2A 0B 1B 2B 0C 1C 2C 0D 1D 2D 0E 1E 2E 0F 1F 2F
  216. ; 0G 1G 2G 0H 1H 2H 0I 1I 2I 0J 1J 2J 0K 1K 2K 0L)
  217. vperm2i128 ymmF, ymmG, ymmC, 0x31 ; ymmF=(1L 2L 0M 1M 2M 0N 1N 2N 0O 1O 2O 0P 1P 2P 0Q 1Q
  218. ; 2Q 0R 1R 2R 0S 1S 2S 0T 1T 2T 0U 1U 2U 0V 1V 2V)
  219. cmp rcx, byte SIZEOF_YMMWORD
  220. jb short .column_st64
  221. test rdi, SIZEOF_YMMWORD-1
  222. jnz short .out1
  223. ; --(aligned)-------------------
  224. vmovntdq YMMWORD [rdi+0*SIZEOF_YMMWORD], ymmA
  225. vmovntdq YMMWORD [rdi+1*SIZEOF_YMMWORD], ymmD
  226. vmovntdq YMMWORD [rdi+2*SIZEOF_YMMWORD], ymmF
  227. jmp short .out0
  228. .out1: ; --(unaligned)-----------------
  229. vmovdqu YMMWORD [rdi+0*SIZEOF_YMMWORD], ymmA
  230. vmovdqu YMMWORD [rdi+1*SIZEOF_YMMWORD], ymmD
  231. vmovdqu YMMWORD [rdi+2*SIZEOF_YMMWORD], ymmF
  232. .out0:
  233. add rdi, byte RGB_PIXELSIZE*SIZEOF_YMMWORD ; outptr
  234. sub rcx, byte SIZEOF_YMMWORD
  235. jz near .endcolumn
  236. add rsi, byte SIZEOF_YMMWORD ; inptr0
  237. dec al ; Yctr
  238. jnz near .Yloop_2nd
  239. add rbx, byte SIZEOF_YMMWORD ; inptr1
  240. add rdx, byte SIZEOF_YMMWORD ; inptr2
  241. jmp near .columnloop
  242. .column_st64:
  243. lea rcx, [rcx+rcx*2] ; imul ecx, RGB_PIXELSIZE
  244. cmp rcx, byte 2*SIZEOF_YMMWORD
  245. jb short .column_st32
  246. vmovdqu YMMWORD [rdi+0*SIZEOF_YMMWORD], ymmA
  247. vmovdqu YMMWORD [rdi+1*SIZEOF_YMMWORD], ymmD
  248. add rdi, byte 2*SIZEOF_YMMWORD ; outptr
  249. vmovdqa ymmA, ymmF
  250. sub rcx, byte 2*SIZEOF_YMMWORD
  251. jmp short .column_st31
  252. .column_st32:
  253. cmp rcx, byte SIZEOF_YMMWORD
  254. jb short .column_st31
  255. vmovdqu YMMWORD [rdi+0*SIZEOF_YMMWORD], ymmA
  256. add rdi, byte SIZEOF_YMMWORD ; outptr
  257. vmovdqa ymmA, ymmD
  258. sub rcx, byte SIZEOF_YMMWORD
  259. jmp short .column_st31
  260. .column_st31:
  261. cmp rcx, byte SIZEOF_XMMWORD
  262. jb short .column_st15
  263. vmovdqu XMMWORD [rdi+0*SIZEOF_XMMWORD], xmmA
  264. add rdi, byte SIZEOF_XMMWORD ; outptr
  265. vperm2i128 ymmA, ymmA, ymmA, 1
  266. sub rcx, byte SIZEOF_XMMWORD
  267. .column_st15:
  268. ; Store the lower 8 bytes of xmmA to the output when it has enough
  269. ; space.
  270. cmp rcx, byte SIZEOF_MMWORD
  271. jb short .column_st7
  272. vmovq XMM_MMWORD [rdi], xmmA
  273. add rdi, byte SIZEOF_MMWORD
  274. sub rcx, byte SIZEOF_MMWORD
  275. vpsrldq xmmA, xmmA, SIZEOF_MMWORD
  276. .column_st7:
  277. ; Store the lower 4 bytes of xmmA to the output when it has enough
  278. ; space.
  279. cmp rcx, byte SIZEOF_DWORD
  280. jb short .column_st3
  281. vmovd XMM_DWORD [rdi], xmmA
  282. add rdi, byte SIZEOF_DWORD
  283. sub rcx, byte SIZEOF_DWORD
  284. vpsrldq xmmA, xmmA, SIZEOF_DWORD
  285. .column_st3:
  286. ; Store the lower 2 bytes of rax to the output when it has enough
  287. ; space.
  288. vmovd eax, xmmA
  289. cmp rcx, byte SIZEOF_WORD
  290. jb short .column_st1
  291. mov word [rdi], ax
  292. add rdi, byte SIZEOF_WORD
  293. sub rcx, byte SIZEOF_WORD
  294. shr rax, 16
  295. .column_st1:
  296. ; Store the lower 1 byte of rax to the output when it has enough
  297. ; space.
  298. test rcx, rcx
  299. jz short .endcolumn
  300. mov byte [rdi], al
  301. %else ; RGB_PIXELSIZE == 4 ; -----------
  302. %ifdef RGBX_FILLER_0XFF
  303. vpcmpeqb ymm6, ymm6, ymm6 ; ymm6=XE=X(02468ACE********GIKMOQSU********)
  304. vpcmpeqb ymm7, ymm7, ymm7 ; ymm7=XO=X(13579BDF********HJLNPRTV********)
  305. %else
  306. vpxor ymm6, ymm6, ymm6 ; ymm6=XE=X(02468ACE********GIKMOQSU********)
  307. vpxor ymm7, ymm7, ymm7 ; ymm7=XO=X(13579BDF********HJLNPRTV********)
  308. %endif
  309. ; ymmA=(00 02 04 06 08 0A 0C 0E ** 0G 0I 0K 0M 0O 0Q 0S 0U **)
  310. ; ymmB=(01 03 05 07 09 0B 0D 0F ** 0H 0J 0L 0N 0P 0R 0T 0V **)
  311. ; ymmC=(10 12 14 16 18 1A 1C 1E ** 1G 1I 1K 1M 1O 1Q 1S 1U **)
  312. ; ymmD=(11 13 15 17 19 1B 1D 1F ** 1H 1J 1L 1N 1P 1R 1T 1V **)
  313. ; ymmE=(20 22 24 26 28 2A 2C 2E ** 2G 2I 2K 2M 2O 2Q 2S 2U **)
  314. ; ymmF=(21 23 25 27 29 2B 2D 2F ** 2H 2J 2L 2N 2P 2R 2T 2V **)
  315. ; ymmG=(30 32 34 36 38 3A 3C 3E ** 3G 3I 3K 3M 3O 3Q 3S 3U **)
  316. ; ymmH=(31 33 35 37 39 3B 3D 3F ** 3H 3J 3L 3N 3P 3R 3T 3V **)
  317. vpunpcklbw ymmA, ymmA, ymmC ; ymmA=(00 10 02 12 04 14 06 16 08 18 0A 1A 0C 1C 0E 1E
  318. ; 0G 1G 0I 1I 0K 1K 0M 1M 0O 1O 0Q 1Q 0S 1S 0U 1U)
  319. vpunpcklbw ymmE, ymmE, ymmG ; ymmE=(20 30 22 32 24 34 26 36 28 38 2A 3A 2C 3C 2E 3E
  320. ; 2G 3G 2I 3I 2K 3K 2M 3M 2O 3O 2Q 3Q 2S 3S 2U 3U)
  321. vpunpcklbw ymmB, ymmB, ymmD ; ymmB=(01 11 03 13 05 15 07 17 09 19 0B 1B 0D 1D 0F 1F
  322. ; 0H 1H 0J 1J 0L 1L 0N 1N 0P 1P 0R 1R 0T 1T 0V 1V)
  323. vpunpcklbw ymmF, ymmF, ymmH ; ymmF=(21 31 23 33 25 35 27 37 29 39 2B 3B 2D 3D 2F 3F
  324. ; 2H 3H 2J 3J 2L 3L 2N 3N 2P 3P 2R 3R 2T 3T 2V 3V)
  325. vpunpckhwd ymmC, ymmA, ymmE ; ymmC=(08 18 28 38 0A 1A 2A 3A 0C 1C 2C 3C 0E 1E 2E 3E
  326. ; 0O 1O 2O 3O 0Q 1Q 2Q 3Q 0S 1S 2S 3S 0U 1U 2U 3U)
  327. vpunpcklwd ymmA, ymmA, ymmE ; ymmA=(00 10 20 30 02 12 22 32 04 14 24 34 06 16 26 36
  328. ; 0G 1G 2G 3G 0I 1I 2I 3I 0K 1K 2K 3K 0M 1M 2M 3M)
  329. vpunpckhwd ymmG, ymmB, ymmF ; ymmG=(09 19 29 39 0B 1B 2B 3B 0D 1D 2D 3D 0F 1F 2F 3F
  330. ; 0P 1P 2P 3P 0R 1R 2R 3R 0T 1T 2T 3T 0V 1V 2V 3V)
  331. vpunpcklwd ymmB, ymmB, ymmF ; ymmB=(01 11 21 31 03 13 23 33 05 15 25 35 07 17 27 37
  332. ; 0H 1H 2H 3H 0J 1J 2J 3J 0L 1L 2L 3L 0N 1N 2N 3N)
  333. vpunpckhdq ymmE, ymmA, ymmB ; ymmE=(04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
  334. ; 0K 1K 2K 3K 0L 1L 2L 3L 0M 1M 2M 3M 0N 1N 2N 3N)
  335. vpunpckldq ymmB, ymmA, ymmB ; ymmB=(00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
  336. ; 0G 1G 2G 3G 0H 1H 2H 3H 0I 1I 2I 3I 0J 1J 2J 3J)
  337. vpunpckhdq ymmF, ymmC, ymmG ; ymmF=(0C 1C 2C 3C 0D 1D 2D 3D 0E 1E 2E 3E 0F 1F 2F 3F
  338. ; 0S 1S 2S 3S 0T 1T 2T 3T 0U 1U 2U 3U 0V 1V 2V 3V)
  339. vpunpckldq ymmG, ymmC, ymmG ; ymmG=(08 18 28 38 09 19 29 39 0A 1A 2A 3A 0B 1B 2B 3B
  340. ; 0O 1O 2O 3O 0P 1P 2P 3P 0Q 1Q 2Q 3Q 0R 1R 2R 3R)
  341. vperm2i128 ymmA, ymmB, ymmE, 0x20 ; ymmA=(00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
  342. ; 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37)
  343. vperm2i128 ymmD, ymmG, ymmF, 0x20 ; ymmD=(08 18 28 38 09 19 29 39 0A 1A 2A 3A 0B 1B 2B 3B
  344. ; 0C 1C 2C 3C 0D 1D 2D 3D 0E 1E 2E 3E 0F 1F 2F 3F)
  345. vperm2i128 ymmC, ymmB, ymmE, 0x31 ; ymmC=(0G 1G 2G 3G 0H 1H 2H 3H 0I 1I 2I 3I 0J 1J 2J 3J
  346. ; 0K 1K 2K 3K 0L 1L 2L 3L 0M 1M 2M 3M 0N 1N 2N 3N)
  347. vperm2i128 ymmH, ymmG, ymmF, 0x31 ; ymmH=(0O 1O 2O 3O 0P 1P 2P 3P 0Q 1Q 2Q 3Q 0R 1R 2R 3R
  348. ; 0S 1S 2S 3S 0T 1T 2T 3T 0U 1U 2U 3U 0V 1V 2V 3V)
  349. cmp rcx, byte SIZEOF_YMMWORD
  350. jb short .column_st64
  351. test rdi, SIZEOF_YMMWORD-1
  352. jnz short .out1
  353. ; --(aligned)-------------------
  354. vmovntdq YMMWORD [rdi+0*SIZEOF_YMMWORD], ymmA
  355. vmovntdq YMMWORD [rdi+1*SIZEOF_YMMWORD], ymmD
  356. vmovntdq YMMWORD [rdi+2*SIZEOF_YMMWORD], ymmC
  357. vmovntdq YMMWORD [rdi+3*SIZEOF_YMMWORD], ymmH
  358. jmp short .out0
  359. .out1: ; --(unaligned)-----------------
  360. vmovdqu YMMWORD [rdi+0*SIZEOF_YMMWORD], ymmA
  361. vmovdqu YMMWORD [rdi+1*SIZEOF_YMMWORD], ymmD
  362. vmovdqu YMMWORD [rdi+2*SIZEOF_YMMWORD], ymmC
  363. vmovdqu YMMWORD [rdi+3*SIZEOF_YMMWORD], ymmH
  364. .out0:
  365. add rdi, RGB_PIXELSIZE*SIZEOF_YMMWORD ; outptr
  366. sub rcx, byte SIZEOF_YMMWORD
  367. jz near .endcolumn
  368. add rsi, byte SIZEOF_YMMWORD ; inptr0
  369. dec al
  370. jnz near .Yloop_2nd
  371. add rbx, byte SIZEOF_YMMWORD ; inptr1
  372. add rdx, byte SIZEOF_YMMWORD ; inptr2
  373. jmp near .columnloop
  374. .column_st64:
  375. cmp rcx, byte SIZEOF_YMMWORD/2
  376. jb short .column_st32
  377. vmovdqu YMMWORD [rdi+0*SIZEOF_YMMWORD], ymmA
  378. vmovdqu YMMWORD [rdi+1*SIZEOF_YMMWORD], ymmD
  379. add rdi, byte 2*SIZEOF_YMMWORD ; outptr
  380. vmovdqa ymmA, ymmC
  381. vmovdqa ymmD, ymmH
  382. sub rcx, byte SIZEOF_YMMWORD/2
  383. .column_st32:
  384. cmp rcx, byte SIZEOF_YMMWORD/4
  385. jb short .column_st16
  386. vmovdqu YMMWORD [rdi+0*SIZEOF_YMMWORD], ymmA
  387. add rdi, byte SIZEOF_YMMWORD ; outptr
  388. vmovdqa ymmA, ymmD
  389. sub rcx, byte SIZEOF_YMMWORD/4
  390. .column_st16:
  391. cmp rcx, byte SIZEOF_YMMWORD/8
  392. jb short .column_st15
  393. vmovdqu XMMWORD [rdi+0*SIZEOF_XMMWORD], xmmA
  394. add rdi, byte SIZEOF_XMMWORD ; outptr
  395. vperm2i128 ymmA, ymmA, ymmA, 1
  396. sub rcx, byte SIZEOF_YMMWORD/8
  397. .column_st15:
  398. ; Store two pixels (8 bytes) of ymmA to the output when it has enough
  399. ; space.
  400. cmp rcx, byte SIZEOF_YMMWORD/16
  401. jb short .column_st7
  402. vmovq MMWORD [rdi], xmmA
  403. add rdi, byte SIZEOF_YMMWORD/16*4
  404. sub rcx, byte SIZEOF_YMMWORD/16
  405. vpsrldq xmmA, SIZEOF_YMMWORD/16*4
  406. .column_st7:
  407. ; Store one pixel (4 bytes) of ymmA to the output when it has enough
  408. ; space.
  409. test rcx, rcx
  410. jz short .endcolumn
  411. vmovd XMM_DWORD [rdi], xmmA
  412. %endif ; RGB_PIXELSIZE ; ---------------
  413. .endcolumn:
  414. sfence ; flush the write buffer
  415. .return:
  416. pop rbx
  417. vzeroupper
  418. uncollect_args 4
  419. mov rsp, rbp ; rsp <- aligned rbp
  420. pop rsp ; rsp <- original rbp
  421. pop rbp
  422. ret
  423. ; --------------------------------------------------------------------------
  424. ;
  425. ; Upsample and color convert for the case of 2:1 horizontal and 2:1 vertical.
  426. ;
  427. ; GLOBAL(void)
  428. ; jsimd_h2v2_merged_upsample_avx2(JDIMENSION output_width,
  429. ; JSAMPIMAGE input_buf,
  430. ; JDIMENSION in_row_group_ctr,
  431. ; JSAMPARRAY output_buf);
  432. ;
  433. ; r10d = JDIMENSION output_width
  434. ; r11 = JSAMPIMAGE input_buf
  435. ; r12d = JDIMENSION in_row_group_ctr
  436. ; r13 = JSAMPARRAY output_buf
  437. align 32
  438. GLOBAL_FUNCTION(jsimd_h2v2_merged_upsample_avx2)
  439. EXTN(jsimd_h2v2_merged_upsample_avx2):
  440. push rbp
  441. mov rax, rsp
  442. mov rbp, rsp
  443. collect_args 4
  444. push rbx
  445. mov eax, r10d
  446. mov rdi, r11
  447. mov ecx, r12d
  448. mov rsi, JSAMPARRAY [rdi+0*SIZEOF_JSAMPARRAY]
  449. mov rbx, JSAMPARRAY [rdi+1*SIZEOF_JSAMPARRAY]
  450. mov rdx, JSAMPARRAY [rdi+2*SIZEOF_JSAMPARRAY]
  451. mov rdi, r13
  452. lea rsi, [rsi+rcx*SIZEOF_JSAMPROW]
  453. push rdx ; inptr2
  454. push rbx ; inptr1
  455. push rsi ; inptr00
  456. mov rbx, rsp
  457. push rdi
  458. push rcx
  459. push rax
  460. %ifdef WIN64
  461. mov r8, rcx
  462. mov r9, rdi
  463. mov rcx, rax
  464. mov rdx, rbx
  465. %else
  466. mov rdx, rcx
  467. mov rcx, rdi
  468. mov rdi, rax
  469. mov rsi, rbx
  470. %endif
  471. call EXTN(jsimd_h2v1_merged_upsample_avx2)
  472. pop rax
  473. pop rcx
  474. pop rdi
  475. pop rsi
  476. pop rbx
  477. pop rdx
  478. add rdi, byte SIZEOF_JSAMPROW ; outptr1
  479. add rsi, byte SIZEOF_JSAMPROW ; inptr01
  480. push rdx ; inptr2
  481. push rbx ; inptr1
  482. push rsi ; inptr00
  483. mov rbx, rsp
  484. push rdi
  485. push rcx
  486. push rax
  487. %ifdef WIN64
  488. mov r8, rcx
  489. mov r9, rdi
  490. mov rcx, rax
  491. mov rdx, rbx
  492. %else
  493. mov rdx, rcx
  494. mov rcx, rdi
  495. mov rdi, rax
  496. mov rsi, rbx
  497. %endif
  498. call EXTN(jsimd_h2v1_merged_upsample_avx2)
  499. pop rax
  500. pop rcx
  501. pop rdi
  502. pop rsi
  503. pop rbx
  504. pop rdx
  505. pop rbx
  506. uncollect_args 4
  507. pop rbp
  508. ret
  509. ; For some reason, the OS X linker does not honor the request to align the
  510. ; segment unless we do this.
  511. align 32