60 #define mask16b mask15b
77 #define BY ((int)( 0.098*(1<<RGB2YUV_SHIFT)+0.5))
78 #define BV ((int)(-0.071*(1<<RGB2YUV_SHIFT)+0.5))
79 #define BU ((int)( 0.439*(1<<RGB2YUV_SHIFT)+0.5))
80 #define GY ((int)( 0.504*(1<<RGB2YUV_SHIFT)+0.5))
81 #define GV ((int)(-0.368*(1<<RGB2YUV_SHIFT)+0.5))
82 #define GU ((int)(-0.291*(1<<RGB2YUV_SHIFT)+0.5))
83 #define RY ((int)( 0.257*(1<<RGB2YUV_SHIFT)+0.5))
84 #define RV ((int)( 0.439*(1<<RGB2YUV_SHIFT)+0.5))
85 #define RU ((int)(-0.148*(1<<RGB2YUV_SHIFT)+0.5))
88 #define PREFETCH "prefetchnta"
90 #define MOVNTQ "movntq"
91 #define SFENCE "sfence"
95 static inline void rgb24tobgr32_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
98 const uint8_t *
s =
src;
100 const uint8_t *mm_end;
104 __asm__ volatile(
"movq %0, %%mm7"::
"m"(mask32a):
"memory");
108 "movd (%1), %%mm0 \n\t"
109 "punpckldq 3(%1), %%mm0 \n\t"
110 "movd 6(%1), %%mm1 \n\t"
111 "punpckldq 9(%1), %%mm1 \n\t"
112 "movd 12(%1), %%mm2 \n\t"
113 "punpckldq 15(%1), %%mm2 \n\t"
114 "movd 18(%1), %%mm3 \n\t"
115 "punpckldq 21(%1), %%mm3 \n\t"
116 "por %%mm7, %%mm0 \n\t"
117 "por %%mm7, %%mm1 \n\t"
118 "por %%mm7, %%mm2 \n\t"
119 "por %%mm7, %%mm3 \n\t"
121 MOVNTQ" %%mm1, 8(%0) \n\t"
122 MOVNTQ" %%mm2, 16(%0) \n\t"
129 __asm__ volatile(SFENCE:::
"memory");
130 __asm__ volatile(EMMS:::
"memory");
139 #define STORE_BGR24_MMX \
140 "psrlq $8, %%mm2 \n\t" \
141 "psrlq $8, %%mm3 \n\t" \
142 "psrlq $8, %%mm6 \n\t" \
143 "psrlq $8, %%mm7 \n\t" \
144 "pand "MANGLE(mask24l)", %%mm0\n\t" \
145 "pand "MANGLE(mask24l)", %%mm1\n\t" \
146 "pand "MANGLE(mask24l)", %%mm4\n\t" \
147 "pand "MANGLE(mask24l)", %%mm5\n\t" \
148 "pand "MANGLE(mask24h)", %%mm2\n\t" \
149 "pand "MANGLE(mask24h)", %%mm3\n\t" \
150 "pand "MANGLE(mask24h)", %%mm6\n\t" \
151 "pand "MANGLE(mask24h)", %%mm7\n\t" \
152 "por %%mm2, %%mm0 \n\t" \
153 "por %%mm3, %%mm1 \n\t" \
154 "por %%mm6, %%mm4 \n\t" \
155 "por %%mm7, %%mm5 \n\t" \
157 "movq %%mm1, %%mm2 \n\t" \
158 "movq %%mm4, %%mm3 \n\t" \
159 "psllq $48, %%mm2 \n\t" \
160 "psllq $32, %%mm3 \n\t" \
161 "por %%mm2, %%mm0 \n\t" \
162 "psrlq $16, %%mm1 \n\t" \
163 "psrlq $32, %%mm4 \n\t" \
164 "psllq $16, %%mm5 \n\t" \
165 "por %%mm3, %%mm1 \n\t" \
166 "por %%mm5, %%mm4 \n\t" \
168 MOVNTQ" %%mm0, (%0) \n\t" \
169 MOVNTQ" %%mm1, 8(%0) \n\t" \
170 MOVNTQ" %%mm4, 16(%0)"
173 static inline void rgb32tobgr24_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
176 const uint8_t *
s =
src;
178 const uint8_t *mm_end;
185 "movq (%1), %%mm0 \n\t"
186 "movq 8(%1), %%mm1 \n\t"
187 "movq 16(%1), %%mm4 \n\t"
188 "movq 24(%1), %%mm5 \n\t"
189 "movq %%mm0, %%mm2 \n\t"
190 "movq %%mm1, %%mm3 \n\t"
191 "movq %%mm4, %%mm6 \n\t"
192 "movq %%mm5, %%mm7 \n\t"
200 __asm__ volatile(SFENCE:::
"memory");
201 __asm__ volatile(EMMS:::
"memory");
216 static inline void rgb15to16_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
218 register const uint8_t*
s=
src;
219 register uint8_t* d=
dst;
220 register const uint8_t *end;
221 const uint8_t *mm_end;
224 __asm__ volatile(
"movq %0, %%mm4"::
"m"(mask15s));
229 "movq (%1), %%mm0 \n\t"
230 "movq 8(%1), %%mm2 \n\t"
231 "movq %%mm0, %%mm1 \n\t"
232 "movq %%mm2, %%mm3 \n\t"
233 "pand %%mm4, %%mm0 \n\t"
234 "pand %%mm4, %%mm2 \n\t"
235 "paddw %%mm1, %%mm0 \n\t"
236 "paddw %%mm3, %%mm2 \n\t"
244 __asm__ volatile(SFENCE:::
"memory");
245 __asm__ volatile(EMMS:::
"memory");
248 register unsigned x= *((
const uint32_t *)
s);
249 *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
254 register unsigned short x= *((
const uint16_t *)
s);
255 *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
259 static inline void rgb16to15_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
261 register const uint8_t*
s=
src;
262 register uint8_t* d=
dst;
263 register const uint8_t *end;
264 const uint8_t *mm_end;
267 __asm__ volatile(
"movq %0, %%mm7"::
"m"(mask15rg));
268 __asm__ volatile(
"movq %0, %%mm6"::
"m"(mask15b));
273 "movq (%1), %%mm0 \n\t"
274 "movq 8(%1), %%mm2 \n\t"
275 "movq %%mm0, %%mm1 \n\t"
276 "movq %%mm2, %%mm3 \n\t"
277 "psrlq $1, %%mm0 \n\t"
278 "psrlq $1, %%mm2 \n\t"
279 "pand %%mm7, %%mm0 \n\t"
280 "pand %%mm7, %%mm2 \n\t"
281 "pand %%mm6, %%mm1 \n\t"
282 "pand %%mm6, %%mm3 \n\t"
283 "por %%mm1, %%mm0 \n\t"
284 "por %%mm3, %%mm2 \n\t"
292 __asm__ volatile(SFENCE:::
"memory");
293 __asm__ volatile(EMMS:::
"memory");
296 register uint32_t x= *((
const uint32_t*)
s);
297 *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
302 register uint16_t x= *((
const uint16_t*)
s);
303 *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
307 static inline void rgb32to16_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
309 const uint8_t *
s =
src;
311 const uint8_t *mm_end;
312 uint16_t *d = (uint16_t *)
dst;
316 "movq %3, %%mm5 \n\t"
317 "movq %4, %%mm6 \n\t"
318 "movq %5, %%mm7 \n\t"
323 "movd (%1), %%mm0 \n\t"
324 "movd 4(%1), %%mm3 \n\t"
325 "punpckldq 8(%1), %%mm0 \n\t"
326 "punpckldq 12(%1), %%mm3 \n\t"
327 "movq %%mm0, %%mm1 \n\t"
328 "movq %%mm3, %%mm4 \n\t"
329 "pand %%mm6, %%mm0 \n\t"
330 "pand %%mm6, %%mm3 \n\t"
331 "pmaddwd %%mm7, %%mm0 \n\t"
332 "pmaddwd %%mm7, %%mm3 \n\t"
333 "pand %%mm5, %%mm1 \n\t"
334 "pand %%mm5, %%mm4 \n\t"
335 "por %%mm1, %%mm0 \n\t"
336 "por %%mm4, %%mm3 \n\t"
337 "psrld $5, %%mm0 \n\t"
338 "pslld $11, %%mm3 \n\t"
339 "por %%mm3, %%mm0 \n\t"
347 :
"r" (mm_end),
"m" (mask3216g),
"m" (mask3216br),
"m" (mul3216)
349 __asm__ volatile(SFENCE:::
"memory");
350 __asm__ volatile(EMMS:::
"memory");
352 register int rgb = *(
const uint32_t*)
s;
s += 4;
353 *d++ = ((
rgb&0xFF)>>3) + ((
rgb&0xFC00)>>5) + ((
rgb&0xF80000)>>8);
357 static inline void rgb32tobgr16_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
359 const uint8_t *
s =
src;
361 const uint8_t *mm_end;
362 uint16_t *d = (uint16_t *)
dst;
366 "movq %0, %%mm7 \n\t"
367 "movq %1, %%mm6 \n\t"
368 ::
"m"(red_16mask),
"m"(green_16mask));
373 "movd (%1), %%mm0 \n\t"
374 "movd 4(%1), %%mm3 \n\t"
375 "punpckldq 8(%1), %%mm0 \n\t"
376 "punpckldq 12(%1), %%mm3 \n\t"
377 "movq %%mm0, %%mm1 \n\t"
378 "movq %%mm0, %%mm2 \n\t"
379 "movq %%mm3, %%mm4 \n\t"
380 "movq %%mm3, %%mm5 \n\t"
381 "psllq $8, %%mm0 \n\t"
382 "psllq $8, %%mm3 \n\t"
383 "pand %%mm7, %%mm0 \n\t"
384 "pand %%mm7, %%mm3 \n\t"
385 "psrlq $5, %%mm1 \n\t"
386 "psrlq $5, %%mm4 \n\t"
387 "pand %%mm6, %%mm1 \n\t"
388 "pand %%mm6, %%mm4 \n\t"
389 "psrlq $19, %%mm2 \n\t"
390 "psrlq $19, %%mm5 \n\t"
391 "pand %2, %%mm2 \n\t"
392 "pand %2, %%mm5 \n\t"
393 "por %%mm1, %%mm0 \n\t"
394 "por %%mm4, %%mm3 \n\t"
395 "por %%mm2, %%mm0 \n\t"
396 "por %%mm5, %%mm3 \n\t"
397 "psllq $16, %%mm3 \n\t"
398 "por %%mm3, %%mm0 \n\t"
400 ::
"r"(d),
"r"(
s),
"m"(blue_16mask):
"memory");
404 __asm__ volatile(SFENCE:::
"memory");
405 __asm__ volatile(EMMS:::
"memory");
407 register int rgb = *(
const uint32_t*)
s;
s += 4;
408 *d++ = ((
rgb&0xF8)<<8) + ((
rgb&0xFC00)>>5) + ((
rgb&0xF80000)>>19);
412 static inline void rgb32to15_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
414 const uint8_t *
s =
src;
416 const uint8_t *mm_end;
417 uint16_t *d = (uint16_t *)
dst;
421 "movq %3, %%mm5 \n\t"
422 "movq %4, %%mm6 \n\t"
423 "movq %5, %%mm7 \n\t"
428 "movd (%1), %%mm0 \n\t"
429 "movd 4(%1), %%mm3 \n\t"
430 "punpckldq 8(%1), %%mm0 \n\t"
431 "punpckldq 12(%1), %%mm3 \n\t"
432 "movq %%mm0, %%mm1 \n\t"
433 "movq %%mm3, %%mm4 \n\t"
434 "pand %%mm6, %%mm0 \n\t"
435 "pand %%mm6, %%mm3 \n\t"
436 "pmaddwd %%mm7, %%mm0 \n\t"
437 "pmaddwd %%mm7, %%mm3 \n\t"
438 "pand %%mm5, %%mm1 \n\t"
439 "pand %%mm5, %%mm4 \n\t"
440 "por %%mm1, %%mm0 \n\t"
441 "por %%mm4, %%mm3 \n\t"
442 "psrld $6, %%mm0 \n\t"
443 "pslld $10, %%mm3 \n\t"
444 "por %%mm3, %%mm0 \n\t"
452 :
"r" (mm_end),
"m" (mask3215g),
"m" (mask3216br),
"m" (mul3215)
454 __asm__ volatile(SFENCE:::
"memory");
455 __asm__ volatile(EMMS:::
"memory");
457 register int rgb = *(
const uint32_t*)
s;
s += 4;
458 *d++ = ((
rgb&0xFF)>>3) + ((
rgb&0xF800)>>6) + ((
rgb&0xF80000)>>9);
462 static inline void rgb32tobgr15_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
464 const uint8_t *
s =
src;
466 const uint8_t *mm_end;
467 uint16_t *d = (uint16_t *)
dst;
471 "movq %0, %%mm7 \n\t"
472 "movq %1, %%mm6 \n\t"
473 ::
"m"(red_15mask),
"m"(green_15mask));
478 "movd (%1), %%mm0 \n\t"
479 "movd 4(%1), %%mm3 \n\t"
480 "punpckldq 8(%1), %%mm0 \n\t"
481 "punpckldq 12(%1), %%mm3 \n\t"
482 "movq %%mm0, %%mm1 \n\t"
483 "movq %%mm0, %%mm2 \n\t"
484 "movq %%mm3, %%mm4 \n\t"
485 "movq %%mm3, %%mm5 \n\t"
486 "psllq $7, %%mm0 \n\t"
487 "psllq $7, %%mm3 \n\t"
488 "pand %%mm7, %%mm0 \n\t"
489 "pand %%mm7, %%mm3 \n\t"
490 "psrlq $6, %%mm1 \n\t"
491 "psrlq $6, %%mm4 \n\t"
492 "pand %%mm6, %%mm1 \n\t"
493 "pand %%mm6, %%mm4 \n\t"
494 "psrlq $19, %%mm2 \n\t"
495 "psrlq $19, %%mm5 \n\t"
496 "pand %2, %%mm2 \n\t"
497 "pand %2, %%mm5 \n\t"
498 "por %%mm1, %%mm0 \n\t"
499 "por %%mm4, %%mm3 \n\t"
500 "por %%mm2, %%mm0 \n\t"
501 "por %%mm5, %%mm3 \n\t"
502 "psllq $16, %%mm3 \n\t"
503 "por %%mm3, %%mm0 \n\t"
505 ::
"r"(d),
"r"(
s),
"m"(blue_15mask):
"memory");
509 __asm__ volatile(SFENCE:::
"memory");
510 __asm__ volatile(EMMS:::
"memory");
512 register int rgb = *(
const uint32_t*)
s;
s += 4;
513 *d++ = ((
rgb&0xF8)<<7) + ((
rgb&0xF800)>>6) + ((
rgb&0xF80000)>>19);
517 static inline void rgb24tobgr16_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
519 const uint8_t *
s =
src;
521 const uint8_t *mm_end;
522 uint16_t *d = (uint16_t *)
dst;
526 "movq %0, %%mm7 \n\t"
527 "movq %1, %%mm6 \n\t"
528 ::
"m"(red_16mask),
"m"(green_16mask));
533 "movd (%1), %%mm0 \n\t"
534 "movd 3(%1), %%mm3 \n\t"
535 "punpckldq 6(%1), %%mm0 \n\t"
536 "punpckldq 9(%1), %%mm3 \n\t"
537 "movq %%mm0, %%mm1 \n\t"
538 "movq %%mm0, %%mm2 \n\t"
539 "movq %%mm3, %%mm4 \n\t"
540 "movq %%mm3, %%mm5 \n\t"
541 "psrlq $3, %%mm0 \n\t"
542 "psrlq $3, %%mm3 \n\t"
543 "pand %2, %%mm0 \n\t"
544 "pand %2, %%mm3 \n\t"
545 "psrlq $5, %%mm1 \n\t"
546 "psrlq $5, %%mm4 \n\t"
547 "pand %%mm6, %%mm1 \n\t"
548 "pand %%mm6, %%mm4 \n\t"
549 "psrlq $8, %%mm2 \n\t"
550 "psrlq $8, %%mm5 \n\t"
551 "pand %%mm7, %%mm2 \n\t"
552 "pand %%mm7, %%mm5 \n\t"
553 "por %%mm1, %%mm0 \n\t"
554 "por %%mm4, %%mm3 \n\t"
555 "por %%mm2, %%mm0 \n\t"
556 "por %%mm5, %%mm3 \n\t"
557 "psllq $16, %%mm3 \n\t"
558 "por %%mm3, %%mm0 \n\t"
560 ::
"r"(d),
"r"(
s),
"m"(blue_16mask):
"memory");
564 __asm__ volatile(SFENCE:::
"memory");
565 __asm__ volatile(EMMS:::
"memory");
570 *d++ = (
b>>3) | ((
g&0xFC)<<3) | ((
r&0xF8)<<8);
574 static inline void rgb24to16_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
576 const uint8_t *
s =
src;
578 const uint8_t *mm_end;
579 uint16_t *d = (uint16_t *)
dst;
583 "movq %0, %%mm7 \n\t"
584 "movq %1, %%mm6 \n\t"
585 ::
"m"(red_16mask),
"m"(green_16mask));
590 "movd (%1), %%mm0 \n\t"
591 "movd 3(%1), %%mm3 \n\t"
592 "punpckldq 6(%1), %%mm0 \n\t"
593 "punpckldq 9(%1), %%mm3 \n\t"
594 "movq %%mm0, %%mm1 \n\t"
595 "movq %%mm0, %%mm2 \n\t"
596 "movq %%mm3, %%mm4 \n\t"
597 "movq %%mm3, %%mm5 \n\t"
598 "psllq $8, %%mm0 \n\t"
599 "psllq $8, %%mm3 \n\t"
600 "pand %%mm7, %%mm0 \n\t"
601 "pand %%mm7, %%mm3 \n\t"
602 "psrlq $5, %%mm1 \n\t"
603 "psrlq $5, %%mm4 \n\t"
604 "pand %%mm6, %%mm1 \n\t"
605 "pand %%mm6, %%mm4 \n\t"
606 "psrlq $19, %%mm2 \n\t"
607 "psrlq $19, %%mm5 \n\t"
608 "pand %2, %%mm2 \n\t"
609 "pand %2, %%mm5 \n\t"
610 "por %%mm1, %%mm0 \n\t"
611 "por %%mm4, %%mm3 \n\t"
612 "por %%mm2, %%mm0 \n\t"
613 "por %%mm5, %%mm3 \n\t"
614 "psllq $16, %%mm3 \n\t"
615 "por %%mm3, %%mm0 \n\t"
617 ::
"r"(d),
"r"(
s),
"m"(blue_16mask):
"memory");
621 __asm__ volatile(SFENCE:::
"memory");
622 __asm__ volatile(EMMS:::
"memory");
627 *d++ = (
b>>3) | ((
g&0xFC)<<3) | ((
r&0xF8)<<8);
631 static inline void rgb24tobgr15_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
633 const uint8_t *
s =
src;
635 const uint8_t *mm_end;
636 uint16_t *d = (uint16_t *)
dst;
640 "movq %0, %%mm7 \n\t"
641 "movq %1, %%mm6 \n\t"
642 ::
"m"(red_15mask),
"m"(green_15mask));
647 "movd (%1), %%mm0 \n\t"
648 "movd 3(%1), %%mm3 \n\t"
649 "punpckldq 6(%1), %%mm0 \n\t"
650 "punpckldq 9(%1), %%mm3 \n\t"
651 "movq %%mm0, %%mm1 \n\t"
652 "movq %%mm0, %%mm2 \n\t"
653 "movq %%mm3, %%mm4 \n\t"
654 "movq %%mm3, %%mm5 \n\t"
655 "psrlq $3, %%mm0 \n\t"
656 "psrlq $3, %%mm3 \n\t"
657 "pand %2, %%mm0 \n\t"
658 "pand %2, %%mm3 \n\t"
659 "psrlq $6, %%mm1 \n\t"
660 "psrlq $6, %%mm4 \n\t"
661 "pand %%mm6, %%mm1 \n\t"
662 "pand %%mm6, %%mm4 \n\t"
663 "psrlq $9, %%mm2 \n\t"
664 "psrlq $9, %%mm5 \n\t"
665 "pand %%mm7, %%mm2 \n\t"
666 "pand %%mm7, %%mm5 \n\t"
667 "por %%mm1, %%mm0 \n\t"
668 "por %%mm4, %%mm3 \n\t"
669 "por %%mm2, %%mm0 \n\t"
670 "por %%mm5, %%mm3 \n\t"
671 "psllq $16, %%mm3 \n\t"
672 "por %%mm3, %%mm0 \n\t"
674 ::
"r"(d),
"r"(
s),
"m"(blue_15mask):
"memory");
678 __asm__ volatile(SFENCE:::
"memory");
679 __asm__ volatile(EMMS:::
"memory");
684 *d++ = (
b>>3) | ((
g&0xF8)<<2) | ((
r&0xF8)<<7);
688 static inline void rgb24to15_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
690 const uint8_t *
s =
src;
692 const uint8_t *mm_end;
693 uint16_t *d = (uint16_t *)
dst;
697 "movq %0, %%mm7 \n\t"
698 "movq %1, %%mm6 \n\t"
699 ::
"m"(red_15mask),
"m"(green_15mask));
704 "movd (%1), %%mm0 \n\t"
705 "movd 3(%1), %%mm3 \n\t"
706 "punpckldq 6(%1), %%mm0 \n\t"
707 "punpckldq 9(%1), %%mm3 \n\t"
708 "movq %%mm0, %%mm1 \n\t"
709 "movq %%mm0, %%mm2 \n\t"
710 "movq %%mm3, %%mm4 \n\t"
711 "movq %%mm3, %%mm5 \n\t"
712 "psllq $7, %%mm0 \n\t"
713 "psllq $7, %%mm3 \n\t"
714 "pand %%mm7, %%mm0 \n\t"
715 "pand %%mm7, %%mm3 \n\t"
716 "psrlq $6, %%mm1 \n\t"
717 "psrlq $6, %%mm4 \n\t"
718 "pand %%mm6, %%mm1 \n\t"
719 "pand %%mm6, %%mm4 \n\t"
720 "psrlq $19, %%mm2 \n\t"
721 "psrlq $19, %%mm5 \n\t"
722 "pand %2, %%mm2 \n\t"
723 "pand %2, %%mm5 \n\t"
724 "por %%mm1, %%mm0 \n\t"
725 "por %%mm4, %%mm3 \n\t"
726 "por %%mm2, %%mm0 \n\t"
727 "por %%mm5, %%mm3 \n\t"
728 "psllq $16, %%mm3 \n\t"
729 "por %%mm3, %%mm0 \n\t"
731 ::
"r"(d),
"r"(
s),
"m"(blue_15mask):
"memory");
735 __asm__ volatile(SFENCE:::
"memory");
736 __asm__ volatile(EMMS:::
"memory");
741 *d++ = (
b>>3) | ((
g&0xF8)<<2) | ((
r&0xF8)<<7);
745 static inline void rgb15tobgr24_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
748 const uint16_t *mm_end;
750 const uint16_t *
s = (
const uint16_t*)
src;
751 end =
s + src_size/2;
757 "movq (%1), %%mm0 \n\t"
758 "movq (%1), %%mm1 \n\t"
759 "movq (%1), %%mm2 \n\t"
760 "pand %2, %%mm0 \n\t"
761 "pand %3, %%mm1 \n\t"
762 "pand %4, %%mm2 \n\t"
763 "psllq $5, %%mm0 \n\t"
764 "pmulhw "MANGLE(mul15_mid)
", %%mm0 \n\t"
765 "pmulhw "MANGLE(mul15_mid)
", %%mm1 \n\t"
766 "pmulhw "MANGLE(mul15_hi)
", %%mm2 \n\t"
767 "movq %%mm0, %%mm3 \n\t"
768 "movq %%mm1, %%mm4 \n\t"
769 "movq %%mm2, %%mm5 \n\t"
770 "punpcklwd %5, %%mm0 \n\t"
771 "punpcklwd %5, %%mm1 \n\t"
772 "punpcklwd %5, %%mm2 \n\t"
773 "punpckhwd %5, %%mm3 \n\t"
774 "punpckhwd %5, %%mm4 \n\t"
775 "punpckhwd %5, %%mm5 \n\t"
776 "psllq $8, %%mm1 \n\t"
777 "psllq $16, %%mm2 \n\t"
778 "por %%mm1, %%mm0 \n\t"
779 "por %%mm2, %%mm0 \n\t"
780 "psllq $8, %%mm4 \n\t"
781 "psllq $16, %%mm5 \n\t"
782 "por %%mm4, %%mm3 \n\t"
783 "por %%mm5, %%mm3 \n\t"
785 "movq %%mm0, %%mm6 \n\t"
786 "movq %%mm3, %%mm7 \n\t"
788 "movq 8(%1), %%mm0 \n\t"
789 "movq 8(%1), %%mm1 \n\t"
790 "movq 8(%1), %%mm2 \n\t"
791 "pand %2, %%mm0 \n\t"
792 "pand %3, %%mm1 \n\t"
793 "pand %4, %%mm2 \n\t"
794 "psllq $5, %%mm0 \n\t"
795 "pmulhw "MANGLE(mul15_mid)
", %%mm0 \n\t"
796 "pmulhw "MANGLE(mul15_mid)
", %%mm1 \n\t"
797 "pmulhw "MANGLE(mul15_hi)
", %%mm2 \n\t"
798 "movq %%mm0, %%mm3 \n\t"
799 "movq %%mm1, %%mm4 \n\t"
800 "movq %%mm2, %%mm5 \n\t"
801 "punpcklwd %5, %%mm0 \n\t"
802 "punpcklwd %5, %%mm1 \n\t"
803 "punpcklwd %5, %%mm2 \n\t"
804 "punpckhwd %5, %%mm3 \n\t"
805 "punpckhwd %5, %%mm4 \n\t"
806 "punpckhwd %5, %%mm5 \n\t"
807 "psllq $8, %%mm1 \n\t"
808 "psllq $16, %%mm2 \n\t"
809 "por %%mm1, %%mm0 \n\t"
810 "por %%mm2, %%mm0 \n\t"
811 "psllq $8, %%mm4 \n\t"
812 "psllq $16, %%mm5 \n\t"
813 "por %%mm4, %%mm3 \n\t"
814 "por %%mm5, %%mm3 \n\t"
817 :
"r"(
s),
"m"(mask15b),
"m"(mask15g),
"m"(mask15r),
"m"(mmx_null)
822 "movq %%mm0, %%mm4 \n\t"
823 "movq %%mm3, %%mm5 \n\t"
824 "movq %%mm6, %%mm0 \n\t"
825 "movq %%mm7, %%mm1 \n\t"
827 "movq %%mm4, %%mm6 \n\t"
828 "movq %%mm5, %%mm7 \n\t"
829 "movq %%mm0, %%mm2 \n\t"
830 "movq %%mm1, %%mm3 \n\t"
840 __asm__ volatile(SFENCE:::
"memory");
841 __asm__ volatile(EMMS:::
"memory");
843 register uint16_t bgr;
845 *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
846 *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);
847 *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);
851 static inline void rgb16tobgr24_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
854 const uint16_t *mm_end;
855 uint8_t *d = (uint8_t *)
dst;
856 const uint16_t *
s = (
const uint16_t *)
src;
857 end =
s + src_size/2;
863 "movq (%1), %%mm0 \n\t"
864 "movq (%1), %%mm1 \n\t"
865 "movq (%1), %%mm2 \n\t"
866 "pand %2, %%mm0 \n\t"
867 "pand %3, %%mm1 \n\t"
868 "pand %4, %%mm2 \n\t"
869 "psllq $5, %%mm0 \n\t"
870 "psrlq $1, %%mm2 \n\t"
871 "pmulhw "MANGLE(mul15_mid)
", %%mm0 \n\t"
872 "pmulhw "MANGLE(mul16_mid)
", %%mm1 \n\t"
873 "pmulhw "MANGLE(mul15_hi)
", %%mm2 \n\t"
874 "movq %%mm0, %%mm3 \n\t"
875 "movq %%mm1, %%mm4 \n\t"
876 "movq %%mm2, %%mm5 \n\t"
877 "punpcklwd %5, %%mm0 \n\t"
878 "punpcklwd %5, %%mm1 \n\t"
879 "punpcklwd %5, %%mm2 \n\t"
880 "punpckhwd %5, %%mm3 \n\t"
881 "punpckhwd %5, %%mm4 \n\t"
882 "punpckhwd %5, %%mm5 \n\t"
883 "psllq $8, %%mm1 \n\t"
884 "psllq $16, %%mm2 \n\t"
885 "por %%mm1, %%mm0 \n\t"
886 "por %%mm2, %%mm0 \n\t"
887 "psllq $8, %%mm4 \n\t"
888 "psllq $16, %%mm5 \n\t"
889 "por %%mm4, %%mm3 \n\t"
890 "por %%mm5, %%mm3 \n\t"
892 "movq %%mm0, %%mm6 \n\t"
893 "movq %%mm3, %%mm7 \n\t"
895 "movq 8(%1), %%mm0 \n\t"
896 "movq 8(%1), %%mm1 \n\t"
897 "movq 8(%1), %%mm2 \n\t"
898 "pand %2, %%mm0 \n\t"
899 "pand %3, %%mm1 \n\t"
900 "pand %4, %%mm2 \n\t"
901 "psllq $5, %%mm0 \n\t"
902 "psrlq $1, %%mm2 \n\t"
903 "pmulhw "MANGLE(mul15_mid)
", %%mm0 \n\t"
904 "pmulhw "MANGLE(mul16_mid)
", %%mm1 \n\t"
905 "pmulhw "MANGLE(mul15_hi)
", %%mm2 \n\t"
906 "movq %%mm0, %%mm3 \n\t"
907 "movq %%mm1, %%mm4 \n\t"
908 "movq %%mm2, %%mm5 \n\t"
909 "punpcklwd %5, %%mm0 \n\t"
910 "punpcklwd %5, %%mm1 \n\t"
911 "punpcklwd %5, %%mm2 \n\t"
912 "punpckhwd %5, %%mm3 \n\t"
913 "punpckhwd %5, %%mm4 \n\t"
914 "punpckhwd %5, %%mm5 \n\t"
915 "psllq $8, %%mm1 \n\t"
916 "psllq $16, %%mm2 \n\t"
917 "por %%mm1, %%mm0 \n\t"
918 "por %%mm2, %%mm0 \n\t"
919 "psllq $8, %%mm4 \n\t"
920 "psllq $16, %%mm5 \n\t"
921 "por %%mm4, %%mm3 \n\t"
922 "por %%mm5, %%mm3 \n\t"
924 :
"r"(
s),
"m"(mask16b),
"m"(mask16g),
"m"(mask16r),
"m"(mmx_null)
929 "movq %%mm0, %%mm4 \n\t"
930 "movq %%mm3, %%mm5 \n\t"
931 "movq %%mm6, %%mm0 \n\t"
932 "movq %%mm7, %%mm1 \n\t"
934 "movq %%mm4, %%mm6 \n\t"
935 "movq %%mm5, %%mm7 \n\t"
936 "movq %%mm0, %%mm2 \n\t"
937 "movq %%mm1, %%mm3 \n\t"
947 __asm__ volatile(SFENCE:::
"memory");
948 __asm__ volatile(EMMS:::
"memory");
950 register uint16_t bgr;
952 *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
953 *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);
954 *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);
966 "packuswb %%mm7, %%mm0 \n\t" \
967 "packuswb %%mm7, %%mm1 \n\t" \
968 "packuswb %%mm7, %%mm2 \n\t" \
969 "punpcklbw %%mm1, %%mm0 \n\t" \
970 "punpcklbw %%mm6, %%mm2 \n\t" \
971 "movq %%mm0, %%mm3 \n\t" \
972 "punpcklwd %%mm2, %%mm0 \n\t" \
973 "punpckhwd %%mm2, %%mm3 \n\t" \
974 MOVNTQ" %%mm0, (%0) \n\t" \
975 MOVNTQ" %%mm3, 8(%0) \n\t" \
977 static inline void rgb15to32_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
980 const uint16_t *mm_end;
982 const uint16_t *
s = (
const uint16_t *)
src;
983 end =
s + src_size/2;
985 __asm__ volatile(
"pxor %%mm7,%%mm7 \n\t":::
"memory");
986 __asm__ volatile(
"pcmpeqd %%mm6,%%mm6 \n\t":::
"memory");
991 "movq (%1), %%mm0 \n\t"
992 "movq (%1), %%mm1 \n\t"
993 "movq (%1), %%mm2 \n\t"
994 "pand %2, %%mm0 \n\t"
995 "pand %3, %%mm1 \n\t"
996 "pand %4, %%mm2 \n\t"
997 "psllq $5, %%mm0 \n\t"
998 "pmulhw %5, %%mm0 \n\t"
999 "pmulhw %5, %%mm1 \n\t"
1000 "pmulhw "MANGLE(mul15_hi)
", %%mm2 \n\t"
1002 ::
"r"(d),
"r"(
s),
"m"(mask15b),
"m"(mask15g),
"m"(mask15r) ,
"m"(mul15_mid)
1008 __asm__ volatile(SFENCE:::
"memory");
1009 __asm__ volatile(EMMS:::
"memory");
1011 register uint16_t bgr;
1013 *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
1014 *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);
1015 *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);
1020 static inline void rgb16to32_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
1022 const uint16_t *end;
1023 const uint16_t *mm_end;
1025 const uint16_t *
s = (
const uint16_t*)
src;
1026 end =
s + src_size/2;
1028 __asm__ volatile(
"pxor %%mm7,%%mm7 \n\t":::
"memory");
1029 __asm__ volatile(
"pcmpeqd %%mm6,%%mm6 \n\t":::
"memory");
1031 while (
s < mm_end) {
1034 "movq (%1), %%mm0 \n\t"
1035 "movq (%1), %%mm1 \n\t"
1036 "movq (%1), %%mm2 \n\t"
1037 "pand %2, %%mm0 \n\t"
1038 "pand %3, %%mm1 \n\t"
1039 "pand %4, %%mm2 \n\t"
1040 "psllq $5, %%mm0 \n\t"
1041 "psrlq $1, %%mm2 \n\t"
1042 "pmulhw %5, %%mm0 \n\t"
1043 "pmulhw "MANGLE(mul16_mid)
", %%mm1 \n\t"
1044 "pmulhw "MANGLE(mul15_hi)
", %%mm2 \n\t"
1046 ::
"r"(d),
"r"(
s),
"m"(mask16b),
"m"(mask16g),
"m"(mask16r),
"m"(mul15_mid)
1052 __asm__ volatile(SFENCE:::
"memory");
1053 __asm__ volatile(EMMS:::
"memory");
1055 register uint16_t bgr;
1057 *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
1058 *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);
1059 *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);
1064 static inline void rgb24tobgr24_mmxext(
const uint8_t *
src, uint8_t *
dst,
int src_size)
1066 x86_reg mmx_size= 23 - src_size;
1068 "test %%"FF_REG_a
", %%"FF_REG_a
" \n\t"
1070 "movq "MANGLE(mask24r)
", %%mm5 \n\t"
1071 "movq "MANGLE(mask24g)
", %%mm6 \n\t"
1072 "movq "MANGLE(mask24b)
", %%mm7 \n\t"
1075 PREFETCH" 32(%1, %%"FF_REG_a
") \n\t"
1076 "movq (%1, %%"FF_REG_a
"), %%mm0 \n\t"
1077 "movq (%1, %%"FF_REG_a
"), %%mm1 \n\t"
1078 "movq 2(%1, %%"FF_REG_a
"), %%mm2 \n\t"
1079 "psllq $16, %%mm0 \n\t"
1080 "pand %%mm5, %%mm0 \n\t"
1081 "pand %%mm6, %%mm1 \n\t"
1082 "pand %%mm7, %%mm2 \n\t"
1083 "por %%mm0, %%mm1 \n\t"
1084 "por %%mm2, %%mm1 \n\t"
1085 "movq 6(%1, %%"FF_REG_a
"), %%mm0 \n\t"
1086 MOVNTQ" %%mm1,(%2, %%"FF_REG_a
") \n\t"
1087 "movq 8(%1, %%"FF_REG_a
"), %%mm1 \n\t"
1088 "movq 10(%1, %%"FF_REG_a
"), %%mm2 \n\t"
1089 "pand %%mm7, %%mm0 \n\t"
1090 "pand %%mm5, %%mm1 \n\t"
1091 "pand %%mm6, %%mm2 \n\t"
1092 "por %%mm0, %%mm1 \n\t"
1093 "por %%mm2, %%mm1 \n\t"
1094 "movq 14(%1, %%"FF_REG_a
"), %%mm0 \n\t"
1095 MOVNTQ" %%mm1, 8(%2, %%"FF_REG_a
")\n\t"
1096 "movq 16(%1, %%"FF_REG_a
"), %%mm1 \n\t"
1097 "movq 18(%1, %%"FF_REG_a
"), %%mm2 \n\t"
1098 "pand %%mm6, %%mm0 \n\t"
1099 "pand %%mm7, %%mm1 \n\t"
1100 "pand %%mm5, %%mm2 \n\t"
1101 "por %%mm0, %%mm1 \n\t"
1102 "por %%mm2, %%mm1 \n\t"
1103 MOVNTQ" %%mm1, 16(%2, %%"FF_REG_a
") \n\t"
1104 "add $24, %%"FF_REG_a
" \n\t"
1108 :
"r" (
src-mmx_size),
"r"(
dst-mmx_size)
1112 __asm__ volatile(SFENCE:::
"memory");
1113 __asm__ volatile(EMMS:::
"memory");
1115 if (mmx_size==23)
return;
1119 src_size= 23-mmx_size;
1122 for (
unsigned i = 0;
i < src_size;
i +=3) {
1131 static inline void yuvPlanartoyuy2_mmxext(
const uint8_t *ysrc,
const uint8_t *usrc,
const uint8_t *vsrc, uint8_t *
dst,
1133 int lumStride,
int chromStride,
int dstStride,
int vertLumPerChroma)
1136 for (
int y = 0; y <
height; y++) {
1139 "xor %%"FF_REG_a
", %%"FF_REG_a
" \n\t"
1142 PREFETCH" 32(%1, %%"FF_REG_a
", 2) \n\t"
1143 PREFETCH" 32(%2, %%"FF_REG_a
") \n\t"
1144 PREFETCH" 32(%3, %%"FF_REG_a
") \n\t"
1145 "movq (%2, %%"FF_REG_a
"), %%mm0 \n\t"
1146 "movq %%mm0, %%mm2 \n\t"
1147 "movq (%3, %%"FF_REG_a
"), %%mm1 \n\t"
1148 "punpcklbw %%mm1, %%mm0 \n\t"
1149 "punpckhbw %%mm1, %%mm2 \n\t"
1151 "movq (%1, %%"FF_REG_a
",2), %%mm3 \n\t"
1152 "movq 8(%1, %%"FF_REG_a
",2), %%mm5 \n\t"
1153 "movq %%mm3, %%mm4 \n\t"
1154 "movq %%mm5, %%mm6 \n\t"
1155 "punpcklbw %%mm0, %%mm3 \n\t"
1156 "punpckhbw %%mm0, %%mm4 \n\t"
1157 "punpcklbw %%mm2, %%mm5 \n\t"
1158 "punpckhbw %%mm2, %%mm6 \n\t"
1160 MOVNTQ" %%mm3, (%0, %%"FF_REG_a
", 4) \n\t"
1161 MOVNTQ" %%mm4, 8(%0, %%"FF_REG_a
", 4) \n\t"
1162 MOVNTQ" %%mm5, 16(%0, %%"FF_REG_a
", 4) \n\t"
1163 MOVNTQ" %%mm6, 24(%0, %%"FF_REG_a
", 4) \n\t"
1165 "add $8, %%"FF_REG_a
" \n\t"
1166 "cmp %4, %%"FF_REG_a
" \n\t"
1168 ::
"r"(
dst),
"r"(ysrc),
"r"(usrc),
"r"(vsrc),
"g" (chromWidth)
1171 if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
1172 usrc += chromStride;
1173 vsrc += chromStride;
1187 static inline void yv12toyuy2_mmxext(
const uint8_t *ysrc,
const uint8_t *usrc,
const uint8_t *vsrc, uint8_t *
dst,
1189 int lumStride,
int chromStride,
int dstStride)
1192 yuvPlanartoyuy2_mmxext(ysrc, usrc, vsrc,
dst,
width,
height, lumStride, chromStride, dstStride, 2);
1195 static inline void yuvPlanartouyvy_mmxext(
const uint8_t *ysrc,
const uint8_t *usrc,
const uint8_t *vsrc, uint8_t *
dst,
1197 int lumStride,
int chromStride,
int dstStride,
int vertLumPerChroma)
1200 for (
int y = 0; y <
height; y++) {
1203 "xor %%"FF_REG_a
", %%"FF_REG_a
" \n\t"
1206 PREFETCH" 32(%1, %%"FF_REG_a
", 2) \n\t"
1207 PREFETCH" 32(%2, %%"FF_REG_a
") \n\t"
1208 PREFETCH" 32(%3, %%"FF_REG_a
") \n\t"
1209 "movq (%2, %%"FF_REG_a
"), %%mm0 \n\t"
1210 "movq %%mm0, %%mm2 \n\t"
1211 "movq (%3, %%"FF_REG_a
"), %%mm1 \n\t"
1212 "punpcklbw %%mm1, %%mm0 \n\t"
1213 "punpckhbw %%mm1, %%mm2 \n\t"
1215 "movq (%1, %%"FF_REG_a
",2), %%mm3 \n\t"
1216 "movq 8(%1, %%"FF_REG_a
",2), %%mm5 \n\t"
1217 "movq %%mm0, %%mm4 \n\t"
1218 "movq %%mm2, %%mm6 \n\t"
1219 "punpcklbw %%mm3, %%mm0 \n\t"
1220 "punpckhbw %%mm3, %%mm4 \n\t"
1221 "punpcklbw %%mm5, %%mm2 \n\t"
1222 "punpckhbw %%mm5, %%mm6 \n\t"
1224 MOVNTQ" %%mm0, (%0, %%"FF_REG_a
", 4) \n\t"
1225 MOVNTQ" %%mm4, 8(%0, %%"FF_REG_a
", 4) \n\t"
1226 MOVNTQ" %%mm2, 16(%0, %%"FF_REG_a
", 4) \n\t"
1227 MOVNTQ" %%mm6, 24(%0, %%"FF_REG_a
", 4) \n\t"
1229 "add $8, %%"FF_REG_a
" \n\t"
1230 "cmp %4, %%"FF_REG_a
" \n\t"
1232 ::
"r"(
dst),
"r"(ysrc),
"r"(usrc),
"r"(vsrc),
"g" (chromWidth)
1235 if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
1236 usrc += chromStride;
1237 vsrc += chromStride;
1251 static inline void yv12touyvy_mmxext(
const uint8_t *ysrc,
const uint8_t *usrc,
const uint8_t *vsrc, uint8_t *
dst,
1253 int lumStride,
int chromStride,
int dstStride)
1256 yuvPlanartouyvy_mmxext(ysrc, usrc, vsrc,
dst,
width,
height, lumStride, chromStride, dstStride, 2);
1262 static inline void yuv422ptouyvy_mmxext(
const uint8_t *ysrc,
const uint8_t *usrc,
const uint8_t *vsrc, uint8_t *
dst,
1264 int lumStride,
int chromStride,
int dstStride)
1266 yuvPlanartouyvy_mmxext(ysrc, usrc, vsrc,
dst,
width,
height, lumStride, chromStride, dstStride, 1);
1272 static inline void yuv422ptoyuy2_mmxext(
const uint8_t *ysrc,
const uint8_t *usrc,
const uint8_t *vsrc, uint8_t *
dst,
1274 int lumStride,
int chromStride,
int dstStride)
1276 yuvPlanartoyuy2_mmxext(ysrc, usrc, vsrc,
dst,
width,
height, lumStride, chromStride, dstStride, 1);
1283 static inline void yuy2toyv12_mmxext(
const uint8_t *
src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1285 int lumStride,
int chromStride,
int srcStride)
1288 for (
int y = 0; y <
height; y += 2) {
1290 "xor %%"FF_REG_a
", %%"FF_REG_a
"\n\t"
1291 "pcmpeqw %%mm7, %%mm7 \n\t"
1292 "psrlw $8, %%mm7 \n\t"
1295 PREFETCH" 64(%0, %%"FF_REG_a
", 4) \n\t"
1296 "movq (%0, %%"FF_REG_a
", 4), %%mm0 \n\t"
1297 "movq 8(%0, %%"FF_REG_a
", 4), %%mm1 \n\t"
1298 "movq %%mm0, %%mm2 \n\t"
1299 "movq %%mm1, %%mm3 \n\t"
1300 "psrlw $8, %%mm0 \n\t"
1301 "psrlw $8, %%mm1 \n\t"
1302 "pand %%mm7, %%mm2 \n\t"
1303 "pand %%mm7, %%mm3 \n\t"
1304 "packuswb %%mm1, %%mm0 \n\t"
1305 "packuswb %%mm3, %%mm2 \n\t"
1307 MOVNTQ" %%mm2, (%1, %%"FF_REG_a
", 2) \n\t"
1309 "movq 16(%0, %%"FF_REG_a
", 4), %%mm1 \n\t"
1310 "movq 24(%0, %%"FF_REG_a
", 4), %%mm2 \n\t"
1311 "movq %%mm1, %%mm3 \n\t"
1312 "movq %%mm2, %%mm4 \n\t"
1313 "psrlw $8, %%mm1 \n\t"
1314 "psrlw $8, %%mm2 \n\t"
1315 "pand %%mm7, %%mm3 \n\t"
1316 "pand %%mm7, %%mm4 \n\t"
1317 "packuswb %%mm2, %%mm1 \n\t"
1318 "packuswb %%mm4, %%mm3 \n\t"
1320 MOVNTQ" %%mm3, 8(%1, %%"FF_REG_a
", 2) \n\t"
1322 "movq %%mm0, %%mm2 \n\t"
1323 "movq %%mm1, %%mm3 \n\t"
1324 "psrlw $8, %%mm0 \n\t"
1325 "psrlw $8, %%mm1 \n\t"
1326 "pand %%mm7, %%mm2 \n\t"
1327 "pand %%mm7, %%mm3 \n\t"
1328 "packuswb %%mm1, %%mm0 \n\t"
1329 "packuswb %%mm3, %%mm2 \n\t"
1331 MOVNTQ" %%mm0, (%3, %%"FF_REG_a
") \n\t"
1332 MOVNTQ" %%mm2, (%2, %%"FF_REG_a
") \n\t"
1334 "add $8, %%"FF_REG_a
" \n\t"
1335 "cmp %4, %%"FF_REG_a
" \n\t"
1337 ::
"r"(
src),
"r"(ydst),
"r"(udst),
"r"(vdst),
"g" (chromWidth)
1338 :
"memory",
"%"FF_REG_a
1345 "xor %%"FF_REG_a
", %%"FF_REG_a
"\n\t"
1348 PREFETCH" 64(%0, %%"FF_REG_a
", 4) \n\t"
1349 "movq (%0, %%"FF_REG_a
", 4), %%mm0 \n\t"
1350 "movq 8(%0, %%"FF_REG_a
", 4), %%mm1 \n\t"
1351 "movq 16(%0, %%"FF_REG_a
", 4), %%mm2 \n\t"
1352 "movq 24(%0, %%"FF_REG_a
", 4), %%mm3 \n\t"
1353 "pand %%mm7, %%mm0 \n\t"
1354 "pand %%mm7, %%mm1 \n\t"
1355 "pand %%mm7, %%mm2 \n\t"
1356 "pand %%mm7, %%mm3 \n\t"
1357 "packuswb %%mm1, %%mm0 \n\t"
1358 "packuswb %%mm3, %%mm2 \n\t"
1360 MOVNTQ" %%mm0, (%1, %%"FF_REG_a
", 2) \n\t"
1361 MOVNTQ" %%mm2, 8(%1, %%"FF_REG_a
", 2) \n\t"
1363 "add $8, %%"FF_REG_a
"\n\t"
1364 "cmp %4, %%"FF_REG_a
"\n\t"
1367 ::
"r"(
src),
"r"(ydst),
"r"(udst),
"r"(vdst),
"g" (chromWidth)
1368 :
"memory",
"%"FF_REG_a
1370 udst += chromStride;
1371 vdst += chromStride;
1380 static inline void planar2x_mmxext(
const uint8_t *
src, uint8_t *
dst,
int srcWidth,
int srcHeight,
int srcStride,
int dstStride)
1385 for (
int x = 0; x < srcWidth - 1; x++) {
1389 dst[2*srcWidth-1]=
src[srcWidth-1];
1393 for (
int y = 1; y < srcHeight; y++) {
1394 x86_reg mmxSize= srcWidth&~15;
1398 "mov %4, %%"FF_REG_a
" \n\t"
1399 "movq "MANGLE(mmx_ff)
", %%mm0 \n\t"
1400 "movq (%0, %%"FF_REG_a
"), %%mm4 \n\t"
1401 "movq %%mm4, %%mm2 \n\t"
1402 "psllq $8, %%mm4 \n\t"
1403 "pand %%mm0, %%mm2 \n\t"
1404 "por %%mm2, %%mm4 \n\t"
1405 "movq (%1, %%"FF_REG_a
"), %%mm5 \n\t"
1406 "movq %%mm5, %%mm3 \n\t"
1407 "psllq $8, %%mm5 \n\t"
1408 "pand %%mm0, %%mm3 \n\t"
1409 "por %%mm3, %%mm5 \n\t"
1411 "movq (%0, %%"FF_REG_a
"), %%mm0 \n\t"
1412 "movq (%1, %%"FF_REG_a
"), %%mm1 \n\t"
1413 "movq 1(%0, %%"FF_REG_a
"), %%mm2 \n\t"
1414 "movq 1(%1, %%"FF_REG_a
"), %%mm3 \n\t"
1415 PAVGB" %%mm0, %%mm5 \n\t"
1416 PAVGB" %%mm0, %%mm3 \n\t"
1417 PAVGB" %%mm0, %%mm5 \n\t"
1418 PAVGB" %%mm0, %%mm3 \n\t"
1419 PAVGB" %%mm1, %%mm4 \n\t"
1420 PAVGB" %%mm1, %%mm2 \n\t"
1421 PAVGB" %%mm1, %%mm4 \n\t"
1422 PAVGB" %%mm1, %%mm2 \n\t"
1423 "movq %%mm5, %%mm7 \n\t"
1424 "movq %%mm4, %%mm6 \n\t"
1425 "punpcklbw %%mm3, %%mm5 \n\t"
1426 "punpckhbw %%mm3, %%mm7 \n\t"
1427 "punpcklbw %%mm2, %%mm4 \n\t"
1428 "punpckhbw %%mm2, %%mm6 \n\t"
1429 MOVNTQ" %%mm5, (%2, %%"FF_REG_a
", 2) \n\t"
1430 MOVNTQ" %%mm7, 8(%2, %%"FF_REG_a
", 2) \n\t"
1431 MOVNTQ" %%mm4, (%3, %%"FF_REG_a
", 2) \n\t"
1432 MOVNTQ" %%mm6, 8(%3, %%"FF_REG_a
", 2) \n\t"
1433 "add $8, %%"FF_REG_a
" \n\t"
1434 "movq -1(%0, %%"FF_REG_a
"), %%mm4 \n\t"
1435 "movq -1(%1, %%"FF_REG_a
"), %%mm5 \n\t"
1437 ::
"r" (
src + mmxSize ),
"r" (
src + srcStride + mmxSize ),
1438 "r" (
dst + mmxSize*2),
"r" (
dst + dstStride + mmxSize*2),
1445 dst[0] = (
src[0] * 3 +
src[srcStride]) >> 2;
1446 dst[dstStride] = (
src[0] + 3 *
src[srcStride]) >> 2;
1449 for (
int x = mmxSize - 1; x < srcWidth - 1; x++) {
1450 dst[2*x +1]= (3*
src[x+0] +
src[x+srcStride+1])>>2;
1451 dst[2*x+dstStride+2]= (
src[x+0] + 3*
src[x+srcStride+1])>>2;
1452 dst[2*x+dstStride+1]= (
src[x+1] + 3*
src[x+srcStride ])>>2;
1453 dst[2*x +2]= (3*
src[x+1] +
src[x+srcStride ])>>2;
1455 dst[srcWidth*2 -1 ]= (3*
src[srcWidth-1] +
src[srcWidth-1 + srcStride])>>2;
1456 dst[srcWidth*2 -1 + dstStride]= (
src[srcWidth-1] + 3*
src[srcWidth-1 + srcStride])>>2;
1465 for (
int x = 0; x < srcWidth - 1; x++) {
1469 dst[2*srcWidth-1]=
src[srcWidth-1];
1483 #if ARCH_X86_32 && HAVE_7REGS
1484 static inline void rgb24toyv12_mmxext(
const uint8_t *
src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1486 int lumStride,
int chromStride,
int srcStride,
1489 #define BGR2Y_IDX "16*4+16*32"
1490 #define BGR2U_IDX "16*4+16*33"
1491 #define BGR2V_IDX "16*4+16*34"
1498 ydst += 2*lumStride;
1499 udst += chromStride;
1500 vdst += chromStride;
1504 for (y = 0; y <
height - 2; y += 2) {
1505 for (
int i = 0;
i < 2;
i++) {
1507 "mov %2, %%"FF_REG_a
"\n\t"
1508 "movq "BGR2Y_IDX
"(%3), %%mm6 \n\t"
1509 "movq "MANGLE(ff_w1111)
", %%mm5 \n\t"
1510 "pxor %%mm7, %%mm7 \n\t"
1511 "lea (%%"FF_REG_a
", %%"FF_REG_a
", 2), %%"FF_REG_d
" \n\t"
1514 PREFETCH" 64(%0, %%"FF_REG_d
") \n\t"
1515 "movd (%0, %%"FF_REG_d
"), %%mm0 \n\t"
1516 "movd 3(%0, %%"FF_REG_d
"), %%mm1 \n\t"
1517 "punpcklbw %%mm7, %%mm0 \n\t"
1518 "punpcklbw %%mm7, %%mm1 \n\t"
1519 "movd 6(%0, %%"FF_REG_d
"), %%mm2 \n\t"
1520 "movd 9(%0, %%"FF_REG_d
"), %%mm3 \n\t"
1521 "punpcklbw %%mm7, %%mm2 \n\t"
1522 "punpcklbw %%mm7, %%mm3 \n\t"
1523 "pmaddwd %%mm6, %%mm0 \n\t"
1524 "pmaddwd %%mm6, %%mm1 \n\t"
1525 "pmaddwd %%mm6, %%mm2 \n\t"
1526 "pmaddwd %%mm6, %%mm3 \n\t"
1527 "psrad $8, %%mm0 \n\t"
1528 "psrad $8, %%mm1 \n\t"
1529 "psrad $8, %%mm2 \n\t"
1530 "psrad $8, %%mm3 \n\t"
1531 "packssdw %%mm1, %%mm0 \n\t"
1532 "packssdw %%mm3, %%mm2 \n\t"
1533 "pmaddwd %%mm5, %%mm0 \n\t"
1534 "pmaddwd %%mm5, %%mm2 \n\t"
1535 "packssdw %%mm2, %%mm0 \n\t"
1536 "psraw $7, %%mm0 \n\t"
1538 "movd 12(%0, %%"FF_REG_d
"), %%mm4 \n\t"
1539 "movd 15(%0, %%"FF_REG_d
"), %%mm1 \n\t"
1540 "punpcklbw %%mm7, %%mm4 \n\t"
1541 "punpcklbw %%mm7, %%mm1 \n\t"
1542 "movd 18(%0, %%"FF_REG_d
"), %%mm2 \n\t"
1543 "movd 21(%0, %%"FF_REG_d
"), %%mm3 \n\t"
1544 "punpcklbw %%mm7, %%mm2 \n\t"
1545 "punpcklbw %%mm7, %%mm3 \n\t"
1546 "pmaddwd %%mm6, %%mm4 \n\t"
1547 "pmaddwd %%mm6, %%mm1 \n\t"
1548 "pmaddwd %%mm6, %%mm2 \n\t"
1549 "pmaddwd %%mm6, %%mm3 \n\t"
1550 "psrad $8, %%mm4 \n\t"
1551 "psrad $8, %%mm1 \n\t"
1552 "psrad $8, %%mm2 \n\t"
1553 "psrad $8, %%mm3 \n\t"
1554 "packssdw %%mm1, %%mm4 \n\t"
1555 "packssdw %%mm3, %%mm2 \n\t"
1556 "pmaddwd %%mm5, %%mm4 \n\t"
1557 "pmaddwd %%mm5, %%mm2 \n\t"
1558 "add $24, %%"FF_REG_d
"\n\t"
1559 "packssdw %%mm2, %%mm4 \n\t"
1560 "psraw $7, %%mm4 \n\t"
1562 "packuswb %%mm4, %%mm0 \n\t"
1563 "paddusb "MANGLE(ff_bgr2YOffset)
", %%mm0 \n\t"
1565 MOVNTQ" %%mm0, (%1, %%"FF_REG_a
") \n\t"
1566 "add $8, %%"FF_REG_a
" \n\t"
1570 :
"%"FF_REG_a,
"%"FF_REG_d
1577 "mov %4, %%"FF_REG_a
"\n\t"
1578 "movq "MANGLE(ff_w1111)
", %%mm5 \n\t"
1579 "movq "BGR2U_IDX
"(%5), %%mm6 \n\t"
1580 "pxor %%mm7, %%mm7 \n\t"
1581 "lea (%%"FF_REG_a
", %%"FF_REG_a
", 2), %%"FF_REG_d
" \n\t"
1582 "add %%"FF_REG_d
", %%"FF_REG_d
"\n\t"
1585 PREFETCH" 64(%0, %%"FF_REG_d
") \n\t"
1586 PREFETCH" 64(%1, %%"FF_REG_d
") \n\t"
1587 "movq (%0, %%"FF_REG_d
"), %%mm0 \n\t"
1588 "movq (%1, %%"FF_REG_d
"), %%mm1 \n\t"
1589 "movq 6(%0, %%"FF_REG_d
"), %%mm2 \n\t"
1590 "movq 6(%1, %%"FF_REG_d
"), %%mm3 \n\t"
1591 PAVGB" %%mm1, %%mm0 \n\t"
1592 PAVGB" %%mm3, %%mm2 \n\t"
1593 "movq %%mm0, %%mm1 \n\t"
1594 "movq %%mm2, %%mm3 \n\t"
1595 "psrlq $24, %%mm0 \n\t"
1596 "psrlq $24, %%mm2 \n\t"
1597 PAVGB" %%mm1, %%mm0 \n\t"
1598 PAVGB" %%mm3, %%mm2 \n\t"
1599 "punpcklbw %%mm7, %%mm0 \n\t"
1600 "punpcklbw %%mm7, %%mm2 \n\t"
1601 "movq "BGR2V_IDX
"(%5), %%mm1 \n\t"
1602 "movq "BGR2V_IDX
"(%5), %%mm3 \n\t"
1604 "pmaddwd %%mm0, %%mm1 \n\t"
1605 "pmaddwd %%mm2, %%mm3 \n\t"
1606 "pmaddwd %%mm6, %%mm0 \n\t"
1607 "pmaddwd %%mm6, %%mm2 \n\t"
1608 "psrad $8, %%mm0 \n\t"
1609 "psrad $8, %%mm1 \n\t"
1610 "psrad $8, %%mm2 \n\t"
1611 "psrad $8, %%mm3 \n\t"
1612 "packssdw %%mm2, %%mm0 \n\t"
1613 "packssdw %%mm3, %%mm1 \n\t"
1614 "pmaddwd %%mm5, %%mm0 \n\t"
1615 "pmaddwd %%mm5, %%mm1 \n\t"
1616 "packssdw %%mm1, %%mm0 \n\t"
1617 "psraw $7, %%mm0 \n\t"
1619 "movq 12(%0, %%"FF_REG_d
"), %%mm4 \n\t"
1620 "movq 12(%1, %%"FF_REG_d
"), %%mm1 \n\t"
1621 "movq 18(%0, %%"FF_REG_d
"), %%mm2 \n\t"
1622 "movq 18(%1, %%"FF_REG_d
"), %%mm3 \n\t"
1623 PAVGB" %%mm1, %%mm4 \n\t"
1624 PAVGB" %%mm3, %%mm2 \n\t"
1625 "movq %%mm4, %%mm1 \n\t"
1626 "movq %%mm2, %%mm3 \n\t"
1627 "psrlq $24, %%mm4 \n\t"
1628 "psrlq $24, %%mm2 \n\t"
1629 PAVGB" %%mm1, %%mm4 \n\t"
1630 PAVGB" %%mm3, %%mm2 \n\t"
1631 "punpcklbw %%mm7, %%mm4 \n\t"
1632 "punpcklbw %%mm7, %%mm2 \n\t"
1633 "movq "BGR2V_IDX
"(%5), %%mm1 \n\t"
1634 "movq "BGR2V_IDX
"(%5), %%mm3 \n\t"
1636 "pmaddwd %%mm4, %%mm1 \n\t"
1637 "pmaddwd %%mm2, %%mm3 \n\t"
1638 "pmaddwd %%mm6, %%mm4 \n\t"
1639 "pmaddwd %%mm6, %%mm2 \n\t"
1640 "psrad $8, %%mm4 \n\t"
1641 "psrad $8, %%mm1 \n\t"
1642 "psrad $8, %%mm2 \n\t"
1643 "psrad $8, %%mm3 \n\t"
1644 "packssdw %%mm2, %%mm4 \n\t"
1645 "packssdw %%mm3, %%mm1 \n\t"
1646 "pmaddwd %%mm5, %%mm4 \n\t"
1647 "pmaddwd %%mm5, %%mm1 \n\t"
1648 "add $24, %%"FF_REG_d
"\n\t"
1649 "packssdw %%mm1, %%mm4 \n\t"
1650 "psraw $7, %%mm4 \n\t"
1652 "movq %%mm0, %%mm1 \n\t"
1653 "punpckldq %%mm4, %%mm0 \n\t"
1654 "punpckhdq %%mm4, %%mm1 \n\t"
1655 "packsswb %%mm1, %%mm0 \n\t"
1656 "paddb "MANGLE(ff_bgr2UVOffset)
", %%mm0 \n\t"
1657 "movd %%mm0, (%2, %%"FF_REG_a
") \n\t"
1658 "punpckhdq %%mm0, %%mm0 \n\t"
1659 "movd %%mm0, (%3, %%"FF_REG_a
") \n\t"
1660 "add $4, %%"FF_REG_a
" \n\t"
1662 : :
"r" (
src+chromWidth*6),
"r" (
src+srcStride+chromWidth*6),
"r" (udst+chromWidth),
"r" (vdst+chromWidth),
"g" (-chromWidth),
"r"(
rgb2yuv)
1664 :
"%"FF_REG_a,
"%"FF_REG_d
1667 udst += chromStride;
1668 vdst += chromStride;
1680 static inline void vu9_to_vu12_mmxext(
const uint8_t *
src1,
const uint8_t *
src2,
1681 uint8_t *dst1, uint8_t *dst2,
1683 int srcStride1,
int srcStride2,
1684 int dstStride1,
int dstStride2)
1691 ::
"m"(*(
src1+srcStride1)),
"m"(*(
src2+srcStride2)):
"memory");
1693 const uint8_t* s1=
src1+srcStride1*(y>>1);
1694 uint8_t* d=dst1+dstStride1*y;
1696 for (;x<
w-31;x+=32) {
1699 "movq (%1,%2), %%mm0 \n\t"
1700 "movq 8(%1,%2), %%mm2 \n\t"
1701 "movq 16(%1,%2), %%mm4 \n\t"
1702 "movq 24(%1,%2), %%mm6 \n\t"
1703 "movq %%mm0, %%mm1 \n\t"
1704 "movq %%mm2, %%mm3 \n\t"
1705 "movq %%mm4, %%mm5 \n\t"
1706 "movq %%mm6, %%mm7 \n\t"
1707 "punpcklbw %%mm0, %%mm0 \n\t"
1708 "punpckhbw %%mm1, %%mm1 \n\t"
1709 "punpcklbw %%mm2, %%mm2 \n\t"
1710 "punpckhbw %%mm3, %%mm3 \n\t"
1711 "punpcklbw %%mm4, %%mm4 \n\t"
1712 "punpckhbw %%mm5, %%mm5 \n\t"
1713 "punpcklbw %%mm6, %%mm6 \n\t"
1714 "punpckhbw %%mm7, %%mm7 \n\t"
1715 MOVNTQ" %%mm0, (%0,%2,2) \n\t"
1716 MOVNTQ" %%mm1, 8(%0,%2,2) \n\t"
1717 MOVNTQ" %%mm2, 16(%0,%2,2) \n\t"
1718 MOVNTQ" %%mm3, 24(%0,%2,2) \n\t"
1719 MOVNTQ" %%mm4, 32(%0,%2,2) \n\t"
1720 MOVNTQ" %%mm5, 40(%0,%2,2) \n\t"
1721 MOVNTQ" %%mm6, 48(%0,%2,2) \n\t"
1722 MOVNTQ" %%mm7, 56(%0,%2,2)"
1723 ::
"r"(d),
"r"(s1),
"r"(x)
1726 for (;x<
w;x++) d[2*x]=d[2*x+1]=s1[x];
1729 const uint8_t* s2=
src2+srcStride2*(y>>1);
1730 uint8_t* d=dst2+dstStride2*y;
1732 for (;x<
w-31;x+=32) {
1735 "movq (%1,%2), %%mm0 \n\t"
1736 "movq 8(%1,%2), %%mm2 \n\t"
1737 "movq 16(%1,%2), %%mm4 \n\t"
1738 "movq 24(%1,%2), %%mm6 \n\t"
1739 "movq %%mm0, %%mm1 \n\t"
1740 "movq %%mm2, %%mm3 \n\t"
1741 "movq %%mm4, %%mm5 \n\t"
1742 "movq %%mm6, %%mm7 \n\t"
1743 "punpcklbw %%mm0, %%mm0 \n\t"
1744 "punpckhbw %%mm1, %%mm1 \n\t"
1745 "punpcklbw %%mm2, %%mm2 \n\t"
1746 "punpckhbw %%mm3, %%mm3 \n\t"
1747 "punpcklbw %%mm4, %%mm4 \n\t"
1748 "punpckhbw %%mm5, %%mm5 \n\t"
1749 "punpcklbw %%mm6, %%mm6 \n\t"
1750 "punpckhbw %%mm7, %%mm7 \n\t"
1751 MOVNTQ" %%mm0, (%0,%2,2) \n\t"
1752 MOVNTQ" %%mm1, 8(%0,%2,2) \n\t"
1753 MOVNTQ" %%mm2, 16(%0,%2,2) \n\t"
1754 MOVNTQ" %%mm3, 24(%0,%2,2) \n\t"
1755 MOVNTQ" %%mm4, 32(%0,%2,2) \n\t"
1756 MOVNTQ" %%mm5, 40(%0,%2,2) \n\t"
1757 MOVNTQ" %%mm6, 48(%0,%2,2) \n\t"
1758 MOVNTQ" %%mm7, 56(%0,%2,2)"
1759 ::
"r"(d),
"r"(s2),
"r"(x)
1762 for (;x<
w;x++) d[2*x]=d[2*x+1]=s2[x];
1771 static inline void yvu9_to_yuy2_mmxext(
const uint8_t *
src1,
const uint8_t *
src2,
const uint8_t *src3,
1774 int srcStride1,
int srcStride2,
1775 int srcStride3,
int dstStride)
1779 for (
int y = 0; y <
h; y++) {
1780 const uint8_t* yp=
src1+srcStride1*y;
1781 const uint8_t* up=
src2+srcStride2*(y>>2);
1782 const uint8_t* vp=src3+srcStride3*(y>>2);
1783 uint8_t* d=
dst+dstStride*y;
1790 "movq (%1, %0, 4), %%mm0 \n\t"
1791 "movq (%2, %0), %%mm1 \n\t"
1792 "movq (%3, %0), %%mm2 \n\t"
1793 "movq %%mm0, %%mm3 \n\t"
1794 "movq %%mm1, %%mm4 \n\t"
1795 "movq %%mm2, %%mm5 \n\t"
1796 "punpcklbw %%mm1, %%mm1 \n\t"
1797 "punpcklbw %%mm2, %%mm2 \n\t"
1798 "punpckhbw %%mm4, %%mm4 \n\t"
1799 "punpckhbw %%mm5, %%mm5 \n\t"
1801 "movq %%mm1, %%mm6 \n\t"
1802 "punpcklbw %%mm2, %%mm1 \n\t"
1803 "punpcklbw %%mm1, %%mm0 \n\t"
1804 "punpckhbw %%mm1, %%mm3 \n\t"
1805 MOVNTQ" %%mm0, (%4, %0, 8) \n\t"
1806 MOVNTQ" %%mm3, 8(%4, %0, 8) \n\t"
1808 "punpckhbw %%mm2, %%mm6 \n\t"
1809 "movq 8(%1, %0, 4), %%mm0 \n\t"
1810 "movq %%mm0, %%mm3 \n\t"
1811 "punpcklbw %%mm6, %%mm0 \n\t"
1812 "punpckhbw %%mm6, %%mm3 \n\t"
1813 MOVNTQ" %%mm0, 16(%4, %0, 8) \n\t"
1814 MOVNTQ" %%mm3, 24(%4, %0, 8) \n\t"
1816 "movq %%mm4, %%mm6 \n\t"
1817 "movq 16(%1, %0, 4), %%mm0 \n\t"
1818 "movq %%mm0, %%mm3 \n\t"
1819 "punpcklbw %%mm5, %%mm4 \n\t"
1820 "punpcklbw %%mm4, %%mm0 \n\t"
1821 "punpckhbw %%mm4, %%mm3 \n\t"
1822 MOVNTQ" %%mm0, 32(%4, %0, 8) \n\t"
1823 MOVNTQ" %%mm3, 40(%4, %0, 8) \n\t"
1825 "punpckhbw %%mm5, %%mm6 \n\t"
1826 "movq 24(%1, %0, 4), %%mm0 \n\t"
1827 "movq %%mm0, %%mm3 \n\t"
1828 "punpcklbw %%mm6, %%mm0 \n\t"
1829 "punpckhbw %%mm6, %%mm3 \n\t"
1830 MOVNTQ" %%mm0, 48(%4, %0, 8) \n\t"
1831 MOVNTQ" %%mm3, 56(%4, %0, 8) \n\t"
1834 :
"r"(yp),
"r" (up),
"r"(vp),
"r"(d)
1838 const int x2 = x<<2;
1841 d[8*x+2] = yp[x2+1];
1843 d[8*x+4] = yp[x2+2];
1845 d[8*x+6] = yp[x2+3];
1856 static void extract_even_mmxext(
const uint8_t *
src, uint8_t *
dst,
x86_reg count)
1865 "pcmpeqw %%mm7, %%mm7 \n\t"
1866 "psrlw $8, %%mm7 \n\t"
1868 "movq -30(%1, %0, 2), %%mm0 \n\t"
1869 "movq -22(%1, %0, 2), %%mm1 \n\t"
1870 "movq -14(%1, %0, 2), %%mm2 \n\t"
1871 "movq -6(%1, %0, 2), %%mm3 \n\t"
1872 "pand %%mm7, %%mm0 \n\t"
1873 "pand %%mm7, %%mm1 \n\t"
1874 "pand %%mm7, %%mm2 \n\t"
1875 "pand %%mm7, %%mm3 \n\t"
1876 "packuswb %%mm1, %%mm0 \n\t"
1877 "packuswb %%mm3, %%mm2 \n\t"
1878 MOVNTQ" %%mm0,-15(%2, %0) \n\t"
1879 MOVNTQ" %%mm2,- 7(%2, %0) \n\t"
1888 dst[count]=
src[2*count];
1893 static void extract_odd_mmxext(
const uint8_t *
src, uint8_t *
dst,
x86_reg count)
1903 "pcmpeqw %%mm7, %%mm7 \n\t"
1904 "psrlw $8, %%mm7 \n\t"
1906 "movq -32(%1, %0, 2), %%mm0 \n\t"
1907 "movq -24(%1, %0, 2), %%mm1 \n\t"
1908 "movq -16(%1, %0, 2), %%mm2 \n\t"
1909 "movq -8(%1, %0, 2), %%mm3 \n\t"
1910 "pand %%mm7, %%mm0 \n\t"
1911 "pand %%mm7, %%mm1 \n\t"
1912 "pand %%mm7, %%mm2 \n\t"
1913 "pand %%mm7, %%mm3 \n\t"
1914 "packuswb %%mm1, %%mm0 \n\t"
1915 "packuswb %%mm3, %%mm2 \n\t"
1916 MOVNTQ" %%mm0,-16(%2, %0) \n\t"
1917 MOVNTQ" %%mm2,- 8(%2, %0) \n\t"
1926 dst[count]=
src[2*count];
1932 static void extract_even2_mmxext(
const uint8_t *
src, uint8_t *dst0, uint8_t *dst1,
x86_reg count)
1941 "pcmpeqw %%mm7, %%mm7 \n\t"
1942 "psrlw $8, %%mm7 \n\t"
1944 "movq -28(%1, %0, 4), %%mm0 \n\t"
1945 "movq -20(%1, %0, 4), %%mm1 \n\t"
1946 "movq -12(%1, %0, 4), %%mm2 \n\t"
1947 "movq -4(%1, %0, 4), %%mm3 \n\t"
1948 "pand %%mm7, %%mm0 \n\t"
1949 "pand %%mm7, %%mm1 \n\t"
1950 "pand %%mm7, %%mm2 \n\t"
1951 "pand %%mm7, %%mm3 \n\t"
1952 "packuswb %%mm1, %%mm0 \n\t"
1953 "packuswb %%mm3, %%mm2 \n\t"
1954 "movq %%mm0, %%mm1 \n\t"
1955 "movq %%mm2, %%mm3 \n\t"
1956 "psrlw $8, %%mm0 \n\t"
1957 "psrlw $8, %%mm2 \n\t"
1958 "pand %%mm7, %%mm1 \n\t"
1959 "pand %%mm7, %%mm3 \n\t"
1960 "packuswb %%mm2, %%mm0 \n\t"
1961 "packuswb %%mm3, %%mm1 \n\t"
1962 MOVNTQ" %%mm0,- 7(%3, %0) \n\t"
1963 MOVNTQ" %%mm1,- 7(%2, %0) \n\t"
1967 :
"r"(
src),
"r"(dst0),
"r"(dst1)
1972 dst0[count]=
src[4*count+0];
1973 dst1[count]=
src[4*count+2];
1979 static void extract_even2avg_mmxext(
const uint8_t *
src0,
const uint8_t *
src1, uint8_t *dst0, uint8_t *dst1,
x86_reg count)
1990 "pcmpeqw %%mm7, %%mm7 \n\t"
1991 "psrlw $8, %%mm7 \n\t"
1993 "movq -28(%1, %0, 4), %%mm0 \n\t"
1994 "movq -20(%1, %0, 4), %%mm1 \n\t"
1995 "movq -12(%1, %0, 4), %%mm2 \n\t"
1996 "movq -4(%1, %0, 4), %%mm3 \n\t"
1997 PAVGB" -28(%2, %0, 4), %%mm0 \n\t"
1998 PAVGB" -20(%2, %0, 4), %%mm1 \n\t"
1999 PAVGB" -12(%2, %0, 4), %%mm2 \n\t"
2000 PAVGB" - 4(%2, %0, 4), %%mm3 \n\t"
2001 "pand %%mm7, %%mm0 \n\t"
2002 "pand %%mm7, %%mm1 \n\t"
2003 "pand %%mm7, %%mm2 \n\t"
2004 "pand %%mm7, %%mm3 \n\t"
2005 "packuswb %%mm1, %%mm0 \n\t"
2006 "packuswb %%mm3, %%mm2 \n\t"
2007 "movq %%mm0, %%mm1 \n\t"
2008 "movq %%mm2, %%mm3 \n\t"
2009 "psrlw $8, %%mm0 \n\t"
2010 "psrlw $8, %%mm2 \n\t"
2011 "pand %%mm7, %%mm1 \n\t"
2012 "pand %%mm7, %%mm3 \n\t"
2013 "packuswb %%mm2, %%mm0 \n\t"
2014 "packuswb %%mm3, %%mm1 \n\t"
2015 MOVNTQ" %%mm0,- 7(%4, %0) \n\t"
2016 MOVNTQ" %%mm1,- 7(%3, %0) \n\t"
2020 :
"r"(
src0),
"r"(
src1),
"r"(dst0),
"r"(dst1)
2026 dst0[count]= (
src0[4*count+0]+
src1[4*count+0])>>1;
2027 dst1[count]= (
src0[4*count+2]+
src1[4*count+2])>>1;
2032 static void extract_odd2_mmxext(
const uint8_t *
src, uint8_t *dst0, uint8_t *dst1,
x86_reg count)
2041 "pcmpeqw %%mm7, %%mm7 \n\t"
2042 "psrlw $8, %%mm7 \n\t"
2044 "movq -28(%1, %0, 4), %%mm0 \n\t"
2045 "movq -20(%1, %0, 4), %%mm1 \n\t"
2046 "movq -12(%1, %0, 4), %%mm2 \n\t"
2047 "movq -4(%1, %0, 4), %%mm3 \n\t"
2048 "psrlw $8, %%mm0 \n\t"
2049 "psrlw $8, %%mm1 \n\t"
2050 "psrlw $8, %%mm2 \n\t"
2051 "psrlw $8, %%mm3 \n\t"
2052 "packuswb %%mm1, %%mm0 \n\t"
2053 "packuswb %%mm3, %%mm2 \n\t"
2054 "movq %%mm0, %%mm1 \n\t"
2055 "movq %%mm2, %%mm3 \n\t"
2056 "psrlw $8, %%mm0 \n\t"
2057 "psrlw $8, %%mm2 \n\t"
2058 "pand %%mm7, %%mm1 \n\t"
2059 "pand %%mm7, %%mm3 \n\t"
2060 "packuswb %%mm2, %%mm0 \n\t"
2061 "packuswb %%mm3, %%mm1 \n\t"
2062 MOVNTQ" %%mm0,- 7(%3, %0) \n\t"
2063 MOVNTQ" %%mm1,- 7(%2, %0) \n\t"
2067 :
"r"(
src),
"r"(dst0),
"r"(dst1)
2073 dst0[count]=
src[4*count+0];
2074 dst1[count]=
src[4*count+2];
2079 static void extract_odd2avg_mmxext(
const uint8_t *
src0,
const uint8_t *
src1, uint8_t *dst0, uint8_t *dst1,
x86_reg count)
2090 "pcmpeqw %%mm7, %%mm7 \n\t"
2091 "psrlw $8, %%mm7 \n\t"
2093 "movq -28(%1, %0, 4), %%mm0 \n\t"
2094 "movq -20(%1, %0, 4), %%mm1 \n\t"
2095 "movq -12(%1, %0, 4), %%mm2 \n\t"
2096 "movq -4(%1, %0, 4), %%mm3 \n\t"
2097 PAVGB" -28(%2, %0, 4), %%mm0 \n\t"
2098 PAVGB" -20(%2, %0, 4), %%mm1 \n\t"
2099 PAVGB" -12(%2, %0, 4), %%mm2 \n\t"
2100 PAVGB" - 4(%2, %0, 4), %%mm3 \n\t"
2101 "psrlw $8, %%mm0 \n\t"
2102 "psrlw $8, %%mm1 \n\t"
2103 "psrlw $8, %%mm2 \n\t"
2104 "psrlw $8, %%mm3 \n\t"
2105 "packuswb %%mm1, %%mm0 \n\t"
2106 "packuswb %%mm3, %%mm2 \n\t"
2107 "movq %%mm0, %%mm1 \n\t"
2108 "movq %%mm2, %%mm3 \n\t"
2109 "psrlw $8, %%mm0 \n\t"
2110 "psrlw $8, %%mm2 \n\t"
2111 "pand %%mm7, %%mm1 \n\t"
2112 "pand %%mm7, %%mm3 \n\t"
2113 "packuswb %%mm2, %%mm0 \n\t"
2114 "packuswb %%mm3, %%mm1 \n\t"
2115 MOVNTQ" %%mm0,- 7(%4, %0) \n\t"
2116 MOVNTQ" %%mm1,- 7(%3, %0) \n\t"
2120 :
"r"(
src0),
"r"(
src1),
"r"(dst0),
"r"(dst1)
2128 dst0[count]= (
src0[4*count+0]+
src1[4*count+0])>>1;
2129 dst1[count]= (
src0[4*count+2]+
src1[4*count+2])>>1;
2134 static void yuyvtoyuv420_mmxext(uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
const uint8_t *
src,
2136 int lumStride,
int chromStride,
int srcStride)
2140 for (
int y = 0; y <
height; y++) {
2141 extract_even_mmxext(
src, ydst,
width);
2143 extract_odd2avg_mmxext(
src-srcStride,
src, udst, vdst, chromWidth);
2158 static void yuyvtoyuv422_mmxext(uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
const uint8_t *
src,
2160 int lumStride,
int chromStride,
int srcStride)
2164 for (
int y = 0; y <
height; y++) {
2165 extract_even_mmxext(
src, ydst,
width);
2166 extract_odd2_mmxext(
src, udst, vdst, chromWidth);
2180 static void uyvytoyuv420_mmxext(uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
const uint8_t *
src,
2182 int lumStride,
int chromStride,
int srcStride)
2186 for (
int y = 0; y <
height; y++) {
2187 extract_odd_mmxext(
src, ydst,
width);
2189 extract_even2avg_mmxext(
src-srcStride,
src, udst, vdst, chromWidth);
2205 static void uyvytoyuv422_mmxext(uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
const uint8_t *
src,
2207 int lumStride,
int chromStride,
int srcStride)
2211 for (
int y = 0; y <
height; y++) {
2212 extract_odd_mmxext(
src, ydst,
width);
2213 extract_even2_mmxext(
src, udst, vdst, chromWidth);
2228 static av_cold void rgb2rgb_init_mmxext(
void)
2260 #if ARCH_X86_32 && HAVE_7REGS
2269 static void interleave_bytes_sse2(
const uint8_t *
src1,
const uint8_t *
src2, uint8_t *dest,
2271 int src2Stride,
int dstStride)
2275 if (!((((intptr_t)
src1) | ((intptr_t)
src2) | ((intptr_t)dest))&15)) {
2277 "xor %%"FF_REG_a
", %%"FF_REG_a
" \n\t"
2279 PREFETCH" 64(%1, %%"FF_REG_a
") \n\t"
2280 PREFETCH" 64(%2, %%"FF_REG_a
") \n\t"
2281 "movdqa (%1, %%"FF_REG_a
"), %%xmm0 \n\t"
2282 "movdqa (%1, %%"FF_REG_a
"), %%xmm1 \n\t"
2283 "movdqa (%2, %%"FF_REG_a
"), %%xmm2 \n\t"
2284 "punpcklbw %%xmm2, %%xmm0 \n\t"
2285 "punpckhbw %%xmm2, %%xmm1 \n\t"
2286 "movntdq %%xmm0, (%0, %%"FF_REG_a
", 2) \n\t"
2287 "movntdq %%xmm1, 16(%0, %%"FF_REG_a
", 2) \n\t"
2288 "add $16, %%"FF_REG_a
" \n\t"
2289 "cmp %3, %%"FF_REG_a
" \n\t"
2292 :
"memory",
XMM_CLOBBERS(
"xmm0",
"xmm1",
"xmm2",)
"%"FF_REG_a
2296 "xor %%"FF_REG_a
", %%"FF_REG_a
" \n\t"
2298 PREFETCH" 64(%1, %%"FF_REG_a
") \n\t"
2299 PREFETCH" 64(%2, %%"FF_REG_a
") \n\t"
2300 "movq (%1, %%"FF_REG_a
"), %%mm0 \n\t"
2301 "movq 8(%1, %%"FF_REG_a
"), %%mm2 \n\t"
2302 "movq %%mm0, %%mm1 \n\t"
2303 "movq %%mm2, %%mm3 \n\t"
2304 "movq (%2, %%"FF_REG_a
"), %%mm4 \n\t"
2305 "movq 8(%2, %%"FF_REG_a
"), %%mm5 \n\t"
2306 "punpcklbw %%mm4, %%mm0 \n\t"
2307 "punpckhbw %%mm4, %%mm1 \n\t"
2308 "punpcklbw %%mm5, %%mm2 \n\t"
2309 "punpckhbw %%mm5, %%mm3 \n\t"
2310 MOVNTQ" %%mm0, (%0, %%"FF_REG_a
", 2) \n\t"
2311 MOVNTQ" %%mm1, 8(%0, %%"FF_REG_a
", 2) \n\t"
2312 MOVNTQ" %%mm2, 16(%0, %%"FF_REG_a
", 2) \n\t"
2313 MOVNTQ" %%mm3, 24(%0, %%"FF_REG_a
", 2) \n\t"
2314 "add $16, %%"FF_REG_a
" \n\t"
2315 "cmp %3, %%"FF_REG_a
" \n\t"
2318 :
"memory",
"%"FF_REG_a
2353 void ff_shuffle_bytes_2103_avx2(
const uint8_t *
src, uint8_t *
dst,
int src_size);
2354 void ff_shuffle_bytes_0321_avx2(
const uint8_t *
src, uint8_t *
dst,
int src_size);
2355 void ff_shuffle_bytes_1230_avx2(
const uint8_t *
src, uint8_t *
dst,
int src_size);
2356 void ff_shuffle_bytes_3012_avx2(
const uint8_t *
src, uint8_t *
dst,
int src_size);
2357 void ff_shuffle_bytes_3210_avx2(
const uint8_t *
src, uint8_t *
dst,
int src_size);
2359 void ff_uyvytoyuv422_sse2(uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2361 int lumStride,
int chromStride,
int srcStride);
2362 void ff_uyvytoyuv422_avx(uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2364 int lumStride,
int chromStride,
int srcStride);
2365 void ff_uyvytoyuv422_avx2(uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2367 int lumStride,
int chromStride,
int srcStride);
2370 #define DEINTERLEAVE_BYTES(cpuext) \
2371 void ff_nv12ToUV_ ## cpuext(uint8_t *dstU, uint8_t *dstV, \
2372 const uint8_t *unused, \
2373 const uint8_t *src1, \
2374 const uint8_t *src2, \
2376 uint32_t *unused2, \
2378 static void deinterleave_bytes_ ## cpuext(const uint8_t *src, uint8_t *dst1, uint8_t *dst2, \
2379 int width, int height, int srcStride, \
2380 int dst1Stride, int dst2Stride) \
2382 for (int h = 0; h < height; h++) { \
2384 ff_nv12ToUV_ ## cpuext(dst1, dst2, NULL, src, NULL, width - 15, NULL, NULL); \
2385 for (int w = (width & (~15)); w < width; w++) { \
2386 dst1[w] = src[2*w+0]; \
2387 dst2[w] = src[2*w+1]; \
2390 dst1 += dst1Stride; \
2391 dst2 += dst2Stride; \
2395 #if HAVE_SSE2_EXTERNAL
2398 #if HAVE_AVX_EXTERNAL
2408 rgb2rgb_init_mmxext();
2413 #if HAVE_SSE2_EXTERNAL
2428 #if HAVE_AVX_EXTERNAL