FFmpeg: libavcodec/x86/mpegvideo

00001 /*
00002  * The simplest mpeg encoder (well, it was the simplest!)
00003  * Copyright (c) 2000,2001 Fabrice Bellard
00004  *
00005  * Optimized for ia32 CPUs by Nick Kurshev <nickols_k@mail.ru>
00006  * h263, mpeg1, mpeg2 dequantizer & draw_edges by Michael Niedermayer <michaelni@gmx.at>
00007  *
00008  * This file is part of FFmpeg.
00009  *
00010  * FFmpeg is free software; you can redistribute it and/or
00011  * modify it under the terms of the GNU Lesser General Public
00012  * License as published by the Free Software Foundation; either
00013  * version 2.1 of the License, or (at your option) any later version.
00014  *
00015  * FFmpeg is distributed in the hope that it will be useful,
00016  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00017  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018  * Lesser General Public License for more details.
00019  *
00020  * You should have received a copy of the GNU Lesser General Public
00021  * License along with FFmpeg; if not, write to the Free Software
00022  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
00023  */
00024 
00025 #include "libavutil/cpu.h"
00026 #include "libavutil/x86/asm.h"
00027 #include "libavcodec/avcodec.h"
00028 #include "libavcodec/dsputil.h"
00029 #include "libavcodec/mpegvideo.h"
00030 #include "dsputil_mmx.h"
00031 
00032 #if HAVE_INLINE_ASM
00033 
00034 extern uint16_t ff_inv_zigzag_direct16[64];
00035 
00036 
00037 static void dct_unquantize_h263_intra_mmx(MpegEncContext *s,
00038                                   DCTELEM *block, int n, int qscale)
00039 {
00040     x86_reg level, qmul, qadd, nCoeffs;
00041 
00042     qmul = qscale << 1;
00043 
00044     assert(s->block_last_index[n]>=0 || s->h263_aic);
00045 
00046     if (!s->h263_aic) {
00047         if (n < 4)
00048             level = block[0] * s->y_dc_scale;
00049         else
00050             level = block[0] * s->c_dc_scale;
00051         qadd = (qscale - 1) | 1;
00052     }else{
00053         qadd = 0;
00054         level= block[0];
00055     }
00056     if(s->ac_pred)
00057         nCoeffs=63;
00058     else
00059         nCoeffs= s->inter_scantable.raster_end[ s->block_last_index[n] ];
00060 //printf("%d %d  ", qmul, qadd);
00061 __asm__ volatile(
00062                 "movd %1, %%mm6                 \n\t" //qmul
00063                 "packssdw %%mm6, %%mm6          \n\t"
00064                 "packssdw %%mm6, %%mm6          \n\t"
00065                 "movd %2, %%mm5                 \n\t" //qadd
00066                 "pxor %%mm7, %%mm7              \n\t"
00067                 "packssdw %%mm5, %%mm5          \n\t"
00068                 "packssdw %%mm5, %%mm5          \n\t"
00069                 "psubw %%mm5, %%mm7             \n\t"
00070                 "pxor %%mm4, %%mm4              \n\t"
00071                 ".p2align 4                     \n\t"
00072                 "1:                             \n\t"
00073                 "movq (%0, %3), %%mm0           \n\t"
00074                 "movq 8(%0, %3), %%mm1          \n\t"
00075 
00076                 "pmullw %%mm6, %%mm0            \n\t"
00077                 "pmullw %%mm6, %%mm1            \n\t"
00078 
00079                 "movq (%0, %3), %%mm2           \n\t"
00080                 "movq 8(%0, %3), %%mm3          \n\t"
00081 
00082                 "pcmpgtw %%mm4, %%mm2           \n\t" // block[i] < 0 ? -1 : 0
00083                 "pcmpgtw %%mm4, %%mm3           \n\t" // block[i] < 0 ? -1 : 0
00084 
00085                 "pxor %%mm2, %%mm0              \n\t"
00086                 "pxor %%mm3, %%mm1              \n\t"
00087 
00088                 "paddw %%mm7, %%mm0             \n\t"
00089                 "paddw %%mm7, %%mm1             \n\t"
00090 
00091                 "pxor %%mm0, %%mm2              \n\t"
00092                 "pxor %%mm1, %%mm3              \n\t"
00093 
00094                 "pcmpeqw %%mm7, %%mm0           \n\t" // block[i] == 0 ? -1 : 0
00095                 "pcmpeqw %%mm7, %%mm1           \n\t" // block[i] == 0 ? -1 : 0
00096 
00097                 "pandn %%mm2, %%mm0             \n\t"
00098                 "pandn %%mm3, %%mm1             \n\t"
00099 
00100                 "movq %%mm0, (%0, %3)           \n\t"
00101                 "movq %%mm1, 8(%0, %3)          \n\t"
00102 
00103                 "add $16, %3                    \n\t"
00104                 "jng 1b                         \n\t"
00105                 ::"r" (block+nCoeffs), "rm"(qmul), "rm" (qadd), "r" (2*(-nCoeffs))
00106                 : "memory"
00107         );
00108         block[0]= level;
00109 }
00110 
00111 
00112 static void dct_unquantize_h263_inter_mmx(MpegEncContext *s,
00113                                   DCTELEM *block, int n, int qscale)
00114 {
00115     x86_reg qmul, qadd, nCoeffs;
00116 
00117     qmul = qscale << 1;
00118     qadd = (qscale - 1) | 1;
00119 
00120     assert(s->block_last_index[n]>=0 || s->h263_aic);
00121 
00122     nCoeffs= s->inter_scantable.raster_end[ s->block_last_index[n] ];
00123 //printf("%d %d  ", qmul, qadd);
00124 __asm__ volatile(
00125                 "movd %1, %%mm6                 \n\t" //qmul
00126                 "packssdw %%mm6, %%mm6          \n\t"
00127                 "packssdw %%mm6, %%mm6          \n\t"
00128                 "movd %2, %%mm5                 \n\t" //qadd
00129                 "pxor %%mm7, %%mm7              \n\t"
00130                 "packssdw %%mm5, %%mm5          \n\t"
00131                 "packssdw %%mm5, %%mm5          \n\t"
00132                 "psubw %%mm5, %%mm7             \n\t"
00133                 "pxor %%mm4, %%mm4              \n\t"
00134                 ".p2align 4                     \n\t"
00135                 "1:                             \n\t"
00136                 "movq (%0, %3), %%mm0           \n\t"
00137                 "movq 8(%0, %3), %%mm1          \n\t"
00138 
00139                 "pmullw %%mm6, %%mm0            \n\t"
00140                 "pmullw %%mm6, %%mm1            \n\t"
00141 
00142                 "movq (%0, %3), %%mm2           \n\t"
00143                 "movq 8(%0, %3), %%mm3          \n\t"
00144 
00145                 "pcmpgtw %%mm4, %%mm2           \n\t" // block[i] < 0 ? -1 : 0
00146                 "pcmpgtw %%mm4, %%mm3           \n\t" // block[i] < 0 ? -1 : 0
00147 
00148                 "pxor %%mm2, %%mm0              \n\t"
00149                 "pxor %%mm3, %%mm1              \n\t"
00150 
00151                 "paddw %%mm7, %%mm0             \n\t"
00152                 "paddw %%mm7, %%mm1             \n\t"
00153 
00154                 "pxor %%mm0, %%mm2              \n\t"
00155                 "pxor %%mm1, %%mm3              \n\t"
00156 
00157                 "pcmpeqw %%mm7, %%mm0           \n\t" // block[i] == 0 ? -1 : 0
00158                 "pcmpeqw %%mm7, %%mm1           \n\t" // block[i] == 0 ? -1 : 0
00159 
00160                 "pandn %%mm2, %%mm0             \n\t"
00161                 "pandn %%mm3, %%mm1             \n\t"
00162 
00163                 "movq %%mm0, (%0, %3)           \n\t"
00164                 "movq %%mm1, 8(%0, %3)          \n\t"
00165 
00166                 "add $16, %3                    \n\t"
00167                 "jng 1b                         \n\t"
00168                 ::"r" (block+nCoeffs), "rm"(qmul), "rm" (qadd), "r" (2*(-nCoeffs))
00169                 : "memory"
00170         );
00171 }
00172 
00173 
00174 /*
00175   NK:
00176   Note: looking at PARANOID:
00177   "enable all paranoid tests for rounding, overflows, etc..."
00178 
00179 #ifdef PARANOID
00180                 if (level < -2048 || level > 2047)
00181                     fprintf(stderr, "unquant error %d %d\n", i, level);
00182 #endif
00183   We can suppose that result of two multiplications can't be greater than 0xFFFF
00184   i.e. is 16-bit, so we use here only PMULLW instruction and can avoid
00185   a complex multiplication.
00186 =====================================================
00187  Full formula for multiplication of 2 integer numbers
00188  which are represent as high:low words:
00189  input: value1 = high1:low1
00190         value2 = high2:low2
00191  output: value3 = value1*value2
00192  value3=high3:low3 (on overflow: modulus 2^32 wrap-around)
00193  this mean that for 0x123456 * 0x123456 correct result is 0x766cb0ce4
00194  but this algorithm will compute only 0x66cb0ce4
00195  this limited by 16-bit size of operands
00196  ---------------------------------
00197  tlow1 = high1*low2
00198  tlow2 = high2*low1
00199  tlow1 = tlow1 + tlow2
00200  high3:low3 = low1*low2
00201  high3 += tlow1
00202 */
00203 static void dct_unquantize_mpeg1_intra_mmx(MpegEncContext *s,
00204                                      DCTELEM *block, int n, int qscale)
00205 {
00206     x86_reg nCoeffs;
00207     const uint16_t *quant_matrix;
00208     int block0;
00209 
00210     assert(s->block_last_index[n]>=0);
00211 
00212     nCoeffs= s->intra_scantable.raster_end[ s->block_last_index[n] ]+1;
00213 
00214     if (n < 4)
00215         block0 = block[0] * s->y_dc_scale;
00216     else
00217         block0 = block[0] * s->c_dc_scale;
00218     /* XXX: only mpeg1 */
00219     quant_matrix = s->intra_matrix;
00220 __asm__ volatile(
00221                 "pcmpeqw %%mm7, %%mm7           \n\t"
00222                 "psrlw $15, %%mm7               \n\t"
00223                 "movd %2, %%mm6                 \n\t"
00224                 "packssdw %%mm6, %%mm6          \n\t"
00225                 "packssdw %%mm6, %%mm6          \n\t"
00226                 "mov %3, %%"REG_a"              \n\t"
00227                 ".p2align 4                     \n\t"
00228                 "1:                             \n\t"
00229                 "movq (%0, %%"REG_a"), %%mm0    \n\t"
00230                 "movq 8(%0, %%"REG_a"), %%mm1   \n\t"
00231                 "movq (%1, %%"REG_a"), %%mm4    \n\t"
00232                 "movq 8(%1, %%"REG_a"), %%mm5   \n\t"
00233                 "pmullw %%mm6, %%mm4            \n\t" // q=qscale*quant_matrix[i]
00234                 "pmullw %%mm6, %%mm5            \n\t" // q=qscale*quant_matrix[i]
00235                 "pxor %%mm2, %%mm2              \n\t"
00236                 "pxor %%mm3, %%mm3              \n\t"
00237                 "pcmpgtw %%mm0, %%mm2           \n\t" // block[i] < 0 ? -1 : 0
00238                 "pcmpgtw %%mm1, %%mm3           \n\t" // block[i] < 0 ? -1 : 0
00239                 "pxor %%mm2, %%mm0              \n\t"
00240                 "pxor %%mm3, %%mm1              \n\t"
00241                 "psubw %%mm2, %%mm0             \n\t" // abs(block[i])
00242                 "psubw %%mm3, %%mm1             \n\t" // abs(block[i])
00243                 "pmullw %%mm4, %%mm0            \n\t" // abs(block[i])*q
00244                 "pmullw %%mm5, %%mm1            \n\t" // abs(block[i])*q
00245                 "pxor %%mm4, %%mm4              \n\t"
00246                 "pxor %%mm5, %%mm5              \n\t" // FIXME slow
00247                 "pcmpeqw (%0, %%"REG_a"), %%mm4 \n\t" // block[i] == 0 ? -1 : 0
00248                 "pcmpeqw 8(%0, %%"REG_a"), %%mm5\n\t" // block[i] == 0 ? -1 : 0
00249                 "psraw $3, %%mm0                \n\t"
00250                 "psraw $3, %%mm1                \n\t"
00251                 "psubw %%mm7, %%mm0             \n\t"
00252                 "psubw %%mm7, %%mm1             \n\t"
00253                 "por %%mm7, %%mm0               \n\t"
00254                 "por %%mm7, %%mm1               \n\t"
00255                 "pxor %%mm2, %%mm0              \n\t"
00256                 "pxor %%mm3, %%mm1              \n\t"
00257                 "psubw %%mm2, %%mm0             \n\t"
00258                 "psubw %%mm3, %%mm1             \n\t"
00259                 "pandn %%mm0, %%mm4             \n\t"
00260                 "pandn %%mm1, %%mm5             \n\t"
00261                 "movq %%mm4, (%0, %%"REG_a")    \n\t"
00262                 "movq %%mm5, 8(%0, %%"REG_a")   \n\t"
00263 
00264                 "add $16, %%"REG_a"             \n\t"
00265                 "js 1b                          \n\t"
00266                 ::"r" (block+nCoeffs), "r"(quant_matrix+nCoeffs), "rm" (qscale), "g" (-2*nCoeffs)
00267                 : "%"REG_a, "memory"
00268         );
00269     block[0]= block0;
00270 }
00271 
00272 static void dct_unquantize_mpeg1_inter_mmx(MpegEncContext *s,
00273                                      DCTELEM *block, int n, int qscale)
00274 {
00275     x86_reg nCoeffs;
00276     const uint16_t *quant_matrix;
00277 
00278     assert(s->block_last_index[n]>=0);
00279 
00280     nCoeffs= s->intra_scantable.raster_end[ s->block_last_index[n] ]+1;
00281 
00282         quant_matrix = s->inter_matrix;
00283 __asm__ volatile(
00284                 "pcmpeqw %%mm7, %%mm7           \n\t"
00285                 "psrlw $15, %%mm7               \n\t"
00286                 "movd %2, %%mm6                 \n\t"
00287                 "packssdw %%mm6, %%mm6          \n\t"
00288                 "packssdw %%mm6, %%mm6          \n\t"
00289                 "mov %3, %%"REG_a"              \n\t"
00290                 ".p2align 4                     \n\t"
00291                 "1:                             \n\t"
00292                 "movq (%0, %%"REG_a"), %%mm0    \n\t"
00293                 "movq 8(%0, %%"REG_a"), %%mm1   \n\t"
00294                 "movq (%1, %%"REG_a"), %%mm4    \n\t"
00295                 "movq 8(%1, %%"REG_a"), %%mm5   \n\t"
00296                 "pmullw %%mm6, %%mm4            \n\t" // q=qscale*quant_matrix[i]
00297                 "pmullw %%mm6, %%mm5            \n\t" // q=qscale*quant_matrix[i]
00298                 "pxor %%mm2, %%mm2              \n\t"
00299                 "pxor %%mm3, %%mm3              \n\t"
00300                 "pcmpgtw %%mm0, %%mm2           \n\t" // block[i] < 0 ? -1 : 0
00301                 "pcmpgtw %%mm1, %%mm3           \n\t" // block[i] < 0 ? -1 : 0
00302                 "pxor %%mm2, %%mm0              \n\t"
00303                 "pxor %%mm3, %%mm1              \n\t"
00304                 "psubw %%mm2, %%mm0             \n\t" // abs(block[i])
00305                 "psubw %%mm3, %%mm1             \n\t" // abs(block[i])
00306                 "paddw %%mm0, %%mm0             \n\t" // abs(block[i])*2
00307                 "paddw %%mm1, %%mm1             \n\t" // abs(block[i])*2
00308                 "paddw %%mm7, %%mm0             \n\t" // abs(block[i])*2 + 1
00309                 "paddw %%mm7, %%mm1             \n\t" // abs(block[i])*2 + 1
00310                 "pmullw %%mm4, %%mm0            \n\t" // (abs(block[i])*2 + 1)*q
00311                 "pmullw %%mm5, %%mm1            \n\t" // (abs(block[i])*2 + 1)*q
00312                 "pxor %%mm4, %%mm4              \n\t"
00313                 "pxor %%mm5, %%mm5              \n\t" // FIXME slow
00314                 "pcmpeqw (%0, %%"REG_a"), %%mm4 \n\t" // block[i] == 0 ? -1 : 0
00315                 "pcmpeqw 8(%0, %%"REG_a"), %%mm5\n\t" // block[i] == 0 ? -1 : 0
00316                 "psraw $4, %%mm0                \n\t"
00317                 "psraw $4, %%mm1                \n\t"
00318                 "psubw %%mm7, %%mm0             \n\t"
00319                 "psubw %%mm7, %%mm1             \n\t"
00320                 "por %%mm7, %%mm0               \n\t"
00321                 "por %%mm7, %%mm1               \n\t"
00322                 "pxor %%mm2, %%mm0              \n\t"
00323                 "pxor %%mm3, %%mm1              \n\t"
00324                 "psubw %%mm2, %%mm0             \n\t"
00325                 "psubw %%mm3, %%mm1             \n\t"
00326                 "pandn %%mm0, %%mm4             \n\t"
00327                 "pandn %%mm1, %%mm5             \n\t"
00328                 "movq %%mm4, (%0, %%"REG_a")    \n\t"
00329                 "movq %%mm5, 8(%0, %%"REG_a")   \n\t"
00330 
00331                 "add $16, %%"REG_a"             \n\t"
00332                 "js 1b                          \n\t"
00333                 ::"r" (block+nCoeffs), "r"(quant_matrix+nCoeffs), "rm" (qscale), "g" (-2*nCoeffs)
00334                 : "%"REG_a, "memory"
00335         );
00336 }
00337 
00338 static void dct_unquantize_mpeg2_intra_mmx(MpegEncContext *s,
00339                                      DCTELEM *block, int n, int qscale)
00340 {
00341     x86_reg nCoeffs;
00342     const uint16_t *quant_matrix;
00343     int block0;
00344 
00345     assert(s->block_last_index[n]>=0);
00346 
00347     if(s->alternate_scan) nCoeffs= 63; //FIXME
00348     else nCoeffs= s->intra_scantable.raster_end[ s->block_last_index[n] ];
00349 
00350     if (n < 4)
00351         block0 = block[0] * s->y_dc_scale;
00352     else
00353         block0 = block[0] * s->c_dc_scale;
00354     quant_matrix = s->intra_matrix;
00355 __asm__ volatile(
00356                 "pcmpeqw %%mm7, %%mm7           \n\t"
00357                 "psrlw $15, %%mm7               \n\t"
00358                 "movd %2, %%mm6                 \n\t"
00359                 "packssdw %%mm6, %%mm6          \n\t"
00360                 "packssdw %%mm6, %%mm6          \n\t"
00361                 "mov %3, %%"REG_a"              \n\t"
00362                 ".p2align 4                     \n\t"
00363                 "1:                             \n\t"
00364                 "movq (%0, %%"REG_a"), %%mm0    \n\t"
00365                 "movq 8(%0, %%"REG_a"), %%mm1   \n\t"
00366                 "movq (%1, %%"REG_a"), %%mm4    \n\t"
00367                 "movq 8(%1, %%"REG_a"), %%mm5   \n\t"
00368                 "pmullw %%mm6, %%mm4            \n\t" // q=qscale*quant_matrix[i]
00369                 "pmullw %%mm6, %%mm5            \n\t" // q=qscale*quant_matrix[i]
00370                 "pxor %%mm2, %%mm2              \n\t"
00371                 "pxor %%mm3, %%mm3              \n\t"
00372                 "pcmpgtw %%mm0, %%mm2           \n\t" // block[i] < 0 ? -1 : 0
00373                 "pcmpgtw %%mm1, %%mm3           \n\t" // block[i] < 0 ? -1 : 0
00374                 "pxor %%mm2, %%mm0              \n\t"
00375                 "pxor %%mm3, %%mm1              \n\t"
00376                 "psubw %%mm2, %%mm0             \n\t" // abs(block[i])
00377                 "psubw %%mm3, %%mm1             \n\t" // abs(block[i])
00378                 "pmullw %%mm4, %%mm0            \n\t" // abs(block[i])*q
00379                 "pmullw %%mm5, %%mm1            \n\t" // abs(block[i])*q
00380                 "pxor %%mm4, %%mm4              \n\t"
00381                 "pxor %%mm5, %%mm5              \n\t" // FIXME slow
00382                 "pcmpeqw (%0, %%"REG_a"), %%mm4 \n\t" // block[i] == 0 ? -1 : 0
00383                 "pcmpeqw 8(%0, %%"REG_a"), %%mm5\n\t" // block[i] == 0 ? -1 : 0
00384                 "psraw $3, %%mm0                \n\t"
00385                 "psraw $3, %%mm1                \n\t"
00386                 "pxor %%mm2, %%mm0              \n\t"
00387                 "pxor %%mm3, %%mm1              \n\t"
00388                 "psubw %%mm2, %%mm0             \n\t"
00389                 "psubw %%mm3, %%mm1             \n\t"
00390                 "pandn %%mm0, %%mm4             \n\t"
00391                 "pandn %%mm1, %%mm5             \n\t"
00392                 "movq %%mm4, (%0, %%"REG_a")    \n\t"
00393                 "movq %%mm5, 8(%0, %%"REG_a")   \n\t"
00394 
00395                 "add $16, %%"REG_a"             \n\t"
00396                 "jng 1b                         \n\t"
00397                 ::"r" (block+nCoeffs), "r"(quant_matrix+nCoeffs), "rm" (qscale), "g" (-2*nCoeffs)
00398                 : "%"REG_a, "memory"
00399         );
00400     block[0]= block0;
00401         //Note, we do not do mismatch control for intra as errors cannot accumulate
00402 }
00403 
00404 static void dct_unquantize_mpeg2_inter_mmx(MpegEncContext *s,
00405                                      DCTELEM *block, int n, int qscale)
00406 {
00407     x86_reg nCoeffs;
00408     const uint16_t *quant_matrix;
00409 
00410     assert(s->block_last_index[n]>=0);
00411 
00412     if(s->alternate_scan) nCoeffs= 63; //FIXME
00413     else nCoeffs= s->intra_scantable.raster_end[ s->block_last_index[n] ];
00414 
00415         quant_matrix = s->inter_matrix;
00416 __asm__ volatile(
00417                 "pcmpeqw %%mm7, %%mm7           \n\t"
00418                 "psrlq $48, %%mm7               \n\t"
00419                 "movd %2, %%mm6                 \n\t"
00420                 "packssdw %%mm6, %%mm6          \n\t"
00421                 "packssdw %%mm6, %%mm6          \n\t"
00422                 "mov %3, %%"REG_a"              \n\t"
00423                 ".p2align 4                     \n\t"
00424                 "1:                             \n\t"
00425                 "movq (%0, %%"REG_a"), %%mm0    \n\t"
00426                 "movq 8(%0, %%"REG_a"), %%mm1   \n\t"
00427                 "movq (%1, %%"REG_a"), %%mm4    \n\t"
00428                 "movq 8(%1, %%"REG_a"), %%mm5   \n\t"
00429                 "pmullw %%mm6, %%mm4            \n\t" // q=qscale*quant_matrix[i]
00430                 "pmullw %%mm6, %%mm5            \n\t" // q=qscale*quant_matrix[i]
00431                 "pxor %%mm2, %%mm2              \n\t"
00432                 "pxor %%mm3, %%mm3              \n\t"
00433                 "pcmpgtw %%mm0, %%mm2           \n\t" // block[i] < 0 ? -1 : 0
00434                 "pcmpgtw %%mm1, %%mm3           \n\t" // block[i] < 0 ? -1 : 0
00435                 "pxor %%mm2, %%mm0              \n\t"
00436                 "pxor %%mm3, %%mm1              \n\t"
00437                 "psubw %%mm2, %%mm0             \n\t" // abs(block[i])
00438                 "psubw %%mm3, %%mm1             \n\t" // abs(block[i])
00439                 "paddw %%mm0, %%mm0             \n\t" // abs(block[i])*2
00440                 "paddw %%mm1, %%mm1             \n\t" // abs(block[i])*2
00441                 "pmullw %%mm4, %%mm0            \n\t" // abs(block[i])*2*q
00442                 "pmullw %%mm5, %%mm1            \n\t" // abs(block[i])*2*q
00443                 "paddw %%mm4, %%mm0             \n\t" // (abs(block[i])*2 + 1)*q
00444                 "paddw %%mm5, %%mm1             \n\t" // (abs(block[i])*2 + 1)*q
00445                 "pxor %%mm4, %%mm4              \n\t"
00446                 "pxor %%mm5, %%mm5              \n\t" // FIXME slow
00447                 "pcmpeqw (%0, %%"REG_a"), %%mm4 \n\t" // block[i] == 0 ? -1 : 0
00448                 "pcmpeqw 8(%0, %%"REG_a"), %%mm5\n\t" // block[i] == 0 ? -1 : 0
00449                 "psrlw $4, %%mm0                \n\t"
00450                 "psrlw $4, %%mm1                \n\t"
00451                 "pxor %%mm2, %%mm0              \n\t"
00452                 "pxor %%mm3, %%mm1              \n\t"
00453                 "psubw %%mm2, %%mm0             \n\t"
00454                 "psubw %%mm3, %%mm1             \n\t"
00455                 "pandn %%mm0, %%mm4             \n\t"
00456                 "pandn %%mm1, %%mm5             \n\t"
00457                 "pxor %%mm4, %%mm7              \n\t"
00458                 "pxor %%mm5, %%mm7              \n\t"
00459                 "movq %%mm4, (%0, %%"REG_a")    \n\t"
00460                 "movq %%mm5, 8(%0, %%"REG_a")   \n\t"
00461 
00462                 "add $16, %%"REG_a"             \n\t"
00463                 "jng 1b                         \n\t"
00464                 "movd 124(%0, %3), %%mm0        \n\t"
00465                 "movq %%mm7, %%mm6              \n\t"
00466                 "psrlq $32, %%mm7               \n\t"
00467                 "pxor %%mm6, %%mm7              \n\t"
00468                 "movq %%mm7, %%mm6              \n\t"
00469                 "psrlq $16, %%mm7               \n\t"
00470                 "pxor %%mm6, %%mm7              \n\t"
00471                 "pslld $31, %%mm7               \n\t"
00472                 "psrlq $15, %%mm7               \n\t"
00473                 "pxor %%mm7, %%mm0              \n\t"
00474                 "movd %%mm0, 124(%0, %3)        \n\t"
00475 
00476                 ::"r" (block+nCoeffs), "r"(quant_matrix+nCoeffs), "rm" (qscale), "r" (-2*nCoeffs)
00477                 : "%"REG_a, "memory"
00478         );
00479 }
00480 
00481 static void  denoise_dct_mmx(MpegEncContext *s, DCTELEM *block){
00482     const int intra= s->mb_intra;
00483     int *sum= s->dct_error_sum[intra];
00484     uint16_t *offset= s->dct_offset[intra];
00485 
00486     s->dct_count[intra]++;
00487 
00488     __asm__ volatile(
00489         "pxor %%mm7, %%mm7                      \n\t"
00490         "1:                                     \n\t"
00491         "pxor %%mm0, %%mm0                      \n\t"
00492         "pxor %%mm1, %%mm1                      \n\t"
00493         "movq (%0), %%mm2                       \n\t"
00494         "movq 8(%0), %%mm3                      \n\t"
00495         "pcmpgtw %%mm2, %%mm0                   \n\t"
00496         "pcmpgtw %%mm3, %%mm1                   \n\t"
00497         "pxor %%mm0, %%mm2                      \n\t"
00498         "pxor %%mm1, %%mm3                      \n\t"
00499         "psubw %%mm0, %%mm2                     \n\t"
00500         "psubw %%mm1, %%mm3                     \n\t"
00501         "movq %%mm2, %%mm4                      \n\t"
00502         "movq %%mm3, %%mm5                      \n\t"
00503         "psubusw (%2), %%mm2                    \n\t"
00504         "psubusw 8(%2), %%mm3                   \n\t"
00505         "pxor %%mm0, %%mm2                      \n\t"
00506         "pxor %%mm1, %%mm3                      \n\t"
00507         "psubw %%mm0, %%mm2                     \n\t"
00508         "psubw %%mm1, %%mm3                     \n\t"
00509         "movq %%mm2, (%0)                       \n\t"
00510         "movq %%mm3, 8(%0)                      \n\t"
00511         "movq %%mm4, %%mm2                      \n\t"
00512         "movq %%mm5, %%mm3                      \n\t"
00513         "punpcklwd %%mm7, %%mm4                 \n\t"
00514         "punpckhwd %%mm7, %%mm2                 \n\t"
00515         "punpcklwd %%mm7, %%mm5                 \n\t"
00516         "punpckhwd %%mm7, %%mm3                 \n\t"
00517         "paddd (%1), %%mm4                      \n\t"
00518         "paddd 8(%1), %%mm2                     \n\t"
00519         "paddd 16(%1), %%mm5                    \n\t"
00520         "paddd 24(%1), %%mm3                    \n\t"
00521         "movq %%mm4, (%1)                       \n\t"
00522         "movq %%mm2, 8(%1)                      \n\t"
00523         "movq %%mm5, 16(%1)                     \n\t"
00524         "movq %%mm3, 24(%1)                     \n\t"
00525         "add $16, %0                            \n\t"
00526         "add $32, %1                            \n\t"
00527         "add $16, %2                            \n\t"
00528         "cmp %3, %0                             \n\t"
00529             " jb 1b                             \n\t"
00530         : "+r" (block), "+r" (sum), "+r" (offset)
00531         : "r"(block+64)
00532     );
00533 }
00534 
00535 static void  denoise_dct_sse2(MpegEncContext *s, DCTELEM *block){
00536     const int intra= s->mb_intra;
00537     int *sum= s->dct_error_sum[intra];
00538     uint16_t *offset= s->dct_offset[intra];
00539 
00540     s->dct_count[intra]++;
00541 
00542     __asm__ volatile(
00543         "pxor %%xmm7, %%xmm7                    \n\t"
00544         "1:                                     \n\t"
00545         "pxor %%xmm0, %%xmm0                    \n\t"
00546         "pxor %%xmm1, %%xmm1                    \n\t"
00547         "movdqa (%0), %%xmm2                    \n\t"
00548         "movdqa 16(%0), %%xmm3                  \n\t"
00549         "pcmpgtw %%xmm2, %%xmm0                 \n\t"
00550         "pcmpgtw %%xmm3, %%xmm1                 \n\t"
00551         "pxor %%xmm0, %%xmm2                    \n\t"
00552         "pxor %%xmm1, %%xmm3                    \n\t"
00553         "psubw %%xmm0, %%xmm2                   \n\t"
00554         "psubw %%xmm1, %%xmm3                   \n\t"
00555         "movdqa %%xmm2, %%xmm4                  \n\t"
00556         "movdqa %%xmm3, %%xmm5                  \n\t"
00557         "psubusw (%2), %%xmm2                   \n\t"
00558         "psubusw 16(%2), %%xmm3                 \n\t"
00559         "pxor %%xmm0, %%xmm2                    \n\t"
00560         "pxor %%xmm1, %%xmm3                    \n\t"
00561         "psubw %%xmm0, %%xmm2                   \n\t"
00562         "psubw %%xmm1, %%xmm3                   \n\t"
00563         "movdqa %%xmm2, (%0)                    \n\t"
00564         "movdqa %%xmm3, 16(%0)                  \n\t"
00565         "movdqa %%xmm4, %%xmm6                  \n\t"
00566         "movdqa %%xmm5, %%xmm0                  \n\t"
00567         "punpcklwd %%xmm7, %%xmm4               \n\t"
00568         "punpckhwd %%xmm7, %%xmm6               \n\t"
00569         "punpcklwd %%xmm7, %%xmm5               \n\t"
00570         "punpckhwd %%xmm7, %%xmm0               \n\t"
00571         "paddd (%1), %%xmm4                     \n\t"
00572         "paddd 16(%1), %%xmm6                   \n\t"
00573         "paddd 32(%1), %%xmm5                   \n\t"
00574         "paddd 48(%1), %%xmm0                   \n\t"
00575         "movdqa %%xmm4, (%1)                    \n\t"
00576         "movdqa %%xmm6, 16(%1)                  \n\t"
00577         "movdqa %%xmm5, 32(%1)                  \n\t"
00578         "movdqa %%xmm0, 48(%1)                  \n\t"
00579         "add $32, %0                            \n\t"
00580         "add $64, %1                            \n\t"
00581         "add $32, %2                            \n\t"
00582         "cmp %3, %0                             \n\t"
00583             " jb 1b                             \n\t"
00584         : "+r" (block), "+r" (sum), "+r" (offset)
00585         : "r"(block+64)
00586           XMM_CLOBBERS_ONLY("%xmm0", "%xmm1", "%xmm2", "%xmm3",
00587                             "%xmm4", "%xmm5", "%xmm6", "%xmm7")
00588     );
00589 }
00590 
00591 #if HAVE_SSSE3
00592 #define HAVE_SSSE3_BAK
00593 #endif
00594 #undef HAVE_SSSE3
00595 #define HAVE_SSSE3 0
00596 
00597 #undef HAVE_SSE2
00598 #undef HAVE_MMXEXT
00599 #define HAVE_SSE2 0
00600 #define HAVE_MMXEXT 0
00601 #define RENAME(a) a ## _MMX
00602 #define RENAMEl(a) a ## _mmx
00603 #include "mpegvideo_mmx_template.c"
00604 
00605 #undef HAVE_MMXEXT
00606 #define HAVE_MMXEXT 1
00607 #undef RENAME
00608 #undef RENAMEl
00609 #define RENAME(a) a ## _MMX2
00610 #define RENAMEl(a) a ## _mmx2
00611 #include "mpegvideo_mmx_template.c"
00612 
00613 #undef HAVE_SSE2
00614 #define HAVE_SSE2 1
00615 #undef RENAME
00616 #undef RENAMEl
00617 #define RENAME(a) a ## _SSE2
00618 #define RENAMEl(a) a ## _sse2
00619 #include "mpegvideo_mmx_template.c"
00620 
00621 #ifdef HAVE_SSSE3_BAK
00622 #undef HAVE_SSSE3
00623 #define HAVE_SSSE3 1
00624 #undef RENAME
00625 #undef RENAMEl
00626 #define RENAME(a) a ## _SSSE3
00627 #define RENAMEl(a) a ## _sse2
00628 #include "mpegvideo_mmx_template.c"
00629 #endif
00630 
00631 #endif /* HAVE_INLINE_ASM */
00632 
00633 void ff_MPV_common_init_mmx(MpegEncContext *s)
00634 {
00635 #if HAVE_INLINE_ASM
00636     int mm_flags = av_get_cpu_flags();
00637 
00638     if (mm_flags & AV_CPU_FLAG_MMX) {
00639         const int dct_algo = s->avctx->dct_algo;
00640 
00641         s->dct_unquantize_h263_intra = dct_unquantize_h263_intra_mmx;
00642         s->dct_unquantize_h263_inter = dct_unquantize_h263_inter_mmx;
00643         s->dct_unquantize_mpeg1_intra = dct_unquantize_mpeg1_intra_mmx;
00644         s->dct_unquantize_mpeg1_inter = dct_unquantize_mpeg1_inter_mmx;
00645         if(!(s->flags & CODEC_FLAG_BITEXACT))
00646             s->dct_unquantize_mpeg2_intra = dct_unquantize_mpeg2_intra_mmx;
00647         s->dct_unquantize_mpeg2_inter = dct_unquantize_mpeg2_inter_mmx;
00648 
00649         if (mm_flags & AV_CPU_FLAG_SSE2) {
00650             s->denoise_dct= denoise_dct_sse2;
00651         } else {
00652                 s->denoise_dct= denoise_dct_mmx;
00653         }
00654 
00655         if(dct_algo==FF_DCT_AUTO || dct_algo==FF_DCT_MMX){
00656 #if HAVE_SSSE3
00657             if(mm_flags & AV_CPU_FLAG_SSSE3){
00658                 s->dct_quantize= dct_quantize_SSSE3;
00659             } else
00660 #endif
00661             if(mm_flags & AV_CPU_FLAG_SSE2){
00662                 s->dct_quantize= dct_quantize_SSE2;
00663             } else if (mm_flags & AV_CPU_FLAG_MMXEXT) {
00664                 s->dct_quantize= dct_quantize_MMX2;
00665             } else {
00666                 s->dct_quantize= dct_quantize_MMX;
00667             }
00668         }
00669     }
00670 #endif /* HAVE_INLINE_ASM */
00671 }
libavcodec/x86/mpegvideo_mmx.c
