FFmpeg
Loading...
Searching...
No Matches
hevcdsp_msa.c
Go to the documentation of this file.
1/*
2 * Copyright (c) 2015 - 2017 Manojkumar Bhosale (Manojkumar.Bhosale@imgtec.com)
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21#include "libavutil/mips/generic_macros_msa.h"
22#include "libavcodec/mips/hevcdsp_mips.h"
23#include "libavcodec/mips/hevc_macros_msa.h"
24
25static const uint8_t ff_hevc_mask_arr[16 * 2] __attribute__((aligned(0x40))) = {
26 /* 8 width cases */
27 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
28 /* 4 width cases */
29 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20
30};
31
32static void hevc_copy_4w_msa(const uint8_t *src, int32_t src_stride,
33 int16_t *dst, int32_t dst_stride,
34 int32_t height)
35{
36 v16i8 zero = { 0 };
37
38 if (2 == height) {
39 v16i8 src0, src1;
40 v8i16 in0;
41
42 LD_SB2(src, src_stride, src0, src1);
43
44 src0 = (v16i8) __msa_ilvr_w((v4i32) src1, (v4i32) src0);
45 in0 = (v8i16) __msa_ilvr_b(zero, src0);
46 in0 <<= 6;
47 ST_D2(in0, 0, 1, dst, dst_stride);
48 } else if (4 == height) {
49 v16i8 src0, src1, src2, src3;
50 v8i16 in0, in1;
51
52 LD_SB4(src, src_stride, src0, src1, src2, src3);
53
54 ILVR_W2_SB(src1, src0, src3, src2, src0, src1);
55 ILVR_B2_SH(zero, src0, zero, src1, in0, in1);
56 in0 <<= 6;
57 in1 <<= 6;
58 ST_D4(in0, in1, 0, 1, 0, 1, dst, dst_stride);
59 } else if (0 == height % 8) {
60 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
61 v8i16 in0, in1, in2, in3;
62 uint32_t loop_cnt;
63
64 for (loop_cnt = (height >> 3); loop_cnt--;) {
65 LD_SB8(src, src_stride,
66 src0, src1, src2, src3, src4, src5, src6, src7);
67 src += (8 * src_stride);
68
69 ILVR_W4_SB(src1, src0, src3, src2, src5, src4, src7, src6,
70 src0, src1, src2, src3);
71 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
72 in0, in1, in2, in3);
73 SLLI_4V(in0, in1, in2, in3, 6);
74 ST_D8(in0, in1, in2, in3, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
75 dst += (8 * dst_stride);
76 }
77 }
78}
79
80static void hevc_copy_6w_msa(const uint8_t *src, int32_t src_stride,
81 int16_t *dst, int32_t dst_stride,
82 int32_t height)
83{
84 uint32_t loop_cnt = (height >> 3);
85 uint32_t res = height & 0x07;
86 v16i8 zero = { 0 };
87 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
88 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
89
90 for (; loop_cnt--; ) {
91 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
92 src += (8 * src_stride);
93
94 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
95 in0, in1, in2, in3);
96 ILVR_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7,
97 in4, in5, in6, in7);
98 SLLI_4V(in0, in1, in2, in3, 6);
99 SLLI_4V(in4, in5, in6, in7, 6);
100 ST12x8_UB(in0, in1, in2, in3, in4, in5, in6, in7, dst, 2 * dst_stride);
101 dst += (8 * dst_stride);
102 }
103 for (; res--; ) {
104 uint64_t out0;
105 uint32_t out1;
106 src0 = LD_SB(src);
107 src += src_stride;
108 in0 = (v8i16)__msa_ilvr_b((v16i8) zero, (v16i8) src0);
109 in0 = in0 << 6;
110 out0 = __msa_copy_u_d((v2i64) in0, 0);
111 out1 = __msa_copy_u_w((v4i32) in0, 2);
112 SD(out0, dst);
113 SW(out1, dst + 4);
114 dst += dst_stride;
115 }
116}
117
118static void hevc_copy_8w_msa(const uint8_t *src, int32_t src_stride,
119 int16_t *dst, int32_t dst_stride,
120 int32_t height)
121{
122 v16i8 zero = { 0 };
123
124 if (2 == height) {
125 v16i8 src0, src1;
126 v8i16 in0, in1;
127
128 LD_SB2(src, src_stride, src0, src1);
129
130 ILVR_B2_SH(zero, src0, zero, src1, in0, in1);
131 in0 <<= 6;
132 in1 <<= 6;
133 ST_SH2(in0, in1, dst, dst_stride);
134 } else if (4 == height) {
135 v16i8 src0, src1, src2, src3;
136 v8i16 in0, in1, in2, in3;
137
138 LD_SB4(src, src_stride, src0, src1, src2, src3);
139
140 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
141 in0, in1, in2, in3);
142 SLLI_4V(in0, in1, in2, in3, 6);
143 ST_SH4(in0, in1, in2, in3, dst, dst_stride);
144 } else if (6 == height) {
145 v16i8 src0, src1, src2, src3, src4, src5;
146 v8i16 in0, in1, in2, in3, in4, in5;
147
148 LD_SB6(src, src_stride, src0, src1, src2, src3, src4, src5);
149
150 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
151 in0, in1, in2, in3);
152 ILVR_B2_SH(zero, src4, zero, src5, in4, in5);
153 SLLI_4V(in0, in1, in2, in3, 6);
154 in4 <<= 6;
155 in5 <<= 6;
156 ST_SH6(in0, in1, in2, in3, in4, in5, dst, dst_stride);
157 } else if (0 == height % 8) {
158 uint32_t loop_cnt;
159 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
160 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
161
162 for (loop_cnt = (height >> 3); loop_cnt--;) {
163 LD_SB8(src, src_stride,
164 src0, src1, src2, src3, src4, src5, src6, src7);
165 src += (8 * src_stride);
166
167 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
168 in0, in1, in2, in3);
169 ILVR_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7,
170 in4, in5, in6, in7);
171 SLLI_4V(in0, in1, in2, in3, 6);
172 SLLI_4V(in4, in5, in6, in7, 6);
173 ST_SH8(in0, in1, in2, in3, in4, in5, in6, in7, dst, dst_stride);
174 dst += (8 * dst_stride);
175 }
176 }
177}
178
179static void hevc_copy_12w_msa(const uint8_t *src, int32_t src_stride,
180 int16_t *dst, int32_t dst_stride,
181 int32_t height)
182{
183 uint32_t loop_cnt;
184 uint32_t res = height & 0x07;
185 v16i8 zero = { 0 };
186 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
187 v8i16 in0, in1, in0_r, in1_r, in2_r, in3_r;
188
189 for (loop_cnt = (height >> 3); loop_cnt--;) {
190 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
191 src += (8 * src_stride);
192
193 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
194 in0_r, in1_r, in2_r, in3_r);
195 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
196 ILVL_W2_SB(src1, src0, src3, src2, src0, src1);
197 ILVR_B2_SH(zero, src0, zero, src1, in0, in1);
198 in0 <<= 6;
199 in1 <<= 6;
200 ST_SH4(in0_r, in1_r, in2_r, in3_r, dst, dst_stride);
201 ST_D4(in0, in1, 0, 1, 0, 1, dst + 8, dst_stride);
202 dst += (4 * dst_stride);
203
204 ILVR_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7,
205 in0_r, in1_r, in2_r, in3_r);
206 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
207 ILVL_W2_SB(src5, src4, src7, src6, src0, src1);
208 ILVR_B2_SH(zero, src0, zero, src1, in0, in1);
209 in0 <<= 6;
210 in1 <<= 6;
211 ST_SH4(in0_r, in1_r, in2_r, in3_r, dst, dst_stride);
212 ST_D4(in0, in1, 0, 1, 0, 1, dst + 8, dst_stride);
213 dst += (4 * dst_stride);
214 }
215 for (; res--; ) {
216 uint64_t out0;
217 src0 = LD_SB(src);
218 src += src_stride;
219 in0_r = (v8i16)__msa_ilvr_b((v16i8) zero, (v16i8) src0);
220 in0 = (v8i16)__msa_ilvl_b((v16i8) zero, (v16i8) src0);
221 in0_r = in0_r << 6;
222 in0 = in0 << 6;
223 ST_UH(in0_r, dst);
224 out0 = __msa_copy_u_d((v2i64) in0, 0);
225 SD(out0, dst + 8);
226 dst += dst_stride;
227 }
228}
229
230static void hevc_copy_16w_msa(const uint8_t *src, int32_t src_stride,
231 int16_t *dst, int32_t dst_stride,
232 int32_t height)
233{
234 v16i8 zero = { 0 };
235
236 if (4 == height) {
237 v16i8 src0, src1, src2, src3;
238 v8i16 in0_r, in1_r, in2_r, in3_r;
239 v8i16 in0_l, in1_l, in2_l, in3_l;
240
241 LD_SB4(src, src_stride, src0, src1, src2, src3);
242
243 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
244 in0_r, in1_r, in2_r, in3_r);
245 ILVL_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
246 in0_l, in1_l, in2_l, in3_l);
247 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
248 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
249 ST_SH4(in0_r, in1_r, in2_r, in3_r, dst, dst_stride);
250 ST_SH4(in0_l, in1_l, in2_l, in3_l, (dst + 8), dst_stride);
251 } else if (12 == height) {
252 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
253 v16i8 src8, src9, src10, src11;
254 v8i16 in0_r, in1_r, in2_r, in3_r;
255 v8i16 in0_l, in1_l, in2_l, in3_l;
256
257 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
258 src += (8 * src_stride);
259 LD_SB4(src, src_stride, src8, src9, src10, src11);
260
261 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
262 in0_r, in1_r, in2_r, in3_r);
263 ILVL_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
264 in0_l, in1_l, in2_l, in3_l);
265 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
266 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
267 ST_SH4(in0_r, in1_r, in2_r, in3_r, dst, dst_stride);
268 ST_SH4(in0_l, in1_l, in2_l, in3_l, (dst + 8), dst_stride);
269 dst += (4 * dst_stride);
270
271 ILVR_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7,
272 in0_r, in1_r, in2_r, in3_r);
273 ILVL_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7,
274 in0_l, in1_l, in2_l, in3_l);
275 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
276 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
277 ST_SH4(in0_r, in1_r, in2_r, in3_r, dst, dst_stride);
278 ST_SH4(in0_l, in1_l, in2_l, in3_l, (dst + 8), dst_stride);
279 dst += (4 * dst_stride);
280
281 ILVR_B4_SH(zero, src8, zero, src9, zero, src10, zero, src11,
282 in0_r, in1_r, in2_r, in3_r);
283 ILVL_B4_SH(zero, src8, zero, src9, zero, src10, zero, src11,
284 in0_l, in1_l, in2_l, in3_l);
285 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
286 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
287 ST_SH4(in0_r, in1_r, in2_r, in3_r, dst, dst_stride);
288 ST_SH4(in0_l, in1_l, in2_l, in3_l, (dst + 8), dst_stride);
289 } else if (0 == (height % 8)) {
290 uint32_t loop_cnt;
291 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
292 v8i16 in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l;
293
294 for (loop_cnt = (height >> 3); loop_cnt--;) {
295 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6,
296 src7);
297 src += (8 * src_stride);
298 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, in0_r,
299 in1_r, in2_r, in3_r);
300 ILVL_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, in0_l,
301 in1_l, in2_l, in3_l);
302 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
303 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
304 ST_SH4(in0_r, in1_r, in2_r, in3_r, dst, dst_stride);
305 ST_SH4(in0_l, in1_l, in2_l, in3_l, (dst + 8), dst_stride);
306 dst += (4 * dst_stride);
307
308 ILVR_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7, in0_r,
309 in1_r, in2_r, in3_r);
310 ILVL_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7, in0_l,
311 in1_l, in2_l, in3_l);
312 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
313 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
314 ST_SH4(in0_r, in1_r, in2_r, in3_r, dst, dst_stride);
315 ST_SH4(in0_l, in1_l, in2_l, in3_l, (dst + 8), dst_stride);
316 dst += (4 * dst_stride);
317 }
318 }
319}
320
321static void hevc_copy_24w_msa(const uint8_t *src, int32_t src_stride,
322 int16_t *dst, int32_t dst_stride,
323 int32_t height)
324{
325 uint32_t loop_cnt;
326 v16i8 zero = { 0 };
327 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
328 v8i16 in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l;
329
330 for (loop_cnt = (height >> 2); loop_cnt--;) {
331 LD_SB4(src, src_stride, src0, src1, src2, src3);
332 LD_SB4((src + 16), src_stride, src4, src5, src6, src7);
333 src += (4 * src_stride);
334 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, in0_r, in1_r,
335 in2_r, in3_r);
336 ILVL_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, in0_l, in1_l,
337 in2_l, in3_l);
338 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
339 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
340 ST_SH4(in0_r, in1_r, in2_r, in3_r, dst, dst_stride);
341 ST_SH4(in0_l, in1_l, in2_l, in3_l, (dst + 8), dst_stride);
342 ILVR_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7, in0_r, in1_r,
343 in2_r, in3_r);
344 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
345 ST_SH4(in0_r, in1_r, in2_r, in3_r, (dst + 16), dst_stride);
346 dst += (4 * dst_stride);
347 }
348}
349
350static void hevc_copy_32w_msa(const uint8_t *src, int32_t src_stride,
351 int16_t *dst, int32_t dst_stride,
352 int32_t height)
353{
354 uint32_t loop_cnt;
355 v16i8 zero = { 0 };
356 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
357 v8i16 in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l;
358
359 for (loop_cnt = (height >> 2); loop_cnt--;) {
360 LD_SB4(src, src_stride, src0, src2, src4, src6);
361 LD_SB4((src + 16), src_stride, src1, src3, src5, src7);
362 src += (4 * src_stride);
363
364 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, in0_r, in1_r,
365 in2_r, in3_r);
366 ILVL_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, in0_l, in1_l,
367 in2_l, in3_l);
368 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
369 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
370 ST_SH4(in0_r, in0_l, in1_r, in1_l, dst, 8);
371 dst += dst_stride;
372 ST_SH4(in2_r, in2_l, in3_r, in3_l, dst, 8);
373 dst += dst_stride;
374
375 ILVR_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7, in0_r, in1_r,
376 in2_r, in3_r);
377 ILVL_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7, in0_l, in1_l,
378 in2_l, in3_l);
379 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
380 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
381 ST_SH4(in0_r, in0_l, in1_r, in1_l, dst, 8);
382 dst += dst_stride;
383 ST_SH4(in2_r, in2_l, in3_r, in3_l, dst, 8);
384 dst += dst_stride;
385 }
386}
387
388static void hevc_copy_48w_msa(const uint8_t *src, int32_t src_stride,
389 int16_t *dst, int32_t dst_stride,
390 int32_t height)
391{
392 uint32_t loop_cnt;
393 v16i8 zero = { 0 };
394 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
395 v16i8 src8, src9, src10, src11;
396 v8i16 in0_r, in1_r, in2_r, in3_r, in4_r, in5_r;
397 v8i16 in0_l, in1_l, in2_l, in3_l, in4_l, in5_l;
398
399 for (loop_cnt = (height >> 2); loop_cnt--;) {
400 LD_SB3(src, 16, src0, src1, src2);
401 src += src_stride;
402 LD_SB3(src, 16, src3, src4, src5);
403 src += src_stride;
404 LD_SB3(src, 16, src6, src7, src8);
405 src += src_stride;
406 LD_SB3(src, 16, src9, src10, src11);
407 src += src_stride;
408
409 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
410 in0_r, in1_r, in2_r, in3_r);
411 ILVL_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
412 in0_l, in1_l, in2_l, in3_l);
413 ILVR_B2_SH(zero, src4, zero, src5, in4_r, in5_r);
414 ILVL_B2_SH(zero, src4, zero, src5, in4_l, in5_l);
415 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
416 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
417 SLLI_4V(in4_r, in5_r, in4_l, in5_l, 6);
418 ST_SH6(in0_r, in0_l, in1_r, in1_l, in2_r, in2_l, dst, 8);
419 dst += dst_stride;
420 ST_SH6(in3_r, in3_l, in4_r, in4_l, in5_r, in5_l, dst, 8);
421 dst += dst_stride;
422
423 ILVR_B4_SH(zero, src6, zero, src7, zero, src8, zero, src9,
424 in0_r, in1_r, in2_r, in3_r);
425 ILVL_B4_SH(zero, src6, zero, src7, zero, src8, zero, src9,
426 in0_l, in1_l, in2_l, in3_l);
427 ILVR_B2_SH(zero, src10, zero, src11, in4_r, in5_r);
428 ILVL_B2_SH(zero, src10, zero, src11, in4_l, in5_l);
429 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
430 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
431 SLLI_4V(in4_r, in5_r, in4_l, in5_l, 6);
432 ST_SH6(in0_r, in0_l, in1_r, in1_l, in2_r, in2_l, dst, 8);
433 dst += dst_stride;
434 ST_SH6(in3_r, in3_l, in4_r, in4_l, in5_r, in5_l, dst, 8);
435 dst += dst_stride;
436 }
437}
438
439static void hevc_copy_64w_msa(const uint8_t *src, int32_t src_stride,
440 int16_t *dst, int32_t dst_stride,
441 int32_t height)
442{
443 uint32_t loop_cnt;
444 v16i8 zero = { 0 };
445 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
446 v8i16 in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l;
447
448 for (loop_cnt = (height >> 1); loop_cnt--;) {
449 LD_SB4(src, 16, src0, src1, src2, src3);
450 src += src_stride;
451 LD_SB4(src, 16, src4, src5, src6, src7);
452 src += src_stride;
453
454 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
455 in0_r, in1_r, in2_r, in3_r);
456 ILVL_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
457 in0_l, in1_l, in2_l, in3_l);
458 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
459 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
460 ST_SH4(in0_r, in0_l, in1_r, in1_l, dst, 8);
461 ST_SH4(in2_r, in2_l, in3_r, in3_l, (dst + 32), 8);
462 dst += dst_stride;
463
464 ILVR_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7,
465 in0_r, in1_r, in2_r, in3_r);
466 ILVL_B4_SH(zero, src4, zero, src5, zero, src6, zero, src7,
467 in0_l, in1_l, in2_l, in3_l);
468 SLLI_4V(in0_r, in1_r, in2_r, in3_r, 6);
469 SLLI_4V(in0_l, in1_l, in2_l, in3_l, 6);
470 ST_SH4(in0_r, in0_l, in1_r, in1_l, dst, 8);
471 ST_SH4(in2_r, in2_l, in3_r, in3_l, (dst + 32), 8);
472 dst += dst_stride;
473 }
474}
475
476static void hevc_hz_8t_4w_msa(const uint8_t *src, int32_t src_stride,
477 int16_t *dst, int32_t dst_stride,
478 const int8_t *filter, int32_t height)
479{
480 uint32_t loop_cnt;
481 uint32_t res = (height & 0x07) >> 1;
482 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
483 v8i16 filt0, filt1, filt2, filt3;
484 v16i8 mask1, mask2, mask3;
485 v16i8 vec0, vec1, vec2, vec3;
486 v8i16 dst0, dst1, dst2, dst3;
487 v8i16 filter_vec, const_vec;
488 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
489
490 src -= 3;
491 const_vec = __msa_ldi_h(128);
492 const_vec <<= 6;
493
494 filter_vec = LD_SH(filter);
495 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
496
497 mask1 = mask0 + 2;
498 mask2 = mask0 + 4;
499 mask3 = mask0 + 6;
500
501 for (loop_cnt = (height >> 3); loop_cnt--;) {
502 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
503 src += (8 * src_stride);
504 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
505
506 VSHF_B4_SB(src0, src1, mask0, mask1, mask2, mask3,
507 vec0, vec1, vec2, vec3);
508 dst0 = const_vec;
509 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
510 dst0, dst0, dst0, dst0);
511 VSHF_B4_SB(src2, src3, mask0, mask1, mask2, mask3,
512 vec0, vec1, vec2, vec3);
513 dst1 = const_vec;
514 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
515 dst1, dst1, dst1, dst1);
516 VSHF_B4_SB(src4, src5, mask0, mask1, mask2, mask3,
517 vec0, vec1, vec2, vec3);
518 dst2 = const_vec;
519 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
520 dst2, dst2, dst2, dst2);
521 VSHF_B4_SB(src6, src7, mask0, mask1, mask2, mask3,
522 vec0, vec1, vec2, vec3);
523 dst3 = const_vec;
524 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
525 dst3, dst3, dst3, dst3);
526
527 ST_D8(dst0, dst1, dst2, dst3, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
528 dst += (8 * dst_stride);
529 }
530 for (; res--; ) {
531 LD_SB2(src, src_stride, src0, src1);
532 src += 2 * src_stride;
533 XORI_B2_128_SB(src0, src1);
534 VSHF_B4_SB(src0, src1, mask0, mask1, mask2, mask3,
535 vec0, vec1, vec2, vec3);
536 dst0 = const_vec;
537 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
538 dst0, dst0, dst0, dst0);
539 ST_D2(dst0, 0, 1, dst, dst_stride);
540 dst += 2 * dst_stride;
541 }
542}
543
544static void hevc_hz_8t_8w_msa(const uint8_t *src, int32_t src_stride,
545 int16_t *dst, int32_t dst_stride,
546 const int8_t *filter, int32_t height)
547{
548 uint32_t loop_cnt;
549 v16i8 src0, src1, src2, src3;
550 v8i16 filt0, filt1, filt2, filt3;
551 v16i8 mask1, mask2, mask3;
552 v16i8 vec0, vec1, vec2, vec3;
553 v8i16 dst0, dst1, dst2, dst3;
554 v8i16 filter_vec, const_vec;
555 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
556
557 src -= 3;
558 const_vec = __msa_ldi_h(128);
559 const_vec <<= 6;
560
561 filter_vec = LD_SH(filter);
562 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
563
564 mask1 = mask0 + 2;
565 mask2 = mask0 + 4;
566 mask3 = mask0 + 6;
567
568 for (loop_cnt = (height >> 2); loop_cnt--;) {
569 LD_SB4(src, src_stride, src0, src1, src2, src3);
570 src += (4 * src_stride);
571 XORI_B4_128_SB(src0, src1, src2, src3);
572
573 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
574 vec0, vec1, vec2, vec3);
575 dst0 = const_vec;
576 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
577 dst0, dst0, dst0, dst0);
578 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
579 vec0, vec1, vec2, vec3);
580 dst1 = const_vec;
581 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
582 dst1, dst1, dst1, dst1);
583 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
584 vec0, vec1, vec2, vec3);
585 dst2 = const_vec;
586 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
587 dst2, dst2, dst2, dst2);
588 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
589 vec0, vec1, vec2, vec3);
590 dst3 = const_vec;
591 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
592 dst3, dst3, dst3, dst3);
593
594 ST_SH4(dst0, dst1, dst2, dst3, dst, dst_stride);
595 dst += (4 * dst_stride);
596 }
597}
598
599static void hevc_hz_8t_12w_msa(const uint8_t *src, int32_t src_stride,
600 int16_t *dst, int32_t dst_stride,
601 const int8_t *filter, int32_t height)
602{
603 uint32_t loop_cnt;
604 int64_t res0, res1, res2, res3;
605 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
606 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
607 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
608 v8i16 filt0, filt1, filt2, filt3, dst0, dst1, dst2, dst3, dst4, dst5;
609 v8i16 filter_vec, const_vec;
610
611 src -= 3;
612 const_vec = __msa_ldi_h(128);
613 const_vec <<= 6;
614
615 filter_vec = LD_SH(filter);
616 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
617
618 mask0 = LD_SB(ff_hevc_mask_arr);
619 mask1 = mask0 + 2;
620 mask2 = mask0 + 4;
621 mask3 = mask0 + 6;
622 mask4 = LD_SB(ff_hevc_mask_arr + 16);
623 mask5 = mask4 + 2;
624 mask6 = mask4 + 4;
625 mask7 = mask4 + 6;
626
627 for (loop_cnt = 4; loop_cnt--;) {
628 LD_SB4(src, src_stride, src0, src1, src2, src3);
629 LD_SB4(src + 8, src_stride, src4, src5, src6, src7);
630 src += (4 * src_stride);
631 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
632
633 dst0 = const_vec;
634 dst1 = const_vec;
635 dst2 = const_vec;
636 dst3 = const_vec;
637 dst4 = const_vec;
638 dst5 = const_vec;
639 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
640 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
641 VSHF_B2_SB(src4, src5, src6, src7, mask4, mask4, vec4, vec5);
642 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
643 dst1, dst2, dst3);
644 DPADD_SB2_SH(vec4, vec5, filt0, filt0, dst4, dst5);
645 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0, vec1);
646 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2, vec3);
647 VSHF_B2_SB(src4, src5, src6, src7, mask5, mask5, vec4, vec5);
648 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
649 dst1, dst2, dst3);
650 DPADD_SB2_SH(vec4, vec5, filt1, filt1, dst4, dst5);
651 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec0, vec1);
652 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec2, vec3);
653 VSHF_B2_SB(src4, src5, src6, src7, mask6, mask6, vec4, vec5);
654 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
655 dst1, dst2, dst3);
656 DPADD_SB2_SH(vec4, vec5, filt2, filt2, dst4, dst5);
657 VSHF_B2_SB(src0, src0, src1, src1, mask3, mask3, vec0, vec1);
658 VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec2, vec3);
659 VSHF_B2_SB(src4, src5, src6, src7, mask7, mask7, vec4, vec5);
660 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
661 dst1, dst2, dst3);
662 DPADD_SB2_SH(vec4, vec5, filt3, filt3, dst4, dst5);
663
664 res0 = __msa_copy_s_d((v2i64) dst4, 0);
665 res1 = __msa_copy_s_d((v2i64) dst4, 1);
666 res2 = __msa_copy_s_d((v2i64) dst5, 0);
667 res3 = __msa_copy_s_d((v2i64) dst5, 1);
668 ST_SH4(dst0, dst1, dst2, dst3, dst, dst_stride);
669 SD4(res0, res1, res2, res3, (dst + 8), dst_stride);
670 dst += (4 * dst_stride);
671 }
672}
673
674static void hevc_hz_8t_16w_msa(const uint8_t *src, int32_t src_stride,
675 int16_t *dst, int32_t dst_stride,
676 const int8_t *filter, int32_t height)
677{
678 uint32_t loop_cnt;
679 v16i8 src0, src1, src2, src3;
680 v8i16 filt0, filt1, filt2, filt3;
681 v16i8 mask1, mask2, mask3;
682 v16i8 vec0, vec1, vec2, vec3;
683 v8i16 dst0, dst1, dst2, dst3;
684 v8i16 filter_vec, const_vec;
685 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
686
687 src -= 3;
688 const_vec = __msa_ldi_h(128);
689 const_vec <<= 6;
690
691 filter_vec = LD_SH(filter);
692 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
693
694 mask1 = mask0 + 2;
695 mask2 = mask0 + 4;
696 mask3 = mask0 + 6;
697
698 for (loop_cnt = (height >> 1); loop_cnt--;) {
699 LD_SB2(src, src_stride, src0, src2);
700 LD_SB2(src + 8, src_stride, src1, src3);
701 src += (2 * src_stride);
702 XORI_B4_128_SB(src0, src1, src2, src3);
703
704 dst0 = const_vec;
705 dst1 = const_vec;
706 dst2 = const_vec;
707 dst3 = const_vec;
708 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
709 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
710 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
711 dst1, dst2, dst3);
712 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0, vec1);
713 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2, vec3);
714 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
715 dst1, dst2, dst3);
716 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec0, vec1);
717 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec2, vec3);
718 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
719 dst1, dst2, dst3);
720 VSHF_B2_SB(src0, src0, src1, src1, mask3, mask3, vec0, vec1);
721 VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec2, vec3);
722 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
723 dst1, dst2, dst3);
724
725 ST_SH2(dst0, dst2, dst, dst_stride);
726 ST_SH2(dst1, dst3, dst + 8, dst_stride);
727 dst += (2 * dst_stride);
728 }
729}
730
731static void hevc_hz_8t_24w_msa(const uint8_t *src, int32_t src_stride,
732 int16_t *dst, int32_t dst_stride,
733 const int8_t *filter, int32_t height)
734{
735 uint32_t loop_cnt;
736 v16i8 src0, src1, src2, src3;
737 v8i16 filt0, filt1, filt2, filt3;
738 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
739 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
740 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
741 v8i16 filter_vec, const_vec;
742 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
743
744 src -= 3;
745 filter_vec = LD_SH(filter);
746 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
747
748 mask1 = mask0 + 2;
749 mask2 = mask0 + 4;
750 mask3 = mask0 + 6;
751 mask4 = mask0 + 8;
752 mask5 = mask0 + 10;
753 mask6 = mask0 + 12;
754 mask7 = mask0 + 14;
755
756 const_vec = __msa_ldi_h(128);
757 const_vec <<= 6;
758
759 for (loop_cnt = (height >> 1); loop_cnt--;) {
760 LD_SB2(src, 16, src0, src1);
761 src += src_stride;
762 LD_SB2(src, 16, src2, src3);
763 src += src_stride;
764 XORI_B4_128_SB(src0, src1, src2, src3);
765
766 dst0 = const_vec;
767 dst1 = const_vec;
768 dst2 = const_vec;
769 dst3 = const_vec;
770 dst4 = const_vec;
771 dst5 = const_vec;
772 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask4, vec0, vec1);
773 VSHF_B2_SB(src1, src1, src2, src2, mask0, mask0, vec2, vec3);
774 VSHF_B2_SB(src2, src3, src3, src3, mask4, mask0, vec4, vec5);
775 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
776 dst1, dst2, dst3);
777 DPADD_SB2_SH(vec4, vec5, filt0, filt0, dst4, dst5);
778 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask5, vec0, vec1);
779 VSHF_B2_SB(src1, src1, src2, src2, mask1, mask1, vec2, vec3);
780 VSHF_B2_SB(src2, src3, src3, src3, mask5, mask1, vec4, vec5);
781 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
782 dst1, dst2, dst3);
783 DPADD_SB2_SH(vec4, vec5, filt1, filt1, dst4, dst5);
784 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask6, vec0, vec1);
785 VSHF_B2_SB(src1, src1, src2, src2, mask2, mask2, vec2, vec3);
786 VSHF_B2_SB(src2, src3, src3, src3, mask6, mask2, vec4, vec5);
787 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
788 dst1, dst2, dst3);
789 DPADD_SB2_SH(vec4, vec5, filt2, filt2, dst4, dst5);
790 VSHF_B2_SB(src0, src0, src0, src1, mask3, mask7, vec0, vec1);
791 VSHF_B2_SB(src1, src1, src2, src2, mask3, mask3, vec2, vec3);
792 VSHF_B2_SB(src2, src3, src3, src3, mask7, mask3, vec4, vec5);
793 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
794 dst1, dst2, dst3);
795 DPADD_SB2_SH(vec4, vec5, filt3, filt3, dst4, dst5);
796
797 ST_SH2(dst0, dst1, dst, 8);
798 ST_SH(dst2, dst + 16);
799 dst += dst_stride;
800 ST_SH2(dst3, dst4, dst, 8);
801 ST_SH(dst5, dst + 16);
802 dst += dst_stride;
803 }
804}
805
806static void hevc_hz_8t_32w_msa(const uint8_t *src, int32_t src_stride,
807 int16_t *dst, int32_t dst_stride,
808 const int8_t *filter, int32_t height)
809{
810 uint32_t loop_cnt;
811 v16i8 src0, src1, src2;
812 v8i16 filt0, filt1, filt2, filt3;
813 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
814 v16i8 vec0, vec1, vec2, vec3;
815 v8i16 dst0, dst1, dst2, dst3;
816 v8i16 filter_vec, const_vec;
817 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
818
819 src -= 3;
820 filter_vec = LD_SH(filter);
821 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
822
823 mask1 = mask0 + 2;
824 mask2 = mask0 + 4;
825 mask3 = mask0 + 6;
826 mask4 = mask0 + 8;
827 mask5 = mask0 + 10;
828 mask6 = mask0 + 12;
829 mask7 = mask0 + 14;
830
831 const_vec = __msa_ldi_h(128);
832 const_vec <<= 6;
833
834 for (loop_cnt = height; loop_cnt--;) {
835 LD_SB2(src, 16, src0, src1);
836 src2 = LD_SB(src + 24);
837 src += src_stride;
838 XORI_B3_128_SB(src0, src1, src2);
839
840 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
841 vec0, vec1, vec2, vec3);
842 dst0 = const_vec;
843 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
844 dst0, dst0, dst0, dst0);
845 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
846 vec0, vec1, vec2, vec3);
847 dst1 = const_vec;
848 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
849 dst1, dst1, dst1, dst1);
850 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
851 vec0, vec1, vec2, vec3);
852 dst2 = const_vec;
853 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
854 dst2, dst2, dst2, dst2);
855 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
856 vec0, vec1, vec2, vec3);
857 dst3 = const_vec;
858 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
859 dst3, dst3, dst3, dst3);
860
861 ST_SH4(dst0, dst1, dst2, dst3, dst, 8);
862 dst += dst_stride;
863 }
864}
865
866static void hevc_hz_8t_48w_msa(const uint8_t *src, int32_t src_stride,
867 int16_t *dst, int32_t dst_stride,
868 const int8_t *filter, int32_t height)
869{
870 uint32_t loop_cnt;
871 v16i8 src0, src1, src2, src3;
872 v8i16 filt0, filt1, filt2, filt3;
873 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
874 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
875 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
876 v8i16 filter_vec, const_vec;
877 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
878
879 src -= 3;
880 filter_vec = LD_SH(filter);
881 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
882
883 mask1 = mask0 + 2;
884 mask2 = mask0 + 4;
885 mask3 = mask0 + 6;
886 mask4 = mask0 + 8;
887 mask5 = mask0 + 10;
888 mask6 = mask0 + 12;
889 mask7 = mask0 + 14;
890
891 const_vec = __msa_ldi_h(128);
892 const_vec <<= 6;
893
894 for (loop_cnt = height; loop_cnt--;) {
895 LD_SB3(src, 16, src0, src1, src2);
896 src3 = LD_SB(src + 40);
897 src += src_stride;
898 XORI_B4_128_SB(src0, src1, src2, src3);
899
900 dst0 = const_vec;
901 dst1 = const_vec;
902 dst2 = const_vec;
903 dst3 = const_vec;
904 dst4 = const_vec;
905 dst5 = const_vec;
906 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask4, vec0, vec1);
907 VSHF_B2_SB(src1, src1, src1, src2, mask0, mask4, vec2, vec3);
908 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
909 dst1, dst2, dst3);
910 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask5, vec0, vec1);
911 VSHF_B2_SB(src1, src1, src1, src2, mask1, mask5, vec2, vec3);
912 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
913 dst1, dst2, dst3);
914 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask6, vec0, vec1);
915 VSHF_B2_SB(src1, src1, src1, src2, mask2, mask6, vec2, vec3);
916 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
917 dst1, dst2, dst3);
918 VSHF_B2_SB(src0, src0, src0, src1, mask3, mask7, vec0, vec1);
919 VSHF_B2_SB(src1, src1, src1, src2, mask3, mask7, vec2, vec3);
920 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
921 dst1, dst2, dst3);
922 ST_SH4(dst0, dst1, dst2, dst3, dst, 8);
923
924 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec4, vec5);
925 DPADD_SB2_SH(vec4, vec5, filt0, filt0, dst4, dst5);
926 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec4, vec5);
927 DPADD_SB2_SH(vec4, vec5, filt1, filt1, dst4, dst5);
928 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec4, vec5);
929 DPADD_SB2_SH(vec4, vec5, filt2, filt2, dst4, dst5);
930 VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec4, vec5);
931 DPADD_SB2_SH(vec4, vec5, filt3, filt3, dst4, dst5);
932 ST_SH2(dst4, dst5, (dst + 32), 8);
933 dst += dst_stride;
934 }
935}
936
937static void hevc_hz_8t_64w_msa(const uint8_t *src, int32_t src_stride,
938 int16_t *dst, int32_t dst_stride,
939 const int8_t *filter, int32_t height)
940{
941 uint32_t loop_cnt;
942 v16i8 src0, src1, src2, src3, src4;
943 v8i16 filt0, filt1, filt2, filt3;
944 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
945 v16i8 vec0, vec1, vec2, vec3;
946 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
947 v8i16 filter_vec, const_vec;
948 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
949
950 src -= 3;
951
952 filter_vec = LD_SH(filter);
953 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
954
955 mask1 = mask0 + 2;
956 mask2 = mask0 + 4;
957 mask3 = mask0 + 6;
958 mask4 = mask0 + 8;
959 mask5 = mask0 + 10;
960 mask6 = mask0 + 12;
961 mask7 = mask0 + 14;
962
963 const_vec = __msa_ldi_h(128);
964 const_vec <<= 6;
965
966 for (loop_cnt = height; loop_cnt--;) {
967 LD_SB4(src, 16, src0, src1, src2, src3);
968 src4 = LD_SB(src + 56);
969 src += src_stride;
970 XORI_B5_128_SB(src0, src1, src2, src3, src4);
971
972 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
973 vec0, vec1, vec2, vec3);
974 dst0 = const_vec;
975 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
976 dst0, dst0, dst0, dst0);
977 ST_SH(dst0, dst);
978
979 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
980 vec0, vec1, vec2, vec3);
981 dst1 = const_vec;
982 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
983 dst1, dst1, dst1, dst1);
984 ST_SH(dst1, dst + 8);
985
986 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
987 vec0, vec1, vec2, vec3);
988 dst2 = const_vec;
989 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
990 dst2, dst2, dst2, dst2);
991 ST_SH(dst2, dst + 16);
992
993 VSHF_B4_SB(src1, src2, mask4, mask5, mask6, mask7,
994 vec0, vec1, vec2, vec3);
995 dst3 = const_vec;
996 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
997 dst3, dst3, dst3, dst3);
998 ST_SH(dst3, dst + 24);
999
1000 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1001 vec0, vec1, vec2, vec3);
1002 dst4 = const_vec;
1003 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
1004 dst4, dst4, dst4, dst4);
1005 ST_SH(dst4, dst + 32);
1006
1007 VSHF_B4_SB(src2, src3, mask4, mask5, mask6, mask7,
1008 vec0, vec1, vec2, vec3);
1009 dst5 = const_vec;
1010 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
1011 dst5, dst5, dst5, dst5);
1012 ST_SH(dst5, dst + 40);
1013
1014 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
1015 vec0, vec1, vec2, vec3);
1016 dst6 = const_vec;
1017 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
1018 dst6, dst6, dst6, dst6);
1019 ST_SH(dst6, dst + 48);
1020
1021 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3,
1022 vec0, vec1, vec2, vec3);
1023 dst7 = const_vec;
1024 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
1025 dst7, dst7, dst7, dst7);
1026 ST_SH(dst7, dst + 56);
1027 dst += dst_stride;
1028 }
1029}
1030
1031static void hevc_vt_8t_4w_msa(const uint8_t *src, int32_t src_stride,
1032 int16_t *dst, int32_t dst_stride,
1033 const int8_t *filter, int32_t height)
1034{
1035 int32_t loop_cnt;
1036 int32_t res = (height & 0x07) >> 1;
1037 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
1038 v16i8 src9, src10, src11, src12, src13, src14;
1039 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1040 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1041 v16i8 src1110_r, src1211_r, src1312_r, src1413_r;
1042 v16i8 src2110, src4332, src6554, src8776, src10998;
1043 v16i8 src12111110, src14131312;
1044 v8i16 dst10, dst32, dst54, dst76;
1045 v8i16 filt0, filt1, filt2, filt3;
1046 v8i16 filter_vec, const_vec;
1047
1048 src -= (3 * src_stride);
1049
1050 const_vec = __msa_ldi_h(128);
1051 const_vec <<= 6;
1052
1053 filter_vec = LD_SH(filter);
1054 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1055
1056 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1057 src += (7 * src_stride);
1058 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1059 src10_r, src32_r, src54_r, src21_r);
1060 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1061 ILVR_D3_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r,
1062 src2110, src4332, src6554);
1063 XORI_B3_128_SB(src2110, src4332, src6554);
1064
1065 for (loop_cnt = (height >> 3); loop_cnt--;) {
1066 LD_SB8(src, src_stride,
1067 src7, src8, src9, src10, src11, src12, src13, src14);
1068 src += (8 * src_stride);
1069
1070 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1071 src76_r, src87_r, src98_r, src109_r);
1072 ILVR_B4_SB(src11, src10, src12, src11, src13, src12, src14, src13,
1073 src1110_r, src1211_r, src1312_r, src1413_r);
1074 ILVR_D4_SB(src87_r, src76_r, src109_r, src98_r,
1075 src1211_r, src1110_r, src1413_r, src1312_r,
1076 src8776, src10998, src12111110, src14131312);
1077 XORI_B4_128_SB(src8776, src10998, src12111110, src14131312);
1078
1079 dst10 = const_vec;
1080 DPADD_SB4_SH(src2110, src4332, src6554, src8776,
1081 filt0, filt1, filt2, filt3, dst10, dst10, dst10, dst10);
1082 dst32 = const_vec;
1083 DPADD_SB4_SH(src4332, src6554, src8776, src10998,
1084 filt0, filt1, filt2, filt3, dst32, dst32, dst32, dst32);
1085 dst54 = const_vec;
1086 DPADD_SB4_SH(src6554, src8776, src10998, src12111110,
1087 filt0, filt1, filt2, filt3, dst54, dst54, dst54, dst54);
1088 dst76 = const_vec;
1089 DPADD_SB4_SH(src8776, src10998, src12111110, src14131312,
1090 filt0, filt1, filt2, filt3, dst76, dst76, dst76, dst76);
1091
1092 ST_D8(dst10, dst32, dst54, dst76, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
1093 dst += (8 * dst_stride);
1094
1095 src2110 = src10998;
1096 src4332 = src12111110;
1097 src6554 = src14131312;
1098 src6 = src14;
1099 }
1100 for (; res--; ) {
1101 LD_SB2(src, src_stride, src7, src8);
1102 src += 2 * src_stride;
1103 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
1104 src8776 = (v16i8)__msa_ilvr_d((v2i64) src87_r, src76_r);
1105 src8776 = (v16i8)__msa_xori_b((v16i8) src8776, 128);
1106 dst10 = const_vec;
1107 DPADD_SB4_SH(src2110, src4332, src6554, src8776,
1108 filt0, filt1, filt2, filt3, dst10, dst10, dst10, dst10);
1109 ST_D2(dst10, 0, 1, dst, dst_stride);
1110 dst += 2 * dst_stride;
1111 src2110 = src4332;
1112 src4332 = src6554;
1113 src6554 = src8776;
1114 src6 = src8;
1115 }
1116}
1117
1118static void hevc_vt_8t_8w_msa(const uint8_t *src, int32_t src_stride,
1119 int16_t *dst, int32_t dst_stride,
1120 const int8_t *filter, int32_t height)
1121{
1122 int32_t loop_cnt;
1123 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1124 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1125 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1126 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
1127 v8i16 filter_vec, const_vec;
1128 v8i16 filt0, filt1, filt2, filt3;
1129
1130 src -= (3 * src_stride);
1131 const_vec = __msa_ldi_h(128);
1132 const_vec <<= 6;
1133
1134 filter_vec = LD_SH(filter);
1135 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1136
1137 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1138 src += (7 * src_stride);
1139 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1140 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1141 src10_r, src32_r, src54_r, src21_r);
1142 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1143
1144 for (loop_cnt = (height >> 2); loop_cnt--;) {
1145 LD_SB4(src, src_stride, src7, src8, src9, src10);
1146 src += (4 * src_stride);
1147 XORI_B4_128_SB(src7, src8, src9, src10);
1148 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1149 src76_r, src87_r, src98_r, src109_r);
1150
1151 dst0_r = const_vec;
1152 DPADD_SB4_SH(src10_r, src32_r, src54_r, src76_r,
1153 filt0, filt1, filt2, filt3,
1154 dst0_r, dst0_r, dst0_r, dst0_r);
1155 dst1_r = const_vec;
1156 DPADD_SB4_SH(src21_r, src43_r, src65_r, src87_r,
1157 filt0, filt1, filt2, filt3,
1158 dst1_r, dst1_r, dst1_r, dst1_r);
1159 dst2_r = const_vec;
1160 DPADD_SB4_SH(src32_r, src54_r, src76_r, src98_r,
1161 filt0, filt1, filt2, filt3,
1162 dst2_r, dst2_r, dst2_r, dst2_r);
1163 dst3_r = const_vec;
1164 DPADD_SB4_SH(src43_r, src65_r, src87_r, src109_r,
1165 filt0, filt1, filt2, filt3,
1166 dst3_r, dst3_r, dst3_r, dst3_r);
1167
1168 ST_SH4(dst0_r, dst1_r, dst2_r, dst3_r, dst, dst_stride);
1169 dst += (4 * dst_stride);
1170
1171 src10_r = src54_r;
1172 src32_r = src76_r;
1173 src54_r = src98_r;
1174 src21_r = src65_r;
1175 src43_r = src87_r;
1176 src65_r = src109_r;
1177 src6 = src10;
1178 }
1179}
1180
1181static void hevc_vt_8t_12w_msa(const uint8_t *src, int32_t src_stride,
1182 int16_t *dst, int32_t dst_stride,
1183 const int8_t *filter, int32_t height)
1184{
1185 int32_t loop_cnt;
1186 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1187 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1188 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1189 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
1190 v16i8 src10_l, src32_l, src54_l, src76_l, src98_l;
1191 v16i8 src21_l, src43_l, src65_l, src87_l, src109_l;
1192 v16i8 src2110, src4332, src6554, src8776, src10998;
1193 v8i16 dst0_l, dst1_l;
1194 v8i16 filter_vec, const_vec;
1195 v8i16 filt0, filt1, filt2, filt3;
1196
1197 src -= (3 * src_stride);
1198 const_vec = __msa_ldi_h(128);
1199 const_vec <<= 6;
1200
1201 filter_vec = LD_SH(filter);
1202 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1203
1204 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1205 src += (7 * src_stride);
1206 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1207 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1208 src10_r, src32_r, src54_r, src21_r);
1209 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1210 ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1211 src10_l, src32_l, src54_l, src21_l);
1212 ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1213 ILVR_D3_SB(src21_l, src10_l, src43_l, src32_l, src65_l, src54_l,
1214 src2110, src4332, src6554);
1215
1216 for (loop_cnt = (height >> 2); loop_cnt--;) {
1217 LD_SB4(src, src_stride, src7, src8, src9, src10);
1218 src += (4 * src_stride);
1219 XORI_B4_128_SB(src7, src8, src9, src10);
1220 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1221 src76_r, src87_r, src98_r, src109_r);
1222 ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1223 src76_l, src87_l, src98_l, src109_l);
1224 ILVR_D2_SB(src87_l, src76_l, src109_l, src98_l, src8776, src10998);
1225
1226 dst0_r = const_vec;
1227 DPADD_SB4_SH(src10_r, src32_r, src54_r, src76_r,
1228 filt0, filt1, filt2, filt3,
1229 dst0_r, dst0_r, dst0_r, dst0_r);
1230 dst1_r = const_vec;
1231 DPADD_SB4_SH(src21_r, src43_r, src65_r, src87_r,
1232 filt0, filt1, filt2, filt3,
1233 dst1_r, dst1_r, dst1_r, dst1_r);
1234 dst2_r = const_vec;
1235 DPADD_SB4_SH(src32_r, src54_r, src76_r, src98_r,
1236 filt0, filt1, filt2, filt3,
1237 dst2_r, dst2_r, dst2_r, dst2_r);
1238 dst3_r = const_vec;
1239 DPADD_SB4_SH(src43_r, src65_r, src87_r, src109_r,
1240 filt0, filt1, filt2, filt3,
1241 dst3_r, dst3_r, dst3_r, dst3_r);
1242 dst0_l = const_vec;
1243 DPADD_SB4_SH(src2110, src4332, src6554, src8776,
1244 filt0, filt1, filt2, filt3,
1245 dst0_l, dst0_l, dst0_l, dst0_l);
1246 dst1_l = const_vec;
1247 DPADD_SB4_SH(src4332, src6554, src8776, src10998,
1248 filt0, filt1, filt2, filt3,
1249 dst1_l, dst1_l, dst1_l, dst1_l);
1250
1251 ST_SH4(dst0_r, dst1_r, dst2_r, dst3_r, dst, dst_stride);
1252 ST_D4(dst0_l, dst1_l, 0, 1, 0, 1, dst + 8, dst_stride);
1253 dst += (4 * dst_stride);
1254
1255 src10_r = src54_r;
1256 src32_r = src76_r;
1257 src54_r = src98_r;
1258 src21_r = src65_r;
1259 src43_r = src87_r;
1260 src65_r = src109_r;
1261 src2110 = src6554;
1262 src4332 = src8776;
1263 src6554 = src10998;
1264 src6 = src10;
1265 }
1266}
1267
1268static void hevc_vt_8t_16multx4mult_msa(const uint8_t *src,
1269 int32_t src_stride,
1270 int16_t *dst,
1271 int32_t dst_stride,
1272 const int8_t *filter,
1273 int32_t height,
1274 int32_t width)
1275{
1276 const uint8_t *src_tmp;
1277 int16_t *dst_tmp;
1278 int32_t loop_cnt, cnt;
1279 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1280 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1281 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1282 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
1283 v16i8 src10_l, src32_l, src54_l, src76_l, src98_l;
1284 v16i8 src21_l, src43_l, src65_l, src87_l, src109_l;
1285 v8i16 dst0_l, dst1_l, dst2_l, dst3_l;
1286 v8i16 filter_vec, const_vec;
1287 v8i16 filt0, filt1, filt2, filt3;
1288
1289 src -= (3 * src_stride);
1290 const_vec = __msa_ldi_h(128);
1291 const_vec <<= 6;
1292
1293 filter_vec = LD_SH(filter);
1294 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1295
1296 for (cnt = width >> 4; cnt--;) {
1297 src_tmp = src;
1298 dst_tmp = dst;
1299
1300 LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6);
1301 src_tmp += (7 * src_stride);
1302 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1303 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1304 src10_r, src32_r, src54_r, src21_r);
1305 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1306 ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1307 src10_l, src32_l, src54_l, src21_l);
1308 ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1309
1310 for (loop_cnt = (height >> 2); loop_cnt--;) {
1311 LD_SB4(src_tmp, src_stride, src7, src8, src9, src10);
1312 src_tmp += (4 * src_stride);
1313 XORI_B4_128_SB(src7, src8, src9, src10);
1314 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1315 src76_r, src87_r, src98_r, src109_r);
1316 ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1317 src76_l, src87_l, src98_l, src109_l);
1318
1319 dst0_r = const_vec;
1320 DPADD_SB4_SH(src10_r, src32_r, src54_r, src76_r,
1321 filt0, filt1, filt2, filt3,
1322 dst0_r, dst0_r, dst0_r, dst0_r);
1323 dst1_r = const_vec;
1324 DPADD_SB4_SH(src21_r, src43_r, src65_r, src87_r,
1325 filt0, filt1, filt2, filt3,
1326 dst1_r, dst1_r, dst1_r, dst1_r);
1327 dst2_r = const_vec;
1328 DPADD_SB4_SH(src32_r, src54_r, src76_r, src98_r,
1329 filt0, filt1, filt2, filt3,
1330 dst2_r, dst2_r, dst2_r, dst2_r);
1331 dst3_r = const_vec;
1332 DPADD_SB4_SH(src43_r, src65_r, src87_r, src109_r,
1333 filt0, filt1, filt2, filt3,
1334 dst3_r, dst3_r, dst3_r, dst3_r);
1335 dst0_l = const_vec;
1336 DPADD_SB4_SH(src10_l, src32_l, src54_l, src76_l,
1337 filt0, filt1, filt2, filt3,
1338 dst0_l, dst0_l, dst0_l, dst0_l);
1339 dst1_l = const_vec;
1340 DPADD_SB4_SH(src21_l, src43_l, src65_l, src87_l,
1341 filt0, filt1, filt2, filt3,
1342 dst1_l, dst1_l, dst1_l, dst1_l);
1343 dst2_l = const_vec;
1344 DPADD_SB4_SH(src32_l, src54_l, src76_l, src98_l,
1345 filt0, filt1, filt2, filt3,
1346 dst2_l, dst2_l, dst2_l, dst2_l);
1347 dst3_l = const_vec;
1348 DPADD_SB4_SH(src43_l, src65_l, src87_l, src109_l,
1349 filt0, filt1, filt2, filt3,
1350 dst3_l, dst3_l, dst3_l, dst3_l);
1351
1352 ST_SH4(dst0_r, dst1_r, dst2_r, dst3_r, dst_tmp, dst_stride);
1353 ST_SH4(dst0_l, dst1_l, dst2_l, dst3_l, dst_tmp + 8, dst_stride);
1354 dst_tmp += (4 * dst_stride);
1355
1356 src10_r = src54_r;
1357 src32_r = src76_r;
1358 src54_r = src98_r;
1359 src21_r = src65_r;
1360 src43_r = src87_r;
1361 src65_r = src109_r;
1362 src10_l = src54_l;
1363 src32_l = src76_l;
1364 src54_l = src98_l;
1365 src21_l = src65_l;
1366 src43_l = src87_l;
1367 src65_l = src109_l;
1368 src6 = src10;
1369 }
1370
1371 src += 16;
1372 dst += 16;
1373 }
1374}
1375
1376static void hevc_vt_8t_16w_msa(const uint8_t *src, int32_t src_stride,
1377 int16_t *dst, int32_t dst_stride,
1378 const int8_t *filter, int32_t height)
1379{
1380 hevc_vt_8t_16multx4mult_msa(src, src_stride, dst, dst_stride,
1381 filter, height, 16);
1382}
1383
1384static void hevc_vt_8t_24w_msa(const uint8_t *src, int32_t src_stride,
1385 int16_t *dst, int32_t dst_stride,
1386 const int8_t *filter, int32_t height)
1387{
1388 hevc_vt_8t_16multx4mult_msa(src, src_stride, dst, dst_stride,
1389 filter, height, 16);
1390 hevc_vt_8t_8w_msa(src + 16, src_stride, dst + 16, dst_stride,
1391 filter, height);
1392}
1393
1394static void hevc_vt_8t_32w_msa(const uint8_t *src, int32_t src_stride,
1395 int16_t *dst, int32_t dst_stride,
1396 const int8_t *filter, int32_t height)
1397{
1398 hevc_vt_8t_16multx4mult_msa(src, src_stride, dst, dst_stride,
1399 filter, height, 32);
1400}
1401
1402static void hevc_vt_8t_48w_msa(const uint8_t *src, int32_t src_stride,
1403 int16_t *dst, int32_t dst_stride,
1404 const int8_t *filter, int32_t height)
1405{
1406 hevc_vt_8t_16multx4mult_msa(src, src_stride, dst, dst_stride,
1407 filter, height, 48);
1408}
1409
1410static void hevc_vt_8t_64w_msa(const uint8_t *src, int32_t src_stride,
1411 int16_t *dst, int32_t dst_stride,
1412 const int8_t *filter, int32_t height)
1413{
1414 hevc_vt_8t_16multx4mult_msa(src, src_stride, dst, dst_stride,
1415 filter, height, 64);
1416}
1417
1418static void hevc_hv_8t_4w_msa(const uint8_t *src, int32_t src_stride,
1419 int16_t *dst, int32_t dst_stride,
1420 const int8_t *filter_x, const int8_t *filter_y,
1421 int32_t height)
1422{
1423 uint32_t loop_cnt;
1424 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1425 v8i16 filt0, filt1, filt2, filt3;
1426 v8i16 filt_h0, filt_h1, filt_h2, filt_h3;
1427 v16i8 mask1, mask2, mask3;
1428 v8i16 filter_vec, const_vec;
1429 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1430 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1431 v8i16 dst30, dst41, dst52, dst63, dst66, dst97, dst108;
1432 v4i32 dst0_r, dst1_r, dst2_r, dst3_r;
1433 v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst98_r;
1434 v8i16 dst21_r, dst43_r, dst65_r, dst87_r, dst109_r;
1435 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
1436
1437 src -= ((3 * src_stride) + 3);
1438 filter_vec = LD_SH(filter_x);
1439 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1440
1441 filter_vec = LD_SH(filter_y);
1442 UNPCK_R_SB_SH(filter_vec, filter_vec);
1443
1444 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
1445
1446 mask1 = mask0 + 2;
1447 mask2 = mask0 + 4;
1448 mask3 = mask0 + 6;
1449
1450 const_vec = __msa_ldi_h(128);
1451 const_vec <<= 6;
1452
1453 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1454 src += (7 * src_stride);
1455 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1456
1457 VSHF_B4_SB(src0, src3, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
1458 VSHF_B4_SB(src1, src4, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
1459 VSHF_B4_SB(src2, src5, mask0, mask1, mask2, mask3,
1460 vec8, vec9, vec10, vec11);
1461 VSHF_B4_SB(src3, src6, mask0, mask1, mask2, mask3,
1462 vec12, vec13, vec14, vec15);
1463 dst30 = const_vec;
1464 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
1465 dst30, dst30, dst30, dst30);
1466 dst41 = const_vec;
1467 DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3,
1468 dst41, dst41, dst41, dst41);
1469 dst52 = const_vec;
1470 DPADD_SB4_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3,
1471 dst52, dst52, dst52, dst52);
1472 dst63 = const_vec;
1473 DPADD_SB4_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2, filt3,
1474 dst63, dst63, dst63, dst63);
1475
1476 ILVRL_H2_SH(dst41, dst30, dst10_r, dst43_r);
1477 ILVRL_H2_SH(dst52, dst41, dst21_r, dst54_r);
1478 ILVRL_H2_SH(dst63, dst52, dst32_r, dst65_r);
1479 dst66 = (v8i16) __msa_splati_d((v2i64) dst63, 1);
1480
1481 for (loop_cnt = height >> 2; loop_cnt--;) {
1482 LD_SB4(src, src_stride, src7, src8, src9, src10);
1483 src += (4 * src_stride);
1484 XORI_B4_128_SB(src7, src8, src9, src10);
1485
1486 VSHF_B4_SB(src7, src9, mask0, mask1, mask2, mask3,
1487 vec0, vec1, vec2, vec3);
1488 VSHF_B4_SB(src8, src10, mask0, mask1, mask2, mask3,
1489 vec4, vec5, vec6, vec7);
1490 dst97 = const_vec;
1491 dst108 = const_vec;
1492 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
1493 dst97, dst97, dst97, dst97);
1494 DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3,
1495 dst108, dst108, dst108, dst108);
1496
1497 dst76_r = __msa_ilvr_h(dst97, dst66);
1498 ILVRL_H2_SH(dst108, dst97, dst87_r, dst109_r);
1499 dst66 = (v8i16) __msa_splati_d((v2i64) dst97, 1);
1500 dst98_r = __msa_ilvr_h(dst66, dst108);
1501
1502 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r,
1503 filt_h0, filt_h1, filt_h2, filt_h3);
1504 dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r,
1505 filt_h0, filt_h1, filt_h2, filt_h3);
1506 dst2_r = HEVC_FILT_8TAP(dst32_r, dst54_r, dst76_r, dst98_r,
1507 filt_h0, filt_h1, filt_h2, filt_h3);
1508 dst3_r = HEVC_FILT_8TAP(dst43_r, dst65_r, dst87_r, dst109_r,
1509 filt_h0, filt_h1, filt_h2, filt_h3);
1510 SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
1511 PCKEV_H2_SW(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst2_r);
1512 ST_D4(dst0_r, dst2_r, 0, 1, 0, 1, dst, dst_stride);
1513 dst += (4 * dst_stride);
1514
1515 dst10_r = dst54_r;
1516 dst32_r = dst76_r;
1517 dst54_r = dst98_r;
1518 dst21_r = dst65_r;
1519 dst43_r = dst87_r;
1520 dst65_r = dst109_r;
1521 dst66 = (v8i16) __msa_splati_d((v2i64) dst108, 1);
1522 }
1523}
1524
1525static void hevc_hv_8t_8multx1mult_msa(const uint8_t *src,
1526 int32_t src_stride,
1527 int16_t *dst,
1528 int32_t dst_stride,
1529 const int8_t *filter_x,
1530 const int8_t *filter_y,
1531 int32_t height, int32_t width)
1532{
1533 uint32_t loop_cnt, cnt;
1534 const uint8_t *src_tmp;
1535 int16_t *dst_tmp;
1536 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
1537 v8i16 filt0, filt1, filt2, filt3;
1538 v8i16 filt_h0, filt_h1, filt_h2, filt_h3;
1539 v16i8 mask1, mask2, mask3;
1540 v8i16 filter_vec, const_vec;
1541 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1542 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1543 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1544 v4i32 dst0_r, dst0_l;
1545 v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
1546 v8i16 dst10_l, dst32_l, dst54_l, dst76_l;
1547 v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
1548
1549 src -= ((3 * src_stride) + 3);
1550 filter_vec = LD_SH(filter_x);
1551 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1552
1553 filter_vec = LD_SH(filter_y);
1554 UNPCK_R_SB_SH(filter_vec, filter_vec);
1555
1556 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
1557
1558 mask1 = mask0 + 2;
1559 mask2 = mask0 + 4;
1560 mask3 = mask0 + 6;
1561
1562 const_vec = __msa_ldi_h(128);
1563 const_vec <<= 6;
1564
1565 for (cnt = width >> 3; cnt--;) {
1566 src_tmp = src;
1567 dst_tmp = dst;
1568
1569 LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6);
1570 src_tmp += (7 * src_stride);
1571 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1572
1573 /* row 0 row 1 row 2 row 3 */
1574 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1575 vec0, vec1, vec2, vec3);
1576 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1577 vec4, vec5, vec6, vec7);
1578 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1579 vec8, vec9, vec10, vec11);
1580 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
1581 vec12, vec13, vec14, vec15);
1582 dst0 = const_vec;
1583 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
1584 dst0, dst0, dst0, dst0);
1585 dst1 = const_vec;
1586 DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3,
1587 dst1, dst1, dst1, dst1);
1588 dst2 = const_vec;
1589 DPADD_SB4_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3,
1590 dst2, dst2, dst2, dst2);
1591 dst3 = const_vec;
1592 DPADD_SB4_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2, filt3,
1593 dst3, dst3, dst3, dst3);
1594
1595 /* row 4 row 5 row 6 */
1596 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3,
1597 vec0, vec1, vec2, vec3);
1598 VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3,
1599 vec4, vec5, vec6, vec7);
1600 VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3,
1601 vec8, vec9, vec10, vec11);
1602 dst4 = const_vec;
1603 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
1604 dst4, dst4, dst4, dst4);
1605 dst5 = const_vec;
1606 DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3,
1607 dst5, dst5, dst5, dst5);
1608 dst6 = const_vec;
1609 DPADD_SB4_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3,
1610 dst6, dst6, dst6, dst6);
1611
1612 for (loop_cnt = height; loop_cnt--;) {
1613 src7 = LD_SB(src_tmp);
1614 src7 = (v16i8) __msa_xori_b((v16u8) src7, 128);
1615 src_tmp += src_stride;
1616
1617 VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3,
1618 vec0, vec1, vec2, vec3);
1619 dst7 = const_vec;
1620 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
1621 dst7, dst7, dst7, dst7);
1622
1623 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
1624 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
1625 ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
1626 ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
1627 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r,
1628 filt_h0, filt_h1, filt_h2, filt_h3);
1629 dst0_l = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l,
1630 filt_h0, filt_h1, filt_h2, filt_h3);
1631 dst0_r >>= 6;
1632 dst0_l >>= 6;
1633
1634 dst0_r = (v4i32) __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
1635 ST_SW(dst0_r, dst_tmp);
1636 dst_tmp += dst_stride;
1637
1638 dst0 = dst1;
1639 dst1 = dst2;
1640 dst2 = dst3;
1641 dst3 = dst4;
1642 dst4 = dst5;
1643 dst5 = dst6;
1644 dst6 = dst7;
1645 }
1646
1647 src += 8;
1648 dst += 8;
1649 }
1650}
1651
1652static void hevc_hv_8t_8w_msa(const uint8_t *src, int32_t src_stride,
1653 int16_t *dst, int32_t dst_stride,
1654 const int8_t *filter_x, const int8_t *filter_y,
1655 int32_t height)
1656{
1657 hevc_hv_8t_8multx1mult_msa(src, src_stride, dst, dst_stride,
1658 filter_x, filter_y, height, 8);
1659}
1660
1661static void hevc_hv_8t_12w_msa(const uint8_t *src, int32_t src_stride,
1662 int16_t *dst, int32_t dst_stride,
1663 const int8_t *filter_x, const int8_t *filter_y,
1664 int32_t height)
1665{
1666 uint32_t loop_cnt;
1667 const uint8_t *src_tmp;
1668 int16_t *dst_tmp;
1669 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1670 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1671 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1672 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1673 v8i16 filt0, filt1, filt2, filt3, filt_h0, filt_h1, filt_h2, filt_h3;
1674 v8i16 filter_vec, const_vec;
1675 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1676 v8i16 dst30, dst41, dst52, dst63, dst66, dst97, dst108;
1677 v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst98_r, dst21_r, dst43_r;
1678 v8i16 dst65_r, dst87_r, dst109_r, dst10_l, dst32_l, dst54_l, dst76_l;
1679 v4i32 dst0_r, dst0_l, dst1_r, dst2_r, dst3_r;
1680
1681 src -= ((3 * src_stride) + 3);
1682 filter_vec = LD_SH(filter_x);
1683 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1684
1685 filter_vec = LD_SH(filter_y);
1686 UNPCK_R_SB_SH(filter_vec, filter_vec);
1687
1688 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
1689
1690 mask0 = LD_SB(ff_hevc_mask_arr);
1691 mask1 = mask0 + 2;
1692 mask2 = mask0 + 4;
1693 mask3 = mask0 + 6;
1694
1695 const_vec = __msa_ldi_h(128);
1696 const_vec <<= 6;
1697
1698 src_tmp = src;
1699 dst_tmp = dst;
1700
1701 LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6);
1702 src_tmp += (7 * src_stride);
1703 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1704
1705 /* row 0 row 1 row 2 row 3 */
1706 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
1707 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
1708 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3, vec8, vec9, vec10,
1709 vec11);
1710 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3, vec12, vec13, vec14,
1711 vec15);
1712 dst0 = const_vec;
1713 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3, dst0, dst0,
1714 dst0, dst0);
1715 dst1 = const_vec;
1716 DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3, dst1, dst1,
1717 dst1, dst1);
1718 dst2 = const_vec;
1719 DPADD_SB4_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3, dst2,
1720 dst2, dst2, dst2);
1721 dst3 = const_vec;
1722 DPADD_SB4_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2, filt3, dst3,
1723 dst3, dst3, dst3);
1724
1725 /* row 4 row 5 row 6 */
1726 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
1727 VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
1728 VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3, vec8, vec9, vec10,
1729 vec11);
1730 dst4 = const_vec;
1731 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3, dst4, dst4,
1732 dst4, dst4);
1733 dst5 = const_vec;
1734 DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3, dst5, dst5,
1735 dst5, dst5);
1736 dst6 = const_vec;
1737 DPADD_SB4_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3, dst6,
1738 dst6, dst6, dst6);
1739
1740 for (loop_cnt = height; loop_cnt--;) {
1741 src7 = LD_SB(src_tmp);
1742 src7 = (v16i8) __msa_xori_b((v16u8) src7, 128);
1743 src_tmp += src_stride;
1744
1745 VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
1746 vec3);
1747 dst7 = const_vec;
1748 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3, dst7,
1749 dst7, dst7, dst7);
1750
1751 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
1752 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
1753 ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
1754 ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
1755 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0,
1756 filt_h1, filt_h2, filt_h3);
1757 dst0_l = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l, filt_h0,
1758 filt_h1, filt_h2, filt_h3);
1759 dst0_r >>= 6;
1760 dst0_l >>= 6;
1761
1762 dst0_r = (v4i32) __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
1763 ST_SW(dst0_r, dst_tmp);
1764 dst_tmp += dst_stride;
1765
1766 dst0 = dst1;
1767 dst1 = dst2;
1768 dst2 = dst3;
1769 dst3 = dst4;
1770 dst4 = dst5;
1771 dst5 = dst6;
1772 dst6 = dst7;
1773 }
1774
1775 src += 8;
1776 dst += 8;
1777
1778 mask4 = LD_SB(ff_hevc_mask_arr + 16);
1779 mask5 = mask4 + 2;
1780 mask6 = mask4 + 4;
1781 mask7 = mask4 + 6;
1782
1783 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1784 src += (7 * src_stride);
1785 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1786
1787 VSHF_B4_SB(src0, src3, mask4, mask5, mask6, mask7, vec0, vec1, vec2, vec3);
1788 VSHF_B4_SB(src1, src4, mask4, mask5, mask6, mask7, vec4, vec5, vec6, vec7);
1789 VSHF_B4_SB(src2, src5, mask4, mask5, mask6, mask7, vec8, vec9, vec10,
1790 vec11);
1791 VSHF_B4_SB(src3, src6, mask4, mask5, mask6, mask7, vec12, vec13, vec14,
1792 vec15);
1793 dst30 = const_vec;
1794 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3, dst30,
1795 dst30, dst30, dst30);
1796 dst41 = const_vec;
1797 DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3, dst41,
1798 dst41, dst41, dst41);
1799 dst52 = const_vec;
1800 DPADD_SB4_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3, dst52,
1801 dst52, dst52, dst52);
1802 dst63 = const_vec;
1803 DPADD_SB4_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2, filt3, dst63,
1804 dst63, dst63, dst63);
1805
1806 ILVRL_H2_SH(dst41, dst30, dst10_r, dst43_r);
1807 ILVRL_H2_SH(dst52, dst41, dst21_r, dst54_r);
1808 ILVRL_H2_SH(dst63, dst52, dst32_r, dst65_r);
1809
1810 dst66 = (v8i16) __msa_splati_d((v2i64) dst63, 1);
1811
1812 for (loop_cnt = height >> 2; loop_cnt--;) {
1813 LD_SB4(src, src_stride, src7, src8, src9, src10);
1814 src += (4 * src_stride);
1815 XORI_B4_128_SB(src7, src8, src9, src10);
1816
1817 VSHF_B4_SB(src7, src9, mask4, mask5, mask6, mask7, vec0, vec1, vec2,
1818 vec3);
1819 VSHF_B4_SB(src8, src10, mask4, mask5, mask6, mask7, vec4, vec5, vec6,
1820 vec7);
1821 dst97 = const_vec;
1822 dst108 = const_vec;
1823 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3, dst97,
1824 dst97, dst97, dst97);
1825 DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3, dst108,
1826 dst108, dst108, dst108);
1827
1828 dst76_r = __msa_ilvr_h(dst97, dst66);
1829 ILVRL_H2_SH(dst108, dst97, dst87_r, dst109_r);
1830 dst66 = (v8i16) __msa_splati_d((v2i64) dst97, 1);
1831 dst98_r = __msa_ilvr_h(dst66, dst108);
1832
1833 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0,
1834 filt_h1, filt_h2, filt_h3);
1835 dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r, filt_h0,
1836 filt_h1, filt_h2, filt_h3);
1837 dst2_r = HEVC_FILT_8TAP(dst32_r, dst54_r, dst76_r, dst98_r, filt_h0,
1838 filt_h1, filt_h2, filt_h3);
1839 dst3_r = HEVC_FILT_8TAP(dst43_r, dst65_r, dst87_r, dst109_r, filt_h0,
1840 filt_h1, filt_h2, filt_h3);
1841 SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
1842 PCKEV_H2_SW(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst2_r);
1843 ST_D4(dst0_r, dst2_r, 0, 1, 0, 1, dst, dst_stride);
1844 dst += (4 * dst_stride);
1845
1846 dst10_r = dst54_r;
1847 dst32_r = dst76_r;
1848 dst54_r = dst98_r;
1849 dst21_r = dst65_r;
1850 dst43_r = dst87_r;
1851 dst65_r = dst109_r;
1852 dst66 = (v8i16) __msa_splati_d((v2i64) dst108, 1);
1853 }
1854}
1855
1856static void hevc_hv_8t_16w_msa(const uint8_t *src, int32_t src_stride,
1857 int16_t *dst, int32_t dst_stride,
1858 const int8_t *filter_x, const int8_t *filter_y,
1859 int32_t height)
1860{
1861 hevc_hv_8t_8multx1mult_msa(src, src_stride, dst, dst_stride,
1862 filter_x, filter_y, height, 16);
1863}
1864
1865static void hevc_hv_8t_24w_msa(const uint8_t *src, int32_t src_stride,
1866 int16_t *dst, int32_t dst_stride,
1867 const int8_t *filter_x, const int8_t *filter_y,
1868 int32_t height)
1869{
1870 hevc_hv_8t_8multx1mult_msa(src, src_stride, dst, dst_stride,
1871 filter_x, filter_y, height, 24);
1872}
1873
1874static void hevc_hv_8t_32w_msa(const uint8_t *src, int32_t src_stride,
1875 int16_t *dst, int32_t dst_stride,
1876 const int8_t *filter_x, const int8_t *filter_y,
1877 int32_t height)
1878{
1879 hevc_hv_8t_8multx1mult_msa(src, src_stride, dst, dst_stride,
1880 filter_x, filter_y, height, 32);
1881}
1882
1883static void hevc_hv_8t_48w_msa(const uint8_t *src, int32_t src_stride,
1884 int16_t *dst, int32_t dst_stride,
1885 const int8_t *filter_x, const int8_t *filter_y,
1886 int32_t height)
1887{
1888 hevc_hv_8t_8multx1mult_msa(src, src_stride, dst, dst_stride,
1889 filter_x, filter_y, height, 48);
1890}
1891
1892static void hevc_hv_8t_64w_msa(const uint8_t *src, int32_t src_stride,
1893 int16_t *dst, int32_t dst_stride,
1894 const int8_t *filter_x, const int8_t *filter_y,
1895 int32_t height)
1896{
1897 hevc_hv_8t_8multx1mult_msa(src, src_stride, dst, dst_stride,
1898 filter_x, filter_y, height, 64);
1899}
1900
1901static void hevc_hz_4t_4x2_msa(const uint8_t *src,
1902 int32_t src_stride,
1903 int16_t *dst,
1904 int32_t dst_stride,
1905 const int8_t *filter)
1906{
1907 v8i16 filt0, filt1;
1908 v16i8 src0, src1;
1909 v16i8 mask1, vec0, vec1;
1910 v8i16 dst0;
1911 v8i16 filter_vec, const_vec;
1912 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
1913
1914 src -= 1;
1915
1916 filter_vec = LD_SH(filter);
1917 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
1918
1919 mask1 = mask0 + 2;
1920
1921 const_vec = __msa_ldi_h(128);
1922 const_vec <<= 6;
1923
1924 LD_SB2(src, src_stride, src0, src1);
1925 XORI_B2_128_SB(src0, src1);
1926
1927 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
1928 dst0 = const_vec;
1929 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
1930
1931 ST_D2(dst0, 0, 1, dst, dst_stride);
1932}
1933
1934static void hevc_hz_4t_4x4_msa(const uint8_t *src,
1935 int32_t src_stride,
1936 int16_t *dst,
1937 int32_t dst_stride,
1938 const int8_t *filter)
1939{
1940 v8i16 filt0, filt1;
1941 v16i8 src0, src1, src2, src3;
1942 v16i8 mask1, vec0, vec1;
1943 v8i16 dst0, dst1;
1944 v8i16 filter_vec, const_vec;
1945 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
1946
1947 src -= 1;
1948
1949 filter_vec = LD_SH(filter);
1950 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
1951
1952 mask1 = mask0 + 2;
1953
1954 const_vec = __msa_ldi_h(128);
1955 const_vec <<= 6;
1956
1957 LD_SB4(src, src_stride, src0, src1, src2, src3);
1958 XORI_B4_128_SB(src0, src1, src2, src3);
1959
1960 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
1961 dst0 = const_vec;
1962 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
1963
1964 VSHF_B2_SB(src2, src3, src2, src3, mask0, mask1, vec0, vec1);
1965 dst1 = const_vec;
1966 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
1967
1968 ST_D4(dst0, dst1, 0, 1, 0, 1, dst, dst_stride);
1969}
1970
1971static void hevc_hz_4t_4x8multiple_msa(const uint8_t *src,
1972 int32_t src_stride,
1973 int16_t *dst,
1974 int32_t dst_stride,
1975 const int8_t *filter,
1976 int32_t height)
1977{
1978 uint32_t loop_cnt;
1979 v8i16 filt0, filt1;
1980 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
1981 v16i8 mask1, vec0, vec1;
1982 v8i16 dst0, dst1, dst2, dst3;
1983 v8i16 filter_vec, const_vec;
1984 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
1985
1986 src -= 1;
1987
1988 filter_vec = LD_SH(filter);
1989 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
1990
1991 mask1 = mask0 + 2;
1992
1993 const_vec = __msa_ldi_h(128);
1994 const_vec <<= 6;
1995
1996 for (loop_cnt = (height >> 3); loop_cnt--;) {
1997 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
1998 src += (8 * src_stride);
1999
2000 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2001
2002 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
2003 dst0 = const_vec;
2004 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
2005 VSHF_B2_SB(src2, src3, src2, src3, mask0, mask1, vec0, vec1);
2006 dst1 = const_vec;
2007 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
2008 VSHF_B2_SB(src4, src5, src4, src5, mask0, mask1, vec0, vec1);
2009 dst2 = const_vec;
2010 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
2011 VSHF_B2_SB(src6, src7, src6, src7, mask0, mask1, vec0, vec1);
2012 dst3 = const_vec;
2013 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
2014
2015 ST_D8(dst0, dst1, dst2, dst3, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
2016 dst += (8 * dst_stride);
2017 }
2018}
2019
2020static void hevc_hz_4t_4w_msa(const uint8_t *src,
2021 int32_t src_stride,
2022 int16_t *dst,
2023 int32_t dst_stride,
2024 const int8_t *filter,
2025 int32_t height)
2026{
2027 if (2 == height) {
2028 hevc_hz_4t_4x2_msa(src, src_stride, dst, dst_stride, filter);
2029 } else if (4 == height) {
2030 hevc_hz_4t_4x4_msa(src, src_stride, dst, dst_stride, filter);
2031 } else if (0 == height % 8) {
2032 hevc_hz_4t_4x8multiple_msa(src, src_stride, dst, dst_stride,
2033 filter, height);
2034 }
2035}
2036
2037static void hevc_hz_4t_6w_msa(const uint8_t *src,
2038 int32_t src_stride,
2039 int16_t *dst,
2040 int32_t dst_stride,
2041 const int8_t *filter,
2042 int32_t height)
2043{
2044 uint32_t loop_cnt;
2045 uint64_t dst_val0, dst_val1, dst_val2, dst_val3;
2046 uint32_t dst_val_int0, dst_val_int1, dst_val_int2, dst_val_int3;
2047 v8i16 filt0, filt1, dst0, dst1, dst2, dst3;
2048 v16i8 src0, src1, src2, src3;
2049 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2050 v16i8 mask1;
2051 v16i8 vec0, vec1;
2052 v8i16 filter_vec, const_vec;
2053
2054 src -= 1;
2055
2056 filter_vec = LD_SH(filter);
2057 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2058
2059 mask1 = mask0 + 2;
2060
2061 const_vec = __msa_ldi_h(128);
2062 const_vec <<= 6;
2063
2064 for (loop_cnt = 2; loop_cnt--;) {
2065 LD_SB4(src, src_stride, src0, src1, src2, src3);
2066 src += (4 * src_stride);
2067
2068 XORI_B4_128_SB(src0, src1, src2, src3);
2069
2070 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
2071 dst0 = const_vec;
2072 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
2073 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
2074 dst1 = const_vec;
2075 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
2076 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
2077 dst2 = const_vec;
2078 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
2079 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
2080 dst3 = const_vec;
2081 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
2082
2083 dst_val0 = __msa_copy_u_d((v2i64) dst0, 0);
2084 dst_val1 = __msa_copy_u_d((v2i64) dst1, 0);
2085 dst_val2 = __msa_copy_u_d((v2i64) dst2, 0);
2086 dst_val3 = __msa_copy_u_d((v2i64) dst3, 0);
2087
2088 dst_val_int0 = __msa_copy_u_w((v4i32) dst0, 2);
2089 dst_val_int1 = __msa_copy_u_w((v4i32) dst1, 2);
2090 dst_val_int2 = __msa_copy_u_w((v4i32) dst2, 2);
2091 dst_val_int3 = __msa_copy_u_w((v4i32) dst3, 2);
2092
2093 SD(dst_val0, dst);
2094 SW(dst_val_int0, dst + 4);
2095 dst += dst_stride;
2096 SD(dst_val1, dst);
2097 SW(dst_val_int1, dst + 4);
2098 dst += dst_stride;
2099 SD(dst_val2, dst);
2100 SW(dst_val_int2, dst + 4);
2101 dst += dst_stride;
2102 SD(dst_val3, dst);
2103 SW(dst_val_int3, dst + 4);
2104 dst += dst_stride;
2105 }
2106}
2107
2108static void hevc_hz_4t_8x2multiple_msa(const uint8_t *src,
2109 int32_t src_stride,
2110 int16_t *dst,
2111 int32_t dst_stride,
2112 const int8_t *filter,
2113 int32_t height)
2114{
2115 uint32_t loop_cnt;
2116 v8i16 filt0, filt1, dst0, dst1;
2117 v16i8 src0, src1;
2118 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2119 v16i8 mask1;
2120 v16i8 vec0, vec1;
2121 v8i16 filter_vec, const_vec;
2122
2123 src -= 1;
2124
2125 filter_vec = LD_SH(filter);
2126 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2127
2128 mask1 = mask0 + 2;
2129
2130 const_vec = __msa_ldi_h(128);
2131 const_vec <<= 6;
2132
2133 for (loop_cnt = (height >> 1); loop_cnt--;) {
2134 LD_SB2(src, src_stride, src0, src1);
2135 src += (2 * src_stride);
2136
2137 XORI_B2_128_SB(src0, src1);
2138
2139 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
2140 dst0 = const_vec;
2141 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
2142
2143 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
2144 dst1 = const_vec;
2145 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
2146
2147 ST_SH2(dst0, dst1, dst, dst_stride);
2148 dst += (2 * dst_stride);
2149 }
2150}
2151
2152static void hevc_hz_4t_8x4multiple_msa(const uint8_t *src,
2153 int32_t src_stride,
2154 int16_t *dst,
2155 int32_t dst_stride,
2156 const int8_t *filter,
2157 int32_t height)
2158{
2159 uint32_t loop_cnt;
2160 v8i16 filt0, filt1;
2161 v16i8 src0, src1, src2, src3;
2162 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2163 v16i8 mask1;
2164 v16i8 vec0, vec1;
2165 v8i16 dst0, dst1, dst2, dst3;
2166 v8i16 filter_vec, const_vec;
2167
2168 src -= 1;
2169
2170 filter_vec = LD_SH(filter);
2171 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2172
2173 mask1 = mask0 + 2;
2174
2175 const_vec = __msa_ldi_h(128);
2176 const_vec <<= 6;
2177
2178 for (loop_cnt = (height >> 2); loop_cnt--;) {
2179 LD_SB4(src, src_stride, src0, src1, src2, src3);
2180 src += (4 * src_stride);
2181
2182 XORI_B4_128_SB(src0, src1, src2, src3);
2183
2184 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
2185 dst0 = const_vec;
2186 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
2187
2188 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
2189 dst1 = const_vec;
2190 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
2191
2192 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
2193 dst2 = const_vec;
2194 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
2195
2196 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
2197 dst3 = const_vec;
2198 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
2199
2200 ST_SH4(dst0, dst1, dst2, dst3, dst, dst_stride);
2201 dst += (4 * dst_stride);
2202 }
2203}
2204
2205static void hevc_hz_4t_8w_msa(const uint8_t *src,
2206 int32_t src_stride,
2207 int16_t *dst,
2208 int32_t dst_stride,
2209 const int8_t *filter,
2210 int32_t height)
2211{
2212 if (2 == height || 6 == height) {
2213 hevc_hz_4t_8x2multiple_msa(src, src_stride, dst, dst_stride,
2214 filter, height);
2215 } else {
2216 hevc_hz_4t_8x4multiple_msa(src, src_stride, dst, dst_stride,
2217 filter, height);
2218 }
2219}
2220
2221static void hevc_hz_4t_12w_msa(const uint8_t *src,
2222 int32_t src_stride,
2223 int16_t *dst,
2224 int32_t dst_stride,
2225 const int8_t *filter,
2226 int32_t height)
2227{
2228 uint32_t loop_cnt;
2229 v8i16 filt0, filt1;
2230 v16i8 src0, src1, src2, src3;
2231 v16i8 mask1;
2232 v16i8 vec0, vec1;
2233 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
2234 v8i16 filter_vec, const_vec;
2235 v16i8 mask3;
2236 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2237 v16i8 mask2 = {
2238 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28
2239 };
2240
2241 src -= 1;
2242
2243 filter_vec = LD_SH(filter);
2244 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2245
2246 mask1 = mask0 + 2;
2247 mask3 = mask2 + 2;
2248
2249 const_vec = __msa_ldi_h(128);
2250 const_vec <<= 6;
2251
2252 for (loop_cnt = (height >> 2); loop_cnt--;) {
2253 LD_SB4(src, src_stride, src0, src1, src2, src3);
2254 src += (4 * src_stride);
2255 XORI_B4_128_SB(src0, src1, src2, src3);
2256
2257 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
2258 dst0 = const_vec;
2259 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
2260 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
2261 dst1 = const_vec;
2262 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
2263 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
2264 dst2 = const_vec;
2265 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
2266 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
2267 dst3 = const_vec;
2268 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
2269 VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
2270 dst4 = const_vec;
2271 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
2272 VSHF_B2_SB(src2, src3, src2, src3, mask2, mask3, vec0, vec1);
2273 dst5 = const_vec;
2274 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
2275
2276 ST_SH4(dst0, dst1, dst2, dst3, dst, dst_stride);
2277 ST_D4(dst4, dst5, 0, 1, 0, 1, dst + 8, dst_stride);
2278 dst += (4 * dst_stride);
2279 }
2280}
2281
2282static void hevc_hz_4t_16w_msa(const uint8_t *src,
2283 int32_t src_stride,
2284 int16_t *dst,
2285 int32_t dst_stride,
2286 const int8_t *filter,
2287 int32_t height)
2288{
2289 uint32_t loop_cnt;
2290 v16i8 src0, src1, src2, src3;
2291 v16i8 src4, src5, src6, src7;
2292 v8i16 filt0, filt1;
2293 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2294 v16i8 mask1;
2295 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
2296 v16i8 vec0, vec1;
2297 v8i16 filter_vec, const_vec;
2298
2299 src -= 1;
2300
2301 filter_vec = LD_SH(filter);
2302 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2303
2304 mask1 = mask0 + 2;
2305
2306 const_vec = __msa_ldi_h(128);
2307 const_vec <<= 6;
2308
2309 for (loop_cnt = (height >> 2); loop_cnt--;) {
2310 LD_SB4(src, src_stride, src0, src2, src4, src6);
2311 LD_SB4(src + 8, src_stride, src1, src3, src5, src7);
2312 src += (4 * src_stride);
2313
2314 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2315
2316 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
2317 dst0 = const_vec;
2318 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
2319
2320 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
2321 dst1 = const_vec;
2322 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
2323
2324 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
2325 dst2 = const_vec;
2326 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
2327
2328 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
2329 dst3 = const_vec;
2330 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
2331
2332 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
2333 dst4 = const_vec;
2334 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
2335
2336 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
2337 dst5 = const_vec;
2338 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
2339
2340 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
2341 dst6 = const_vec;
2342 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst6, dst6);
2343
2344 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
2345 dst7 = const_vec;
2346 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst7, dst7);
2347
2348 ST_SH4(dst0, dst2, dst4, dst6, dst, dst_stride);
2349 ST_SH4(dst1, dst3, dst5, dst7, dst + 8, dst_stride);
2350 dst += (4 * dst_stride);
2351 }
2352}
2353
2354static void hevc_hz_4t_24w_msa(const uint8_t *src,
2355 int32_t src_stride,
2356 int16_t *dst,
2357 int32_t dst_stride,
2358 const int8_t *filter,
2359 int32_t height)
2360{
2361 uint32_t loop_cnt;
2362 int16_t *dst_tmp = dst + 16;
2363 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
2364 v8i16 filt0, filt1;
2365 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2366 v16i8 mask1, mask00, mask11;
2367 v16i8 vec0, vec1;
2368 v8i16 dst0, dst1, dst2, dst3;
2369 v8i16 filter_vec, const_vec;
2370
2371 src -= 1;
2372
2373 filter_vec = LD_SH(filter);
2374 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2375
2376 mask1 = mask0 + 2;
2377 mask00 = mask0 + 8;
2378 mask11 = mask0 + 10;
2379
2380 const_vec = __msa_ldi_h(128);
2381 const_vec <<= 6;
2382
2383 for (loop_cnt = (height >> 2); loop_cnt--;) {
2384 /* 16 width */
2385 LD_SB4(src, src_stride, src0, src2, src4, src6);
2386 LD_SB4(src + 16, src_stride, src1, src3, src5, src7);
2387 src += (4 * src_stride);
2388
2389 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2390
2391 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
2392 dst0 = const_vec;
2393 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
2394
2395 VSHF_B2_SB(src0, src1, src0, src1, mask00, mask11, vec0, vec1);
2396 dst1 = const_vec;
2397 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
2398
2399 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
2400 dst2 = const_vec;
2401 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
2402
2403 VSHF_B2_SB(src2, src3, src2, src3, mask00, mask11, vec0, vec1);
2404 dst3 = const_vec;
2405 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
2406
2407 ST_SH2(dst0, dst1, dst, 8);
2408 dst += dst_stride;
2409 ST_SH2(dst2, dst3, dst, 8);
2410 dst += dst_stride;
2411
2412 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
2413 dst0 = const_vec;
2414 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
2415
2416 VSHF_B2_SB(src4, src5, src4, src5, mask00, mask11, vec0, vec1);
2417 dst1 = const_vec;
2418 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
2419
2420 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
2421 dst2 = const_vec;
2422 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
2423
2424 VSHF_B2_SB(src6, src7, src6, src7, mask00, mask11, vec0, vec1);
2425 dst3 = const_vec;
2426 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
2427
2428 ST_SH2(dst0, dst1, dst, 8);
2429 dst += dst_stride;
2430 ST_SH2(dst2, dst3, dst, 8);
2431 dst += dst_stride;
2432
2433 /* 8 width */
2434 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
2435 dst0 = const_vec;
2436 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
2437
2438 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
2439 dst1 = const_vec;
2440 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
2441
2442 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
2443 dst2 = const_vec;
2444 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
2445
2446 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
2447 dst3 = const_vec;
2448 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
2449
2450 ST_SH4(dst0, dst1, dst2, dst3, dst_tmp, dst_stride);
2451 dst_tmp += (4 * dst_stride);
2452 }
2453}
2454
2455static void hevc_hz_4t_32w_msa(const uint8_t *src,
2456 int32_t src_stride,
2457 int16_t *dst,
2458 int32_t dst_stride,
2459 const int8_t *filter,
2460 int32_t height)
2461{
2462 uint32_t loop_cnt;
2463 v16i8 src0, src1, src2;
2464 v8i16 filt0, filt1;
2465 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2466 v16i8 mask1, mask2, mask3;
2467 v8i16 dst0, dst1, dst2, dst3;
2468 v16i8 vec0, vec1, vec2, vec3;
2469 v8i16 filter_vec, const_vec;
2470
2471 src -= 1;
2472
2473 filter_vec = LD_SH(filter);
2474 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2475
2476 const_vec = __msa_ldi_h(128);
2477 const_vec <<= 6;
2478
2479 mask1 = mask0 + 2;
2480 mask2 = mask0 + 8;
2481 mask3 = mask0 + 10;
2482
2483 for (loop_cnt = height; loop_cnt--;) {
2484 LD_SB2(src, 16, src0, src1);
2485 src2 = LD_SB(src + 24);
2486 src += src_stride;
2487
2488 XORI_B3_128_SB(src0, src1, src2);
2489
2490 dst0 = const_vec;
2491 dst1 = const_vec;
2492 dst2 = const_vec;
2493 dst3 = const_vec;
2494 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask2, vec0, vec1);
2495 VSHF_B2_SB(src1, src1, src2, src2, mask0, mask0, vec2, vec3);
2496 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
2497 dst1, dst2, dst3);
2498 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask3, vec0, vec1);
2499 VSHF_B2_SB(src1, src1, src2, src2, mask1, mask1, vec2, vec3);
2500 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
2501 dst1, dst2, dst3);
2502 ST_SH4(dst0, dst1, dst2, dst3, dst, 8);
2503 dst += dst_stride;
2504 }
2505}
2506
2507static void hevc_vt_4t_4x2_msa(const uint8_t *src,
2508 int32_t src_stride,
2509 int16_t *dst,
2510 int32_t dst_stride,
2511 const int8_t *filter)
2512{
2513 v16i8 src0, src1, src2, src3, src4;
2514 v16i8 src10_r, src32_r, src21_r, src43_r;
2515 v16i8 src2110, src4332;
2516 v8i16 dst10;
2517 v8i16 filt0, filt1;
2518 v8i16 filter_vec, const_vec;
2519
2520 src -= src_stride;
2521
2522 const_vec = __msa_ldi_h(128);
2523 const_vec <<= 6;
2524
2525 filter_vec = LD_SH(filter);
2526 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2527
2528 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
2529 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
2530 src10_r, src21_r, src32_r, src43_r);
2531
2532 ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332);
2533 XORI_B2_128_SB(src2110, src4332);
2534 dst10 = const_vec;
2535 DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
2536
2537 ST_D2(dst10, 0, 1, dst, dst_stride);
2538}
2539
2540static void hevc_vt_4t_4x4_msa(const uint8_t *src,
2541 int32_t src_stride,
2542 int16_t *dst,
2543 int32_t dst_stride,
2544 const int8_t *filter,
2545 int32_t height)
2546{
2547 v16i8 src0, src1, src2, src3, src4, src5, src6;
2548 v16i8 src10_r, src32_r, src54_r, src21_r, src43_r, src65_r;
2549 v16i8 src2110, src4332, src6554;
2550 v8i16 dst10, dst32;
2551 v8i16 filt0, filt1;
2552 v8i16 filter_vec, const_vec;
2553
2554 src -= src_stride;
2555
2556 const_vec = __msa_ldi_h(128);
2557 const_vec <<= 6;
2558
2559 filter_vec = LD_SH(filter);
2560 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2561
2562 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
2563 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
2564 src10_r, src21_r, src32_r, src43_r);
2565 ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
2566 ILVR_D3_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r,
2567 src2110, src4332, src6554);
2568 XORI_B3_128_SB(src2110, src4332, src6554);
2569 dst10 = const_vec;
2570 DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
2571 dst32 = const_vec;
2572 DPADD_SB2_SH(src4332, src6554, filt0, filt1, dst32, dst32);
2573
2574 ST_D4(dst10, dst32, 0, 1, 0, 1, dst, dst_stride);
2575}
2576
2577static void hevc_vt_4t_4x8_msa(const uint8_t *src,
2578 int32_t src_stride,
2579 int16_t *dst,
2580 int32_t dst_stride,
2581 const int8_t *filter,
2582 int32_t height)
2583{
2584 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2585 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
2586 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
2587 v16i8 src2110, src4332, src6554, src8776, src10998;
2588 v8i16 dst10, dst32, dst54, dst76;
2589 v8i16 filt0, filt1;
2590 v8i16 filter_vec, const_vec;
2591
2592 src -= src_stride;
2593 const_vec = __msa_ldi_h(128);
2594 const_vec <<= 6;
2595
2596 filter_vec = LD_SH(filter);
2597 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2598
2599 LD_SB3(src, src_stride, src0, src1, src2);
2600 src += (3 * src_stride);
2601
2602 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2603 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
2604 src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
2605
2606 LD_SB8(src, src_stride, src3, src4, src5, src6, src7, src8, src9, src10);
2607 src += (8 * src_stride);
2608 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
2609 src32_r, src43_r, src54_r, src65_r);
2610 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
2611 src76_r, src87_r, src98_r, src109_r);
2612 ILVR_D4_SB(src43_r, src32_r, src65_r, src54_r, src87_r, src76_r, src109_r,
2613 src98_r, src4332, src6554, src8776, src10998);
2614 XORI_B4_128_SB(src4332, src6554, src8776, src10998);
2615 dst10 = const_vec;
2616 dst32 = const_vec;
2617 dst54 = const_vec;
2618 dst76 = const_vec;
2619 DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
2620 DPADD_SB2_SH(src4332, src6554, filt0, filt1, dst32, dst32);
2621 DPADD_SB2_SH(src6554, src8776, filt0, filt1, dst54, dst54);
2622 DPADD_SB2_SH(src8776, src10998, filt0, filt1, dst76, dst76);
2623 ST_D8(dst10, dst32, dst54, dst76, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
2624}
2625
2626static void hevc_vt_4t_4x16_msa(const uint8_t *src, int32_t src_stride,
2627 int16_t *dst, int32_t dst_stride,
2628 const int8_t *filter, int32_t height)
2629{
2630 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2631 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r;
2632 v16i8 src65_r, src87_r, src109_r, src2110, src4332, src6554, src8776;
2633 v16i8 src10998;
2634 v8i16 dst10, dst32, dst54, dst76, filt0, filt1, filter_vec, const_vec;
2635
2636 src -= src_stride;
2637 const_vec = __msa_ldi_h(128);
2638 const_vec <<= 6;
2639
2640 filter_vec = LD_SH(filter);
2641 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2642
2643 LD_SB3(src, src_stride, src0, src1, src2);
2644 src += (3 * src_stride);
2645
2646 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2647 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
2648 src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
2649
2650 LD_SB8(src, src_stride, src3, src4, src5, src6, src7, src8, src9, src10);
2651 src += (8 * src_stride);
2652 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5, src32_r, src43_r,
2653 src54_r, src65_r);
2654 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r,
2655 src87_r, src98_r, src109_r);
2656 ILVR_D4_SB(src43_r, src32_r, src65_r, src54_r, src87_r, src76_r, src109_r,
2657 src98_r, src4332, src6554, src8776, src10998);
2658 XORI_B4_128_SB(src4332, src6554, src8776, src10998);
2659
2660 dst10 = const_vec;
2661 dst32 = const_vec;
2662 dst54 = const_vec;
2663 dst76 = const_vec;
2664 DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
2665 DPADD_SB2_SH(src4332, src6554, filt0, filt1, dst32, dst32);
2666 DPADD_SB2_SH(src6554, src8776, filt0, filt1, dst54, dst54);
2667 DPADD_SB2_SH(src8776, src10998, filt0, filt1, dst76, dst76);
2668 ST_D8(dst10, dst32, dst54, dst76, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
2669 dst += (8 * dst_stride);
2670
2671 src2 = src10;
2672 src2110 = src10998;
2673
2674 LD_SB8(src, src_stride, src3, src4, src5, src6, src7, src8, src9, src10);
2675 src += (8 * src_stride);
2676
2677 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5, src32_r, src43_r,
2678 src54_r, src65_r);
2679 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r,
2680 src87_r, src98_r, src109_r);
2681 ILVR_D4_SB(src43_r, src32_r, src65_r, src54_r, src87_r, src76_r, src109_r,
2682 src98_r, src4332, src6554, src8776, src10998);
2683 XORI_B4_128_SB(src4332, src6554, src8776, src10998);
2684
2685 dst10 = const_vec;
2686 dst32 = const_vec;
2687 dst54 = const_vec;
2688 dst76 = const_vec;
2689 DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
2690 DPADD_SB2_SH(src4332, src6554, filt0, filt1, dst32, dst32);
2691 DPADD_SB2_SH(src6554, src8776, filt0, filt1, dst54, dst54);
2692 DPADD_SB2_SH(src8776, src10998, filt0, filt1, dst76, dst76);
2693 ST_D8(dst10, dst32, dst54, dst76, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
2694}
2695
2696static void hevc_vt_4t_4w_msa(const uint8_t *src,
2697 int32_t src_stride,
2698 int16_t *dst,
2699 int32_t dst_stride,
2700 const int8_t *filter,
2701 int32_t height)
2702{
2703 if (2 == height) {
2704 hevc_vt_4t_4x2_msa(src, src_stride, dst, dst_stride, filter);
2705 } else if (4 == height) {
2706 hevc_vt_4t_4x4_msa(src, src_stride, dst, dst_stride, filter, height);
2707 } else if (8 == height) {
2708 hevc_vt_4t_4x8_msa(src, src_stride, dst, dst_stride, filter, height);
2709 } else if (16 == height) {
2710 hevc_vt_4t_4x16_msa(src, src_stride, dst, dst_stride, filter, height);
2711 }
2712}
2713
2714static void hevc_vt_4t_6w_msa(const uint8_t *src,
2715 int32_t src_stride,
2716 int16_t *dst,
2717 int32_t dst_stride,
2718 const int8_t *filter,
2719 int32_t height)
2720{
2721 int32_t loop_cnt;
2722 int32_t res = height & 0x03;
2723 uint32_t dst_val_int0, dst_val_int1, dst_val_int2, dst_val_int3;
2724 uint64_t dst_val0, dst_val1, dst_val2, dst_val3;
2725 v16i8 src0, src1, src2, src3, src4;
2726 v16i8 src10_r, src32_r, src21_r, src43_r;
2727 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
2728 v8i16 filt0, filt1;
2729 v8i16 filter_vec, const_vec;
2730
2731 src -= src_stride;
2732 const_vec = __msa_ldi_h(128);
2733 const_vec <<= 6;
2734
2735 filter_vec = LD_SH(filter);
2736 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2737
2738 LD_SB3(src, src_stride, src0, src1, src2);
2739 src += (3 * src_stride);
2740 XORI_B3_128_SB(src0, src1, src2);
2741 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2742
2743 for (loop_cnt = (height >> 2); loop_cnt--;) {
2744 LD_SB2(src, src_stride, src3, src4);
2745 src += (2 * src_stride);
2746 XORI_B2_128_SB(src3, src4);
2747 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2748
2749 dst0_r = const_vec;
2750 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
2751 dst1_r = const_vec;
2752 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
2753
2754 LD_SB2(src, src_stride, src1, src2);
2755 src += (2 * src_stride);
2756 XORI_B2_128_SB(src1, src2);
2757 ILVR_B2_SB(src1, src4, src2, src1, src10_r, src21_r);
2758
2759 dst2_r = const_vec;
2760 DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, dst2_r, dst2_r);
2761 dst3_r = const_vec;
2762 DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, dst3_r, dst3_r);
2763
2764 dst_val0 = __msa_copy_u_d((v2i64) dst0_r, 0);
2765 dst_val1 = __msa_copy_u_d((v2i64) dst1_r, 0);
2766 dst_val2 = __msa_copy_u_d((v2i64) dst2_r, 0);
2767 dst_val3 = __msa_copy_u_d((v2i64) dst3_r, 0);
2768
2769 dst_val_int0 = __msa_copy_u_w((v4i32) dst0_r, 2);
2770 dst_val_int1 = __msa_copy_u_w((v4i32) dst1_r, 2);
2771 dst_val_int2 = __msa_copy_u_w((v4i32) dst2_r, 2);
2772 dst_val_int3 = __msa_copy_u_w((v4i32) dst3_r, 2);
2773
2774 SD(dst_val0, dst);
2775 SW(dst_val_int0, dst + 4);
2776 dst += dst_stride;
2777 SD(dst_val1, dst);
2778 SW(dst_val_int1, dst + 4);
2779 dst += dst_stride;
2780 SD(dst_val2, dst);
2781 SW(dst_val_int2, dst + 4);
2782 dst += dst_stride;
2783 SD(dst_val3, dst);
2784 SW(dst_val_int3, dst + 4);
2785 dst += dst_stride;
2786 }
2787 if (res) {
2788 LD_SB2(src, src_stride, src3, src4);
2789 XORI_B2_128_SB(src3, src4);
2790 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2791
2792 dst0_r = const_vec;
2793 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
2794 dst1_r = const_vec;
2795 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
2796
2797 dst_val0 = __msa_copy_u_d((v2i64) dst0_r, 0);
2798 dst_val1 = __msa_copy_u_d((v2i64) dst1_r, 0);
2799
2800 dst_val_int0 = __msa_copy_u_w((v4i32) dst0_r, 2);
2801 dst_val_int1 = __msa_copy_u_w((v4i32) dst1_r, 2);
2802
2803 SD(dst_val0, dst);
2804 SW(dst_val_int0, dst + 4);
2805 dst += dst_stride;
2806 SD(dst_val1, dst);
2807 SW(dst_val_int1, dst + 4);
2808 dst += dst_stride;
2809 }
2810}
2811
2812static void hevc_vt_4t_8x2_msa(const uint8_t *src,
2813 int32_t src_stride,
2814 int16_t *dst,
2815 int32_t dst_stride,
2816 const int8_t *filter)
2817{
2818 v16i8 src0, src1, src2, src3, src4;
2819 v16i8 src10_r, src32_r, src21_r, src43_r;
2820 v8i16 dst0_r, dst1_r;
2821 v8i16 filt0, filt1;
2822 v8i16 filter_vec, const_vec;
2823
2824 src -= src_stride;
2825 const_vec = __msa_ldi_h(128);
2826 const_vec <<= 6;
2827
2828 filter_vec = LD_SH(filter);
2829 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2830
2831 LD_SB3(src, src_stride, src0, src1, src2);
2832 src += (3 * src_stride);
2833 XORI_B3_128_SB(src0, src1, src2);
2834 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2835
2836 LD_SB2(src, src_stride, src3, src4);
2837 XORI_B2_128_SB(src3, src4);
2838 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2839 dst0_r = const_vec;
2840 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
2841 dst1_r = const_vec;
2842 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
2843
2844 ST_SH2(dst0_r, dst1_r, dst, dst_stride);
2845}
2846
2847static void hevc_vt_4t_8x6_msa(const uint8_t *src,
2848 int32_t src_stride,
2849 int16_t *dst,
2850 int32_t dst_stride,
2851 const int8_t *filter)
2852{
2853 v16i8 src0, src1, src2, src3, src4;
2854 v16i8 src10_r, src32_r, src21_r, src43_r;
2855 v8i16 dst0_r, dst1_r;
2856 v8i16 filt0, filt1;
2857 v8i16 filter_vec, const_vec;
2858
2859 src -= src_stride;
2860 const_vec = __msa_ldi_h(128);
2861 const_vec <<= 6;
2862
2863 filter_vec = LD_SH(filter);
2864 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2865
2866 LD_SB3(src, src_stride, src0, src1, src2);
2867 src += (3 * src_stride);
2868 XORI_B3_128_SB(src0, src1, src2);
2869 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2870
2871 LD_SB2(src, src_stride, src3, src4);
2872 src += (2 * src_stride);
2873 XORI_B2_128_SB(src3, src4);
2874
2875 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2876 dst0_r = const_vec;
2877 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
2878 dst1_r = const_vec;
2879 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
2880
2881 ST_SH2(dst0_r, dst1_r, dst, dst_stride);
2882 dst += (2 * dst_stride);
2883
2884 LD_SB2(src, src_stride, src1, src2);
2885 src += (2 * src_stride);
2886 XORI_B2_128_SB(src1, src2);
2887
2888 ILVR_B2_SB(src1, src4, src2, src1, src10_r, src21_r);
2889 dst0_r = const_vec;
2890 DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, dst0_r, dst0_r);
2891 dst1_r = const_vec;
2892 DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, dst1_r, dst1_r);
2893
2894 ST_SH2(dst0_r, dst1_r, dst, dst_stride);
2895 dst += (2 * dst_stride);
2896
2897 LD_SB2(src, src_stride, src3, src4);
2898 XORI_B2_128_SB(src3, src4);
2899
2900 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2901 dst0_r = const_vec;
2902 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
2903 dst1_r = const_vec;
2904 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
2905
2906 ST_SH2(dst0_r, dst1_r, dst, dst_stride);
2907}
2908
2909static void hevc_vt_4t_8x4multiple_msa(const uint8_t *src,
2910 int32_t src_stride,
2911 int16_t *dst,
2912 int32_t dst_stride,
2913 const int8_t *filter,
2914 int32_t height)
2915{
2916 int32_t loop_cnt;
2917 v16i8 src0, src1, src2, src3, src4, src5, src6;
2918 v16i8 src10_r, src32_r, src21_r, src43_r, src54_r, src65_r;
2919 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
2920 v8i16 filt0, filt1;
2921 v8i16 filter_vec, const_vec;
2922
2923 src -= src_stride;
2924 const_vec = __msa_ldi_h(128);
2925 const_vec <<= 6;
2926
2927 filter_vec = LD_SH(filter);
2928 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2929
2930 LD_SB3(src, src_stride, src0, src1, src2);
2931 src += (3 * src_stride);
2932 XORI_B3_128_SB(src0, src1, src2);
2933 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2934
2935 for (loop_cnt = (height >> 2); loop_cnt--;) {
2936 LD_SB4(src, src_stride, src3, src4, src5, src6);
2937 src += (4 * src_stride);
2938 XORI_B4_128_SB(src3, src4, src5, src6);
2939 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2940 ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
2941 dst0_r = const_vec;
2942 dst1_r = const_vec;
2943 dst2_r = const_vec;
2944 dst3_r = const_vec;
2945 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
2946 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
2947 DPADD_SB2_SH(src32_r, src54_r, filt0, filt1, dst2_r, dst2_r);
2948 DPADD_SB2_SH(src43_r, src65_r, filt0, filt1, dst3_r, dst3_r);
2949 ST_SH4(dst0_r, dst1_r, dst2_r, dst3_r, dst, dst_stride);
2950 dst += (4 * dst_stride);
2951
2952 src2 = src6;
2953 src10_r = src54_r;
2954 src21_r = src65_r;
2955 }
2956}
2957
2958static void hevc_vt_4t_8w_msa(const uint8_t *src,
2959 int32_t src_stride,
2960 int16_t *dst,
2961 int32_t dst_stride,
2962 const int8_t *filter,
2963 int32_t height)
2964{
2965 if (2 == height) {
2966 hevc_vt_4t_8x2_msa(src, src_stride, dst, dst_stride, filter);
2967 } else if (6 == height) {
2968 hevc_vt_4t_8x6_msa(src, src_stride, dst, dst_stride, filter);
2969 } else {
2970 hevc_vt_4t_8x4multiple_msa(src, src_stride, dst, dst_stride,
2971 filter, height);
2972 }
2973}
2974
2975static void hevc_vt_4t_12w_msa(const uint8_t *src,
2976 int32_t src_stride,
2977 int16_t *dst,
2978 int32_t dst_stride,
2979 const int8_t *filter,
2980 int32_t height)
2981{
2982 int32_t loop_cnt;
2983 v16i8 src0, src1, src2, src3, src4, src5, src6;
2984 v16i8 src10_r, src32_r, src21_r, src43_r;
2985 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
2986 v16i8 src10_l, src32_l, src54_l, src21_l, src43_l, src65_l;
2987 v16i8 src2110, src4332;
2988 v16i8 src54_r, src65_r, src6554;
2989 v8i16 dst0_l, dst1_l;
2990 v8i16 filt0, filt1;
2991 v8i16 filter_vec, const_vec;
2992
2993 src -= (1 * src_stride);
2994 const_vec = __msa_ldi_h(128);
2995 const_vec <<= 6;
2996
2997 filter_vec = LD_SH(filter);
2998 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2999
3000 LD_SB3(src, src_stride, src0, src1, src2);
3001 src += (3 * src_stride);
3002 XORI_B3_128_SB(src0, src1, src2);
3003 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3004 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
3005 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_l, (v2i64) src10_l);
3006
3007 for (loop_cnt = 4; loop_cnt--;) {
3008 LD_SB2(src, src_stride, src3, src4);
3009 src += (2 * src_stride);
3010 LD_SB2(src, src_stride, src5, src6);
3011 src += (2 * src_stride);
3012 XORI_B2_128_SB(src3, src4);
3013 XORI_B2_128_SB(src5, src6);
3014
3015 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3016 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
3017 src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_l, (v2i64) src32_l);
3018 ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
3019 ILVL_B2_SB(src5, src4, src6, src5, src54_l, src65_l);
3020 src6554 = (v16i8) __msa_ilvr_d((v2i64) src65_l, (v2i64) src54_l);
3021
3022 dst0_r = const_vec;
3023 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3024 dst1_r = const_vec;
3025 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3026 dst2_r = const_vec;
3027 DPADD_SB2_SH(src32_r, src54_r, filt0, filt1, dst2_r, dst2_r);
3028 dst3_r = const_vec;
3029 DPADD_SB2_SH(src43_r, src65_r, filt0, filt1, dst3_r, dst3_r);
3030 dst0_l = const_vec;
3031 DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst0_l, dst0_l);
3032 dst1_l = const_vec;
3033 DPADD_SB2_SH(src4332, src6554, filt0, filt1, dst1_l, dst1_l);
3034
3035 ST_SH4(dst0_r, dst1_r, dst2_r, dst3_r, dst, dst_stride);
3036 ST_D4(dst0_l, dst1_l, 0, 1, 0, 1, dst + 8, dst_stride);
3037 dst += (4 * dst_stride);
3038
3039 src2 = src6;
3040 src10_r = src54_r;
3041 src21_r = src65_r;
3042 src2110 = src6554;
3043 }
3044}
3045
3046static void hevc_vt_4t_16w_msa(const uint8_t *src,
3047 int32_t src_stride,
3048 int16_t *dst,
3049 int32_t dst_stride,
3050 const int8_t *filter,
3051 int32_t height)
3052{
3053 int32_t loop_cnt;
3054 v16i8 src0, src1, src2, src3, src4, src5;
3055 v16i8 src10_r, src32_r, src21_r, src43_r;
3056 v16i8 src10_l, src32_l, src21_l, src43_l;
3057 v8i16 dst0_r, dst1_r, dst0_l, dst1_l;
3058 v8i16 filt0, filt1;
3059 v8i16 filter_vec, const_vec;
3060
3061 src -= src_stride;
3062 const_vec = __msa_ldi_h(128);
3063 const_vec <<= 6;
3064
3065 filter_vec = LD_SH(filter);
3066 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3067
3068 LD_SB3(src, src_stride, src0, src1, src2);
3069 src += (3 * src_stride);
3070 XORI_B3_128_SB(src0, src1, src2);
3071 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3072 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
3073
3074 for (loop_cnt = (height >> 2); loop_cnt--;) {
3075 LD_SB2(src, src_stride, src3, src4);
3076 src += (2 * src_stride);
3077 XORI_B2_128_SB(src3, src4);
3078 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3079 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
3080 dst0_r = const_vec;
3081 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3082 dst0_l = const_vec;
3083 DPADD_SB2_SH(src10_l, src32_l, filt0, filt1, dst0_l, dst0_l);
3084 dst1_r = const_vec;
3085 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3086 dst1_l = const_vec;
3087 DPADD_SB2_SH(src21_l, src43_l, filt0, filt1, dst1_l, dst1_l);
3088 ST_SH2(dst0_r, dst0_l, dst, 8);
3089 dst += dst_stride;
3090 ST_SH2(dst1_r, dst1_l, dst, 8);
3091 dst += dst_stride;
3092
3093 LD_SB2(src, src_stride, src5, src2);
3094 src += (2 * src_stride);
3095 XORI_B2_128_SB(src5, src2);
3096 ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
3097 ILVL_B2_SB(src5, src4, src2, src5, src10_l, src21_l);
3098 dst0_r = const_vec;
3099 DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, dst0_r, dst0_r);
3100 dst0_l = const_vec;
3101 DPADD_SB2_SH(src32_l, src10_l, filt0, filt1, dst0_l, dst0_l);
3102 dst1_r = const_vec;
3103 DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, dst1_r, dst1_r);
3104 dst1_l = const_vec;
3105 DPADD_SB2_SH(src43_l, src21_l, filt0, filt1, dst1_l, dst1_l);
3106 ST_SH2(dst0_r, dst0_l, dst, 8);
3107 dst += dst_stride;
3108 ST_SH2(dst1_r, dst1_l, dst, 8);
3109 dst += dst_stride;
3110 }
3111}
3112
3113static void hevc_vt_4t_24w_msa(const uint8_t *src,
3114 int32_t src_stride,
3115 int16_t *dst,
3116 int32_t dst_stride,
3117 const int8_t *filter,
3118 int32_t height)
3119{
3120 int32_t loop_cnt;
3121 v16i8 src0, src1, src2, src3, src4, src5;
3122 v16i8 src6, src7, src8, src9, src10, src11;
3123 v16i8 src10_r, src32_r, src76_r, src98_r;
3124 v16i8 src21_r, src43_r, src87_r, src109_r;
3125 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
3126 v16i8 src10_l, src32_l, src21_l, src43_l;
3127 v8i16 dst0_l, dst1_l;
3128 v8i16 filt0, filt1;
3129 v8i16 filter_vec, const_vec;
3130
3131 src -= src_stride;
3132 const_vec = __msa_ldi_h(128);
3133 const_vec <<= 6;
3134
3135 filter_vec = LD_SH(filter);
3136 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3137
3138 LD_SB3(src, src_stride, src0, src1, src2);
3139 XORI_B3_128_SB(src0, src1, src2);
3140 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3141 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
3142
3143 LD_SB3(src + 16, src_stride, src6, src7, src8);
3144 src += (3 * src_stride);
3145 XORI_B3_128_SB(src6, src7, src8);
3146 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3147
3148 for (loop_cnt = (height >> 2); loop_cnt--;) {
3149 LD_SB2(src, src_stride, src3, src4);
3150 XORI_B2_128_SB(src3, src4);
3151 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3152 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
3153
3154 LD_SB2(src + 16, src_stride, src9, src10);
3155 src += (2 * src_stride);
3156 XORI_B2_128_SB(src9, src10);
3157 ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
3158
3159 dst0_r = const_vec;
3160 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3161 dst0_l = const_vec;
3162 DPADD_SB2_SH(src10_l, src32_l, filt0, filt1, dst0_l, dst0_l);
3163 dst1_r = const_vec;
3164 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3165 dst1_l = const_vec;
3166 DPADD_SB2_SH(src21_l, src43_l, filt0, filt1, dst1_l, dst1_l);
3167 dst2_r = const_vec;
3168 DPADD_SB2_SH(src76_r, src98_r, filt0, filt1, dst2_r, dst2_r);
3169 dst3_r = const_vec;
3170 DPADD_SB2_SH(src87_r, src109_r, filt0, filt1, dst3_r, dst3_r);
3171
3172 ST_SH2(dst0_r, dst0_l, dst, 8);
3173 ST_SH(dst2_r, dst + 16);
3174 dst += dst_stride;
3175 ST_SH2(dst1_r, dst1_l, dst, 8);
3176 ST_SH(dst3_r, dst + 16);
3177 dst += dst_stride;
3178
3179 LD_SB2(src, src_stride, src5, src2);
3180 XORI_B2_128_SB(src5, src2);
3181 ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
3182 ILVL_B2_SB(src5, src4, src2, src5, src10_l, src21_l);
3183
3184 LD_SB2(src + 16, src_stride, src11, src8);
3185 src += (2 * src_stride);
3186 XORI_B2_128_SB(src11, src8);
3187 ILVR_B2_SB(src11, src10, src8, src11, src76_r, src87_r);
3188
3189 dst0_r = const_vec;
3190 DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, dst0_r, dst0_r);
3191 dst0_l = const_vec;
3192 DPADD_SB2_SH(src32_l, src10_l, filt0, filt1, dst0_l, dst0_l);
3193 dst1_r = const_vec;
3194 DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, dst1_r, dst1_r);
3195 dst1_l = const_vec;
3196 DPADD_SB2_SH(src43_l, src21_l, filt0, filt1, dst1_l, dst1_l);
3197 dst2_r = const_vec;
3198 DPADD_SB2_SH(src98_r, src76_r, filt0, filt1, dst2_r, dst2_r);
3199 dst3_r = const_vec;
3200 DPADD_SB2_SH(src109_r, src87_r, filt0, filt1, dst3_r, dst3_r);
3201
3202 ST_SH2(dst0_r, dst0_l, dst, 8);
3203 ST_SH(dst2_r, dst + 16);
3204 dst += dst_stride;
3205 ST_SH2(dst1_r, dst1_l, dst, 8);
3206 ST_SH(dst3_r, dst + 16);
3207 dst += dst_stride;
3208 }
3209}
3210
3211static void hevc_vt_4t_32w_msa(const uint8_t *src,
3212 int32_t src_stride,
3213 int16_t *dst,
3214 int32_t dst_stride,
3215 const int8_t *filter,
3216 int32_t height)
3217{
3218 int32_t loop_cnt;
3219 v16i8 src0, src1, src2, src3, src4, src5;
3220 v16i8 src6, src7, src8, src9, src10, src11;
3221 v16i8 src10_r, src32_r, src76_r, src98_r;
3222 v16i8 src21_r, src43_r, src87_r, src109_r;
3223 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
3224 v16i8 src10_l, src32_l, src76_l, src98_l;
3225 v16i8 src21_l, src43_l, src87_l, src109_l;
3226 v8i16 dst0_l, dst1_l, dst2_l, dst3_l;
3227 v8i16 filt0, filt1;
3228 v8i16 filter_vec, const_vec;
3229
3230 src -= src_stride;
3231 const_vec = __msa_ldi_h(128);
3232 const_vec <<= 6;
3233
3234 filter_vec = LD_SH(filter);
3235 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3236
3237 LD_SB3(src, src_stride, src0, src1, src2);
3238 XORI_B3_128_SB(src0, src1, src2);
3239 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3240 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
3241
3242 LD_SB3(src + 16, src_stride, src6, src7, src8);
3243 src += (3 * src_stride);
3244 XORI_B3_128_SB(src6, src7, src8);
3245 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3246 ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
3247
3248 for (loop_cnt = (height >> 2); loop_cnt--;) {
3249 LD_SB2(src, src_stride, src3, src4);
3250 XORI_B2_128_SB(src3, src4);
3251 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3252 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
3253
3254 LD_SB2(src + 16, src_stride, src9, src10);
3255 src += (2 * src_stride);
3256 XORI_B2_128_SB(src9, src10);
3257 ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
3258 ILVL_B2_SB(src9, src8, src10, src9, src98_l, src109_l);
3259
3260 dst0_r = const_vec;
3261 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3262 dst0_l = const_vec;
3263 DPADD_SB2_SH(src10_l, src32_l, filt0, filt1, dst0_l, dst0_l);
3264 dst1_r = const_vec;
3265 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3266 dst1_l = const_vec;
3267 DPADD_SB2_SH(src21_l, src43_l, filt0, filt1, dst1_l, dst1_l);
3268 dst2_r = const_vec;
3269 DPADD_SB2_SH(src76_r, src98_r, filt0, filt1, dst2_r, dst2_r);
3270 dst2_l = const_vec;
3271 DPADD_SB2_SH(src76_l, src98_l, filt0, filt1, dst2_l, dst2_l);
3272 dst3_r = const_vec;
3273 DPADD_SB2_SH(src87_r, src109_r, filt0, filt1, dst3_r, dst3_r);
3274 dst3_l = const_vec;
3275 DPADD_SB2_SH(src87_l, src109_l, filt0, filt1, dst3_l, dst3_l);
3276
3277 ST_SH4(dst0_r, dst0_l, dst2_r, dst2_l, dst, 8);
3278 dst += dst_stride;
3279 ST_SH4(dst1_r, dst1_l, dst3_r, dst3_l, dst, 8);
3280 dst += dst_stride;
3281
3282 LD_SB2(src, src_stride, src5, src2);
3283 XORI_B2_128_SB(src5, src2);
3284 ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
3285 ILVL_B2_SB(src5, src4, src2, src5, src10_l, src21_l);
3286
3287 LD_SB2(src + 16, src_stride, src11, src8);
3288 src += (2 * src_stride);
3289 XORI_B2_128_SB(src11, src8);
3290 ILVR_B2_SB(src11, src10, src8, src11, src76_r, src87_r);
3291 ILVL_B2_SB(src11, src10, src8, src11, src76_l, src87_l);
3292
3293 dst0_r = const_vec;
3294 DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, dst0_r, dst0_r);
3295 dst0_l = const_vec;
3296 DPADD_SB2_SH(src32_l, src10_l, filt0, filt1, dst0_l, dst0_l);
3297 dst1_r = const_vec;
3298 DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, dst1_r, dst1_r);
3299 dst1_l = const_vec;
3300 DPADD_SB2_SH(src43_l, src21_l, filt0, filt1, dst1_l, dst1_l);
3301 dst2_r = const_vec;
3302 DPADD_SB2_SH(src98_r, src76_r, filt0, filt1, dst2_r, dst2_r);
3303 dst2_l = const_vec;
3304 DPADD_SB2_SH(src98_l, src76_l, filt0, filt1, dst2_l, dst2_l);
3305 dst3_r = const_vec;
3306 DPADD_SB2_SH(src109_r, src87_r, filt0, filt1, dst3_r, dst3_r);
3307 dst3_l = const_vec;
3308 DPADD_SB2_SH(src109_l, src87_l, filt0, filt1, dst3_l, dst3_l);
3309
3310 ST_SH4(dst0_r, dst0_l, dst2_r, dst2_l, dst, 8);
3311 dst += dst_stride;
3312 ST_SH4(dst1_r, dst1_l, dst3_r, dst3_l, dst, 8);
3313 dst += dst_stride;
3314 }
3315}
3316
3317static void hevc_hv_4t_4x2_msa(const uint8_t *src,
3318 int32_t src_stride,
3319 int16_t *dst,
3320 int32_t dst_stride,
3321 const int8_t *filter_x,
3322 const int8_t *filter_y)
3323{
3324 v16i8 src0, src1, src2, src3, src4;
3325 v8i16 filt0, filt1;
3326 v8i16 filt_h0, filt_h1;
3327 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
3328 v16i8 mask1;
3329 v8i16 filter_vec, const_vec;
3330 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
3331 v8i16 dst20, dst31, dst42, dst10, dst32, dst21, dst43;
3332 v4i32 dst0, dst1;
3333
3334 src -= (src_stride + 1);
3335 filter_vec = LD_SH(filter_x);
3336 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3337
3338 filter_vec = LD_SH(filter_y);
3339 UNPCK_R_SB_SH(filter_vec, filter_vec);
3340
3341 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3342
3343 mask1 = mask0 + 2;
3344
3345 const_vec = __msa_ldi_h(128);
3346 const_vec <<= 6;
3347
3348 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
3349 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3350 VSHF_B2_SB(src0, src2, src0, src2, mask0, mask1, vec0, vec1);
3351 VSHF_B2_SB(src1, src3, src1, src3, mask0, mask1, vec2, vec3);
3352 VSHF_B2_SB(src2, src4, src2, src4, mask0, mask1, vec4, vec5);
3353
3354 dst20 = const_vec;
3355 dst31 = const_vec;
3356 dst42 = const_vec;
3357 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst20, dst20);
3358 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst31, dst31);
3359 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst42, dst42);
3360 ILVRL_H2_SH(dst31, dst20, dst10, dst32);
3361 ILVRL_H2_SH(dst42, dst31, dst21, dst43);
3362
3363 dst0 = HEVC_FILT_4TAP(dst10, dst32, filt_h0, filt_h1);
3364 dst1 = HEVC_FILT_4TAP(dst21, dst43, filt_h0, filt_h1);
3365 dst0 >>= 6;
3366 dst1 >>= 6;
3367 dst0 = (v4i32) __msa_pckev_h((v8i16) dst1, (v8i16) dst0);
3368 ST_D2(dst0, 0, 1, dst, dst_stride);
3369}
3370
3371static void hevc_hv_4t_4x4_msa(const uint8_t *src,
3372 int32_t src_stride,
3373 int16_t *dst,
3374 int32_t dst_stride,
3375 const int8_t *filter_x,
3376 const int8_t *filter_y)
3377{
3378 v16i8 src0, src1, src2, src3, src4, src5, src6;
3379 v8i16 filt0, filt1;
3380 v8i16 filt_h0, filt_h1;
3381 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
3382 v16i8 mask1;
3383 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3384 v8i16 filter_vec, const_vec;
3385 v8i16 dst30, dst41, dst52, dst63, dst10, dst32, dst54, dst21, dst43, dst65;
3386 v4i32 dst0, dst1, dst2, dst3;
3387
3388 src -= (src_stride + 1);
3389
3390 filter_vec = LD_SH(filter_x);
3391 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3392
3393 filter_vec = LD_SH(filter_y);
3394 UNPCK_R_SB_SH(filter_vec, filter_vec);
3395
3396 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3397
3398 mask1 = mask0 + 2;
3399
3400 const_vec = __msa_ldi_h(128);
3401 const_vec <<= 6;
3402
3403 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
3404 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
3405
3406 VSHF_B2_SB(src0, src3, src0, src3, mask0, mask1, vec0, vec1);
3407 VSHF_B2_SB(src1, src4, src1, src4, mask0, mask1, vec2, vec3);
3408 VSHF_B2_SB(src2, src5, src2, src5, mask0, mask1, vec4, vec5);
3409 VSHF_B2_SB(src3, src6, src3, src6, mask0, mask1, vec6, vec7);
3410
3411 dst30 = const_vec;
3412 dst41 = const_vec;
3413 dst52 = const_vec;
3414 dst63 = const_vec;
3415 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst30, dst30);
3416 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst41, dst41);
3417 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst52, dst52);
3418 DPADD_SB2_SH(vec6, vec7, filt0, filt1, dst63, dst63);
3419
3420 ILVRL_H2_SH(dst41, dst30, dst10, dst43);
3421 ILVRL_H2_SH(dst52, dst41, dst21, dst54);
3422 ILVRL_H2_SH(dst63, dst52, dst32, dst65);
3423
3424 dst0 = HEVC_FILT_4TAP(dst10, dst32, filt_h0, filt_h1);
3425 dst1 = HEVC_FILT_4TAP(dst21, dst43, filt_h0, filt_h1);
3426 dst2 = HEVC_FILT_4TAP(dst32, dst54, filt_h0, filt_h1);
3427 dst3 = HEVC_FILT_4TAP(dst43, dst65, filt_h0, filt_h1);
3428 SRA_4V(dst0, dst1, dst2, dst3, 6);
3429 PCKEV_H2_SW(dst1, dst0, dst3, dst2, dst0, dst2);
3430 ST_D4(dst0, dst2, 0, 1, 0, 1, dst, dst_stride);
3431}
3432
3433
3434static void hevc_hv_4t_4multx8mult_msa(const uint8_t *src,
3435 int32_t src_stride,
3436 int16_t *dst,
3437 int32_t dst_stride,
3438 const int8_t *filter_x,
3439 const int8_t *filter_y,
3440 int32_t height)
3441{
3442 uint32_t loop_cnt;
3443 v16i8 src0, src1, src2, src3, src4, src5, src6;
3444 v16i8 src7, src8, src9, src10;
3445 v8i16 filt0, filt1;
3446 v8i16 filt_h0, filt_h1;
3447 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
3448 v16i8 mask1;
3449 v8i16 filter_vec, const_vec;
3450 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3451 v8i16 dst10, dst21, dst22, dst73, dst84, dst95, dst106;
3452 v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst98_r;
3453 v8i16 dst21_r, dst43_r, dst65_r, dst87_r, dst109_r;
3454 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3455
3456 src -= (src_stride + 1);
3457 filter_vec = LD_SH(filter_x);
3458 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3459
3460 filter_vec = LD_SH(filter_y);
3461 UNPCK_R_SB_SH(filter_vec, filter_vec);
3462
3463 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3464
3465 mask1 = mask0 + 2;
3466
3467 const_vec = __msa_ldi_h(128);
3468 const_vec <<= 6;
3469
3470 LD_SB3(src, src_stride, src0, src1, src2);
3471 src += (3 * src_stride);
3472 XORI_B3_128_SB(src0, src1, src2);
3473 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
3474 VSHF_B2_SB(src1, src2, src1, src2, mask0, mask1, vec2, vec3);
3475 dst10 = const_vec;
3476 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst10, dst10);
3477 dst21 = const_vec;
3478 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst21, dst21);
3479 ILVRL_H2_SH(dst21, dst10, dst10_r, dst21_r);
3480 dst22 = (v8i16) __msa_splati_d((v2i64) dst21, 1);
3481
3482 for (loop_cnt = height >> 3; loop_cnt--;) {
3483 LD_SB8(src, src_stride,
3484 src3, src4, src5, src6, src7, src8, src9, src10);
3485 src += (8 * src_stride);
3486 XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
3487
3488 VSHF_B2_SB(src3, src7, src3, src7, mask0, mask1, vec0, vec1);
3489 VSHF_B2_SB(src4, src8, src4, src8, mask0, mask1, vec2, vec3);
3490 VSHF_B2_SB(src5, src9, src5, src9, mask0, mask1, vec4, vec5);
3491 VSHF_B2_SB(src6, src10, src6, src10, mask0, mask1, vec6, vec7);
3492
3493 dst73 = const_vec;
3494 dst84 = const_vec;
3495 dst95 = const_vec;
3496 dst106 = const_vec;
3497 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst73, dst73);
3498 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst84, dst84);
3499 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst95, dst95);
3500 DPADD_SB2_SH(vec6, vec7, filt0, filt1, dst106, dst106);
3501
3502 dst32_r = __msa_ilvr_h(dst73, dst22);
3503 ILVRL_H2_SH(dst84, dst73, dst43_r, dst87_r);
3504 ILVRL_H2_SH(dst95, dst84, dst54_r, dst98_r);
3505 ILVRL_H2_SH(dst106, dst95, dst65_r, dst109_r);
3506 dst22 = (v8i16) __msa_splati_d((v2i64) dst73, 1);
3507 dst76_r = __msa_ilvr_h(dst22, dst106);
3508
3509 dst0 = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3510 dst1 = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3511 dst2 = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3512 dst3 = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3513 dst4 = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
3514 dst5 = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
3515 dst6 = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
3516 dst7 = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
3517 SRA_4V(dst0, dst1, dst2, dst3, 6);
3518 SRA_4V(dst4, dst5, dst6, dst7, 6);
3519 PCKEV_H4_SW(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
3520 dst0, dst1, dst2, dst3);
3521 ST_D8(dst0, dst1, dst2, dst3, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
3522 dst += (8 * dst_stride);
3523
3524 dst10_r = dst98_r;
3525 dst21_r = dst109_r;
3526 dst22 = (v8i16) __msa_splati_d((v2i64) dst106, 1);
3527 }
3528}
3529
3530static void hevc_hv_4t_4w_msa(const uint8_t *src,
3531 int32_t src_stride,
3532 int16_t *dst,
3533 int32_t dst_stride,
3534 const int8_t *filter_x,
3535 const int8_t *filter_y,
3536 int32_t height)
3537{
3538 if (2 == height) {
3539 hevc_hv_4t_4x2_msa(src, src_stride, dst, dst_stride,
3540 filter_x, filter_y);
3541 } else if (4 == height) {
3542 hevc_hv_4t_4x4_msa(src, src_stride, dst, dst_stride,
3543 filter_x, filter_y);
3544 } else if (0 == (height % 8)) {
3545 hevc_hv_4t_4multx8mult_msa(src, src_stride, dst, dst_stride,
3546 filter_x, filter_y, height);
3547 }
3548}
3549
3550static void hevc_hv_4t_6w_msa(const uint8_t *src,
3551 int32_t src_stride,
3552 int16_t *dst,
3553 int32_t dst_stride,
3554 const int8_t *filter_x,
3555 const int8_t *filter_y,
3556 int32_t height)
3557{
3558 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3559 v8i16 filt0, filt1;
3560 v8i16 filt_h0, filt_h1;
3561 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
3562 v16i8 mask1;
3563 v8i16 filter_vec, const_vec;
3564 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3565 v8i16 dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6, dsth7, dsth8, dsth9;
3566 v8i16 dsth10, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
3567 v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst98_r, dst21_r, dst43_r;
3568 v8i16 dst65_r, dst87_r, dst109_r, dst10_l, dst32_l, dst54_l, dst76_l;
3569 v8i16 dst98_l, dst21_l, dst43_l, dst65_l, dst87_l, dst109_l;
3570 v8i16 dst1021_l, dst3243_l, dst5465_l, dst7687_l, dst98109_l;
3571 v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r, dst6_r, dst7_r;
3572 v4i32 dst0_l, dst1_l, dst2_l, dst3_l;
3573
3574 src -= (src_stride + 1);
3575 filter_vec = LD_SH(filter_x);
3576 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3577
3578 filter_vec = LD_SH(filter_y);
3579 UNPCK_R_SB_SH(filter_vec, filter_vec);
3580
3581 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3582
3583 mask1 = mask0 + 2;
3584
3585 const_vec = __msa_ldi_h(128);
3586 const_vec <<= 6;
3587
3588 LD_SB3(src, src_stride, src0, src1, src2);
3589 src += (3 * src_stride);
3590 XORI_B3_128_SB(src0, src1, src2);
3591
3592 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3593 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3594 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3595
3596 dsth0 = const_vec;
3597 dsth1 = const_vec;
3598 dsth2 = const_vec;
3599 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dsth0, dsth0);
3600 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dsth1, dsth1);
3601 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dsth2, dsth2);
3602
3603 ILVRL_H2_SH(dsth1, dsth0, dst10_r, dst10_l);
3604 ILVRL_H2_SH(dsth2, dsth1, dst21_r, dst21_l);
3605
3606 LD_SB8(src, src_stride, src3, src4, src5, src6, src7, src8, src9, src10);
3607 XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
3608
3609 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3610 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
3611 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
3612 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
3613
3614 dsth3 = const_vec;
3615 dsth4 = const_vec;
3616 dsth5 = const_vec;
3617 dsth6 = const_vec;
3618 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dsth3, dsth3);
3619 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dsth4, dsth4);
3620 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dsth5, dsth5);
3621 DPADD_SB2_SH(vec6, vec7, filt0, filt1, dsth6, dsth6);
3622
3623 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
3624 VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec2, vec3);
3625 VSHF_B2_SB(src9, src9, src9, src9, mask0, mask1, vec4, vec5);
3626 VSHF_B2_SB(src10, src10, src10, src10, mask0, mask1, vec6, vec7);
3627
3628 dsth7 = const_vec;
3629 dsth8 = const_vec;
3630 dsth9 = const_vec;
3631 dsth10 = const_vec;
3632 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dsth7, dsth7);
3633 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dsth8, dsth8);
3634 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dsth9, dsth9);
3635 DPADD_SB2_SH(vec6, vec7, filt0, filt1, dsth10, dsth10);
3636
3637 ILVRL_H2_SH(dsth3, dsth2, dst32_r, dst32_l);
3638 ILVRL_H2_SH(dsth4, dsth3, dst43_r, dst43_l);
3639 ILVRL_H2_SH(dsth5, dsth4, dst54_r, dst54_l);
3640 ILVRL_H2_SH(dsth6, dsth5, dst65_r, dst65_l);
3641 ILVRL_H2_SH(dsth7, dsth6, dst76_r, dst76_l);
3642 ILVRL_H2_SH(dsth8, dsth7, dst87_r, dst87_l);
3643 ILVRL_H2_SH(dsth9, dsth8, dst98_r, dst98_l);
3644 ILVRL_H2_SH(dsth10, dsth9, dst109_r, dst109_l);
3645
3646 PCKEV_D2_SH(dst21_l, dst10_l, dst43_l, dst32_l, dst1021_l, dst3243_l);
3647 PCKEV_D2_SH(dst65_l, dst54_l, dst87_l, dst76_l, dst5465_l, dst7687_l);
3648 dst98109_l = (v8i16) __msa_pckev_d((v2i64) dst109_l, (v2i64) dst98_l);
3649
3650 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3651 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3652 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3653 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3654 dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
3655 dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
3656 dst6_r = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
3657 dst7_r = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
3658 dst0_l = HEVC_FILT_4TAP(dst1021_l, dst3243_l, filt_h0, filt_h1);
3659 dst1_l = HEVC_FILT_4TAP(dst3243_l, dst5465_l, filt_h0, filt_h1);
3660 dst2_l = HEVC_FILT_4TAP(dst5465_l, dst7687_l, filt_h0, filt_h1);
3661 dst3_l = HEVC_FILT_4TAP(dst7687_l, dst98109_l, filt_h0, filt_h1);
3662 SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
3663 SRA_4V(dst4_r, dst5_r, dst6_r, dst7_r, 6);
3664 SRA_4V(dst0_l, dst1_l, dst2_l, dst3_l, 6);
3665 PCKEV_H2_SH(dst1_r, dst0_r, dst3_r, dst2_r, tmp0, tmp1);
3666 PCKEV_H2_SH(dst5_r, dst4_r, dst7_r, dst6_r, tmp2, tmp3);
3667 PCKEV_H2_SH(dst1_l, dst0_l, dst3_l, dst2_l, tmp4, tmp5);
3668 ST_D4(tmp0, tmp1, 0, 1, 0, 1, dst, dst_stride);
3669 ST_W4(tmp4, 0, 1, 2, 3, dst + 4, dst_stride);
3670 dst += 4 * dst_stride;
3671 ST_D4(tmp2, tmp3, 0, 1, 0, 1, dst, dst_stride);
3672 ST_W4(tmp5, 0, 1, 2, 3, dst + 4, dst_stride);
3673}
3674
3675static void hevc_hv_4t_8x2_msa(const uint8_t *src,
3676 int32_t src_stride,
3677 int16_t *dst,
3678 int32_t dst_stride,
3679 const int8_t *filter_x,
3680 const int8_t *filter_y)
3681{
3682 v16i8 src0, src1, src2, src3, src4;
3683 v8i16 filt0, filt1;
3684 v8i16 filt_h0, filt_h1;
3685 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
3686 v16i8 mask1;
3687 v8i16 filter_vec, const_vec;
3688 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
3689 v8i16 dst0, dst1, dst2, dst3, dst4;
3690 v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
3691 v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
3692 v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
3693
3694 src -= (src_stride + 1);
3695
3696 filter_vec = LD_SH(filter_x);
3697 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3698
3699 filter_vec = LD_SH(filter_y);
3700 UNPCK_R_SB_SH(filter_vec, filter_vec);
3701
3702 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3703
3704 mask1 = mask0 + 2;
3705
3706 const_vec = __msa_ldi_h(128);
3707 const_vec <<= 6;
3708
3709 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
3710 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3711
3712 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3713 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3714 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3715 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
3716 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec8, vec9);
3717
3718 dst0 = const_vec;
3719 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
3720 dst1 = const_vec;
3721 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
3722 dst2 = const_vec;
3723 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
3724 dst3 = const_vec;
3725 DPADD_SB2_SH(vec6, vec7, filt0, filt1, dst3, dst3);
3726 dst4 = const_vec;
3727 DPADD_SB2_SH(vec8, vec9, filt0, filt1, dst4, dst4);
3728
3729 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
3730 ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
3731 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
3732 ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
3733 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3734 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
3735 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3736 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
3737 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
3738 PCKEV_H2_SW(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r);
3739 ST_SW2(dst0_r, dst1_r, dst, dst_stride);
3740}
3741
3742static void hevc_hv_4t_8multx4_msa(const uint8_t *src, int32_t src_stride,
3743 int16_t *dst, int32_t dst_stride,
3744 const int8_t *filter_x,
3745 const int8_t *filter_y, int32_t width8mult)
3746{
3747 int32_t cnt;
3748 v16i8 src0, src1, src2, src3, src4, src5, src6, mask0, mask1;
3749 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3750 v8i16 filt0, filt1, filt_h0, filt_h1, filter_vec, const_vec;
3751 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6;
3752 v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
3753 v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
3754 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
3755
3756 src -= (src_stride + 1);
3757
3758 filter_vec = LD_SH(filter_x);
3759 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3760
3761 filter_vec = LD_SH(filter_y);
3762 UNPCK_R_SB_SH(filter_vec, filter_vec);
3763
3764 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3765
3766 mask0 = LD_SB(ff_hevc_mask_arr);
3767 mask1 = mask0 + 2;
3768
3769 const_vec = __msa_ldi_h(128);
3770 const_vec <<= 6;
3771
3772 for (cnt = width8mult; cnt--;) {
3773 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
3774 src += 8;
3775 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
3776
3777 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3778 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3779 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3780
3781 dst0 = const_vec;
3782 dst1 = const_vec;
3783 dst2 = const_vec;
3784 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
3785 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
3786 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
3787
3788 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
3789 ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
3790
3791 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3792 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
3793 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
3794 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
3795 dst3 = const_vec;
3796 dst4 = const_vec;
3797 dst5 = const_vec;
3798 dst6 = const_vec;
3799 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
3800 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst4, dst4);
3801 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst5, dst5);
3802 DPADD_SB2_SH(vec6, vec7, filt0, filt1, dst6, dst6);
3803 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
3804 ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
3805 ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
3806 ILVRL_H2_SH(dst6, dst5, dst65_r, dst65_l);
3807 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3808 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
3809 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3810 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
3811
3812 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3813 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
3814 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3815 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
3816 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
3817 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
3818 PCKEV_H2_SW(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r);
3819 PCKEV_H2_SW(dst2_l, dst2_r, dst3_l, dst3_r, dst2_r, dst3_r);
3820
3821 ST_SW4(dst0_r, dst1_r, dst2_r, dst3_r, dst, dst_stride);
3822 dst += 8;
3823 }
3824}
3825
3826static void hevc_hv_4t_8x6_msa(const uint8_t *src,
3827 int32_t src_stride,
3828 int16_t *dst,
3829 int32_t dst_stride,
3830 const int8_t *filter_x,
3831 const int8_t *filter_y)
3832{
3833 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3834 v8i16 filt0, filt1;
3835 v8i16 filt_h0, filt_h1;
3836 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
3837 v16i8 mask1;
3838 v8i16 filter_vec, const_vec;
3839 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
3840 v16i8 vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17;
3841 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
3842 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
3843 v4i32 dst4_r, dst4_l, dst5_r, dst5_l;
3844 v8i16 dst10_r, dst32_r, dst10_l, dst32_l;
3845 v8i16 dst21_r, dst43_r, dst21_l, dst43_l;
3846 v8i16 dst54_r, dst54_l, dst65_r, dst65_l;
3847 v8i16 dst76_r, dst76_l, dst87_r, dst87_l;
3848
3849 src -= (src_stride + 1);
3850
3851 filter_vec = LD_SH(filter_x);
3852 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3853
3854 filter_vec = LD_SH(filter_y);
3855 UNPCK_R_SB_SH(filter_vec, filter_vec);
3856
3857 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3858
3859 mask1 = mask0 + 2;
3860
3861 const_vec = __msa_ldi_h(128);
3862 const_vec <<= 6;
3863
3864 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
3865 src += (5 * src_stride);
3866 LD_SB4(src, src_stride, src5, src6, src7, src8);
3867
3868 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3869 XORI_B4_128_SB(src5, src6, src7, src8);
3870
3871 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3872 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3873 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3874 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
3875 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec8, vec9);
3876 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec10, vec11);
3877 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec12, vec13);
3878 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec14, vec15);
3879 VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec16, vec17);
3880
3881 dst0 = const_vec;
3882 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
3883 dst1 = const_vec;
3884 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
3885 dst2 = const_vec;
3886 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
3887 dst3 = const_vec;
3888 DPADD_SB2_SH(vec6, vec7, filt0, filt1, dst3, dst3);
3889 dst4 = const_vec;
3890 DPADD_SB2_SH(vec8, vec9, filt0, filt1, dst4, dst4);
3891 dst5 = const_vec;
3892 DPADD_SB2_SH(vec10, vec11, filt0, filt1, dst5, dst5);
3893 dst6 = const_vec;
3894 DPADD_SB2_SH(vec12, vec13, filt0, filt1, dst6, dst6);
3895 dst7 = const_vec;
3896 DPADD_SB2_SH(vec14, vec15, filt0, filt1, dst7, dst7);
3897 dst8 = const_vec;
3898 DPADD_SB2_SH(vec16, vec17, filt0, filt1, dst8, dst8);
3899
3900 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
3901 ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
3902 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
3903 ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
3904 ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
3905 ILVRL_H2_SH(dst6, dst5, dst65_r, dst65_l);
3906 ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
3907 ILVRL_H2_SH(dst8, dst7, dst87_r, dst87_l);
3908
3909 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3910 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
3911 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3912 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
3913 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3914 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
3915 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3916 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
3917 dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
3918 dst4_l = HEVC_FILT_4TAP(dst54_l, dst76_l, filt_h0, filt_h1);
3919 dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
3920 dst5_l = HEVC_FILT_4TAP(dst65_l, dst87_l, filt_h0, filt_h1);
3921
3922 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
3923 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
3924 SRA_4V(dst4_r, dst4_l, dst5_r, dst5_l, 6);
3925
3926 PCKEV_H4_SW(dst0_l, dst0_r, dst1_l, dst1_r,
3927 dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r, dst2_r, dst3_r);
3928 PCKEV_H2_SW(dst4_l, dst4_r, dst5_l, dst5_r, dst4_r, dst5_r);
3929
3930 ST_SW2(dst0_r, dst1_r, dst, dst_stride);
3931 dst += (2 * dst_stride);
3932 ST_SW2(dst2_r, dst3_r, dst, dst_stride);
3933 dst += (2 * dst_stride);
3934 ST_SW2(dst4_r, dst5_r, dst, dst_stride);
3935}
3936
3937static void hevc_hv_4t_8multx4mult_msa(const uint8_t *src,
3938 int32_t src_stride,
3939 int16_t *dst,
3940 int32_t dst_stride,
3941 const int8_t *filter_x,
3942 const int8_t *filter_y,
3943 int32_t height,
3944 int32_t width8mult)
3945{
3946 uint32_t loop_cnt, cnt;
3947 const uint8_t *src_tmp;
3948 int16_t *dst_tmp;
3949 v16i8 src0, src1, src2, src3, src4, src5, src6;
3950 v8i16 filt0, filt1;
3951 v8i16 filt_h0, filt_h1;
3952 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
3953 v16i8 mask1;
3954 v8i16 filter_vec, const_vec;
3955 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3956 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6;
3957 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
3958 v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
3959 v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
3960
3961 src -= (src_stride + 1);
3962
3963 filter_vec = LD_SH(filter_x);
3964 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3965
3966 filter_vec = LD_SH(filter_y);
3967 UNPCK_R_SB_SH(filter_vec, filter_vec);
3968
3969 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3970
3971 mask1 = mask0 + 2;
3972
3973 const_vec = __msa_ldi_h(128);
3974 const_vec <<= 6;
3975
3976 for (cnt = width8mult; cnt--;) {
3977 src_tmp = src;
3978 dst_tmp = dst;
3979
3980 LD_SB3(src_tmp, src_stride, src0, src1, src2);
3981 src_tmp += (3 * src_stride);
3982
3983 XORI_B3_128_SB(src0, src1, src2);
3984
3985 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3986 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3987 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3988
3989 dst0 = const_vec;
3990 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
3991 dst1 = const_vec;
3992 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
3993 dst2 = const_vec;
3994 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
3995
3996 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
3997 ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
3998
3999 for (loop_cnt = height >> 2; loop_cnt--;) {
4000 LD_SB4(src_tmp, src_stride, src3, src4, src5, src6);
4001 src_tmp += (4 * src_stride);
4002 XORI_B4_128_SB(src3, src4, src5, src6);
4003
4004 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
4005 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
4006 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
4007 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
4008
4009 dst3 = const_vec;
4010 dst4 = const_vec;
4011 dst5 = const_vec;
4012 dst6 = const_vec;
4013 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
4014 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst4, dst4);
4015 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst5, dst5);
4016 DPADD_SB2_SH(vec6, vec7, filt0, filt1, dst6, dst6);
4017
4018 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
4019 ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
4020 ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
4021 ILVRL_H2_SH(dst6, dst5, dst65_r, dst65_l);
4022
4023 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
4024 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
4025 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
4026 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
4027 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
4028 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
4029 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
4030 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
4031
4032 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
4033 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
4034
4035 PCKEV_H4_SW(dst0_l, dst0_r, dst1_l, dst1_r,
4036 dst2_l, dst2_r, dst3_l, dst3_r,
4037 dst0_r, dst1_r, dst2_r, dst3_r);
4038
4039 ST_SW4(dst0_r, dst1_r, dst2_r, dst3_r, dst_tmp, dst_stride);
4040 dst_tmp += (4 * dst_stride);
4041
4042 dst10_r = dst54_r;
4043 dst10_l = dst54_l;
4044 dst21_r = dst65_r;
4045 dst21_l = dst65_l;
4046 dst2 = dst6;
4047 }
4048
4049 src += 8;
4050 dst += 8;
4051 }
4052}
4053
4054static void hevc_hv_4t_8w_msa(const uint8_t *src,
4055 int32_t src_stride,
4056 int16_t *dst,
4057 int32_t dst_stride,
4058 const int8_t *filter_x,
4059 const int8_t *filter_y,
4060 int32_t height)
4061{
4062
4063 if (2 == height) {
4064 hevc_hv_4t_8x2_msa(src, src_stride, dst, dst_stride,
4065 filter_x, filter_y);
4066 } else if (4 == height) {
4067 hevc_hv_4t_8multx4_msa(src, src_stride, dst, dst_stride,
4068 filter_x, filter_y, 1);
4069 } else if (6 == height) {
4070 hevc_hv_4t_8x6_msa(src, src_stride, dst, dst_stride,
4071 filter_x, filter_y);
4072 } else if (0 == (height % 4)) {
4073 hevc_hv_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
4074 filter_x, filter_y, height, 1);
4075 }
4076}
4077
4078static void hevc_hv_4t_12w_msa(const uint8_t *src,
4079 int32_t src_stride,
4080 int16_t *dst,
4081 int32_t dst_stride,
4082 const int8_t *filter_x,
4083 const int8_t *filter_y,
4084 int32_t height)
4085{
4086 uint32_t loop_cnt;
4087 const uint8_t *src_tmp;
4088 int16_t *dst_tmp;
4089 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
4090 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
4091 v16i8 mask0, mask1, mask2, mask3;
4092 v8i16 filt0, filt1, filt_h0, filt_h1, filter_vec, const_vec;
4093 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst10, dst21, dst22, dst73;
4094 v8i16 dst84, dst95, dst106, dst76_r, dst98_r, dst87_r, dst109_r;
4095 v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
4096 v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
4097 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
4098 v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4099
4100 src -= (src_stride + 1);
4101
4102 filter_vec = LD_SH(filter_x);
4103 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4104
4105 filter_vec = LD_SH(filter_y);
4106 UNPCK_R_SB_SH(filter_vec, filter_vec);
4107
4108 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
4109
4110 mask0 = LD_SB(ff_hevc_mask_arr);
4111 mask1 = mask0 + 2;
4112
4113 const_vec = __msa_ldi_h(128);
4114 const_vec <<= 6;
4115
4116 src_tmp = src;
4117 dst_tmp = dst;
4118
4119 LD_SB3(src_tmp, src_stride, src0, src1, src2);
4120 src_tmp += (3 * src_stride);
4121
4122 XORI_B3_128_SB(src0, src1, src2);
4123
4124 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
4125 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
4126 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
4127
4128 dst0 = const_vec;
4129 dst1 = const_vec;
4130 dst2 = const_vec;
4131 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
4132 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
4133 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
4134
4135 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
4136 ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
4137
4138 for (loop_cnt = 4; loop_cnt--;) {
4139 LD_SB4(src_tmp, src_stride, src3, src4, src5, src6);
4140 src_tmp += (4 * src_stride);
4141 XORI_B4_128_SB(src3, src4, src5, src6);
4142
4143 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
4144 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
4145 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
4146 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
4147
4148 dst3 = const_vec;
4149 dst4 = const_vec;
4150 dst5 = const_vec;
4151 dst6 = const_vec;
4152 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
4153 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst4, dst4);
4154 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst5, dst5);
4155 DPADD_SB2_SH(vec6, vec7, filt0, filt1, dst6, dst6);
4156
4157 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
4158 ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
4159 ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
4160 ILVRL_H2_SH(dst6, dst5, dst65_r, dst65_l);
4161
4162 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
4163 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
4164 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
4165 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
4166 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
4167 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
4168 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
4169 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
4170
4171 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
4172 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
4173 PCKEV_H4_SW(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l,
4174 dst3_r, dst0_r, dst1_r, dst2_r, dst3_r);
4175 ST_SW4(dst0_r, dst1_r, dst2_r, dst3_r, dst_tmp, dst_stride);
4176 dst_tmp += (4 * dst_stride);
4177
4178 dst10_r = dst54_r;
4179 dst10_l = dst54_l;
4180 dst21_r = dst65_r;
4181 dst21_l = dst65_l;
4182 dst2 = dst6;
4183 }
4184
4185 src += 8;
4186 dst += 8;
4187
4188 mask2 = LD_SB(ff_hevc_mask_arr + 16);
4189 mask3 = mask2 + 2;
4190
4191 LD_SB3(src, src_stride, src0, src1, src2);
4192 src += (3 * src_stride);
4193 XORI_B3_128_SB(src0, src1, src2);
4194 VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
4195 VSHF_B2_SB(src1, src2, src1, src2, mask2, mask3, vec2, vec3);
4196 dst10 = const_vec;
4197 dst21 = const_vec;
4198 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst10, dst10);
4199 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst21, dst21);
4200 ILVRL_H2_SH(dst21, dst10, dst10_r, dst21_r);
4201 dst22 = (v8i16) __msa_splati_d((v2i64) dst21, 1);
4202
4203 for (loop_cnt = 2; loop_cnt--;) {
4204 LD_SB8(src, src_stride, src3, src4, src5, src6, src7, src8, src9,
4205 src10);
4206 src += (8 * src_stride);
4207 XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
4208 VSHF_B2_SB(src3, src7, src3, src7, mask2, mask3, vec0, vec1);
4209 VSHF_B2_SB(src4, src8, src4, src8, mask2, mask3, vec2, vec3);
4210 VSHF_B2_SB(src5, src9, src5, src9, mask2, mask3, vec4, vec5);
4211 VSHF_B2_SB(src6, src10, src6, src10, mask2, mask3, vec6, vec7);
4212
4213 dst73 = const_vec;
4214 dst84 = const_vec;
4215 dst95 = const_vec;
4216 dst106 = const_vec;
4217 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst73, dst73);
4218 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst84, dst84);
4219 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst95, dst95);
4220 DPADD_SB2_SH(vec6, vec7, filt0, filt1, dst106, dst106);
4221
4222 dst32_r = __msa_ilvr_h(dst73, dst22);
4223 ILVRL_H2_SH(dst84, dst73, dst43_r, dst87_r);
4224 ILVRL_H2_SH(dst95, dst84, dst54_r, dst98_r);
4225 ILVRL_H2_SH(dst106, dst95, dst65_r, dst109_r);
4226 dst22 = (v8i16) __msa_splati_d((v2i64) dst73, 1);
4227 dst76_r = __msa_ilvr_h(dst22, dst106);
4228
4229 tmp0 = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
4230 tmp1 = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
4231 tmp2 = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
4232 tmp3 = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
4233 tmp4 = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
4234 tmp5 = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
4235 tmp6 = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
4236 tmp7 = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
4237
4238 SRA_4V(tmp0, tmp1, tmp2, tmp3, 6);
4239 SRA_4V(tmp4, tmp5, tmp6, tmp7, 6);
4240 PCKEV_H4_SW(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, tmp0, tmp1,
4241 tmp2, tmp3);
4242 ST_D8(tmp0, tmp1, tmp2, tmp3, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
4243 dst += (8 * dst_stride);
4244
4245 dst10_r = dst98_r;
4246 dst21_r = dst109_r;
4247 dst22 = (v8i16) __msa_splati_d((v2i64) dst106, 1);
4248 }
4249}
4250
4251static void hevc_hv_4t_16w_msa(const uint8_t *src,
4252 int32_t src_stride,
4253 int16_t *dst,
4254 int32_t dst_stride,
4255 const int8_t *filter_x,
4256 const int8_t *filter_y,
4257 int32_t height)
4258{
4259 if (4 == height) {
4260 hevc_hv_4t_8multx4_msa(src, src_stride, dst, dst_stride,
4261 filter_x, filter_y, 2);
4262 } else {
4263 hevc_hv_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
4264 filter_x, filter_y, height, 2);
4265 }
4266}
4267
4268static void hevc_hv_4t_24w_msa(const uint8_t *src,
4269 int32_t src_stride,
4270 int16_t *dst,
4271 int32_t dst_stride,
4272 const int8_t *filter_x,
4273 const int8_t *filter_y,
4274 int32_t height)
4275{
4276 hevc_hv_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
4277 filter_x, filter_y, height, 3);
4278}
4279
4280static void hevc_hv_4t_32w_msa(const uint8_t *src,
4281 int32_t src_stride,
4282 int16_t *dst,
4283 int32_t dst_stride,
4284 const int8_t *filter_x,
4285 const int8_t *filter_y,
4286 int32_t height)
4287{
4288 hevc_hv_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
4289 filter_x, filter_y, height, 4);
4290}
4291
4292#define MC_COPY(WIDTH) \
4293void ff_hevc_put_hevc_pel_pixels##WIDTH##_8_msa(int16_t *dst, \
4294 const uint8_t *src, \
4295 ptrdiff_t src_stride, \
4296 int height, \
4297 intptr_t mx, \
4298 intptr_t my, \
4299 int width) \
4300{ \
4301 hevc_copy_##WIDTH##w_msa(src, src_stride, dst, MAX_PB_SIZE, height); \
4302}
4303
4304MC_COPY(4);
4305MC_COPY(6);
4306MC_COPY(8);
4307MC_COPY(12);
4308MC_COPY(16);
4309MC_COPY(24);
4310MC_COPY(32);
4311MC_COPY(48);
4312MC_COPY(64);
4313
4314#undef MC_COPY
4315
4316#define MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR) \
4317void ff_hevc_put_hevc_##PEL##_##DIR##WIDTH##_8_msa(int16_t *dst, \
4318 const uint8_t *src, \
4319 ptrdiff_t src_stride, \
4320 int height, \
4321 intptr_t mx, \
4322 intptr_t my, \
4323 int width) \
4324{ \
4325 const int8_t *filter = ff_hevc_##PEL##_filters[FILT_DIR]; \
4326 \
4327 hevc_##DIR1##_##TAP##t_##WIDTH##w_msa(src, src_stride, dst, \
4328 MAX_PB_SIZE, filter, height); \
4329}
4330
4331MC(qpel, h, 4, 8, hz, mx);
4332MC(qpel, h, 8, 8, hz, mx);
4333MC(qpel, h, 12, 8, hz, mx);
4334MC(qpel, h, 16, 8, hz, mx);
4335MC(qpel, h, 24, 8, hz, mx);
4336MC(qpel, h, 32, 8, hz, mx);
4337MC(qpel, h, 48, 8, hz, mx);
4338MC(qpel, h, 64, 8, hz, mx);
4339
4340MC(qpel, v, 4, 8, vt, my);
4341MC(qpel, v, 8, 8, vt, my);
4342MC(qpel, v, 12, 8, vt, my);
4343MC(qpel, v, 16, 8, vt, my);
4344MC(qpel, v, 24, 8, vt, my);
4345MC(qpel, v, 32, 8, vt, my);
4346MC(qpel, v, 48, 8, vt, my);
4347MC(qpel, v, 64, 8, vt, my);
4348
4349MC(epel, h, 4, 4, hz, mx);
4350MC(epel, h, 6, 4, hz, mx);
4351MC(epel, h, 8, 4, hz, mx);
4352MC(epel, h, 12, 4, hz, mx);
4353MC(epel, h, 16, 4, hz, mx);
4354MC(epel, h, 24, 4, hz, mx);
4355MC(epel, h, 32, 4, hz, mx);
4356
4357MC(epel, v, 4, 4, vt, my);
4358MC(epel, v, 6, 4, vt, my);
4359MC(epel, v, 8, 4, vt, my);
4360MC(epel, v, 12, 4, vt, my);
4361MC(epel, v, 16, 4, vt, my);
4362MC(epel, v, 24, 4, vt, my);
4363MC(epel, v, 32, 4, vt, my);
4364
4365#undef MC
4366
4367#define MC_HV(PEL, WIDTH, TAP) \
4368void ff_hevc_put_hevc_##PEL##_hv##WIDTH##_8_msa(int16_t *dst, \
4369 const uint8_t *src, \
4370 ptrdiff_t src_stride, \
4371 int height, \
4372 intptr_t mx, \
4373 intptr_t my, \
4374 int width) \
4375{ \
4376 const int8_t *filter_x = ff_hevc_##PEL##_filters[mx]; \
4377 const int8_t *filter_y = ff_hevc_##PEL##_filters[my]; \
4378 \
4379 hevc_hv_##TAP##t_##WIDTH##w_msa(src, src_stride, dst, MAX_PB_SIZE, \
4380 filter_x, filter_y, height); \
4381}
4382
4383MC_HV(qpel, 4, 8);
4384MC_HV(qpel, 8, 8);
4385MC_HV(qpel, 12, 8);
4386MC_HV(qpel, 16, 8);
4387MC_HV(qpel, 24, 8);
4388MC_HV(qpel, 32, 8);
4389MC_HV(qpel, 48, 8);
4390MC_HV(qpel, 64, 8);
4391
4392MC_HV(epel, 4, 4);
4393MC_HV(epel, 6, 4);
4394MC_HV(epel, 8, 4);
4395MC_HV(epel, 12, 4);
4396MC_HV(epel, 16, 4);
4397MC_HV(epel, 24, 4);
4398MC_HV(epel, 32, 4);
4399
4400#undef MC_HV
my
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t my
Definition dsp.h:57
mx
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t mx
Definition dsp.h:57
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition dsp.h:87
int32_t
int32_t
Definition audioconvert.c:56
SD
#define SD
Definition ccaption_dec.c:944
int64_t
long long int64_t
Definition coverity.c:34
aligned
static int aligned(int val)
Definition dashdec.c:178
generic_macros_msa.h
ILVR_B2_SB
#define ILVR_B2_SB(...)
Definition generic_macros_msa.h:1338
PCKEV_H2_SW
#define PCKEV_H2_SW(...)
Definition generic_macros_msa.h:1760
XORI_B8_128_SB
#define XORI_B8_128_SB(...)
Definition generic_macros_msa.h:1880
SPLATI_W4_SH
#define SPLATI_W4_SH(...)
Definition generic_macros_msa.h:1700
ST_SW4
#define ST_SW4(...)
Definition generic_macros_msa.h:377
ST_D4
#define ST_D4(in0, in1, idx0, idx1, idx2, idx3, pdst, stride)
Definition generic_macros_msa.h:499
ILVR_D3_SB
#define ILVR_D3_SB(...)
Definition generic_macros_msa.h:1452
SPLATI_H2_SH
#define SPLATI_H2_SH(...)
Definition generic_macros_msa.h:1656
XORI_B2_128_SB
#define XORI_B2_128_SB(...)
Definition generic_macros_msa.h:1835
XORI_B7_128_SB
#define XORI_B7_128_SB(...)
Definition generic_macros_msa.h:1873
ILVR_B4_SB
#define ILVR_B4_SB(...)
Definition generic_macros_msa.h:1360
ST_SH2
#define ST_SH2(...)
Definition generic_macros_msa.h:366
SPLATI_W2_SH
#define SPLATI_W2_SH(...)
Definition generic_macros_msa.h:1692
ST_SH8
#define ST_SH8(...)
Definition generic_macros_msa.h:392
VSHF_B4_SB
#define VSHF_B4_SB(...)
Definition generic_macros_msa.h:680
ST_SH
#define ST_SH(...)
Definition generic_macros_msa.h:43
DPADD_SB2_SH
#define DPADD_SB2_SH(...)
Definition generic_macros_msa.h:833
ILVR_B4_SH
#define ILVR_B4_SH(...)
Definition generic_macros_msa.h:1362
LD_SB7
#define LD_SB7(...)
Definition generic_macros_msa.h:327
ST_SH6
#define ST_SH6(...)
Definition generic_macros_msa.h:384
ILVL_B4_SB
#define ILVL_B4_SB(...)
Definition generic_macros_msa.h:1274
ILVR_D4_SB
#define ILVR_D4_SB(...)
Definition generic_macros_msa.h:1460
XORI_B4_128_SB
#define XORI_B4_128_SB(...)
Definition generic_macros_msa.h:1851
ST12x8_UB
#define ST12x8_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride)
Definition generic_macros_msa.h:527
LD_SB3
#define LD_SB3(...)
Definition generic_macros_msa.h:289
ILVRL_H2_SH
#define ILVRL_H2_SH(...)
Definition generic_macros_msa.h:1508
UNPCK_R_SB_SH
#define UNPCK_R_SB_SH(in, out)
Definition generic_macros_msa.h:2156
ST_D8
#define ST_D8(in0, in1, in2, in3, idx0, idx1, idx2, idx3, idx4, idx5, idx6, idx7, pdst, stride)
Definition generic_macros_msa.h:511
XORI_B3_128_SB
#define XORI_B3_128_SB(...)
Definition generic_macros_msa.h:1843
LD_SB5
#define LD_SB5(...)
Definition generic_macros_msa.h:308
PCKEV_D2_SH
#define PCKEV_D2_SH(...)
Definition generic_macros_msa.h:1789
PCKEV_H2_SH
#define PCKEV_H2_SH(...)
Definition generic_macros_msa.h:1759
ILVR_D2_SB
#define ILVR_D2_SB(...)
Definition generic_macros_msa.h:1444
ST_SW2
#define ST_SW2(...)
Definition generic_macros_msa.h:367
ST_SW
#define ST_SW(...)
Definition generic_macros_msa.h:45
SRA_4V
#define SRA_4V(in0, in1, in2, in3, shift)
Definition generic_macros_msa.h:1939
LD_SB8
#define LD_SB8(...)
Definition generic_macros_msa.h:336
SPLATI_H4_SH
#define SPLATI_H4_SH(...)
Definition generic_macros_msa.h:1674
PCKEV_H4_SW
#define PCKEV_H4_SW(...)
Definition generic_macros_msa.h:1769
SLLI_4V
#define SLLI_4V(in0, in1, in2, in3, shift)
Definition generic_macros_msa.h:1921
SW
#define SW(val, pdst)
Definition generic_macros_msa.h:167
SD4
#define SD4(in0, in1, in2, in3, pdst, stride)
Definition generic_macros_msa.h:256
LD_SB4
#define LD_SB4(...)
Definition generic_macros_msa.h:297
LD_SB6
#define LD_SB6(...)
Definition generic_macros_msa.h:316
VSHF_B2_SB
#define VSHF_B2_SB(...)
Definition generic_macros_msa.h:662
ILVR_W2_SB
#define ILVR_W2_SB(...)
Definition generic_macros_msa.h:1417
ILVR_B2_SH
#define ILVR_B2_SH(...)
Definition generic_macros_msa.h:1340
DPADD_SB4_SH
#define DPADD_SB4_SH(...)
Definition generic_macros_msa.h:841
LD_SB
#define LD_SB(...)
Definition generic_macros_msa.h:33
LD_SH
#define LD_SH(...)
Definition generic_macros_msa.h:35
LD_SB2
#define LD_SB2(...)
Definition generic_macros_msa.h:278
XORI_B5_128_SB
#define XORI_B5_128_SB(...)
Definition generic_macros_msa.h:1859
ST_UH
#define ST_UH(...)
Definition generic_macros_msa.h:42
ILVL_B4_SH
#define ILVL_B4_SH(...)
Definition generic_macros_msa.h:1276
ST_D2
#define ST_D2(in, idx0, idx1, pdst, stride)
Definition generic_macros_msa.h:491
ST_SH4
#define ST_SH4(...)
Definition generic_macros_msa.h:376
ILVR_W4_SB
#define ILVR_W4_SB(...)
Definition generic_macros_msa.h:1426
ILVL_B2_SB
#define ILVL_B2_SB(...)
Definition generic_macros_msa.h:1263
ILVL_W2_SB
#define ILVL_W2_SB(...)
Definition generic_macros_msa.h:1319
ILVL_B2_SH
#define ILVL_B2_SH(...)
Definition generic_macros_msa.h:1265
src2
const pixel * src2
Definition h264pred_template.c:421
hevc_macros_msa.h
HEVC_FILT_4TAP
#define HEVC_FILT_4TAP(in0, in1, filt0, filt1)
Definition hevc_macros_msa.h:64
HEVC_FILT_8TAP
#define HEVC_FILT_8TAP(in0, in1, in2, in3, filt0, filt1, filt2, filt3)
Definition hevc_macros_msa.h:35
ff_hevc_mask_arr
static const uint8_t ff_hevc_mask_arr[16 *2]
Definition hevc_mc_bi_lsx.c:26
hevcdsp_mips.h
hevc_vt_4t_8x6_msa
static void hevc_vt_4t_8x6_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter)
Definition hevcdsp_msa.c:2847
hevc_vt_4t_32w_msa
static void hevc_vt_4t_32w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:3211
hevc_vt_8t_48w_msa
static void hevc_vt_8t_48w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:1402
hevc_hv_8t_32w_msa
static void hevc_hv_8t_32w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:1874
hevc_vt_4t_4x4_msa
static void hevc_vt_4t_4x4_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2540
hevc_vt_8t_24w_msa
static void hevc_vt_8t_24w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:1384
hevc_vt_4t_12w_msa
static void hevc_vt_4t_12w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2975
hevc_hz_4t_12w_msa
static void hevc_hz_4t_12w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2221
hevc_hv_4t_16w_msa
static void hevc_hv_4t_16w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:4251
hevc_vt_4t_4x8_msa
static void hevc_vt_4t_4x8_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2577
hevc_hz_8t_64w_msa
static void hevc_hz_8t_64w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:937
hevc_copy_8w_msa
static void hevc_copy_8w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_msa.c:118
hevc_copy_24w_msa
static void hevc_copy_24w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_msa.c:321
hevc_hz_4t_8x2multiple_msa
static void hevc_hz_4t_8x2multiple_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2108
hevc_hv_4t_8x2_msa
static void hevc_hv_4t_8x2_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y)
Definition hevcdsp_msa.c:3675
hevc_vt_4t_8x2_msa
static void hevc_vt_4t_8x2_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter)
Definition hevcdsp_msa.c:2812
hevc_hz_4t_8w_msa
static void hevc_hz_4t_8w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2205
hevc_copy_32w_msa
static void hevc_copy_32w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_msa.c:350
hevc_hz_4t_4x4_msa
static void hevc_hz_4t_4x4_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter)
Definition hevcdsp_msa.c:1934
hevc_hv_4t_4w_msa
static void hevc_hv_4t_4w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:3530
hevc_vt_4t_16w_msa
static void hevc_vt_4t_16w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:3046
hevc_hz_4t_32w_msa
static void hevc_hz_4t_32w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2455
hevc_vt_4t_8w_msa
static void hevc_vt_4t_8w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2958
MC_HV
#define MC_HV(PEL, WIDTH, TAP)
Definition hevcdsp_msa.c:4367
hevc_hv_8t_24w_msa
static void hevc_hv_8t_24w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:1865
hevc_vt_4t_6w_msa
static void hevc_vt_4t_6w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2714
hevc_hz_8t_24w_msa
static void hevc_hz_8t_24w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:731
hevc_vt_8t_16w_msa
static void hevc_vt_8t_16w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:1376
hevc_vt_4t_8x4multiple_msa
static void hevc_vt_4t_8x4multiple_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2909
hevc_hz_4t_4x8multiple_msa
static void hevc_hz_4t_4x8multiple_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:1971
hevc_hv_4t_24w_msa
static void hevc_hv_4t_24w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:4268
hevc_hz_8t_8w_msa
static void hevc_hz_8t_8w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:544
hevc_hz_8t_16w_msa
static void hevc_hz_8t_16w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:674
hevc_hv_8t_64w_msa
static void hevc_hv_8t_64w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:1892
hevc_vt_4t_24w_msa
static void hevc_vt_4t_24w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:3113
hevc_hv_4t_6w_msa
static void hevc_hv_4t_6w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:3550
hevc_copy_12w_msa
static void hevc_copy_12w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_msa.c:179
MC
#define MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR)
Definition hevcdsp_msa.c:4316
hevc_copy_16w_msa
static void hevc_copy_16w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_msa.c:230
hevc_hv_4t_4multx8mult_msa
static void hevc_hv_4t_4multx8mult_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:3434
hevc_hv_8t_48w_msa
static void hevc_hv_8t_48w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:1883
hevc_vt_8t_4w_msa
static void hevc_vt_8t_4w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:1031
hevc_hv_4t_4x4_msa
static void hevc_hv_4t_4x4_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y)
Definition hevcdsp_msa.c:3371
hevc_hv_8t_4w_msa
static void hevc_hv_8t_4w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:1418
MC_COPY
#define MC_COPY(WIDTH)
Definition hevcdsp_msa.c:4292
hevc_vt_8t_64w_msa
static void hevc_vt_8t_64w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:1410
hevc_hv_4t_8w_msa
static void hevc_hv_4t_8w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:4054
hevc_hv_8t_12w_msa
static void hevc_hv_8t_12w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:1661
hevc_hv_8t_8w_msa
static void hevc_hv_8t_8w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:1652
hevc_hv_4t_8multx4mult_msa
static void hevc_hv_4t_8multx4mult_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t width8mult)
Definition hevcdsp_msa.c:3937
hevc_hv_8t_16w_msa
static void hevc_hv_8t_16w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:1856
hevc_hz_8t_32w_msa
static void hevc_hz_8t_32w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:806
hevc_vt_8t_16multx4mult_msa
static void hevc_vt_8t_16multx4mult_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t width)
Definition hevcdsp_msa.c:1268
hevc_hz_4t_16w_msa
static void hevc_hz_4t_16w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2282
hevc_hv_4t_4x2_msa
static void hevc_hv_4t_4x2_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y)
Definition hevcdsp_msa.c:3317
hevc_copy_6w_msa
static void hevc_copy_6w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_msa.c:80
hevc_hz_4t_6w_msa
static void hevc_hz_4t_6w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2037
hevc_vt_4t_4x16_msa
static void hevc_vt_4t_4x16_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2626
hevc_copy_48w_msa
static void hevc_copy_48w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_msa.c:388
hevc_hz_4t_24w_msa
static void hevc_hz_4t_24w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2354
hevc_hv_4t_8multx4_msa
static void hevc_hv_4t_8multx4_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t width8mult)
Definition hevcdsp_msa.c:3742
hevc_hv_4t_32w_msa
static void hevc_hv_4t_32w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:4280
hevc_copy_64w_msa
static void hevc_copy_64w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_msa.c:439
hevc_hv_4t_12w_msa
static void hevc_hv_4t_12w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_msa.c:4078
hevc_hz_4t_8x4multiple_msa
static void hevc_hz_4t_8x4multiple_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2152
hevc_hv_8t_8multx1mult_msa
static void hevc_hv_8t_8multx1mult_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t width)
Definition hevcdsp_msa.c:1525
hevc_copy_4w_msa
static void hevc_copy_4w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_msa.c:32
hevc_vt_8t_12w_msa
static void hevc_vt_8t_12w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:1181
hevc_vt_4t_4x2_msa
static void hevc_vt_4t_4x2_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter)
Definition hevcdsp_msa.c:2507
hevc_vt_8t_32w_msa
static void hevc_vt_8t_32w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:1394
hevc_hz_4t_4x2_msa
static void hevc_hz_4t_4x2_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter)
Definition hevcdsp_msa.c:1901
hevc_vt_8t_8w_msa
static void hevc_vt_8t_8w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:1118
hevc_hz_4t_4w_msa
static void hevc_hz_4t_4w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2020
hevc_hz_8t_4w_msa
static void hevc_hz_8t_4w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:476
hevc_hz_8t_12w_msa
static void hevc_hz_8t_12w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:599
hevc_hz_8t_48w_msa
static void hevc_hz_8t_48w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:866
hevc_vt_4t_4w_msa
static void hevc_vt_4t_4w_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_msa.c:2696
hevc_hv_4t_8x6_msa
static void hevc_hv_4t_8x6_msa(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y)
Definition hevcdsp_msa.c:3826
zero
static int zero(InterplayACMContext *s, unsigned ind, unsigned col)
Definition interplayacm.c:121
filter
void(* filter)(uint8_t *src, ptrdiff_t stride, int qscale)
Definition h263dsp.c:29
src1
#define src1
Definition h264pred.c:141
src0
#define src0
Definition h264pred.c:140
src
#define src
Definition vp8dsp.c:248
height
#define height
Definition dsp.h:89
width
#define width
Definition dsp.h:89
ST_W4
#define ST_W4(in, idx0, idx1, idx2, idx3, pdst, stride)
Definition vp8_lpf_lsx.c:234
h
h
Definition vp9dsp_template.c:2070
Generated on Sun Aug 16 2026 20:20:49 for FFmpeg by doxygen 1.13.2