nengel@2: /*
nengel@2:  * H.26L/H.264/AVC/JVT/14496-10/... loop filter
nengel@2:  * Copyright (c) 2003 Michael Niedermayer <michaelni@gmx.at>
nengel@2:  *
nengel@2:  * This file is part of FFmpeg.
nengel@2:  *
nengel@2:  * FFmpeg is free software; you can redistribute it and/or
nengel@2:  * modify it under the terms of the GNU Lesser General Public
nengel@2:  * License as published by the Free Software Foundation; either
nengel@2:  * version 2.1 of the License, or (at your option) any later version.
nengel@2:  *
nengel@2:  * FFmpeg is distributed in the hope that it will be useful,
nengel@2:  * but WITHOUT ANY WARRANTY; without even the implied warranty of
nengel@2:  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
nengel@2:  * Lesser General Public License for more details.
nengel@2:  *
nengel@2:  * You should have received a copy of the GNU Lesser General Public
nengel@2:  * License along with FFmpeg; if not, write to the Free Software
nengel@2:  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
nengel@2:  */
nengel@2: 
nengel@2: /**
nengel@2:  * @file
nengel@2:  * H.264 / AVC / MPEG4 part10 loop filter.
nengel@2:  * @author Michael Niedermayer <michaelni@gmx.at>
nengel@2:  */
nengel@2: 
nengel@2: #include "dsputil.h"
nengel@2: #include "mathops.h"
nengel@2: #include "rectangle.h"
nengel@2: #include "h264_types.h"
nengel@2: #include "h264_misc.h"
nengel@2: #include "h264_data.h"
nengel@2: //#undef NDEBUG
nengel@2: #include <assert.h>
nengel@2: 
nengel@2: /* Deblocking filter (p153) */
nengel@2: static const uint8_t alpha_table[52*3] = {
nengel@2:      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
nengel@2:      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
nengel@2:      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
nengel@2:      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
nengel@2:      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
nengel@2:      0,  0,  0,  0,  0,  0,  4,  4,  5,  6,
nengel@2:      7,  8,  9, 10, 12, 13, 15, 17, 20, 22,
nengel@2:     25, 28, 32, 36, 40, 45, 50, 56, 63, 71,
nengel@2:     80, 90,101,113,127,144,162,182,203,226,
nengel@2:    255,255,
nengel@2:    255,255,255,255,255,255,255,255,255,255,255,255,255,
nengel@2:    255,255,255,255,255,255,255,255,255,255,255,255,255,
nengel@2:    255,255,255,255,255,255,255,255,255,255,255,255,255,
nengel@2:    255,255,255,255,255,255,255,255,255,255,255,255,255,
nengel@2: };
nengel@2: static const uint8_t beta_table[52*3] = {
nengel@2:      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
nengel@2:      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
nengel@2:      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
nengel@2:      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
nengel@2:      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
nengel@2:      0,  0,  0,  0,  0,  0,  2,  2,  2,  3,
nengel@2:      3,  3,  3,  4,  4,  4,  6,  6,  7,  7,
nengel@2:      8,  8,  9,  9, 10, 10, 11, 11, 12, 12,
nengel@2:     13, 13, 14, 14, 15, 15, 16, 16, 17, 17,
nengel@2:     18, 18,
nengel@2:     18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
nengel@2:     18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
nengel@2:     18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
nengel@2:     18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
nengel@2: };
nengel@2: static const uint8_t tc0_table[52*3][4] = {
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 },
nengel@2:     {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 0 }, {-1, 0, 0, 1 },
nengel@2:     {-1, 0, 0, 1 }, {-1, 0, 0, 1 }, {-1, 0, 0, 1 }, {-1, 0, 1, 1 }, {-1, 0, 1, 1 }, {-1, 1, 1, 1 },
nengel@2:     {-1, 1, 1, 1 }, {-1, 1, 1, 1 }, {-1, 1, 1, 1 }, {-1, 1, 1, 2 }, {-1, 1, 1, 2 }, {-1, 1, 1, 2 },
nengel@2:     {-1, 1, 1, 2 }, {-1, 1, 2, 3 }, {-1, 1, 2, 3 }, {-1, 2, 2, 3 }, {-1, 2, 2, 4 }, {-1, 2, 3, 4 },
nengel@2:     {-1, 2, 3, 4 }, {-1, 3, 3, 5 }, {-1, 3, 4, 6 }, {-1, 3, 4, 6 }, {-1, 4, 5, 7 }, {-1, 4, 5, 8 },
nengel@2:     {-1, 4, 6, 9 }, {-1, 5, 7,10 }, {-1, 6, 8,11 }, {-1, 6, 8,13 }, {-1, 7,10,14 }, {-1, 8,11,16 },
nengel@2:     {-1, 9,12,18 }, {-1,10,13,20 }, {-1,11,15,23 }, {-1,13,17,25 },
nengel@2:     {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 },
nengel@2:     {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 },
nengel@2:     {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 },
nengel@2:     {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 },
nengel@2:     {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 },
nengel@2:     {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 },
nengel@2:     {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 },
nengel@2:     {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 },
nengel@2:     {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 }, {-1,13,17,25 },
nengel@2: };
nengel@2: 
nengel@2: av_always_inline static void filter_mb_edgev( uint8_t *pix, int stride, int16_t bS[4], unsigned int qp, MBRecContext *mrc, H264Slice *s) {
nengel@2:     const unsigned int index_a = qp + s->slice_alpha_c0_offset;
nengel@2:     const int alpha = alpha_table[index_a];
nengel@2:     const int beta  = beta_table[qp + s->slice_beta_offset];
nengel@2:     if (alpha ==0 || beta == 0) return;
nengel@2: 
nengel@2:     if( bS[0] < 4 ) {
nengel@2:         int8_t tc[4];
nengel@2:         tc[0] = tc0_table[index_a][bS[0]];
nengel@2:         tc[1] = tc0_table[index_a][bS[1]];
nengel@2:         tc[2] = tc0_table[index_a][bS[2]];
nengel@2:         tc[3] = tc0_table[index_a][bS[3]];
nengel@2:         mrc->hdsp.h264_h_loop_filter_luma(pix, stride, alpha, beta, tc);
nengel@2:     } else {
nengel@2:         mrc->hdsp.h264_h_loop_filter_luma_intra(pix, stride, alpha, beta);
nengel@2:     }
nengel@2: }
nengel@2: 
nengel@2: av_always_inline static void filter_mb_edgecv( uint8_t *pix, int stride, int16_t bS[4], unsigned int qp, MBRecContext *mrc, H264Slice *s ) {
nengel@2:     const unsigned int index_a = qp + s->slice_alpha_c0_offset;
nengel@2:     const int alpha = alpha_table[index_a];
nengel@2:     const int beta  = beta_table[qp + s->slice_beta_offset];
nengel@2:     if (alpha ==0 || beta == 0) return;
nengel@2: 
nengel@2:     if( bS[0] < 4 ) {
nengel@2:         int8_t tc[4];
nengel@2:         tc[0] = tc0_table[index_a][bS[0]]+1;
nengel@2:         tc[1] = tc0_table[index_a][bS[1]]+1;
nengel@2:         tc[2] = tc0_table[index_a][bS[2]]+1;
nengel@2:         tc[3] = tc0_table[index_a][bS[3]]+1;
nengel@2:         mrc->hdsp.h264_h_loop_filter_chroma(pix, stride, alpha, beta, tc);
nengel@2:     } else {
nengel@2:         mrc->hdsp.h264_h_loop_filter_chroma_intra(pix, stride, alpha, beta);
nengel@2:     }
nengel@2: }
nengel@2: 
nengel@2: 
nengel@2: av_always_inline static void filter_mb_edgeh( uint8_t *pix, int stride, int16_t bS[4], unsigned int qp, MBRecContext *mrc, H264Slice *s ) {
nengel@2:     const unsigned int index_a = qp + s->slice_alpha_c0_offset;
nengel@2:     const int alpha = alpha_table[index_a];
nengel@2:     const int beta  = beta_table[qp + s->slice_beta_offset];
nengel@2:     if (alpha ==0 || beta == 0) return;
nengel@2: 
nengel@2:     if( bS[0] < 4 ) {
nengel@2:         int8_t tc[4];
nengel@2:         tc[0] = tc0_table[index_a][bS[0]];
nengel@2:         tc[1] = tc0_table[index_a][bS[1]];
nengel@2:         tc[2] = tc0_table[index_a][bS[2]];
nengel@2:         tc[3] = tc0_table[index_a][bS[3]];
nengel@2:         mrc->hdsp.h264_v_loop_filter_luma(pix, stride, alpha, beta, tc);
nengel@2:     } else {
nengel@2:         mrc->hdsp.h264_v_loop_filter_luma_intra(pix, stride, alpha, beta);
nengel@2:     }
nengel@2: }
nengel@2: 
nengel@2: av_always_inline static void filter_mb_edgech( uint8_t *pix, int stride, int16_t bS[4], unsigned int qp, MBRecContext *mrc, H264Slice *s ) {
nengel@2:     const unsigned int index_a = qp + s->slice_alpha_c0_offset;
nengel@2:     const int alpha = alpha_table[index_a];
nengel@2:     const int beta  = beta_table[qp + s->slice_beta_offset];
nengel@2:     if (alpha ==0 || beta == 0) return;
nengel@2: 
nengel@2:     if( bS[0] < 4 ) {
nengel@2:         int8_t tc[4];
nengel@2:         tc[0] = tc0_table[index_a][bS[0]]+1;
nengel@2:         tc[1] = tc0_table[index_a][bS[1]]+1;
nengel@2:         tc[2] = tc0_table[index_a][bS[2]]+1;
nengel@2:         tc[3] = tc0_table[index_a][bS[3]]+1;
nengel@2:         mrc->hdsp.h264_v_loop_filter_chroma(pix, stride, alpha, beta, tc);
nengel@2:     } else {
nengel@2:         mrc->hdsp.h264_v_loop_filter_chroma_intra(pix, stride, alpha, beta);
nengel@2:     }
nengel@2: }
nengel@2: 
nengel@2: static av_always_inline void filter_mb_dir(MBRecContext *mrc, MBRecState *mrs, H264Slice *s, H264Mb *m, uint8_t *img_y, uint8_t *img_cb, uint8_t *img_cr, int dir) {
nengel@2:     const int mbm_type = dir == 0 ? mrs->left_type : mrs->top_type;
nengel@2:     const int qp_xy= m->qscale_mb_xy;
nengel@2:     const int qp_dir = dir == 0 ? m->qscale_left_mb_xy : m->qscale_top_mb_xy;
nengel@2:     const int linesize = mrc->linesize;
nengel@2:     const int uvlinesize = mrc->uvlinesize;
nengel@2:     const int mb_type = m->mb_type;
nengel@2:     int edge;
nengel@2:     const int edges = mrs->edges[dir];
nengel@2: 
nengel@2:     if(mbm_type){
nengel@2:         int16_t* bS=mrs->bS[dir][0];
nengel@2:         /* Filter edge */
nengel@2:         // Do not use s->qscale as luma quantizer because it has not the same
nengel@2:         // value in IPCM macroblocks.
nengel@2:         if(bS[0]+bS[1]+bS[2]+bS[3]){
nengel@2:             int qp = ( qp_xy + qp_dir + 1 ) >> 1;
nengel@2:             if( dir == 0 ) {
nengel@2:                 filter_mb_edgev( &img_y[0], linesize, bS, qp, mrc, s );
nengel@2:                 {
nengel@2:                     int qp= ( get_chroma_qp(s, 0, qp_xy) + get_chroma_qp( s, 0, qp_dir) + 1 ) >> 1;
nengel@2:                     filter_mb_edgecv( &img_cb[0], uvlinesize, bS, qp, mrc, s);
nengel@2:                     filter_mb_edgecv( &img_cr[0], uvlinesize, bS, qp, mrc, s);
nengel@2:                 }
nengel@2:             } else {
nengel@2:                 filter_mb_edgeh( &img_y[0], linesize, bS, qp, mrc, s );
nengel@2:                 {
nengel@2:                     int qp= ( get_chroma_qp(s, 0, qp_xy) + get_chroma_qp( s, 0, qp_dir) + 1 ) >> 1;
nengel@2:                     filter_mb_edgech( &img_cb[0], uvlinesize, bS, qp, mrc, s);
nengel@2:                     filter_mb_edgech( &img_cr[0], uvlinesize, bS, qp, mrc, s);
nengel@2:                 }
nengel@2:             }
nengel@2:         }
nengel@2:     }
nengel@2: 
nengel@2:     for( edge = 1; edge < edges; edge++ ) {
nengel@2:         int16_t* bS=mrs->bS[dir][edge];
nengel@2:         int qp = qp_xy;
nengel@2: 
nengel@2:         if( IS_8x8DCT(mb_type & (edge<<24)) ) // (edge&1) && IS_8x8DCT(mb_type)
nengel@2:             continue;
nengel@2: 
nengel@2:         if(bS[0]+bS[1]+bS[2]+bS[3] == 0)
nengel@2:             continue;
nengel@2: 
nengel@2:         /* Filter edge */
nengel@2:         // Do not use s->qscale as luma quantizer because it has not the same
nengel@2:         // value in IPCM macroblocks.
nengel@2: 
nengel@2:         if( dir == 0 ) {
nengel@2:             filter_mb_edgev( &img_y[4*edge], linesize, bS, qp, mrc, s);
nengel@2:             if( (edge&1) == 0 ) {
nengel@2:                 filter_mb_edgecv( &img_cb[2*edge], uvlinesize, bS, get_chroma_qp(s, 0, qp_xy), mrc, s);
nengel@2:                 filter_mb_edgecv( &img_cr[2*edge], uvlinesize, bS, get_chroma_qp(s, 1, qp_xy), mrc, s);
nengel@2:             }
nengel@2:         } else {
nengel@2:             filter_mb_edgeh( &img_y[4*edge*linesize], linesize, bS, qp, mrc, s );
nengel@2:             if( (edge&1) == 0 ) {
nengel@2:                 filter_mb_edgech( &img_cb[2*edge*uvlinesize], uvlinesize, bS, get_chroma_qp(s, 0, qp_xy), mrc, s);
nengel@2:                 filter_mb_edgech( &img_cr[2*edge*uvlinesize], uvlinesize, bS, get_chroma_qp(s, 1, qp_xy), mrc, s);
nengel@2:             }
nengel@2:         }
nengel@2:     }
nengel@2: }
nengel@2: 
nengel@2: static int check_mv(MBRecContext *mrc, MBRecState *mrs, H264Slice *s, long b_idx, long bn_idx, int mvy_limit){
nengel@2:     int v;
nengel@2:     v= mrs->ref_cache[0][b_idx] != mrs->ref_cache[0][bn_idx];
nengel@2:     if(!v && mrs->ref_cache[0][b_idx]!=-1)
nengel@2:         // absolute value >= 7 | ...
nengel@2:         v= ((unsigned) (mrs->mv_cache[0][b_idx][0] - mrs->mv_cache[0][bn_idx][0] + 3) >= 7U) |
nengel@2:         ((FFABS( mrs->mv_cache[0][b_idx][1] - mrs->mv_cache[0][bn_idx][1] )) >= mvy_limit);
nengel@2: 
nengel@2:     if(s->list_count==2){
nengel@2:         if(!v)
nengel@2:             v = (mrs->ref_cache[1][b_idx] != mrs->ref_cache[1][bn_idx]) |
nengel@2:             ((unsigned) (mrs->mv_cache[1][b_idx][0] - mrs->mv_cache[1][bn_idx][0] + 3) >= 7U) |
nengel@2:             ((FFABS( mrs->mv_cache[1][b_idx][1] - mrs->mv_cache[1][bn_idx][1] )) >= mvy_limit);
nengel@2: 
nengel@2:         if(v){
nengel@2:             if((mrs->ref_cache[0][b_idx] != mrs->ref_cache[1][bn_idx]) |
nengel@2:                 (mrs->ref_cache[1][b_idx] != mrs->ref_cache[0][bn_idx]))
nengel@2:                 return 1;
nengel@2:             return
nengel@2:             ((unsigned) (mrs->mv_cache[0][b_idx][0] - mrs->mv_cache[1][bn_idx][0] + 3) >= 7U) |
nengel@2:             ((FFABS( mrs->mv_cache[0][b_idx][1] - mrs->mv_cache[1][bn_idx][1] )) >= mvy_limit) |
nengel@2:             ((unsigned) (mrs->mv_cache[1][b_idx][0] - mrs->mv_cache[0][bn_idx][0] + 3) >= 7U) |
nengel@2:             ((FFABS( mrs->mv_cache[1][b_idx][1] - mrs->mv_cache[0][bn_idx][1] )) >= mvy_limit);
nengel@2:         }
nengel@2:     }
nengel@2: 
nengel@2:     return v;
nengel@2: }
nengel@2: 
nengel@2: static void calc_bS_values(MBRecContext *mrc, MBRecState *mrs, H264Slice *s, H264Mb *m, int mvy_limit, int dir) {
nengel@2:     int mb_type = m->mb_type;
nengel@2:     int edge;
nengel@2:     const int mbm_type = dir == 0 ? mrs->left_type : mrs->top_type;
nengel@2: 
nengel@2:     // how often to recheck mv-based bS when iterating between edges
nengel@2:     static const uint8_t mask_edge_tab[2][8]={{0,3,3,3,1,1,1,1},
nengel@2:     {0,3,1,1,3,3,3,3}};
nengel@2:     const int mask_edge = mask_edge_tab[dir][(mb_type>>3)&7];
nengel@2:     const int edges = mask_edge== 3 && !(m->cbp&15) ? 1 : 4;
nengel@2:     // how often to recheck mv-based bS when iterating along each edge
nengel@2:     const int mask_par0 = mb_type & (MB_TYPE_16x16 | (MB_TYPE_8x16 >> dir));
nengel@2: 
nengel@2:     mrs->edges[dir]= edges;
nengel@2: 
nengel@2:     if(mbm_type){
nengel@2:         int16_t* bS=mrs->bS[dir][0];
nengel@2:         if( IS_INTRA(mb_type|mbm_type)) {
nengel@2:             AV_WN64A(bS, 0x0004000400040004ULL);
nengel@2:         } else {
nengel@2:             int i;
nengel@2:             int mv_done;
nengel@2:             if( mask_par0 && ((mbm_type & (MB_TYPE_16x16 | (MB_TYPE_8x16 >> dir)))) ) {
nengel@2:                 int b_idx= 8 + 4;
nengel@2:                 int bn_idx= b_idx - (dir ? 8:1);
nengel@2: 
nengel@2:                 bS[0] = bS[1] = bS[2] = bS[3] = check_mv(mrc, mrs, s, 8 + 4, bn_idx, mvy_limit);
nengel@2:                 mv_done = 1;
nengel@2:             }
nengel@2:             else
nengel@2:                 mv_done = 0;
nengel@2: 
nengel@2:             for( i = 0; i < 4; i++ ) {
nengel@2:                 int x = dir == 0 ? 0 : i;
nengel@2:                 int y = dir == 0 ? i    : 0;
nengel@2:                 int b_idx= 8 + 4 + x + 8*y;
nengel@2:                 int bn_idx= b_idx - (dir ? 8:1);
nengel@2: 
nengel@2:                 if( mrs->non_zero_count_cache[b_idx] |
nengel@2:                     mrs->non_zero_count_cache[bn_idx] ) {
nengel@2:                     bS[i] = 2;
nengel@2:                 }
nengel@2:                 else if(!mv_done)
nengel@2:                 {
nengel@2:                     bS[i] = check_mv(mrc, mrs, s, b_idx, bn_idx, mvy_limit);
nengel@2:                 }
nengel@2:             }
nengel@2:         }
nengel@2:     }
nengel@2: 
nengel@2:     /* Calculate bS */
nengel@2:     for( edge = 1; edge < edges; edge++ ) {
nengel@2:         int16_t* bS=mrs->bS[dir][edge];
nengel@2: 
nengel@2:         if( IS_8x8DCT(mb_type & (edge<<24)) ) // (edge&1) && IS_8x8DCT(mb_type)
nengel@2:             continue;
nengel@2: 
nengel@2:         if( IS_INTRA(mb_type)) {
nengel@2:             AV_WN64A(bS, 0x0003000300030003ULL);
nengel@2:         } else {
nengel@2:             int i;
nengel@2:             int mv_done;
nengel@2: 
nengel@2:             if( edge & mask_edge ) {
nengel@2:                 AV_ZERO64(bS);
nengel@2:                 mv_done = 1;
nengel@2:             }
nengel@2:             else if( mask_par0 ) {
nengel@2:                 int b_idx= 8 + 4 + edge * (dir ? 8:1);
nengel@2:                 int bn_idx= b_idx - (dir ? 8:1);
nengel@2: 
nengel@2:                 bS[0] = bS[1] = bS[2] = bS[3] = check_mv(mrc, mrs, s, b_idx, bn_idx, mvy_limit);
nengel@2:                 mv_done = 1;
nengel@2:             }
nengel@2:             else
nengel@2:                 mv_done = 0;
nengel@2: 
nengel@2:             for( i = 0; i < 4; i++ ) {
nengel@2:                 int x = dir == 0 ? edge : i;
nengel@2:                 int y = dir == 0 ? i    : edge;
nengel@2:                 int b_idx= 8 + 4 + x + 8*y;
nengel@2:                 int bn_idx= b_idx - (dir ? 8:1);
nengel@2: 
nengel@2:                 if( mrs->non_zero_count_cache[b_idx] |
nengel@2:                     mrs->non_zero_count_cache[bn_idx] ) {
nengel@2:                     bS[i] = 2;
nengel@2:                 }
nengel@2:                 else if(!mv_done)
nengel@2:                 {
nengel@2:                     bS[i] = check_mv(mrc, mrs, s, b_idx, bn_idx, mvy_limit);
nengel@2:                 }
nengel@2:             }
nengel@2: 
nengel@2:             if(bS[0]+bS[1]+bS[2]+bS[3] == 0)
nengel@2:                 continue;
nengel@2:         }
nengel@2: 
nengel@2:     }
nengel@2: }
nengel@2: 
nengel@2: 
nengel@2: /**
nengel@2: *
nengel@2: * @return zero if the loop filter can be skiped
nengel@2: */
nengel@2: static int fill_filter_caches(MBRecContext *mrc, MBRecState *mrs, H264Slice *s, H264Mb *m, int mb_type){
nengel@2:     H264Mb *m_top = m - mrc->mb_width;
nengel@2:     H264Mb *m_left = m - 1;
nengel@2:     const int mb_x = m->mb_x;
nengel@2:     const int mb_y = m->mb_y;
nengel@2:     int top_type, left_type;
nengel@2:     int qp, top_qp, left_qp;
nengel@2:     int qp_thresh = s->qp_thresh; //FIXME strictly we should store qp_thresh for each mb of a slice
nengel@2: 
nengel@2:     qp = m->qscale_mb_xy ;
nengel@2:     left_qp = m->qscale_left_mb_xy ;
nengel@2:     top_qp  = m->qscale_top_mb_xy ;
nengel@2: 
nengel@2:     //for sufficiently low qp, filtering wouldn't do anything
nengel@2:     //this is a conservative estimate: could also check beta_offset and more accurate chroma_qp
nengel@2:     if(qp <= qp_thresh
nengel@2:         && (!(mb_x+mb_y) || ((qp + left_qp + 1)>>1) <= qp_thresh)
nengel@2:         && ( mb_y==0 || ((qp + top_qp + 1)>>1) <= qp_thresh)){
nengel@2:         return 0;
nengel@2:     }
nengel@2: 
nengel@2:     if(IS_INTRA(mb_type)){
nengel@2:         return 1;
nengel@2:     }
nengel@2: 
nengel@2:     {
nengel@2:         int list;
nengel@2:         for(list=0; list<s->list_count; list++){
nengel@2:             int8_t *ref;
nengel@2: 
nengel@2:             if(!USES_LIST(mb_type, list)){
nengel@2:                 fill_rectangle( mrs->mv_cache[list][scan8[0]], 4, 4, 8, pack16to32(0,0), 4);
nengel@2:                 fill_rectangle( mrs->mv_cache[list][scan8[0]], 4, 4, 8, pack16to32(0,0), 4);
nengel@2:                 AV_WN32A(&mrs->ref_cache[list][scan8[ 0]], ((LIST_NOT_USED)&0xFF)*0x01010101u);
nengel@2:                 AV_WN32A(&mrs->ref_cache[list][scan8[ 2]], ((LIST_NOT_USED)&0xFF)*0x01010101u);
nengel@2:                 AV_WN32A(&mrs->ref_cache[list][scan8[ 8]], ((LIST_NOT_USED)&0xFF)*0x01010101u);
nengel@2:                 AV_WN32A(&mrs->ref_cache[list][scan8[10]], ((LIST_NOT_USED)&0xFF)*0x01010101u);
nengel@2:                 continue;
nengel@2:             }
nengel@2: 
nengel@2:             ref = &mrs->ref_index[list][4*mb_x];
nengel@2:             {
nengel@2:                 int (*ref2frm)[64] =(void *) (s->ref2frm[0] +  2);
nengel@2:                 AV_WN32A(&mrs->ref_cache[list][scan8[ 0]], (pack16to32(ref2frm[list][ref[0]],ref2frm[list][ref[1]])&0x00FF00FF)*0x0101);
nengel@2:                 AV_WN32A(&mrs->ref_cache[list][scan8[ 2]], (pack16to32(ref2frm[list][ref[0]],ref2frm[list][ref[1]])&0x00FF00FF)*0x0101);
nengel@2:                 ref += 2;
nengel@2: 
nengel@2:                 AV_WN32A(&mrs->ref_cache[list][scan8[ 8]], (pack16to32(ref2frm[list][ref[0]],ref2frm[list][ref[1]])&0x00FF00FF)*0x0101);
nengel@2:                 AV_WN32A(&mrs->ref_cache[list][scan8[10]], (pack16to32(ref2frm[list][ref[0]],ref2frm[list][ref[1]])&0x00FF00FF)*0x0101);
nengel@2:             }
nengel@2:         }
nengel@2:     }
nengel@2: 
nengel@2:     /*
nengel@2:     0 . T T. T T T T
nengel@2:     1 L . .L . . . .
nengel@2:     2 L . .L . . . .
nengel@2:     3 . T TL . . . .
nengel@2:     4 L . .L . . . .
nengel@2:     5 L . .. . . . .
nengel@2:     */
nengel@2: 
nengel@2:     if (IS_SKIP(mb_type)){
nengel@2:         memset(mrs->non_zero_count_cache + 8, 0, 8*5); //FIXME ugly, remove pfui
nengel@2:     }
nengel@2: 
nengel@2:     //FIXME constraint_intra_pred & partitioning & nnz (let us hope this is just a typo in the spec)
nengel@2:     top_type  = mrs->top_type;
nengel@2:     left_type = mrs->left_type;
nengel@2:     if(top_type){
nengel@2:         AV_COPY32(&mrs->non_zero_count_cache[4+8*0], &m_top->non_zero_count[3*4]);
nengel@2:     }
nengel@2: 
nengel@2:     if(left_type){
nengel@2:         mrs->non_zero_count_cache[3+8*1]= m_left->non_zero_count[3+0*4];
nengel@2:         mrs->non_zero_count_cache[3+8*2]= m_left->non_zero_count[3+1*4];
nengel@2:         mrs->non_zero_count_cache[3+8*3]= m_left->non_zero_count[3+2*4];
nengel@2:         mrs->non_zero_count_cache[3+8*4]= m_left->non_zero_count[3+3*4];
nengel@2:     }
nengel@2: 
nengel@2:     if(IS_INTER(mb_type) || IS_DIRECT(mb_type)){
nengel@2:         int list;
nengel@2:         for(list=0; list<s->list_count; list++){
nengel@2:             if(USES_LIST(top_type, list)){
nengel@2:                 const int b_xy= 4*mb_x + 3*mrc->b_stride;
nengel@2:                 const int b8_x= 4*mb_x + 2;
nengel@2:                 int (*ref2frm)[64] = (void *) (s->ref2frm[0] +  2);
nengel@2:                 AV_COPY128(mrs->mv_cache[list][scan8[0] + 0 - 1*8], mrs->motion_val_top[list][b_xy + 0]);
nengel@2: 
nengel@2:                 mrs->ref_cache[list][scan8[0] + 0 - 1*8]=
nengel@2:                 mrs->ref_cache[list][scan8[0] + 1 - 1*8]= ref2frm[list][mrs->ref_index_top[list][b8_x + 0]];
nengel@2:                 mrs->ref_cache[list][scan8[0] + 2 - 1*8]=
nengel@2:                 mrs->ref_cache[list][scan8[0] + 3 - 1*8]= ref2frm[list][mrs->ref_index_top[list][b8_x + 1]];
nengel@2:             }else{
nengel@2:                 AV_ZERO128(mrs->mv_cache[list][scan8[0] + 0 - 1*8]);
nengel@2:                 AV_WN32A(&mrs->ref_cache[list][scan8[0] + 0 - 1*8], ((LIST_NOT_USED)&0xFF)*0x01010101u);
nengel@2:             }
nengel@2: 
nengel@2:             if(USES_LIST(left_type, list)){
nengel@2:                 const int b_x = 4*(mb_x-1) + 3;
nengel@2:                 const int b8_x= 4*(mb_x-1) + 1;
nengel@2:                 int (*ref2frm)[64] = (void *) (s->ref2frm[0] +  2);
nengel@2:                 AV_COPY32(mrs->mv_cache[list][scan8[0] - 1 + 0 ], mrs->motion_val[list][b_x + mrc->b_stride*0]);
nengel@2:                 AV_COPY32(mrs->mv_cache[list][scan8[0] - 1 + 8 ], mrs->motion_val[list][b_x + mrc->b_stride*1]);
nengel@2:                 AV_COPY32(mrs->mv_cache[list][scan8[0] - 1 +16 ], mrs->motion_val[list][b_x + mrc->b_stride*2]);
nengel@2:                 AV_COPY32(mrs->mv_cache[list][scan8[0] - 1 +24 ], mrs->motion_val[list][b_x + mrc->b_stride*3]);
nengel@2: 
nengel@2:                 mrs->ref_cache[list][scan8[0] - 1 + 0 ]=
nengel@2:                 mrs->ref_cache[list][scan8[0] - 1 + 8 ]= ref2frm[list][mrs->ref_index[list][b8_x + 2*0]];
nengel@2:                 mrs->ref_cache[list][scan8[0] - 1 +16 ]=
nengel@2:                 mrs->ref_cache[list][scan8[0] - 1 +24 ]= ref2frm[list][mrs->ref_index[list][b8_x + 2*1]];
nengel@2: 
nengel@2:             }else{
nengel@2:                 AV_ZERO32(mrs->mv_cache [list][scan8[0] - 1 + 0 ]);
nengel@2:                 AV_ZERO32(mrs->mv_cache [list][scan8[0] - 1 + 8 ]);
nengel@2:                 AV_ZERO32(mrs->mv_cache [list][scan8[0] - 1 +16 ]);
nengel@2:                 AV_ZERO32(mrs->mv_cache [list][scan8[0] - 1 +24 ]);
nengel@2: 
nengel@2:                 mrs->ref_cache[list][scan8[0] - 1 + 0  ]=
nengel@2:                 mrs->ref_cache[list][scan8[0] - 1 + 8  ]=
nengel@2:                 mrs->ref_cache[list][scan8[0] - 1 + 16 ]=
nengel@2:                 mrs->ref_cache[list][scan8[0] - 1 + 24 ]= LIST_NOT_USED;
nengel@2:             }
nengel@2:         }
nengel@2:     }
nengel@2:     return 1;
nengel@2: }
nengel@2: 
nengel@2: void ff_h264_filter_mb(MBRecContext *mrc, MBRecState *mrs, H264Slice *s, H264Mb *m, uint8_t *img_y, uint8_t *img_cb, uint8_t *img_cr) {
nengel@2:     if (fill_filter_caches(mrc, mrs, s, m, m->mb_type)){
nengel@2:         calc_bS_values(mrc, mrs, s, m, 4, 0);
nengel@2:         calc_bS_values(mrc, mrs, s, m, 4, 1);
nengel@2:         filter_mb_dir(mrc, mrs, s, m, img_y, img_cb, img_cr, 0);
nengel@2:         filter_mb_dir(mrc, mrs, s, m, img_y, img_cb, img_cr, 1);
nengel@2:     }
nengel@2: }