h264: simplify reference picture marking process.
[vaapi:gstreamer-vaapi.git] / gst-libs / gst / vaapi / gstvaapidecoder_h264.c
1 /*
2  *  gstvaapidecoder_h264.c - H.264 decoder
3  *
4  *  Copyright (C) 2011-2012 Intel Corporation
5  *
6  *  This library is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU Lesser General Public License
8  *  as published by the Free Software Foundation; either version 2.1
9  *  of the License, or (at your option) any later version.
10  *
11  *  This library is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  *  Lesser General Public License for more details.
15  *
16  *  You should have received a copy of the GNU Lesser General Public
17  *  License along with this library; if not, write to the Free
18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19  *  Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * SECTION:gstvaapidecoder_h264
24  * @short_description: H.264 decoder
25  */
26
27 #include "sysdeps.h"
28 #include <string.h>
29 #include <gst/base/gstadapter.h>
30 #include <gst/codecparsers/gsth264parser.h>
31 #include "gstvaapidecoder_h264.h"
32 #include "gstvaapidecoder_objects.h"
33 #include "gstvaapidecoder_priv.h"
34 #include "gstvaapidisplay_priv.h"
35 #include "gstvaapiobject_priv.h"
36
37 #define DEBUG 1
38 #include "gstvaapidebug.h"
39
40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
41 #define USE_STRICT_DPB_ORDERING 0
42
43 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
44 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
45 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
46 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
47
48 // Used for field_poc[]
49 #define TOP_FIELD       0
50 #define BOTTOM_FIELD    1
51
52 /* ------------------------------------------------------------------------- */
53 /* --- H.264 Pictures                                                    --- */
54 /* ------------------------------------------------------------------------- */
55
56 #define GST_VAAPI_TYPE_PICTURE_H264 \
57     (gst_vaapi_picture_h264_get_type())
58
59 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
60     ((GstVaapiPictureH264 *)(obj))
61
62 #define GST_VAAPI_PICTURE_H264(obj)                             \
63     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
64                                 GST_VAAPI_TYPE_PICTURE_H264,    \
65                                 GstVaapiPictureH264))
66
67 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
68     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
69                              GST_VAAPI_TYPE_PICTURE_H264,       \
70                              GstVaapiPictureH264Class))
71
72 #define GST_VAAPI_IS_PICTURE_H264(obj) \
73     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
74
75 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
76     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
77
78 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
79     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
80                                GST_VAAPI_TYPE_PICTURE_H264,     \
81                                GstVaapiPictureH264Class))
82
83 /*
84  * Extended picture flags:
85  *
86  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
87  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
88  *     "used for short-term reference"
89  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
90  *     "used for long-term reference"
91  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
92  *     reference picture (short-term reference or long-term reference)
93  */
94 enum {
95     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
96
97     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
98         GST_VAAPI_PICTURE_FLAG_REFERENCE),
99     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
100         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
101     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
102         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
103         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
104 };
105
106 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
107     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
108
109 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
110     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
111       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
112      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
113
114 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
115     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
116       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
117      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
118
119 struct _GstVaapiPictureH264 {
120     GstVaapiPicture             base;
121     GstH264PPS                 *pps;
122     gint32                      field_poc[2];
123     gint32                      frame_num;              // Original frame_num from slice_header()
124     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
125     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
126     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
127     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
128     guint                       output_flag             : 1;
129     guint                       output_needed           : 1;
130 };
131
132 struct _GstVaapiPictureH264Class {
133     /*< private >*/
134     GstVaapiPictureClass        parent_class;
135 };
136
137 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
138                             gst_vaapi_picture_h264,
139                             GST_VAAPI_TYPE_PICTURE)
140
141 static void
142 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
143 {
144 }
145
146 static gboolean
147 gst_vaapi_picture_h264_create(
148     GstVaapiPictureH264                      *picture,
149     const GstVaapiCodecObjectConstructorArgs *args
150 )
151 {
152     return TRUE;
153 }
154
155 static void
156 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
157 {
158     picture->field_poc[0]       = G_MAXINT32;
159     picture->field_poc[1]       = G_MAXINT32;
160     picture->output_needed      = FALSE;
161 }
162
163 static inline GstVaapiPictureH264 *
164 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
165 {
166     GstVaapiCodecObject *object;
167
168     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
169
170     object = gst_vaapi_codec_object_new(
171         GST_VAAPI_TYPE_PICTURE_H264,
172         GST_VAAPI_CODEC_BASE(decoder),
173         NULL, sizeof(VAPictureParameterBufferH264),
174         NULL, 0
175     );
176     if (!object)
177         return NULL;
178     return GST_VAAPI_PICTURE_H264_CAST(object);
179 }
180
181 static inline GstVaapiSliceH264 *
182 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
183 {
184     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
185
186     if (G_UNLIKELY(picture->base.slices->len < 1))
187         return NULL;
188     return g_ptr_array_index(picture->base.slices,
189         picture->base.slices->len - 1);
190 }
191
192 /* ------------------------------------------------------------------------- */
193 /* --- Slices                                                            --- */
194 /* ------------------------------------------------------------------------- */
195
196 #define GST_VAAPI_TYPE_SLICE_H264 \
197     (gst_vaapi_slice_h264_get_type())
198
199 #define GST_VAAPI_SLICE_H264_CAST(obj) \
200     ((GstVaapiSliceH264 *)(obj))
201
202 #define GST_VAAPI_SLICE_H264(obj)                               \
203     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
204                                 GST_VAAPI_TYPE_SLICE_H264,      \
205                                 GstVaapiSliceH264))
206
207 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
208     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
209                              GST_VAAPI_TYPE_SLICE_H264,         \
210                              GstVaapiSliceH264Class))
211
212 #define GST_VAAPI_IS_SLICE_H264(obj) \
213     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
214
215 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
216     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
217
218 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
219     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
220                                GST_VAAPI_TYPE_SLICE_H264,       \
221                                GstVaapiSliceH264Class))
222
223 struct _GstVaapiSliceH264 {
224     GstVaapiSlice               base;
225     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
226 };
227
228 struct _GstVaapiSliceH264Class {
229     /*< private >*/
230     GstVaapiSliceClass          parent_class;
231 };
232
233 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
234                             gst_vaapi_slice_h264,
235                             GST_VAAPI_TYPE_SLICE)
236
237 static void
238 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
239 {
240 }
241
242 static gboolean
243 gst_vaapi_slice_h264_create(
244     GstVaapiSliceH264                        *slice,
245     const GstVaapiCodecObjectConstructorArgs *args
246 )
247 {
248     return TRUE;
249 }
250
251 static void
252 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
253 {
254 }
255
256 static inline GstVaapiSliceH264 *
257 gst_vaapi_slice_h264_new(
258     GstVaapiDecoderH264 *decoder,
259     const guint8        *data,
260     guint                data_size
261 )
262 {
263     GstVaapiCodecObject *object;
264
265     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
266
267     object = gst_vaapi_codec_object_new(
268         GST_VAAPI_TYPE_SLICE_H264,
269         GST_VAAPI_CODEC_BASE(decoder),
270         NULL, sizeof(VASliceParameterBufferH264),
271         data, data_size
272     );
273     if (!object)
274         return NULL;
275     return GST_VAAPI_SLICE_H264_CAST(object);
276 }
277
278 /* ------------------------------------------------------------------------- */
279 /* --- H.264 Decoder                                                     --- */
280 /* ------------------------------------------------------------------------- */
281
282 G_DEFINE_TYPE(GstVaapiDecoderH264,
283               gst_vaapi_decoder_h264,
284               GST_VAAPI_TYPE_DECODER)
285
286 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
287     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
288                                  GST_VAAPI_TYPE_DECODER_H264,   \
289                                  GstVaapiDecoderH264Private))
290
291 struct _GstVaapiDecoderH264Private {
292     GstAdapter                 *adapter;
293     GstH264NalParser           *parser;
294     /* Last decoded SPS. May not be the last activated one. Just here because
295        it may not fit stack memory allocation in decode_sps() */
296     GstH264SPS                  last_sps;
297     /* Last decoded PPS. May not be the last activated one. Just here because
298        it may not fit stack memory allocation in decode_pps() */
299     GstH264PPS                  last_pps;
300     GstVaapiPictureH264        *current_picture;
301     GstVaapiPictureH264        *dpb[16];
302     guint                       dpb_count;
303     guint                       dpb_size;
304     GstVaapiProfile             profile;
305     GstVaapiEntrypoint          entrypoint;
306     GstVaapiChromaType          chroma_type;
307     GstVaapiPictureH264        *short_ref[16];
308     guint                       short_ref_count;
309     GstVaapiPictureH264        *long_ref[16];
310     guint                       long_ref_count;
311     GstVaapiPictureH264        *RefPicList0[32];
312     guint                       RefPicList0_count;
313     GstVaapiPictureH264        *RefPicList1[32];
314     guint                       RefPicList1_count;
315     guint                       nal_length_size;
316     guint                       width;
317     guint                       height;
318     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
319     gint32                      poc_msb;                // PicOrderCntMsb
320     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
321     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
322     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
323     gint32                      frame_num_offset;       // FrameNumOffset
324     gint32                      frame_num;              // frame_num (from slice_header())
325     gint32                      prev_frame_num;         // prevFrameNum
326     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
327     gboolean                    prev_pic_structure;     // previous picture structure
328     guint                       is_constructed          : 1;
329     guint                       is_opened               : 1;
330     guint                       is_avc                  : 1;
331     guint                       has_context             : 1;
332 };
333
334 static gboolean
335 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
336
337 /* Get number of reference frames to use */
338 static guint
339 get_max_dec_frame_buffering(GstH264SPS *sps)
340 {
341     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
342
343     /* Table A-1 - Level limits */
344     switch (sps->level_idc) {
345     case 10: MaxDpbMbs = 396;    break;
346     case 11: MaxDpbMbs = 900;    break;
347     case 12: MaxDpbMbs = 2376;   break;
348     case 13: MaxDpbMbs = 2376;   break;
349     case 20: MaxDpbMbs = 2376;   break;
350     case 21: MaxDpbMbs = 4752;   break;
351     case 22: MaxDpbMbs = 8100;   break;
352     case 30: MaxDpbMbs = 8100;   break;
353     case 31: MaxDpbMbs = 18000;  break;
354     case 32: MaxDpbMbs = 20480;  break;
355     case 40: MaxDpbMbs = 32768;  break;
356     case 41: MaxDpbMbs = 32768;  break;
357     case 42: MaxDpbMbs = 34816;  break;
358     case 50: MaxDpbMbs = 110400; break;
359     case 51: MaxDpbMbs = 184320; break;
360     default:
361         g_assert(0 && "unhandled level");
362         break;
363     }
364
365     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
366                   (sps->pic_height_in_map_units_minus1 + 1) *
367                   (sps->frame_mbs_only_flag ? 1 : 2));
368     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
369
370     /* VUI parameters */
371     if (sps->vui_parameters_present_flag) {
372         GstH264VUIParams * const vui_params = &sps->vui_parameters;
373         if (vui_params->bitstream_restriction_flag)
374             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
375         else {
376             switch (sps->profile_idc) {
377             case 44:  // CAVLC 4:4:4 Intra profile
378             case 86:  // Scalable High profile
379             case 100: // High profile
380             case 110: // High 10 profile
381             case 122: // High 4:2:2 profile
382             case 244: // High 4:4:4 Predictive profile
383                 if (sps->constraint_set3_flag)
384                     max_dec_frame_buffering = 0;
385                 break;
386             }
387         }
388     }
389
390     if (max_dec_frame_buffering > 16)
391         max_dec_frame_buffering = 16;
392     else if (max_dec_frame_buffering < sps->num_ref_frames)
393         max_dec_frame_buffering = sps->num_ref_frames;
394     return MAX(1, max_dec_frame_buffering);
395 }
396
397 static void
398 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
399 {
400     GstVaapiDecoderH264Private * const priv = decoder->priv;
401     guint i, num_pictures = --priv->dpb_count;
402
403     if (USE_STRICT_DPB_ORDERING) {
404         for (i = index; i < num_pictures; i++)
405             gst_vaapi_picture_replace(&priv->dpb[i], priv->dpb[i + 1]);
406     }
407     else if (index != num_pictures)
408         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
409     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
410 }
411
412 static inline gboolean
413 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
414 {
415     /* XXX: update cropping rectangle */
416     picture->output_needed = FALSE;
417     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
418 }
419
420 static gboolean
421 dpb_bump(GstVaapiDecoderH264 *decoder)
422 {
423     GstVaapiDecoderH264Private * const priv = decoder->priv;
424     guint i, lowest_poc_index;
425     gboolean success;
426
427     for (i = 0; i < priv->dpb_count; i++) {
428         if (priv->dpb[i]->output_needed)
429             break;
430     }
431     if (i == priv->dpb_count)
432         return FALSE;
433
434     lowest_poc_index = i++;
435     for (; i < priv->dpb_count; i++) {
436         GstVaapiPictureH264 * const picture = priv->dpb[i];
437         if (picture->output_needed && picture->base.poc < priv->dpb[lowest_poc_index]->base.poc)
438             lowest_poc_index = i;
439     }
440
441     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
442     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
443         dpb_remove_index(decoder, lowest_poc_index);
444     return success;
445 }
446
447 static void
448 dpb_clear(GstVaapiDecoderH264 *decoder)
449 {
450     GstVaapiDecoderH264Private * const priv = decoder->priv;
451     guint i;
452
453     for (i = 0; i < priv->dpb_count; i++)
454         gst_vaapi_picture_replace(&priv->dpb[i], NULL);
455     priv->dpb_count = 0;
456 }
457
458 static void
459 dpb_flush(GstVaapiDecoderH264 *decoder)
460 {
461     while (dpb_bump(decoder))
462         ;
463     dpb_clear(decoder);
464 }
465
466 static gboolean
467 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
468 {
469     GstVaapiDecoderH264Private * const priv = decoder->priv;
470     guint i;
471
472     // Remove all unused pictures
473     if (GST_VAAPI_PICTURE_IS_IDR(picture))
474         dpb_flush(decoder);
475     else {
476         i = 0;
477         while (i < priv->dpb_count) {
478             GstVaapiPictureH264 * const picture = priv->dpb[i];
479             if (!picture->output_needed &&
480                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
481                 dpb_remove_index(decoder, i);
482             else
483                 i++;
484         }
485     }
486
487     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
488     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
489         while (priv->dpb_count == priv->dpb_size) {
490             if (!dpb_bump(decoder))
491                 return FALSE;
492         }
493         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
494         if (picture->output_flag)
495             picture->output_needed = TRUE;
496     }
497
498     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
499     else {
500         if (!picture->output_flag)
501             return TRUE;
502         while (priv->dpb_count == priv->dpb_size) {
503             for (i = 0; i < priv->dpb_count; i++) {
504                 if (priv->dpb[i]->output_needed &&
505                     priv->dpb[i]->base.poc < picture->base.poc)
506                     break;
507             }
508             if (i == priv->dpb_count)
509                 return dpb_output(decoder, picture);
510             if (!dpb_bump(decoder))
511                 return FALSE;
512         }
513         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
514         picture->output_needed = TRUE;
515     }
516     return TRUE;
517 }
518
519 static inline void
520 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
521 {
522     GstVaapiDecoderH264Private * const priv = decoder->priv;
523
524     priv->dpb_size = get_max_dec_frame_buffering(sps);
525     GST_DEBUG("DPB size %u", priv->dpb_size);
526 }
527
528 static GstVaapiDecoderStatus
529 get_status(GstH264ParserResult result)
530 {
531     GstVaapiDecoderStatus status;
532
533     switch (result) {
534     case GST_H264_PARSER_OK:
535         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
536         break;
537     case GST_H264_PARSER_NO_NAL_END:
538         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
539         break;
540     case GST_H264_PARSER_ERROR:
541         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
542         break;
543     default:
544         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
545         break;
546     }
547     return status;
548 }
549
550 static void
551 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
552 {
553     GstVaapiDecoderH264Private * const priv = decoder->priv;
554
555     gst_vaapi_picture_replace(&priv->current_picture, NULL);
556
557     dpb_clear(decoder);
558
559     if (priv->parser) {
560         gst_h264_nal_parser_free(priv->parser);
561         priv->parser = NULL;
562     }
563
564     if (priv->adapter) {
565         gst_adapter_clear(priv->adapter);
566         g_object_unref(priv->adapter);
567         priv->adapter = NULL;
568     }
569 }
570
571 static gboolean
572 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
573 {
574     GstVaapiDecoderH264Private * const priv = decoder->priv;
575
576     gst_vaapi_decoder_h264_close(decoder);
577
578     priv->adapter = gst_adapter_new();
579     if (!priv->adapter)
580         return FALSE;
581
582     priv->parser = gst_h264_nal_parser_new();
583     if (!priv->parser)
584         return FALSE;
585     return TRUE;
586 }
587
588 static void
589 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
590 {
591     gst_vaapi_decoder_h264_close(decoder);
592 }
593
594 static gboolean
595 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
596 {
597     if (!GST_VAAPI_DECODER_CODEC(decoder))
598         return FALSE;
599     return TRUE;
600 }
601
602 static guint
603 h264_get_profile(GstH264SPS *sps)
604 {
605     guint profile = 0;
606
607     switch (sps->profile_idc) {
608     case 66:
609         profile = GST_VAAPI_PROFILE_H264_BASELINE;
610         break;
611     case 77:
612         profile = GST_VAAPI_PROFILE_H264_MAIN;
613         break;
614     case 100:
615         profile = GST_VAAPI_PROFILE_H264_HIGH;
616         break;
617     }
618     return profile;
619 }
620
621 static guint
622 h264_get_chroma_type(GstH264SPS *sps)
623 {
624     guint chroma_type = 0;
625
626     switch (sps->chroma_format_idc) {
627     case 1:
628         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
629         break;
630     case 2:
631         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
632         break;
633     case 3:
634         if (!sps->separate_colour_plane_flag)
635             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
636         break;
637     }
638     return chroma_type;
639 }
640
641 static GstVaapiProfile
642 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
643 {
644     GstVaapiDecoderH264Private * const priv = decoder->priv;
645     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
646     GstVaapiProfile profile, profiles[2];
647     guint i, n_profiles = 0;
648
649     profile = h264_get_profile(sps);
650     if (!profile)
651         return GST_VAAPI_PROFILE_UNKNOWN;
652
653     profiles[n_profiles++] = profile;
654     switch (profile) {
655     case GST_VAAPI_PROFILE_H264_MAIN:
656         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
657         break;
658     default:
659         break;
660     }
661
662     /* If the preferred profile (profiles[0]) matches one that we already
663        found, then just return it now instead of searching for it again */
664     if (profiles[0] == priv->profile)
665         return priv->profile;
666
667     for (i = 0; i < n_profiles; i++) {
668         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
669             return profiles[i];
670     }
671     return GST_VAAPI_PROFILE_UNKNOWN;
672 }
673
674 static GstVaapiDecoderStatus
675 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
676 {
677     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
678     GstVaapiDecoderH264Private * const priv = decoder->priv;
679     GstVaapiContextInfo info;
680     GstVaapiProfile profile;
681     GstVaapiChromaType chroma_type;
682     gboolean reset_context = FALSE;
683
684     profile = get_profile(decoder, sps);
685     if (!profile) {
686         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
687         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
688     }
689
690     if (priv->profile != profile) {
691         GST_DEBUG("profile changed");
692         reset_context = TRUE;
693         priv->profile = profile;
694     }
695
696     chroma_type = h264_get_chroma_type(sps);
697     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
698         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
699         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
700     }
701
702     if (priv->chroma_type != chroma_type) {
703         GST_DEBUG("chroma format changed");
704         reset_context     = TRUE;
705         priv->chroma_type = chroma_type;
706     }
707
708     if (priv->width != sps->width || priv->height != sps->height) {
709         GST_DEBUG("size changed");
710         reset_context = TRUE;
711         priv->width   = sps->width;
712         priv->height  = sps->height;
713     }
714
715     gst_vaapi_decoder_set_pixel_aspect_ratio(
716         base_decoder,
717         sps->vui_parameters.par_n,
718         sps->vui_parameters.par_d
719     );
720
721     if (!reset_context && priv->has_context)
722         return GST_VAAPI_DECODER_STATUS_SUCCESS;
723
724     info.profile    = priv->profile;
725     info.entrypoint = priv->entrypoint;
726     info.width      = priv->width;
727     info.height     = priv->height;
728     info.ref_frames = get_max_dec_frame_buffering(sps);
729
730     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
731         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
732     priv->has_context = TRUE;
733
734     /* Reset DPB */
735     dpb_reset(decoder, sps);
736     return GST_VAAPI_DECODER_STATUS_SUCCESS;
737 }
738
739 static void
740 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
741 {
742     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
743     guint i, j;
744
745     /* There are always 6 4x4 scaling lists */
746     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
747     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
748
749     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
750         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
751                sizeof(iq_matrix->ScalingList4x4));
752     else {
753         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
754             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
755                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
756         }
757     }
758 }
759
760 static void
761 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
762 {
763     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
764     const GstH264SPS * const sps = pps->sequence;
765     guint i, j, n;
766
767     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
768     if (!pps->transform_8x8_mode_flag)
769         return;
770
771     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
772     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
773
774     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
775         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
776                sizeof(iq_matrix->ScalingList8x8));
777     else {
778         n = (sps->chroma_format_idc != 3) ? 2 : 6;
779         for (i = 0; i < n; i++) {
780             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
781                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
782         }
783     }
784 }
785
786 static GstVaapiDecoderStatus
787 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
788 {
789     GstVaapiPicture * const base_picture = &picture->base;
790     GstH264PPS * const pps = picture->pps;
791     GstH264SPS * const sps = pps->sequence;
792     VAIQMatrixBufferH264 *iq_matrix;
793
794     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
795     if (!base_picture->iq_matrix) {
796         GST_ERROR("failed to allocate IQ matrix");
797         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
798     }
799     iq_matrix = base_picture->iq_matrix->param;
800
801     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
802        is not large enough to hold lists for 4:4:4 */
803     if (sps->chroma_format_idc == 3)
804         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
805
806     fill_iq_matrix_4x4(iq_matrix, pps);
807     fill_iq_matrix_8x8(iq_matrix, pps);
808
809     return GST_VAAPI_DECODER_STATUS_SUCCESS;
810 }
811
812 static GstVaapiDecoderStatus
813 decode_current_picture(GstVaapiDecoderH264 *decoder)
814 {
815     GstVaapiDecoderH264Private * const priv = decoder->priv;
816     GstVaapiPictureH264 * const picture = priv->current_picture;
817     GstVaapiDecoderStatus status;
818
819     if (!picture)
820         return GST_VAAPI_DECODER_STATUS_SUCCESS;
821
822     status = ensure_context(decoder, picture->pps->sequence);
823     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
824         return status;
825
826     if (!exec_ref_pic_marking(decoder, picture))
827         goto error;
828     if (!dpb_add(decoder, picture))
829         goto error;
830     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
831         goto error;
832     gst_vaapi_picture_replace(&priv->current_picture, NULL);
833     return GST_VAAPI_DECODER_STATUS_SUCCESS;
834
835 error:
836     gst_vaapi_picture_replace(&priv->current_picture, NULL);
837     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
838 }
839
840 static GstVaapiDecoderStatus
841 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
842 {
843     GstVaapiDecoderH264Private * const priv = decoder->priv;
844     GstH264SPS * const sps = &priv->last_sps;
845     GstH264ParserResult result;
846
847     GST_DEBUG("decode SPS");
848
849     memset(sps, 0, sizeof(*sps));
850     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
851     if (result != GST_H264_PARSER_OK)
852         return get_status(result);
853
854     return GST_VAAPI_DECODER_STATUS_SUCCESS;
855 }
856
857 static GstVaapiDecoderStatus
858 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
859 {
860     GstVaapiDecoderH264Private * const priv = decoder->priv;
861     GstH264PPS * const pps = &priv->last_pps;
862     GstH264ParserResult result;
863
864     GST_DEBUG("decode PPS");
865
866     memset(pps, 0, sizeof(*pps));
867     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
868     if (result != GST_H264_PARSER_OK)
869         return get_status(result);
870
871     return GST_VAAPI_DECODER_STATUS_SUCCESS;
872 }
873
874 static GstVaapiDecoderStatus
875 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
876 {
877     GstVaapiDecoderH264Private * const priv = decoder->priv;
878     GstH264SEIMessage sei;
879     GstH264ParserResult result;
880
881     GST_DEBUG("decode SEI");
882
883     memset(&sei, 0, sizeof(sei));
884     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
885     if (result != GST_H264_PARSER_OK) {
886         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
887         return get_status(result);
888     }
889
890     return GST_VAAPI_DECODER_STATUS_SUCCESS;
891 }
892
893 static GstVaapiDecoderStatus
894 decode_sequence_end(GstVaapiDecoderH264 *decoder)
895 {
896     GstVaapiDecoderStatus status;
897
898     GST_DEBUG("decode sequence-end");
899
900     status = decode_current_picture(decoder);
901     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
902         return status;
903
904     dpb_flush(decoder);
905     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
906 }
907
908 /* 8.2.1.1 - Decoding process for picture order count type 0 */
909 static void
910 init_picture_poc_0(
911     GstVaapiDecoderH264 *decoder,
912     GstVaapiPictureH264 *picture,
913     GstH264SliceHdr     *slice_hdr
914 )
915 {
916     GstVaapiDecoderH264Private * const priv = decoder->priv;
917     GstH264PPS * const pps = slice_hdr->pps;
918     GstH264SPS * const sps = pps->sequence;
919     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
920     gint32 temp_poc;
921
922     GST_DEBUG("decode picture order count type 0");
923
924     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
925         priv->prev_poc_msb = 0;
926         priv->prev_poc_lsb = 0;
927     }
928     else if (priv->prev_pic_has_mmco5) {
929         priv->prev_poc_msb = 0;
930         priv->prev_poc_lsb =
931             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
932              0 : priv->field_poc[TOP_FIELD]);
933     }
934     else {
935         priv->prev_poc_msb = priv->poc_msb;
936         priv->prev_poc_lsb = priv->poc_lsb;
937     }
938
939     // (8-3)
940     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
941     if (priv->poc_lsb < priv->prev_poc_lsb &&
942         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
943         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
944     else if (priv->poc_lsb > priv->prev_poc_lsb &&
945              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
946         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
947     else
948         priv->poc_msb = priv->prev_poc_msb;
949
950     temp_poc = priv->poc_msb + priv->poc_lsb;
951     switch (picture->base.structure) {
952     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
953         // (8-4, 8-5)
954         priv->field_poc[TOP_FIELD] = temp_poc;
955         priv->field_poc[BOTTOM_FIELD] = temp_poc +
956             slice_hdr->delta_pic_order_cnt_bottom;
957         break;
958     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
959         // (8-4)
960         priv->field_poc[TOP_FIELD] = temp_poc;
961         break;
962     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
963         // (8-5)
964         priv->field_poc[BOTTOM_FIELD] = temp_poc;
965         break;
966     }
967 }
968
969 /* 8.2.1.2 - Decoding process for picture order count type 1 */
970 static void
971 init_picture_poc_1(
972     GstVaapiDecoderH264 *decoder,
973     GstVaapiPictureH264 *picture,
974     GstH264SliceHdr     *slice_hdr
975 )
976 {
977     GstVaapiDecoderH264Private * const priv = decoder->priv;
978     GstH264PPS * const pps = slice_hdr->pps;
979     GstH264SPS * const sps = pps->sequence;
980     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
981     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
982     guint i;
983
984     GST_DEBUG("decode picture order count type 1");
985
986     if (priv->prev_pic_has_mmco5)
987         prev_frame_num_offset = 0;
988     else
989         prev_frame_num_offset = priv->frame_num_offset;
990
991     // (8-6)
992     if (GST_VAAPI_PICTURE_IS_IDR(picture))
993         priv->frame_num_offset = 0;
994     else if (priv->prev_frame_num > priv->frame_num)
995         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
996     else
997         priv->frame_num_offset = prev_frame_num_offset;
998
999     // (8-7)
1000     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1001         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1002     else
1003         abs_frame_num = 0;
1004     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1005         abs_frame_num = abs_frame_num - 1;
1006
1007     if (abs_frame_num > 0) {
1008         gint32 expected_delta_per_poc_cycle;
1009         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1010
1011         expected_delta_per_poc_cycle = 0;
1012         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1013             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1014
1015         // (8-8)
1016         poc_cycle_cnt = (abs_frame_num - 1) /
1017             sps->num_ref_frames_in_pic_order_cnt_cycle;
1018         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1019             sps->num_ref_frames_in_pic_order_cnt_cycle;
1020
1021         // (8-9)
1022         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1023         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1024             expected_poc += sps->offset_for_ref_frame[i];
1025     }
1026     else
1027         expected_poc = 0;
1028     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1029         expected_poc += sps->offset_for_non_ref_pic;
1030
1031     // (8-10)
1032     switch (picture->base.structure) {
1033     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1034         priv->field_poc[TOP_FIELD] = expected_poc +
1035             slice_hdr->delta_pic_order_cnt[0];
1036         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1037             sps->offset_for_top_to_bottom_field +
1038             slice_hdr->delta_pic_order_cnt[1];
1039         break;
1040     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1041         priv->field_poc[TOP_FIELD] = expected_poc +
1042             slice_hdr->delta_pic_order_cnt[0];
1043         break;
1044     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1045         priv->field_poc[BOTTOM_FIELD] = expected_poc + 
1046             sps->offset_for_top_to_bottom_field +
1047             slice_hdr->delta_pic_order_cnt[0];
1048         break;
1049     }
1050 }
1051
1052 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1053 static void
1054 init_picture_poc_2(
1055     GstVaapiDecoderH264 *decoder,
1056     GstVaapiPictureH264 *picture,
1057     GstH264SliceHdr     *slice_hdr
1058 )
1059 {
1060     GstVaapiDecoderH264Private * const priv = decoder->priv;
1061     GstH264PPS * const pps = slice_hdr->pps;
1062     GstH264SPS * const sps = pps->sequence;
1063     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1064     gint32 prev_frame_num_offset, temp_poc;
1065
1066     GST_DEBUG("decode picture order count type 2");
1067
1068     if (priv->prev_pic_has_mmco5)
1069         prev_frame_num_offset = 0;
1070     else
1071         prev_frame_num_offset = priv->frame_num_offset;
1072
1073     // (8-11)
1074     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1075         priv->frame_num_offset = 0;
1076     else if (priv->prev_frame_num > priv->frame_num)
1077         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1078     else
1079         priv->frame_num_offset = prev_frame_num_offset;
1080
1081     // (8-12)
1082     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1083         temp_poc = 0;
1084     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1085         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1086     else
1087         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1088
1089     // (8-13)
1090     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1091         priv->field_poc[TOP_FIELD] = temp_poc;
1092     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1093         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1094 }
1095
1096 /* 8.2.1 - Decoding process for picture order count */
1097 static void
1098 init_picture_poc(
1099     GstVaapiDecoderH264 *decoder,
1100     GstVaapiPictureH264 *picture,
1101     GstH264SliceHdr     *slice_hdr
1102 )
1103 {
1104     GstVaapiDecoderH264Private * const priv = decoder->priv;
1105     GstH264PPS * const pps = slice_hdr->pps;
1106     GstH264SPS * const sps = pps->sequence;
1107
1108     switch (sps->pic_order_cnt_type) {
1109     case 0:
1110         init_picture_poc_0(decoder, picture, slice_hdr);
1111         break;
1112     case 1:
1113         init_picture_poc_1(decoder, picture, slice_hdr);
1114         break;
1115     case 2:
1116         init_picture_poc_2(decoder, picture, slice_hdr);
1117         break;
1118     }
1119
1120     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1121         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1122     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1123         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1124     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1125 }
1126
1127 static int
1128 compare_picture_pic_num_dec(const void *a, const void *b)
1129 {
1130     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1131     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1132
1133     return picB->pic_num - picA->pic_num;
1134 }
1135
1136 static int
1137 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1138 {
1139     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1140     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1141
1142     return picA->long_term_pic_num - picB->long_term_pic_num;
1143 }
1144
1145 static int
1146 compare_picture_poc_dec(const void *a, const void *b)
1147 {
1148     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1149     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1150
1151     return picB->base.poc - picA->base.poc;
1152 }
1153
1154 static int
1155 compare_picture_poc_inc(const void *a, const void *b)
1156 {
1157     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1158     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1159
1160     return picA->base.poc - picB->base.poc;
1161 }
1162
1163 static int
1164 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1165 {
1166     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1167     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1168
1169     return picB->frame_num_wrap - picA->frame_num_wrap;
1170 }
1171
1172 static int
1173 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1174 {
1175     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1176     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1177
1178     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1179 }
1180
1181 /* 8.2.4.1 - Decoding process for picture numbers */
1182 static void
1183 init_picture_refs_pic_num(
1184     GstVaapiDecoderH264 *decoder,
1185     GstVaapiPictureH264 *picture,
1186     GstH264SliceHdr     *slice_hdr
1187 )
1188 {
1189     GstVaapiDecoderH264Private * const priv = decoder->priv;
1190     GstH264PPS * const pps = slice_hdr->pps;
1191     GstH264SPS * const sps = pps->sequence;
1192     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1193     guint i;
1194
1195     GST_DEBUG("decode picture numbers");
1196
1197     for (i = 0; i < priv->short_ref_count; i++) {
1198         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1199
1200         // (8-27)
1201         if (pic->frame_num > priv->frame_num)
1202             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1203         else
1204             pic->frame_num_wrap = pic->frame_num;
1205
1206         // (8-28, 8-30, 8-31)
1207         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1208             pic->pic_num = pic->frame_num_wrap;
1209         else {
1210             if (pic->base.structure == picture->base.structure)
1211                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1212             else
1213                 pic->pic_num = 2 * pic->frame_num_wrap;
1214         }
1215     }
1216
1217     for (i = 0; i < priv->long_ref_count; i++) {
1218         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1219
1220         // (8-29, 8-32, 8-33)
1221         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1222             pic->long_term_pic_num = pic->long_term_frame_idx;
1223         else {
1224             if (pic->base.structure == picture->base.structure)
1225                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1226             else
1227                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1228         }
1229     }
1230 }
1231
1232 #define SORT_REF_LIST(list, n, compare_func) \
1233     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1234
1235 static void
1236 init_picture_refs_p_slice(
1237     GstVaapiDecoderH264 *decoder,
1238     GstVaapiPictureH264 *picture,
1239     GstH264SliceHdr     *slice_hdr
1240 )
1241 {
1242     GstVaapiDecoderH264Private * const priv = decoder->priv;
1243     GstVaapiPictureH264 **ref_list;
1244     guint i;
1245
1246     GST_DEBUG("decode reference picture list for P and SP slices");
1247
1248     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1249         /* 8.2.4.2.1 - P and SP slices in frames */
1250         if (priv->short_ref_count > 0) {
1251             ref_list = priv->RefPicList0;
1252             for (i = 0; i < priv->short_ref_count; i++)
1253                 ref_list[i] = priv->short_ref[i];
1254             SORT_REF_LIST(ref_list, i, pic_num_dec);
1255             priv->RefPicList0_count += i;
1256         }
1257
1258         if (priv->long_ref_count > 0) {
1259             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1260             for (i = 0; i < priv->long_ref_count; i++)
1261                 ref_list[i] = priv->long_ref[i];
1262             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1263             priv->RefPicList0_count += i;
1264         }
1265     }
1266     else {
1267         /* 8.2.4.2.2 - P and SP slices in fields */
1268         GstVaapiPictureH264 *short_ref[32];
1269         guint short_ref_count = 0;
1270         GstVaapiPictureH264 *long_ref[32];
1271         guint long_ref_count = 0;
1272
1273         // XXX: handle second field if current field is marked as
1274         // "used for short-term reference"
1275         if (priv->short_ref_count > 0) {
1276             for (i = 0; i < priv->short_ref_count; i++)
1277                 short_ref[i] = priv->short_ref[i];
1278             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1279             short_ref_count = i;
1280         }
1281
1282         // XXX: handle second field if current field is marked as
1283         // "used for long-term reference"
1284         if (priv->long_ref_count > 0) {
1285             for (i = 0; i < priv->long_ref_count; i++)
1286                 long_ref[i] = priv->long_ref[i];
1287             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1288             long_ref_count = i;
1289         }
1290
1291         // XXX: handle 8.2.4.2.5
1292     }
1293 }
1294
1295 static void
1296 init_picture_refs_b_slice(
1297     GstVaapiDecoderH264 *decoder,
1298     GstVaapiPictureH264 *picture,
1299     GstH264SliceHdr     *slice_hdr
1300 )
1301 {
1302     GstVaapiDecoderH264Private * const priv = decoder->priv;
1303     GstVaapiPictureH264 **ref_list;
1304     guint i, n;
1305
1306     GST_DEBUG("decode reference picture list for B slices");
1307
1308     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1309         /* 8.2.4.2.3 - B slices in frames */
1310
1311         /* RefPicList0 */
1312         if (priv->short_ref_count > 0) {
1313             // 1. Short-term references
1314             ref_list = priv->RefPicList0;
1315             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1316                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1317                     ref_list[n++] = priv->short_ref[i];
1318             }
1319             SORT_REF_LIST(ref_list, n, poc_dec);
1320             priv->RefPicList0_count += n;
1321
1322             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1323             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1324                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1325                     ref_list[n++] = priv->short_ref[i];
1326             }
1327             SORT_REF_LIST(ref_list, n, poc_inc);
1328             priv->RefPicList0_count += n;
1329         }
1330
1331         if (priv->long_ref_count > 0) {
1332             // 2. Long-term references
1333             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1334             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1335                 ref_list[n++] = priv->long_ref[i];
1336             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1337             priv->RefPicList0_count += n;
1338         }
1339
1340         /* RefPicList1 */
1341         if (priv->short_ref_count > 0) {
1342             // 1. Short-term references
1343             ref_list = priv->RefPicList1;
1344             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1345                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1346                     ref_list[n++] = priv->short_ref[i];
1347             }
1348             SORT_REF_LIST(ref_list, n, poc_inc);
1349             priv->RefPicList1_count += n;
1350
1351             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1352             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1353                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1354                     ref_list[n++] = priv->short_ref[i];
1355             }
1356             SORT_REF_LIST(ref_list, n, poc_dec);
1357             priv->RefPicList1_count += n;
1358         }
1359
1360         if (priv->long_ref_count > 0) {
1361             // 2. Long-term references
1362             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1363             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1364                 ref_list[n++] = priv->long_ref[i];
1365             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1366             priv->RefPicList1_count += n;
1367         }
1368     }
1369     else {
1370         /* 8.2.4.2.4 - B slices in fields */
1371         GstVaapiPictureH264 *short_ref0[32];
1372         guint short_ref0_count = 0;
1373         GstVaapiPictureH264 *short_ref1[32];
1374         guint short_ref1_count = 0;
1375         GstVaapiPictureH264 *long_ref[32];
1376         guint long_ref_count = 0;
1377
1378         /* refFrameList0ShortTerm */
1379         if (priv->short_ref_count > 0) {
1380             ref_list = short_ref0;
1381             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1382                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1383                     ref_list[n++] = priv->short_ref[i];
1384             }
1385             SORT_REF_LIST(ref_list, n, poc_dec);
1386             short_ref0_count += n;
1387
1388             ref_list = &short_ref0[short_ref0_count];
1389             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1390                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1391                     ref_list[n++] = priv->short_ref[i];
1392             }
1393             SORT_REF_LIST(ref_list, n, poc_inc);
1394             short_ref0_count += n;
1395         }
1396
1397         /* refFrameList1ShortTerm */
1398         if (priv->short_ref_count > 0) {
1399             ref_list = short_ref1;
1400             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1401                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1402                     ref_list[n++] = priv->short_ref[i];
1403             }
1404             SORT_REF_LIST(ref_list, n, poc_inc);
1405             short_ref1_count += n;
1406
1407             ref_list = &short_ref1[short_ref1_count];
1408             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1409                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1410                     ref_list[n++] = priv->short_ref[i];
1411             }
1412             SORT_REF_LIST(ref_list, n, poc_dec);
1413             short_ref1_count += n;
1414         }
1415
1416         /* refFrameListLongTerm */
1417         if (priv->long_ref_count > 0) {
1418             for (i = 0; i < priv->long_ref_count; i++)
1419                 long_ref[i] = priv->long_ref[i];
1420             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1421             long_ref_count = i;
1422         }
1423
1424         // XXX: handle 8.2.4.2.5
1425     }
1426
1427     /* Check whether RefPicList1 is identical to RefPicList0, then
1428        swap if necessary */
1429     if (priv->RefPicList1_count > 1 &&
1430         priv->RefPicList1_count == priv->RefPicList0_count &&
1431         memcmp(priv->RefPicList0, priv->RefPicList1,
1432                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1433         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1434         priv->RefPicList1[0] = priv->RefPicList1[1];
1435         priv->RefPicList1[1] = tmp;
1436     }
1437 }
1438
1439 #undef SORT_REF_LIST
1440
1441 static gboolean
1442 remove_reference_at(
1443     GstVaapiDecoderH264  *decoder,
1444     GstVaapiPictureH264 **pictures,
1445     guint                *picture_count,
1446     guint                 index
1447 )
1448 {
1449     guint num_pictures = *picture_count;
1450     GstVaapiPictureH264 *picture;
1451
1452     g_return_val_if_fail(index < num_pictures, FALSE);
1453
1454     picture = pictures[index];
1455     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
1456
1457     if (index != --num_pictures)
1458         pictures[index] = pictures[num_pictures];
1459     pictures[num_pictures] = NULL;
1460     *picture_count = num_pictures;
1461     return TRUE;
1462 }
1463
1464 static gint
1465 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1466 {
1467     GstVaapiDecoderH264Private * const priv = decoder->priv;
1468     guint i;
1469
1470     for (i = 0; i < priv->short_ref_count; i++) {
1471         if (priv->short_ref[i]->pic_num == pic_num)
1472             return i;
1473     }
1474     GST_ERROR("found no short-term reference picture with PicNum = %d",
1475               pic_num);
1476     return -1;
1477 }
1478
1479 static gint
1480 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1481 {
1482     GstVaapiDecoderH264Private * const priv = decoder->priv;
1483     guint i;
1484
1485     for (i = 0; i < priv->long_ref_count; i++) {
1486         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1487             return i;
1488     }
1489     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1490               long_term_pic_num);
1491     return -1;
1492 }
1493
1494 static void
1495 exec_picture_refs_modification_1(
1496     GstVaapiDecoderH264           *decoder,
1497     GstVaapiPictureH264           *picture,
1498     GstH264SliceHdr               *slice_hdr,
1499     guint                          list
1500 )
1501 {
1502     GstVaapiDecoderH264Private * const priv = decoder->priv;
1503     GstH264PPS * const pps = slice_hdr->pps;
1504     GstH264SPS * const sps = pps->sequence;
1505     GstH264RefPicListModification *ref_pic_list_modification;
1506     guint num_ref_pic_list_modifications;
1507     GstVaapiPictureH264 **ref_list;
1508     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1509     guint i, j, n, num_refs;
1510     gint found_ref_idx;
1511     gint32 MaxPicNum, CurrPicNum, picNumPred;
1512
1513     GST_DEBUG("modification process of reference picture list %u", list);
1514
1515     if (list == 0) {
1516         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1517         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1518         ref_list                       = priv->RefPicList0;
1519         ref_list_count_ptr             = &priv->RefPicList0_count;
1520         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1521     }
1522     else {
1523         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1524         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1525         ref_list                       = priv->RefPicList1;
1526         ref_list_count_ptr             = &priv->RefPicList1_count;
1527         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1528     }
1529     ref_list_count = *ref_list_count_ptr;
1530
1531     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1532         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1533         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1534     }
1535     else {
1536         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1537         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1538     }
1539
1540     picNumPred = CurrPicNum;
1541
1542     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1543         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1544         if (l->modification_of_pic_nums_idc == 3)
1545             break;
1546
1547         /* 8.2.4.3.1 - Short-term reference pictures */
1548         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1549             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1550             gint32 picNum, picNumNoWrap;
1551
1552             // (8-34)
1553             if (l->modification_of_pic_nums_idc == 0) {
1554                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1555                 if (picNumNoWrap < 0)
1556                     picNumNoWrap += MaxPicNum;
1557             }
1558
1559             // (8-35)
1560             else {
1561                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1562                 if (picNumNoWrap >= MaxPicNum)
1563                     picNumNoWrap -= MaxPicNum;
1564             }
1565             picNumPred = picNumNoWrap;
1566
1567             // (8-36)
1568             picNum = picNumNoWrap;
1569             if (picNum > CurrPicNum)
1570                 picNum -= MaxPicNum;
1571
1572             // (8-37)
1573             for (j = num_refs; j > ref_list_idx; j--)
1574                 ref_list[j] = ref_list[j - 1];
1575             found_ref_idx = find_short_term_reference(decoder, picNum);
1576             ref_list[ref_list_idx++] =
1577                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1578             n = ref_list_idx;
1579             for (j = ref_list_idx; j <= num_refs; j++) {
1580                 gint32 PicNumF;
1581                 if (!ref_list[j])
1582                     continue;
1583                 PicNumF =
1584                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1585                     ref_list[j]->pic_num : MaxPicNum;
1586                 if (PicNumF != picNum)
1587                     ref_list[n++] = ref_list[j];
1588             }
1589         }
1590
1591         /* 8.2.4.3.2 - Long-term reference pictures */
1592         else {
1593
1594             for (j = num_refs; j > ref_list_idx; j--)
1595                 ref_list[j] = ref_list[j - 1];
1596             found_ref_idx =
1597                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1598             ref_list[ref_list_idx++] =
1599                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1600             n = ref_list_idx;
1601             for (j = ref_list_idx; j <= num_refs; j++) {
1602                 gint32 LongTermPicNumF;
1603                 if (!ref_list[j])
1604                     continue;
1605                 LongTermPicNumF =
1606                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1607                     ref_list[j]->long_term_pic_num : INT_MAX;
1608                 if (LongTermPicNumF != l->value.long_term_pic_num)
1609                     ref_list[n++] = ref_list[j];
1610             }
1611         }
1612     }
1613
1614 #if DEBUG
1615     for (i = 0; i < num_refs; i++)
1616         if (!ref_list[i])
1617             GST_ERROR("list %u entry %u is empty", list, i);
1618 #endif
1619     *ref_list_count_ptr = num_refs;
1620 }
1621
1622 /* 8.2.4.3 - Modification process for reference picture lists */
1623 static void
1624 exec_picture_refs_modification(
1625     GstVaapiDecoderH264 *decoder,
1626     GstVaapiPictureH264 *picture,
1627     GstH264SliceHdr     *slice_hdr
1628 )
1629 {
1630     GST_DEBUG("execute ref_pic_list_modification()");
1631
1632     /* RefPicList0 */
1633     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1634         slice_hdr->ref_pic_list_modification_flag_l0)
1635         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1636
1637     /* RefPicList1 */
1638     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1639         slice_hdr->ref_pic_list_modification_flag_l1)
1640         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1641 }
1642
1643 static void
1644 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
1645 {
1646     GstVaapiDecoderH264Private * const priv = decoder->priv;
1647     guint i, short_ref_count, long_ref_count;
1648
1649     short_ref_count = 0;
1650     long_ref_count  = 0;
1651     for (i = 0; i < priv->dpb_count; i++) {
1652         GstVaapiPictureH264 * const picture = priv->dpb[i];
1653
1654         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1655             priv->short_ref[short_ref_count++] = picture;
1656         else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1657             priv->long_ref[long_ref_count++] = picture;
1658     }
1659
1660     for (i = short_ref_count; i < priv->short_ref_count; i++)
1661         priv->short_ref[i] = NULL;
1662     priv->short_ref_count = short_ref_count;
1663
1664     for (i = long_ref_count; i < priv->long_ref_count; i++)
1665         priv->long_ref[i] = NULL;
1666     priv->long_ref_count = long_ref_count;
1667 }
1668
1669 static gboolean
1670 init_picture_refs(
1671     GstVaapiDecoderH264 *decoder,
1672     GstVaapiPictureH264 *picture,
1673     GstH264SliceHdr     *slice_hdr
1674 )
1675 {
1676     GstVaapiDecoderH264Private * const priv = decoder->priv;
1677     GstVaapiPicture * const base_picture = &picture->base;
1678     guint i, num_refs;
1679
1680     init_picture_ref_lists(decoder);
1681     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1682
1683     priv->RefPicList0_count = 0;
1684     priv->RefPicList1_count = 0;
1685
1686     switch (base_picture->type) {
1687     case GST_VAAPI_PICTURE_TYPE_P:
1688     case GST_VAAPI_PICTURE_TYPE_SP:
1689         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1690         break;
1691     case GST_VAAPI_PICTURE_TYPE_B:
1692         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1693         break;
1694     default:
1695         break;
1696     }
1697
1698     exec_picture_refs_modification(decoder, picture, slice_hdr);
1699
1700     switch (base_picture->type) {
1701     case GST_VAAPI_PICTURE_TYPE_B:
1702         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1703         for (i = priv->RefPicList1_count; i < num_refs; i++)
1704             priv->RefPicList1[i] = NULL;
1705         priv->RefPicList1_count = num_refs;
1706
1707         // fall-through
1708     case GST_VAAPI_PICTURE_TYPE_P:
1709     case GST_VAAPI_PICTURE_TYPE_SP:
1710         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1711         for (i = priv->RefPicList0_count; i < num_refs; i++)
1712             priv->RefPicList0[i] = NULL;
1713         priv->RefPicList0_count = num_refs;
1714         break;
1715     default:
1716         break;
1717     }
1718     return TRUE;
1719 }
1720
1721 static gboolean
1722 init_picture(
1723     GstVaapiDecoderH264 *decoder,
1724     GstVaapiPictureH264 *picture,
1725     GstH264SliceHdr     *slice_hdr,
1726     GstH264NalUnit      *nalu
1727 )
1728 {
1729     GstVaapiDecoderH264Private * const priv = decoder->priv;
1730     GstVaapiPicture * const base_picture = &picture->base;
1731
1732     priv->prev_frame_num        = priv->frame_num;
1733     priv->frame_num             = slice_hdr->frame_num;
1734     picture->frame_num          = priv->frame_num;
1735     picture->frame_num_wrap     = priv->frame_num;
1736     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1737     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1738
1739     /* Reset decoder state for IDR pictures */
1740     if (nalu->type == GST_H264_NAL_SLICE_IDR) {
1741         GST_DEBUG("<IDR>");
1742         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
1743     }
1744
1745     /* Initialize slice type */
1746     switch (slice_hdr->type % 5) {
1747     case GST_H264_P_SLICE:
1748         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1749         break;
1750     case GST_H264_B_SLICE:
1751         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1752         break;
1753     case GST_H264_I_SLICE:
1754         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1755         break;
1756     case GST_H264_SP_SLICE:
1757         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1758         break;
1759     case GST_H264_SI_SLICE:
1760         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1761         break;
1762     }
1763
1764     /* Initialize picture structure */
1765     if (!slice_hdr->field_pic_flag)
1766         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1767     else if (!slice_hdr->bottom_field_flag)
1768         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
1769     else
1770         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
1771
1772     if (nalu->ref_idc) {
1773         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1774             &slice_hdr->dec_ref_pic_marking;
1775
1776         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
1777             dec_ref_pic_marking->long_term_reference_flag)
1778             GST_VAAPI_PICTURE_FLAG_SET(picture,
1779                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
1780         else
1781             GST_VAAPI_PICTURE_FLAG_SET(picture,
1782                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
1783     }
1784
1785     init_picture_poc(decoder, picture, slice_hdr);
1786
1787     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1788         dpb_flush(decoder);
1789
1790     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1791         GST_ERROR("failed to initialize references");
1792         return FALSE;
1793     }
1794     return TRUE;
1795 }
1796
1797 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1798 static gboolean
1799 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1800 {
1801     GstVaapiDecoderH264Private * const priv = decoder->priv;
1802     GstH264PPS * const pps = priv->current_picture->pps;
1803     GstH264SPS * const sps = pps->sequence;
1804     guint i, max_num_ref_frames, lowest_frame_num_index;
1805     gint32 lowest_frame_num;
1806
1807     GST_DEBUG("reference picture marking process (sliding window)");
1808
1809     max_num_ref_frames = sps->num_ref_frames;
1810     if (max_num_ref_frames == 0)
1811         max_num_ref_frames = 1;
1812
1813     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1814         return TRUE;
1815     if (priv->short_ref_count < 1)
1816         return FALSE;
1817
1818     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1819     lowest_frame_num_index = 0;
1820     for (i = 1; i < priv->short_ref_count; i++) {
1821         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1822             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1823             lowest_frame_num_index = i;
1824         }
1825     }
1826
1827     remove_reference_at(
1828         decoder,
1829         priv->short_ref, &priv->short_ref_count,
1830         lowest_frame_num_index
1831     );
1832     return TRUE;
1833 }
1834
1835 static inline gint32
1836 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
1837 {
1838     gint32 pic_num;
1839
1840     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1841         pic_num = picture->frame_num_wrap;
1842     else
1843         pic_num = 2 * picture->frame_num_wrap + 1;
1844     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1845     return pic_num;
1846 }
1847
1848 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
1849 static void
1850 exec_ref_pic_marking_adaptive_mmco_1(
1851     GstVaapiDecoderH264  *decoder,
1852     GstVaapiPictureH264  *picture,
1853     GstH264RefPicMarking *ref_pic_marking
1854 )
1855 {
1856     GstVaapiDecoderH264Private * const priv = decoder->priv;
1857     gint32 i, picNumX;
1858
1859     picNumX = get_picNumX(picture, ref_pic_marking);
1860     i = find_short_term_reference(decoder, picNumX);
1861     if (i < 0)
1862         return;
1863     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1864 }
1865
1866 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
1867 static void
1868 exec_ref_pic_marking_adaptive_mmco_2(
1869     GstVaapiDecoderH264  *decoder,
1870     GstVaapiPictureH264  *picture,
1871     GstH264RefPicMarking *ref_pic_marking
1872 )
1873 {
1874     GstVaapiDecoderH264Private * const priv = decoder->priv;
1875     gint32 i;
1876
1877     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
1878     if (i < 0)
1879         return;
1880     remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1881 }
1882
1883 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
1884 static void
1885 exec_ref_pic_marking_adaptive_mmco_3(
1886     GstVaapiDecoderH264  *decoder,
1887     GstVaapiPictureH264  *picture,
1888     GstH264RefPicMarking *ref_pic_marking
1889 )
1890 {
1891     GstVaapiDecoderH264Private * const priv = decoder->priv;
1892     gint32 i, picNumX;
1893
1894     for (i = 0; i < priv->long_ref_count; i++) {
1895         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
1896             break;
1897     }
1898     if (i != priv->long_ref_count)
1899         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1900
1901     picNumX = get_picNumX(picture, ref_pic_marking);
1902     i = find_short_term_reference(decoder, picNumX);
1903     if (i < 0)
1904         return;
1905
1906     picture = priv->short_ref[i];
1907     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1908     priv->long_ref[priv->long_ref_count++] = picture;
1909
1910     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
1911     GST_VAAPI_PICTURE_FLAG_SET(picture,
1912         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
1913 }
1914
1915 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
1916  * as "unused for reference" */
1917 static void
1918 exec_ref_pic_marking_adaptive_mmco_4(
1919     GstVaapiDecoderH264  *decoder,
1920     GstVaapiPictureH264  *picture,
1921     GstH264RefPicMarking *ref_pic_marking
1922 )
1923 {
1924     GstVaapiDecoderH264Private * const priv = decoder->priv;
1925     gint32 i, long_term_frame_idx;
1926
1927     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
1928
1929     for (i = 0; i < priv->long_ref_count; i++) {
1930         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
1931             continue;
1932         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1933         i--;
1934     }
1935 }
1936
1937 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
1938 static void
1939 exec_ref_pic_marking_adaptive_mmco_5(
1940     GstVaapiDecoderH264  *decoder,
1941     GstVaapiPictureH264  *picture,
1942     GstH264RefPicMarking *ref_pic_marking
1943 )
1944 {
1945     GstVaapiDecoderH264Private * const priv = decoder->priv;
1946
1947     dpb_flush(decoder);
1948
1949     priv->prev_pic_has_mmco5 = TRUE;
1950
1951     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
1952     priv->frame_num = 0;
1953     priv->frame_num_offset = 0;
1954     picture->frame_num = 0;
1955
1956     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
1957     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1958         picture->field_poc[TOP_FIELD] -= picture->base.poc;
1959     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1960         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
1961     picture->base.poc = 0;
1962 }
1963
1964 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
1965 static void
1966 exec_ref_pic_marking_adaptive_mmco_6(
1967     GstVaapiDecoderH264  *decoder,
1968     GstVaapiPictureH264  *picture,
1969     GstH264RefPicMarking *ref_pic_marking
1970 )
1971 {
1972     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
1973     GST_VAAPI_PICTURE_FLAG_SET(picture,
1974         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
1975 }
1976
1977 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1978 static gboolean
1979 exec_ref_pic_marking_adaptive(
1980     GstVaapiDecoderH264     *decoder,
1981     GstVaapiPictureH264     *picture,
1982     GstH264DecRefPicMarking *dec_ref_pic_marking
1983 )
1984 {
1985     guint i;
1986
1987     GST_DEBUG("reference picture marking process (adaptive memory control)");
1988
1989     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
1990         GstVaapiDecoderH264  *decoder,
1991         GstVaapiPictureH264  *picture,
1992         GstH264RefPicMarking *ref_pic_marking
1993     );
1994
1995     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
1996         NULL,
1997         exec_ref_pic_marking_adaptive_mmco_1,
1998         exec_ref_pic_marking_adaptive_mmco_2,
1999         exec_ref_pic_marking_adaptive_mmco_3,
2000         exec_ref_pic_marking_adaptive_mmco_4,
2001         exec_ref_pic_marking_adaptive_mmco_5,
2002         exec_ref_pic_marking_adaptive_mmco_6,
2003     };
2004
2005     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2006         GstH264RefPicMarking * const ref_pic_marking =
2007             &dec_ref_pic_marking->ref_pic_marking[i];
2008
2009         const guint mmco = ref_pic_marking->memory_management_control_operation;
2010         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2011             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2012         else {
2013             GST_ERROR("unhandled MMCO %u", mmco);
2014             return FALSE;
2015         }
2016     }
2017     return TRUE;
2018 }
2019
2020 /* 8.2.5 - Execute reference picture marking process */
2021 static gboolean
2022 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2023 {
2024     GstVaapiDecoderH264Private * const priv = decoder->priv;
2025
2026     priv->prev_pic_has_mmco5 = FALSE;
2027     priv->prev_pic_structure = picture->base.structure;
2028
2029     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2030         return TRUE;
2031
2032     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2033         GstVaapiSliceH264 * const slice =
2034             gst_vaapi_picture_h264_get_last_slice(picture);
2035         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2036             &slice->slice_hdr.dec_ref_pic_marking;
2037         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2038             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2039                 return FALSE;
2040         }
2041         else {
2042             if (!exec_ref_pic_marking_sliding_window(decoder))
2043                 return FALSE;
2044         }
2045     }
2046     return TRUE;
2047 }
2048
2049 static void
2050 vaapi_init_picture(VAPictureH264 *pic)
2051 {
2052     pic->picture_id           = VA_INVALID_ID;
2053     pic->frame_idx            = 0;
2054     pic->flags                = VA_PICTURE_H264_INVALID;
2055     pic->TopFieldOrderCnt     = 0;
2056     pic->BottomFieldOrderCnt  = 0;
2057 }
2058
2059 static void
2060 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture)
2061 {
2062     pic->picture_id = picture->base.surface_id;
2063     pic->flags = 0;
2064
2065     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2066         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2067         pic->frame_idx = picture->long_term_frame_idx;
2068     }
2069     else {
2070         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2071             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2072         pic->frame_idx = picture->frame_num;
2073     }
2074
2075     switch (picture->base.structure) {
2076     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2077         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2078         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2079         break;
2080     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2081         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2082         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2083         pic->BottomFieldOrderCnt = 0;
2084         break;
2085     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2086         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2087         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2088         pic->TopFieldOrderCnt = 0;
2089         break;
2090     }
2091 }
2092
2093 static gboolean
2094 fill_picture(
2095     GstVaapiDecoderH264 *decoder,
2096     GstVaapiPictureH264 *picture,
2097     GstH264SliceHdr     *slice_hdr,
2098     GstH264NalUnit      *nalu
2099 )
2100 {
2101     GstVaapiDecoderH264Private * const priv = decoder->priv;
2102     GstVaapiPicture * const base_picture = &picture->base;
2103     GstH264PPS * const pps = picture->pps;
2104     GstH264SPS * const sps = pps->sequence;
2105     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2106     guint i, n;
2107
2108     /* Fill in VAPictureParameterBufferH264 */
2109     vaapi_fill_picture(&pic_param->CurrPic, picture);
2110     for (i = 0, n = 0; i < priv->short_ref_count; i++, n++)
2111         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->short_ref[i]);
2112     for (i = 0; i < priv->long_ref_count; i++, n++)
2113         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->long_ref[i]);
2114     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2115         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2116
2117 #define COPY_FIELD(s, f) \
2118     pic_param->f = (s)->f
2119
2120 #define COPY_BFM(a, s, f) \
2121     pic_param->a.bits.f = (s)->f
2122
2123     pic_param->picture_width_in_mbs_minus1  = ((priv->width + 15) >> 4)  - 1;
2124     pic_param->picture_height_in_mbs_minus1 = ((priv->height + 15) >> 4) - 1;
2125     pic_param->frame_num                    = priv->frame_num;
2126
2127     COPY_FIELD(sps, bit_depth_luma_minus8);
2128     COPY_FIELD(sps, bit_depth_chroma_minus8);
2129     COPY_FIELD(sps, num_ref_frames);
2130     COPY_FIELD(pps, num_slice_groups_minus1);
2131     COPY_FIELD(pps, slice_group_map_type);
2132     COPY_FIELD(pps, slice_group_change_rate_minus1);
2133     COPY_FIELD(pps, pic_init_qp_minus26);
2134     COPY_FIELD(pps, pic_init_qs_minus26);
2135     COPY_FIELD(pps, chroma_qp_index_offset);
2136     COPY_FIELD(pps, second_chroma_qp_index_offset);
2137
2138     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2139     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2140     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2141
2142     COPY_BFM(seq_fields, sps, chroma_format_idc);
2143     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2144     COPY_BFM(seq_fields, sps, frame_mbs_only_flag); 
2145     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag); 
2146     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag); 
2147     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2148     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2149     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2150     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2151
2152     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2153     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2154     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2155
2156     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2157     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2158     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2159     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2160     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2161     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2162     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2163     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2164     return TRUE;
2165 }
2166
2167 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2168 static gboolean
2169 is_new_picture(
2170     GstVaapiDecoderH264 *decoder,
2171     GstH264NalUnit      *nalu,
2172     GstH264SliceHdr     *slice_hdr
2173 )
2174 {
2175     GstVaapiDecoderH264Private * const priv = decoder->priv;
2176     GstH264PPS * const pps = slice_hdr->pps;
2177     GstH264SPS * const sps = pps->sequence;
2178     GstVaapiSliceH264 *slice;
2179     GstH264SliceHdr *prev_slice_hdr;
2180
2181     if (!priv->current_picture)
2182         return TRUE;
2183
2184     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2185     if (!slice)
2186         return FALSE;
2187     prev_slice_hdr = &slice->slice_hdr;
2188
2189 #define CHECK_EXPR(expr, field_name) do {              \
2190         if (!(expr)) {                                 \
2191             GST_DEBUG(field_name " differs in value"); \
2192             return TRUE;                               \
2193         }                                              \
2194     } while (0)
2195
2196 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2197     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2198
2199     /* frame_num differs in value, regardless of inferred values to 0 */
2200     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2201
2202     /* pic_parameter_set_id differs in value */
2203     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2204
2205     /* field_pic_flag differs in value */
2206     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2207
2208     /* bottom_field_flag is present in both and differs in value */
2209     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2210         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2211
2212     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2213     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2214                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2215
2216     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2217        value or delta_pic_order_cnt_bottom differs in value */
2218     if (sps->pic_order_cnt_type == 0) {
2219         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2220         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2221             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2222     }
2223
2224     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2225        differs in value or delta_pic_order_cnt[1] differs in value */
2226     else if (sps->pic_order_cnt_type == 1) {
2227         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2228         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2229     }
2230
2231     /* IdrPicFlag differs in value */
2232     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_IDR(priv->current_picture) ^
2233                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2234
2235     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2236     if (GST_VAAPI_PICTURE_IS_IDR(priv->current_picture))
2237         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2238
2239 #undef CHECK_EXPR
2240 #undef CHECK_VALUE
2241     return FALSE;
2242 }
2243
2244 static GstVaapiDecoderStatus
2245 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2246 {
2247     GstVaapiDecoderH264Private * const priv = decoder->priv;
2248     GstVaapiPictureH264 *picture;
2249     GstVaapiDecoderStatus status;
2250     GstH264PPS * const pps = slice_hdr->pps;
2251     GstH264SPS * const sps = pps->sequence;
2252
2253     status = decode_current_picture(decoder);
2254     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2255         return status;
2256
2257     status = ensure_context(decoder, sps);
2258     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2259         return status;
2260
2261     picture = gst_vaapi_picture_h264_new(decoder);
2262     if (!picture) {
2263         GST_ERROR("failed to allocate picture");
2264         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2265     }
2266     priv->current_picture = picture;
2267
2268     picture->pps = pps;
2269
2270     status = ensure_quant_matrix(decoder, picture);
2271     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2272         GST_ERROR("failed to reset quantizer matrix");
2273         return status;
2274     }
2275
2276     if (!init_picture(decoder, picture, slice_hdr, nalu))
2277         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2278     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2279         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2280     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2281 }
2282
2283 static inline guint
2284 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2285 {
2286     guint epb_count;
2287
2288     epb_count = slice_hdr->n_emulation_prevention_bytes;
2289     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2290 }
2291
2292 static gboolean
2293 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2294 {
2295     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2296     GstH264PPS * const pps = slice_hdr->pps;
2297     GstH264SPS * const sps = pps->sequence;
2298     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2299     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2300     guint num_weight_tables = 0;
2301     gint i, j;
2302
2303     if (pps->weighted_pred_flag &&
2304         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2305         num_weight_tables = 1;
2306     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2307         num_weight_tables = 2;
2308     else
2309         num_weight_tables = 0;
2310
2311     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2312     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2313     slice_param->luma_weight_l0_flag      = 0;
2314     slice_param->chroma_weight_l0_flag    = 0;
2315     slice_param->luma_weight_l1_flag      = 0;
2316     slice_param->chroma_weight_l1_flag    = 0;
2317
2318     if (num_weight_tables < 1)
2319         return TRUE;
2320
2321     slice_param->luma_weight_l0_flag = 1;
2322     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2323         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2324         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2325     }
2326
2327     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2328     if (slice_param->chroma_weight_l0_flag) {
2329         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2330             for (j = 0; j < 2; j++) {
2331                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2332                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2333             }
2334         }
2335     }
2336
2337     if (num_weight_tables < 2)
2338         return TRUE;
2339
2340     slice_param->luma_weight_l1_flag = 1;
2341     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2342         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2343         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2344     }
2345
2346     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2347     if (slice_param->chroma_weight_l1_flag) {
2348         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2349             for (j = 0; j < 2; j++) {
2350                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2351                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2352             }
2353         }
2354     }
2355     return TRUE;
2356 }
2357
2358 static gboolean
2359 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2360 {
2361     GstVaapiDecoderH264Private * const priv = decoder->priv;
2362     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2363     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2364     guint i, num_ref_lists = 0;
2365
2366     slice_param->num_ref_idx_l0_active_minus1 = 0;
2367     slice_param->num_ref_idx_l1_active_minus1 = 0;
2368
2369     if (GST_H264_IS_B_SLICE(slice_hdr))
2370         num_ref_lists = 2;
2371     else if (GST_H264_IS_I_SLICE(slice_hdr))
2372         num_ref_lists = 0;
2373     else
2374         num_ref_lists = 1;
2375
2376     if (num_ref_lists < 1)
2377         return TRUE;
2378
2379     slice_param->num_ref_idx_l0_active_minus1 =
2380         slice_hdr->num_ref_idx_l0_active_minus1;
2381
2382     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2383         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i]);
2384     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2385         vaapi_init_picture(&slice_param->RefPicList0[i]);
2386
2387     if (num_ref_lists < 2)
2388         return TRUE;
2389
2390     slice_param->num_ref_idx_l1_active_minus1 =
2391         slice_hdr->num_ref_idx_l1_active_minus1;
2392
2393     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2394         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i]);
2395     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2396         vaapi_init_picture(&slice_param->RefPicList1[i]);
2397     return TRUE;
2398 }
2399
2400 static gboolean
2401 fill_slice(
2402     GstVaapiDecoderH264 *decoder,
2403     GstVaapiSliceH264   *slice,
2404     GstH264NalUnit      *nalu
2405 )
2406 {
2407     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2408     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2409
2410     /* Fill in VASliceParameterBufferH264 */
2411     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2412     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2413     slice_param->slice_type                     = slice_hdr->type % 5;
2414     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2415     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2416     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2417     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2418     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2419     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2420
2421     if (!fill_RefPicList(decoder, slice))
2422         return FALSE;
2423     if (!fill_pred_weight_table(decoder, slice))
2424         return FALSE;
2425     return TRUE;
2426 }
2427
2428 static GstVaapiDecoderStatus
2429 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2430 {
2431     GstVaapiDecoderH264Private * const priv = decoder->priv;
2432     GstVaapiDecoderStatus status;
2433     GstVaapiPictureH264 *picture;
2434     GstVaapiSliceH264 *slice = NULL;
2435     GstH264SliceHdr *slice_hdr;
2436     GstH264ParserResult result;
2437
2438     GST_DEBUG("slice (%u bytes)", nalu->size);
2439
2440     slice = gst_vaapi_slice_h264_new(
2441         decoder,
2442         nalu->data + nalu->offset,
2443         nalu->size
2444     );
2445     if (!slice) {
2446         GST_ERROR("failed to allocate slice");
2447         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2448     }
2449
2450     slice_hdr = &slice->slice_hdr;
2451     memset(slice_hdr, 0, sizeof(*slice_hdr));
2452     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2453     if (result != GST_H264_PARSER_OK) {
2454         status = get_status(result);
2455         goto error;
2456     }
2457
2458     if (is_new_picture(decoder, nalu, slice_hdr)) {
2459         status = decode_picture(decoder, nalu, slice_hdr);
2460         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2461             goto error;
2462     }
2463     picture = priv->current_picture;
2464
2465     if (!fill_slice(decoder, slice, nalu)) {
2466         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2467         goto error;
2468     }
2469     gst_vaapi_picture_add_slice(
2470         GST_VAAPI_PICTURE_CAST(picture),
2471         GST_VAAPI_SLICE_CAST(slice)
2472     );
2473     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2474
2475 error:
2476     if (slice)
2477         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2478     return status;
2479 }
2480
2481 static inline gint
2482 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2483 {
2484     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2485                                                      0xffffff00, 0x00000100,
2486                                                      ofs, size,
2487                                                      scp);
2488 }
2489
2490 static GstVaapiDecoderStatus
2491 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2492 {
2493     GstVaapiDecoderStatus status;
2494
2495     switch (nalu->type) {
2496     case GST_H264_NAL_SLICE_IDR:
2497         /* fall-through. IDR specifics are handled in init_picture() */
2498     case GST_H264_NAL_SLICE:
2499         status = decode_slice(decoder, nalu);
2500         break;
2501     case GST_H264_NAL_SPS:
2502         status = decode_sps(decoder, nalu);
2503         break;
2504     case GST_H264_NAL_PPS:
2505         status = decode_pps(decoder, nalu);
2506         break;
2507     case GST_H264_NAL_SEI:
2508         status = decode_sei(decoder, nalu);
2509         break;
2510     case GST_H264_NAL_SEQ_END:
2511         status = decode_sequence_end(decoder);
2512         break;
2513     case GST_H264_NAL_AU_DELIMITER:
2514         /* skip all Access Unit NALs */
2515         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2516         break;
2517     case GST_H264_NAL_FILLER_DATA:
2518         /* skip all Filler Data NALs */
2519         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2520         break;
2521     default:
2522         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2523         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2524         break;
2525     }
2526     return status;
2527 }
2528
2529 static GstVaapiDecoderStatus
2530 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2531 {
2532     GstVaapiDecoderH264Private * const priv = decoder->priv;
2533     GstVaapiDecoderStatus status;
2534     GstH264ParserResult result;
2535     GstH264NalUnit nalu;
2536     gboolean is_eos;
2537     const guchar *buf;
2538     guint i, buf_size, nalu_size, size;
2539     guint32 start_code;
2540     gint ofs;
2541
2542     buf      = GST_BUFFER_DATA(buffer);
2543     buf_size = GST_BUFFER_SIZE(buffer);
2544     is_eos   = GST_BUFFER_IS_EOS(buffer);
2545     if (buf && buf_size > 0)
2546         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2547
2548     size = gst_adapter_available(priv->adapter);
2549     do {
2550         if (size == 0) {
2551             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2552             break;
2553         }
2554
2555         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2556         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2557             break;
2558
2559         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2560         if (priv->is_avc) {
2561             if (size < priv->nal_length_size)
2562                 break;
2563             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2564
2565             nalu_size = 0;
2566             for (i = 0; i < priv->nal_length_size; i++)
2567                 nalu_size = (nalu_size << 8) | buf[i];
2568
2569             buf_size = priv->nal_length_size + nalu_size;
2570             if (size < buf_size)
2571                 break;
2572             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2573             size -= buf_size;
2574
2575             buf      = GST_BUFFER_DATA(buffer);
2576             buf_size = GST_BUFFER_SIZE(buffer);
2577
2578             result = gst_h264_parser_identify_nalu_avc(
2579                 priv->parser,
2580                 buf, 0, buf_size, priv->nal_length_size,
2581                 &nalu
2582             );
2583         }
2584         else {
2585             if (size < 4)
2586                 break;
2587             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
2588             if (ofs < 0)
2589                 break;
2590             gst_adapter_flush(priv->adapter, ofs);
2591             size -= ofs;
2592
2593             ofs = G_UNLIKELY(size < 8) ? -1 :
2594                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
2595             if (ofs < 0) {
2596                 // Assume the whole NAL unit is present if end-of-stream
2597                 if (!is_eos)
2598                     break;
2599                 ofs = size;
2600             }
2601             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
2602             size -= ofs;
2603
2604             buf      = GST_BUFFER_DATA(buffer);
2605             buf_size = GST_BUFFER_SIZE(buffer);
2606
2607             result = gst_h264_parser_identify_nalu_unchecked(
2608                 priv->parser,
2609                 buf, 0, buf_size,
2610                 &nalu
2611             );
2612         }
2613         status = get_status(result);
2614         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
2615             status = decode_nalu(decoder, &nalu);
2616         gst_buffer_unref(buffer);
2617     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
2618
2619     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
2620                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
2621         status = decode_sequence_end(decoder);
2622     return status;
2623 }
2624
2625 static GstVaapiDecoderStatus
2626 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2627 {
2628     GstVaapiDecoderH264Private * const priv = decoder->priv;
2629     GstVaapiDecoderStatus status;
2630     GstH264NalUnit nalu;
2631     GstH264ParserResult result;
2632     guchar *buf;
2633     guint buf_size;
2634     guint i, ofs, num_sps, num_pps;
2635
2636     buf      = GST_BUFFER_DATA(buffer);
2637     buf_size = GST_BUFFER_SIZE(buffer);
2638     if (!buf || buf_size == 0)
2639         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2640
2641     if (buf_size < 8)
2642         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2643
2644     if (buf[0] != 1) {
2645         GST_ERROR("failed to decode codec-data, not in avcC format");
2646         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2647     }
2648
2649     priv->nal_length_size = (buf[4] & 0x03) + 1;
2650
2651     num_sps = buf[5] & 0x1f;
2652     ofs = 6;
2653
2654     for (i = 0; i < num_sps; i++) {
2655         result = gst_h264_parser_identify_nalu_avc(
2656             priv->parser,
2657             buf, ofs, buf_size, 2,
2658             &nalu
2659         );
2660         if (result != GST_H264_PARSER_OK)
2661             return get_status(result);
2662
2663         status = decode_sps(decoder, &nalu);
2664         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2665             return status;
2666         ofs = nalu.offset + nalu.size;
2667     }
2668
2669     num_pps = buf[ofs];
2670     ofs++;
2671
2672     for (i = 0; i < num_pps; i++) {
2673         result = gst_h264_parser_identify_nalu_avc(
2674             priv->parser,
2675             buf, ofs, buf_size, 2,
2676             &nalu
2677         );
2678         if (result != GST_H264_PARSER_OK)
2679             return get_status(result);
2680
2681         status = decode_pps(decoder, &nalu);
2682         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2683             return status;
2684         ofs = nalu.offset + nalu.size;
2685     }
2686
2687     priv->is_avc = TRUE;
2688     return status;
2689 }
2690
2691 GstVaapiDecoderStatus
2692 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2693 {
2694     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2695     GstVaapiDecoderH264Private * const priv = decoder->priv;
2696     GstVaapiDecoderStatus status;
2697     GstBuffer *codec_data;
2698
2699     g_return_val_if_fail(priv->is_constructed,
2700                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2701
2702     if (!priv->is_opened) {
2703         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2704         if (!priv->is_opened)
2705             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2706
2707         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2708         if (codec_data) {
2709             status = decode_codec_data(decoder, codec_data);
2710             if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2711                 return status;