decoder: h264: cope with new gst_h264_parser_parse_sei() interface.
[vaapi:gstreamer-vaapi.git] / gst-libs / gst / vaapi / gstvaapidecoder_h264.c
1 /*
2  *  gstvaapidecoder_h264.c - H.264 decoder
3  *
4  *  Copyright (C) 2011-2014 Intel Corporation
5  *    Author: Gwenole Beauchesne <gwenole.beauchesne@intel.com>
6  *
7  *  This library is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU Lesser General Public License
9  *  as published by the Free Software Foundation; either version 2.1
10  *  of the License, or (at your option) any later version.
11  *
12  *  This library is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  *  Lesser General Public License for more details.
16  *
17  *  You should have received a copy of the GNU Lesser General Public
18  *  License along with this library; if not, write to the Free
19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20  *  Boston, MA 02110-1301 USA
21  */
22
23 /**
24  * SECTION:gstvaapidecoder_h264
25  * @short_description: H.264 decoder
26  */
27
28 #include "sysdeps.h"
29 #include <string.h>
30 #include <gst/base/gstadapter.h>
31 #include <gst/codecparsers/gsth264parser.h>
32 #include "gstvaapidecoder_h264.h"
33 #include "gstvaapidecoder_objects.h"
34 #include "gstvaapidecoder_priv.h"
35 #include "gstvaapidisplay_priv.h"
36 #include "gstvaapiobject_priv.h"
37 #include "gstvaapiutils_h264_priv.h"
38
39 #define DEBUG 1
40 #include "gstvaapidebug.h"
41
42 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
43 #define USE_STRICT_DPB_ORDERING 0
44
45 typedef struct _GstVaapiDecoderH264Private      GstVaapiDecoderH264Private;
46 typedef struct _GstVaapiDecoderH264Class        GstVaapiDecoderH264Class;
47 typedef struct _GstVaapiFrameStore              GstVaapiFrameStore;
48 typedef struct _GstVaapiFrameStoreClass         GstVaapiFrameStoreClass;
49 typedef struct _GstVaapiParserInfoH264          GstVaapiParserInfoH264;
50 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
51
52 // Used for field_poc[]
53 #define TOP_FIELD       0
54 #define BOTTOM_FIELD    1
55
56 /* ------------------------------------------------------------------------- */
57 /* --- H.264 Parser Info                                                 --- */
58 /* ------------------------------------------------------------------------- */
59
60 #define GST_VAAPI_PARSER_INFO_H264(obj) \
61     ((GstVaapiParserInfoH264 *)(obj))
62
63 struct _GstVaapiParserInfoH264 {
64     GstVaapiMiniObject  parent_instance;
65     GstH264NalUnit      nalu;
66     union {
67         GstH264SPS      sps;
68         GstH264PPS      pps;
69         GstH264SliceHdr slice_hdr;
70     }                   data;
71     guint               state;
72 };
73
74 static inline const GstVaapiMiniObjectClass *
75 gst_vaapi_parser_info_h264_class(void)
76 {
77     static const GstVaapiMiniObjectClass GstVaapiParserInfoH264Class = {
78         sizeof(GstVaapiParserInfoH264),
79         NULL
80     };
81     return &GstVaapiParserInfoH264Class;
82 }
83
84 static inline GstVaapiParserInfoH264 *
85 gst_vaapi_parser_info_h264_new(void)
86 {
87     return (GstVaapiParserInfoH264 *)
88         gst_vaapi_mini_object_new(gst_vaapi_parser_info_h264_class());
89 }
90
91 #define gst_vaapi_parser_info_h264_ref(pi) \
92     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(pi))
93
94 #define gst_vaapi_parser_info_h264_unref(pi) \
95     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(pi))
96
97 #define gst_vaapi_parser_info_h264_replace(old_pi_ptr, new_pi)          \
98     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_pi_ptr),  \
99         (GstVaapiMiniObject *)(new_pi))
100
101 /* ------------------------------------------------------------------------- */
102 /* --- H.264 Pictures                                                    --- */
103 /* ------------------------------------------------------------------------- */
104
105 /*
106  * Extended picture flags:
107  *
108  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
109  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
110  *     "used for short-term reference"
111  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
112  *     "used for long-term reference"
113  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
114  *     reference picture (short-term reference or long-term reference)
115  */
116 enum {
117     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
118
119     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
120         GST_VAAPI_PICTURE_FLAG_REFERENCE),
121     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
122         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
123     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
124         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
125         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
126 };
127
128 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
129     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
130
131 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
132     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
133       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
134      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
135
136 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
137     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
138       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
139      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
140
141 struct _GstVaapiPictureH264 {
142     GstVaapiPicture             base;
143     GstH264PPS                 *pps;
144     GstH264SliceHdr            *last_slice_hdr;
145     guint                       structure;
146     gint32                      field_poc[2];
147     gint32                      frame_num;              // Original frame_num from slice_header()
148     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
149     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
150     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
151     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
152     GstVaapiPictureH264        *other_field;            // Temporary for ref pic marking: other field in the same frame store
153     guint                       output_flag             : 1;
154     guint                       output_needed           : 1;
155 };
156
157 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264, gst_vaapi_picture_h264);
158
159 void
160 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *picture)
161 {
162     gst_vaapi_picture_destroy(GST_VAAPI_PICTURE(picture));
163 }
164
165 gboolean
166 gst_vaapi_picture_h264_create(
167     GstVaapiPictureH264                      *picture,
168     const GstVaapiCodecObjectConstructorArgs *args
169 )
170 {
171     if (!gst_vaapi_picture_create(GST_VAAPI_PICTURE(picture), args))
172         return FALSE;
173
174     picture->field_poc[0]       = G_MAXINT32;
175     picture->field_poc[1]       = G_MAXINT32;
176     picture->output_needed      = FALSE;
177     return TRUE;
178 }
179
180 static inline GstVaapiPictureH264 *
181 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
182 {
183     return (GstVaapiPictureH264 *)gst_vaapi_codec_object_new(
184         &GstVaapiPictureH264Class,
185         GST_VAAPI_CODEC_BASE(decoder),
186         NULL, sizeof(VAPictureParameterBufferH264),
187         NULL, 0,
188         0);
189 }
190
191 static inline void
192 gst_vaapi_picture_h264_set_reference(
193     GstVaapiPictureH264 *picture,
194     guint                reference_flags,
195     gboolean             other_field
196 )
197 {
198     if (!picture)
199         return;
200     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
201     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
202
203     if (!other_field || !(picture = picture->other_field))
204         return;
205     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
206     GST_VAAPI_PICTURE_FLAG_SET(picture, reference_flags);
207 }
208
209 static inline GstVaapiPictureH264 *
210 gst_vaapi_picture_h264_new_field(GstVaapiPictureH264 *picture)
211 {
212     g_return_val_if_fail(picture, NULL);
213
214     return (GstVaapiPictureH264 *)gst_vaapi_picture_new_field(&picture->base);
215 }
216
217 /* ------------------------------------------------------------------------- */
218 /* --- Frame Buffers (DPB)                                               --- */
219 /* ------------------------------------------------------------------------- */
220
221 struct _GstVaapiFrameStore {
222     /*< private >*/
223     GstVaapiMiniObject          parent_instance;
224
225     guint                       structure;
226     GstVaapiPictureH264        *buffers[2];
227     guint                       num_buffers;
228     guint                       output_needed;
229 };
230
231 static void
232 gst_vaapi_frame_store_finalize(gpointer object)
233 {
234     GstVaapiFrameStore * const fs = object;
235     guint i;
236
237     for (i = 0; i < fs->num_buffers; i++)
238         gst_vaapi_picture_replace(&fs->buffers[i], NULL);
239 }
240
241 static GstVaapiFrameStore *
242 gst_vaapi_frame_store_new(GstVaapiPictureH264 *picture)
243 {
244     GstVaapiFrameStore *fs;
245
246     static const GstVaapiMiniObjectClass GstVaapiFrameStoreClass = {
247         sizeof(GstVaapiFrameStore),
248         gst_vaapi_frame_store_finalize
249     };
250
251     fs = (GstVaapiFrameStore *)
252         gst_vaapi_mini_object_new(&GstVaapiFrameStoreClass);
253     if (!fs)
254         return NULL;
255
256     fs->structure       = picture->structure;
257     fs->buffers[0]      = gst_vaapi_picture_ref(picture);
258     fs->buffers[1]      = NULL;
259     fs->num_buffers     = 1;
260     fs->output_needed   = picture->output_needed;
261     return fs;
262 }
263
264 static gboolean
265 gst_vaapi_frame_store_add(GstVaapiFrameStore *fs, GstVaapiPictureH264 *picture)
266 {
267     guint field;
268
269     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
270     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FRAME(picture), FALSE);
271     g_return_val_if_fail(!GST_VAAPI_PICTURE_IS_FIRST_FIELD(picture), FALSE);
272
273     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], picture);
274     if (picture->output_flag) {
275         picture->output_needed = TRUE;
276         fs->output_needed++;
277     }
278
279     fs->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
280
281     field = picture->structure == GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD ?
282         TOP_FIELD : BOTTOM_FIELD;
283     g_return_val_if_fail(fs->buffers[0]->field_poc[field] == G_MAXINT32, FALSE);
284     fs->buffers[0]->field_poc[field] = picture->field_poc[field];
285     g_return_val_if_fail(picture->field_poc[!field] == G_MAXINT32, FALSE);
286     picture->field_poc[!field] = fs->buffers[0]->field_poc[!field];
287     return TRUE;
288 }
289
290 static gboolean
291 gst_vaapi_frame_store_split_fields(GstVaapiFrameStore *fs)
292 {
293     GstVaapiPictureH264 * const first_field = fs->buffers[0];
294     GstVaapiPictureH264 *second_field;
295
296     g_return_val_if_fail(fs->num_buffers == 1, FALSE);
297
298     first_field->base.structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
299     GST_VAAPI_PICTURE_FLAG_SET(first_field, GST_VAAPI_PICTURE_FLAG_INTERLACED);
300
301     second_field = gst_vaapi_picture_h264_new_field(first_field);
302     if (!second_field)
303         return FALSE;
304     gst_vaapi_picture_replace(&fs->buffers[fs->num_buffers++], second_field);
305     gst_vaapi_picture_unref(second_field);
306
307     second_field->frame_num    = first_field->frame_num;
308     second_field->field_poc[0] = first_field->field_poc[0];
309     second_field->field_poc[1] = first_field->field_poc[1];
310     second_field->output_flag  = first_field->output_flag;
311     if (second_field->output_flag) {
312         second_field->output_needed = TRUE;
313         fs->output_needed++;
314     }
315     return TRUE;
316 }
317
318 static inline gboolean
319 gst_vaapi_frame_store_has_frame(GstVaapiFrameStore *fs)
320 {
321     return fs->structure == GST_VAAPI_PICTURE_STRUCTURE_FRAME;
322 }
323
324 static inline gboolean
325 gst_vaapi_frame_store_has_reference(GstVaapiFrameStore *fs)
326 {
327     guint i;
328
329     for (i = 0; i < fs->num_buffers; i++) {
330         if (GST_VAAPI_PICTURE_IS_REFERENCE(fs->buffers[i]))
331             return TRUE;
332     }
333     return FALSE;
334 }
335
336 #define gst_vaapi_frame_store_ref(fs) \
337     gst_vaapi_mini_object_ref(GST_VAAPI_MINI_OBJECT(fs))
338
339 #define gst_vaapi_frame_store_unref(fs) \
340     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(fs))
341
342 #define gst_vaapi_frame_store_replace(old_fs_p, new_fs)                 \
343     gst_vaapi_mini_object_replace((GstVaapiMiniObject **)(old_fs_p),    \
344         (GstVaapiMiniObject *)(new_fs))
345
346 /* ------------------------------------------------------------------------- */
347 /* --- H.264 Decoder                                                     --- */
348 /* ------------------------------------------------------------------------- */
349
350 #define GST_VAAPI_DECODER_H264_CAST(decoder) \
351     ((GstVaapiDecoderH264 *)(decoder))
352
353 typedef enum {
354     GST_H264_VIDEO_STATE_GOT_SPS        = 1 << 0,
355     GST_H264_VIDEO_STATE_GOT_PPS        = 1 << 1,
356     GST_H264_VIDEO_STATE_GOT_SLICE      = 1 << 2,
357
358     GST_H264_VIDEO_STATE_VALID_PICTURE_HEADERS = (
359         GST_H264_VIDEO_STATE_GOT_SPS |
360         GST_H264_VIDEO_STATE_GOT_PPS),
361     GST_H264_VIDEO_STATE_VALID_PICTURE = (
362         GST_H264_VIDEO_STATE_VALID_PICTURE_HEADERS |
363         GST_H264_VIDEO_STATE_GOT_SLICE)
364 } GstH264VideoState;
365
366 struct _GstVaapiDecoderH264Private {
367     GstH264NalParser           *parser;
368     guint                       parser_state;
369     guint                       decoder_state;
370     GstVaapiPictureH264        *current_picture;
371     GstVaapiParserInfoH264     *prev_slice_pi;
372     GstVaapiFrameStore         *prev_frame;
373     GstVaapiFrameStore         *dpb[16];
374     guint                       dpb_count;
375     guint                       dpb_size;
376     GstVaapiProfile             profile;
377     GstVaapiEntrypoint          entrypoint;
378     GstVaapiChromaType          chroma_type;
379     GstVaapiPictureH264        *short_ref[32];
380     guint                       short_ref_count;
381     GstVaapiPictureH264        *long_ref[32];
382     guint                       long_ref_count;
383     GstVaapiPictureH264        *RefPicList0[32];
384     guint                       RefPicList0_count;
385     GstVaapiPictureH264        *RefPicList1[32];
386     guint                       RefPicList1_count;
387     guint                       nal_length_size;
388     guint                       mb_width;
389     guint                       mb_height;
390     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
391     gint32                      poc_msb;                // PicOrderCntMsb
392     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
393     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
394     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
395     gint32                      frame_num_offset;       // FrameNumOffset
396     gint32                      frame_num;              // frame_num (from slice_header())
397     gint32                      prev_frame_num;         // prevFrameNum
398     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
399     gboolean                    prev_pic_structure;     // previous picture structure
400     guint                       is_opened               : 1;
401     guint                       is_avcC                 : 1;
402     guint                       has_context             : 1;
403     guint                       progressive_sequence    : 1;
404 };
405
406 /**
407  * GstVaapiDecoderH264:
408  *
409  * A decoder based on H264.
410  */
411 struct _GstVaapiDecoderH264 {
412     /*< private >*/
413     GstVaapiDecoder             parent_instance;
414     GstVaapiDecoderH264Private  priv;
415 };
416
417 /**
418  * GstVaapiDecoderH264Class:
419  *
420  * A decoder class based on H264.
421  */
422 struct _GstVaapiDecoderH264Class {
423     /*< private >*/
424     GstVaapiDecoderClass parent_class;
425 };
426
427 static gboolean
428 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
429
430 /* Get number of reference frames to use */
431 static guint
432 get_max_dec_frame_buffering(GstH264SPS *sps)
433 {
434     guint max_dec_frame_buffering, PicSizeMbs;
435     GstVaapiLevelH264 level;
436     const GstVaapiH264LevelLimits *level_limits;
437
438     /* Table A-1 - Level limits */
439     if (G_UNLIKELY(sps->level_idc == 11 && sps->constraint_set3_flag))
440         level = GST_VAAPI_LEVEL_H264_L1b;
441     else
442         level = gst_vaapi_utils_h264_get_level(sps->level_idc);
443     level_limits = gst_vaapi_utils_h264_get_level_limits(level);
444     if (!level_limits)
445         return 16;
446
447     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
448                   (sps->pic_height_in_map_units_minus1 + 1) *
449                   (sps->frame_mbs_only_flag ? 1 : 2));
450     max_dec_frame_buffering = level_limits->MaxDpbMbs / PicSizeMbs;
451
452     /* VUI parameters */
453     if (sps->vui_parameters_present_flag) {
454         GstH264VUIParams * const vui_params = &sps->vui_parameters;
455         if (vui_params->bitstream_restriction_flag)
456             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
457         else {
458             switch (sps->profile_idc) {
459             case 44:  // CAVLC 4:4:4 Intra profile
460             case GST_H264_PROFILE_SCALABLE_HIGH:
461             case GST_H264_PROFILE_HIGH:
462             case GST_H264_PROFILE_HIGH10:
463             case GST_H264_PROFILE_HIGH_422:
464             case GST_H264_PROFILE_HIGH_444:
465                 if (sps->constraint_set3_flag)
466                     max_dec_frame_buffering = 0;
467                 break;
468             }
469         }
470     }
471
472     if (max_dec_frame_buffering > 16)
473         max_dec_frame_buffering = 16;
474     else if (max_dec_frame_buffering < sps->num_ref_frames)
475         max_dec_frame_buffering = sps->num_ref_frames;
476     return MAX(1, max_dec_frame_buffering);
477 }
478
479 static void
480 array_remove_index_fast(void *array, guint *array_length_ptr, guint index)
481 {
482     gpointer * const entries = array;
483     guint num_entries = *array_length_ptr;
484
485     g_return_if_fail(index < num_entries);
486
487     if (index != --num_entries)
488         entries[index] = entries[num_entries];
489     entries[num_entries] = NULL;
490     *array_length_ptr = num_entries;
491 }
492
493 #if 1
494 static inline void
495 array_remove_index(void *array, guint *array_length_ptr, guint index)
496 {
497     array_remove_index_fast(array, array_length_ptr, index);
498 }
499 #else
500 static void
501 array_remove_index(void *array, guint *array_length_ptr, guint index)
502 {
503     gpointer * const entries = array;
504     const guint num_entries = *array_length_ptr - 1;
505     guint i;
506
507     g_return_if_fail(index <= num_entries);
508
509     for (i = index; i < num_entries; i++)
510         entries[i] = entries[i + 1];
511     entries[num_entries] = NULL;
512     *array_length_ptr = num_entries;
513 }
514 #endif
515
516 #define ARRAY_REMOVE_INDEX(array, index) \
517     array_remove_index(array, &array##_count, index)
518
519 static void
520 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
521 {
522     GstVaapiDecoderH264Private * const priv = &decoder->priv;
523     guint i, num_frames = --priv->dpb_count;
524
525     if (USE_STRICT_DPB_ORDERING) {
526         for (i = index; i < num_frames; i++)
527             gst_vaapi_frame_store_replace(&priv->dpb[i], priv->dpb[i + 1]);
528     }
529     else if (index != num_frames)
530         gst_vaapi_frame_store_replace(&priv->dpb[index], priv->dpb[num_frames]);
531     gst_vaapi_frame_store_replace(&priv->dpb[num_frames], NULL);
532 }
533
534 static gboolean
535 dpb_output(
536     GstVaapiDecoderH264 *decoder,
537     GstVaapiFrameStore  *fs,
538     GstVaapiPictureH264 *picture
539 )
540 {
541     picture->output_needed = FALSE;
542
543     if (fs) {
544         if (--fs->output_needed > 0)
545             return TRUE;
546         picture = fs->buffers[0];
547     }
548     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
549 }
550
551 static inline void
552 dpb_evict(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture, guint i)
553 {
554     GstVaapiDecoderH264Private * const priv = &decoder->priv;
555     GstVaapiFrameStore * const fs = priv->dpb[i];
556
557     if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
558         dpb_remove_index(decoder, i);
559 }
560
561 static gboolean
562 dpb_bump(GstVaapiDecoderH264 *decoder)
563 {
564     GstVaapiDecoderH264Private * const priv = &decoder->priv;
565     GstVaapiPictureH264 *found_picture = NULL;
566     guint i, j, found_index;
567     gboolean success;
568
569     for (i = 0; i < priv->dpb_count; i++) {
570         GstVaapiFrameStore * const fs = priv->dpb[i];
571         if (!fs->output_needed)
572             continue;
573         for (j = 0; j < fs->num_buffers; j++) {
574             GstVaapiPictureH264 * const picture = fs->buffers[j];
575             if (!picture->output_needed)
576                 continue;
577             if (!found_picture || found_picture->base.poc > picture->base.poc)
578                 found_picture = picture, found_index = i;
579         }
580     }
581     if (!found_picture)
582         return FALSE;
583
584     success = dpb_output(decoder, priv->dpb[found_index], found_picture);
585     dpb_evict(decoder, found_picture, found_index);
586     return success;
587 }
588
589 static void
590 dpb_clear(GstVaapiDecoderH264 *decoder)
591 {
592     GstVaapiDecoderH264Private * const priv = &decoder->priv;
593     guint i;
594
595     for (i = 0; i < priv->dpb_count; i++)
596         gst_vaapi_frame_store_replace(&priv->dpb[i], NULL);
597     priv->dpb_count = 0;
598
599     gst_vaapi_frame_store_replace(&priv->prev_frame, NULL);
600 }
601
602 static void
603 dpb_flush(GstVaapiDecoderH264 *decoder)
604 {
605     while (dpb_bump(decoder))
606         ;
607     dpb_clear(decoder);
608 }
609
610 static gboolean
611 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
612 {
613     GstVaapiDecoderH264Private * const priv = &decoder->priv;
614     GstVaapiFrameStore *fs;
615     guint i, j;
616
617     // Remove all unused pictures
618     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
619         i = 0;
620         while (i < priv->dpb_count) {
621             GstVaapiFrameStore * const fs = priv->dpb[i];
622             if (!fs->output_needed && !gst_vaapi_frame_store_has_reference(fs))
623                 dpb_remove_index(decoder, i);
624             else
625                 i++;
626         }
627     }
628
629     // Check if picture is the second field and the first field is still in DPB
630     fs = priv->prev_frame;
631     if (fs && !gst_vaapi_frame_store_has_frame(fs))
632         return gst_vaapi_frame_store_add(fs, picture);
633
634     // Create new frame store, and split fields if necessary
635     fs = gst_vaapi_frame_store_new(picture);
636     if (!fs)
637         return FALSE;
638     gst_vaapi_frame_store_replace(&priv->prev_frame, fs);
639     gst_vaapi_frame_store_unref(fs);
640
641     if (!priv->progressive_sequence && gst_vaapi_frame_store_has_frame(fs)) {
642         if (!gst_vaapi_frame_store_split_fields(fs))
643             return FALSE;
644     }
645
646     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
647     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
648         while (priv->dpb_count == priv->dpb_size) {
649             if (!dpb_bump(decoder))
650                 return FALSE;
651         }
652         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
653         if (picture->output_flag) {
654             picture->output_needed = TRUE;
655             fs->output_needed++;
656         }
657     }
658
659     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
660     else {
661         if (!picture->output_flag)
662             return TRUE;
663         while (priv->dpb_count == priv->dpb_size) {
664             gboolean found_picture = FALSE;
665             for (i = 0; !found_picture && i < priv->dpb_count; i++) {
666                 GstVaapiFrameStore * const fs = priv->dpb[i];
667                 if (!fs->output_needed)
668                     continue;
669                 for (j = 0; !found_picture && j < fs->num_buffers; j++)
670                     found_picture = fs->buffers[j]->output_needed &&
671                         fs->buffers[j]->base.poc < picture->base.poc;
672             }
673             if (!found_picture)
674                 return dpb_output(decoder, NULL, picture);
675             if (!dpb_bump(decoder))
676                 return FALSE;
677         }
678         gst_vaapi_frame_store_replace(&priv->dpb[priv->dpb_count++], fs);
679         picture->output_needed = TRUE;
680         fs->output_needed++;
681     }
682     return TRUE;
683 }
684
685 static inline void
686 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
687 {
688     GstVaapiDecoderH264Private * const priv = &decoder->priv;
689
690     priv->dpb_size = get_max_dec_frame_buffering(sps);
691     GST_DEBUG("DPB size %u", priv->dpb_size);
692 }
693
694 static GstVaapiDecoderStatus
695 get_status(GstH264ParserResult result)
696 {
697     GstVaapiDecoderStatus status;
698
699     switch (result) {
700     case GST_H264_PARSER_OK:
701         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
702         break;
703     case GST_H264_PARSER_NO_NAL_END:
704         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
705         break;
706     case GST_H264_PARSER_ERROR:
707         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
708         break;
709     default:
710         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
711         break;
712     }
713     return status;
714 }
715
716 static void
717 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
718 {
719     GstVaapiDecoderH264Private * const priv = &decoder->priv;
720
721     gst_vaapi_picture_replace(&priv->current_picture, NULL);
722     gst_vaapi_parser_info_h264_replace(&priv->prev_slice_pi, NULL);
723
724     dpb_clear(decoder);
725
726     if (priv->parser) {
727         gst_h264_nal_parser_free(priv->parser);
728         priv->parser = NULL;
729     }
730 }
731
732 static gboolean
733 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder)
734 {
735     GstVaapiDecoderH264Private * const priv = &decoder->priv;
736
737     gst_vaapi_decoder_h264_close(decoder);
738
739     priv->parser = gst_h264_nal_parser_new();
740     if (!priv->parser)
741         return FALSE;
742     return TRUE;
743 }
744
745 static void
746 gst_vaapi_decoder_h264_destroy(GstVaapiDecoder *base_decoder)
747 {
748     GstVaapiDecoderH264 * const decoder =
749         GST_VAAPI_DECODER_H264_CAST(base_decoder);
750
751     gst_vaapi_decoder_h264_close(decoder);
752 }
753
754 static gboolean
755 gst_vaapi_decoder_h264_create(GstVaapiDecoder *base_decoder)
756 {
757     GstVaapiDecoderH264 * const decoder =
758         GST_VAAPI_DECODER_H264_CAST(base_decoder);
759     GstVaapiDecoderH264Private * const priv = &decoder->priv;
760
761     priv->profile               = GST_VAAPI_PROFILE_UNKNOWN;
762     priv->entrypoint            = GST_VAAPI_ENTRYPOINT_VLD;
763     priv->chroma_type           = GST_VAAPI_CHROMA_TYPE_YUV420;
764     priv->prev_pic_structure    = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
765     priv->progressive_sequence  = TRUE;
766     return TRUE;
767 }
768
769 static void
770 fill_profiles(GstVaapiProfile profiles[16], guint *n_profiles_ptr,
771     GstVaapiProfile profile)
772 {
773     guint n_profiles = *n_profiles_ptr;
774
775     profiles[n_profiles++] = profile;
776     switch (profile) {
777     case GST_VAAPI_PROFILE_H264_MAIN:
778         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
779         break;
780     default:
781         break;
782     }
783     *n_profiles_ptr = n_profiles;
784 }
785
786 static GstVaapiProfile
787 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
788 {
789     GstVaapiDecoderH264Private * const priv = &decoder->priv;
790     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
791     GstVaapiProfile profile, profiles[4];
792     guint i, n_profiles = 0;
793
794     profile = gst_vaapi_utils_h264_get_profile(sps->profile_idc);
795     if (!profile)
796         return GST_VAAPI_PROFILE_UNKNOWN;
797
798     fill_profiles(profiles, &n_profiles, profile);
799     switch (profile) {
800     case GST_VAAPI_PROFILE_H264_BASELINE:
801         if (sps->constraint_set1_flag) { // A.2.2 (main profile)
802             fill_profiles(profiles, &n_profiles,
803                 GST_VAAPI_PROFILE_H264_CONSTRAINED_BASELINE);
804             fill_profiles(profiles, &n_profiles,
805                 GST_VAAPI_PROFILE_H264_MAIN);
806         }
807         break;
808     case GST_VAAPI_PROFILE_H264_EXTENDED:
809         if (sps->constraint_set1_flag) { // A.2.2 (main profile)
810             fill_profiles(profiles, &n_profiles,
811                 GST_VAAPI_PROFILE_H264_MAIN);
812         }
813         break;
814     default:
815         break;
816     }
817
818     /* If the preferred profile (profiles[0]) matches one that we already
819        found, then just return it now instead of searching for it again */
820     if (profiles[0] == priv->profile)
821         return priv->profile;
822
823     for (i = 0; i < n_profiles; i++) {
824         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
825             return profiles[i];
826     }
827     return GST_VAAPI_PROFILE_UNKNOWN;
828 }
829
830 static GstVaapiDecoderStatus
831 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
832 {
833     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
834     GstVaapiDecoderH264Private * const priv = &decoder->priv;
835     GstVaapiContextInfo info;
836     GstVaapiProfile profile;
837     GstVaapiChromaType chroma_type;
838     gboolean reset_context = FALSE;
839     guint mb_width, mb_height;
840
841     profile = get_profile(decoder, sps);
842     if (!profile) {
843         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
844         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
845     }
846
847     if (priv->profile != profile) {
848         GST_DEBUG("profile changed");
849         reset_context = TRUE;
850         priv->profile = profile;
851     }
852
853     chroma_type = gst_vaapi_utils_h264_get_chroma_type(sps->chroma_format_idc);
854     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
855         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
856         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
857     }
858
859     if (priv->chroma_type != chroma_type) {
860         GST_DEBUG("chroma format changed");
861         reset_context     = TRUE;
862         priv->chroma_type = chroma_type;
863     }
864
865     mb_width  = sps->pic_width_in_mbs_minus1 + 1;
866     mb_height = (sps->pic_height_in_map_units_minus1 + 1) <<
867         !sps->frame_mbs_only_flag;
868     if (priv->mb_width != mb_width || priv->mb_height != mb_height) {
869         GST_DEBUG("size changed");
870         reset_context   = TRUE;
871         priv->mb_width  = mb_width;
872         priv->mb_height = mb_height;
873     }
874
875     priv->progressive_sequence = sps->frame_mbs_only_flag;
876 #if 0
877     /* XXX: we only output complete frames for now */
878     gst_vaapi_decoder_set_interlaced(base_decoder, !priv->progressive_sequence);
879 #endif
880
881     gst_vaapi_decoder_set_pixel_aspect_ratio(
882         base_decoder,
883         sps->vui_parameters.par_n,
884         sps->vui_parameters.par_d
885     );
886
887     if (!reset_context && priv->has_context)
888         return GST_VAAPI_DECODER_STATUS_SUCCESS;
889
890     /* XXX: fix surface size when cropping is implemented */
891     info.profile    = priv->profile;
892     info.entrypoint = priv->entrypoint;
893     info.width      = sps->width;
894     info.height     = sps->height;
895     info.ref_frames = get_max_dec_frame_buffering(sps);
896
897     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
898         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
899     priv->has_context = TRUE;
900
901     /* Reset DPB */
902     dpb_reset(decoder, sps);
903     return GST_VAAPI_DECODER_STATUS_SUCCESS;
904 }
905
906 static void
907 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
908 {
909     guint i;
910
911     /* There are always 6 4x4 scaling lists */
912     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
913     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
914
915     for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++)
916         gst_h264_video_quant_matrix_4x4_get_raster_from_zigzag(
917             iq_matrix->ScalingList4x4[i], pps->scaling_lists_4x4[i]);
918 }
919
920 static void
921 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
922 {
923     const GstH264SPS * const sps = pps->sequence;
924     guint i, n;
925
926     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
927     if (!pps->transform_8x8_mode_flag)
928         return;
929
930     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
931     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
932
933     n = (sps->chroma_format_idc != 3) ? 2 : 6;
934     for (i = 0; i < n; i++) {
935         gst_h264_video_quant_matrix_8x8_get_raster_from_zigzag(
936             iq_matrix->ScalingList8x8[i], pps->scaling_lists_8x8[i]);
937     }
938 }
939
940 static GstVaapiDecoderStatus
941 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
942 {
943     GstVaapiPicture * const base_picture = &picture->base;
944     GstH264PPS * const pps = picture->pps;
945     GstH264SPS * const sps = pps->sequence;
946     VAIQMatrixBufferH264 *iq_matrix;
947
948     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
949     if (!base_picture->iq_matrix) {
950         GST_ERROR("failed to allocate IQ matrix");
951         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
952     }
953     iq_matrix = base_picture->iq_matrix->param;
954
955     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
956        is not large enough to hold lists for 4:4:4 */
957     if (sps->chroma_format_idc == 3)
958         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
959
960     fill_iq_matrix_4x4(iq_matrix, pps);
961     fill_iq_matrix_8x8(iq_matrix, pps);
962
963     return GST_VAAPI_DECODER_STATUS_SUCCESS;
964 }
965
966 static inline gboolean
967 is_valid_state(guint state, guint ref_state)
968 {
969     return (state & ref_state) == ref_state;
970 }
971
972 static GstVaapiDecoderStatus
973 decode_current_picture(GstVaapiDecoderH264 *decoder)
974 {
975     GstVaapiDecoderH264Private * const priv = &decoder->priv;
976     GstVaapiPictureH264 * const picture = priv->current_picture;
977
978     if (!is_valid_state(priv->decoder_state, GST_H264_VIDEO_STATE_VALID_PICTURE))
979         goto drop_frame;
980     priv->decoder_state = 0;
981
982     if (!picture)
983         return GST_VAAPI_DECODER_STATUS_SUCCESS;
984
985     if (!exec_ref_pic_marking(decoder, picture))
986         goto error;
987     if (!dpb_add(decoder, picture))
988         goto error;
989     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
990         goto error;
991     if (priv->prev_frame && gst_vaapi_frame_store_has_frame(priv->prev_frame))
992         gst_vaapi_picture_replace(&priv->current_picture, NULL);
993     return GST_VAAPI_DECODER_STATUS_SUCCESS;
994
995 error:
996     /* XXX: fix for cases where first field failed to be decoded */
997     gst_vaapi_picture_replace(&priv->current_picture, NULL);
998     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
999
1000 drop_frame:
1001     priv->decoder_state = 0;
1002     return GST_VAAPI_DECODER_STATUS_DROP_FRAME;
1003 }
1004
1005 static GstVaapiDecoderStatus
1006 parse_sps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1007 {
1008     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1009     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1010     GstH264SPS * const sps = &pi->data.sps;
1011     GstH264ParserResult result;
1012
1013     GST_DEBUG("parse SPS");
1014
1015     priv->parser_state = 0;
1016
1017     /* Variables that don't have inferred values per the H.264
1018        standard but that should get a default value anyway */
1019     sps->log2_max_pic_order_cnt_lsb_minus4 = 0;
1020
1021     result = gst_h264_parser_parse_sps(priv->parser, &pi->nalu, sps, TRUE);
1022     if (result != GST_H264_PARSER_OK)
1023         return get_status(result);
1024
1025     priv->parser_state |= GST_H264_VIDEO_STATE_GOT_SPS;
1026     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1027 }
1028
1029 static GstVaapiDecoderStatus
1030 parse_pps(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1031 {
1032     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1033     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1034     GstH264PPS * const pps = &pi->data.pps;
1035     GstH264ParserResult result;
1036
1037     GST_DEBUG("parse PPS");
1038
1039     priv->parser_state &= GST_H264_VIDEO_STATE_GOT_SPS;
1040
1041     /* Variables that don't have inferred values per the H.264
1042        standard but that should get a default value anyway */
1043     pps->slice_group_map_type = 0;
1044     pps->slice_group_change_rate_minus1 = 0;
1045
1046     result = gst_h264_parser_parse_pps(priv->parser, &pi->nalu, pps);
1047     if (result != GST_H264_PARSER_OK)
1048         return get_status(result);
1049
1050     priv->parser_state |= GST_H264_VIDEO_STATE_GOT_PPS;
1051     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1052 }
1053
1054 static GstVaapiDecoderStatus
1055 parse_sei(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1056 {
1057     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1058     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1059     GArray *sei_messages = NULL;
1060     GstH264ParserResult result;
1061
1062     GST_DEBUG("parse SEI");
1063
1064     result = gst_h264_parser_parse_sei(priv->parser, &pi->nalu, &sei_messages);
1065     if (result != GST_H264_PARSER_OK) {
1066         GST_WARNING("failed to parse SEI messages");
1067         g_array_unref(sei_messages);
1068         return get_status(result);
1069     }
1070
1071     g_array_unref(sei_messages);
1072     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1073 }
1074
1075 static GstVaapiDecoderStatus
1076 parse_slice(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
1077 {
1078     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1079     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
1080     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
1081     GstH264ParserResult result;
1082
1083     GST_DEBUG("parse slice");
1084
1085     priv->parser_state &= (GST_H264_VIDEO_STATE_GOT_SPS|
1086                            GST_H264_VIDEO_STATE_GOT_PPS);
1087
1088     /* Variables that don't have inferred values per the H.264
1089        standard but that should get a default value anyway */
1090     slice_hdr->cabac_init_idc = 0;
1091     slice_hdr->direct_spatial_mv_pred_flag = 0;
1092
1093     result = gst_h264_parser_parse_slice_hdr(priv->parser, &pi->nalu,
1094         slice_hdr, TRUE, TRUE);
1095     if (result != GST_H264_PARSER_OK)
1096         return get_status(result);
1097
1098     priv->parser_state |= GST_H264_VIDEO_STATE_GOT_SLICE;
1099     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1100 }
1101
1102 static GstVaapiDecoderStatus
1103 decode_sequence_end(GstVaapiDecoderH264 *decoder)
1104 {
1105     GstVaapiDecoderStatus status;
1106
1107     GST_DEBUG("decode sequence-end");
1108
1109     status = decode_current_picture(decoder);
1110     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
1111         return status;
1112
1113     dpb_flush(decoder);
1114     return GST_VAAPI_DECODER_STATUS_SUCCESS;
1115 }
1116
1117 /* 8.2.1.1 - Decoding process for picture order count type 0 */
1118 static void
1119 init_picture_poc_0(
1120     GstVaapiDecoderH264 *decoder,
1121     GstVaapiPictureH264 *picture,
1122     GstH264SliceHdr     *slice_hdr
1123 )
1124 {
1125     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1126     GstH264PPS * const pps = slice_hdr->pps;
1127     GstH264SPS * const sps = pps->sequence;
1128     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
1129     gint32 temp_poc;
1130
1131     GST_DEBUG("decode picture order count type 0");
1132
1133     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1134         priv->prev_poc_msb = 0;
1135         priv->prev_poc_lsb = 0;
1136     }
1137     else if (priv->prev_pic_has_mmco5) {
1138         priv->prev_poc_msb = 0;
1139         priv->prev_poc_lsb =
1140             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
1141              0 : priv->field_poc[TOP_FIELD]);
1142     }
1143     else {
1144         priv->prev_poc_msb = priv->poc_msb;
1145         priv->prev_poc_lsb = priv->poc_lsb;
1146     }
1147
1148     // (8-3)
1149     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
1150     if (priv->poc_lsb < priv->prev_poc_lsb &&
1151         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
1152         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
1153     else if (priv->poc_lsb > priv->prev_poc_lsb &&
1154              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
1155         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
1156     else
1157         priv->poc_msb = priv->prev_poc_msb;
1158
1159     temp_poc = priv->poc_msb + priv->poc_lsb;
1160     switch (picture->structure) {
1161     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1162         // (8-4, 8-5)
1163         priv->field_poc[TOP_FIELD] = temp_poc;
1164         priv->field_poc[BOTTOM_FIELD] = temp_poc +
1165             slice_hdr->delta_pic_order_cnt_bottom;
1166         break;
1167     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1168         // (8-4)
1169         priv->field_poc[TOP_FIELD] = temp_poc;
1170         break;
1171     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1172         // (8-5)
1173         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1174         break;
1175     }
1176 }
1177
1178 /* 8.2.1.2 - Decoding process for picture order count type 1 */
1179 static void
1180 init_picture_poc_1(
1181     GstVaapiDecoderH264 *decoder,
1182     GstVaapiPictureH264 *picture,
1183     GstH264SliceHdr     *slice_hdr
1184 )
1185 {
1186     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1187     GstH264PPS * const pps = slice_hdr->pps;
1188     GstH264SPS * const sps = pps->sequence;
1189     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1190     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
1191     guint i;
1192
1193     GST_DEBUG("decode picture order count type 1");
1194
1195     if (priv->prev_pic_has_mmco5)
1196         prev_frame_num_offset = 0;
1197     else
1198         prev_frame_num_offset = priv->frame_num_offset;
1199
1200     // (8-6)
1201     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1202         priv->frame_num_offset = 0;
1203     else if (priv->prev_frame_num > priv->frame_num)
1204         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1205     else
1206         priv->frame_num_offset = prev_frame_num_offset;
1207
1208     // (8-7)
1209     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
1210         abs_frame_num = priv->frame_num_offset + priv->frame_num;
1211     else
1212         abs_frame_num = 0;
1213     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1214         abs_frame_num = abs_frame_num - 1;
1215
1216     if (abs_frame_num > 0) {
1217         gint32 expected_delta_per_poc_cycle;
1218         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1219
1220         expected_delta_per_poc_cycle = 0;
1221         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1222             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1223
1224         // (8-8)
1225         poc_cycle_cnt = (abs_frame_num - 1) /
1226             sps->num_ref_frames_in_pic_order_cnt_cycle;
1227         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1228             sps->num_ref_frames_in_pic_order_cnt_cycle;
1229
1230         // (8-9)
1231         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1232         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1233             expected_poc += sps->offset_for_ref_frame[i];
1234     }
1235     else
1236         expected_poc = 0;
1237     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1238         expected_poc += sps->offset_for_non_ref_pic;
1239
1240     // (8-10)
1241     switch (picture->structure) {
1242     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1243         priv->field_poc[TOP_FIELD] = expected_poc +
1244             slice_hdr->delta_pic_order_cnt[0];
1245         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1246             sps->offset_for_top_to_bottom_field +
1247             slice_hdr->delta_pic_order_cnt[1];
1248         break;
1249     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1250         priv->field_poc[TOP_FIELD] = expected_poc +
1251             slice_hdr->delta_pic_order_cnt[0];
1252         break;
1253     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1254         priv->field_poc[BOTTOM_FIELD] = expected_poc + 
1255             sps->offset_for_top_to_bottom_field +
1256             slice_hdr->delta_pic_order_cnt[0];
1257         break;
1258     }
1259 }
1260
1261 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1262 static void
1263 init_picture_poc_2(
1264     GstVaapiDecoderH264 *decoder,
1265     GstVaapiPictureH264 *picture,
1266     GstH264SliceHdr     *slice_hdr
1267 )
1268 {
1269     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1270     GstH264PPS * const pps = slice_hdr->pps;
1271     GstH264SPS * const sps = pps->sequence;
1272     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1273     gint32 prev_frame_num_offset, temp_poc;
1274
1275     GST_DEBUG("decode picture order count type 2");
1276
1277     if (priv->prev_pic_has_mmco5)
1278         prev_frame_num_offset = 0;
1279     else
1280         prev_frame_num_offset = priv->frame_num_offset;
1281
1282     // (8-11)
1283     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1284         priv->frame_num_offset = 0;
1285     else if (priv->prev_frame_num > priv->frame_num)
1286         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1287     else
1288         priv->frame_num_offset = prev_frame_num_offset;
1289
1290     // (8-12)
1291     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1292         temp_poc = 0;
1293     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1294         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1295     else
1296         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1297
1298     // (8-13)
1299     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1300         priv->field_poc[TOP_FIELD] = temp_poc;
1301     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1302         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1303 }
1304
1305 /* 8.2.1 - Decoding process for picture order count */
1306 static void
1307 init_picture_poc(
1308     GstVaapiDecoderH264 *decoder,
1309     GstVaapiPictureH264 *picture,
1310     GstH264SliceHdr     *slice_hdr
1311 )
1312 {
1313     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1314     GstH264PPS * const pps = slice_hdr->pps;
1315     GstH264SPS * const sps = pps->sequence;
1316
1317     switch (sps->pic_order_cnt_type) {
1318     case 0:
1319         init_picture_poc_0(decoder, picture, slice_hdr);
1320         break;
1321     case 1:
1322         init_picture_poc_1(decoder, picture, slice_hdr);
1323         break;
1324     case 2:
1325         init_picture_poc_2(decoder, picture, slice_hdr);
1326         break;
1327     }
1328
1329     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1330         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1331     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1332         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1333     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1334 }
1335
1336 static int
1337 compare_picture_pic_num_dec(const void *a, const void *b)
1338 {
1339     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1340     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1341
1342     return picB->pic_num - picA->pic_num;
1343 }
1344
1345 static int
1346 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1347 {
1348     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1349     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1350
1351     return picA->long_term_pic_num - picB->long_term_pic_num;
1352 }
1353
1354 static int
1355 compare_picture_poc_dec(const void *a, const void *b)
1356 {
1357     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1358     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1359
1360     return picB->base.poc - picA->base.poc;
1361 }
1362
1363 static int
1364 compare_picture_poc_inc(const void *a, const void *b)
1365 {
1366     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1367     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1368
1369     return picA->base.poc - picB->base.poc;
1370 }
1371
1372 static int
1373 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1374 {
1375     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1376     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1377
1378     return picB->frame_num_wrap - picA->frame_num_wrap;
1379 }
1380
1381 static int
1382 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1383 {
1384     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1385     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1386
1387     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1388 }
1389
1390 /* 8.2.4.1 - Decoding process for picture numbers */
1391 static void
1392 init_picture_refs_pic_num(
1393     GstVaapiDecoderH264 *decoder,
1394     GstVaapiPictureH264 *picture,
1395     GstH264SliceHdr     *slice_hdr
1396 )
1397 {
1398     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1399     GstH264PPS * const pps = slice_hdr->pps;
1400     GstH264SPS * const sps = pps->sequence;
1401     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1402     guint i;
1403
1404     GST_DEBUG("decode picture numbers");
1405
1406     for (i = 0; i < priv->short_ref_count; i++) {
1407         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1408
1409         // (8-27)
1410         if (pic->frame_num > priv->frame_num)
1411             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1412         else
1413             pic->frame_num_wrap = pic->frame_num;
1414
1415         // (8-28, 8-30, 8-31)
1416         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1417             pic->pic_num = pic->frame_num_wrap;
1418         else {
1419             if (pic->structure == picture->structure)
1420                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1421             else
1422                 pic->pic_num = 2 * pic->frame_num_wrap;
1423         }
1424     }
1425
1426     for (i = 0; i < priv->long_ref_count; i++) {
1427         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1428
1429         // (8-29, 8-32, 8-33)
1430         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1431             pic->long_term_pic_num = pic->long_term_frame_idx;
1432         else {
1433             if (pic->structure == picture->structure)
1434                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1435             else
1436                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1437         }
1438     }
1439 }
1440
1441 #define SORT_REF_LIST(list, n, compare_func) \
1442     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1443
1444 static void
1445 init_picture_refs_fields_1(
1446     guint                picture_structure,
1447     GstVaapiPictureH264 *RefPicList[32],
1448     guint               *RefPicList_count,
1449     GstVaapiPictureH264 *ref_list[32],
1450     guint                ref_list_count
1451 )
1452 {
1453     guint i, j, n;
1454
1455     i = 0;
1456     j = 0;
1457     n = *RefPicList_count;
1458     do {
1459         g_assert(n < 32);
1460         for (; i < ref_list_count; i++) {
1461             if (ref_list[i]->structure == picture_structure) {
1462                 RefPicList[n++] = ref_list[i++];
1463                 break;
1464             }
1465         }
1466         for (; j < ref_list_count; j++) {
1467             if (ref_list[j]->structure != picture_structure) {
1468                 RefPicList[n++] = ref_list[j++];
1469                 break;
1470             }
1471         }
1472     } while (i < ref_list_count || j < ref_list_count);
1473     *RefPicList_count = n;
1474 }
1475
1476 static inline void
1477 init_picture_refs_fields(
1478     GstVaapiPictureH264 *picture,
1479     GstVaapiPictureH264 *RefPicList[32],
1480     guint               *RefPicList_count,
1481     GstVaapiPictureH264 *short_ref[32],
1482     guint                short_ref_count,
1483     GstVaapiPictureH264 *long_ref[32],
1484     guint                long_ref_count
1485 )
1486 {
1487     guint n = 0;
1488
1489     /* 8.2.4.2.5 - reference picture lists in fields */
1490     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1491         short_ref, short_ref_count);
1492     init_picture_refs_fields_1(picture->structure, RefPicList, &n,
1493         long_ref, long_ref_count);
1494     *RefPicList_count = n;
1495 }
1496
1497 static void
1498 init_picture_refs_p_slice(
1499     GstVaapiDecoderH264 *decoder,
1500     GstVaapiPictureH264 *picture,
1501     GstH264SliceHdr     *slice_hdr
1502 )
1503 {
1504     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1505     GstVaapiPictureH264 **ref_list;
1506     guint i;
1507
1508     GST_DEBUG("decode reference picture list for P and SP slices");
1509
1510     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1511         /* 8.2.4.2.1 - P and SP slices in frames */
1512         if (priv->short_ref_count > 0) {
1513             ref_list = priv->RefPicList0;
1514             for (i = 0; i < priv->short_ref_count; i++)
1515                 ref_list[i] = priv->short_ref[i];
1516             SORT_REF_LIST(ref_list, i, pic_num_dec);
1517             priv->RefPicList0_count += i;
1518         }
1519
1520         if (priv->long_ref_count > 0) {
1521             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1522             for (i = 0; i < priv->long_ref_count; i++)
1523                 ref_list[i] = priv->long_ref[i];
1524             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1525             priv->RefPicList0_count += i;
1526         }
1527     }
1528     else {
1529         /* 8.2.4.2.2 - P and SP slices in fields */
1530         GstVaapiPictureH264 *short_ref[32];
1531         guint short_ref_count = 0;
1532         GstVaapiPictureH264 *long_ref[32];
1533         guint long_ref_count = 0;
1534
1535         if (priv->short_ref_count > 0) {
1536             for (i = 0; i < priv->short_ref_count; i++)
1537                 short_ref[i] = priv->short_ref[i];
1538             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1539             short_ref_count = i;
1540         }
1541
1542         if (priv->long_ref_count > 0) {
1543             for (i = 0; i < priv->long_ref_count; i++)
1544                 long_ref[i] = priv->long_ref[i];
1545             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1546             long_ref_count = i;
1547         }
1548
1549         init_picture_refs_fields(
1550             picture,
1551             priv->RefPicList0, &priv->RefPicList0_count,
1552             short_ref,          short_ref_count,
1553             long_ref,           long_ref_count
1554         );
1555     }
1556 }
1557
1558 static void
1559 init_picture_refs_b_slice(
1560     GstVaapiDecoderH264 *decoder,
1561     GstVaapiPictureH264 *picture,
1562     GstH264SliceHdr     *slice_hdr
1563 )
1564 {
1565     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1566     GstVaapiPictureH264 **ref_list;
1567     guint i, n;
1568
1569     GST_DEBUG("decode reference picture list for B slices");
1570
1571     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1572         /* 8.2.4.2.3 - B slices in frames */
1573
1574         /* RefPicList0 */
1575         if (priv->short_ref_count > 0) {
1576             // 1. Short-term references
1577             ref_list = priv->RefPicList0;
1578             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1579                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1580                     ref_list[n++] = priv->short_ref[i];
1581             }
1582             SORT_REF_LIST(ref_list, n, poc_dec);
1583             priv->RefPicList0_count += n;
1584
1585             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1586             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1587                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1588                     ref_list[n++] = priv->short_ref[i];
1589             }
1590             SORT_REF_LIST(ref_list, n, poc_inc);
1591             priv->RefPicList0_count += n;
1592         }
1593
1594         if (priv->long_ref_count > 0) {
1595             // 2. Long-term references
1596             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1597             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1598                 ref_list[n++] = priv->long_ref[i];
1599             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1600             priv->RefPicList0_count += n;
1601         }
1602
1603         /* RefPicList1 */
1604         if (priv->short_ref_count > 0) {
1605             // 1. Short-term references
1606             ref_list = priv->RefPicList1;
1607             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1608                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1609                     ref_list[n++] = priv->short_ref[i];
1610             }
1611             SORT_REF_LIST(ref_list, n, poc_inc);
1612             priv->RefPicList1_count += n;
1613
1614             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1615             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1616                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1617                     ref_list[n++] = priv->short_ref[i];
1618             }
1619             SORT_REF_LIST(ref_list, n, poc_dec);
1620             priv->RefPicList1_count += n;
1621         }
1622
1623         if (priv->long_ref_count > 0) {
1624             // 2. Long-term references
1625             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1626             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1627                 ref_list[n++] = priv->long_ref[i];
1628             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1629             priv->RefPicList1_count += n;
1630         }
1631     }
1632     else {
1633         /* 8.2.4.2.4 - B slices in fields */
1634         GstVaapiPictureH264 *short_ref0[32];
1635         guint short_ref0_count = 0;
1636         GstVaapiPictureH264 *short_ref1[32];
1637         guint short_ref1_count = 0;
1638         GstVaapiPictureH264 *long_ref[32];
1639         guint long_ref_count = 0;
1640
1641         /* refFrameList0ShortTerm */
1642         if (priv->short_ref_count > 0) {
1643             ref_list = short_ref0;
1644             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1645                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1646                     ref_list[n++] = priv->short_ref[i];
1647             }
1648             SORT_REF_LIST(ref_list, n, poc_dec);
1649             short_ref0_count += n;
1650
1651             ref_list = &short_ref0[short_ref0_count];
1652             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1653                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1654                     ref_list[n++] = priv->short_ref[i];
1655             }
1656             SORT_REF_LIST(ref_list, n, poc_inc);
1657             short_ref0_count += n;
1658         }
1659
1660         /* refFrameList1ShortTerm */
1661         if (priv->short_ref_count > 0) {
1662             ref_list = short_ref1;
1663             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1664                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1665                     ref_list[n++] = priv->short_ref[i];
1666             }
1667             SORT_REF_LIST(ref_list, n, poc_inc);
1668             short_ref1_count += n;
1669
1670             ref_list = &short_ref1[short_ref1_count];
1671             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1672                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1673                     ref_list[n++] = priv->short_ref[i];
1674             }
1675             SORT_REF_LIST(ref_list, n, poc_dec);
1676             short_ref1_count += n;
1677         }
1678
1679         /* refFrameListLongTerm */
1680         if (priv->long_ref_count > 0) {
1681             for (i = 0; i < priv->long_ref_count; i++)
1682                 long_ref[i] = priv->long_ref[i];
1683             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1684             long_ref_count = i;
1685         }
1686
1687         init_picture_refs_fields(
1688             picture,
1689             priv->RefPicList0, &priv->RefPicList0_count,
1690             short_ref0,         short_ref0_count,
1691             long_ref,           long_ref_count
1692         );
1693
1694         init_picture_refs_fields(
1695             picture,
1696             priv->RefPicList1, &priv->RefPicList1_count,
1697             short_ref1,         short_ref1_count,
1698             long_ref,           long_ref_count
1699         );
1700    }
1701
1702     /* Check whether RefPicList1 is identical to RefPicList0, then
1703        swap if necessary */
1704     if (priv->RefPicList1_count > 1 &&
1705         priv->RefPicList1_count == priv->RefPicList0_count &&
1706         memcmp(priv->RefPicList0, priv->RefPicList1,
1707                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1708         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1709         priv->RefPicList1[0] = priv->RefPicList1[1];
1710         priv->RefPicList1[1] = tmp;
1711     }
1712 }
1713
1714 #undef SORT_REF_LIST
1715
1716 static gint
1717 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1718 {
1719     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1720     guint i;
1721
1722     for (i = 0; i < priv->short_ref_count; i++) {
1723         if (priv->short_ref[i]->pic_num == pic_num)
1724             return i;
1725     }
1726     GST_ERROR("found no short-term reference picture with PicNum = %d",
1727               pic_num);
1728     return -1;
1729 }
1730
1731 static gint
1732 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1733 {
1734     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1735     guint i;
1736
1737     for (i = 0; i < priv->long_ref_count; i++) {
1738         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1739             return i;
1740     }
1741     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1742               long_term_pic_num);
1743     return -1;
1744 }
1745
1746 static void
1747 exec_picture_refs_modification_1(
1748     GstVaapiDecoderH264           *decoder,
1749     GstVaapiPictureH264           *picture,
1750     GstH264SliceHdr               *slice_hdr,
1751     guint                          list
1752 )
1753 {
1754     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1755     GstH264PPS * const pps = slice_hdr->pps;
1756     GstH264SPS * const sps = pps->sequence;
1757     GstH264RefPicListModification *ref_pic_list_modification;
1758     guint num_ref_pic_list_modifications;
1759     GstVaapiPictureH264 **ref_list;
1760     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1761     guint i, j, n, num_refs;
1762     gint found_ref_idx;
1763     gint32 MaxPicNum, CurrPicNum, picNumPred;
1764
1765     GST_DEBUG("modification process of reference picture list %u", list);
1766
1767     if (list == 0) {
1768         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1769         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1770         ref_list                       = priv->RefPicList0;
1771         ref_list_count_ptr             = &priv->RefPicList0_count;
1772         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1773     }
1774     else {
1775         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1776         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1777         ref_list                       = priv->RefPicList1;
1778         ref_list_count_ptr             = &priv->RefPicList1_count;
1779         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1780     }
1781     ref_list_count = *ref_list_count_ptr;
1782
1783     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1784         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1785         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1786     }
1787     else {
1788         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1789         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1790     }
1791
1792     picNumPred = CurrPicNum;
1793
1794     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1795         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1796         if (l->modification_of_pic_nums_idc == 3)
1797             break;
1798
1799         /* 8.2.4.3.1 - Short-term reference pictures */
1800         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1801             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1802             gint32 picNum, picNumNoWrap;
1803
1804             // (8-34)
1805             if (l->modification_of_pic_nums_idc == 0) {
1806                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1807                 if (picNumNoWrap < 0)
1808                     picNumNoWrap += MaxPicNum;
1809             }
1810
1811             // (8-35)
1812             else {
1813                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1814                 if (picNumNoWrap >= MaxPicNum)
1815                     picNumNoWrap -= MaxPicNum;
1816             }
1817             picNumPred = picNumNoWrap;
1818
1819             // (8-36)
1820             picNum = picNumNoWrap;
1821             if (picNum > CurrPicNum)
1822                 picNum -= MaxPicNum;
1823
1824             // (8-37)
1825             for (j = num_refs; j > ref_list_idx; j--)
1826                 ref_list[j] = ref_list[j - 1];
1827             found_ref_idx = find_short_term_reference(decoder, picNum);
1828             ref_list[ref_list_idx++] =
1829                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1830             n = ref_list_idx;
1831             for (j = ref_list_idx; j <= num_refs; j++) {
1832                 gint32 PicNumF;
1833                 if (!ref_list[j])
1834                     continue;
1835                 PicNumF =
1836                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1837                     ref_list[j]->pic_num : MaxPicNum;
1838                 if (PicNumF != picNum)
1839                     ref_list[n++] = ref_list[j];
1840             }
1841         }
1842
1843         /* 8.2.4.3.2 - Long-term reference pictures */
1844         else {
1845
1846             for (j = num_refs; j > ref_list_idx; j--)
1847                 ref_list[j] = ref_list[j - 1];
1848             found_ref_idx =
1849                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1850             ref_list[ref_list_idx++] =
1851                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1852             n = ref_list_idx;
1853             for (j = ref_list_idx; j <= num_refs; j++) {
1854                 gint32 LongTermPicNumF;
1855                 if (!ref_list[j])
1856                     continue;
1857                 LongTermPicNumF =
1858                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1859                     ref_list[j]->long_term_pic_num : INT_MAX;
1860                 if (LongTermPicNumF != l->value.long_term_pic_num)
1861                     ref_list[n++] = ref_list[j];
1862             }
1863         }
1864     }
1865
1866 #if DEBUG
1867     for (i = 0; i < num_refs; i++)
1868         if (!ref_list[i])
1869             GST_ERROR("list %u entry %u is empty", list, i);
1870 #endif
1871     *ref_list_count_ptr = num_refs;
1872 }
1873
1874 /* 8.2.4.3 - Modification process for reference picture lists */
1875 static void
1876 exec_picture_refs_modification(
1877     GstVaapiDecoderH264 *decoder,
1878     GstVaapiPictureH264 *picture,
1879     GstH264SliceHdr     *slice_hdr
1880 )
1881 {
1882     GST_DEBUG("execute ref_pic_list_modification()");
1883
1884     /* RefPicList0 */
1885     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1886         slice_hdr->ref_pic_list_modification_flag_l0)
1887         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1888
1889     /* RefPicList1 */
1890     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1891         slice_hdr->ref_pic_list_modification_flag_l1)
1892         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1893 }
1894
1895 static void
1896 init_picture_ref_lists(GstVaapiDecoderH264 *decoder)
1897 {
1898     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1899     guint i, j, short_ref_count, long_ref_count;
1900
1901     short_ref_count = 0;
1902     long_ref_count  = 0;
1903     if (GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture)) {
1904         for (i = 0; i < priv->dpb_count; i++) {
1905             GstVaapiFrameStore * const fs = priv->dpb[i];
1906             GstVaapiPictureH264 *picture;
1907             if (!gst_vaapi_frame_store_has_frame(fs))
1908                 continue;
1909             picture = fs->buffers[0];
1910             if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1911                 priv->short_ref[short_ref_count++] = picture;
1912             else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1913                 priv->long_ref[long_ref_count++] = picture;
1914             picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1915             picture->other_field = fs->buffers[1];
1916         }
1917     }
1918     else {
1919         for (i = 0; i < priv->dpb_count; i++) {
1920             GstVaapiFrameStore * const fs = priv->dpb[i];
1921             for (j = 0; j < fs->num_buffers; j++) {
1922                 GstVaapiPictureH264 * const picture = fs->buffers[j];
1923                 if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
1924                     priv->short_ref[short_ref_count++] = picture;
1925                 else if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
1926                     priv->long_ref[long_ref_count++] = picture;
1927                 picture->structure = picture->base.structure;
1928                 picture->other_field = fs->buffers[j ^ 1];
1929             }
1930         }
1931     }
1932
1933     for (i = short_ref_count; i < priv->short_ref_count; i++)
1934         priv->short_ref[i] = NULL;
1935     priv->short_ref_count = short_ref_count;
1936
1937     for (i = long_ref_count; i < priv->long_ref_count; i++)
1938         priv->long_ref[i] = NULL;
1939     priv->long_ref_count = long_ref_count;
1940 }
1941
1942 static void
1943 init_picture_refs(
1944     GstVaapiDecoderH264 *decoder,
1945     GstVaapiPictureH264 *picture,
1946     GstH264SliceHdr     *slice_hdr
1947 )
1948 {
1949     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1950     GstVaapiPicture * const base_picture = &picture->base;
1951     guint i, num_refs;
1952
1953     init_picture_ref_lists(decoder);
1954     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1955
1956     priv->RefPicList0_count = 0;
1957     priv->RefPicList1_count = 0;
1958
1959     switch (base_picture->type) {
1960     case GST_VAAPI_PICTURE_TYPE_P:
1961     case GST_VAAPI_PICTURE_TYPE_SP:
1962         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1963         break;
1964     case GST_VAAPI_PICTURE_TYPE_B:
1965         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1966         break;
1967     default:
1968         break;
1969     }
1970
1971     exec_picture_refs_modification(decoder, picture, slice_hdr);
1972
1973     switch (base_picture->type) {
1974     case GST_VAAPI_PICTURE_TYPE_B:
1975         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1976         for (i = priv->RefPicList1_count; i < num_refs; i++)
1977             priv->RefPicList1[i] = NULL;
1978         priv->RefPicList1_count = num_refs;
1979
1980         // fall-through
1981     case GST_VAAPI_PICTURE_TYPE_P:
1982     case GST_VAAPI_PICTURE_TYPE_SP:
1983         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1984         for (i = priv->RefPicList0_count; i < num_refs; i++)
1985             priv->RefPicList0[i] = NULL;
1986         priv->RefPicList0_count = num_refs;
1987         break;
1988     default:
1989         break;
1990     }
1991 }
1992
1993 static gboolean
1994 init_picture(
1995     GstVaapiDecoderH264 *decoder,
1996     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
1997 {
1998     GstVaapiDecoderH264Private * const priv = &decoder->priv;
1999     GstVaapiPicture * const base_picture = &picture->base;
2000     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2001
2002     priv->prev_frame_num        = priv->frame_num;
2003     priv->frame_num             = slice_hdr->frame_num;
2004     picture->frame_num          = priv->frame_num;
2005     picture->frame_num_wrap     = priv->frame_num;
2006     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
2007     base_picture->pts           = GST_VAAPI_DECODER_CODEC_FRAME(decoder)->pts;
2008
2009     /* Reset decoder state for IDR pictures */
2010     if (pi->nalu.type == GST_H264_NAL_SLICE_IDR) {
2011         GST_DEBUG("<IDR>");
2012         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
2013         dpb_flush(decoder);
2014     }
2015
2016     /* Initialize slice type */
2017     switch (slice_hdr->type % 5) {
2018     case GST_H264_P_SLICE:
2019         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
2020         break;
2021     case GST_H264_B_SLICE:
2022         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
2023         break;
2024     case GST_H264_I_SLICE:
2025         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
2026         break;
2027     case GST_H264_SP_SLICE:
2028         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
2029         break;
2030     case GST_H264_SI_SLICE:
2031         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
2032         break;
2033     }
2034
2035     /* Initialize picture structure */
2036     if (!slice_hdr->field_pic_flag)
2037         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
2038     else {
2039         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_INTERLACED);
2040         if (!slice_hdr->bottom_field_flag)
2041             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
2042         else
2043             base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
2044     }
2045     picture->structure = base_picture->structure;
2046
2047     /* Initialize reference flags */
2048     if (pi->nalu.ref_idc) {
2049         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2050             &slice_hdr->dec_ref_pic_marking;
2051
2052         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
2053             dec_ref_pic_marking->long_term_reference_flag)
2054             GST_VAAPI_PICTURE_FLAG_SET(picture,
2055                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
2056         else
2057             GST_VAAPI_PICTURE_FLAG_SET(picture,
2058                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
2059     }
2060
2061     init_picture_poc(decoder, picture, slice_hdr);
2062     init_picture_refs(decoder, picture, slice_hdr);
2063     return TRUE;
2064 }
2065
2066 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
2067 static gboolean
2068 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
2069 {
2070     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2071     GstH264PPS * const pps = priv->current_picture->pps;
2072     GstH264SPS * const sps = pps->sequence;
2073     GstVaapiPictureH264 *ref_picture;
2074     guint i, m, max_num_ref_frames;
2075
2076     GST_DEBUG("reference picture marking process (sliding window)");
2077
2078     if (!GST_VAAPI_PICTURE_IS_FIRST_FIELD(priv->current_picture))
2079         return TRUE;
2080
2081     max_num_ref_frames = sps->num_ref_frames;
2082     if (max_num_ref_frames == 0)
2083         max_num_ref_frames = 1;
2084     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture))
2085         max_num_ref_frames <<= 1;
2086
2087     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
2088         return TRUE;
2089     if (priv->short_ref_count < 1)
2090         return FALSE;
2091
2092     for (m = 0, i = 1; i < priv->short_ref_count; i++) {
2093         GstVaapiPictureH264 * const picture = priv->short_ref[i];
2094         if (picture->frame_num_wrap < priv->short_ref[m]->frame_num_wrap)
2095             m = i;
2096     }
2097
2098     ref_picture = priv->short_ref[m];
2099     gst_vaapi_picture_h264_set_reference(ref_picture, 0, TRUE);
2100     ARRAY_REMOVE_INDEX(priv->short_ref, m);
2101
2102     /* Both fields need to be marked as "unused for reference", so
2103        remove the other field from the short_ref[] list as well */
2104     if (!GST_VAAPI_PICTURE_IS_FRAME(priv->current_picture) && ref_picture->other_field) {
2105         for (i = 0; i < priv->short_ref_count; i++) {
2106             if (priv->short_ref[i] == ref_picture->other_field) {
2107                 ARRAY_REMOVE_INDEX(priv->short_ref, i);
2108                 break;
2109             }
2110         }
2111     }
2112     return TRUE;
2113 }
2114
2115 static inline gint32
2116 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
2117 {
2118     gint32 pic_num;
2119
2120     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
2121         pic_num = picture->frame_num_wrap;
2122     else
2123         pic_num = 2 * picture->frame_num_wrap + 1;
2124     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
2125     return pic_num;
2126 }
2127
2128 /* 8.2.5.4.1. Mark short-term reference picture as "unused for reference" */
2129 static void
2130 exec_ref_pic_marking_adaptive_mmco_1(
2131     GstVaapiDecoderH264  *decoder,
2132     GstVaapiPictureH264  *picture,
2133     GstH264RefPicMarking *ref_pic_marking
2134 )
2135 {
2136     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2137     gint32 i, picNumX;
2138
2139     picNumX = get_picNumX(picture, ref_pic_marking);
2140     i = find_short_term_reference(decoder, picNumX);
2141     if (i < 0)
2142         return;
2143
2144     gst_vaapi_picture_h264_set_reference(priv->short_ref[i], 0,
2145         GST_VAAPI_PICTURE_IS_FRAME(picture));
2146     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2147 }
2148
2149 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
2150 static void
2151 exec_ref_pic_marking_adaptive_mmco_2(
2152     GstVaapiDecoderH264  *decoder,
2153     GstVaapiPictureH264  *picture,
2154     GstH264RefPicMarking *ref_pic_marking
2155 )
2156 {
2157     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2158     gint32 i;
2159
2160     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
2161     if (i < 0)
2162         return;
2163
2164     gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0,
2165         GST_VAAPI_PICTURE_IS_FRAME(picture));
2166     ARRAY_REMOVE_INDEX(priv->long_ref, i);
2167 }
2168
2169 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
2170 static void
2171 exec_ref_pic_marking_adaptive_mmco_3(
2172     GstVaapiDecoderH264  *decoder,
2173     GstVaapiPictureH264  *picture,
2174     GstH264RefPicMarking *ref_pic_marking
2175 )
2176 {
2177     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2178     GstVaapiPictureH264 *ref_picture;
2179     gint32 i, picNumX;
2180
2181     for (i = 0; i < priv->long_ref_count; i++) {
2182         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
2183             break;
2184     }
2185     if (i != priv->long_ref_count) {
2186         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, TRUE);
2187         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2188     }
2189
2190     picNumX = get_picNumX(picture, ref_pic_marking);
2191     i = find_short_term_reference(decoder, picNumX);
2192     if (i < 0)
2193         return;
2194
2195     ref_picture = priv->short_ref[i];
2196     ARRAY_REMOVE_INDEX(priv->short_ref, i);
2197     priv->long_ref[priv->long_ref_count++] = ref_picture;
2198
2199     ref_picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2200     gst_vaapi_picture_h264_set_reference(ref_picture,
2201         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE,
2202         GST_VAAPI_PICTURE_IS_FRAME(picture));
2203 }
2204
2205 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
2206  * as "unused for reference" */
2207 static void
2208 exec_ref_pic_marking_adaptive_mmco_4(
2209     GstVaapiDecoderH264  *decoder,
2210     GstVaapiPictureH264  *picture,
2211     GstH264RefPicMarking *ref_pic_marking
2212 )
2213 {
2214     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2215     gint32 i, long_term_frame_idx;
2216
2217     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
2218
2219     for (i = 0; i < priv->long_ref_count; i++) {
2220         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
2221             continue;
2222         gst_vaapi_picture_h264_set_reference(priv->long_ref[i], 0, FALSE);
2223         ARRAY_REMOVE_INDEX(priv->long_ref, i);
2224         i--;
2225     }
2226 }
2227
2228 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
2229 static void
2230 exec_ref_pic_marking_adaptive_mmco_5(
2231     GstVaapiDecoderH264  *decoder,
2232     GstVaapiPictureH264  *picture,
2233     GstH264RefPicMarking *ref_pic_marking
2234 )
2235 {
2236     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2237
2238     dpb_flush(decoder);
2239
2240     priv->prev_pic_has_mmco5 = TRUE;
2241
2242     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
2243     priv->frame_num = 0;
2244     priv->frame_num_offset = 0;
2245     picture->frame_num = 0;
2246
2247     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
2248     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
2249         picture->field_poc[TOP_FIELD] -= picture->base.poc;
2250     if (picture->structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
2251         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
2252     picture->base.poc = 0;
2253 }
2254
2255 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
2256 static void
2257 exec_ref_pic_marking_adaptive_mmco_6(
2258     GstVaapiDecoderH264  *decoder,
2259     GstVaapiPictureH264  *picture,
2260     GstH264RefPicMarking *ref_pic_marking
2261 )
2262 {
2263     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
2264     gst_vaapi_picture_h264_set_reference(picture,
2265         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE, FALSE);
2266 }
2267
2268 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
2269 static gboolean
2270 exec_ref_pic_marking_adaptive(
2271     GstVaapiDecoderH264     *decoder,
2272     GstVaapiPictureH264     *picture,
2273     GstH264DecRefPicMarking *dec_ref_pic_marking
2274 )
2275 {
2276     guint i;
2277
2278     GST_DEBUG("reference picture marking process (adaptive memory control)");
2279
2280     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
2281         GstVaapiDecoderH264  *decoder,
2282         GstVaapiPictureH264  *picture,
2283         GstH264RefPicMarking *ref_pic_marking
2284     );
2285
2286     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
2287         NULL,
2288         exec_ref_pic_marking_adaptive_mmco_1,
2289         exec_ref_pic_marking_adaptive_mmco_2,
2290         exec_ref_pic_marking_adaptive_mmco_3,
2291         exec_ref_pic_marking_adaptive_mmco_4,
2292         exec_ref_pic_marking_adaptive_mmco_5,
2293         exec_ref_pic_marking_adaptive_mmco_6,
2294     };
2295
2296     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
2297         GstH264RefPicMarking * const ref_pic_marking =
2298             &dec_ref_pic_marking->ref_pic_marking[i];
2299
2300         const guint mmco = ref_pic_marking->memory_management_control_operation;
2301         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
2302             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
2303         else {
2304             GST_ERROR("unhandled MMCO %u", mmco);
2305             return FALSE;
2306         }
2307     }
2308     return TRUE;
2309 }
2310
2311 /* 8.2.5 - Execute reference picture marking process */
2312 static gboolean
2313 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2314 {
2315     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2316
2317     priv->prev_pic_has_mmco5 = FALSE;
2318     priv->prev_pic_structure = picture->structure;
2319
2320     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2321         return TRUE;
2322
2323     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2324         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2325             &picture->last_slice_hdr->dec_ref_pic_marking;
2326         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2327             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2328                 return FALSE;
2329         }
2330         else {
2331             if (!exec_ref_pic_marking_sliding_window(decoder))
2332                 return FALSE;
2333         }
2334     }
2335     return TRUE;
2336 }
2337
2338 static void
2339 vaapi_init_picture(VAPictureH264 *pic)
2340 {
2341     pic->picture_id           = VA_INVALID_ID;
2342     pic->frame_idx            = 0;
2343     pic->flags                = VA_PICTURE_H264_INVALID;
2344     pic->TopFieldOrderCnt     = 0;
2345     pic->BottomFieldOrderCnt  = 0;
2346 }
2347
2348 static void
2349 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture,
2350     guint picture_structure)
2351 {
2352     if (!picture_structure)
2353         picture_structure = picture->structure;
2354
2355     pic->picture_id = picture->base.surface_id;
2356     pic->flags = 0;
2357
2358     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2359         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2360         pic->frame_idx = picture->long_term_frame_idx;
2361     }
2362     else {
2363         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2364             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2365         pic->frame_idx = picture->frame_num;
2366     }
2367
2368     switch (picture_structure) {
2369     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2370         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2371         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2372         break;
2373     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2374         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2375         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2376         pic->BottomFieldOrderCnt = 0;
2377         break;
2378     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2379         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2380         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2381         pic->TopFieldOrderCnt = 0;
2382         break;
2383     }
2384 }
2385
2386 static gboolean
2387 fill_picture(GstVaapiDecoderH264 *decoder,
2388     GstVaapiPictureH264 *picture, GstVaapiParserInfoH264 *pi)
2389 {
2390     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2391     GstVaapiPicture * const base_picture = &picture->base;
2392     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2393     GstH264PPS * const pps = picture->pps;
2394     GstH264SPS * const sps = pps->sequence;
2395     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2396     guint i, n;
2397
2398     /* Fill in VAPictureParameterBufferH264 */
2399     vaapi_fill_picture(&pic_param->CurrPic, picture, 0);
2400
2401     for (i = 0, n = 0; i < priv->dpb_count; i++) {
2402         GstVaapiFrameStore * const fs = priv->dpb[i];
2403         if (gst_vaapi_frame_store_has_reference(fs))
2404             vaapi_fill_picture(&pic_param->ReferenceFrames[n++],
2405                 fs->buffers[0], fs->structure);
2406     }
2407     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2408         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2409
2410 #define COPY_FIELD(s, f) \
2411     pic_param->f = (s)->f
2412
2413 #define COPY_BFM(a, s, f) \
2414     pic_param->a.bits.f = (s)->f
2415
2416     pic_param->picture_width_in_mbs_minus1  = priv->mb_width - 1;
2417     pic_param->picture_height_in_mbs_minus1 = priv->mb_height - 1;
2418     pic_param->frame_num                    = priv->frame_num;
2419
2420     COPY_FIELD(sps, bit_depth_luma_minus8);
2421     COPY_FIELD(sps, bit_depth_chroma_minus8);
2422     COPY_FIELD(sps, num_ref_frames);
2423     COPY_FIELD(pps, num_slice_groups_minus1);
2424     COPY_FIELD(pps, slice_group_map_type);
2425     COPY_FIELD(pps, slice_group_change_rate_minus1);
2426     COPY_FIELD(pps, pic_init_qp_minus26);
2427     COPY_FIELD(pps, pic_init_qs_minus26);
2428     COPY_FIELD(pps, chroma_qp_index_offset);
2429     COPY_FIELD(pps, second_chroma_qp_index_offset);
2430
2431     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2432     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2433     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2434
2435     COPY_BFM(seq_fields, sps, chroma_format_idc);
2436     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2437     COPY_BFM(seq_fields, sps, frame_mbs_only_flag); 
2438     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag); 
2439     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag); 
2440     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2441     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2442     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2443     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2444
2445     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2446     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2447     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2448
2449     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2450     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2451     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2452     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2453     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2454     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2455     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2456     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2457     return TRUE;
2458 }
2459
2460 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2461 static gboolean
2462 is_new_picture(GstVaapiParserInfoH264 *pi, GstVaapiParserInfoH264 *prev_pi)
2463 {
2464     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2465     GstH264PPS * const pps = slice_hdr->pps;
2466     GstH264SPS * const sps = pps->sequence;
2467     GstH264SliceHdr *prev_slice_hdr;
2468
2469     if (!prev_pi)
2470         return TRUE;
2471     prev_slice_hdr = &prev_pi->data.slice_hdr;
2472
2473 #define CHECK_EXPR(expr, field_name) do {              \
2474         if (!(expr)) {                                 \
2475             GST_DEBUG(field_name " differs in value"); \
2476             return TRUE;                               \
2477         }                                              \
2478     } while (0)
2479
2480 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2481     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2482
2483     /* frame_num differs in value, regardless of inferred values to 0 */
2484     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2485
2486     /* pic_parameter_set_id differs in value */
2487     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2488
2489     /* field_pic_flag differs in value */
2490     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2491
2492     /* bottom_field_flag is present in both and differs in value */
2493     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2494         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2495
2496     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2497     CHECK_EXPR((pi->nalu.ref_idc != 0) ==
2498                (prev_pi->nalu.ref_idc != 0), "nal_ref_idc");
2499
2500     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2501        value or delta_pic_order_cnt_bottom differs in value */
2502     if (sps->pic_order_cnt_type == 0) {
2503         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2504         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2505             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2506     }
2507
2508     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2509        differs in value or delta_pic_order_cnt[1] differs in value */
2510     else if (sps->pic_order_cnt_type == 1) {
2511         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2512         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2513     }
2514
2515     /* IdrPicFlag differs in value */
2516     CHECK_VALUE(&pi->nalu, &prev_pi->nalu, idr_pic_flag);
2517
2518     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2519     if (pi->nalu.idr_pic_flag)
2520         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2521
2522 #undef CHECK_EXPR
2523 #undef CHECK_VALUE
2524     return FALSE;
2525 }
2526
2527 static GstVaapiDecoderStatus
2528 decode_picture(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
2529 {
2530     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2531     GstVaapiParserInfoH264 * const pi = unit->parsed_info;
2532     GstH264SliceHdr * const slice_hdr = &pi->data.slice_hdr;
2533     GstH264PPS * const pps = slice_hdr->pps;
2534     GstH264SPS * const sps = pps->sequence;
2535     GstVaapiPictureH264 *picture;
2536     GstVaapiDecoderStatus status;
2537
2538     status = ensure_context(decoder, sps);
2539     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2540         return status;
2541
2542     priv->decoder_state = 0;
2543
2544     if (priv->current_picture) {
2545         /* Re-use current picture where the first field was decoded */
2546         picture = gst_vaapi_picture_h264_new_field(priv->current_picture);
2547         if (!picture) {
2548             GST_ERROR("failed to allocate field picture");
2549             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2550         }
2551     }
2552     else {
2553         /* Create new picture */
2554         picture = gst_vaapi_picture_h264_new(decoder);
2555         if (!picture) {
2556             GST_ERROR("failed to allocate picture");
2557             return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2558         }
2559     }
2560     gst_vaapi_picture_replace(&priv->current_picture, picture);
2561     gst_vaapi_picture_unref(picture);
2562
2563     /* Update cropping rectangle */
2564     if (sps->frame_cropping_flag) {
2565         GstVaapiRectangle crop_rect;
2566         crop_rect.x = sps->crop_rect_x;
2567         crop_rect.y = sps->crop_rect_y;
2568         crop_rect.width = sps->crop_rect_width;
2569         crop_rect.height = sps->crop_rect_height;
2570         gst_vaapi_picture_set_crop_rect(&picture->base, &crop_rect);
2571     }
2572
2573     picture->pps = pps;
2574
2575     status = ensure_quant_matrix(decoder, picture);
2576     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2577         GST_ERROR("failed to reset quantizer matrix");
2578         return status;
2579     }
2580
2581     if (!init_picture(decoder, picture, pi))
2582         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2583     if (!fill_picture(decoder, picture, pi))
2584         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2585
2586     priv->decoder_state = pi->state;
2587     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2588 }
2589
2590 static inline guint
2591 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr)
2592 {
2593     guint epb_count;
2594
2595     epb_count = slice_hdr->n_emulation_prevention_bytes;
2596     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2597 }
2598
2599 static gboolean
2600 fill_pred_weight_table(GstVaapiDecoderH264 *decoder,
2601     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2602 {
2603     VASliceParameterBufferH264 * const slice_param = slice->param;
2604     GstH264PPS * const pps = slice_hdr->pps;
2605     GstH264SPS * const sps = pps->sequence;
2606     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2607     guint num_weight_tables = 0;
2608     gint i, j;
2609
2610     if (pps->weighted_pred_flag &&
2611         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2612         num_weight_tables = 1;
2613     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2614         num_weight_tables = 2;
2615     else
2616         num_weight_tables = 0;
2617
2618     slice_param->luma_log2_weight_denom   = 0;
2619     slice_param->chroma_log2_weight_denom = 0;
2620     slice_param->luma_weight_l0_flag      = 0;
2621     slice_param->chroma_weight_l0_flag    = 0;
2622     slice_param->luma_weight_l1_flag      = 0;
2623     slice_param->chroma_weight_l1_flag    = 0;
2624
2625     if (num_weight_tables < 1)
2626         return TRUE;
2627
2628     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2629     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2630
2631     slice_param->luma_weight_l0_flag = 1;
2632     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2633         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2634         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2635     }
2636
2637     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2638     if (slice_param->chroma_weight_l0_flag) {
2639         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2640             for (j = 0; j < 2; j++) {
2641                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2642                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2643             }
2644         }
2645     }
2646
2647     if (num_weight_tables < 2)
2648         return TRUE;
2649
2650     slice_param->luma_weight_l1_flag = 1;
2651     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2652         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2653         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2654     }
2655
2656     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2657     if (slice_param->chroma_weight_l1_flag) {
2658         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2659             for (j = 0; j < 2; j++) {
2660                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2661                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2662             }
2663         }
2664     }
2665     return TRUE;
2666 }
2667
2668 static gboolean
2669 fill_RefPicList(GstVaapiDecoderH264 *decoder,
2670     GstVaapiSlice *slice, GstH264SliceHdr *slice_hdr)
2671 {
2672     GstVaapiDecoderH264Private * const priv = &decoder->priv;
2673     VASliceParameterBufferH264 * const slice_param = slice->param;
2674     guint i, num_ref_lists = 0;
2675
2676     slice_param->num_ref_idx_l0_active_minus1 = 0;
2677     slice_param->num_ref_idx_l1_active_minus1 = 0;
2678
2679     if (GST_H264_IS_B_SLICE(slice_hdr))
2680         num_ref_lists = 2;
2681     else if (GST_H264_IS_I_SLICE(slice_hdr))
2682         num_ref_lists = 0;
2683     else
2684         num_ref_lists = 1;
2685
2686     if (num_ref_lists < 1)
2687         return TRUE;
2688
2689     slice_param->num_ref_idx_l0_active_minus1 =
2690         slice_hdr->num_ref_idx_l0_active_minus1;
2691
2692     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2693         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i], 0);
2694     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2695         vaapi_init_picture(&slice_param->RefPicList0[i]);
2696
2697     if (num_ref_lists < 2)
2698         return TRUE;
2699
2700     slice_param->num_ref_idx_l1_active_minus1 =
2701         slice_hdr->num_ref_idx_l1_active_minus1;
2702
2703     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2704         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i], 0);
2705     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2706         vaapi_init_picture(&slice_param->RefPicList1[i]);
2707     return TRUE;
2708</