h264: introduce GST_VAAPI_PICTURE_FLAG_IDR flag.
[vaapi:windyuan-gstreamer-vaapi.git] / gst-libs / gst / vaapi / gstvaapidecoder_h264.c
1 /*
2  *  gstvaapidecoder_h264.c - H.264 decoder
3  *
4  *  Copyright (C) 2011-2012 Intel Corporation
5  *
6  *  This library is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU Lesser General Public License
8  *  as published by the Free Software Foundation; either version 2.1
9  *  of the License, or (at your option) any later version.
10  *
11  *  This library is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  *  Lesser General Public License for more details.
15  *
16  *  You should have received a copy of the GNU Lesser General Public
17  *  License along with this library; if not, write to the Free
18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19  *  Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * SECTION:gstvaapidecoder_h264
24  * @short_description: H.264 decoder
25  */
26
27 #include "sysdeps.h"
28 #include <string.h>
29 #include <gst/base/gstadapter.h>
30 #include <gst/codecparsers/gsth264parser.h>
31 #include "gstvaapidecoder_h264.h"
32 #include "gstvaapidecoder_objects.h"
33 #include "gstvaapidecoder_priv.h"
34 #include "gstvaapidisplay_priv.h"
35 #include "gstvaapiobject_priv.h"
36
37 #define DEBUG 1
38 #include "gstvaapidebug.h"
39
40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
41 #define USE_STRICT_DPB_ORDERING 0
42
43 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
44 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
45 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
46 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
47
48 /* ------------------------------------------------------------------------- */
49 /* --- H.264 Pictures                                                    --- */
50 /* ------------------------------------------------------------------------- */
51
52 #define GST_VAAPI_TYPE_PICTURE_H264 \
53     (gst_vaapi_picture_h264_get_type())
54
55 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
56     ((GstVaapiPictureH264 *)(obj))
57
58 #define GST_VAAPI_PICTURE_H264(obj)                             \
59     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
60                                 GST_VAAPI_TYPE_PICTURE_H264,    \
61                                 GstVaapiPictureH264))
62
63 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
64     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
65                              GST_VAAPI_TYPE_PICTURE_H264,       \
66                              GstVaapiPictureH264Class))
67
68 #define GST_VAAPI_IS_PICTURE_H264(obj) \
69     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
70
71 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
72     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
73
74 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
75     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
76                                GST_VAAPI_TYPE_PICTURE_H264,     \
77                                GstVaapiPictureH264Class))
78
79 /*
80  * Extended picture flags:
81  *
82  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
83  */
84 enum {
85     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
86 };
87
88 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
89     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
90
91 struct _GstVaapiPictureH264 {
92     GstVaapiPicture             base;
93     VAPictureH264               info;
94     GstH264PPS                 *pps;
95     gint32                      poc;
96     gint32                      frame_num;              // Original frame_num from slice_header()
97     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
98     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
99     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
100     guint                       is_long_term            : 1;
101     guint                       output_flag             : 1;
102     guint                       output_needed           : 1;
103 };
104
105 struct _GstVaapiPictureH264Class {
106     /*< private >*/
107     GstVaapiPictureClass        parent_class;
108 };
109
110 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
111                             gst_vaapi_picture_h264,
112                             GST_VAAPI_TYPE_PICTURE)
113
114 static void
115 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
116 {
117 }
118
119 static gboolean
120 gst_vaapi_picture_h264_create(
121     GstVaapiPictureH264                      *picture,
122     const GstVaapiCodecObjectConstructorArgs *args
123 )
124 {
125     return TRUE;
126 }
127
128 static void
129 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
130 {
131     VAPictureH264 *va_pic;
132
133     va_pic                      = &picture->info;
134     va_pic->flags               = 0;
135     va_pic->TopFieldOrderCnt    = 0;
136     va_pic->BottomFieldOrderCnt = 0;
137
138     picture->poc                = 0;
139     picture->is_long_term       = FALSE;
140     picture->output_needed      = FALSE;
141 }
142
143 static inline GstVaapiPictureH264 *
144 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
145 {
146     GstVaapiCodecObject *object;
147
148     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
149
150     object = gst_vaapi_codec_object_new(
151         GST_VAAPI_TYPE_PICTURE_H264,
152         GST_VAAPI_CODEC_BASE(decoder),
153         NULL, sizeof(VAPictureParameterBufferH264),
154         NULL, 0
155     );
156     if (!object)
157         return NULL;
158     return GST_VAAPI_PICTURE_H264_CAST(object);
159 }
160
161 static inline GstVaapiSliceH264 *
162 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
163 {
164     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
165
166     if (G_UNLIKELY(picture->base.slices->len < 1))
167         return NULL;
168     return g_ptr_array_index(picture->base.slices,
169         picture->base.slices->len - 1);
170 }
171
172 /* ------------------------------------------------------------------------- */
173 /* --- Slices                                                            --- */
174 /* ------------------------------------------------------------------------- */
175
176 #define GST_VAAPI_TYPE_SLICE_H264 \
177     (gst_vaapi_slice_h264_get_type())
178
179 #define GST_VAAPI_SLICE_H264_CAST(obj) \
180     ((GstVaapiSliceH264 *)(obj))
181
182 #define GST_VAAPI_SLICE_H264(obj)                               \
183     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
184                                 GST_VAAPI_TYPE_SLICE_H264,      \
185                                 GstVaapiSliceH264))
186
187 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
188     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
189                              GST_VAAPI_TYPE_SLICE_H264,         \
190                              GstVaapiSliceH264Class))
191
192 #define GST_VAAPI_IS_SLICE_H264(obj) \
193     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
194
195 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
196     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
197
198 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
199     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
200                                GST_VAAPI_TYPE_SLICE_H264,       \
201                                GstVaapiSliceH264Class))
202
203 struct _GstVaapiSliceH264 {
204     GstVaapiSlice               base;
205     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
206 };
207
208 struct _GstVaapiSliceH264Class {
209     /*< private >*/
210     GstVaapiSliceClass          parent_class;
211 };
212
213 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
214                             gst_vaapi_slice_h264,
215                             GST_VAAPI_TYPE_SLICE)
216
217 static void
218 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
219 {
220 }
221
222 static gboolean
223 gst_vaapi_slice_h264_create(
224     GstVaapiSliceH264                        *slice,
225     const GstVaapiCodecObjectConstructorArgs *args
226 )
227 {
228     return TRUE;
229 }
230
231 static void
232 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
233 {
234 }
235
236 static inline GstVaapiSliceH264 *
237 gst_vaapi_slice_h264_new(
238     GstVaapiDecoderH264 *decoder,
239     const guint8        *data,
240     guint                data_size
241 )
242 {
243     GstVaapiCodecObject *object;
244
245     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
246
247     object = gst_vaapi_codec_object_new(
248         GST_VAAPI_TYPE_SLICE_H264,
249         GST_VAAPI_CODEC_BASE(decoder),
250         NULL, sizeof(VASliceParameterBufferH264),
251         data, data_size
252     );
253     if (!object)
254         return NULL;
255     return GST_VAAPI_SLICE_H264_CAST(object);
256 }
257
258 /* ------------------------------------------------------------------------- */
259 /* --- H.264 Decoder                                                     --- */
260 /* ------------------------------------------------------------------------- */
261
262 G_DEFINE_TYPE(GstVaapiDecoderH264,
263               gst_vaapi_decoder_h264,
264               GST_VAAPI_TYPE_DECODER)
265
266 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
267     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
268                                  GST_VAAPI_TYPE_DECODER_H264,   \
269                                  GstVaapiDecoderH264Private))
270
271 // Used for field_poc[]
272 #define TOP_FIELD       0
273 #define BOTTOM_FIELD    1
274
275 struct _GstVaapiDecoderH264Private {
276     GstAdapter                 *adapter;
277     GstH264NalParser           *parser;
278     /* Last decoded SPS. May not be the last activated one. Just here because
279        it may not fit stack memory allocation in decode_sps() */
280     GstH264SPS                  last_sps;
281     /* Last decoded PPS. May not be the last activated one. Just here because
282        it may not fit stack memory allocation in decode_pps() */
283     GstH264PPS                  last_pps;
284     GstVaapiPictureH264        *current_picture;
285     GstVaapiPictureH264        *dpb[16];
286     guint                       dpb_count;
287     guint                       dpb_size;
288     GstVaapiProfile             profile;
289     GstVaapiEntrypoint          entrypoint;
290     GstVaapiChromaType          chroma_type;
291     GstVaapiPictureH264        *short_ref[32];
292     guint                       short_ref_count;
293     GstVaapiPictureH264        *long_ref[32];
294     guint                       long_ref_count;
295     GstVaapiPictureH264        *RefPicList0[32];
296     guint                       RefPicList0_count;
297     GstVaapiPictureH264        *RefPicList1[32];
298     guint                       RefPicList1_count;
299     guint                       nal_length_size;
300     guint                       width;
301     guint                       height;
302     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
303     gint32                      poc_msb;                // PicOrderCntMsb
304     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
305     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
306     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
307     gint32                      frame_num_offset;       // FrameNumOffset
308     gint32                      frame_num;              // frame_num (from slice_header())
309     gint32                      prev_frame_num;         // prevFrameNum
310     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
311     gboolean                    prev_pic_structure;     // previous picture structure
312     guint                       is_constructed          : 1;
313     guint                       is_opened               : 1;
314     guint                       is_avc                  : 1;
315     guint                       has_context             : 1;
316 };
317
318 static gboolean
319 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
320
321 static void
322 clear_references(
323     GstVaapiDecoderH264  *decoder,
324     GstVaapiPictureH264 **pictures,
325     guint                *picture_count
326 );
327
328 /* Get number of reference frames to use */
329 static guint
330 get_max_dec_frame_buffering(GstH264SPS *sps)
331 {
332     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
333
334     /* Table A-1 - Level limits */
335     switch (sps->level_idc) {
336     case 10: MaxDpbMbs = 396;    break;
337     case 11: MaxDpbMbs = 900;    break;
338     case 12: MaxDpbMbs = 2376;   break;
339     case 13: MaxDpbMbs = 2376;   break;
340     case 20: MaxDpbMbs = 2376;   break;
341     case 21: MaxDpbMbs = 4752;   break;
342     case 22: MaxDpbMbs = 8100;   break;
343     case 30: MaxDpbMbs = 8100;   break;
344     case 31: MaxDpbMbs = 18000;  break;
345     case 32: MaxDpbMbs = 20480;  break;
346     case 40: MaxDpbMbs = 32768;  break;
347     case 41: MaxDpbMbs = 32768;  break;
348     case 42: MaxDpbMbs = 34816;  break;
349     case 50: MaxDpbMbs = 110400; break;
350     case 51: MaxDpbMbs = 184320; break;
351     default:
352         g_assert(0 && "unhandled level");
353         break;
354     }
355
356     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
357                   (sps->pic_height_in_map_units_minus1 + 1) *
358                   (sps->frame_mbs_only_flag ? 1 : 2));
359     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
360
361     /* VUI parameters */
362     if (sps->vui_parameters_present_flag) {
363         GstH264VUIParams * const vui_params = &sps->vui_parameters;
364         if (vui_params->bitstream_restriction_flag)
365             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
366         else {
367             switch (sps->profile_idc) {
368             case 44:  // CAVLC 4:4:4 Intra profile
369             case 86:  // Scalable High profile
370             case 100: // High profile
371             case 110: // High 10 profile
372             case 122: // High 4:2:2 profile
373             case 244: // High 4:4:4 Predictive profile
374                 if (sps->constraint_set3_flag)
375                     max_dec_frame_buffering = 0;
376                 break;
377             }
378         }
379     }
380
381     if (max_dec_frame_buffering > 16)
382         max_dec_frame_buffering = 16;
383     else if (max_dec_frame_buffering < sps->num_ref_frames)
384         max_dec_frame_buffering = sps->num_ref_frames;
385     return MAX(1, max_dec_frame_buffering);
386 }
387
388 static void
389 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
390 {
391     GstVaapiDecoderH264Private * const priv = decoder->priv;
392     guint i, num_pictures = --priv->dpb_count;
393
394     if (USE_STRICT_DPB_ORDERING) {
395         for (i = index; i < num_pictures; i++)
396             gst_vaapi_picture_replace(&priv->dpb[i], priv->dpb[i + 1]);
397     }
398     else if (index != num_pictures)
399         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
400     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
401 }
402
403 static inline gboolean
404 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
405 {
406     /* XXX: update cropping rectangle */
407     picture->output_needed = FALSE;
408     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
409 }
410
411 static gboolean
412 dpb_bump(GstVaapiDecoderH264 *decoder)
413 {
414     GstVaapiDecoderH264Private * const priv = decoder->priv;
415     guint i, lowest_poc_index;
416     gboolean success;
417
418     for (i = 0; i < priv->dpb_count; i++) {
419         if (priv->dpb[i]->output_needed)
420             break;
421     }
422     if (i == priv->dpb_count)
423         return FALSE;
424
425     lowest_poc_index = i++;
426     for (; i < priv->dpb_count; i++) {
427         GstVaapiPictureH264 * const picture = priv->dpb[i];
428         if (picture->output_needed && picture->poc < priv->dpb[lowest_poc_index]->poc)
429             lowest_poc_index = i;
430     }
431
432     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
433     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
434         dpb_remove_index(decoder, lowest_poc_index);
435     return success;
436 }
437
438 static void
439 dpb_flush(GstVaapiDecoderH264 *decoder)
440 {
441     GstVaapiDecoderH264Private * const priv = decoder->priv;
442
443     while (dpb_bump(decoder))
444         ;
445     clear_references(decoder, priv->dpb, &priv->dpb_count);
446 }
447
448 static gboolean
449 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
450 {
451     GstVaapiDecoderH264Private * const priv = decoder->priv;
452     guint i;
453
454     // Remove all unused pictures
455     if (GST_VAAPI_PICTURE_IS_IDR(picture))
456         dpb_flush(decoder);
457     else {
458         i = 0;
459         while (i < priv->dpb_count) {
460             GstVaapiPictureH264 * const picture = priv->dpb[i];
461             if (!picture->output_needed &&
462                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
463                 dpb_remove_index(decoder, i);
464             else
465                 i++;
466         }
467     }
468
469     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
470     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
471         while (priv->dpb_count == priv->dpb_size) {
472             if (!dpb_bump(decoder))
473                 return FALSE;
474         }
475         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
476         if (picture->output_flag)
477             picture->output_needed = TRUE;
478     }
479
480     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
481     else {
482         if (!picture->output_flag)
483             return TRUE;
484         while (priv->dpb_count == priv->dpb_size) {
485             for (i = 0; i < priv->dpb_count; i++) {
486                 if (priv->dpb[i]->output_needed &&
487                     priv->dpb[i]->poc < picture->poc)
488                     break;
489             }
490             if (i == priv->dpb_count)
491                 return dpb_output(decoder, picture);
492             if (!dpb_bump(decoder))
493                 return FALSE;
494         }
495         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
496         picture->output_needed = TRUE;
497     }
498     return TRUE;
499 }
500
501 static inline void
502 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
503 {
504     GstVaapiDecoderH264Private * const priv = decoder->priv;
505
506     priv->dpb_size = get_max_dec_frame_buffering(sps);
507     GST_DEBUG("DPB size %u", priv->dpb_size);
508 }
509
510 static GstVaapiDecoderStatus
511 get_status(GstH264ParserResult result)
512 {
513     GstVaapiDecoderStatus status;
514
515     switch (result) {
516     case GST_H264_PARSER_OK:
517         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
518         break;
519     case GST_H264_PARSER_NO_NAL_END:
520         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
521         break;
522     case GST_H264_PARSER_ERROR:
523         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
524         break;
525     default:
526         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
527         break;
528     }
529     return status;
530 }
531
532 static void
533 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
534 {
535     GstVaapiDecoderH264Private * const priv = decoder->priv;
536
537     gst_vaapi_picture_replace(&priv->current_picture, NULL);
538     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
539     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
540     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
541
542     if (priv->parser) {
543         gst_h264_nal_parser_free(priv->parser);
544         priv->parser = NULL;
545     }
546
547     if (priv->adapter) {
548         gst_adapter_clear(priv->adapter);
549         g_object_unref(priv->adapter);
550         priv->adapter = NULL;
551     }
552 }
553
554 static gboolean
555 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
556 {
557     GstVaapiDecoderH264Private * const priv = decoder->priv;
558
559     gst_vaapi_decoder_h264_close(decoder);
560
561     priv->adapter = gst_adapter_new();
562     if (!priv->adapter)
563         return FALSE;
564
565     priv->parser = gst_h264_nal_parser_new();
566     if (!priv->parser)
567         return FALSE;
568     return TRUE;
569 }
570
571 static void
572 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
573 {
574     gst_vaapi_decoder_h264_close(decoder);
575 }
576
577 static gboolean
578 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
579 {
580     if (!GST_VAAPI_DECODER_CODEC(decoder))
581         return FALSE;
582     return TRUE;
583 }
584
585 static guint
586 h264_get_profile(GstH264SPS *sps)
587 {
588     guint profile = 0;
589
590     switch (sps->profile_idc) {
591     case 66:
592         profile = GST_VAAPI_PROFILE_H264_BASELINE;
593         break;
594     case 77:
595         profile = GST_VAAPI_PROFILE_H264_MAIN;
596         break;
597     case 100:
598         profile = GST_VAAPI_PROFILE_H264_HIGH;
599         break;
600     }
601     return profile;
602 }
603
604 static guint
605 h264_get_chroma_type(GstH264SPS *sps)
606 {
607     guint chroma_type = 0;
608
609     switch (sps->chroma_format_idc) {
610     case 1:
611         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
612         break;
613     case 2:
614         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
615         break;
616     case 3:
617         if (!sps->separate_colour_plane_flag)
618             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
619         break;
620     }
621     return chroma_type;
622 }
623
624 static GstVaapiProfile
625 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
626 {
627     GstVaapiDecoderH264Private * const priv = decoder->priv;
628     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
629     GstVaapiProfile profile, profiles[2];
630     guint i, n_profiles = 0;
631
632     profile = h264_get_profile(sps);
633     if (!profile)
634         return GST_VAAPI_PROFILE_UNKNOWN;
635
636     profiles[n_profiles++] = profile;
637     switch (profile) {
638     case GST_VAAPI_PROFILE_H264_MAIN:
639         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
640         break;
641     default:
642         break;
643     }
644
645     /* If the preferred profile (profiles[0]) matches one that we already
646        found, then just return it now instead of searching for it again */
647     if (profiles[0] == priv->profile)
648         return priv->profile;
649
650     for (i = 0; i < n_profiles; i++) {
651         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
652             return profiles[i];
653     }
654     return GST_VAAPI_PROFILE_UNKNOWN;
655 }
656
657 static GstVaapiDecoderStatus
658 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
659 {
660     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
661     GstVaapiDecoderH264Private * const priv = decoder->priv;
662     GstVaapiContextInfo info;
663     GstVaapiProfile profile;
664     GstVaapiChromaType chroma_type;
665     gboolean reset_context = FALSE;
666
667     profile = get_profile(decoder, sps);
668     if (!profile) {
669         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
670         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
671     }
672
673     if (priv->profile != profile) {
674         GST_DEBUG("profile changed");
675         reset_context = TRUE;
676         priv->profile = profile;
677     }
678
679     chroma_type = h264_get_chroma_type(sps);
680     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
681         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
682         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
683     }
684
685     if (priv->chroma_type != chroma_type) {
686         GST_DEBUG("chroma format changed");
687         reset_context     = TRUE;
688         priv->chroma_type = chroma_type;
689     }
690
691     if (priv->width != sps->width || priv->height != sps->height) {
692         GST_DEBUG("size changed");
693         reset_context = TRUE;
694         priv->width   = sps->width;
695         priv->height  = sps->height;
696     }
697
698     gst_vaapi_decoder_set_pixel_aspect_ratio(
699         base_decoder,
700         sps->vui_parameters.par_n,
701         sps->vui_parameters.par_d
702     );
703
704     if (!reset_context && priv->has_context)
705         return GST_VAAPI_DECODER_STATUS_SUCCESS;
706
707     info.profile    = priv->profile;
708     info.entrypoint = priv->entrypoint;
709     info.width      = priv->width;
710     info.height     = priv->height;
711     info.ref_frames = get_max_dec_frame_buffering(sps);
712
713     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
714         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
715     priv->has_context = TRUE;
716
717     /* Reset DPB */
718     dpb_reset(decoder, sps);
719     return GST_VAAPI_DECODER_STATUS_SUCCESS;
720 }
721
722 static void
723 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
724 {
725     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
726     guint i, j;
727
728     /* There are always 6 4x4 scaling lists */
729     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
730     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
731
732     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
733         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
734                sizeof(iq_matrix->ScalingList4x4));
735     else {
736         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
737             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
738                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
739         }
740     }
741 }
742
743 static void
744 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
745 {
746     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
747     const GstH264SPS * const sps = pps->sequence;
748     guint i, j, n;
749
750     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
751     if (!pps->transform_8x8_mode_flag)
752         return;
753
754     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
755     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
756
757     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
758         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
759                sizeof(iq_matrix->ScalingList8x8));
760     else {
761         n = (sps->chroma_format_idc != 3) ? 2 : 6;
762         for (i = 0; i < n; i++) {
763             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
764                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
765         }
766     }
767 }
768
769 static GstVaapiDecoderStatus
770 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
771 {
772     GstVaapiPicture * const base_picture = &picture->base;
773     GstH264PPS * const pps = picture->pps;
774     GstH264SPS * const sps = pps->sequence;
775     VAIQMatrixBufferH264 *iq_matrix;
776
777     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
778     if (!base_picture->iq_matrix) {
779         GST_ERROR("failed to allocate IQ matrix");
780         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
781     }
782     iq_matrix = base_picture->iq_matrix->param;
783
784     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
785        is not large enough to hold lists for 4:4:4 */
786     if (sps->chroma_format_idc == 3)
787         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
788
789     fill_iq_matrix_4x4(iq_matrix, pps);
790     fill_iq_matrix_8x8(iq_matrix, pps);
791
792     return GST_VAAPI_DECODER_STATUS_SUCCESS;
793 }
794
795 static GstVaapiDecoderStatus
796 decode_current_picture(GstVaapiDecoderH264 *decoder)
797 {
798     GstVaapiDecoderH264Private * const priv = decoder->priv;
799     GstVaapiPictureH264 * const picture = priv->current_picture;
800     GstVaapiDecoderStatus status;
801
802     if (!picture)
803         return GST_VAAPI_DECODER_STATUS_SUCCESS;
804
805     status = ensure_context(decoder, picture->pps->sequence);
806     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
807         return status;
808
809     if (!decode_picture_end(decoder, picture))
810         goto error;
811     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
812         goto error;
813     gst_vaapi_picture_replace(&priv->current_picture, NULL);
814     return GST_VAAPI_DECODER_STATUS_SUCCESS;
815
816 error:
817     gst_vaapi_picture_replace(&priv->current_picture, NULL);
818     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
819 }
820
821 static GstVaapiDecoderStatus
822 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
823 {
824     GstVaapiDecoderH264Private * const priv = decoder->priv;
825     GstH264SPS * const sps = &priv->last_sps;
826     GstH264ParserResult result;
827
828     GST_DEBUG("decode SPS");
829
830     memset(sps, 0, sizeof(*sps));
831     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
832     if (result != GST_H264_PARSER_OK)
833         return get_status(result);
834
835     return GST_VAAPI_DECODER_STATUS_SUCCESS;
836 }
837
838 static GstVaapiDecoderStatus
839 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
840 {
841     GstVaapiDecoderH264Private * const priv = decoder->priv;
842     GstH264PPS * const pps = &priv->last_pps;
843     GstH264ParserResult result;
844
845     GST_DEBUG("decode PPS");
846
847     memset(pps, 0, sizeof(*pps));
848     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
849     if (result != GST_H264_PARSER_OK)
850         return get_status(result);
851
852     return GST_VAAPI_DECODER_STATUS_SUCCESS;
853 }
854
855 static GstVaapiDecoderStatus
856 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
857 {
858     GstVaapiDecoderH264Private * const priv = decoder->priv;
859     GstH264SEIMessage sei;
860     GstH264ParserResult result;
861
862     GST_DEBUG("decode SEI");
863
864     memset(&sei, 0, sizeof(sei));
865     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
866     if (result != GST_H264_PARSER_OK) {
867         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
868         return get_status(result);
869     }
870
871     return GST_VAAPI_DECODER_STATUS_SUCCESS;
872 }
873
874 static GstVaapiDecoderStatus
875 decode_sequence_end(GstVaapiDecoderH264 *decoder)
876 {
877     GstVaapiDecoderStatus status;
878
879     GST_DEBUG("decode sequence-end");
880
881     status = decode_current_picture(decoder);
882     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
883         return status;
884
885     dpb_flush(decoder);
886     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
887 }
888
889 /* 8.2.1.1 - Decoding process for picture order count type 0 */
890 static void
891 init_picture_poc_0(
892     GstVaapiDecoderH264 *decoder,
893     GstVaapiPictureH264 *picture,
894     GstH264SliceHdr     *slice_hdr
895 )
896 {
897     GstVaapiDecoderH264Private * const priv = decoder->priv;
898     GstH264PPS * const pps = slice_hdr->pps;
899     GstH264SPS * const sps = pps->sequence;
900     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
901     gint32 temp_poc;
902
903     GST_DEBUG("decode picture order count type 0");
904
905     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
906         priv->prev_poc_msb = 0;
907         priv->prev_poc_lsb = 0;
908     }
909     else if (priv->prev_pic_has_mmco5) {
910         priv->prev_poc_msb = 0;
911         priv->prev_poc_lsb =
912             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
913              0 : priv->field_poc[TOP_FIELD]);
914     }
915     else {
916         priv->prev_poc_msb = priv->poc_msb;
917         priv->prev_poc_lsb = priv->poc_lsb;
918     }
919
920     // (8-3)
921     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
922     if (priv->poc_lsb < priv->prev_poc_lsb &&
923         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
924         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
925     else if (priv->poc_lsb > priv->prev_poc_lsb &&
926              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
927         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
928     else
929         priv->poc_msb = priv->prev_poc_msb;
930
931     temp_poc = priv->poc_msb + priv->poc_lsb;
932     switch (picture->base.structure) {
933     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
934         // (8-4, 8-5)
935         priv->field_poc[TOP_FIELD] = temp_poc;
936         priv->field_poc[BOTTOM_FIELD] = temp_poc +
937             slice_hdr->delta_pic_order_cnt_bottom;
938         break;
939     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
940         // (8-4)
941         priv->field_poc[TOP_FIELD] = temp_poc;
942         break;
943     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
944         // (8-5)
945         priv->field_poc[BOTTOM_FIELD] = temp_poc;
946         break;
947     }
948 }
949
950 /* 8.2.1.2 - Decoding process for picture order count type 1 */
951 static void
952 init_picture_poc_1(
953     GstVaapiDecoderH264 *decoder,
954     GstVaapiPictureH264 *picture,
955     GstH264SliceHdr     *slice_hdr
956 )
957 {
958     GstVaapiDecoderH264Private * const priv = decoder->priv;
959     GstH264PPS * const pps = slice_hdr->pps;
960     GstH264SPS * const sps = pps->sequence;
961     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
962     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
963     guint i;
964
965     GST_DEBUG("decode picture order count type 1");
966
967     if (priv->prev_pic_has_mmco5)
968         prev_frame_num_offset = 0;
969     else
970         prev_frame_num_offset = priv->frame_num_offset;
971
972     // (8-6)
973     if (GST_VAAPI_PICTURE_IS_IDR(picture))
974         priv->frame_num_offset = 0;
975     else if (priv->prev_frame_num > priv->frame_num)
976         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
977     else
978         priv->frame_num_offset = prev_frame_num_offset;
979
980     // (8-7)
981     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
982         abs_frame_num = priv->frame_num_offset + priv->frame_num;
983     else
984         abs_frame_num = 0;
985     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
986         abs_frame_num = abs_frame_num - 1;
987
988     if (abs_frame_num > 0) {
989         gint32 expected_delta_per_poc_cycle;
990         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
991
992         expected_delta_per_poc_cycle = 0;
993         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
994             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
995
996         // (8-8)
997         poc_cycle_cnt = (abs_frame_num - 1) /
998             sps->num_ref_frames_in_pic_order_cnt_cycle;
999         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1000             sps->num_ref_frames_in_pic_order_cnt_cycle;
1001
1002         // (8-9)
1003         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1004         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1005             expected_poc += sps->offset_for_ref_frame[i];
1006     }
1007     else
1008         expected_poc = 0;
1009     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1010         expected_poc += sps->offset_for_non_ref_pic;
1011
1012     // (8-10)
1013     switch (picture->base.structure) {
1014     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1015         priv->field_poc[TOP_FIELD] = expected_poc +
1016             slice_hdr->delta_pic_order_cnt[0];
1017         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1018             sps->offset_for_top_to_bottom_field +
1019             slice_hdr->delta_pic_order_cnt[1];
1020         break;
1021     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1022         priv->field_poc[TOP_FIELD] = expected_poc +
1023             slice_hdr->delta_pic_order_cnt[0];
1024         break;
1025     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1026         priv->field_poc[BOTTOM_FIELD] = expected_poc + 
1027             sps->offset_for_top_to_bottom_field +
1028             slice_hdr->delta_pic_order_cnt[0];
1029         break;
1030     }
1031 }
1032
1033 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1034 static void
1035 init_picture_poc_2(
1036     GstVaapiDecoderH264 *decoder,
1037     GstVaapiPictureH264 *picture,
1038     GstH264SliceHdr     *slice_hdr
1039 )
1040 {
1041     GstVaapiDecoderH264Private * const priv = decoder->priv;
1042     GstH264PPS * const pps = slice_hdr->pps;
1043     GstH264SPS * const sps = pps->sequence;
1044     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1045     gint32 prev_frame_num_offset, temp_poc;
1046
1047     GST_DEBUG("decode picture order count type 2");
1048
1049     if (priv->prev_pic_has_mmco5)
1050         prev_frame_num_offset = 0;
1051     else
1052         prev_frame_num_offset = priv->frame_num_offset;
1053
1054     // (8-11)
1055     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1056         priv->frame_num_offset = 0;
1057     else if (priv->prev_frame_num > priv->frame_num)
1058         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1059     else
1060         priv->frame_num_offset = prev_frame_num_offset;
1061
1062     // (8-12)
1063     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1064         temp_poc = 0;
1065     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1066         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1067     else
1068         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1069
1070     // (8-13)
1071     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1072         priv->field_poc[TOP_FIELD] = temp_poc;
1073     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1074         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1075 }
1076
1077 /* 8.2.1 - Decoding process for picture order count */
1078 static void
1079 init_picture_poc(
1080     GstVaapiDecoderH264 *decoder,
1081     GstVaapiPictureH264 *picture,
1082     GstH264SliceHdr     *slice_hdr
1083 )
1084 {
1085     GstVaapiDecoderH264Private * const priv = decoder->priv;
1086     VAPictureH264 * const pic = &picture->info;
1087     GstH264PPS * const pps = slice_hdr->pps;
1088     GstH264SPS * const sps = pps->sequence;
1089
1090     switch (sps->pic_order_cnt_type) {
1091     case 0:
1092         init_picture_poc_0(decoder, picture, slice_hdr);
1093         break;
1094     case 1:
1095         init_picture_poc_1(decoder, picture, slice_hdr);
1096         break;
1097     case 2:
1098         init_picture_poc_2(decoder, picture, slice_hdr);
1099         break;
1100     }
1101
1102     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1103         pic->TopFieldOrderCnt = priv->field_poc[TOP_FIELD];
1104     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1105         pic->BottomFieldOrderCnt = priv->field_poc[BOTTOM_FIELD];
1106     picture->poc = MIN(pic->TopFieldOrderCnt, pic->BottomFieldOrderCnt);
1107 }
1108
1109 static int
1110 compare_picture_pic_num_dec(const void *a, const void *b)
1111 {
1112     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1113     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1114
1115     return picB->pic_num - picA->pic_num;
1116 }
1117
1118 static int
1119 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1120 {
1121     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1122     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1123
1124     return picA->long_term_pic_num - picB->long_term_pic_num;
1125 }
1126
1127 static int
1128 compare_picture_poc_dec(const void *a, const void *b)
1129 {
1130     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1131     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1132
1133     return picB->poc - picA->poc;
1134 }
1135
1136 static int
1137 compare_picture_poc_inc(const void *a, const void *b)
1138 {
1139     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1140     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1141
1142     return picA->poc - picB->poc;
1143 }
1144
1145 static int
1146 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1147 {
1148     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1149     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1150
1151     return picB->frame_num_wrap - picA->frame_num_wrap;
1152 }
1153
1154 static int
1155 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1156 {
1157     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1158     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1159
1160     return picA->info.frame_idx - picB->info.frame_idx;
1161 }
1162
1163 /* 8.2.4.1 - Decoding process for picture numbers */
1164 static void
1165 init_picture_refs_pic_num(
1166     GstVaapiDecoderH264 *decoder,
1167     GstVaapiPictureH264 *picture,
1168     GstH264SliceHdr     *slice_hdr
1169 )
1170 {
1171     GstVaapiDecoderH264Private * const priv = decoder->priv;
1172     GstH264PPS * const pps = slice_hdr->pps;
1173     GstH264SPS * const sps = pps->sequence;
1174     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1175     guint i;
1176
1177     GST_DEBUG("decode picture numbers");
1178
1179     for (i = 0; i < priv->short_ref_count; i++) {
1180         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1181
1182         // (8-27)
1183         if (pic->frame_num > priv->frame_num)
1184             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1185         else
1186             pic->frame_num_wrap = pic->frame_num;
1187
1188         // (8-28, 8-30, 8-31)
1189         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1190             pic->pic_num = pic->frame_num_wrap;
1191         else {
1192             if (pic->base.structure == picture->base.structure)
1193                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1194             else
1195                 pic->pic_num = 2 * pic->frame_num_wrap;
1196         }
1197     }
1198
1199     for (i = 0; i < priv->long_ref_count; i++) {
1200         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1201
1202         // (8-29, 8-32, 8-33)
1203         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1204             pic->long_term_pic_num = pic->info.frame_idx;
1205         else {
1206             if (pic->base.structure == picture->base.structure)
1207                 pic->long_term_pic_num = 2 * pic->info.frame_idx + 1;
1208             else
1209                 pic->long_term_pic_num = 2 * pic->info.frame_idx;
1210         }
1211     }
1212 }
1213
1214 #define SORT_REF_LIST(list, n, compare_func) \
1215     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1216
1217 static void
1218 init_picture_refs_p_slice(
1219     GstVaapiDecoderH264 *decoder,
1220     GstVaapiPictureH264 *picture,
1221     GstH264SliceHdr     *slice_hdr
1222 )
1223 {
1224     GstVaapiDecoderH264Private * const priv = decoder->priv;
1225     GstVaapiPictureH264 **ref_list;
1226     guint i;
1227
1228     GST_DEBUG("decode reference picture list for P and SP slices");
1229
1230     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1231         /* 8.2.4.2.1 - P and SP slices in frames */
1232         if (priv->short_ref_count > 0) {
1233             ref_list = priv->RefPicList0;
1234             for (i = 0; i < priv->short_ref_count; i++)
1235                 ref_list[i] = priv->short_ref[i];
1236             SORT_REF_LIST(ref_list, i, pic_num_dec);
1237             priv->RefPicList0_count += i;
1238         }
1239
1240         if (priv->long_ref_count > 0) {
1241             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1242             for (i = 0; i < priv->long_ref_count; i++)
1243                 ref_list[i] = priv->long_ref[i];
1244             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1245             priv->RefPicList0_count += i;
1246         }
1247     }
1248     else {
1249         /* 8.2.4.2.2 - P and SP slices in fields */
1250         GstVaapiPictureH264 *short_ref[32];
1251         guint short_ref_count = 0;
1252         GstVaapiPictureH264 *long_ref[32];
1253         guint long_ref_count = 0;
1254
1255         // XXX: handle second field if current field is marked as
1256         // "used for short-term reference"
1257         if (priv->short_ref_count > 0) {
1258             for (i = 0; i < priv->short_ref_count; i++)
1259                 short_ref[i] = priv->short_ref[i];
1260             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1261             short_ref_count = i;
1262         }
1263
1264         // XXX: handle second field if current field is marked as
1265         // "used for long-term reference"
1266         if (priv->long_ref_count > 0) {
1267             for (i = 0; i < priv->long_ref_count; i++)
1268                 long_ref[i] = priv->long_ref[i];
1269             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1270             long_ref_count = i;
1271         }
1272
1273         // XXX: handle 8.2.4.2.5
1274     }
1275 }
1276
1277 static void
1278 init_picture_refs_b_slice(
1279     GstVaapiDecoderH264 *decoder,
1280     GstVaapiPictureH264 *picture,
1281     GstH264SliceHdr     *slice_hdr
1282 )
1283 {
1284     GstVaapiDecoderH264Private * const priv = decoder->priv;
1285     GstVaapiPictureH264 **ref_list;
1286     guint i, n;
1287
1288     GST_DEBUG("decode reference picture list for B slices");
1289
1290     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1291         /* 8.2.4.2.3 - B slices in frames */
1292
1293         /* RefPicList0 */
1294         if (priv->short_ref_count > 0) {
1295             // 1. Short-term references
1296             ref_list = priv->RefPicList0;
1297             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1298                 if (priv->short_ref[i]->poc < picture->poc)
1299                     ref_list[n++] = priv->short_ref[i];
1300             }
1301             SORT_REF_LIST(ref_list, n, poc_dec);
1302             priv->RefPicList0_count += n;
1303
1304             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1305             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1306                 if (priv->short_ref[i]->poc >= picture->poc)
1307                     ref_list[n++] = priv->short_ref[i];
1308             }
1309             SORT_REF_LIST(ref_list, n, poc_inc);
1310             priv->RefPicList0_count += n;
1311         }
1312
1313         if (priv->long_ref_count > 0) {
1314             // 2. Long-term references
1315             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1316             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1317                 ref_list[n++] = priv->long_ref[i];
1318             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1319             priv->RefPicList0_count += n;
1320         }
1321
1322         /* RefPicList1 */
1323         if (priv->short_ref_count > 0) {
1324             // 1. Short-term references
1325             ref_list = priv->RefPicList1;
1326             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1327                 if (priv->short_ref[i]->poc > picture->poc)
1328                     ref_list[n++] = priv->short_ref[i];
1329             }
1330             SORT_REF_LIST(ref_list, n, poc_inc);
1331             priv->RefPicList1_count += n;
1332
1333             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1334             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1335                 if (priv->short_ref[i]->poc <= picture->poc)
1336                     ref_list[n++] = priv->short_ref[i];
1337             }
1338             SORT_REF_LIST(ref_list, n, poc_dec);
1339             priv->RefPicList1_count += n;
1340         }
1341
1342         if (priv->long_ref_count > 0) {
1343             // 2. Long-term references
1344             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1345             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1346                 ref_list[n++] = priv->long_ref[i];
1347             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1348             priv->RefPicList1_count += n;
1349         }
1350     }
1351     else {
1352         /* 8.2.4.2.4 - B slices in fields */
1353         GstVaapiPictureH264 *short_ref0[32];
1354         guint short_ref0_count = 0;
1355         GstVaapiPictureH264 *short_ref1[32];
1356         guint short_ref1_count = 0;
1357         GstVaapiPictureH264 *long_ref[32];
1358         guint long_ref_count = 0;
1359
1360         /* refFrameList0ShortTerm */
1361         if (priv->short_ref_count > 0) {
1362             ref_list = short_ref0;
1363             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1364                 if (priv->short_ref[i]->poc <= picture->poc)
1365                     ref_list[n++] = priv->short_ref[i];
1366             }
1367             SORT_REF_LIST(ref_list, n, poc_dec);
1368             short_ref0_count += n;
1369
1370             ref_list = &short_ref0[short_ref0_count];
1371             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1372                 if (priv->short_ref[i]->poc > picture->poc)
1373                     ref_list[n++] = priv->short_ref[i];
1374             }
1375             SORT_REF_LIST(ref_list, n, poc_inc);
1376             short_ref0_count += n;
1377         }
1378
1379         /* refFrameList1ShortTerm */
1380         if (priv->short_ref_count > 0) {
1381             ref_list = short_ref1;
1382             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1383                 if (priv->short_ref[i]->poc > picture->poc)
1384                     ref_list[n++] = priv->short_ref[i];
1385             }
1386             SORT_REF_LIST(ref_list, n, poc_inc);
1387             short_ref1_count += n;
1388
1389             ref_list = &short_ref1[short_ref1_count];
1390             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1391                 if (priv->short_ref[i]->poc <= picture->poc)
1392                     ref_list[n++] = priv->short_ref[i];
1393             }
1394             SORT_REF_LIST(ref_list, n, poc_dec);
1395             short_ref1_count += n;
1396         }
1397
1398         /* refFrameListLongTerm */
1399         if (priv->long_ref_count > 0) {
1400             for (i = 0; i < priv->long_ref_count; i++)
1401                 long_ref[i] = priv->long_ref[i];
1402             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1403             long_ref_count = i;
1404         }
1405
1406         // XXX: handle 8.2.4.2.5
1407     }
1408
1409     /* Check whether RefPicList1 is identical to RefPicList0, then
1410        swap if necessary */
1411     if (priv->RefPicList1_count > 1 &&
1412         priv->RefPicList1_count == priv->RefPicList0_count &&
1413         memcmp(priv->RefPicList0, priv->RefPicList1,
1414                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1415         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1416         priv->RefPicList1[0] = priv->RefPicList1[1];
1417         priv->RefPicList1[1] = tmp;
1418     }
1419 }
1420
1421 #undef SORT_REF_LIST
1422
1423 static void
1424 clear_references(
1425     GstVaapiDecoderH264  *decoder,
1426     GstVaapiPictureH264 **pictures,
1427     guint                *picture_count
1428 )
1429 {
1430     const guint num_pictures = *picture_count;
1431     guint i;
1432
1433     for (i = 0; i < num_pictures; i++)
1434         gst_vaapi_picture_replace(&pictures[i], NULL);
1435     *picture_count = 0;
1436 }
1437
1438 static gboolean
1439 remove_reference_at(
1440     GstVaapiDecoderH264  *decoder,
1441     GstVaapiPictureH264 **pictures,
1442     guint                *picture_count,
1443     guint                 index
1444 )
1445 {
1446     guint num_pictures = *picture_count;
1447     GstVaapiPictureH264 *picture;
1448
1449     g_return_val_if_fail(index < num_pictures, FALSE);
1450
1451     picture = pictures[index];
1452     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1453     picture->is_long_term = FALSE;
1454     picture->info.flags &= ~(VA_PICTURE_H264_SHORT_TERM_REFERENCE |
1455                              VA_PICTURE_H264_LONG_TERM_REFERENCE);
1456
1457     if (index != --num_pictures)
1458         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1459     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1460     *picture_count = num_pictures;
1461     return TRUE;
1462 }
1463
1464 static gint
1465 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1466 {
1467     GstVaapiDecoderH264Private * const priv = decoder->priv;
1468     guint i;
1469
1470     for (i = 0; i < priv->short_ref_count; i++) {
1471         if (priv->short_ref[i]->pic_num == pic_num)
1472             return i;
1473     }
1474     GST_ERROR("found no short-term reference picture with PicNum = %d",
1475               pic_num);
1476     return -1;
1477 }
1478
1479 static gint
1480 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1481 {
1482     GstVaapiDecoderH264Private * const priv = decoder->priv;
1483     guint i;
1484
1485     for (i = 0; i < priv->long_ref_count; i++) {
1486         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1487             return i;
1488     }
1489     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1490               long_term_pic_num);
1491     return -1;
1492 }
1493
1494 static void
1495 exec_picture_refs_modification_1(
1496     GstVaapiDecoderH264           *decoder,
1497     GstVaapiPictureH264           *picture,
1498     GstH264SliceHdr               *slice_hdr,
1499     guint                          list
1500 )
1501 {
1502     GstVaapiDecoderH264Private * const priv = decoder->priv;
1503     GstH264PPS * const pps = slice_hdr->pps;
1504     GstH264SPS * const sps = pps->sequence;
1505     GstH264RefPicListModification *ref_pic_list_modification;
1506     guint num_ref_pic_list_modifications;
1507     GstVaapiPictureH264 **ref_list;
1508     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1509     guint i, j, n, num_refs;
1510     gint found_ref_idx;
1511     gint32 MaxPicNum, CurrPicNum, picNumPred;
1512
1513     GST_DEBUG("modification process of reference picture list %u", list);
1514
1515     if (list == 0) {
1516         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1517         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1518         ref_list                       = priv->RefPicList0;
1519         ref_list_count_ptr             = &priv->RefPicList0_count;
1520         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1521     }
1522     else {
1523         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1524         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1525         ref_list                       = priv->RefPicList1;
1526         ref_list_count_ptr             = &priv->RefPicList1_count;
1527         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1528     }
1529     ref_list_count = *ref_list_count_ptr;
1530
1531     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1532         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1533         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1534     }
1535     else {
1536         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1537         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1538     }
1539
1540     picNumPred = CurrPicNum;
1541
1542     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1543         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1544         if (l->modification_of_pic_nums_idc == 3)
1545             break;
1546
1547         /* 8.2.4.3.1 - Short-term reference pictures */
1548         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1549             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1550             gint32 picNum, picNumNoWrap;
1551
1552             // (8-34)
1553             if (l->modification_of_pic_nums_idc == 0) {
1554                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1555                 if (picNumNoWrap < 0)
1556                     picNumNoWrap += MaxPicNum;
1557             }
1558
1559             // (8-35)
1560             else {
1561                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1562                 if (picNumNoWrap >= MaxPicNum)
1563                     picNumNoWrap -= MaxPicNum;
1564             }
1565             picNumPred = picNumNoWrap;
1566
1567             // (8-36)
1568             picNum = picNumNoWrap;
1569             if (picNum > CurrPicNum)
1570                 picNum -= MaxPicNum;
1571
1572             // (8-37)
1573             for (j = num_refs; j > ref_list_idx; j--)
1574                 ref_list[j] = ref_list[j - 1];
1575             found_ref_idx = find_short_term_reference(decoder, picNum);
1576             ref_list[ref_list_idx++] =
1577                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1578             n = ref_list_idx;
1579             for (j = ref_list_idx; j <= num_refs; j++) {
1580                 gint32 PicNumF;
1581                 if (!ref_list[j])
1582                     continue;
1583                 PicNumF = ref_list[j]->is_long_term ?
1584                     MaxPicNum : ref_list[j]->pic_num;
1585                 if (PicNumF != picNum)
1586                     ref_list[n++] = ref_list[j];
1587             }
1588         }
1589
1590         /* 8.2.4.3.2 - Long-term reference pictures */
1591         else {
1592
1593             for (j = num_refs; j > ref_list_idx; j--)
1594                 ref_list[j] = ref_list[j - 1];
1595             found_ref_idx =
1596                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1597             ref_list[ref_list_idx++] =
1598                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1599             n = ref_list_idx;
1600             for (j = ref_list_idx; j <= num_refs; j++) {
1601                 gint32 LongTermPicNumF;
1602                 if (!ref_list[j])
1603                     continue;
1604                 LongTermPicNumF = ref_list[j]->is_long_term ?
1605                     ref_list[j]->long_term_pic_num : INT_MAX;
1606                 if (LongTermPicNumF != l->value.long_term_pic_num)
1607                     ref_list[n++] = ref_list[j];
1608             }
1609         }
1610     }
1611
1612 #if DEBUG
1613     for (i = 0; i < num_refs; i++)
1614         if (!ref_list[i])
1615             GST_ERROR("list %u entry %u is empty", list, i);
1616 #endif
1617     *ref_list_count_ptr = num_refs;
1618 }
1619
1620 /* 8.2.4.3 - Modification process for reference picture lists */
1621 static void
1622 exec_picture_refs_modification(
1623     GstVaapiDecoderH264 *decoder,
1624     GstVaapiPictureH264 *picture,
1625     GstH264SliceHdr     *slice_hdr
1626 )
1627 {
1628     GST_DEBUG("execute ref_pic_list_modification()");
1629
1630     /* RefPicList0 */
1631     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1632         slice_hdr->ref_pic_list_modification_flag_l0)
1633         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1634
1635     /* RefPicList1 */
1636     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1637         slice_hdr->ref_pic_list_modification_flag_l1)
1638         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1639 }
1640
1641 static gboolean
1642 init_picture_refs(
1643     GstVaapiDecoderH264 *decoder,
1644     GstVaapiPictureH264 *picture,
1645     GstH264SliceHdr     *slice_hdr
1646 )
1647 {
1648     GstVaapiDecoderH264Private * const priv = decoder->priv;
1649     GstVaapiPicture * const base_picture = &picture->base;
1650     guint i, num_refs;
1651
1652     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1653
1654     priv->RefPicList0_count = 0;
1655     priv->RefPicList1_count = 0;
1656
1657     switch (base_picture->type) {
1658     case GST_VAAPI_PICTURE_TYPE_P:
1659     case GST_VAAPI_PICTURE_TYPE_SP:
1660         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1661         break;
1662     case GST_VAAPI_PICTURE_TYPE_B:
1663         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1664         break;
1665     default:
1666         break;
1667     }
1668
1669     exec_picture_refs_modification(decoder, picture, slice_hdr);
1670
1671     switch (base_picture->type) {
1672     case GST_VAAPI_PICTURE_TYPE_B:
1673         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1674         for (i = priv->RefPicList1_count; i < num_refs; i++)
1675             priv->RefPicList1[i] = NULL;
1676         priv->RefPicList1_count = num_refs;
1677
1678         // fall-through
1679     case GST_VAAPI_PICTURE_TYPE_P:
1680     case GST_VAAPI_PICTURE_TYPE_SP:
1681         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1682         for (i = priv->RefPicList0_count; i < num_refs; i++)
1683             priv->RefPicList0[i] = NULL;
1684         priv->RefPicList0_count = num_refs;
1685         break;
1686     default:
1687         break;
1688     }
1689     return TRUE;
1690 }
1691
1692 static gboolean
1693 init_picture(
1694     GstVaapiDecoderH264 *decoder,
1695     GstVaapiPictureH264 *picture,
1696     GstH264SliceHdr     *slice_hdr,
1697     GstH264NalUnit      *nalu
1698 )
1699 {
1700     GstVaapiDecoderH264Private * const priv = decoder->priv;
1701     GstVaapiPicture * const base_picture = &picture->base;
1702     VAPictureH264 *pic;
1703
1704     priv->prev_frame_num        = priv->frame_num;
1705     priv->frame_num             = slice_hdr->frame_num;
1706     picture->frame_num          = priv->frame_num;
1707     picture->frame_num_wrap     = priv->frame_num;
1708     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1709     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1710
1711     /* Reset decoder state for IDR pictures */
1712     if (nalu->type == GST_H264_NAL_SLICE_IDR) {
1713         GST_DEBUG("<IDR>");
1714         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
1715         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1716         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1717     }
1718
1719     /* Initialize VA picture info */
1720     pic = &picture->info;
1721     pic->picture_id = picture->base.surface_id;
1722     pic->frame_idx  = priv->frame_num;
1723
1724     /* Initialize slice type */
1725     switch (slice_hdr->type % 5) {
1726     case GST_H264_P_SLICE:
1727         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1728         break;
1729     case GST_H264_B_SLICE:
1730         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1731         break;
1732     case GST_H264_I_SLICE:
1733         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1734         break;
1735     case GST_H264_SP_SLICE:
1736         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1737         break;
1738     case GST_H264_SI_SLICE:
1739         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1740         break;
1741     }
1742
1743     /* Initialize picture structure */
1744     if (!slice_hdr->field_pic_flag)
1745         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1746     else if (!slice_hdr->bottom_field_flag)
1747         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
1748     else
1749         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
1750
1751     if (nalu->ref_idc) {
1752         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1753             &slice_hdr->dec_ref_pic_marking;
1754         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1755         if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
1756             if (dec_ref_pic_marking->long_term_reference_flag)
1757                 picture->is_long_term = TRUE;
1758         }
1759     }
1760
1761     init_picture_poc(decoder, picture, slice_hdr);
1762     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1763         GST_ERROR("failed to initialize references");
1764         return FALSE;
1765     }
1766     return TRUE;
1767 }
1768
1769 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1770 static gboolean
1771 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1772 {
1773     GstVaapiDecoderH264Private * const priv = decoder->priv;
1774     GstH264PPS * const pps = priv->current_picture->pps;
1775     GstH264SPS * const sps = pps->sequence;
1776     guint i, max_num_ref_frames, lowest_frame_num_index;
1777     gint32 lowest_frame_num;
1778
1779     GST_DEBUG("reference picture marking process (sliding window)");
1780
1781     max_num_ref_frames = sps->num_ref_frames;
1782     if (max_num_ref_frames == 0)
1783         max_num_ref_frames = 1;
1784
1785     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1786         return TRUE;
1787     if (priv->short_ref_count < 1)
1788         return FALSE;
1789
1790     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1791     lowest_frame_num_index = 0;
1792     for (i = 1; i < priv->short_ref_count; i++) {
1793         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1794             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1795             lowest_frame_num_index = i;
1796         }
1797     }
1798
1799     remove_reference_at(
1800         decoder,
1801         priv->short_ref, &priv->short_ref_count,
1802         lowest_frame_num_index
1803     );
1804     return TRUE;
1805 }
1806
1807 static inline gint32
1808 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
1809 {
1810     gint32 pic_num;
1811
1812     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1813         pic_num = picture->frame_num_wrap;
1814     else
1815         pic_num = 2 * picture->frame_num_wrap + 1;
1816     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1817     return pic_num;
1818 }
1819
1820 /* 8.2.5.4.1. Mark-term reference picture as "unused for reference" */
1821 static void
1822 exec_ref_pic_marking_adaptive_mmco_1(
1823     GstVaapiDecoderH264  *decoder,
1824     GstVaapiPictureH264  *picture,
1825     GstH264RefPicMarking *ref_pic_marking
1826 )
1827 {
1828     GstVaapiDecoderH264Private * const priv = decoder->priv;
1829     gint32 i, picNumX;
1830
1831     picNumX = get_picNumX(picture, ref_pic_marking);
1832     i = find_short_term_reference(decoder, picNumX);
1833     if (i < 0)
1834         return;
1835     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1836 }
1837
1838 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
1839 static void
1840 exec_ref_pic_marking_adaptive_mmco_2(
1841     GstVaapiDecoderH264  *decoder,
1842     GstVaapiPictureH264  *picture,
1843     GstH264RefPicMarking *ref_pic_marking
1844 )
1845 {
1846     GstVaapiDecoderH264Private * const priv = decoder->priv;
1847     gint32 i;
1848
1849     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
1850     if (i < 0)
1851         return;
1852     remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1853 }
1854
1855 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
1856 static void
1857 exec_ref_pic_marking_adaptive_mmco_3(
1858     GstVaapiDecoderH264  *decoder,
1859     GstVaapiPictureH264  *picture,
1860     GstH264RefPicMarking *ref_pic_marking
1861 )
1862 {
1863     GstVaapiDecoderH264Private * const priv = decoder->priv;
1864     VAPictureH264 *pic;
1865     gint32 i, picNumX;
1866
1867     for (i = 0; i < priv->long_ref_count; i++) {
1868         if (priv->long_ref[i]->info.frame_idx == ref_pic_marking->long_term_frame_idx)
1869             break;
1870     }
1871     if (i != priv->long_ref_count)
1872         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1873
1874     picNumX = get_picNumX(picture, ref_pic_marking);
1875     i = find_short_term_reference(decoder, picNumX);
1876     if (i < 0)
1877         return;
1878
1879     picture = gst_vaapi_picture_ref(priv->short_ref[i]);
1880     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1881     gst_vaapi_picture_replace(&priv->long_ref[priv->long_ref_count++], picture);
1882     gst_vaapi_picture_unref(picture);
1883
1884     picture->is_long_term = TRUE;
1885     pic = &picture->info;
1886     pic->frame_idx = ref_pic_marking->long_term_frame_idx;
1887     pic->flags &= ~VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1888     pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1889     GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1890 }
1891
1892 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
1893  * as "unused for reference" */
1894 static void
1895 exec_ref_pic_marking_adaptive_mmco_4(
1896     GstVaapiDecoderH264  *decoder,
1897     GstVaapiPictureH264  *picture,
1898     GstH264RefPicMarking *ref_pic_marking
1899 )
1900 {
1901     GstVaapiDecoderH264Private * const priv = decoder->priv;
1902     gint32 i, long_term_frame_idx;
1903
1904     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
1905
1906     for (i = 0; i < priv->long_ref_count; i++) {
1907         if ((gint32)priv->long_ref[i]->info.frame_idx <= long_term_frame_idx)
1908             continue;
1909         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1910         i--;
1911     }
1912 }
1913
1914 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
1915 static void
1916 exec_ref_pic_marking_adaptive_mmco_5(
1917     GstVaapiDecoderH264  *decoder,
1918     GstVaapiPictureH264  *picture,
1919     GstH264RefPicMarking *ref_pic_marking
1920 )
1921 {
1922     GstVaapiDecoderH264Private * const priv = decoder->priv;
1923     VAPictureH264 * const pic = &picture->info;
1924
1925     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1926     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1927     dpb_flush(decoder);
1928
1929     priv->prev_pic_has_mmco5 = TRUE;
1930
1931     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
1932     priv->frame_num = 0;
1933     priv->frame_num_offset = 0;
1934     picture->frame_num = 0;
1935
1936     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
1937     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1938         pic->TopFieldOrderCnt -= picture->poc;
1939     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1940         pic->BottomFieldOrderCnt -= picture->poc;
1941     picture->poc = 0;
1942 }
1943
1944 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
1945 static void
1946 exec_ref_pic_marking_adaptive_mmco_6(
1947     GstVaapiDecoderH264  *decoder,
1948     GstVaapiPictureH264  *picture,
1949     GstH264RefPicMarking *ref_pic_marking
1950 )
1951 {
1952     picture->is_long_term = TRUE;
1953     picture->info.frame_idx = ref_pic_marking->long_term_frame_idx;
1954 }
1955
1956 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1957 static gboolean
1958 exec_ref_pic_marking_adaptive(
1959     GstVaapiDecoderH264     *decoder,
1960     GstVaapiPictureH264     *picture,
1961     GstH264DecRefPicMarking *dec_ref_pic_marking
1962 )
1963 {
1964     guint i;
1965
1966     GST_DEBUG("reference picture marking process (adaptive memory control)");
1967
1968     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
1969         GstVaapiDecoderH264  *decoder,
1970         GstVaapiPictureH264  *picture,
1971         GstH264RefPicMarking *ref_pic_marking
1972     );
1973
1974     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
1975         NULL,
1976         exec_ref_pic_marking_adaptive_mmco_1,
1977         exec_ref_pic_marking_adaptive_mmco_2,
1978         exec_ref_pic_marking_adaptive_mmco_3,
1979         exec_ref_pic_marking_adaptive_mmco_4,
1980         exec_ref_pic_marking_adaptive_mmco_5,
1981         exec_ref_pic_marking_adaptive_mmco_6,
1982     };
1983
1984     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1985         GstH264RefPicMarking * const ref_pic_marking =
1986             &dec_ref_pic_marking->ref_pic_marking[i];
1987
1988         const guint mmco = ref_pic_marking->memory_management_control_operation;
1989         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
1990             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
1991         else {
1992             GST_ERROR("unhandled MMCO %u", mmco);
1993             return FALSE;
1994         }
1995     }
1996     return TRUE;
1997 }
1998
1999 /* 8.2.5 - Execute reference picture marking process */
2000 static gboolean
2001 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2002 {
2003     GstVaapiDecoderH264Private * const priv = decoder->priv;
2004     GstVaapiPictureH264 **picture_ptr;
2005
2006     priv->prev_pic_has_mmco5 = FALSE;
2007     priv->prev_pic_structure = picture->base.structure;
2008
2009     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2010         return TRUE;
2011
2012     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2013         GstVaapiSliceH264 * const slice =
2014             gst_vaapi_picture_h264_get_last_slice(picture);
2015         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2016             &slice->slice_hdr.dec_ref_pic_marking;
2017         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2018             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2019                 return FALSE;
2020         }
2021         else {
2022             if (!exec_ref_pic_marking_sliding_window(decoder))
2023                 return FALSE;
2024         }
2025     }
2026
2027     if (picture->is_long_term) {
2028         picture_ptr = &priv->long_ref[priv->long_ref_count++];
2029         picture->info.flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2030     }
2031     else {
2032         picture_ptr = &priv->short_ref[priv->short_ref_count++];
2033         picture->info.flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2034     }
2035     gst_vaapi_picture_replace(picture_ptr, picture);
2036     return TRUE;
2037 }
2038
2039 static void
2040 vaapi_init_picture(VAPictureH264 *pic)
2041 {
2042     pic->picture_id           = VA_INVALID_ID;
2043     pic->frame_idx            = 0;
2044     pic->flags                = VA_PICTURE_H264_INVALID;
2045     pic->TopFieldOrderCnt     = 0;
2046     pic->BottomFieldOrderCnt  = 0;
2047 }
2048
2049 static void
2050 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture)
2051 {
2052     pic->picture_id = picture->base.surface_id;
2053     pic->flags = 0;
2054
2055     if (picture->info.flags & VA_PICTURE_H264_LONG_TERM_REFERENCE) {
2056         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2057         pic->frame_idx = picture->info.frame_idx;
2058     }
2059     else {
2060         if (picture->info.flags & VA_PICTURE_H264_SHORT_TERM_REFERENCE)
2061             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2062         pic->frame_idx = picture->frame_num;
2063     }
2064
2065     switch (picture->base.structure) {
2066     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2067         pic->TopFieldOrderCnt = picture->info.TopFieldOrderCnt;
2068         pic->BottomFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2069         break;
2070     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2071         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2072         pic->TopFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2073         pic->BottomFieldOrderCnt = 0;
2074         break;
2075     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2076         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2077         pic->BottomFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2078         pic->TopFieldOrderCnt = 0;
2079         break;
2080     }
2081 }
2082
2083 static gboolean
2084 fill_picture(
2085     GstVaapiDecoderH264 *decoder,
2086     GstVaapiPictureH264 *picture,
2087     GstH264SliceHdr     *slice_hdr,
2088     GstH264NalUnit      *nalu
2089 )
2090 {
2091     GstVaapiDecoderH264Private * const priv = decoder->priv;
2092     GstVaapiPicture * const base_picture = &picture->base;
2093     GstH264PPS * const pps = picture->pps;
2094     GstH264SPS * const sps = pps->sequence;
2095     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2096     guint i, n;
2097
2098     /* Fill in VAPictureParameterBufferH264 */
2099     vaapi_fill_picture(&pic_param->CurrPic, picture);
2100     for (i = 0, n = 0; i < priv->short_ref_count; i++, n++)
2101         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->short_ref[i]);
2102     for (i = 0; i < priv->long_ref_count; i++, n++)
2103         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->long_ref[i]);
2104     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2105         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2106
2107 #define COPY_FIELD(s, f) \
2108     pic_param->f = (s)->f
2109
2110 #define COPY_BFM(a, s, f) \
2111     pic_param->a.bits.f = (s)->f
2112
2113     pic_param->picture_width_in_mbs_minus1  = ((priv->width + 15) >> 4)  - 1;
2114     pic_param->picture_height_in_mbs_minus1 = ((priv->height + 15) >> 4) - 1;
2115     pic_param->frame_num                    = priv->frame_num;
2116
2117     COPY_FIELD(sps, bit_depth_luma_minus8);
2118     COPY_FIELD(sps, bit_depth_chroma_minus8);
2119     COPY_FIELD(sps, num_ref_frames);
2120     COPY_FIELD(pps, num_slice_groups_minus1);
2121     COPY_FIELD(pps, slice_group_map_type);
2122     COPY_FIELD(pps, slice_group_change_rate_minus1);
2123     COPY_FIELD(pps, pic_init_qp_minus26);
2124     COPY_FIELD(pps, pic_init_qs_minus26);
2125     COPY_FIELD(pps, chroma_qp_index_offset);
2126     COPY_FIELD(pps, second_chroma_qp_index_offset);
2127
2128     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2129     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2130     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2131
2132     COPY_BFM(seq_fields, sps, chroma_format_idc);
2133     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2134     COPY_BFM(seq_fields, sps, frame_mbs_only_flag); 
2135     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag); 
2136     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag); 
2137     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2138     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2139     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2140     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2141
2142     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2143     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2144     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2145
2146     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2147     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2148     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2149     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2150     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2151     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2152     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2153     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2154     return TRUE;
2155 }
2156
2157 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2158 static gboolean
2159 is_new_picture(
2160     GstVaapiDecoderH264 *decoder,
2161     GstH264NalUnit      *nalu,
2162     GstH264SliceHdr     *slice_hdr
2163 )
2164 {
2165     GstVaapiDecoderH264Private * const priv = decoder->priv;
2166     GstH264PPS * const pps = slice_hdr->pps;
2167     GstH264SPS * const sps = pps->sequence;
2168     GstVaapiSliceH264 *slice;
2169     GstH264SliceHdr *prev_slice_hdr;
2170
2171     if (!priv->current_picture)
2172         return TRUE;
2173
2174     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2175     if (!slice)
2176         return FALSE;
2177     prev_slice_hdr = &slice->slice_hdr;
2178
2179 #define CHECK_EXPR(expr, field_name) do {              \
2180         if (!(expr)) {                                 \
2181             GST_DEBUG(field_name " differs in value"); \
2182             return TRUE;                               \
2183         }                                              \
2184     } while (0)
2185
2186 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2187     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2188
2189     /* frame_num differs in value, regardless of inferred values to 0 */
2190     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2191
2192     /* pic_parameter_set_id differs in value */
2193     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2194
2195     /* field_pic_flag differs in value */
2196     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2197
2198     /* bottom_field_flag is present in both and differs in value */
2199     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2200         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2201
2202     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2203     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2204                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2205
2206     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2207        value or delta_pic_order_cnt_bottom differs in value */
2208     if (sps->pic_order_cnt_type == 0) {
2209         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2210         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2211             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2212     }
2213
2214     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2215        differs in value or delta_pic_order_cnt[1] differs in value */
2216     else if (sps->pic_order_cnt_type == 1) {
2217         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2218         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2219     }
2220
2221     /* IdrPicFlag differs in value */
2222     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_IDR(priv->current_picture) ^
2223                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2224
2225     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2226     if (GST_VAAPI_PICTURE_IS_IDR(priv->current_picture))
2227         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2228
2229 #undef CHECK_EXPR
2230 #undef CHECK_VALUE
2231     return FALSE;
2232 }
2233
2234 static GstVaapiDecoderStatus
2235 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2236 {
2237     GstVaapiDecoderH264Private * const priv = decoder->priv;
2238     GstVaapiPictureH264 *picture;
2239     GstVaapiDecoderStatus status;
2240     GstH264PPS * const pps = slice_hdr->pps;
2241     GstH264SPS * const sps = pps->sequence;
2242
2243     status = decode_current_picture(decoder);
2244     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2245         return status;
2246
2247     status = ensure_context(decoder, sps);
2248     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2249         return status;
2250
2251     picture = gst_vaapi_picture_h264_new(decoder);
2252     if (!picture) {
2253         GST_ERROR("failed to allocate picture");
2254         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2255     }
2256     priv->current_picture = picture;
2257
2258     picture->pps = pps;
2259
2260     status = ensure_quant_matrix(decoder, picture);
2261     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2262         GST_ERROR("failed to reset quantizer matrix");
2263         return status;
2264     }
2265
2266     if (!init_picture(decoder, picture, slice_hdr, nalu))
2267         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2268     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2269         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2270     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2271 }
2272
2273 static gboolean
2274 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2275 {
2276     if (!exec_ref_pic_marking(decoder, picture))
2277         return FALSE;
2278     if (!dpb_add(decoder, picture))
2279         return FALSE;
2280     return TRUE;
2281 }
2282
2283 static inline guint
2284 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2285 {
2286     guint epb_count;
2287
2288     epb_count = slice_hdr->n_emulation_prevention_bytes;
2289     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2290 }
2291
2292 static gboolean
2293 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2294 {
2295     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2296     GstH264PPS * const pps = slice_hdr->pps;
2297     GstH264SPS * const sps = pps->sequence;
2298     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2299     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2300     guint num_weight_tables = 0;
2301     gint i, j;
2302
2303     if (pps->weighted_pred_flag &&
2304         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2305         num_weight_tables = 1;
2306     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2307         num_weight_tables = 2;
2308     else
2309         num_weight_tables = 0;
2310
2311     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2312     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2313     slice_param->luma_weight_l0_flag      = 0;
2314     slice_param->chroma_weight_l0_flag    = 0;
2315     slice_param->luma_weight_l1_flag      = 0;
2316     slice_param->chroma_weight_l1_flag    = 0;
2317
2318     if (num_weight_tables < 1)
2319         return TRUE;
2320
2321     slice_param->luma_weight_l0_flag = 1;
2322     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2323         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2324         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2325     }
2326
2327     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2328     if (slice_param->chroma_weight_l0_flag) {
2329         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2330             for (j = 0; j < 2; j++) {
2331                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2332                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2333             }
2334         }
2335     }
2336
2337     if (num_weight_tables < 2)
2338         return TRUE;
2339
2340     slice_param->luma_weight_l1_flag = 1;
2341     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2342         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2343         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2344     }
2345
2346     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2347     if (slice_param->chroma_weight_l1_flag) {
2348         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2349             for (j = 0; j < 2; j++) {
2350                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2351                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2352             }
2353         }
2354     }
2355     return TRUE;
2356 }
2357
2358 static gboolean
2359 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2360 {
2361     GstVaapiDecoderH264Private * const priv = decoder->priv;
2362     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2363     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2364     guint i, num_ref_lists = 0;
2365
2366     slice_param->num_ref_idx_l0_active_minus1 = 0;
2367     slice_param->num_ref_idx_l1_active_minus1 = 0;
2368
2369     if (GST_H264_IS_B_SLICE(slice_hdr))
2370         num_ref_lists = 2;
2371     else if (GST_H264_IS_I_SLICE(slice_hdr))
2372         num_ref_lists = 0;
2373     else
2374         num_ref_lists = 1;
2375
2376     if (num_ref_lists < 1)
2377         return TRUE;
2378
2379     slice_param->num_ref_idx_l0_active_minus1 =
2380         slice_hdr->num_ref_idx_l0_active_minus1;
2381
2382     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2383         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i]);
2384     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2385         vaapi_init_picture(&slice_param->RefPicList0[i]);
2386
2387     if (num_ref_lists < 2)
2388         return TRUE;
2389
2390     slice_param->num_ref_idx_l1_active_minus1 =
2391         slice_hdr->num_ref_idx_l1_active_minus1;
2392
2393     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2394         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i]);
2395     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2396         vaapi_init_picture(&slice_param->RefPicList1[i]);
2397     return TRUE;
2398 }
2399
2400 static gboolean
2401 fill_slice(
2402     GstVaapiDecoderH264 *decoder,
2403     GstVaapiSliceH264   *slice,
2404     GstH264NalUnit      *nalu
2405 )
2406 {
2407     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2408     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2409
2410     /* Fill in VASliceParameterBufferH264 */
2411     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2412     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2413     slice_param->slice_type                     = slice_hdr->type % 5;
2414     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2415     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2416     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2417     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2418     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2419     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2420
2421     if (!fill_RefPicList(decoder, slice))
2422         return FALSE;
2423     if (!fill_pred_weight_table(decoder, slice))
2424         return FALSE;
2425     return TRUE;
2426 }
2427
2428 static GstVaapiDecoderStatus
2429 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2430 {
2431     GstVaapiDecoderH264Private * const priv = decoder->priv;
2432     GstVaapiDecoderStatus status;
2433     GstVaapiPictureH264 *picture;
2434     GstVaapiSliceH264 *slice = NULL;
2435     GstH264SliceHdr *slice_hdr;
2436     GstH264ParserResult result;
2437
2438     GST_DEBUG("slice (%u bytes)", nalu->size);
2439
2440     slice = gst_vaapi_slice_h264_new(
2441         decoder,
2442         nalu->data + nalu->offset,
2443         nalu->size
2444     );
2445     if (!slice) {
2446         GST_ERROR("failed to allocate slice");
2447         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2448     }
2449
2450     slice_hdr = &slice->slice_hdr;
2451     memset(slice_hdr, 0, sizeof(*slice_hdr));
2452     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2453     if (result != GST_H264_PARSER_OK) {
2454         status = get_status(result);
2455         goto error;
2456     }
2457
2458     if (is_new_picture(decoder, nalu, slice_hdr)) {
2459         status = decode_picture(decoder, nalu, slice_hdr);
2460         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2461             goto error;
2462     }
2463     picture = priv->current_picture;
2464
2465     if (!fill_slice(decoder, slice, nalu)) {
2466         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2467         goto error;
2468     }
2469     gst_vaapi_picture_add_slice(
2470         GST_VAAPI_PICTURE_CAST(picture),
2471         GST_VAAPI_SLICE_CAST(slice)
2472     );
2473     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2474
2475 error:
2476     if (slice)
2477         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2478     return status;
2479 }
2480
2481 static inline gint
2482 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2483 {
2484     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2485                                                      0xffffff00, 0x00000100,
2486                                                      ofs, size,
2487                                                      scp);
2488 }
2489
2490 static GstVaapiDecoderStatus
2491 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2492 {
2493     GstVaapiDecoderStatus status;
2494
2495     switch (nalu->type) {
2496     case GST_H264_NAL_SLICE_IDR:
2497         /* fall-through. IDR specifics are handled in init_picture() */
2498     case GST_H264_NAL_SLICE:
2499         status = decode_slice(decoder, nalu);
2500         break;
2501     case GST_H264_NAL_SPS:
2502         status = decode_sps(decoder, nalu);
2503         break;
2504     case GST_H264_NAL_PPS:
2505         status = decode_pps(decoder, nalu);
2506         break;
2507     case GST_H264_NAL_SEI:
2508         status = decode_sei(decoder, nalu);
2509         break;
2510     case GST_H264_NAL_SEQ_END:
2511         status = decode_sequence_end(decoder);
2512         break;
2513     case GST_H264_NAL_AU_DELIMITER:
2514         /* skip all Access Unit NALs */
2515         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2516         break;
2517     case GST_H264_NAL_FILLER_DATA:
2518         /* skip all Filler Data NALs */
2519         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2520         break;
2521     default:
2522         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2523         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2524         break;
2525     }
2526     return status;
2527 }
2528
2529 static GstVaapiDecoderStatus
2530 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2531 {
2532     GstVaapiDecoderH264Private * const priv = decoder->priv;
2533     GstVaapiDecoderStatus status;
2534     GstH264ParserResult result;
2535     GstH264NalUnit nalu;
2536     gboolean is_eos;
2537     const guchar *buf;
2538     guint i, buf_size, nalu_size, size;
2539     guint32 start_code;
2540     gint ofs;
2541
2542     buf      = GST_BUFFER_DATA(buffer);
2543     buf_size = GST_BUFFER_SIZE(buffer);
2544     is_eos   = GST_BUFFER_IS_EOS(buffer);
2545     if (buf && buf_size > 0)
2546         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2547
2548     size = gst_adapter_available(priv->adapter);
2549     do {
2550         if (size == 0) {
2551             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2552             break;
2553         }
2554
2555         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2556         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2557             break;
2558
2559         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2560         if (priv->is_avc) {
2561             if (size < priv->nal_length_size)
2562                 break;
2563             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2564
2565             nalu_size = 0;
2566             for (i = 0; i < priv->nal_length_size; i++)
2567                 nalu_size = (nalu_size << 8) | buf[i];
2568
2569             buf_size = priv->nal_length_size + nalu_size;
2570             if (size < buf_size)
2571                 break;
2572             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2573             size -= buf_size;
2574
2575             buf      = GST_BUFFER_DATA(buffer);
2576             buf_size = GST_BUFFER_SIZE(buffer);
2577
2578             result = gst_h264_parser_identify_nalu_avc(
2579                 priv->parser,
2580                 buf, 0, buf_size, priv->nal_length_size,