h264: fill in GstVaapiPicture structure.
[vaapi:gstreamer-vaapi.git] / gst-libs / gst / vaapi / gstvaapidecoder_h264.c
1 /*
2  *  gstvaapidecoder_h264.c - H.264 decoder
3  *
4  *  Copyright (C) 2011-2012 Intel Corporation
5  *
6  *  This library is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU Lesser General Public License
8  *  as published by the Free Software Foundation; either version 2.1
9  *  of the License, or (at your option) any later version.
10  *
11  *  This library is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  *  Lesser General Public License for more details.
15  *
16  *  You should have received a copy of the GNU Lesser General Public
17  *  License along with this library; if not, write to the Free
18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19  *  Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * SECTION:gstvaapidecoder_h264
24  * @short_description: H.264 decoder
25  */
26
27 #include "sysdeps.h"
28 #include <string.h>
29 #include <gst/base/gstadapter.h>
30 #include <gst/codecparsers/gsth264parser.h>
31 #include "gstvaapidecoder_h264.h"
32 #include "gstvaapidecoder_objects.h"
33 #include "gstvaapidecoder_priv.h"
34 #include "gstvaapidisplay_priv.h"
35 #include "gstvaapiobject_priv.h"
36
37 #define DEBUG 1
38 #include "gstvaapidebug.h"
39
40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
41 #define USE_STRICT_DPB_ORDERING 0
42
43 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
44 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
45 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
46 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
47
48 /* ------------------------------------------------------------------------- */
49 /* --- H.264 Pictures                                                    --- */
50 /* ------------------------------------------------------------------------- */
51
52 #define GST_VAAPI_TYPE_PICTURE_H264 \
53     (gst_vaapi_picture_h264_get_type())
54
55 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
56     ((GstVaapiPictureH264 *)(obj))
57
58 #define GST_VAAPI_PICTURE_H264(obj)                             \
59     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
60                                 GST_VAAPI_TYPE_PICTURE_H264,    \
61                                 GstVaapiPictureH264))
62
63 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
64     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
65                              GST_VAAPI_TYPE_PICTURE_H264,       \
66                              GstVaapiPictureH264Class))
67
68 #define GST_VAAPI_IS_PICTURE_H264(obj) \
69     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
70
71 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
72     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
73
74 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
75     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
76                                GST_VAAPI_TYPE_PICTURE_H264,     \
77                                GstVaapiPictureH264Class))
78
79 struct _GstVaapiPictureH264 {
80     GstVaapiPicture             base;
81     VAPictureH264               info;
82     GstH264PPS                 *pps;
83     gint32                      poc;
84     gint32                      frame_num;              // Original frame_num from slice_header()
85     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
86     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
87     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
88     guint                       is_idr                  : 1;
89     guint                       is_long_term            : 1;
90     guint                       output_flag             : 1;
91     guint                       output_needed           : 1;
92 };
93
94 struct _GstVaapiPictureH264Class {
95     /*< private >*/
96     GstVaapiPictureClass        parent_class;
97 };
98
99 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
100                             gst_vaapi_picture_h264,
101                             GST_VAAPI_TYPE_PICTURE)
102
103 static void
104 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
105 {
106 }
107
108 static gboolean
109 gst_vaapi_picture_h264_create(
110     GstVaapiPictureH264                      *picture,
111     const GstVaapiCodecObjectConstructorArgs *args
112 )
113 {
114     return TRUE;
115 }
116
117 static void
118 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
119 {
120     VAPictureH264 *va_pic;
121
122     va_pic                      = &picture->info;
123     va_pic->flags               = 0;
124     va_pic->TopFieldOrderCnt    = 0;
125     va_pic->BottomFieldOrderCnt = 0;
126
127     picture->poc                = 0;
128     picture->is_long_term       = FALSE;
129     picture->is_idr             = FALSE;
130     picture->output_needed      = FALSE;
131 }
132
133 static inline GstVaapiPictureH264 *
134 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
135 {
136     GstVaapiCodecObject *object;
137
138     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
139
140     object = gst_vaapi_codec_object_new(
141         GST_VAAPI_TYPE_PICTURE_H264,
142         GST_VAAPI_CODEC_BASE(decoder),
143         NULL, sizeof(VAPictureParameterBufferH264),
144         NULL, 0
145     );
146     if (!object)
147         return NULL;
148     return GST_VAAPI_PICTURE_H264_CAST(object);
149 }
150
151 static inline GstVaapiSliceH264 *
152 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
153 {
154     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
155
156     if (G_UNLIKELY(picture->base.slices->len < 1))
157         return NULL;
158     return g_ptr_array_index(picture->base.slices,
159         picture->base.slices->len - 1);
160 }
161
162 /* ------------------------------------------------------------------------- */
163 /* --- Slices                                                            --- */
164 /* ------------------------------------------------------------------------- */
165
166 #define GST_VAAPI_TYPE_SLICE_H264 \
167     (gst_vaapi_slice_h264_get_type())
168
169 #define GST_VAAPI_SLICE_H264_CAST(obj) \
170     ((GstVaapiSliceH264 *)(obj))
171
172 #define GST_VAAPI_SLICE_H264(obj)                               \
173     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
174                                 GST_VAAPI_TYPE_SLICE_H264,      \
175                                 GstVaapiSliceH264))
176
177 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
178     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
179                              GST_VAAPI_TYPE_SLICE_H264,         \
180                              GstVaapiSliceH264Class))
181
182 #define GST_VAAPI_IS_SLICE_H264(obj) \
183     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
184
185 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
186     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
187
188 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
189     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
190                                GST_VAAPI_TYPE_SLICE_H264,       \
191                                GstVaapiSliceH264Class))
192
193 struct _GstVaapiSliceH264 {
194     GstVaapiSlice               base;
195     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
196 };
197
198 struct _GstVaapiSliceH264Class {
199     /*< private >*/
200     GstVaapiSliceClass          parent_class;
201 };
202
203 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
204                             gst_vaapi_slice_h264,
205                             GST_VAAPI_TYPE_SLICE)
206
207 static void
208 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
209 {
210 }
211
212 static gboolean
213 gst_vaapi_slice_h264_create(
214     GstVaapiSliceH264                        *slice,
215     const GstVaapiCodecObjectConstructorArgs *args
216 )
217 {
218     return TRUE;
219 }
220
221 static void
222 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
223 {
224 }
225
226 static inline GstVaapiSliceH264 *
227 gst_vaapi_slice_h264_new(
228     GstVaapiDecoderH264 *decoder,
229     const guint8        *data,
230     guint                data_size
231 )
232 {
233     GstVaapiCodecObject *object;
234
235     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
236
237     object = gst_vaapi_codec_object_new(
238         GST_VAAPI_TYPE_SLICE_H264,
239         GST_VAAPI_CODEC_BASE(decoder),
240         NULL, sizeof(VASliceParameterBufferH264),
241         data, data_size
242     );
243     if (!object)
244         return NULL;
245     return GST_VAAPI_SLICE_H264_CAST(object);
246 }
247
248 /* ------------------------------------------------------------------------- */
249 /* --- H.264 Decoder                                                     --- */
250 /* ------------------------------------------------------------------------- */
251
252 G_DEFINE_TYPE(GstVaapiDecoderH264,
253               gst_vaapi_decoder_h264,
254               GST_VAAPI_TYPE_DECODER)
255
256 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
257     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
258                                  GST_VAAPI_TYPE_DECODER_H264,   \
259                                  GstVaapiDecoderH264Private))
260
261 // Used for field_poc[]
262 #define TOP_FIELD       0
263 #define BOTTOM_FIELD    1
264
265 struct _GstVaapiDecoderH264Private {
266     GstAdapter                 *adapter;
267     GstH264NalParser           *parser;
268     /* Last decoded SPS. May not be the last activated one. Just here because
269        it may not fit stack memory allocation in decode_sps() */
270     GstH264SPS                  last_sps;
271     /* Last decoded PPS. May not be the last activated one. Just here because
272        it may not fit stack memory allocation in decode_pps() */
273     GstH264PPS                  last_pps;
274     GstVaapiPictureH264        *current_picture;
275     GstVaapiPictureH264        *dpb[16];
276     guint                       dpb_count;
277     guint                       dpb_size;
278     GstVaapiProfile             profile;
279     GstVaapiEntrypoint          entrypoint;
280     GstVaapiChromaType          chroma_type;
281     GstVaapiPictureH264        *short_ref[32];
282     guint                       short_ref_count;
283     GstVaapiPictureH264        *long_ref[32];
284     guint                       long_ref_count;
285     GstVaapiPictureH264        *RefPicList0[32];
286     guint                       RefPicList0_count;
287     GstVaapiPictureH264        *RefPicList1[32];
288     guint                       RefPicList1_count;
289     guint                       nal_length_size;
290     guint                       width;
291     guint                       height;
292     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
293     gint32                      poc_msb;                // PicOrderCntMsb
294     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
295     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
296     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
297     gint32                      frame_num_offset;       // FrameNumOffset
298     gint32                      frame_num;              // frame_num (from slice_header())
299     gint32                      prev_frame_num;         // prevFrameNum
300     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
301     gboolean                    prev_pic_structure;     // previous picture structure
302     guint                       is_constructed          : 1;
303     guint                       is_opened               : 1;
304     guint                       is_avc                  : 1;
305     guint                       has_context             : 1;
306 };
307
308 static gboolean
309 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
310
311 static void
312 clear_references(
313     GstVaapiDecoderH264  *decoder,
314     GstVaapiPictureH264 **pictures,
315     guint                *picture_count
316 );
317
318 /* Get number of reference frames to use */
319 static guint
320 get_max_dec_frame_buffering(GstH264SPS *sps)
321 {
322     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
323
324     /* Table A-1 - Level limits */
325     switch (sps->level_idc) {
326     case 10: MaxDpbMbs = 396;    break;
327     case 11: MaxDpbMbs = 900;    break;
328     case 12: MaxDpbMbs = 2376;   break;
329     case 13: MaxDpbMbs = 2376;   break;
330     case 20: MaxDpbMbs = 2376;   break;
331     case 21: MaxDpbMbs = 4752;   break;
332     case 22: MaxDpbMbs = 8100;   break;
333     case 30: MaxDpbMbs = 8100;   break;
334     case 31: MaxDpbMbs = 18000;  break;
335     case 32: MaxDpbMbs = 20480;  break;
336     case 40: MaxDpbMbs = 32768;  break;
337     case 41: MaxDpbMbs = 32768;  break;
338     case 42: MaxDpbMbs = 34816;  break;
339     case 50: MaxDpbMbs = 110400; break;
340     case 51: MaxDpbMbs = 184320; break;
341     default:
342         g_assert(0 && "unhandled level");
343         break;
344     }
345
346     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
347                   (sps->pic_height_in_map_units_minus1 + 1) *
348                   (sps->frame_mbs_only_flag ? 1 : 2));
349     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
350
351     /* VUI parameters */
352     if (sps->vui_parameters_present_flag) {
353         GstH264VUIParams * const vui_params = &sps->vui_parameters;
354         if (vui_params->bitstream_restriction_flag)
355             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
356         else {
357             switch (sps->profile_idc) {
358             case 44:  // CAVLC 4:4:4 Intra profile
359             case 86:  // Scalable High profile
360             case 100: // High profile
361             case 110: // High 10 profile
362             case 122: // High 4:2:2 profile
363             case 244: // High 4:4:4 Predictive profile
364                 if (sps->constraint_set3_flag)
365                     max_dec_frame_buffering = 0;
366                 break;
367             }
368         }
369     }
370
371     if (max_dec_frame_buffering > 16)
372         max_dec_frame_buffering = 16;
373     else if (max_dec_frame_buffering < sps->num_ref_frames)
374         max_dec_frame_buffering = sps->num_ref_frames;
375     return MAX(1, max_dec_frame_buffering);
376 }
377
378 static void
379 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
380 {
381     GstVaapiDecoderH264Private * const priv = decoder->priv;
382     guint i, num_pictures = --priv->dpb_count;
383
384     if (USE_STRICT_DPB_ORDERING) {
385         for (i = index; i < num_pictures; i++)
386             gst_vaapi_picture_replace(&priv->dpb[i], priv->dpb[i + 1]);
387     }
388     else if (index != num_pictures)
389         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
390     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
391 }
392
393 static inline gboolean
394 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
395 {
396     /* XXX: update cropping rectangle */
397     picture->output_needed = FALSE;
398     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
399 }
400
401 static gboolean
402 dpb_bump(GstVaapiDecoderH264 *decoder)
403 {
404     GstVaapiDecoderH264Private * const priv = decoder->priv;
405     guint i, lowest_poc_index;
406     gboolean success;
407
408     for (i = 0; i < priv->dpb_count; i++) {
409         if (priv->dpb[i]->output_needed)
410             break;
411     }
412     if (i == priv->dpb_count)
413         return FALSE;
414
415     lowest_poc_index = i++;
416     for (; i < priv->dpb_count; i++) {
417         GstVaapiPictureH264 * const picture = priv->dpb[i];
418         if (picture->output_needed && picture->poc < priv->dpb[lowest_poc_index]->poc)
419             lowest_poc_index = i;
420     }
421
422     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
423     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
424         dpb_remove_index(decoder, lowest_poc_index);
425     return success;
426 }
427
428 static void
429 dpb_flush(GstVaapiDecoderH264 *decoder)
430 {
431     GstVaapiDecoderH264Private * const priv = decoder->priv;
432
433     while (dpb_bump(decoder))
434         ;
435     clear_references(decoder, priv->dpb, &priv->dpb_count);
436 }
437
438 static gboolean
439 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
440 {
441     GstVaapiDecoderH264Private * const priv = decoder->priv;
442     guint i;
443
444     // Remove all unused pictures
445     if (picture->is_idr)
446         dpb_flush(decoder);
447     else {
448         i = 0;
449         while (i < priv->dpb_count) {
450             GstVaapiPictureH264 * const picture = priv->dpb[i];
451             if (!picture->output_needed &&
452                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
453                 dpb_remove_index(decoder, i);
454             else
455                 i++;
456         }
457     }
458
459     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
460     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
461         while (priv->dpb_count == priv->dpb_size) {
462             if (!dpb_bump(decoder))
463                 return FALSE;
464         }
465         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
466         if (picture->output_flag)
467             picture->output_needed = TRUE;
468     }
469
470     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
471     else {
472         if (!picture->output_flag)
473             return TRUE;
474         while (priv->dpb_count == priv->dpb_size) {
475             for (i = 0; i < priv->dpb_count; i++) {
476                 if (priv->dpb[i]->output_needed &&
477                     priv->dpb[i]->poc < picture->poc)
478                     break;
479             }
480             if (i == priv->dpb_count)
481                 return dpb_output(decoder, picture);
482             if (!dpb_bump(decoder))
483                 return FALSE;
484         }
485         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
486         picture->output_needed = TRUE;
487     }
488     return TRUE;
489 }
490
491 static inline void
492 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
493 {
494     GstVaapiDecoderH264Private * const priv = decoder->priv;
495
496     priv->dpb_size = get_max_dec_frame_buffering(sps);
497     GST_DEBUG("DPB size %u", priv->dpb_size);
498 }
499
500 static GstVaapiDecoderStatus
501 get_status(GstH264ParserResult result)
502 {
503     GstVaapiDecoderStatus status;
504
505     switch (result) {
506     case GST_H264_PARSER_OK:
507         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
508         break;
509     case GST_H264_PARSER_NO_NAL_END:
510         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
511         break;
512     case GST_H264_PARSER_ERROR:
513         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
514         break;
515     default:
516         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
517         break;
518     }
519     return status;
520 }
521
522 static void
523 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
524 {
525     GstVaapiDecoderH264Private * const priv = decoder->priv;
526
527     gst_vaapi_picture_replace(&priv->current_picture, NULL);
528     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
529     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
530     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
531
532     if (priv->parser) {
533         gst_h264_nal_parser_free(priv->parser);
534         priv->parser = NULL;
535     }
536
537     if (priv->adapter) {
538         gst_adapter_clear(priv->adapter);
539         g_object_unref(priv->adapter);
540         priv->adapter = NULL;
541     }
542 }
543
544 static gboolean
545 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
546 {
547     GstVaapiDecoderH264Private * const priv = decoder->priv;
548
549     gst_vaapi_decoder_h264_close(decoder);
550
551     priv->adapter = gst_adapter_new();
552     if (!priv->adapter)
553         return FALSE;
554
555     priv->parser = gst_h264_nal_parser_new();
556     if (!priv->parser)
557         return FALSE;
558     return TRUE;
559 }
560
561 static void
562 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
563 {
564     gst_vaapi_decoder_h264_close(decoder);
565 }
566
567 static gboolean
568 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
569 {
570     if (!GST_VAAPI_DECODER_CODEC(decoder))
571         return FALSE;
572     return TRUE;
573 }
574
575 static guint
576 h264_get_profile(GstH264SPS *sps)
577 {
578     guint profile = 0;
579
580     switch (sps->profile_idc) {
581     case 66:
582         profile = GST_VAAPI_PROFILE_H264_BASELINE;
583         break;
584     case 77:
585         profile = GST_VAAPI_PROFILE_H264_MAIN;
586         break;
587     case 100:
588         profile = GST_VAAPI_PROFILE_H264_HIGH;
589         break;
590     }
591     return profile;
592 }
593
594 static guint
595 h264_get_chroma_type(GstH264SPS *sps)
596 {
597     guint chroma_type = 0;
598
599     switch (sps->chroma_format_idc) {
600     case 1:
601         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
602         break;
603     case 2:
604         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
605         break;
606     case 3:
607         if (!sps->separate_colour_plane_flag)
608             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
609         break;
610     }
611     return chroma_type;
612 }
613
614 static GstVaapiProfile
615 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
616 {
617     GstVaapiDecoderH264Private * const priv = decoder->priv;
618     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
619     GstVaapiProfile profile, profiles[2];
620     guint i, n_profiles = 0;
621
622     profile = h264_get_profile(sps);
623     if (!profile)
624         return GST_VAAPI_PROFILE_UNKNOWN;
625
626     profiles[n_profiles++] = profile;
627     switch (profile) {
628     case GST_VAAPI_PROFILE_H264_MAIN:
629         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
630         break;
631     default:
632         break;
633     }
634
635     /* If the preferred profile (profiles[0]) matches one that we already
636        found, then just return it now instead of searching for it again */
637     if (profiles[0] == priv->profile)
638         return priv->profile;
639
640     for (i = 0; i < n_profiles; i++) {
641         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
642             return profiles[i];
643     }
644     return GST_VAAPI_PROFILE_UNKNOWN;
645 }
646
647 static GstVaapiDecoderStatus
648 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
649 {
650     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
651     GstVaapiDecoderH264Private * const priv = decoder->priv;
652     GstVaapiContextInfo info;
653     GstVaapiProfile profile;
654     GstVaapiChromaType chroma_type;
655     gboolean reset_context = FALSE;
656
657     profile = get_profile(decoder, sps);
658     if (!profile) {
659         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
660         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
661     }
662
663     if (priv->profile != profile) {
664         GST_DEBUG("profile changed");
665         reset_context = TRUE;
666         priv->profile = profile;
667     }
668
669     chroma_type = h264_get_chroma_type(sps);
670     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
671         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
672         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
673     }
674
675     if (priv->chroma_type != chroma_type) {
676         GST_DEBUG("chroma format changed");
677         reset_context     = TRUE;
678         priv->chroma_type = chroma_type;
679     }
680
681     if (priv->width != sps->width || priv->height != sps->height) {
682         GST_DEBUG("size changed");
683         reset_context = TRUE;
684         priv->width   = sps->width;
685         priv->height  = sps->height;
686     }
687
688     gst_vaapi_decoder_set_pixel_aspect_ratio(
689         base_decoder,
690         sps->vui_parameters.par_n,
691         sps->vui_parameters.par_d
692     );
693
694     if (!reset_context && priv->has_context)
695         return GST_VAAPI_DECODER_STATUS_SUCCESS;
696
697     info.profile    = priv->profile;
698     info.entrypoint = priv->entrypoint;
699     info.width      = priv->width;
700     info.height     = priv->height;
701     info.ref_frames = get_max_dec_frame_buffering(sps);
702
703     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
704         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
705     priv->has_context = TRUE;
706
707     /* Reset DPB */
708     dpb_reset(decoder, sps);
709     return GST_VAAPI_DECODER_STATUS_SUCCESS;
710 }
711
712 static void
713 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
714 {
715     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
716     guint i, j;
717
718     /* There are always 6 4x4 scaling lists */
719     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
720     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
721
722     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
723         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
724                sizeof(iq_matrix->ScalingList4x4));
725     else {
726         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
727             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
728                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
729         }
730     }
731 }
732
733 static void
734 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
735 {
736     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
737     const GstH264SPS * const sps = pps->sequence;
738     guint i, j, n;
739
740     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
741     if (!pps->transform_8x8_mode_flag)
742         return;
743
744     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
745     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
746
747     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
748         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
749                sizeof(iq_matrix->ScalingList8x8));
750     else {
751         n = (sps->chroma_format_idc != 3) ? 2 : 6;
752         for (i = 0; i < n; i++) {
753             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
754                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
755         }
756     }
757 }
758
759 static GstVaapiDecoderStatus
760 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
761 {
762     GstVaapiPicture * const base_picture = &picture->base;
763     GstH264PPS * const pps = picture->pps;
764     GstH264SPS * const sps = pps->sequence;
765     VAIQMatrixBufferH264 *iq_matrix;
766
767     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
768     if (!base_picture->iq_matrix) {
769         GST_ERROR("failed to allocate IQ matrix");
770         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
771     }
772     iq_matrix = base_picture->iq_matrix->param;
773
774     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
775        is not large enough to hold lists for 4:4:4 */
776     if (sps->chroma_format_idc == 3)
777         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
778
779     fill_iq_matrix_4x4(iq_matrix, pps);
780     fill_iq_matrix_8x8(iq_matrix, pps);
781
782     return GST_VAAPI_DECODER_STATUS_SUCCESS;
783 }
784
785 static GstVaapiDecoderStatus
786 decode_current_picture(GstVaapiDecoderH264 *decoder)
787 {
788     GstVaapiDecoderH264Private * const priv = decoder->priv;
789     GstVaapiPictureH264 * const picture = priv->current_picture;
790     GstVaapiDecoderStatus status;
791
792     if (!picture)
793         return GST_VAAPI_DECODER_STATUS_SUCCESS;
794
795     status = ensure_context(decoder, picture->pps->sequence);
796     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
797         return status;
798
799     if (!decode_picture_end(decoder, picture))
800         goto error;
801     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
802         goto error;
803     gst_vaapi_picture_replace(&priv->current_picture, NULL);
804     return GST_VAAPI_DECODER_STATUS_SUCCESS;
805
806 error:
807     gst_vaapi_picture_replace(&priv->current_picture, NULL);
808     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
809 }
810
811 static GstVaapiDecoderStatus
812 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
813 {
814     GstVaapiDecoderH264Private * const priv = decoder->priv;
815     GstH264SPS * const sps = &priv->last_sps;
816     GstH264ParserResult result;
817
818     GST_DEBUG("decode SPS");
819
820     memset(sps, 0, sizeof(*sps));
821     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
822     if (result != GST_H264_PARSER_OK)
823         return get_status(result);
824
825     return GST_VAAPI_DECODER_STATUS_SUCCESS;
826 }
827
828 static GstVaapiDecoderStatus
829 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
830 {
831     GstVaapiDecoderH264Private * const priv = decoder->priv;
832     GstH264PPS * const pps = &priv->last_pps;
833     GstH264ParserResult result;
834
835     GST_DEBUG("decode PPS");
836
837     memset(pps, 0, sizeof(*pps));
838     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
839     if (result != GST_H264_PARSER_OK)
840         return get_status(result);
841
842     return GST_VAAPI_DECODER_STATUS_SUCCESS;
843 }
844
845 static GstVaapiDecoderStatus
846 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
847 {
848     GstVaapiDecoderH264Private * const priv = decoder->priv;
849     GstH264SEIMessage sei;
850     GstH264ParserResult result;
851
852     GST_DEBUG("decode SEI");
853
854     memset(&sei, 0, sizeof(sei));
855     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
856     if (result != GST_H264_PARSER_OK) {
857         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
858         return get_status(result);
859     }
860
861     return GST_VAAPI_DECODER_STATUS_SUCCESS;
862 }
863
864 static GstVaapiDecoderStatus
865 decode_sequence_end(GstVaapiDecoderH264 *decoder)
866 {
867     GstVaapiDecoderStatus status;
868
869     GST_DEBUG("decode sequence-end");
870
871     status = decode_current_picture(decoder);
872     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
873         return status;
874
875     dpb_flush(decoder);
876     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
877 }
878
879 /* 8.2.1.1 - Decoding process for picture order count type 0 */
880 static void
881 init_picture_poc_0(
882     GstVaapiDecoderH264 *decoder,
883     GstVaapiPictureH264 *picture,
884     GstH264SliceHdr     *slice_hdr
885 )
886 {
887     GstVaapiDecoderH264Private * const priv = decoder->priv;
888     GstH264PPS * const pps = slice_hdr->pps;
889     GstH264SPS * const sps = pps->sequence;
890     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
891     gint32 temp_poc;
892
893     GST_DEBUG("decode picture order count type 0");
894
895     if (picture->is_idr) {
896         priv->prev_poc_msb = 0;
897         priv->prev_poc_lsb = 0;
898     }
899     else if (priv->prev_pic_has_mmco5) {
900         priv->prev_poc_msb = 0;
901         priv->prev_poc_lsb =
902             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
903              0 : priv->field_poc[TOP_FIELD]);
904     }
905     else {
906         priv->prev_poc_msb = priv->poc_msb;
907         priv->prev_poc_lsb = priv->poc_lsb;
908     }
909
910     // (8-3)
911     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
912     if (priv->poc_lsb < priv->prev_poc_lsb &&
913         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
914         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
915     else if (priv->poc_lsb > priv->prev_poc_lsb &&
916              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
917         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
918     else
919         priv->poc_msb = priv->prev_poc_msb;
920
921     temp_poc = priv->poc_msb + priv->poc_lsb;
922     switch (picture->base.structure) {
923     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
924         // (8-4, 8-5)
925         priv->field_poc[TOP_FIELD] = temp_poc;
926         priv->field_poc[BOTTOM_FIELD] = temp_poc +
927             slice_hdr->delta_pic_order_cnt_bottom;
928         break;
929     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
930         // (8-4)
931         priv->field_poc[TOP_FIELD] = temp_poc;
932         break;
933     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
934         // (8-5)
935         priv->field_poc[BOTTOM_FIELD] = temp_poc;
936         break;
937     }
938 }
939
940 /* 8.2.1.2 - Decoding process for picture order count type 1 */
941 static void
942 init_picture_poc_1(
943     GstVaapiDecoderH264 *decoder,
944     GstVaapiPictureH264 *picture,
945     GstH264SliceHdr     *slice_hdr
946 )
947 {
948     GstVaapiDecoderH264Private * const priv = decoder->priv;
949     GstH264PPS * const pps = slice_hdr->pps;
950     GstH264SPS * const sps = pps->sequence;
951     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
952     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
953     guint i;
954
955     GST_DEBUG("decode picture order count type 1");
956
957     if (priv->prev_pic_has_mmco5)
958         prev_frame_num_offset = 0;
959     else
960         prev_frame_num_offset = priv->frame_num_offset;
961
962     // (8-6)
963     if (picture->is_idr)
964         priv->frame_num_offset = 0;
965     else if (priv->prev_frame_num > priv->frame_num)
966         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
967     else
968         priv->frame_num_offset = prev_frame_num_offset;
969
970     // (8-7)
971     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
972         abs_frame_num = priv->frame_num_offset + priv->frame_num;
973     else
974         abs_frame_num = 0;
975     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
976         abs_frame_num = abs_frame_num - 1;
977
978     if (abs_frame_num > 0) {
979         gint32 expected_delta_per_poc_cycle;
980         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
981
982         expected_delta_per_poc_cycle = 0;
983         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
984             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
985
986         // (8-8)
987         poc_cycle_cnt = (abs_frame_num - 1) /
988             sps->num_ref_frames_in_pic_order_cnt_cycle;
989         frame_num_in_poc_cycle = (abs_frame_num - 1) %
990             sps->num_ref_frames_in_pic_order_cnt_cycle;
991
992         // (8-9)
993         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
994         for (i = 0; i <= frame_num_in_poc_cycle; i++)
995             expected_poc += sps->offset_for_ref_frame[i];
996     }
997     else
998         expected_poc = 0;
999     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1000         expected_poc += sps->offset_for_non_ref_pic;
1001
1002     // (8-10)
1003     switch (picture->base.structure) {
1004     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1005         priv->field_poc[TOP_FIELD] = expected_poc +
1006             slice_hdr->delta_pic_order_cnt[0];
1007         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1008             sps->offset_for_top_to_bottom_field +
1009             slice_hdr->delta_pic_order_cnt[1];
1010         break;
1011     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1012         priv->field_poc[TOP_FIELD] = expected_poc +
1013             slice_hdr->delta_pic_order_cnt[0];
1014         break;
1015     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1016         priv->field_poc[BOTTOM_FIELD] = expected_poc + 
1017             sps->offset_for_top_to_bottom_field +
1018             slice_hdr->delta_pic_order_cnt[0];
1019         break;
1020     }
1021 }
1022
1023 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1024 static void
1025 init_picture_poc_2(
1026     GstVaapiDecoderH264 *decoder,
1027     GstVaapiPictureH264 *picture,
1028     GstH264SliceHdr     *slice_hdr
1029 )
1030 {
1031     GstVaapiDecoderH264Private * const priv = decoder->priv;
1032     GstH264PPS * const pps = slice_hdr->pps;
1033     GstH264SPS * const sps = pps->sequence;
1034     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1035     gint32 prev_frame_num_offset, temp_poc;
1036
1037     GST_DEBUG("decode picture order count type 2");
1038
1039     if (priv->prev_pic_has_mmco5)
1040         prev_frame_num_offset = 0;
1041     else
1042         prev_frame_num_offset = priv->frame_num_offset;
1043
1044     // (8-11)
1045     if (picture->is_idr)
1046         priv->frame_num_offset = 0;
1047     else if (priv->prev_frame_num > priv->frame_num)
1048         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1049     else
1050         priv->frame_num_offset = prev_frame_num_offset;
1051
1052     // (8-12)
1053     if (picture->is_idr)
1054         temp_poc = 0;
1055     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1056         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1057     else
1058         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1059
1060     // (8-13)
1061     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1062         priv->field_poc[TOP_FIELD] = temp_poc;
1063     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1064         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1065 }
1066
1067 /* 8.2.1 - Decoding process for picture order count */
1068 static void
1069 init_picture_poc(
1070     GstVaapiDecoderH264 *decoder,
1071     GstVaapiPictureH264 *picture,
1072     GstH264SliceHdr     *slice_hdr
1073 )
1074 {
1075     GstVaapiDecoderH264Private * const priv = decoder->priv;
1076     VAPictureH264 * const pic = &picture->info;
1077     GstH264PPS * const pps = slice_hdr->pps;
1078     GstH264SPS * const sps = pps->sequence;
1079
1080     switch (sps->pic_order_cnt_type) {
1081     case 0:
1082         init_picture_poc_0(decoder, picture, slice_hdr);
1083         break;
1084     case 1:
1085         init_picture_poc_1(decoder, picture, slice_hdr);
1086         break;
1087     case 2:
1088         init_picture_poc_2(decoder, picture, slice_hdr);
1089         break;
1090     }
1091
1092     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1093         pic->TopFieldOrderCnt = priv->field_poc[TOP_FIELD];
1094     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1095         pic->BottomFieldOrderCnt = priv->field_poc[BOTTOM_FIELD];
1096     picture->poc = MIN(pic->TopFieldOrderCnt, pic->BottomFieldOrderCnt);
1097 }
1098
1099 static int
1100 compare_picture_pic_num_dec(const void *a, const void *b)
1101 {
1102     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1103     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1104
1105     return picB->pic_num - picA->pic_num;
1106 }
1107
1108 static int
1109 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1110 {
1111     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1112     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1113
1114     return picA->long_term_pic_num - picB->long_term_pic_num;
1115 }
1116
1117 static int
1118 compare_picture_poc_dec(const void *a, const void *b)
1119 {
1120     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1121     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1122
1123     return picB->poc - picA->poc;
1124 }
1125
1126 static int
1127 compare_picture_poc_inc(const void *a, const void *b)
1128 {
1129     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1130     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1131
1132     return picA->poc - picB->poc;
1133 }
1134
1135 static int
1136 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1137 {
1138     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1139     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1140
1141     return picB->frame_num_wrap - picA->frame_num_wrap;
1142 }
1143
1144 static int
1145 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1146 {
1147     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1148     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1149
1150     return picA->info.frame_idx - picB->info.frame_idx;
1151 }
1152
1153 /* 8.2.4.1 - Decoding process for picture numbers */
1154 static void
1155 init_picture_refs_pic_num(
1156     GstVaapiDecoderH264 *decoder,
1157     GstVaapiPictureH264 *picture,
1158     GstH264SliceHdr     *slice_hdr
1159 )
1160 {
1161     GstVaapiDecoderH264Private * const priv = decoder->priv;
1162     GstH264PPS * const pps = slice_hdr->pps;
1163     GstH264SPS * const sps = pps->sequence;
1164     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1165     guint i;
1166
1167     GST_DEBUG("decode picture numbers");
1168
1169     for (i = 0; i < priv->short_ref_count; i++) {
1170         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1171
1172         // (8-27)
1173         if (pic->frame_num > priv->frame_num)
1174             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1175         else
1176             pic->frame_num_wrap = pic->frame_num;
1177
1178         // (8-28, 8-30, 8-31)
1179         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1180             pic->pic_num = pic->frame_num_wrap;
1181         else {
1182             if (pic->base.structure == picture->base.structure)
1183                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1184             else
1185                 pic->pic_num = 2 * pic->frame_num_wrap;
1186         }
1187     }
1188
1189     for (i = 0; i < priv->long_ref_count; i++) {
1190         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1191
1192         // (8-29, 8-32, 8-33)
1193         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1194             pic->long_term_pic_num = pic->info.frame_idx;
1195         else {
1196             if (pic->base.structure == picture->base.structure)
1197                 pic->long_term_pic_num = 2 * pic->info.frame_idx + 1;
1198             else
1199                 pic->long_term_pic_num = 2 * pic->info.frame_idx;
1200         }
1201     }
1202 }
1203
1204 #define SORT_REF_LIST(list, n, compare_func) \
1205     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1206
1207 static void
1208 init_picture_refs_p_slice(
1209     GstVaapiDecoderH264 *decoder,
1210     GstVaapiPictureH264 *picture,
1211     GstH264SliceHdr     *slice_hdr
1212 )
1213 {
1214     GstVaapiDecoderH264Private * const priv = decoder->priv;
1215     GstVaapiPictureH264 **ref_list;
1216     guint i;
1217
1218     GST_DEBUG("decode reference picture list for P and SP slices");
1219
1220     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1221         /* 8.2.4.2.1 - P and SP slices in frames */
1222         if (priv->short_ref_count > 0) {
1223             ref_list = priv->RefPicList0;
1224             for (i = 0; i < priv->short_ref_count; i++)
1225                 ref_list[i] = priv->short_ref[i];
1226             SORT_REF_LIST(ref_list, i, pic_num_dec);
1227             priv->RefPicList0_count += i;
1228         }
1229
1230         if (priv->long_ref_count > 0) {
1231             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1232             for (i = 0; i < priv->long_ref_count; i++)
1233                 ref_list[i] = priv->long_ref[i];
1234             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1235             priv->RefPicList0_count += i;
1236         }
1237     }
1238     else {
1239         /* 8.2.4.2.2 - P and SP slices in fields */
1240         GstVaapiPictureH264 *short_ref[32];
1241         guint short_ref_count = 0;
1242         GstVaapiPictureH264 *long_ref[32];
1243         guint long_ref_count = 0;
1244
1245         // XXX: handle second field if current field is marked as
1246         // "used for short-term reference"
1247         if (priv->short_ref_count > 0) {
1248             for (i = 0; i < priv->short_ref_count; i++)
1249                 short_ref[i] = priv->short_ref[i];
1250             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1251             short_ref_count = i;
1252         }
1253
1254         // XXX: handle second field if current field is marked as
1255         // "used for long-term reference"
1256         if (priv->long_ref_count > 0) {
1257             for (i = 0; i < priv->long_ref_count; i++)
1258                 long_ref[i] = priv->long_ref[i];
1259             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1260             long_ref_count = i;
1261         }
1262
1263         // XXX: handle 8.2.4.2.5
1264     }
1265 }
1266
1267 static void
1268 init_picture_refs_b_slice(
1269     GstVaapiDecoderH264 *decoder,
1270     GstVaapiPictureH264 *picture,
1271     GstH264SliceHdr     *slice_hdr
1272 )
1273 {
1274     GstVaapiDecoderH264Private * const priv = decoder->priv;
1275     GstVaapiPictureH264 **ref_list;
1276     guint i, n;
1277
1278     GST_DEBUG("decode reference picture list for B slices");
1279
1280     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1281         /* 8.2.4.2.3 - B slices in frames */
1282
1283         /* RefPicList0 */
1284         if (priv->short_ref_count > 0) {
1285             // 1. Short-term references
1286             ref_list = priv->RefPicList0;
1287             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1288                 if (priv->short_ref[i]->poc < picture->poc)
1289                     ref_list[n++] = priv->short_ref[i];
1290             }
1291             SORT_REF_LIST(ref_list, n, poc_dec);
1292             priv->RefPicList0_count += n;
1293
1294             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1295             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1296                 if (priv->short_ref[i]->poc >= picture->poc)
1297                     ref_list[n++] = priv->short_ref[i];
1298             }
1299             SORT_REF_LIST(ref_list, n, poc_inc);
1300             priv->RefPicList0_count += n;
1301         }
1302
1303         if (priv->long_ref_count > 0) {
1304             // 2. Long-term references
1305             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1306             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1307                 ref_list[n++] = priv->long_ref[i];
1308             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1309             priv->RefPicList0_count += n;
1310         }
1311
1312         /* RefPicList1 */
1313         if (priv->short_ref_count > 0) {
1314             // 1. Short-term references
1315             ref_list = priv->RefPicList1;
1316             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1317                 if (priv->short_ref[i]->poc > picture->poc)
1318                     ref_list[n++] = priv->short_ref[i];
1319             }
1320             SORT_REF_LIST(ref_list, n, poc_inc);
1321             priv->RefPicList1_count += n;
1322
1323             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1324             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1325                 if (priv->short_ref[i]->poc <= picture->poc)
1326                     ref_list[n++] = priv->short_ref[i];
1327             }
1328             SORT_REF_LIST(ref_list, n, poc_dec);
1329             priv->RefPicList1_count += n;
1330         }
1331
1332         if (priv->long_ref_count > 0) {
1333             // 2. Long-term references
1334             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1335             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1336                 ref_list[n++] = priv->long_ref[i];
1337             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1338             priv->RefPicList1_count += n;
1339         }
1340     }
1341     else {
1342         /* 8.2.4.2.4 - B slices in fields */
1343         GstVaapiPictureH264 *short_ref0[32];
1344         guint short_ref0_count = 0;
1345         GstVaapiPictureH264 *short_ref1[32];
1346         guint short_ref1_count = 0;
1347         GstVaapiPictureH264 *long_ref[32];
1348         guint long_ref_count = 0;
1349
1350         /* refFrameList0ShortTerm */
1351         if (priv->short_ref_count > 0) {
1352             ref_list = short_ref0;
1353             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1354                 if (priv->short_ref[i]->poc <= picture->poc)
1355                     ref_list[n++] = priv->short_ref[i];
1356             }
1357             SORT_REF_LIST(ref_list, n, poc_dec);
1358             short_ref0_count += n;
1359
1360             ref_list = &short_ref0[short_ref0_count];
1361             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1362                 if (priv->short_ref[i]->poc > picture->poc)
1363                     ref_list[n++] = priv->short_ref[i];
1364             }
1365             SORT_REF_LIST(ref_list, n, poc_inc);
1366             short_ref0_count += n;
1367         }
1368
1369         /* refFrameList1ShortTerm */
1370         if (priv->short_ref_count > 0) {
1371             ref_list = short_ref1;
1372             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1373                 if (priv->short_ref[i]->poc > picture->poc)
1374                     ref_list[n++] = priv->short_ref[i];
1375             }
1376             SORT_REF_LIST(ref_list, n, poc_inc);
1377             short_ref1_count += n;
1378
1379             ref_list = &short_ref1[short_ref1_count];
1380             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1381                 if (priv->short_ref[i]->poc <= picture->poc)
1382                     ref_list[n++] = priv->short_ref[i];
1383             }
1384             SORT_REF_LIST(ref_list, n, poc_dec);
1385             short_ref1_count += n;
1386         }
1387
1388         /* refFrameListLongTerm */
1389         if (priv->long_ref_count > 0) {
1390             for (i = 0; i < priv->long_ref_count; i++)
1391                 long_ref[i] = priv->long_ref[i];
1392             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1393             long_ref_count = i;
1394         }
1395
1396         // XXX: handle 8.2.4.2.5
1397     }
1398
1399     /* Check whether RefPicList1 is identical to RefPicList0, then
1400        swap if necessary */
1401     if (priv->RefPicList1_count > 1 &&
1402         priv->RefPicList1_count == priv->RefPicList0_count &&
1403         memcmp(priv->RefPicList0, priv->RefPicList1,
1404                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1405         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1406         priv->RefPicList1[0] = priv->RefPicList1[1];
1407         priv->RefPicList1[1] = tmp;
1408     }
1409 }
1410
1411 #undef SORT_REF_LIST
1412
1413 static void
1414 clear_references(
1415     GstVaapiDecoderH264  *decoder,
1416     GstVaapiPictureH264 **pictures,
1417     guint                *picture_count
1418 )
1419 {
1420     const guint num_pictures = *picture_count;
1421     guint i;
1422
1423     for (i = 0; i < num_pictures; i++)
1424         gst_vaapi_picture_replace(&pictures[i], NULL);
1425     *picture_count = 0;
1426 }
1427
1428 static gboolean
1429 remove_reference_at(
1430     GstVaapiDecoderH264  *decoder,
1431     GstVaapiPictureH264 **pictures,
1432     guint                *picture_count,
1433     guint                 index
1434 )
1435 {
1436     guint num_pictures = *picture_count;
1437     GstVaapiPictureH264 *picture;
1438
1439     g_return_val_if_fail(index < num_pictures, FALSE);
1440
1441     picture = pictures[index];
1442     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1443     picture->is_long_term = FALSE;
1444     picture->info.flags &= ~(VA_PICTURE_H264_SHORT_TERM_REFERENCE |
1445                              VA_PICTURE_H264_LONG_TERM_REFERENCE);
1446
1447     if (index != --num_pictures)
1448         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1449     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1450     *picture_count = num_pictures;
1451     return TRUE;
1452 }
1453
1454 static gint
1455 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1456 {
1457     GstVaapiDecoderH264Private * const priv = decoder->priv;
1458     guint i;
1459
1460     for (i = 0; i < priv->short_ref_count; i++) {
1461         if (priv->short_ref[i]->pic_num == pic_num)
1462             return i;
1463     }
1464     GST_ERROR("found no short-term reference picture with PicNum = %d",
1465               pic_num);
1466     return -1;
1467 }
1468
1469 static gint
1470 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1471 {
1472     GstVaapiDecoderH264Private * const priv = decoder->priv;
1473     guint i;
1474
1475     for (i = 0; i < priv->long_ref_count; i++) {
1476         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1477             return i;
1478     }
1479     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1480               long_term_pic_num);
1481     return -1;
1482 }
1483
1484 static void
1485 exec_picture_refs_modification_1(
1486     GstVaapiDecoderH264           *decoder,
1487     GstVaapiPictureH264           *picture,
1488     GstH264SliceHdr               *slice_hdr,
1489     guint                          list
1490 )
1491 {
1492     GstVaapiDecoderH264Private * const priv = decoder->priv;
1493     GstH264PPS * const pps = slice_hdr->pps;
1494     GstH264SPS * const sps = pps->sequence;
1495     GstH264RefPicListModification *ref_pic_list_modification;
1496     guint num_ref_pic_list_modifications;
1497     GstVaapiPictureH264 **ref_list;
1498     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1499     guint i, j, n, num_refs;
1500     gint found_ref_idx;
1501     gint32 MaxPicNum, CurrPicNum, picNumPred;
1502
1503     GST_DEBUG("modification process of reference picture list %u", list);
1504
1505     if (list == 0) {
1506         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1507         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1508         ref_list                       = priv->RefPicList0;
1509         ref_list_count_ptr             = &priv->RefPicList0_count;
1510         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1511     }
1512     else {
1513         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1514         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1515         ref_list                       = priv->RefPicList1;
1516         ref_list_count_ptr             = &priv->RefPicList1_count;
1517         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1518     }
1519     ref_list_count = *ref_list_count_ptr;
1520
1521     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1522         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1523         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1524     }
1525     else {
1526         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1527         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1528     }
1529
1530     picNumPred = CurrPicNum;
1531
1532     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1533         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1534         if (l->modification_of_pic_nums_idc == 3)
1535             break;
1536
1537         /* 8.2.4.3.1 - Short-term reference pictures */
1538         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1539             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1540             gint32 picNum, picNumNoWrap;
1541
1542             // (8-34)
1543             if (l->modification_of_pic_nums_idc == 0) {
1544                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1545                 if (picNumNoWrap < 0)
1546                     picNumNoWrap += MaxPicNum;
1547             }
1548
1549             // (8-35)
1550             else {
1551                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1552                 if (picNumNoWrap >= MaxPicNum)
1553                     picNumNoWrap -= MaxPicNum;
1554             }
1555             picNumPred = picNumNoWrap;
1556
1557             // (8-36)
1558             picNum = picNumNoWrap;
1559             if (picNum > CurrPicNum)
1560                 picNum -= MaxPicNum;
1561
1562             // (8-37)
1563             for (j = num_refs; j > ref_list_idx; j--)
1564                 ref_list[j] = ref_list[j - 1];
1565             found_ref_idx = find_short_term_reference(decoder, picNum);
1566             ref_list[ref_list_idx++] =
1567                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1568             n = ref_list_idx;
1569             for (j = ref_list_idx; j <= num_refs; j++) {
1570                 gint32 PicNumF;
1571                 if (!ref_list[j])
1572                     continue;
1573                 PicNumF = ref_list[j]->is_long_term ?
1574                     MaxPicNum : ref_list[j]->pic_num;
1575                 if (PicNumF != picNum)
1576                     ref_list[n++] = ref_list[j];
1577             }
1578         }
1579
1580         /* 8.2.4.3.2 - Long-term reference pictures */
1581         else {
1582
1583             for (j = num_refs; j > ref_list_idx; j--)
1584                 ref_list[j] = ref_list[j - 1];
1585             found_ref_idx =
1586                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1587             ref_list[ref_list_idx++] =
1588                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1589             n = ref_list_idx;
1590             for (j = ref_list_idx; j <= num_refs; j++) {
1591                 gint32 LongTermPicNumF;
1592                 if (!ref_list[j])
1593                     continue;
1594                 LongTermPicNumF = ref_list[j]->is_long_term ?
1595                     ref_list[j]->long_term_pic_num : INT_MAX;
1596                 if (LongTermPicNumF != l->value.long_term_pic_num)
1597                     ref_list[n++] = ref_list[j];
1598             }
1599         }
1600     }
1601
1602 #if DEBUG
1603     for (i = 0; i < num_refs; i++)
1604         if (!ref_list[i])
1605             GST_ERROR("list %u entry %u is empty", list, i);
1606 #endif
1607     *ref_list_count_ptr = num_refs;
1608 }
1609
1610 /* 8.2.4.3 - Modification process for reference picture lists */
1611 static void
1612 exec_picture_refs_modification(
1613     GstVaapiDecoderH264 *decoder,
1614     GstVaapiPictureH264 *picture,
1615     GstH264SliceHdr     *slice_hdr
1616 )
1617 {
1618     GST_DEBUG("execute ref_pic_list_modification()");
1619
1620     /* RefPicList0 */
1621     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1622         slice_hdr->ref_pic_list_modification_flag_l0)
1623         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1624
1625     /* RefPicList1 */
1626     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1627         slice_hdr->ref_pic_list_modification_flag_l1)
1628         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1629 }
1630
1631 static gboolean
1632 init_picture_refs(
1633     GstVaapiDecoderH264 *decoder,
1634     GstVaapiPictureH264 *picture,
1635     GstH264SliceHdr     *slice_hdr
1636 )
1637 {
1638     GstVaapiDecoderH264Private * const priv = decoder->priv;
1639     GstVaapiPicture * const base_picture = &picture->base;
1640     guint i, num_refs;
1641
1642     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1643
1644     priv->RefPicList0_count = 0;
1645     priv->RefPicList1_count = 0;
1646
1647     switch (base_picture->type) {
1648     case GST_VAAPI_PICTURE_TYPE_P:
1649     case GST_VAAPI_PICTURE_TYPE_SP:
1650         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1651         break;
1652     case GST_VAAPI_PICTURE_TYPE_B:
1653         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1654         break;
1655     default:
1656         break;
1657     }
1658
1659     exec_picture_refs_modification(decoder, picture, slice_hdr);
1660
1661     switch (base_picture->type) {
1662     case GST_VAAPI_PICTURE_TYPE_B:
1663         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1664         for (i = priv->RefPicList1_count; i < num_refs; i++)
1665             priv->RefPicList1[i] = NULL;
1666         priv->RefPicList1_count = num_refs;
1667
1668         // fall-through
1669     case GST_VAAPI_PICTURE_TYPE_P:
1670     case GST_VAAPI_PICTURE_TYPE_SP:
1671         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1672         for (i = priv->RefPicList0_count; i < num_refs; i++)
1673             priv->RefPicList0[i] = NULL;
1674         priv->RefPicList0_count = num_refs;
1675         break;
1676     default:
1677         break;
1678     }
1679     return TRUE;
1680 }
1681
1682 static gboolean
1683 init_picture(
1684     GstVaapiDecoderH264 *decoder,
1685     GstVaapiPictureH264 *picture,
1686     GstH264SliceHdr     *slice_hdr,
1687     GstH264NalUnit      *nalu
1688 )
1689 {
1690     GstVaapiDecoderH264Private * const priv = decoder->priv;
1691     GstVaapiPicture * const base_picture = &picture->base;
1692     VAPictureH264 *pic;
1693
1694     priv->prev_frame_num        = priv->frame_num;
1695     priv->frame_num             = slice_hdr->frame_num;
1696     picture->frame_num          = priv->frame_num;
1697     picture->frame_num_wrap     = priv->frame_num;
1698     picture->is_idr             = nalu->type == GST_H264_NAL_SLICE_IDR;
1699     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1700     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1701
1702     /* Reset decoder state for IDR pictures */
1703     if (picture->is_idr) {
1704         GST_DEBUG("<IDR>");
1705         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1706         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1707     }
1708
1709     /* Initialize VA picture info */
1710     pic = &picture->info;
1711     pic->picture_id = picture->base.surface_id;
1712     pic->frame_idx  = priv->frame_num;
1713
1714     /* Initialize slice type */
1715     switch (slice_hdr->type % 5) {
1716     case GST_H264_P_SLICE:
1717         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1718         break;
1719     case GST_H264_B_SLICE:
1720         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1721         break;
1722     case GST_H264_I_SLICE:
1723         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1724         break;
1725     case GST_H264_SP_SLICE:
1726         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1727         break;
1728     case GST_H264_SI_SLICE:
1729         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1730         break;
1731     }
1732
1733     /* Initialize picture structure */
1734     if (!slice_hdr->field_pic_flag)
1735         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1736     else if (!slice_hdr->bottom_field_flag)
1737         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
1738     else
1739         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
1740
1741     if (nalu->ref_idc) {
1742         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1743             &slice_hdr->dec_ref_pic_marking;
1744         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1745         if (picture->is_idr) {
1746             if (dec_ref_pic_marking->long_term_reference_flag)
1747                 picture->is_long_term = TRUE;
1748         }
1749     }
1750
1751     init_picture_poc(decoder, picture, slice_hdr);
1752     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1753         GST_ERROR("failed to initialize references");
1754         return FALSE;
1755     }
1756     return TRUE;
1757 }
1758
1759 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1760 static gboolean
1761 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1762 {
1763     GstVaapiDecoderH264Private * const priv = decoder->priv;
1764     GstH264PPS * const pps = priv->current_picture->pps;
1765     GstH264SPS * const sps = pps->sequence;
1766     guint i, max_num_ref_frames, lowest_frame_num_index;
1767     gint32 lowest_frame_num;
1768
1769     GST_DEBUG("reference picture marking process (sliding window)");
1770
1771     max_num_ref_frames = sps->num_ref_frames;
1772     if (max_num_ref_frames == 0)
1773         max_num_ref_frames = 1;
1774
1775     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1776         return TRUE;
1777     if (priv->short_ref_count < 1)
1778         return FALSE;
1779
1780     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1781     lowest_frame_num_index = 0;
1782     for (i = 1; i < priv->short_ref_count; i++) {
1783         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1784             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1785             lowest_frame_num_index = i;
1786         }
1787     }
1788
1789     remove_reference_at(
1790         decoder,
1791         priv->short_ref, &priv->short_ref_count,
1792         lowest_frame_num_index
1793     );
1794     return TRUE;
1795 }
1796
1797 static inline gint32
1798 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
1799 {
1800     gint32 pic_num;
1801
1802     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1803         pic_num = picture->frame_num_wrap;
1804     else
1805         pic_num = 2 * picture->frame_num_wrap + 1;
1806     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1807     return pic_num;
1808 }
1809
1810 /* 8.2.5.4.1. Mark-term reference picture as "unused for reference" */
1811 static void
1812 exec_ref_pic_marking_adaptive_mmco_1(
1813     GstVaapiDecoderH264  *decoder,
1814     GstVaapiPictureH264  *picture,
1815     GstH264RefPicMarking *ref_pic_marking
1816 )
1817 {
1818     GstVaapiDecoderH264Private * const priv = decoder->priv;
1819     gint32 i, picNumX;
1820
1821     picNumX = get_picNumX(picture, ref_pic_marking);
1822     i = find_short_term_reference(decoder, picNumX);
1823     if (i < 0)
1824         return;
1825     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1826 }
1827
1828 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
1829 static void
1830 exec_ref_pic_marking_adaptive_mmco_2(
1831     GstVaapiDecoderH264  *decoder,
1832     GstVaapiPictureH264  *picture,
1833     GstH264RefPicMarking *ref_pic_marking
1834 )
1835 {
1836     GstVaapiDecoderH264Private * const priv = decoder->priv;
1837     gint32 i;
1838
1839     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
1840     if (i < 0)
1841         return;
1842     remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1843 }
1844
1845 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
1846 static void
1847 exec_ref_pic_marking_adaptive_mmco_3(
1848     GstVaapiDecoderH264  *decoder,
1849     GstVaapiPictureH264  *picture,
1850     GstH264RefPicMarking *ref_pic_marking
1851 )
1852 {
1853     GstVaapiDecoderH264Private * const priv = decoder->priv;
1854     VAPictureH264 *pic;
1855     gint32 i, picNumX;
1856
1857     for (i = 0; i < priv->long_ref_count; i++) {
1858         if (priv->long_ref[i]->info.frame_idx == ref_pic_marking->long_term_frame_idx)
1859             break;
1860     }
1861     if (i != priv->long_ref_count)
1862         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1863
1864     picNumX = get_picNumX(picture, ref_pic_marking);
1865     i = find_short_term_reference(decoder, picNumX);
1866     if (i < 0)
1867         return;
1868
1869     picture = gst_vaapi_picture_ref(priv->short_ref[i]);
1870     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1871     gst_vaapi_picture_replace(&priv->long_ref[priv->long_ref_count++], picture);
1872     gst_vaapi_picture_unref(picture);
1873
1874     picture->is_long_term = TRUE;
1875     pic = &picture->info;
1876     pic->frame_idx = ref_pic_marking->long_term_frame_idx;
1877     pic->flags &= ~VA_PICTURE_H264_SHORT_TERM_REFERENCE;
1878     pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
1879     GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_REFERENCE);
1880 }
1881
1882 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
1883  * as "unused for reference" */
1884 static void
1885 exec_ref_pic_marking_adaptive_mmco_4(
1886     GstVaapiDecoderH264  *decoder,
1887     GstVaapiPictureH264  *picture,
1888     GstH264RefPicMarking *ref_pic_marking
1889 )
1890 {
1891     GstVaapiDecoderH264Private * const priv = decoder->priv;
1892     gint32 i, long_term_frame_idx;
1893
1894     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
1895
1896     for (i = 0; i < priv->long_ref_count; i++) {
1897         if ((gint32)priv->long_ref[i]->info.frame_idx <= long_term_frame_idx)
1898             continue;
1899         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1900         i--;
1901     }
1902 }
1903
1904 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
1905 static void
1906 exec_ref_pic_marking_adaptive_mmco_5(
1907     GstVaapiDecoderH264  *decoder,
1908     GstVaapiPictureH264  *picture,
1909     GstH264RefPicMarking *ref_pic_marking
1910 )
1911 {
1912     GstVaapiDecoderH264Private * const priv = decoder->priv;
1913     VAPictureH264 * const pic = &picture->info;
1914
1915     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1916     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1917     dpb_flush(decoder);
1918
1919     priv->prev_pic_has_mmco5 = TRUE;
1920
1921     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
1922     priv->frame_num = 0;
1923     priv->frame_num_offset = 0;
1924     picture->frame_num = 0;
1925
1926     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
1927     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1928         pic->TopFieldOrderCnt -= picture->poc;
1929     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1930         pic->BottomFieldOrderCnt -= picture->poc;
1931     picture->poc = 0;
1932 }
1933
1934 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
1935 static void
1936 exec_ref_pic_marking_adaptive_mmco_6(
1937     GstVaapiDecoderH264  *decoder,
1938     GstVaapiPictureH264  *picture,
1939     GstH264RefPicMarking *ref_pic_marking
1940 )
1941 {
1942     picture->is_long_term = TRUE;
1943     picture->info.frame_idx = ref_pic_marking->long_term_frame_idx;
1944 }
1945
1946 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1947 static gboolean
1948 exec_ref_pic_marking_adaptive(
1949     GstVaapiDecoderH264     *decoder,
1950     GstVaapiPictureH264     *picture,
1951     GstH264DecRefPicMarking *dec_ref_pic_marking
1952 )
1953 {
1954     guint i;
1955
1956     GST_DEBUG("reference picture marking process (adaptive memory control)");
1957
1958     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
1959         GstVaapiDecoderH264  *decoder,
1960         GstVaapiPictureH264  *picture,
1961         GstH264RefPicMarking *ref_pic_marking
1962     );
1963
1964     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
1965         NULL,
1966         exec_ref_pic_marking_adaptive_mmco_1,
1967         exec_ref_pic_marking_adaptive_mmco_2,
1968         exec_ref_pic_marking_adaptive_mmco_3,
1969         exec_ref_pic_marking_adaptive_mmco_4,
1970         exec_ref_pic_marking_adaptive_mmco_5,
1971         exec_ref_pic_marking_adaptive_mmco_6,
1972     };
1973
1974     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1975         GstH264RefPicMarking * const ref_pic_marking =
1976             &dec_ref_pic_marking->ref_pic_marking[i];
1977
1978         const guint mmco = ref_pic_marking->memory_management_control_operation;
1979         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
1980             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
1981         else {
1982             GST_ERROR("unhandled MMCO %u", mmco);
1983             return FALSE;
1984         }
1985     }
1986     return TRUE;
1987 }
1988
1989 /* 8.2.5 - Execute reference picture marking process */
1990 static gboolean
1991 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
1992 {
1993     GstVaapiDecoderH264Private * const priv = decoder->priv;
1994     GstVaapiPictureH264 **picture_ptr;
1995
1996     priv->prev_pic_has_mmco5 = FALSE;
1997     priv->prev_pic_structure = picture->base.structure;
1998
1999     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2000         return TRUE;
2001
2002     if (!picture->is_idr) {
2003         GstVaapiSliceH264 * const slice =
2004             gst_vaapi_picture_h264_get_last_slice(picture);
2005         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2006             &slice->slice_hdr.dec_ref_pic_marking;
2007         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2008             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2009                 return FALSE;
2010         }
2011         else {
2012             if (!exec_ref_pic_marking_sliding_window(decoder))
2013                 return FALSE;
2014         }
2015     }
2016
2017     if (picture->is_long_term) {
2018         picture_ptr = &priv->long_ref[priv->long_ref_count++];
2019         picture->info.flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2020     }
2021     else {
2022         picture_ptr = &priv->short_ref[priv->short_ref_count++];
2023         picture->info.flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2024     }
2025     gst_vaapi_picture_replace(picture_ptr, picture);
2026     return TRUE;
2027 }
2028
2029 static void
2030 vaapi_init_picture(VAPictureH264 *pic)
2031 {
2032     pic->picture_id           = VA_INVALID_ID;
2033     pic->frame_idx            = 0;
2034     pic->flags                = VA_PICTURE_H264_INVALID;
2035     pic->TopFieldOrderCnt     = 0;
2036     pic->BottomFieldOrderCnt  = 0;
2037 }
2038
2039 static void
2040 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture)
2041 {
2042     pic->picture_id = picture->base.surface_id;
2043     pic->flags = 0;
2044
2045     if (picture->info.flags & VA_PICTURE_H264_LONG_TERM_REFERENCE) {
2046         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2047         pic->frame_idx = picture->info.frame_idx;
2048     }
2049     else {
2050         if (picture->info.flags & VA_PICTURE_H264_SHORT_TERM_REFERENCE)
2051             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2052         pic->frame_idx = picture->frame_num;
2053     }
2054
2055     switch (picture->base.structure) {
2056     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2057         pic->TopFieldOrderCnt = picture->info.TopFieldOrderCnt;
2058         pic->BottomFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2059         break;
2060     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2061         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2062         pic->TopFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2063         pic->BottomFieldOrderCnt = 0;
2064         break;
2065     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2066         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2067         pic->BottomFieldOrderCnt = picture->info.BottomFieldOrderCnt;
2068         pic->TopFieldOrderCnt = 0;
2069         break;
2070     }
2071 }
2072
2073 static gboolean
2074 fill_picture(
2075     GstVaapiDecoderH264 *decoder,
2076     GstVaapiPictureH264 *picture,
2077     GstH264SliceHdr     *slice_hdr,
2078     GstH264NalUnit      *nalu
2079 )
2080 {
2081     GstVaapiDecoderH264Private * const priv = decoder->priv;
2082     GstVaapiPicture * const base_picture = &picture->base;
2083     GstH264PPS * const pps = picture->pps;
2084     GstH264SPS * const sps = pps->sequence;
2085     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2086     guint i, n;
2087
2088     /* Fill in VAPictureParameterBufferH264 */
2089     vaapi_fill_picture(&pic_param->CurrPic, picture);
2090     for (i = 0, n = 0; i < priv->short_ref_count; i++, n++)
2091         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->short_ref[i]);
2092     for (i = 0; i < priv->long_ref_count; i++, n++)
2093         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->long_ref[i]);
2094     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2095         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2096
2097 #define COPY_FIELD(s, f) \
2098     pic_param->f = (s)->f
2099
2100 #define COPY_BFM(a, s, f) \
2101     pic_param->a.bits.f = (s)->f
2102
2103     pic_param->picture_width_in_mbs_minus1  = ((priv->width + 15) >> 4)  - 1;
2104     pic_param->picture_height_in_mbs_minus1 = ((priv->height + 15) >> 4) - 1;
2105     pic_param->frame_num                    = priv->frame_num;
2106
2107     COPY_FIELD(sps, bit_depth_luma_minus8);
2108     COPY_FIELD(sps, bit_depth_chroma_minus8);
2109     COPY_FIELD(sps, num_ref_frames);
2110     COPY_FIELD(pps, num_slice_groups_minus1);
2111     COPY_FIELD(pps, slice_group_map_type);
2112     COPY_FIELD(pps, slice_group_change_rate_minus1);
2113     COPY_FIELD(pps, pic_init_qp_minus26);
2114     COPY_FIELD(pps, pic_init_qs_minus26);
2115     COPY_FIELD(pps, chroma_qp_index_offset);
2116     COPY_FIELD(pps, second_chroma_qp_index_offset);
2117
2118     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2119     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2120     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2121
2122     COPY_BFM(seq_fields, sps, chroma_format_idc);
2123     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2124     COPY_BFM(seq_fields, sps, frame_mbs_only_flag); 
2125     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag); 
2126     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag); 
2127     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2128     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2129     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2130     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2131
2132     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2133     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2134     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2135
2136     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2137     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2138     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2139     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2140     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2141     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2142     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2143     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2144     return TRUE;
2145 }
2146
2147 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2148 static gboolean
2149 is_new_picture(
2150     GstVaapiDecoderH264 *decoder,
2151     GstH264NalUnit      *nalu,
2152     GstH264SliceHdr     *slice_hdr
2153 )
2154 {
2155     GstVaapiDecoderH264Private * const priv = decoder->priv;
2156     GstH264PPS * const pps = slice_hdr->pps;
2157     GstH264SPS * const sps = pps->sequence;
2158     GstVaapiSliceH264 *slice;
2159     GstH264SliceHdr *prev_slice_hdr;
2160
2161     if (!priv->current_picture)
2162         return TRUE;
2163
2164     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2165     if (!slice)
2166         return FALSE;
2167     prev_slice_hdr = &slice->slice_hdr;
2168
2169 #define CHECK_EXPR(expr, field_name) do {              \
2170         if (!(expr)) {                                 \
2171             GST_DEBUG(field_name " differs in value"); \
2172             return TRUE;                               \
2173         }                                              \
2174     } while (0)
2175
2176 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2177     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2178
2179     /* frame_num differs in value, regardless of inferred values to 0 */
2180     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2181
2182     /* pic_parameter_set_id differs in value */
2183     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2184
2185     /* field_pic_flag differs in value */
2186     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2187
2188     /* bottom_field_flag is present in both and differs in value */
2189     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2190         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2191
2192     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2193     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2194                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2195
2196     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2197        value or delta_pic_order_cnt_bottom differs in value */
2198     if (sps->pic_order_cnt_type == 0) {
2199         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2200         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2201             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2202     }
2203
2204     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2205        differs in value or delta_pic_order_cnt[1] differs in value */
2206     else if (sps->pic_order_cnt_type == 1) {
2207         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2208         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2209     }
2210
2211     /* IdrPicFlag differs in value */
2212     CHECK_EXPR(((priv->current_picture->is_idr ^
2213                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2214
2215     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2216     if (priv->current_picture->is_idr)
2217         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2218
2219 #undef CHECK_EXPR
2220 #undef CHECK_VALUE
2221     return FALSE;
2222 }
2223
2224 static GstVaapiDecoderStatus
2225 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2226 {
2227     GstVaapiDecoderH264Private * const priv = decoder->priv;
2228     GstVaapiPictureH264 *picture;
2229     GstVaapiDecoderStatus status;
2230     GstH264PPS * const pps = slice_hdr->pps;
2231     GstH264SPS * const sps = pps->sequence;
2232
2233     status = decode_current_picture(decoder);
2234     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2235         return status;
2236
2237     status = ensure_context(decoder, sps);
2238     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2239         return status;
2240
2241     picture = gst_vaapi_picture_h264_new(decoder);
2242     if (!picture) {
2243         GST_ERROR("failed to allocate picture");
2244         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2245     }
2246     priv->current_picture = picture;
2247
2248     picture->pps = pps;
2249
2250     status = ensure_quant_matrix(decoder, picture);
2251     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2252         GST_ERROR("failed to reset quantizer matrix");
2253         return status;
2254     }
2255
2256     if (!init_picture(decoder, picture, slice_hdr, nalu))
2257         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2258     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2259         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2260     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2261 }
2262
2263 static gboolean
2264 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2265 {
2266     if (!exec_ref_pic_marking(decoder, picture))
2267         return FALSE;
2268     if (!dpb_add(decoder, picture))
2269         return FALSE;
2270     return TRUE;
2271 }
2272
2273 static inline guint
2274 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2275 {
2276     guint epb_count;
2277
2278     epb_count = slice_hdr->n_emulation_prevention_bytes;
2279     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2280 }
2281
2282 static gboolean
2283 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2284 {
2285     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2286     GstH264PPS * const pps = slice_hdr->pps;
2287     GstH264SPS * const sps = pps->sequence;
2288     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2289     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2290     guint num_weight_tables = 0;
2291     gint i, j;
2292
2293     if (pps->weighted_pred_flag &&
2294         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2295         num_weight_tables = 1;
2296     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2297         num_weight_tables = 2;
2298     else
2299         num_weight_tables = 0;
2300
2301     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2302     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2303     slice_param->luma_weight_l0_flag      = 0;
2304     slice_param->chroma_weight_l0_flag    = 0;
2305     slice_param->luma_weight_l1_flag      = 0;
2306     slice_param->chroma_weight_l1_flag    = 0;
2307
2308     if (num_weight_tables < 1)
2309         return TRUE;
2310
2311     slice_param->luma_weight_l0_flag = 1;
2312     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2313         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2314         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2315     }
2316
2317     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2318     if (slice_param->chroma_weight_l0_flag) {
2319         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2320             for (j = 0; j < 2; j++) {
2321                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2322                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2323             }
2324         }
2325     }
2326
2327     if (num_weight_tables < 2)
2328         return TRUE;
2329
2330     slice_param->luma_weight_l1_flag = 1;
2331     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2332         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2333         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2334     }
2335
2336     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2337     if (slice_param->chroma_weight_l1_flag) {
2338         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2339             for (j = 0; j < 2; j++) {
2340                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2341                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2342             }
2343         }
2344     }
2345     return TRUE;
2346 }
2347
2348 static gboolean
2349 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2350 {
2351     GstVaapiDecoderH264Private * const priv = decoder->priv;
2352     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2353     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2354     guint i, num_ref_lists = 0;
2355
2356     slice_param->num_ref_idx_l0_active_minus1 = 0;
2357     slice_param->num_ref_idx_l1_active_minus1 = 0;
2358
2359     if (GST_H264_IS_B_SLICE(slice_hdr))
2360         num_ref_lists = 2;
2361     else if (GST_H264_IS_I_SLICE(slice_hdr))
2362         num_ref_lists = 0;
2363     else
2364         num_ref_lists = 1;
2365
2366     if (num_ref_lists < 1)
2367         return TRUE;
2368
2369     slice_param->num_ref_idx_l0_active_minus1 =
2370         slice_hdr->num_ref_idx_l0_active_minus1;
2371
2372     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2373         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i]);
2374     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2375         vaapi_init_picture(&slice_param->RefPicList0[i]);
2376
2377     if (num_ref_lists < 2)
2378         return TRUE;
2379
2380     slice_param->num_ref_idx_l1_active_minus1 =
2381         slice_hdr->num_ref_idx_l1_active_minus1;
2382
2383     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2384         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i]);
2385     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2386         vaapi_init_picture(&slice_param->RefPicList1[i]);
2387     return TRUE;
2388 }
2389
2390 static gboolean
2391 fill_slice(
2392     GstVaapiDecoderH264 *decoder,
2393     GstVaapiSliceH264   *slice,
2394     GstH264NalUnit      *nalu
2395 )
2396 {
2397     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2398     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2399
2400     /* Fill in VASliceParameterBufferH264 */
2401     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2402     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2403     slice_param->slice_type                     = slice_hdr->type % 5;
2404     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2405     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2406     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2407     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2408     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2409     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2410
2411     if (!fill_RefPicList(decoder, slice))
2412         return FALSE;
2413     if (!fill_pred_weight_table(decoder, slice))
2414         return FALSE;
2415     return TRUE;
2416 }
2417
2418 static GstVaapiDecoderStatus
2419 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2420 {
2421     GstVaapiDecoderH264Private * const priv = decoder->priv;
2422     GstVaapiDecoderStatus status;
2423     GstVaapiPictureH264 *picture;
2424     GstVaapiSliceH264 *slice = NULL;
2425     GstH264SliceHdr *slice_hdr;
2426     GstH264ParserResult result;
2427
2428     GST_DEBUG("slice (%u bytes)", nalu->size);
2429
2430     slice = gst_vaapi_slice_h264_new(
2431         decoder,
2432         nalu->data + nalu->offset,
2433         nalu->size
2434     );
2435     if (!slice) {
2436         GST_ERROR("failed to allocate slice");
2437         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2438     }
2439
2440     slice_hdr = &slice->slice_hdr;
2441     memset(slice_hdr, 0, sizeof(*slice_hdr));
2442     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2443     if (result != GST_H264_PARSER_OK) {
2444         status = get_status(result);
2445         goto error;
2446     }
2447
2448     if (is_new_picture(decoder, nalu, slice_hdr)) {
2449         status = decode_picture(decoder, nalu, slice_hdr);
2450         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2451             goto error;
2452     }
2453     picture = priv->current_picture;
2454
2455     if (!fill_slice(decoder, slice, nalu)) {
2456         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2457         goto error;
2458     }
2459     gst_vaapi_picture_add_slice(
2460         GST_VAAPI_PICTURE_CAST(picture),
2461         GST_VAAPI_SLICE_CAST(slice)
2462     );
2463     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2464
2465 error:
2466     if (slice)
2467         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2468     return status;
2469 }
2470
2471 static inline gint
2472 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2473 {
2474     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2475                                                      0xffffff00, 0x00000100,
2476                                                      ofs, size,
2477                                                      scp);
2478 }
2479
2480 static GstVaapiDecoderStatus
2481 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2482 {
2483     GstVaapiDecoderStatus status;
2484
2485     switch (nalu->type) {
2486     case GST_H264_NAL_SLICE_IDR:
2487         /* fall-through. IDR specifics are handled in init_picture() */
2488     case GST_H264_NAL_SLICE:
2489         status = decode_slice(decoder, nalu);
2490         break;
2491     case GST_H264_NAL_SPS:
2492         status = decode_sps(decoder, nalu);
2493         break;
2494     case GST_H264_NAL_PPS:
2495         status = decode_pps(decoder, nalu);
2496         break;
2497     case GST_H264_NAL_SEI:
2498         status = decode_sei(decoder, nalu);
2499         break;
2500     case GST_H264_NAL_SEQ_END:
2501         status = decode_sequence_end(decoder);
2502         break;
2503     case GST_H264_NAL_AU_DELIMITER:
2504         /* skip all Access Unit NALs */
2505         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2506         break;
2507     case GST_H264_NAL_FILLER_DATA:
2508         /* skip all Filler Data NALs */
2509         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2510         break;
2511     default:
2512         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2513         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2514         break;
2515     }
2516     return status;
2517 }
2518
2519 static GstVaapiDecoderStatus
2520 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2521 {
2522     GstVaapiDecoderH264Private * const priv = decoder->priv;
2523     GstVaapiDecoderStatus status;
2524     GstH264ParserResult result;
2525     GstH264NalUnit nalu;
2526     gboolean is_eos;
2527     const guchar *buf;
2528     guint i, buf_size, nalu_size, size;
2529     guint32 start_code;
2530     gint ofs;
2531
2532     buf      = GST_BUFFER_DATA(buffer);
2533     buf_size = GST_BUFFER_SIZE(buffer);
2534     is_eos   = GST_BUFFER_IS_EOS(buffer);
2535     if (buf && buf_size > 0)
2536         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2537
2538     size = gst_adapter_available(priv->adapter);
2539     do {
2540         if (size == 0) {
2541             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2542             break;
2543         }
2544
2545         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2546         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2547             break;
2548
2549         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2550         if (priv->is_avc) {
2551             if (size < priv->nal_length_size)
2552                 break;
2553             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2554
2555             nalu_size = 0;
2556             for (i = 0; i < priv->nal_length_size; i++)
2557                 nalu_size = (nalu_size << 8) | buf[i];
2558
2559             buf_size = priv->nal_length_size + nalu_size;
2560             if (size < buf_size)
2561                 break;
2562             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2563             size -= buf_size;
2564
2565             buf      = GST_BUFFER_DATA(buffer);
2566             buf_size = GST_BUFFER_SIZE(buffer);
2567
2568             result = gst_h264_parser_identify_nalu_avc(
2569                 priv->parser,
2570                 buf, 0, buf_size, priv->nal_length_size,
2571                 &nalu
2572             );
2573         }
2574         else {
2575             if (size < 4)
2576                 break;
2577             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
2578             if (ofs < 0)
2579                 break;
2580             gst_adapter_flush(priv->adapter, ofs);
2581             size -= ofs;
2582
2583             ofs = G_UNLIKELY(size < 8) ? -1 :
2584                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
2585             if (ofs < 0) {
2586                 // Assume the whole NAL unit is present if end-of-stream
2587                 if (!is_eos)
2588                     break;
2589                 ofs = size;
2590             }
2591             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
2592             size -= ofs;
2593
2594             buf      = GST_BUFFER_DATA(buffer);
2595             buf_size = GST_BUFFER_SIZE(buffer);
2596
2597             result = gst_h264_parser_identify_nalu_unchecked(
2598                 priv->parser,
2599                 buf, 0, buf_size,
2600                 &nalu
2601             );
2602         }
2603         status = get_status(result);
2604         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
2605             status = decode_nalu(decoder, &nalu);
2606         gst_buffer_unref(buffer);
2607     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
2608
2609     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
2610                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
2611         status = decode_sequence_end(decoder);
2612     return status;
2613 }
2614
2615 static GstVaapiDecoderStatus
2616 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2617 {
2618     GstVaapiDecoderH264Private * const priv = decoder->priv;
2619     GstVaapiDecoderStatus status;
2620     GstH264NalUnit nalu;
2621     GstH264ParserResult result;
2622     guchar *buf;
2623     guint buf_size;
2624     guint i, ofs, num_sps, num_pps;
2625
2626     buf      = GST_BUFFER_DATA(buffer);
2627     buf_size = GST_BUFFER_SIZE(buffer);
2628     if (!buf || buf_size == 0)
2629         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2630
2631     if (buf_size < 8)
2632         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2633
2634     if (buf[0] != 1) {
2635         GST_ERROR("failed to decode codec-data, not in avcC format");
2636         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2637     }
2638
2639     priv->nal_length_size = (buf[4] & 0x03) + 1;
2640
2641     num_sps = buf[5] & 0x1f;
2642     ofs = 6;
2643
2644     for (i = 0; i < num_sps; i++) {
2645         result = gst_h264_parser_identify_nalu_avc(
2646             priv->parser,
2647             buf, ofs, buf_size, 2,
2648             &nalu
2649         );
2650         if (result != GST_H264_PARSER_OK)
2651             return get_status(result);
2652
2653         status = decode_sps(decoder, &nalu);
2654         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2655             return status;
2656         ofs = nalu.offset + nalu.size;
2657     }
2658
2659     num_pps = buf[ofs];
2660     ofs++;
2661
2662     for (i = 0; i < num_pps; i++) {
2663         result = gst_h264_parser_identify_nalu_avc(
2664             priv->parser,
2665             buf, ofs, buf_size, 2,
2666             &nalu
2667         );
2668         if (result != GST_H264_PARSER_OK)
2669             return get_status(result);
2670
2671         status = decode_pps(decoder, &nalu);
2672         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2673             return status;
2674         ofs = nalu.offset + nalu.size;
2675     }
2676
2677     priv->is_avc = TRUE;
2678     return status;
2679 }
2680
2681 GstVaapiDecoderStatus
2682 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2683 {
2684     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2685     GstVaapiDecoderH264Private * const priv = decoder->priv;
2686     GstVaapiDecoderStatus status;
2687     GstBuffer *codec_data;
2688
2689     g_return_val_if_fail(priv->is_constructed,
2690                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2691
2692     if (!priv->is_opened) {
2693         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2694         if (!priv->is_opened)
2695             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2696
2697         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2698         if (codec_data) {
2699             status = decode_codec_data(decoder, codec_data);