h264: introduce per-field POC in GstVaapiPictureH264.
[vaapi:gstreamer-vaapi.git] / gst-libs / gst / vaapi / gstvaapidecoder_h264.c
1 /*
2  *  gstvaapidecoder_h264.c - H.264 decoder
3  *
4  *  Copyright (C) 2011-2012 Intel Corporation
5  *
6  *  This library is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU Lesser General Public License
8  *  as published by the Free Software Foundation; either version 2.1
9  *  of the License, or (at your option) any later version.
10  *
11  *  This library is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  *  Lesser General Public License for more details.
15  *
16  *  You should have received a copy of the GNU Lesser General Public
17  *  License along with this library; if not, write to the Free
18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19  *  Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * SECTION:gstvaapidecoder_h264
24  * @short_description: H.264 decoder
25  */
26
27 #include "sysdeps.h"
28 #include <string.h>
29 #include <gst/base/gstadapter.h>
30 #include <gst/codecparsers/gsth264parser.h>
31 #include "gstvaapidecoder_h264.h"
32 #include "gstvaapidecoder_objects.h"
33 #include "gstvaapidecoder_priv.h"
34 #include "gstvaapidisplay_priv.h"
35 #include "gstvaapiobject_priv.h"
36
37 #define DEBUG 1
38 #include "gstvaapidebug.h"
39
40 /* Defined to 1 if strict ordering of DPB is needed. Only useful for debug */
41 #define USE_STRICT_DPB_ORDERING 0
42
43 typedef struct _GstVaapiPictureH264             GstVaapiPictureH264;
44 typedef struct _GstVaapiPictureH264Class        GstVaapiPictureH264Class;
45 typedef struct _GstVaapiSliceH264               GstVaapiSliceH264;
46 typedef struct _GstVaapiSliceH264Class          GstVaapiSliceH264Class;
47
48 // Used for field_poc[]
49 #define TOP_FIELD       0
50 #define BOTTOM_FIELD    1
51
52 /* ------------------------------------------------------------------------- */
53 /* --- H.264 Pictures                                                    --- */
54 /* ------------------------------------------------------------------------- */
55
56 #define GST_VAAPI_TYPE_PICTURE_H264 \
57     (gst_vaapi_picture_h264_get_type())
58
59 #define GST_VAAPI_PICTURE_H264_CAST(obj) \
60     ((GstVaapiPictureH264 *)(obj))
61
62 #define GST_VAAPI_PICTURE_H264(obj)                             \
63     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
64                                 GST_VAAPI_TYPE_PICTURE_H264,    \
65                                 GstVaapiPictureH264))
66
67 #define GST_VAAPI_PICTURE_H264_CLASS(klass)                     \
68     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
69                              GST_VAAPI_TYPE_PICTURE_H264,       \
70                              GstVaapiPictureH264Class))
71
72 #define GST_VAAPI_IS_PICTURE_H264(obj) \
73     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_PICTURE_H264))
74
75 #define GST_VAAPI_IS_PICTURE_H264_CLASS(klass) \
76     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_PICTURE_H264))
77
78 #define GST_VAAPI_PICTURE_H264_GET_CLASS(obj)                   \
79     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
80                                GST_VAAPI_TYPE_PICTURE_H264,     \
81                                GstVaapiPictureH264Class))
82
83 /*
84  * Extended picture flags:
85  *
86  * @GST_VAAPI_PICTURE_FLAG_IDR: flag that specifies an IDR picture
87  * @GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE: flag that specifies
88  *     "used for short-term reference"
89  * @GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE: flag that specifies
90  *     "used for long-term reference"
91  * @GST_VAAPI_PICTURE_FLAGS_REFERENCE: mask covering any kind of
92  *     reference picture (short-term reference or long-term reference)
93  */
94 enum {
95     GST_VAAPI_PICTURE_FLAG_IDR = (GST_VAAPI_PICTURE_FLAG_LAST << 0),
96
97     GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE = (
98         GST_VAAPI_PICTURE_FLAG_REFERENCE),
99     GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE = (
100         GST_VAAPI_PICTURE_FLAG_REFERENCE | (GST_VAAPI_PICTURE_FLAG_LAST << 1)),
101     GST_VAAPI_PICTURE_FLAGS_REFERENCE = (
102         GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE |
103         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE),
104 };
105
106 #define GST_VAAPI_PICTURE_IS_IDR(picture) \
107     (GST_VAAPI_PICTURE_FLAG_IS_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR))
108
109 #define GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture)      \
110     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
111       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
112      GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE)
113
114 #define GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)       \
115     ((GST_VAAPI_PICTURE_FLAGS(picture) &                        \
116       GST_VAAPI_PICTURE_FLAGS_REFERENCE) ==                     \
117      GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE)
118
119 struct _GstVaapiPictureH264 {
120     GstVaapiPicture             base;
121     GstH264PPS                 *pps;
122     gint32                      field_poc[2];
123     gint32                      frame_num;              // Original frame_num from slice_header()
124     gint32                      frame_num_wrap;         // Temporary for ref pic marking: FrameNumWrap
125     gint32                      long_term_frame_idx;    // Temporary for ref pic marking: LongTermFrameIdx
126     gint32                      pic_num;                // Temporary for ref pic marking: PicNum
127     gint32                      long_term_pic_num;      // Temporary for ref pic marking: LongTermPicNum
128     guint                       output_flag             : 1;
129     guint                       output_needed           : 1;
130 };
131
132 struct _GstVaapiPictureH264Class {
133     /*< private >*/
134     GstVaapiPictureClass        parent_class;
135 };
136
137 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiPictureH264,
138                             gst_vaapi_picture_h264,
139                             GST_VAAPI_TYPE_PICTURE)
140
141 static void
142 gst_vaapi_picture_h264_destroy(GstVaapiPictureH264 *decoder)
143 {
144 }
145
146 static gboolean
147 gst_vaapi_picture_h264_create(
148     GstVaapiPictureH264                      *picture,
149     const GstVaapiCodecObjectConstructorArgs *args
150 )
151 {
152     return TRUE;
153 }
154
155 static void
156 gst_vaapi_picture_h264_init(GstVaapiPictureH264 *picture)
157 {
158     picture->field_poc[0]       = G_MAXINT32;
159     picture->field_poc[1]       = G_MAXINT32;
160     picture->output_needed      = FALSE;
161 }
162
163 static inline GstVaapiPictureH264 *
164 gst_vaapi_picture_h264_new(GstVaapiDecoderH264 *decoder)
165 {
166     GstVaapiCodecObject *object;
167
168     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
169
170     object = gst_vaapi_codec_object_new(
171         GST_VAAPI_TYPE_PICTURE_H264,
172         GST_VAAPI_CODEC_BASE(decoder),
173         NULL, sizeof(VAPictureParameterBufferH264),
174         NULL, 0
175     );
176     if (!object)
177         return NULL;
178     return GST_VAAPI_PICTURE_H264_CAST(object);
179 }
180
181 static inline GstVaapiSliceH264 *
182 gst_vaapi_picture_h264_get_last_slice(GstVaapiPictureH264 *picture)
183 {
184     g_return_val_if_fail(GST_VAAPI_IS_PICTURE_H264(picture), NULL);
185
186     if (G_UNLIKELY(picture->base.slices->len < 1))
187         return NULL;
188     return g_ptr_array_index(picture->base.slices,
189         picture->base.slices->len - 1);
190 }
191
192 /* ------------------------------------------------------------------------- */
193 /* --- Slices                                                            --- */
194 /* ------------------------------------------------------------------------- */
195
196 #define GST_VAAPI_TYPE_SLICE_H264 \
197     (gst_vaapi_slice_h264_get_type())
198
199 #define GST_VAAPI_SLICE_H264_CAST(obj) \
200     ((GstVaapiSliceH264 *)(obj))
201
202 #define GST_VAAPI_SLICE_H264(obj)                               \
203     (G_TYPE_CHECK_INSTANCE_CAST((obj),                          \
204                                 GST_VAAPI_TYPE_SLICE_H264,      \
205                                 GstVaapiSliceH264))
206
207 #define GST_VAAPI_SLICE_H264_CLASS(klass)                       \
208     (G_TYPE_CHECK_CLASS_CAST((klass),                           \
209                              GST_VAAPI_TYPE_SLICE_H264,         \
210                              GstVaapiSliceH264Class))
211
212 #define GST_VAAPI_IS_SLICE_H264(obj) \
213     (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_VAAPI_TYPE_SLICE_H264))
214
215 #define GST_VAAPI_IS_SLICE_H264_CLASS(klass) \
216     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_SLICE_H264))
217
218 #define GST_VAAPI_SLICE_H264_GET_CLASS(obj)                     \
219     (G_TYPE_INSTANCE_GET_CLASS((obj),                           \
220                                GST_VAAPI_TYPE_SLICE_H264,       \
221                                GstVaapiSliceH264Class))
222
223 struct _GstVaapiSliceH264 {
224     GstVaapiSlice               base;
225     GstH264SliceHdr             slice_hdr;              // parsed slice_header()
226 };
227
228 struct _GstVaapiSliceH264Class {
229     /*< private >*/
230     GstVaapiSliceClass          parent_class;
231 };
232
233 GST_VAAPI_CODEC_DEFINE_TYPE(GstVaapiSliceH264,
234                             gst_vaapi_slice_h264,
235                             GST_VAAPI_TYPE_SLICE)
236
237 static void
238 gst_vaapi_slice_h264_destroy(GstVaapiSliceH264 *slice)
239 {
240 }
241
242 static gboolean
243 gst_vaapi_slice_h264_create(
244     GstVaapiSliceH264                        *slice,
245     const GstVaapiCodecObjectConstructorArgs *args
246 )
247 {
248     return TRUE;
249 }
250
251 static void
252 gst_vaapi_slice_h264_init(GstVaapiSliceH264 *slice)
253 {
254 }
255
256 static inline GstVaapiSliceH264 *
257 gst_vaapi_slice_h264_new(
258     GstVaapiDecoderH264 *decoder,
259     const guint8        *data,
260     guint                data_size
261 )
262 {
263     GstVaapiCodecObject *object;
264
265     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder), NULL);
266
267     object = gst_vaapi_codec_object_new(
268         GST_VAAPI_TYPE_SLICE_H264,
269         GST_VAAPI_CODEC_BASE(decoder),
270         NULL, sizeof(VASliceParameterBufferH264),
271         data, data_size
272     );
273     if (!object)
274         return NULL;
275     return GST_VAAPI_SLICE_H264_CAST(object);
276 }
277
278 /* ------------------------------------------------------------------------- */
279 /* --- H.264 Decoder                                                     --- */
280 /* ------------------------------------------------------------------------- */
281
282 G_DEFINE_TYPE(GstVaapiDecoderH264,
283               gst_vaapi_decoder_h264,
284               GST_VAAPI_TYPE_DECODER)
285
286 #define GST_VAAPI_DECODER_H264_GET_PRIVATE(obj)                 \
287     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
288                                  GST_VAAPI_TYPE_DECODER_H264,   \
289                                  GstVaapiDecoderH264Private))
290
291 struct _GstVaapiDecoderH264Private {
292     GstAdapter                 *adapter;
293     GstH264NalParser           *parser;
294     /* Last decoded SPS. May not be the last activated one. Just here because
295        it may not fit stack memory allocation in decode_sps() */
296     GstH264SPS                  last_sps;
297     /* Last decoded PPS. May not be the last activated one. Just here because
298        it may not fit stack memory allocation in decode_pps() */
299     GstH264PPS                  last_pps;
300     GstVaapiPictureH264        *current_picture;
301     GstVaapiPictureH264        *dpb[16];
302     guint                       dpb_count;
303     guint                       dpb_size;
304     GstVaapiProfile             profile;
305     GstVaapiEntrypoint          entrypoint;
306     GstVaapiChromaType          chroma_type;
307     GstVaapiPictureH264        *short_ref[32];
308     guint                       short_ref_count;
309     GstVaapiPictureH264        *long_ref[32];
310     guint                       long_ref_count;
311     GstVaapiPictureH264        *RefPicList0[32];
312     guint                       RefPicList0_count;
313     GstVaapiPictureH264        *RefPicList1[32];
314     guint                       RefPicList1_count;
315     guint                       nal_length_size;
316     guint                       width;
317     guint                       height;
318     gint32                      field_poc[2];           // 0:TopFieldOrderCnt / 1:BottomFieldOrderCnt
319     gint32                      poc_msb;                // PicOrderCntMsb
320     gint32                      poc_lsb;                // pic_order_cnt_lsb (from slice_header())
321     gint32                      prev_poc_msb;           // prevPicOrderCntMsb
322     gint32                      prev_poc_lsb;           // prevPicOrderCntLsb
323     gint32                      frame_num_offset;       // FrameNumOffset
324     gint32                      frame_num;              // frame_num (from slice_header())
325     gint32                      prev_frame_num;         // prevFrameNum
326     gboolean                    prev_pic_has_mmco5;     // prevMmco5Pic
327     gboolean                    prev_pic_structure;     // previous picture structure
328     guint                       is_constructed          : 1;
329     guint                       is_opened               : 1;
330     guint                       is_avc                  : 1;
331     guint                       has_context             : 1;
332 };
333
334 static gboolean
335 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture);
336
337 static void
338 clear_references(
339     GstVaapiDecoderH264  *decoder,
340     GstVaapiPictureH264 **pictures,
341     guint                *picture_count
342 );
343
344 /* Get number of reference frames to use */
345 static guint
346 get_max_dec_frame_buffering(GstH264SPS *sps)
347 {
348     guint max_dec_frame_buffering, MaxDpbMbs, PicSizeMbs;
349
350     /* Table A-1 - Level limits */
351     switch (sps->level_idc) {
352     case 10: MaxDpbMbs = 396;    break;
353     case 11: MaxDpbMbs = 900;    break;
354     case 12: MaxDpbMbs = 2376;   break;
355     case 13: MaxDpbMbs = 2376;   break;
356     case 20: MaxDpbMbs = 2376;   break;
357     case 21: MaxDpbMbs = 4752;   break;
358     case 22: MaxDpbMbs = 8100;   break;
359     case 30: MaxDpbMbs = 8100;   break;
360     case 31: MaxDpbMbs = 18000;  break;
361     case 32: MaxDpbMbs = 20480;  break;
362     case 40: MaxDpbMbs = 32768;  break;
363     case 41: MaxDpbMbs = 32768;  break;
364     case 42: MaxDpbMbs = 34816;  break;
365     case 50: MaxDpbMbs = 110400; break;
366     case 51: MaxDpbMbs = 184320; break;
367     default:
368         g_assert(0 && "unhandled level");
369         break;
370     }
371
372     PicSizeMbs = ((sps->pic_width_in_mbs_minus1 + 1) *
373                   (sps->pic_height_in_map_units_minus1 + 1) *
374                   (sps->frame_mbs_only_flag ? 1 : 2));
375     max_dec_frame_buffering = MaxDpbMbs / PicSizeMbs;
376
377     /* VUI parameters */
378     if (sps->vui_parameters_present_flag) {
379         GstH264VUIParams * const vui_params = &sps->vui_parameters;
380         if (vui_params->bitstream_restriction_flag)
381             max_dec_frame_buffering = vui_params->max_dec_frame_buffering;
382         else {
383             switch (sps->profile_idc) {
384             case 44:  // CAVLC 4:4:4 Intra profile
385             case 86:  // Scalable High profile
386             case 100: // High profile
387             case 110: // High 10 profile
388             case 122: // High 4:2:2 profile
389             case 244: // High 4:4:4 Predictive profile
390                 if (sps->constraint_set3_flag)
391                     max_dec_frame_buffering = 0;
392                 break;
393             }
394         }
395     }
396
397     if (max_dec_frame_buffering > 16)
398         max_dec_frame_buffering = 16;
399     else if (max_dec_frame_buffering < sps->num_ref_frames)
400         max_dec_frame_buffering = sps->num_ref_frames;
401     return MAX(1, max_dec_frame_buffering);
402 }
403
404 static void
405 dpb_remove_index(GstVaapiDecoderH264 *decoder, guint index)
406 {
407     GstVaapiDecoderH264Private * const priv = decoder->priv;
408     guint i, num_pictures = --priv->dpb_count;
409
410     if (USE_STRICT_DPB_ORDERING) {
411         for (i = index; i < num_pictures; i++)
412             gst_vaapi_picture_replace(&priv->dpb[i], priv->dpb[i + 1]);
413     }
414     else if (index != num_pictures)
415         gst_vaapi_picture_replace(&priv->dpb[index], priv->dpb[num_pictures]);
416     gst_vaapi_picture_replace(&priv->dpb[num_pictures], NULL);
417 }
418
419 static inline gboolean
420 dpb_output(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
421 {
422     /* XXX: update cropping rectangle */
423     picture->output_needed = FALSE;
424     return gst_vaapi_picture_output(GST_VAAPI_PICTURE_CAST(picture));
425 }
426
427 static gboolean
428 dpb_bump(GstVaapiDecoderH264 *decoder)
429 {
430     GstVaapiDecoderH264Private * const priv = decoder->priv;
431     guint i, lowest_poc_index;
432     gboolean success;
433
434     for (i = 0; i < priv->dpb_count; i++) {
435         if (priv->dpb[i]->output_needed)
436             break;
437     }
438     if (i == priv->dpb_count)
439         return FALSE;
440
441     lowest_poc_index = i++;
442     for (; i < priv->dpb_count; i++) {
443         GstVaapiPictureH264 * const picture = priv->dpb[i];
444         if (picture->output_needed && picture->base.poc < priv->dpb[lowest_poc_index]->base.poc)
445             lowest_poc_index = i;
446     }
447
448     success = dpb_output(decoder, priv->dpb[lowest_poc_index]);
449     if (!GST_VAAPI_PICTURE_IS_REFERENCE(priv->dpb[lowest_poc_index]))
450         dpb_remove_index(decoder, lowest_poc_index);
451     return success;
452 }
453
454 static void
455 dpb_flush(GstVaapiDecoderH264 *decoder)
456 {
457     GstVaapiDecoderH264Private * const priv = decoder->priv;
458
459     while (dpb_bump(decoder))
460         ;
461     clear_references(decoder, priv->dpb, &priv->dpb_count);
462 }
463
464 static gboolean
465 dpb_add(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
466 {
467     GstVaapiDecoderH264Private * const priv = decoder->priv;
468     guint i;
469
470     // Remove all unused pictures
471     if (GST_VAAPI_PICTURE_IS_IDR(picture))
472         dpb_flush(decoder);
473     else {
474         i = 0;
475         while (i < priv->dpb_count) {
476             GstVaapiPictureH264 * const picture = priv->dpb[i];
477             if (!picture->output_needed &&
478                 !GST_VAAPI_PICTURE_IS_REFERENCE(picture))
479                 dpb_remove_index(decoder, i);
480             else
481                 i++;
482         }
483     }
484
485     // C.4.5.1 - Storage and marking of a reference decoded picture into the DPB
486     if (GST_VAAPI_PICTURE_IS_REFERENCE(picture)) {
487         while (priv->dpb_count == priv->dpb_size) {
488             if (!dpb_bump(decoder))
489                 return FALSE;
490         }
491         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
492         if (picture->output_flag)
493             picture->output_needed = TRUE;
494     }
495
496     // C.4.5.2 - Storage and marking of a non-reference decoded picture into the DPB
497     else {
498         if (!picture->output_flag)
499             return TRUE;
500         while (priv->dpb_count == priv->dpb_size) {
501             for (i = 0; i < priv->dpb_count; i++) {
502                 if (priv->dpb[i]->output_needed &&
503                     priv->dpb[i]->base.poc < picture->base.poc)
504                     break;
505             }
506             if (i == priv->dpb_count)
507                 return dpb_output(decoder, picture);
508             if (!dpb_bump(decoder))
509                 return FALSE;
510         }
511         gst_vaapi_picture_replace(&priv->dpb[priv->dpb_count++], picture);
512         picture->output_needed = TRUE;
513     }
514     return TRUE;
515 }
516
517 static inline void
518 dpb_reset(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
519 {
520     GstVaapiDecoderH264Private * const priv = decoder->priv;
521
522     priv->dpb_size = get_max_dec_frame_buffering(sps);
523     GST_DEBUG("DPB size %u", priv->dpb_size);
524 }
525
526 static GstVaapiDecoderStatus
527 get_status(GstH264ParserResult result)
528 {
529     GstVaapiDecoderStatus status;
530
531     switch (result) {
532     case GST_H264_PARSER_OK:
533         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
534         break;
535     case GST_H264_PARSER_NO_NAL_END:
536         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
537         break;
538     case GST_H264_PARSER_ERROR:
539         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
540         break;
541     default:
542         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
543         break;
544     }
545     return status;
546 }
547
548 static void
549 gst_vaapi_decoder_h264_close(GstVaapiDecoderH264 *decoder)
550 {
551     GstVaapiDecoderH264Private * const priv = decoder->priv;
552
553     gst_vaapi_picture_replace(&priv->current_picture, NULL);
554     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
555     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
556     clear_references(decoder, priv->dpb,       &priv->dpb_count      );
557
558     if (priv->parser) {
559         gst_h264_nal_parser_free(priv->parser);
560         priv->parser = NULL;
561     }
562
563     if (priv->adapter) {
564         gst_adapter_clear(priv->adapter);
565         g_object_unref(priv->adapter);
566         priv->adapter = NULL;
567     }
568 }
569
570 static gboolean
571 gst_vaapi_decoder_h264_open(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
572 {
573     GstVaapiDecoderH264Private * const priv = decoder->priv;
574
575     gst_vaapi_decoder_h264_close(decoder);
576
577     priv->adapter = gst_adapter_new();
578     if (!priv->adapter)
579         return FALSE;
580
581     priv->parser = gst_h264_nal_parser_new();
582     if (!priv->parser)
583         return FALSE;
584     return TRUE;
585 }
586
587 static void
588 gst_vaapi_decoder_h264_destroy(GstVaapiDecoderH264 *decoder)
589 {
590     gst_vaapi_decoder_h264_close(decoder);
591 }
592
593 static gboolean
594 gst_vaapi_decoder_h264_create(GstVaapiDecoderH264 *decoder)
595 {
596     if (!GST_VAAPI_DECODER_CODEC(decoder))
597         return FALSE;
598     return TRUE;
599 }
600
601 static guint
602 h264_get_profile(GstH264SPS *sps)
603 {
604     guint profile = 0;
605
606     switch (sps->profile_idc) {
607     case 66:
608         profile = GST_VAAPI_PROFILE_H264_BASELINE;
609         break;
610     case 77:
611         profile = GST_VAAPI_PROFILE_H264_MAIN;
612         break;
613     case 100:
614         profile = GST_VAAPI_PROFILE_H264_HIGH;
615         break;
616     }
617     return profile;
618 }
619
620 static guint
621 h264_get_chroma_type(GstH264SPS *sps)
622 {
623     guint chroma_type = 0;
624
625     switch (sps->chroma_format_idc) {
626     case 1:
627         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV420;
628         break;
629     case 2:
630         chroma_type = GST_VAAPI_CHROMA_TYPE_YUV422;
631         break;
632     case 3:
633         if (!sps->separate_colour_plane_flag)
634             chroma_type = GST_VAAPI_CHROMA_TYPE_YUV444;
635         break;
636     }
637     return chroma_type;
638 }
639
640 static GstVaapiProfile
641 get_profile(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
642 {
643     GstVaapiDecoderH264Private * const priv = decoder->priv;
644     GstVaapiDisplay * const display = GST_VAAPI_DECODER_DISPLAY(decoder);
645     GstVaapiProfile profile, profiles[2];
646     guint i, n_profiles = 0;
647
648     profile = h264_get_profile(sps);
649     if (!profile)
650         return GST_VAAPI_PROFILE_UNKNOWN;
651
652     profiles[n_profiles++] = profile;
653     switch (profile) {
654     case GST_VAAPI_PROFILE_H264_MAIN:
655         profiles[n_profiles++] = GST_VAAPI_PROFILE_H264_HIGH;
656         break;
657     default:
658         break;
659     }
660
661     /* If the preferred profile (profiles[0]) matches one that we already
662        found, then just return it now instead of searching for it again */
663     if (profiles[0] == priv->profile)
664         return priv->profile;
665
666     for (i = 0; i < n_profiles; i++) {
667         if (gst_vaapi_display_has_decoder(display, profiles[i], priv->entrypoint))
668             return profiles[i];
669     }
670     return GST_VAAPI_PROFILE_UNKNOWN;
671 }
672
673 static GstVaapiDecoderStatus
674 ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
675 {
676     GstVaapiDecoder * const base_decoder = GST_VAAPI_DECODER_CAST(decoder);
677     GstVaapiDecoderH264Private * const priv = decoder->priv;
678     GstVaapiContextInfo info;
679     GstVaapiProfile profile;
680     GstVaapiChromaType chroma_type;
681     gboolean reset_context = FALSE;
682
683     profile = get_profile(decoder, sps);
684     if (!profile) {
685         GST_ERROR("unsupported profile_idc %u", sps->profile_idc);
686         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
687     }
688
689     if (priv->profile != profile) {
690         GST_DEBUG("profile changed");
691         reset_context = TRUE;
692         priv->profile = profile;
693     }
694
695     chroma_type = h264_get_chroma_type(sps);
696     if (!chroma_type || chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420) {
697         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
698         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
699     }
700
701     if (priv->chroma_type != chroma_type) {
702         GST_DEBUG("chroma format changed");
703         reset_context     = TRUE;
704         priv->chroma_type = chroma_type;
705     }
706
707     if (priv->width != sps->width || priv->height != sps->height) {
708         GST_DEBUG("size changed");
709         reset_context = TRUE;
710         priv->width   = sps->width;
711         priv->height  = sps->height;
712     }
713
714     gst_vaapi_decoder_set_pixel_aspect_ratio(
715         base_decoder,
716         sps->vui_parameters.par_n,
717         sps->vui_parameters.par_d
718     );
719
720     if (!reset_context && priv->has_context)
721         return GST_VAAPI_DECODER_STATUS_SUCCESS;
722
723     info.profile    = priv->profile;
724     info.entrypoint = priv->entrypoint;
725     info.width      = priv->width;
726     info.height     = priv->height;
727     info.ref_frames = get_max_dec_frame_buffering(sps);
728
729     if (!gst_vaapi_decoder_ensure_context(GST_VAAPI_DECODER(decoder), &info))
730         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
731     priv->has_context = TRUE;
732
733     /* Reset DPB */
734     dpb_reset(decoder, sps);
735     return GST_VAAPI_DECODER_STATUS_SUCCESS;
736 }
737
738 static void
739 fill_iq_matrix_4x4(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
740 {
741     const guint8 (* const ScalingList4x4)[6][16] = &pps->scaling_lists_4x4;
742     guint i, j;
743
744     /* There are always 6 4x4 scaling lists */
745     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4) == 6);
746     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList4x4[0]) == 16);
747
748     if (sizeof(iq_matrix->ScalingList4x4[0][0]) == 1)
749         memcpy(iq_matrix->ScalingList4x4, *ScalingList4x4,
750                sizeof(iq_matrix->ScalingList4x4));
751     else {
752         for (i = 0; i < G_N_ELEMENTS(iq_matrix->ScalingList4x4); i++) {
753             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList4x4[i]); j++)
754                 iq_matrix->ScalingList4x4[i][j] = (*ScalingList4x4)[i][j];
755         }
756     }
757 }
758
759 static void
760 fill_iq_matrix_8x8(VAIQMatrixBufferH264 *iq_matrix, const GstH264PPS *pps)
761 {
762     const guint8 (* const ScalingList8x8)[6][64] = &pps->scaling_lists_8x8;
763     const GstH264SPS * const sps = pps->sequence;
764     guint i, j, n;
765
766     /* If chroma_format_idc != 3, there are up to 2 8x8 scaling lists */
767     if (!pps->transform_8x8_mode_flag)
768         return;
769
770     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8) >= 2);
771     g_assert(G_N_ELEMENTS(iq_matrix->ScalingList8x8[0]) == 64);
772
773     if (sizeof(iq_matrix->ScalingList8x8[0][0]) == 1)
774         memcpy(iq_matrix->ScalingList8x8, *ScalingList8x8,
775                sizeof(iq_matrix->ScalingList8x8));
776     else {
777         n = (sps->chroma_format_idc != 3) ? 2 : 6;
778         for (i = 0; i < n; i++) {
779             for (j = 0; j < G_N_ELEMENTS(iq_matrix->ScalingList8x8[i]); j++)
780                 iq_matrix->ScalingList8x8[i][j] = (*ScalingList8x8)[i][j];
781         }
782     }
783 }
784
785 static GstVaapiDecoderStatus
786 ensure_quant_matrix(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
787 {
788     GstVaapiPicture * const base_picture = &picture->base;
789     GstH264PPS * const pps = picture->pps;
790     GstH264SPS * const sps = pps->sequence;
791     VAIQMatrixBufferH264 *iq_matrix;
792
793     base_picture->iq_matrix = GST_VAAPI_IQ_MATRIX_NEW(H264, decoder);
794     if (!base_picture->iq_matrix) {
795         GST_ERROR("failed to allocate IQ matrix");
796         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
797     }
798     iq_matrix = base_picture->iq_matrix->param;
799
800     /* XXX: we can only support 4:2:0 or 4:2:2 since ScalingLists8x8[]
801        is not large enough to hold lists for 4:4:4 */
802     if (sps->chroma_format_idc == 3)
803         return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CHROMA_FORMAT;
804
805     fill_iq_matrix_4x4(iq_matrix, pps);
806     fill_iq_matrix_8x8(iq_matrix, pps);
807
808     return GST_VAAPI_DECODER_STATUS_SUCCESS;
809 }
810
811 static GstVaapiDecoderStatus
812 decode_current_picture(GstVaapiDecoderH264 *decoder)
813 {
814     GstVaapiDecoderH264Private * const priv = decoder->priv;
815     GstVaapiPictureH264 * const picture = priv->current_picture;
816     GstVaapiDecoderStatus status;
817
818     if (!picture)
819         return GST_VAAPI_DECODER_STATUS_SUCCESS;
820
821     status = ensure_context(decoder, picture->pps->sequence);
822     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
823         return status;
824
825     if (!decode_picture_end(decoder, picture))
826         goto error;
827     if (!gst_vaapi_picture_decode(GST_VAAPI_PICTURE_CAST(picture)))
828         goto error;
829     gst_vaapi_picture_replace(&priv->current_picture, NULL);
830     return GST_VAAPI_DECODER_STATUS_SUCCESS;
831
832 error:
833     gst_vaapi_picture_replace(&priv->current_picture, NULL);
834     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
835 }
836
837 static GstVaapiDecoderStatus
838 decode_sps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
839 {
840     GstVaapiDecoderH264Private * const priv = decoder->priv;
841     GstH264SPS * const sps = &priv->last_sps;
842     GstH264ParserResult result;
843
844     GST_DEBUG("decode SPS");
845
846     memset(sps, 0, sizeof(*sps));
847     result = gst_h264_parser_parse_sps(priv->parser, nalu, sps, TRUE);
848     if (result != GST_H264_PARSER_OK)
849         return get_status(result);
850
851     return GST_VAAPI_DECODER_STATUS_SUCCESS;
852 }
853
854 static GstVaapiDecoderStatus
855 decode_pps(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
856 {
857     GstVaapiDecoderH264Private * const priv = decoder->priv;
858     GstH264PPS * const pps = &priv->last_pps;
859     GstH264ParserResult result;
860
861     GST_DEBUG("decode PPS");
862
863     memset(pps, 0, sizeof(*pps));
864     result = gst_h264_parser_parse_pps(priv->parser, nalu, pps);
865     if (result != GST_H264_PARSER_OK)
866         return get_status(result);
867
868     return GST_VAAPI_DECODER_STATUS_SUCCESS;
869 }
870
871 static GstVaapiDecoderStatus
872 decode_sei(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
873 {
874     GstVaapiDecoderH264Private * const priv = decoder->priv;
875     GstH264SEIMessage sei;
876     GstH264ParserResult result;
877
878     GST_DEBUG("decode SEI");
879
880     memset(&sei, 0, sizeof(sei));
881     result = gst_h264_parser_parse_sei(priv->parser, nalu, &sei);
882     if (result != GST_H264_PARSER_OK) {
883         GST_WARNING("failed to decode SEI, payload type:%d", sei.payloadType);
884         return get_status(result);
885     }
886
887     return GST_VAAPI_DECODER_STATUS_SUCCESS;
888 }
889
890 static GstVaapiDecoderStatus
891 decode_sequence_end(GstVaapiDecoderH264 *decoder)
892 {
893     GstVaapiDecoderStatus status;
894
895     GST_DEBUG("decode sequence-end");
896
897     status = decode_current_picture(decoder);
898     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
899         return status;
900
901     dpb_flush(decoder);
902     return GST_VAAPI_DECODER_STATUS_END_OF_STREAM;
903 }
904
905 /* 8.2.1.1 - Decoding process for picture order count type 0 */
906 static void
907 init_picture_poc_0(
908     GstVaapiDecoderH264 *decoder,
909     GstVaapiPictureH264 *picture,
910     GstH264SliceHdr     *slice_hdr
911 )
912 {
913     GstVaapiDecoderH264Private * const priv = decoder->priv;
914     GstH264PPS * const pps = slice_hdr->pps;
915     GstH264SPS * const sps = pps->sequence;
916     const gint32 MaxPicOrderCntLsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
917     gint32 temp_poc;
918
919     GST_DEBUG("decode picture order count type 0");
920
921     if (GST_VAAPI_PICTURE_IS_IDR(picture)) {
922         priv->prev_poc_msb = 0;
923         priv->prev_poc_lsb = 0;
924     }
925     else if (priv->prev_pic_has_mmco5) {
926         priv->prev_poc_msb = 0;
927         priv->prev_poc_lsb =
928             (priv->prev_pic_structure == GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD ?
929              0 : priv->field_poc[TOP_FIELD]);
930     }
931     else {
932         priv->prev_poc_msb = priv->poc_msb;
933         priv->prev_poc_lsb = priv->poc_lsb;
934     }
935
936     // (8-3)
937     priv->poc_lsb = slice_hdr->pic_order_cnt_lsb;
938     if (priv->poc_lsb < priv->prev_poc_lsb &&
939         (priv->prev_poc_lsb - priv->poc_lsb) >= (MaxPicOrderCntLsb / 2))
940         priv->poc_msb = priv->prev_poc_msb + MaxPicOrderCntLsb;
941     else if (priv->poc_lsb > priv->prev_poc_lsb &&
942              (priv->poc_lsb - priv->prev_poc_lsb) > (MaxPicOrderCntLsb / 2))
943         priv->poc_msb = priv->prev_poc_msb - MaxPicOrderCntLsb;
944     else
945         priv->poc_msb = priv->prev_poc_msb;
946
947     temp_poc = priv->poc_msb + priv->poc_lsb;
948     switch (picture->base.structure) {
949     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
950         // (8-4, 8-5)
951         priv->field_poc[TOP_FIELD] = temp_poc;
952         priv->field_poc[BOTTOM_FIELD] = temp_poc +
953             slice_hdr->delta_pic_order_cnt_bottom;
954         break;
955     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
956         // (8-4)
957         priv->field_poc[TOP_FIELD] = temp_poc;
958         break;
959     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
960         // (8-5)
961         priv->field_poc[BOTTOM_FIELD] = temp_poc;
962         break;
963     }
964 }
965
966 /* 8.2.1.2 - Decoding process for picture order count type 1 */
967 static void
968 init_picture_poc_1(
969     GstVaapiDecoderH264 *decoder,
970     GstVaapiPictureH264 *picture,
971     GstH264SliceHdr     *slice_hdr
972 )
973 {
974     GstVaapiDecoderH264Private * const priv = decoder->priv;
975     GstH264PPS * const pps = slice_hdr->pps;
976     GstH264SPS * const sps = pps->sequence;
977     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
978     gint32 prev_frame_num_offset, abs_frame_num, expected_poc;
979     guint i;
980
981     GST_DEBUG("decode picture order count type 1");
982
983     if (priv->prev_pic_has_mmco5)
984         prev_frame_num_offset = 0;
985     else
986         prev_frame_num_offset = priv->frame_num_offset;
987
988     // (8-6)
989     if (GST_VAAPI_PICTURE_IS_IDR(picture))
990         priv->frame_num_offset = 0;
991     else if (priv->prev_frame_num > priv->frame_num)
992         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
993     else
994         priv->frame_num_offset = prev_frame_num_offset;
995
996     // (8-7)
997     if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
998         abs_frame_num = priv->frame_num_offset + priv->frame_num;
999     else
1000         abs_frame_num = 0;
1001     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture) && abs_frame_num > 0)
1002         abs_frame_num = abs_frame_num - 1;
1003
1004     if (abs_frame_num > 0) {
1005         gint32 expected_delta_per_poc_cycle;
1006         gint32 poc_cycle_cnt, frame_num_in_poc_cycle;
1007
1008         expected_delta_per_poc_cycle = 0;
1009         for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++)
1010             expected_delta_per_poc_cycle += sps->offset_for_ref_frame[i];
1011
1012         // (8-8)
1013         poc_cycle_cnt = (abs_frame_num - 1) /
1014             sps->num_ref_frames_in_pic_order_cnt_cycle;
1015         frame_num_in_poc_cycle = (abs_frame_num - 1) %
1016             sps->num_ref_frames_in_pic_order_cnt_cycle;
1017
1018         // (8-9)
1019         expected_poc = poc_cycle_cnt * expected_delta_per_poc_cycle;
1020         for (i = 0; i <= frame_num_in_poc_cycle; i++)
1021             expected_poc += sps->offset_for_ref_frame[i];
1022     }
1023     else
1024         expected_poc = 0;
1025     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1026         expected_poc += sps->offset_for_non_ref_pic;
1027
1028     // (8-10)
1029     switch (picture->base.structure) {
1030     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
1031         priv->field_poc[TOP_FIELD] = expected_poc +
1032             slice_hdr->delta_pic_order_cnt[0];
1033         priv->field_poc[BOTTOM_FIELD] = priv->field_poc[TOP_FIELD] +
1034             sps->offset_for_top_to_bottom_field +
1035             slice_hdr->delta_pic_order_cnt[1];
1036         break;
1037     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
1038         priv->field_poc[TOP_FIELD] = expected_poc +
1039             slice_hdr->delta_pic_order_cnt[0];
1040         break;
1041     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
1042         priv->field_poc[BOTTOM_FIELD] = expected_poc + 
1043             sps->offset_for_top_to_bottom_field +
1044             slice_hdr->delta_pic_order_cnt[0];
1045         break;
1046     }
1047 }
1048
1049 /* 8.2.1.3 - Decoding process for picture order count type 2 */
1050 static void
1051 init_picture_poc_2(
1052     GstVaapiDecoderH264 *decoder,
1053     GstVaapiPictureH264 *picture,
1054     GstH264SliceHdr     *slice_hdr
1055 )
1056 {
1057     GstVaapiDecoderH264Private * const priv = decoder->priv;
1058     GstH264PPS * const pps = slice_hdr->pps;
1059     GstH264SPS * const sps = pps->sequence;
1060     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1061     gint32 prev_frame_num_offset, temp_poc;
1062
1063     GST_DEBUG("decode picture order count type 2");
1064
1065     if (priv->prev_pic_has_mmco5)
1066         prev_frame_num_offset = 0;
1067     else
1068         prev_frame_num_offset = priv->frame_num_offset;
1069
1070     // (8-11)
1071     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1072         priv->frame_num_offset = 0;
1073     else if (priv->prev_frame_num > priv->frame_num)
1074         priv->frame_num_offset = prev_frame_num_offset + MaxFrameNum;
1075     else
1076         priv->frame_num_offset = prev_frame_num_offset;
1077
1078     // (8-12)
1079     if (GST_VAAPI_PICTURE_IS_IDR(picture))
1080         temp_poc = 0;
1081     else if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
1082         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num) - 1;
1083     else
1084         temp_poc = 2 * (priv->frame_num_offset + priv->frame_num);
1085
1086     // (8-13)
1087     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1088         priv->field_poc[TOP_FIELD] = temp_poc;
1089     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1090         priv->field_poc[BOTTOM_FIELD] = temp_poc;
1091 }
1092
1093 /* 8.2.1 - Decoding process for picture order count */
1094 static void
1095 init_picture_poc(
1096     GstVaapiDecoderH264 *decoder,
1097     GstVaapiPictureH264 *picture,
1098     GstH264SliceHdr     *slice_hdr
1099 )
1100 {
1101     GstVaapiDecoderH264Private * const priv = decoder->priv;
1102     GstH264PPS * const pps = slice_hdr->pps;
1103     GstH264SPS * const sps = pps->sequence;
1104
1105     switch (sps->pic_order_cnt_type) {
1106     case 0:
1107         init_picture_poc_0(decoder, picture, slice_hdr);
1108         break;
1109     case 1:
1110         init_picture_poc_1(decoder, picture, slice_hdr);
1111         break;
1112     case 2:
1113         init_picture_poc_2(decoder, picture, slice_hdr);
1114         break;
1115     }
1116
1117     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1118         picture->field_poc[TOP_FIELD] = priv->field_poc[TOP_FIELD];
1119     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1120         picture->field_poc[BOTTOM_FIELD] = priv->field_poc[BOTTOM_FIELD];
1121     picture->base.poc = MIN(picture->field_poc[0], picture->field_poc[1]);
1122 }
1123
1124 static int
1125 compare_picture_pic_num_dec(const void *a, const void *b)
1126 {
1127     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1128     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1129
1130     return picB->pic_num - picA->pic_num;
1131 }
1132
1133 static int
1134 compare_picture_long_term_pic_num_inc(const void *a, const void *b)
1135 {
1136     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1137     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1138
1139     return picA->long_term_pic_num - picB->long_term_pic_num;
1140 }
1141
1142 static int
1143 compare_picture_poc_dec(const void *a, const void *b)
1144 {
1145     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1146     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1147
1148     return picB->base.poc - picA->base.poc;
1149 }
1150
1151 static int
1152 compare_picture_poc_inc(const void *a, const void *b)
1153 {
1154     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1155     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1156
1157     return picA->base.poc - picB->base.poc;
1158 }
1159
1160 static int
1161 compare_picture_frame_num_wrap_dec(const void *a, const void *b)
1162 {
1163     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1164     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1165
1166     return picB->frame_num_wrap - picA->frame_num_wrap;
1167 }
1168
1169 static int
1170 compare_picture_long_term_frame_idx_inc(const void *a, const void *b)
1171 {
1172     const GstVaapiPictureH264 * const picA = *(GstVaapiPictureH264 **)a;
1173     const GstVaapiPictureH264 * const picB = *(GstVaapiPictureH264 **)b;
1174
1175     return picA->long_term_frame_idx - picB->long_term_frame_idx;
1176 }
1177
1178 /* 8.2.4.1 - Decoding process for picture numbers */
1179 static void
1180 init_picture_refs_pic_num(
1181     GstVaapiDecoderH264 *decoder,
1182     GstVaapiPictureH264 *picture,
1183     GstH264SliceHdr     *slice_hdr
1184 )
1185 {
1186     GstVaapiDecoderH264Private * const priv = decoder->priv;
1187     GstH264PPS * const pps = slice_hdr->pps;
1188     GstH264SPS * const sps = pps->sequence;
1189     const gint32 MaxFrameNum = 1 << (sps->log2_max_frame_num_minus4 + 4);
1190     guint i;
1191
1192     GST_DEBUG("decode picture numbers");
1193
1194     for (i = 0; i < priv->short_ref_count; i++) {
1195         GstVaapiPictureH264 * const pic = priv->short_ref[i];
1196
1197         // (8-27)
1198         if (pic->frame_num > priv->frame_num)
1199             pic->frame_num_wrap = pic->frame_num - MaxFrameNum;
1200         else
1201             pic->frame_num_wrap = pic->frame_num;
1202
1203         // (8-28, 8-30, 8-31)
1204         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1205             pic->pic_num = pic->frame_num_wrap;
1206         else {
1207             if (pic->base.structure == picture->base.structure)
1208                 pic->pic_num = 2 * pic->frame_num_wrap + 1;
1209             else
1210                 pic->pic_num = 2 * pic->frame_num_wrap;
1211         }
1212     }
1213
1214     for (i = 0; i < priv->long_ref_count; i++) {
1215         GstVaapiPictureH264 * const pic = priv->long_ref[i];
1216
1217         // (8-29, 8-32, 8-33)
1218         if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1219             pic->long_term_pic_num = pic->long_term_frame_idx;
1220         else {
1221             if (pic->base.structure == picture->base.structure)
1222                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx + 1;
1223             else
1224                 pic->long_term_pic_num = 2 * pic->long_term_frame_idx;
1225         }
1226     }
1227 }
1228
1229 #define SORT_REF_LIST(list, n, compare_func) \
1230     qsort(list, n, sizeof(*(list)), compare_picture_##compare_func)
1231
1232 static void
1233 init_picture_refs_p_slice(
1234     GstVaapiDecoderH264 *decoder,
1235     GstVaapiPictureH264 *picture,
1236     GstH264SliceHdr     *slice_hdr
1237 )
1238 {
1239     GstVaapiDecoderH264Private * const priv = decoder->priv;
1240     GstVaapiPictureH264 **ref_list;
1241     guint i;
1242
1243     GST_DEBUG("decode reference picture list for P and SP slices");
1244
1245     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1246         /* 8.2.4.2.1 - P and SP slices in frames */
1247         if (priv->short_ref_count > 0) {
1248             ref_list = priv->RefPicList0;
1249             for (i = 0; i < priv->short_ref_count; i++)
1250                 ref_list[i] = priv->short_ref[i];
1251             SORT_REF_LIST(ref_list, i, pic_num_dec);
1252             priv->RefPicList0_count += i;
1253         }
1254
1255         if (priv->long_ref_count > 0) {
1256             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1257             for (i = 0; i < priv->long_ref_count; i++)
1258                 ref_list[i] = priv->long_ref[i];
1259             SORT_REF_LIST(ref_list, i, long_term_pic_num_inc);
1260             priv->RefPicList0_count += i;
1261         }
1262     }
1263     else {
1264         /* 8.2.4.2.2 - P and SP slices in fields */
1265         GstVaapiPictureH264 *short_ref[32];
1266         guint short_ref_count = 0;
1267         GstVaapiPictureH264 *long_ref[32];
1268         guint long_ref_count = 0;
1269
1270         // XXX: handle second field if current field is marked as
1271         // "used for short-term reference"
1272         if (priv->short_ref_count > 0) {
1273             for (i = 0; i < priv->short_ref_count; i++)
1274                 short_ref[i] = priv->short_ref[i];
1275             SORT_REF_LIST(short_ref, i, frame_num_wrap_dec);
1276             short_ref_count = i;
1277         }
1278
1279         // XXX: handle second field if current field is marked as
1280         // "used for long-term reference"
1281         if (priv->long_ref_count > 0) {
1282             for (i = 0; i < priv->long_ref_count; i++)
1283                 long_ref[i] = priv->long_ref[i];
1284             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1285             long_ref_count = i;
1286         }
1287
1288         // XXX: handle 8.2.4.2.5
1289     }
1290 }
1291
1292 static void
1293 init_picture_refs_b_slice(
1294     GstVaapiDecoderH264 *decoder,
1295     GstVaapiPictureH264 *picture,
1296     GstH264SliceHdr     *slice_hdr
1297 )
1298 {
1299     GstVaapiDecoderH264Private * const priv = decoder->priv;
1300     GstVaapiPictureH264 **ref_list;
1301     guint i, n;
1302
1303     GST_DEBUG("decode reference picture list for B slices");
1304
1305     if (GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1306         /* 8.2.4.2.3 - B slices in frames */
1307
1308         /* RefPicList0 */
1309         if (priv->short_ref_count > 0) {
1310             // 1. Short-term references
1311             ref_list = priv->RefPicList0;
1312             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1313                 if (priv->short_ref[i]->base.poc < picture->base.poc)
1314                     ref_list[n++] = priv->short_ref[i];
1315             }
1316             SORT_REF_LIST(ref_list, n, poc_dec);
1317             priv->RefPicList0_count += n;
1318
1319             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1320             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1321                 if (priv->short_ref[i]->base.poc >= picture->base.poc)
1322                     ref_list[n++] = priv->short_ref[i];
1323             }
1324             SORT_REF_LIST(ref_list, n, poc_inc);
1325             priv->RefPicList0_count += n;
1326         }
1327
1328         if (priv->long_ref_count > 0) {
1329             // 2. Long-term references
1330             ref_list = &priv->RefPicList0[priv->RefPicList0_count];
1331             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1332                 ref_list[n++] = priv->long_ref[i];
1333             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1334             priv->RefPicList0_count += n;
1335         }
1336
1337         /* RefPicList1 */
1338         if (priv->short_ref_count > 0) {
1339             // 1. Short-term references
1340             ref_list = priv->RefPicList1;
1341             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1342                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1343                     ref_list[n++] = priv->short_ref[i];
1344             }
1345             SORT_REF_LIST(ref_list, n, poc_inc);
1346             priv->RefPicList1_count += n;
1347
1348             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1349             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1350                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1351                     ref_list[n++] = priv->short_ref[i];
1352             }
1353             SORT_REF_LIST(ref_list, n, poc_dec);
1354             priv->RefPicList1_count += n;
1355         }
1356
1357         if (priv->long_ref_count > 0) {
1358             // 2. Long-term references
1359             ref_list = &priv->RefPicList1[priv->RefPicList1_count];
1360             for (n = 0, i = 0; i < priv->long_ref_count; i++)
1361                 ref_list[n++] = priv->long_ref[i];
1362             SORT_REF_LIST(ref_list, n, long_term_pic_num_inc);
1363             priv->RefPicList1_count += n;
1364         }
1365     }
1366     else {
1367         /* 8.2.4.2.4 - B slices in fields */
1368         GstVaapiPictureH264 *short_ref0[32];
1369         guint short_ref0_count = 0;
1370         GstVaapiPictureH264 *short_ref1[32];
1371         guint short_ref1_count = 0;
1372         GstVaapiPictureH264 *long_ref[32];
1373         guint long_ref_count = 0;
1374
1375         /* refFrameList0ShortTerm */
1376         if (priv->short_ref_count > 0) {
1377             ref_list = short_ref0;
1378             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1379                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1380                     ref_list[n++] = priv->short_ref[i];
1381             }
1382             SORT_REF_LIST(ref_list, n, poc_dec);
1383             short_ref0_count += n;
1384
1385             ref_list = &short_ref0[short_ref0_count];
1386             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1387                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1388                     ref_list[n++] = priv->short_ref[i];
1389             }
1390             SORT_REF_LIST(ref_list, n, poc_inc);
1391             short_ref0_count += n;
1392         }
1393
1394         /* refFrameList1ShortTerm */
1395         if (priv->short_ref_count > 0) {
1396             ref_list = short_ref1;
1397             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1398                 if (priv->short_ref[i]->base.poc > picture->base.poc)
1399                     ref_list[n++] = priv->short_ref[i];
1400             }
1401             SORT_REF_LIST(ref_list, n, poc_inc);
1402             short_ref1_count += n;
1403
1404             ref_list = &short_ref1[short_ref1_count];
1405             for (n = 0, i = 0; i < priv->short_ref_count; i++) {
1406                 if (priv->short_ref[i]->base.poc <= picture->base.poc)
1407                     ref_list[n++] = priv->short_ref[i];
1408             }
1409             SORT_REF_LIST(ref_list, n, poc_dec);
1410             short_ref1_count += n;
1411         }
1412
1413         /* refFrameListLongTerm */
1414         if (priv->long_ref_count > 0) {
1415             for (i = 0; i < priv->long_ref_count; i++)
1416                 long_ref[i] = priv->long_ref[i];
1417             SORT_REF_LIST(long_ref, i, long_term_frame_idx_inc);
1418             long_ref_count = i;
1419         }
1420
1421         // XXX: handle 8.2.4.2.5
1422     }
1423
1424     /* Check whether RefPicList1 is identical to RefPicList0, then
1425        swap if necessary */
1426     if (priv->RefPicList1_count > 1 &&
1427         priv->RefPicList1_count == priv->RefPicList0_count &&
1428         memcmp(priv->RefPicList0, priv->RefPicList1,
1429                priv->RefPicList0_count * sizeof(priv->RefPicList0[0])) == 0) {
1430         GstVaapiPictureH264 * const tmp = priv->RefPicList1[0];
1431         priv->RefPicList1[0] = priv->RefPicList1[1];
1432         priv->RefPicList1[1] = tmp;
1433     }
1434 }
1435
1436 #undef SORT_REF_LIST
1437
1438 static void
1439 clear_references(
1440     GstVaapiDecoderH264  *decoder,
1441     GstVaapiPictureH264 **pictures,
1442     guint                *picture_count
1443 )
1444 {
1445     const guint num_pictures = *picture_count;
1446     guint i;
1447
1448     for (i = 0; i < num_pictures; i++)
1449         gst_vaapi_picture_replace(&pictures[i], NULL);
1450     *picture_count = 0;
1451 }
1452
1453 static gboolean
1454 remove_reference_at(
1455     GstVaapiDecoderH264  *decoder,
1456     GstVaapiPictureH264 **pictures,
1457     guint                *picture_count,
1458     guint                 index
1459 )
1460 {
1461     guint num_pictures = *picture_count;
1462     GstVaapiPictureH264 *picture;
1463
1464     g_return_val_if_fail(index < num_pictures, FALSE);
1465
1466     picture = pictures[index];
1467     GST_VAAPI_PICTURE_FLAG_UNSET(picture, GST_VAAPI_PICTURE_FLAGS_REFERENCE);
1468
1469     if (index != --num_pictures)
1470         gst_vaapi_picture_replace(&pictures[index], pictures[num_pictures]);
1471     gst_vaapi_picture_replace(&pictures[num_pictures], NULL);
1472     *picture_count = num_pictures;
1473     return TRUE;
1474 }
1475
1476 static gint
1477 find_short_term_reference(GstVaapiDecoderH264 *decoder, gint32 pic_num)
1478 {
1479     GstVaapiDecoderH264Private * const priv = decoder->priv;
1480     guint i;
1481
1482     for (i = 0; i < priv->short_ref_count; i++) {
1483         if (priv->short_ref[i]->pic_num == pic_num)
1484             return i;
1485     }
1486     GST_ERROR("found no short-term reference picture with PicNum = %d",
1487               pic_num);
1488     return -1;
1489 }
1490
1491 static gint
1492 find_long_term_reference(GstVaapiDecoderH264 *decoder, gint32 long_term_pic_num)
1493 {
1494     GstVaapiDecoderH264Private * const priv = decoder->priv;
1495     guint i;
1496
1497     for (i = 0; i < priv->long_ref_count; i++) {
1498         if (priv->long_ref[i]->long_term_pic_num == long_term_pic_num)
1499             return i;
1500     }
1501     GST_ERROR("found no long-term reference picture with LongTermPicNum = %d",
1502               long_term_pic_num);
1503     return -1;
1504 }
1505
1506 static void
1507 exec_picture_refs_modification_1(
1508     GstVaapiDecoderH264           *decoder,
1509     GstVaapiPictureH264           *picture,
1510     GstH264SliceHdr               *slice_hdr,
1511     guint                          list
1512 )
1513 {
1514     GstVaapiDecoderH264Private * const priv = decoder->priv;
1515     GstH264PPS * const pps = slice_hdr->pps;
1516     GstH264SPS * const sps = pps->sequence;
1517     GstH264RefPicListModification *ref_pic_list_modification;
1518     guint num_ref_pic_list_modifications;
1519     GstVaapiPictureH264 **ref_list;
1520     guint *ref_list_count_ptr, ref_list_count, ref_list_idx = 0;
1521     guint i, j, n, num_refs;
1522     gint found_ref_idx;
1523     gint32 MaxPicNum, CurrPicNum, picNumPred;
1524
1525     GST_DEBUG("modification process of reference picture list %u", list);
1526
1527     if (list == 0) {
1528         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l0;
1529         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
1530         ref_list                       = priv->RefPicList0;
1531         ref_list_count_ptr             = &priv->RefPicList0_count;
1532         num_refs                       = slice_hdr->num_ref_idx_l0_active_minus1 + 1;
1533     }
1534     else {
1535         ref_pic_list_modification      = slice_hdr->ref_pic_list_modification_l1;
1536         num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
1537         ref_list                       = priv->RefPicList1;
1538         ref_list_count_ptr             = &priv->RefPicList1_count;
1539         num_refs                       = slice_hdr->num_ref_idx_l1_active_minus1 + 1;
1540     }
1541     ref_list_count = *ref_list_count_ptr;
1542
1543     if (!GST_VAAPI_PICTURE_IS_FRAME(picture)) {
1544         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 5); // 2 * MaxFrameNum
1545         CurrPicNum = 2 * slice_hdr->frame_num + 1;              // 2 * frame_num + 1
1546     }
1547     else {
1548         MaxPicNum  = 1 << (sps->log2_max_frame_num_minus4 + 4); // MaxFrameNum
1549         CurrPicNum = slice_hdr->frame_num;                      // frame_num
1550     }
1551
1552     picNumPred = CurrPicNum;
1553
1554     for (i = 0; i < num_ref_pic_list_modifications; i++) {
1555         GstH264RefPicListModification * const l = &ref_pic_list_modification[i];
1556         if (l->modification_of_pic_nums_idc == 3)
1557             break;
1558
1559         /* 8.2.4.3.1 - Short-term reference pictures */
1560         if (l->modification_of_pic_nums_idc == 0 || l->modification_of_pic_nums_idc == 1) {
1561             gint32 abs_diff_pic_num = l->value.abs_diff_pic_num_minus1 + 1;
1562             gint32 picNum, picNumNoWrap;
1563
1564             // (8-34)
1565             if (l->modification_of_pic_nums_idc == 0) {
1566                 picNumNoWrap = picNumPred - abs_diff_pic_num;
1567                 if (picNumNoWrap < 0)
1568                     picNumNoWrap += MaxPicNum;
1569             }
1570
1571             // (8-35)
1572             else {
1573                 picNumNoWrap = picNumPred + abs_diff_pic_num;
1574                 if (picNumNoWrap >= MaxPicNum)
1575                     picNumNoWrap -= MaxPicNum;
1576             }
1577             picNumPred = picNumNoWrap;
1578
1579             // (8-36)
1580             picNum = picNumNoWrap;
1581             if (picNum > CurrPicNum)
1582                 picNum -= MaxPicNum;
1583
1584             // (8-37)
1585             for (j = num_refs; j > ref_list_idx; j--)
1586                 ref_list[j] = ref_list[j - 1];
1587             found_ref_idx = find_short_term_reference(decoder, picNum);
1588             ref_list[ref_list_idx++] =
1589                 found_ref_idx >= 0 ? priv->short_ref[found_ref_idx] : NULL;
1590             n = ref_list_idx;
1591             for (j = ref_list_idx; j <= num_refs; j++) {
1592                 gint32 PicNumF;
1593                 if (!ref_list[j])
1594                     continue;
1595                 PicNumF =
1596                     GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(ref_list[j]) ?
1597                     ref_list[j]->pic_num : MaxPicNum;
1598                 if (PicNumF != picNum)
1599                     ref_list[n++] = ref_list[j];
1600             }
1601         }
1602
1603         /* 8.2.4.3.2 - Long-term reference pictures */
1604         else {
1605
1606             for (j = num_refs; j > ref_list_idx; j--)
1607                 ref_list[j] = ref_list[j - 1];
1608             found_ref_idx =
1609                 find_long_term_reference(decoder, l->value.long_term_pic_num);
1610             ref_list[ref_list_idx++] =
1611                 found_ref_idx >= 0 ? priv->long_ref[found_ref_idx] : NULL;
1612             n = ref_list_idx;
1613             for (j = ref_list_idx; j <= num_refs; j++) {
1614                 gint32 LongTermPicNumF;
1615                 if (!ref_list[j])
1616                     continue;
1617                 LongTermPicNumF =
1618                     GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(ref_list[j]) ?
1619                     ref_list[j]->long_term_pic_num : INT_MAX;
1620                 if (LongTermPicNumF != l->value.long_term_pic_num)
1621                     ref_list[n++] = ref_list[j];
1622             }
1623         }
1624     }
1625
1626 #if DEBUG
1627     for (i = 0; i < num_refs; i++)
1628         if (!ref_list[i])
1629             GST_ERROR("list %u entry %u is empty", list, i);
1630 #endif
1631     *ref_list_count_ptr = num_refs;
1632 }
1633
1634 /* 8.2.4.3 - Modification process for reference picture lists */
1635 static void
1636 exec_picture_refs_modification(
1637     GstVaapiDecoderH264 *decoder,
1638     GstVaapiPictureH264 *picture,
1639     GstH264SliceHdr     *slice_hdr
1640 )
1641 {
1642     GST_DEBUG("execute ref_pic_list_modification()");
1643
1644     /* RefPicList0 */
1645     if (!GST_H264_IS_I_SLICE(slice_hdr) && !GST_H264_IS_SI_SLICE(slice_hdr) &&
1646         slice_hdr->ref_pic_list_modification_flag_l0)
1647         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 0);
1648
1649     /* RefPicList1 */
1650     if (GST_H264_IS_B_SLICE(slice_hdr) &&
1651         slice_hdr->ref_pic_list_modification_flag_l1)
1652         exec_picture_refs_modification_1(decoder, picture, slice_hdr, 1);
1653 }
1654
1655 static gboolean
1656 init_picture_refs(
1657     GstVaapiDecoderH264 *decoder,
1658     GstVaapiPictureH264 *picture,
1659     GstH264SliceHdr     *slice_hdr
1660 )
1661 {
1662     GstVaapiDecoderH264Private * const priv = decoder->priv;
1663     GstVaapiPicture * const base_picture = &picture->base;
1664     guint i, num_refs;
1665
1666     init_picture_refs_pic_num(decoder, picture, slice_hdr);
1667
1668     priv->RefPicList0_count = 0;
1669     priv->RefPicList1_count = 0;
1670
1671     switch (base_picture->type) {
1672     case GST_VAAPI_PICTURE_TYPE_P:
1673     case GST_VAAPI_PICTURE_TYPE_SP:
1674         init_picture_refs_p_slice(decoder, picture, slice_hdr);
1675         break;
1676     case GST_VAAPI_PICTURE_TYPE_B:
1677         init_picture_refs_b_slice(decoder, picture, slice_hdr);
1678         break;
1679     default:
1680         break;
1681     }
1682
1683     exec_picture_refs_modification(decoder, picture, slice_hdr);
1684
1685     switch (base_picture->type) {
1686     case GST_VAAPI_PICTURE_TYPE_B:
1687         num_refs = 1 + slice_hdr->num_ref_idx_l1_active_minus1;
1688         for (i = priv->RefPicList1_count; i < num_refs; i++)
1689             priv->RefPicList1[i] = NULL;
1690         priv->RefPicList1_count = num_refs;
1691
1692         // fall-through
1693     case GST_VAAPI_PICTURE_TYPE_P:
1694     case GST_VAAPI_PICTURE_TYPE_SP:
1695         num_refs = 1 + slice_hdr->num_ref_idx_l0_active_minus1;
1696         for (i = priv->RefPicList0_count; i < num_refs; i++)
1697             priv->RefPicList0[i] = NULL;
1698         priv->RefPicList0_count = num_refs;
1699         break;
1700     default:
1701         break;
1702     }
1703     return TRUE;
1704 }
1705
1706 static gboolean
1707 init_picture(
1708     GstVaapiDecoderH264 *decoder,
1709     GstVaapiPictureH264 *picture,
1710     GstH264SliceHdr     *slice_hdr,
1711     GstH264NalUnit      *nalu
1712 )
1713 {
1714     GstVaapiDecoderH264Private * const priv = decoder->priv;
1715     GstVaapiPicture * const base_picture = &picture->base;
1716
1717     priv->prev_frame_num        = priv->frame_num;
1718     priv->frame_num             = slice_hdr->frame_num;
1719     picture->frame_num          = priv->frame_num;
1720     picture->frame_num_wrap     = priv->frame_num;
1721     picture->output_flag        = TRUE; /* XXX: conformant to Annex A only */
1722     base_picture->pts           = gst_adapter_prev_timestamp(priv->adapter, NULL);
1723
1724     /* Reset decoder state for IDR pictures */
1725     if (nalu->type == GST_H264_NAL_SLICE_IDR) {
1726         GST_DEBUG("<IDR>");
1727         GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_IDR);
1728         clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1729         clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1730     }
1731
1732     /* Initialize slice type */
1733     switch (slice_hdr->type % 5) {
1734     case GST_H264_P_SLICE:
1735         base_picture->type = GST_VAAPI_PICTURE_TYPE_P;
1736         break;
1737     case GST_H264_B_SLICE:
1738         base_picture->type = GST_VAAPI_PICTURE_TYPE_B;
1739         break;
1740     case GST_H264_I_SLICE:
1741         base_picture->type = GST_VAAPI_PICTURE_TYPE_I;
1742         break;
1743     case GST_H264_SP_SLICE:
1744         base_picture->type = GST_VAAPI_PICTURE_TYPE_SP;
1745         break;
1746     case GST_H264_SI_SLICE:
1747         base_picture->type = GST_VAAPI_PICTURE_TYPE_SI;
1748         break;
1749     }
1750
1751     /* Initialize picture structure */
1752     if (!slice_hdr->field_pic_flag)
1753         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
1754     else if (!slice_hdr->bottom_field_flag)
1755         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD;
1756     else
1757         base_picture->structure = GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD;
1758
1759     if (nalu->ref_idc) {
1760         GstH264DecRefPicMarking * const dec_ref_pic_marking =
1761             &slice_hdr->dec_ref_pic_marking;
1762
1763         if (GST_VAAPI_PICTURE_IS_IDR(picture) &&
1764             dec_ref_pic_marking->long_term_reference_flag)
1765             GST_VAAPI_PICTURE_FLAG_SET(picture,
1766                 GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
1767         else
1768             GST_VAAPI_PICTURE_FLAG_SET(picture,
1769                 GST_VAAPI_PICTURE_FLAG_SHORT_TERM_REFERENCE);
1770     }
1771
1772     init_picture_poc(decoder, picture, slice_hdr);
1773     if (!init_picture_refs(decoder, picture, slice_hdr)) {
1774         GST_ERROR("failed to initialize references");
1775         return FALSE;
1776     }
1777     return TRUE;
1778 }
1779
1780 /* 8.2.5.3 - Sliding window decoded reference picture marking process */
1781 static gboolean
1782 exec_ref_pic_marking_sliding_window(GstVaapiDecoderH264 *decoder)
1783 {
1784     GstVaapiDecoderH264Private * const priv = decoder->priv;
1785     GstH264PPS * const pps = priv->current_picture->pps;
1786     GstH264SPS * const sps = pps->sequence;
1787     guint i, max_num_ref_frames, lowest_frame_num_index;
1788     gint32 lowest_frame_num;
1789
1790     GST_DEBUG("reference picture marking process (sliding window)");
1791
1792     max_num_ref_frames = sps->num_ref_frames;
1793     if (max_num_ref_frames == 0)
1794         max_num_ref_frames = 1;
1795
1796     if (priv->short_ref_count + priv->long_ref_count < max_num_ref_frames)
1797         return TRUE;
1798     if (priv->short_ref_count < 1)
1799         return FALSE;
1800
1801     lowest_frame_num = priv->short_ref[0]->frame_num_wrap;
1802     lowest_frame_num_index = 0;
1803     for (i = 1; i < priv->short_ref_count; i++) {
1804         if (priv->short_ref[i]->frame_num_wrap < lowest_frame_num) {
1805             lowest_frame_num = priv->short_ref[i]->frame_num_wrap;
1806             lowest_frame_num_index = i;
1807         }
1808     }
1809
1810     remove_reference_at(
1811         decoder,
1812         priv->short_ref, &priv->short_ref_count,
1813         lowest_frame_num_index
1814     );
1815     return TRUE;
1816 }
1817
1818 static inline gint32
1819 get_picNumX(GstVaapiPictureH264 *picture, GstH264RefPicMarking *ref_pic_marking)
1820 {
1821     gint32 pic_num;
1822
1823     if (GST_VAAPI_PICTURE_IS_FRAME(picture))
1824         pic_num = picture->frame_num_wrap;
1825     else
1826         pic_num = 2 * picture->frame_num_wrap + 1;
1827     pic_num -= ref_pic_marking->difference_of_pic_nums_minus1 + 1;
1828     return pic_num;
1829 }
1830
1831 /* 8.2.5.4.1. Mark-term reference picture as "unused for reference" */
1832 static void
1833 exec_ref_pic_marking_adaptive_mmco_1(
1834     GstVaapiDecoderH264  *decoder,
1835     GstVaapiPictureH264  *picture,
1836     GstH264RefPicMarking *ref_pic_marking
1837 )
1838 {
1839     GstVaapiDecoderH264Private * const priv = decoder->priv;
1840     gint32 i, picNumX;
1841
1842     picNumX = get_picNumX(picture, ref_pic_marking);
1843     i = find_short_term_reference(decoder, picNumX);
1844     if (i < 0)
1845         return;
1846     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1847 }
1848
1849 /* 8.2.5.4.2. Mark long-term reference picture as "unused for reference" */
1850 static void
1851 exec_ref_pic_marking_adaptive_mmco_2(
1852     GstVaapiDecoderH264  *decoder,
1853     GstVaapiPictureH264  *picture,
1854     GstH264RefPicMarking *ref_pic_marking
1855 )
1856 {
1857     GstVaapiDecoderH264Private * const priv = decoder->priv;
1858     gint32 i;
1859
1860     i = find_long_term_reference(decoder, ref_pic_marking->long_term_pic_num);
1861     if (i < 0)
1862         return;
1863     remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1864 }
1865
1866 /* 8.2.5.4.3. Assign LongTermFrameIdx to a short-term reference picture */
1867 static void
1868 exec_ref_pic_marking_adaptive_mmco_3(
1869     GstVaapiDecoderH264  *decoder,
1870     GstVaapiPictureH264  *picture,
1871     GstH264RefPicMarking *ref_pic_marking
1872 )
1873 {
1874     GstVaapiDecoderH264Private * const priv = decoder->priv;
1875     gint32 i, picNumX;
1876
1877     for (i = 0; i < priv->long_ref_count; i++) {
1878         if (priv->long_ref[i]->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
1879             break;
1880     }
1881     if (i != priv->long_ref_count)
1882         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1883
1884     picNumX = get_picNumX(picture, ref_pic_marking);
1885     i = find_short_term_reference(decoder, picNumX);
1886     if (i < 0)
1887         return;
1888
1889     picture = gst_vaapi_picture_ref(priv->short_ref[i]);
1890     remove_reference_at(decoder, priv->short_ref, &priv->short_ref_count, i);
1891     gst_vaapi_picture_replace(&priv->long_ref[priv->long_ref_count++], picture);
1892     gst_vaapi_picture_unref(picture);
1893
1894     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
1895     GST_VAAPI_PICTURE_FLAG_SET(picture,
1896         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
1897 }
1898
1899 /* 8.2.5.4.4. Mark pictures with LongTermFramIdx > max_long_term_frame_idx
1900  * as "unused for reference" */
1901 static void
1902 exec_ref_pic_marking_adaptive_mmco_4(
1903     GstVaapiDecoderH264  *decoder,
1904     GstVaapiPictureH264  *picture,
1905     GstH264RefPicMarking *ref_pic_marking
1906 )
1907 {
1908     GstVaapiDecoderH264Private * const priv = decoder->priv;
1909     gint32 i, long_term_frame_idx;
1910
1911     long_term_frame_idx = ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
1912
1913     for (i = 0; i < priv->long_ref_count; i++) {
1914         if (priv->long_ref[i]->long_term_frame_idx <= long_term_frame_idx)
1915             continue;
1916         remove_reference_at(decoder, priv->long_ref, &priv->long_ref_count, i);
1917         i--;
1918     }
1919 }
1920
1921 /* 8.2.5.4.5. Mark all reference pictures as "unused for reference" */
1922 static void
1923 exec_ref_pic_marking_adaptive_mmco_5(
1924     GstVaapiDecoderH264  *decoder,
1925     GstVaapiPictureH264  *picture,
1926     GstH264RefPicMarking *ref_pic_marking
1927 )
1928 {
1929     GstVaapiDecoderH264Private * const priv = decoder->priv;
1930
1931     clear_references(decoder, priv->short_ref, &priv->short_ref_count);
1932     clear_references(decoder, priv->long_ref,  &priv->long_ref_count );
1933     dpb_flush(decoder);
1934
1935     priv->prev_pic_has_mmco5 = TRUE;
1936
1937     /* The picture shall be inferred to have had frame_num equal to 0 (7.4.3) */
1938     priv->frame_num = 0;
1939     priv->frame_num_offset = 0;
1940     picture->frame_num = 0;
1941
1942     /* Update TopFieldOrderCnt and BottomFieldOrderCnt (8.2.1) */
1943     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD)
1944         picture->field_poc[TOP_FIELD] -= picture->base.poc;
1945     if (picture->base.structure != GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD)
1946         picture->field_poc[BOTTOM_FIELD] -= picture->base.poc;
1947     picture->base.poc = 0;
1948 }
1949
1950 /* 8.2.5.4.6. Assign a long-term frame index to the current picture */
1951 static void
1952 exec_ref_pic_marking_adaptive_mmco_6(
1953     GstVaapiDecoderH264  *decoder,
1954     GstVaapiPictureH264  *picture,
1955     GstH264RefPicMarking *ref_pic_marking
1956 )
1957 {
1958     picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
1959     GST_VAAPI_PICTURE_FLAG_SET(picture,
1960         GST_VAAPI_PICTURE_FLAG_LONG_TERM_REFERENCE);
1961 }
1962
1963 /* 8.2.5.4. Adaptive memory control decoded reference picture marking process */
1964 static gboolean
1965 exec_ref_pic_marking_adaptive(
1966     GstVaapiDecoderH264     *decoder,
1967     GstVaapiPictureH264     *picture,
1968     GstH264DecRefPicMarking *dec_ref_pic_marking
1969 )
1970 {
1971     guint i;
1972
1973     GST_DEBUG("reference picture marking process (adaptive memory control)");
1974
1975     typedef void (*exec_ref_pic_marking_adaptive_mmco_func)(
1976         GstVaapiDecoderH264  *decoder,
1977         GstVaapiPictureH264  *picture,
1978         GstH264RefPicMarking *ref_pic_marking
1979     );
1980
1981     static const exec_ref_pic_marking_adaptive_mmco_func mmco_funcs[] = {
1982         NULL,
1983         exec_ref_pic_marking_adaptive_mmco_1,
1984         exec_ref_pic_marking_adaptive_mmco_2,
1985         exec_ref_pic_marking_adaptive_mmco_3,
1986         exec_ref_pic_marking_adaptive_mmco_4,
1987         exec_ref_pic_marking_adaptive_mmco_5,
1988         exec_ref_pic_marking_adaptive_mmco_6,
1989     };
1990
1991     for (i = 0; i < dec_ref_pic_marking->n_ref_pic_marking; i++) {
1992         GstH264RefPicMarking * const ref_pic_marking =
1993             &dec_ref_pic_marking->ref_pic_marking[i];
1994
1995         const guint mmco = ref_pic_marking->memory_management_control_operation;
1996         if (mmco < G_N_ELEMENTS(mmco_funcs) && mmco_funcs[mmco])
1997             mmco_funcs[mmco](decoder, picture, ref_pic_marking);
1998         else {
1999             GST_ERROR("unhandled MMCO %u", mmco);
2000             return FALSE;
2001         }
2002     }
2003     return TRUE;
2004 }
2005
2006 /* 8.2.5 - Execute reference picture marking process */
2007 static gboolean
2008 exec_ref_pic_marking(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2009 {
2010     GstVaapiDecoderH264Private * const priv = decoder->priv;
2011     GstVaapiPictureH264 **picture_ptr;
2012
2013     priv->prev_pic_has_mmco5 = FALSE;
2014     priv->prev_pic_structure = picture->base.structure;
2015
2016     if (!GST_VAAPI_PICTURE_IS_REFERENCE(picture))
2017         return TRUE;
2018
2019     if (!GST_VAAPI_PICTURE_IS_IDR(picture)) {
2020         GstVaapiSliceH264 * const slice =
2021             gst_vaapi_picture_h264_get_last_slice(picture);
2022         GstH264DecRefPicMarking * const dec_ref_pic_marking =
2023             &slice->slice_hdr.dec_ref_pic_marking;
2024         if (dec_ref_pic_marking->adaptive_ref_pic_marking_mode_flag) {
2025             if (!exec_ref_pic_marking_adaptive(decoder, picture, dec_ref_pic_marking))
2026                 return FALSE;
2027         }
2028         else {
2029             if (!exec_ref_pic_marking_sliding_window(decoder))
2030                 return FALSE;
2031         }
2032     }
2033
2034     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture))
2035         picture_ptr = &priv->long_ref[priv->long_ref_count++];
2036     else
2037         picture_ptr = &priv->short_ref[priv->short_ref_count++];
2038     gst_vaapi_picture_replace(picture_ptr, picture);
2039     return TRUE;
2040 }
2041
2042 static void
2043 vaapi_init_picture(VAPictureH264 *pic)
2044 {
2045     pic->picture_id           = VA_INVALID_ID;
2046     pic->frame_idx            = 0;
2047     pic->flags                = VA_PICTURE_H264_INVALID;
2048     pic->TopFieldOrderCnt     = 0;
2049     pic->BottomFieldOrderCnt  = 0;
2050 }
2051
2052 static void
2053 vaapi_fill_picture(VAPictureH264 *pic, GstVaapiPictureH264 *picture)
2054 {
2055     pic->picture_id = picture->base.surface_id;
2056     pic->flags = 0;
2057
2058     if (GST_VAAPI_PICTURE_IS_LONG_TERM_REFERENCE(picture)) {
2059         pic->flags |= VA_PICTURE_H264_LONG_TERM_REFERENCE;
2060         pic->frame_idx = picture->long_term_frame_idx;
2061     }
2062     else {
2063         if (GST_VAAPI_PICTURE_IS_SHORT_TERM_REFERENCE(picture))
2064             pic->flags |= VA_PICTURE_H264_SHORT_TERM_REFERENCE;
2065         pic->frame_idx = picture->frame_num;
2066     }
2067
2068     switch (picture->base.structure) {
2069     case GST_VAAPI_PICTURE_STRUCTURE_FRAME:
2070         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2071         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2072         break;
2073     case GST_VAAPI_PICTURE_STRUCTURE_TOP_FIELD:
2074         pic->flags |= VA_PICTURE_H264_TOP_FIELD;
2075         pic->TopFieldOrderCnt = picture->field_poc[TOP_FIELD];
2076         pic->BottomFieldOrderCnt = 0;
2077         break;
2078     case GST_VAAPI_PICTURE_STRUCTURE_BOTTOM_FIELD:
2079         pic->flags |= VA_PICTURE_H264_BOTTOM_FIELD;
2080         pic->BottomFieldOrderCnt = picture->field_poc[BOTTOM_FIELD];
2081         pic->TopFieldOrderCnt = 0;
2082         break;
2083     }
2084 }
2085
2086 static gboolean
2087 fill_picture(
2088     GstVaapiDecoderH264 *decoder,
2089     GstVaapiPictureH264 *picture,
2090     GstH264SliceHdr     *slice_hdr,
2091     GstH264NalUnit      *nalu
2092 )
2093 {
2094     GstVaapiDecoderH264Private * const priv = decoder->priv;
2095     GstVaapiPicture * const base_picture = &picture->base;
2096     GstH264PPS * const pps = picture->pps;
2097     GstH264SPS * const sps = pps->sequence;
2098     VAPictureParameterBufferH264 * const pic_param = base_picture->param;
2099     guint i, n;
2100
2101     /* Fill in VAPictureParameterBufferH264 */
2102     vaapi_fill_picture(&pic_param->CurrPic, picture);
2103     for (i = 0, n = 0; i < priv->short_ref_count; i++, n++)
2104         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->short_ref[i]);
2105     for (i = 0; i < priv->long_ref_count; i++, n++)
2106         vaapi_fill_picture(&pic_param->ReferenceFrames[n], priv->long_ref[i]);
2107     for (; n < G_N_ELEMENTS(pic_param->ReferenceFrames); n++)
2108         vaapi_init_picture(&pic_param->ReferenceFrames[n]);
2109
2110 #define COPY_FIELD(s, f) \
2111     pic_param->f = (s)->f
2112
2113 #define COPY_BFM(a, s, f) \
2114     pic_param->a.bits.f = (s)->f
2115
2116     pic_param->picture_width_in_mbs_minus1  = ((priv->width + 15) >> 4)  - 1;
2117     pic_param->picture_height_in_mbs_minus1 = ((priv->height + 15) >> 4) - 1;
2118     pic_param->frame_num                    = priv->frame_num;
2119
2120     COPY_FIELD(sps, bit_depth_luma_minus8);
2121     COPY_FIELD(sps, bit_depth_chroma_minus8);
2122     COPY_FIELD(sps, num_ref_frames);
2123     COPY_FIELD(pps, num_slice_groups_minus1);
2124     COPY_FIELD(pps, slice_group_map_type);
2125     COPY_FIELD(pps, slice_group_change_rate_minus1);
2126     COPY_FIELD(pps, pic_init_qp_minus26);
2127     COPY_FIELD(pps, pic_init_qs_minus26);
2128     COPY_FIELD(pps, chroma_qp_index_offset);
2129     COPY_FIELD(pps, second_chroma_qp_index_offset);
2130
2131     pic_param->seq_fields.value                                         = 0; /* reset all bits */
2132     pic_param->seq_fields.bits.residual_colour_transform_flag           = sps->separate_colour_plane_flag;
2133     pic_param->seq_fields.bits.MinLumaBiPredSize8x8                     = sps->level_idc >= 31; /* A.3.3.2 */
2134
2135     COPY_BFM(seq_fields, sps, chroma_format_idc);
2136     COPY_BFM(seq_fields, sps, gaps_in_frame_num_value_allowed_flag);
2137     COPY_BFM(seq_fields, sps, frame_mbs_only_flag); 
2138     COPY_BFM(seq_fields, sps, mb_adaptive_frame_field_flag); 
2139     COPY_BFM(seq_fields, sps, direct_8x8_inference_flag); 
2140     COPY_BFM(seq_fields, sps, log2_max_frame_num_minus4);
2141     COPY_BFM(seq_fields, sps, pic_order_cnt_type);
2142     COPY_BFM(seq_fields, sps, log2_max_pic_order_cnt_lsb_minus4);
2143     COPY_BFM(seq_fields, sps, delta_pic_order_always_zero_flag);
2144
2145     pic_param->pic_fields.value                                         = 0; /* reset all bits */
2146     pic_param->pic_fields.bits.field_pic_flag                           = slice_hdr->field_pic_flag;
2147     pic_param->pic_fields.bits.reference_pic_flag                       = GST_VAAPI_PICTURE_IS_REFERENCE(picture);
2148
2149     COPY_BFM(pic_fields, pps, entropy_coding_mode_flag);
2150     COPY_BFM(pic_fields, pps, weighted_pred_flag);
2151     COPY_BFM(pic_fields, pps, weighted_bipred_idc);
2152     COPY_BFM(pic_fields, pps, transform_8x8_mode_flag);
2153     COPY_BFM(pic_fields, pps, constrained_intra_pred_flag);
2154     COPY_BFM(pic_fields, pps, pic_order_present_flag);
2155     COPY_BFM(pic_fields, pps, deblocking_filter_control_present_flag);
2156     COPY_BFM(pic_fields, pps, redundant_pic_cnt_present_flag);
2157     return TRUE;
2158 }
2159
2160 /* Detection of the first VCL NAL unit of a primary coded picture (7.4.1.2.4) */
2161 static gboolean
2162 is_new_picture(
2163     GstVaapiDecoderH264 *decoder,
2164     GstH264NalUnit      *nalu,
2165     GstH264SliceHdr     *slice_hdr
2166 )
2167 {
2168     GstVaapiDecoderH264Private * const priv = decoder->priv;
2169     GstH264PPS * const pps = slice_hdr->pps;
2170     GstH264SPS * const sps = pps->sequence;
2171     GstVaapiSliceH264 *slice;
2172     GstH264SliceHdr *prev_slice_hdr;
2173
2174     if (!priv->current_picture)
2175         return TRUE;
2176
2177     slice = gst_vaapi_picture_h264_get_last_slice(priv->current_picture);
2178     if (!slice)
2179         return FALSE;
2180     prev_slice_hdr = &slice->slice_hdr;
2181
2182 #define CHECK_EXPR(expr, field_name) do {              \
2183         if (!(expr)) {                                 \
2184             GST_DEBUG(field_name " differs in value"); \
2185             return TRUE;                               \
2186         }                                              \
2187     } while (0)
2188
2189 #define CHECK_VALUE(new_slice_hdr, old_slice_hdr, field) \
2190     CHECK_EXPR(((new_slice_hdr)->field == (old_slice_hdr)->field), #field)
2191
2192     /* frame_num differs in value, regardless of inferred values to 0 */
2193     CHECK_VALUE(slice_hdr, prev_slice_hdr, frame_num);
2194
2195     /* pic_parameter_set_id differs in value */
2196     CHECK_VALUE(slice_hdr, prev_slice_hdr, pps);
2197
2198     /* field_pic_flag differs in value */
2199     CHECK_VALUE(slice_hdr, prev_slice_hdr, field_pic_flag);
2200
2201     /* bottom_field_flag is present in both and differs in value */
2202     if (slice_hdr->field_pic_flag && prev_slice_hdr->field_pic_flag)
2203         CHECK_VALUE(slice_hdr, prev_slice_hdr, bottom_field_flag);
2204
2205     /* nal_ref_idc differs in value with one of the nal_ref_idc values is 0 */
2206     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_REFERENCE(priv->current_picture) ^
2207                  (nalu->ref_idc != 0)) == 0), "nal_ref_idc");
2208
2209     /* POC type is 0 for both and either pic_order_cnt_lsb differs in
2210        value or delta_pic_order_cnt_bottom differs in value */
2211     if (sps->pic_order_cnt_type == 0) {
2212         CHECK_VALUE(slice_hdr, prev_slice_hdr, pic_order_cnt_lsb);
2213         if (pps->pic_order_present_flag && !slice_hdr->field_pic_flag)
2214             CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt_bottom);
2215     }
2216
2217     /* POC type is 1 for both and either delta_pic_order_cnt[0]
2218        differs in value or delta_pic_order_cnt[1] differs in value */
2219     else if (sps->pic_order_cnt_type == 1) {
2220         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[0]);
2221         CHECK_VALUE(slice_hdr, prev_slice_hdr, delta_pic_order_cnt[1]);
2222     }
2223
2224     /* IdrPicFlag differs in value */
2225     CHECK_EXPR(((GST_VAAPI_PICTURE_IS_IDR(priv->current_picture) ^
2226                  (nalu->type == GST_H264_NAL_SLICE_IDR)) == 0), "IdrPicFlag");
2227
2228     /* IdrPicFlag is equal to 1 for both and idr_pic_id differs in value */
2229     if (GST_VAAPI_PICTURE_IS_IDR(priv->current_picture))
2230         CHECK_VALUE(slice_hdr, prev_slice_hdr, idr_pic_id);
2231
2232 #undef CHECK_EXPR
2233 #undef CHECK_VALUE
2234     return FALSE;
2235 }
2236
2237 static GstVaapiDecoderStatus
2238 decode_picture(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu, GstH264SliceHdr *slice_hdr)
2239 {
2240     GstVaapiDecoderH264Private * const priv = decoder->priv;
2241     GstVaapiPictureH264 *picture;
2242     GstVaapiDecoderStatus status;
2243     GstH264PPS * const pps = slice_hdr->pps;
2244     GstH264SPS * const sps = pps->sequence;
2245
2246     status = decode_current_picture(decoder);
2247     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2248         return status;
2249
2250     status = ensure_context(decoder, sps);
2251     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2252         return status;
2253
2254     picture = gst_vaapi_picture_h264_new(decoder);
2255     if (!picture) {
2256         GST_ERROR("failed to allocate picture");
2257         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2258     }
2259     priv->current_picture = picture;
2260
2261     picture->pps = pps;
2262
2263     status = ensure_quant_matrix(decoder, picture);
2264     if (status != GST_VAAPI_DECODER_STATUS_SUCCESS) {
2265         GST_ERROR("failed to reset quantizer matrix");
2266         return status;
2267     }
2268
2269     if (!init_picture(decoder, picture, slice_hdr, nalu))
2270         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2271     if (!fill_picture(decoder, picture, slice_hdr, nalu))
2272         return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2273     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2274 }
2275
2276 static gboolean
2277 decode_picture_end(GstVaapiDecoderH264 *decoder, GstVaapiPictureH264 *picture)
2278 {
2279     if (!exec_ref_pic_marking(decoder, picture))
2280         return FALSE;
2281     if (!dpb_add(decoder, picture))
2282         return FALSE;
2283     return TRUE;
2284 }
2285
2286 static inline guint
2287 get_slice_data_bit_offset(GstH264SliceHdr *slice_hdr, GstH264NalUnit *nalu)
2288 {
2289     guint epb_count;
2290
2291     epb_count = slice_hdr->n_emulation_prevention_bytes;
2292     return 8 /* nal_unit_type */ + slice_hdr->header_size - epb_count * 8;
2293 }
2294
2295 static gboolean
2296 fill_pred_weight_table(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2297 {
2298     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2299     GstH264PPS * const pps = slice_hdr->pps;
2300     GstH264SPS * const sps = pps->sequence;
2301     GstH264PredWeightTable * const w = &slice_hdr->pred_weight_table;
2302     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2303     guint num_weight_tables = 0;
2304     gint i, j;
2305
2306     if (pps->weighted_pred_flag &&
2307         (GST_H264_IS_P_SLICE(slice_hdr) || GST_H264_IS_SP_SLICE(slice_hdr)))
2308         num_weight_tables = 1;
2309     else if (pps->weighted_bipred_idc == 1 && GST_H264_IS_B_SLICE(slice_hdr))
2310         num_weight_tables = 2;
2311     else
2312         num_weight_tables = 0;
2313
2314     slice_param->luma_log2_weight_denom   = w->luma_log2_weight_denom;
2315     slice_param->chroma_log2_weight_denom = w->chroma_log2_weight_denom;
2316     slice_param->luma_weight_l0_flag      = 0;
2317     slice_param->chroma_weight_l0_flag    = 0;
2318     slice_param->luma_weight_l1_flag      = 0;
2319     slice_param->chroma_weight_l1_flag    = 0;
2320
2321     if (num_weight_tables < 1)
2322         return TRUE;
2323
2324     slice_param->luma_weight_l0_flag = 1;
2325     for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2326         slice_param->luma_weight_l0[i] = w->luma_weight_l0[i];
2327         slice_param->luma_offset_l0[i] = w->luma_offset_l0[i];
2328     }
2329
2330     slice_param->chroma_weight_l0_flag = sps->chroma_array_type != 0;
2331     if (slice_param->chroma_weight_l0_flag) {
2332         for (i = 0; i <= slice_param->num_ref_idx_l0_active_minus1; i++) {
2333             for (j = 0; j < 2; j++) {
2334                 slice_param->chroma_weight_l0[i][j] = w->chroma_weight_l0[i][j];
2335                 slice_param->chroma_offset_l0[i][j] = w->chroma_offset_l0[i][j];
2336             }
2337         }
2338     }
2339
2340     if (num_weight_tables < 2)
2341         return TRUE;
2342
2343     slice_param->luma_weight_l1_flag = 1;
2344     for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2345         slice_param->luma_weight_l1[i] = w->luma_weight_l1[i];
2346         slice_param->luma_offset_l1[i] = w->luma_offset_l1[i];
2347     }
2348
2349     slice_param->chroma_weight_l1_flag = sps->chroma_array_type != 0;
2350     if (slice_param->chroma_weight_l1_flag) {
2351         for (i = 0; i <= slice_param->num_ref_idx_l1_active_minus1; i++) {
2352             for (j = 0; j < 2; j++) {
2353                 slice_param->chroma_weight_l1[i][j] = w->chroma_weight_l1[i][j];
2354                 slice_param->chroma_offset_l1[i][j] = w->chroma_offset_l1[i][j];
2355             }
2356         }
2357     }
2358     return TRUE;
2359 }
2360
2361 static gboolean
2362 fill_RefPicList(GstVaapiDecoderH264 *decoder, GstVaapiSliceH264 *slice)
2363 {
2364     GstVaapiDecoderH264Private * const priv = decoder->priv;
2365     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2366     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2367     guint i, num_ref_lists = 0;
2368
2369     slice_param->num_ref_idx_l0_active_minus1 = 0;
2370     slice_param->num_ref_idx_l1_active_minus1 = 0;
2371
2372     if (GST_H264_IS_B_SLICE(slice_hdr))
2373         num_ref_lists = 2;
2374     else if (GST_H264_IS_I_SLICE(slice_hdr))
2375         num_ref_lists = 0;
2376     else
2377         num_ref_lists = 1;
2378
2379     if (num_ref_lists < 1)
2380         return TRUE;
2381
2382     slice_param->num_ref_idx_l0_active_minus1 =
2383         slice_hdr->num_ref_idx_l0_active_minus1;
2384
2385     for (i = 0; i < priv->RefPicList0_count && priv->RefPicList0[i]; i++)
2386         vaapi_fill_picture(&slice_param->RefPicList0[i], priv->RefPicList0[i]);
2387     for (; i <= slice_param->num_ref_idx_l0_active_minus1; i++)
2388         vaapi_init_picture(&slice_param->RefPicList0[i]);
2389
2390     if (num_ref_lists < 2)
2391         return TRUE;
2392
2393     slice_param->num_ref_idx_l1_active_minus1 =
2394         slice_hdr->num_ref_idx_l1_active_minus1;
2395
2396     for (i = 0; i < priv->RefPicList1_count && priv->RefPicList1[i]; i++)
2397         vaapi_fill_picture(&slice_param->RefPicList1[i], priv->RefPicList1[i]);
2398     for (; i <= slice_param->num_ref_idx_l1_active_minus1; i++)
2399         vaapi_init_picture(&slice_param->RefPicList1[i]);
2400     return TRUE;
2401 }
2402
2403 static gboolean
2404 fill_slice(
2405     GstVaapiDecoderH264 *decoder,
2406     GstVaapiSliceH264   *slice,
2407     GstH264NalUnit      *nalu
2408 )
2409 {
2410     GstH264SliceHdr * const slice_hdr = &slice->slice_hdr;
2411     VASliceParameterBufferH264 * const slice_param = slice->base.param;
2412
2413     /* Fill in VASliceParameterBufferH264 */
2414     slice_param->slice_data_bit_offset          = get_slice_data_bit_offset(slice_hdr, nalu);
2415     slice_param->first_mb_in_slice              = slice_hdr->first_mb_in_slice;
2416     slice_param->slice_type                     = slice_hdr->type % 5;
2417     slice_param->direct_spatial_mv_pred_flag    = slice_hdr->direct_spatial_mv_pred_flag;
2418     slice_param->cabac_init_idc                 = slice_hdr->cabac_init_idc;
2419     slice_param->slice_qp_delta                 = slice_hdr->slice_qp_delta;
2420     slice_param->disable_deblocking_filter_idc  = slice_hdr->disable_deblocking_filter_idc;
2421     slice_param->slice_alpha_c0_offset_div2     = slice_hdr->slice_alpha_c0_offset_div2;
2422     slice_param->slice_beta_offset_div2         = slice_hdr->slice_beta_offset_div2;
2423
2424     if (!fill_RefPicList(decoder, slice))
2425         return FALSE;
2426     if (!fill_pred_weight_table(decoder, slice))
2427         return FALSE;
2428     return TRUE;
2429 }
2430
2431 static GstVaapiDecoderStatus
2432 decode_slice(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2433 {
2434     GstVaapiDecoderH264Private * const priv = decoder->priv;
2435     GstVaapiDecoderStatus status;
2436     GstVaapiPictureH264 *picture;
2437     GstVaapiSliceH264 *slice = NULL;
2438     GstH264SliceHdr *slice_hdr;
2439     GstH264ParserResult result;
2440
2441     GST_DEBUG("slice (%u bytes)", nalu->size);
2442
2443     slice = gst_vaapi_slice_h264_new(
2444         decoder,
2445         nalu->data + nalu->offset,
2446         nalu->size
2447     );
2448     if (!slice) {
2449         GST_ERROR("failed to allocate slice");
2450         return GST_VAAPI_DECODER_STATUS_ERROR_ALLOCATION_FAILED;
2451     }
2452
2453     slice_hdr = &slice->slice_hdr;
2454     memset(slice_hdr, 0, sizeof(*slice_hdr));
2455     result = gst_h264_parser_parse_slice_hdr(priv->parser, nalu, slice_hdr, TRUE, TRUE);
2456     if (result != GST_H264_PARSER_OK) {
2457         status = get_status(result);
2458         goto error;
2459     }
2460
2461     if (is_new_picture(decoder, nalu, slice_hdr)) {
2462         status = decode_picture(decoder, nalu, slice_hdr);
2463         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2464             goto error;
2465     }
2466     picture = priv->current_picture;
2467
2468     if (!fill_slice(decoder, slice, nalu)) {
2469         status = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
2470         goto error;
2471     }
2472     gst_vaapi_picture_add_slice(
2473         GST_VAAPI_PICTURE_CAST(picture),
2474         GST_VAAPI_SLICE_CAST(slice)
2475     );
2476     return GST_VAAPI_DECODER_STATUS_SUCCESS;
2477
2478 error:
2479     if (slice)
2480         gst_mini_object_unref(GST_MINI_OBJECT(slice));
2481     return status;
2482 }
2483
2484 static inline gint
2485 scan_for_start_code(GstAdapter *adapter, guint ofs, guint size, guint32 *scp)
2486 {
2487     return (gint)gst_adapter_masked_scan_uint32_peek(adapter,
2488                                                      0xffffff00, 0x00000100,
2489                                                      ofs, size,
2490                                                      scp);
2491 }
2492
2493 static GstVaapiDecoderStatus
2494 decode_nalu(GstVaapiDecoderH264 *decoder, GstH264NalUnit *nalu)
2495 {
2496     GstVaapiDecoderStatus status;
2497
2498     switch (nalu->type) {
2499     case GST_H264_NAL_SLICE_IDR:
2500         /* fall-through. IDR specifics are handled in init_picture() */
2501     case GST_H264_NAL_SLICE:
2502         status = decode_slice(decoder, nalu);
2503         break;
2504     case GST_H264_NAL_SPS:
2505         status = decode_sps(decoder, nalu);
2506         break;
2507     case GST_H264_NAL_PPS:
2508         status = decode_pps(decoder, nalu);
2509         break;
2510     case GST_H264_NAL_SEI:
2511         status = decode_sei(decoder, nalu);
2512         break;
2513     case GST_H264_NAL_SEQ_END:
2514         status = decode_sequence_end(decoder);
2515         break;
2516     case GST_H264_NAL_AU_DELIMITER:
2517         /* skip all Access Unit NALs */
2518         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2519         break;
2520     case GST_H264_NAL_FILLER_DATA:
2521         /* skip all Filler Data NALs */
2522         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2523         break;
2524     default:
2525         GST_WARNING("unsupported NAL unit type %d", nalu->type);
2526         status = GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2527         break;
2528     }
2529     return status;
2530 }
2531
2532 static GstVaapiDecoderStatus
2533 decode_buffer(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2534 {
2535     GstVaapiDecoderH264Private * const priv = decoder->priv;
2536     GstVaapiDecoderStatus status;
2537     GstH264ParserResult result;
2538     GstH264NalUnit nalu;
2539     gboolean is_eos;
2540     const guchar *buf;
2541     guint i, buf_size, nalu_size, size;
2542     guint32 start_code;
2543     gint ofs;
2544
2545     buf      = GST_BUFFER_DATA(buffer);
2546     buf_size = GST_BUFFER_SIZE(buffer);
2547     is_eos   = GST_BUFFER_IS_EOS(buffer);
2548     if (buf && buf_size > 0)
2549         gst_adapter_push(priv->adapter, gst_buffer_ref(buffer));
2550
2551     size = gst_adapter_available(priv->adapter);
2552     do {
2553         if (size == 0) {
2554             status = GST_VAAPI_DECODER_STATUS_SUCCESS;
2555             break;
2556         }
2557
2558         status = gst_vaapi_decoder_check_status(GST_VAAPI_DECODER(decoder));
2559         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2560             break;
2561
2562         status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2563         if (priv->is_avc) {
2564             if (size < priv->nal_length_size)
2565                 break;
2566             buf = gst_adapter_peek(priv->adapter, priv->nal_length_size);
2567
2568             nalu_size = 0;
2569             for (i = 0; i < priv->nal_length_size; i++)
2570                 nalu_size = (nalu_size << 8) | buf[i];
2571
2572             buf_size = priv->nal_length_size + nalu_size;
2573             if (size < buf_size)
2574                 break;
2575             buffer = gst_adapter_take_buffer(priv->adapter, buf_size);
2576             size -= buf_size;
2577
2578             buf      = GST_BUFFER_DATA(buffer);
2579             buf_size = GST_BUFFER_SIZE(buffer);
2580
2581             result = gst_h264_parser_identify_nalu_avc(
2582                 priv->parser,
2583                 buf, 0, buf_size, priv->nal_length_size,
2584                 &nalu
2585             );
2586         }
2587         else {
2588             if (size < 4)
2589                 break;
2590             ofs = scan_for_start_code(priv->adapter, 0, size, &start_code);
2591             if (ofs < 0)
2592                 break;
2593             gst_adapter_flush(priv->adapter, ofs);
2594             size -= ofs;
2595
2596             ofs = G_UNLIKELY(size < 8) ? -1 :
2597                 scan_for_start_code(priv->adapter, 4, size - 4, NULL);
2598             if (ofs < 0) {
2599                 // Assume the whole NAL unit is present if end-of-stream
2600                 if (!is_eos)
2601                     break;
2602                 ofs = size;
2603             }
2604             buffer = gst_adapter_take_buffer(priv->adapter, ofs);
2605             size -= ofs;
2606
2607             buf      = GST_BUFFER_DATA(buffer);
2608             buf_size = GST_BUFFER_SIZE(buffer);
2609
2610             result = gst_h264_parser_identify_nalu_unchecked(
2611                 priv->parser,
2612                 buf, 0, buf_size,
2613                 &nalu
2614             );
2615         }
2616         status = get_status(result);
2617         if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
2618             status = decode_nalu(decoder, &nalu);
2619         gst_buffer_unref(buffer);
2620     } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
2621
2622     if (is_eos && (status == GST_VAAPI_DECODER_STATUS_SUCCESS ||
2623                    status == GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA))
2624         status = decode_sequence_end(decoder);
2625     return status;
2626 }
2627
2628 static GstVaapiDecoderStatus
2629 decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
2630 {
2631     GstVaapiDecoderH264Private * const priv = decoder->priv;
2632     GstVaapiDecoderStatus status;
2633     GstH264NalUnit nalu;
2634     GstH264ParserResult result;
2635     guchar *buf;
2636     guint buf_size;
2637     guint i, ofs, num_sps, num_pps;
2638
2639     buf      = GST_BUFFER_DATA(buffer);
2640     buf_size = GST_BUFFER_SIZE(buffer);
2641     if (!buf || buf_size == 0)
2642         return GST_VAAPI_DECODER_STATUS_SUCCESS;
2643
2644     if (buf_size < 8)
2645         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
2646
2647     if (buf[0] != 1) {
2648         GST_ERROR("failed to decode codec-data, not in avcC format");
2649         return GST_VAAPI_DECODER_STATUS_ERROR_BITSTREAM_PARSER;
2650     }
2651
2652     priv->nal_length_size = (buf[4] & 0x03) + 1;
2653
2654     num_sps = buf[5] & 0x1f;
2655     ofs = 6;
2656
2657     for (i = 0; i < num_sps; i++) {
2658         result = gst_h264_parser_identify_nalu_avc(
2659             priv->parser,
2660             buf, ofs, buf_size, 2,
2661             &nalu
2662         );
2663         if (result != GST_H264_PARSER_OK)
2664             return get_status(result);
2665
2666         status = decode_sps(decoder, &nalu);
2667         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2668             return status;
2669         ofs = nalu.offset + nalu.size;
2670     }
2671
2672     num_pps = buf[ofs];
2673     ofs++;
2674
2675     for (i = 0; i < num_pps; i++) {
2676         result = gst_h264_parser_identify_nalu_avc(
2677             priv->parser,
2678             buf, ofs, buf_size, 2,
2679             &nalu
2680         );
2681         if (result != GST_H264_PARSER_OK)
2682             return get_status(result);
2683
2684         status = decode_pps(decoder, &nalu);
2685         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
2686             return status;
2687         ofs = nalu.offset + nalu.size;
2688     }
2689
2690     priv->is_avc = TRUE;
2691     return status;
2692 }
2693
2694 GstVaapiDecoderStatus
2695 gst_vaapi_decoder_h264_decode(GstVaapiDecoder *base, GstBuffer *buffer)
2696 {
2697     GstVaapiDecoderH264 * const decoder = GST_VAAPI_DECODER_H264(base);
2698     GstVaapiDecoderH264Private * const priv = decoder->priv;
2699     GstVaapiDecoderStatus status;
2700     GstBuffer *codec_data;
2701
2702     g_return_val_if_fail(priv->is_constructed,
2703                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
2704
2705     if (!priv->is_opened) {
2706         priv->is_opened = gst_vaapi_decoder_h264_open(decoder, buffer);
2707         if (!priv->is_opened)
2708             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
2709
2710         codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
2711       &nb