configure.ac: check old/new h264 encoder API
[vaapi:windyuan-gstreamer-vaapi.git] / gst-libs / gst / vaapi / gstvaapiencoder_h264.c
1 /*
2  *  gstvaapiencoder_h264.c -  H.264 encoder
3  *
4  *  Copyright (C) 2011 Intel Corporation
5  *
6  *  This library is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU Lesser General Public License
8  *  as published by the Free Software Foundation; either version 2.1
9  *  of the License, or (at your option) any later version.
10  *
11  *  This library is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  *  Lesser General Public License for more details.
15  *
16  *  You should have received a copy of the GNU Lesser General Public
17  *  License along with this library; if not, write to the Free
18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19  *  Boston, MA 02110-1301 USA
20  */
21
22 #include "gstvaapiencoder_h264.h"
23 #include "config.h"
24
25 #include <string.h>
26 #include <stdlib.h>
27 #include <va/va.h>
28 #include <va/va_x11.h>
29 #if !HAVE_OLD_H264_ENCODER
30 #include <va/va_enc_h264.h>
31 #endif
32 #include <X11/Xlib.h>
33 #include <glib.h>
34
35 #include "gst/gstclock.h"
36 #include "gst/gstvalue.h"
37
38 #include "gstvaapiobject.h"
39 #include "gstvaapiobject_priv.h"
40 #include "gstvaapicontext.h"
41 #include "gstvaapisurface.h"
42 #include "gstvaapivideobuffer.h"
43 #include "gstvaapidisplay_priv.h"
44
45 GST_DEBUG_CATEGORY_STATIC (gst_vaapi_h264_encoder_debug);
46
47 #define GST_CAT_DEFAULT gst_vaapi_h264_encoder_debug
48
49 #define GST_VAAPI_ENCODER_H264_CAST(encoder)    ((GstVaapiEncoderH264 *)(encoder))
50
51 #define SHARE_CODED_BUF         0
52
53 #define DEFAULT_SURFACE_NUMBER  3
54 #define DEFAULT_CODEDBUF_NUM    5
55 #define DEFAULT_SID_INPUT       0 // suface_ids[0]
56
57 #define REF_RECON_SURFACE_NUM   2
58
59 #define ENTROPY_MODE_CAVLC      0
60 #define ENTROPY_MODE_CABAC      1
61
62 #define BR_CBR          0
63 #define BR_VBR          1
64 #define BR_CQP          2
65
66 #define NAL_REF_IDC_NONE        0
67 #define NAL_REF_IDC_LOW         1
68 #define NAL_REF_IDC_MEDIUM      2
69 #define NAL_REF_IDC_HIGH        3
70
71
72 typedef enum {
73   NAL_UNKNOWN     = 0,
74   NAL_NON_IDR     = 1,
75   NAL_IDR         = 5,    /* ref_idc != 0 */
76   NAL_SEI         = 6,    /* ref_idc == 0 */
77   NAL_SPS         = 7,
78   NAL_PPS         = 8,
79   NAL_AUD         = 9,
80   NAL_FILLER      = 12,
81 }H264_NAL_TYPE;
82
83
84 typedef enum {
85   SLICE_TYPE_P  = 0,
86   SLICE_TYPE_B  = 1,
87   SLICE_TYPE_I  = 2
88 } H264_SLICE_TYPE;
89
90 struct _GstVaapiEncoderH264Private {
91   GstVaapiEncoderH264   *public;
92   guint32           format;   /*NV12, I420,*/
93   gboolean          avc_flag;  /*elementary flag*/
94
95   /* private data*/
96   GQueue           *video_buffer_caches; /*not used for baseline*/
97
98   GstVaapiSurface  *ref_surface1;  /* reference buffer*/
99   GstVaapiSurface  *ref_surface2;  /* for B frames */
100   GstVaapiSurface  *recon_surface; /* reconstruct buffer*/
101
102   VABufferID        seq_param_id;
103   VABufferID        pic_param_id;
104   VABufferID        slice_param_id;
105   VABufferID        packed_seq_param_id;
106   VABufferID        packed_seq_data_id;
107   VABufferID        packed_pic_param_id;
108   VABufferID        packed_pic_data_id;
109 #if HAVE_OLD_H264_ENCODER
110   VAEncSliceParameterBuffer     *slice_param_buffers;
111 #else
112   VAEncSliceParameterBufferH264 *slice_param_buffers;
113 #endif
114   guint32           default_slice_height;
115   guint32           slice_mod_mb_num;
116   guint32           default_cts_offset;
117
118   GstBuffer        *sps_data;
119   GstBuffer        *pps_data;
120
121   GQueue           *queued_buffers;  /* GstVaapiVideoBuffers with surface*/
122
123   guint32           gop_count;
124   guint32           cur_display_num;
125   guint32           cur_decode_num;
126   H264_SLICE_TYPE   cur_slice_type;
127   guint64           last_decode_time;
128   guint32           max_frame_num;
129   guint32           max_pic_order_cnt;
130   guint16           idr_num;
131 };
132
133 G_DEFINE_TYPE(GstVaapiEncoderH264, gst_vaapi_encoder_h264, GST_TYPE_VAAPI_BASE_ENCODER)
134
135 // 4096-1
136 #define H264_BITSTREAM_ALLOC_ALIGN_MASK 0x0FFF
137
138 #define BIT_STREAM_BUFFER(stream)    ((stream)->buffer)
139 #define BIT_STREAM_BIT_SIZE(stream)  ((stream)->bit_size)
140
141 struct _H264Bitstream {
142   guint8   *buffer;
143   guint32   bit_size;
144   guint32   max_bit_capability;
145 };
146
147 typedef struct _H264Bitstream H264Bitstream;
148
149 static const guint8 h264_bit_mask[9] = {0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF};
150
151 /* h264 bitstream functions */
152 static void
153 h264_bitstream_init(H264Bitstream *bitstream, guint32 bit_capability);
154
155 static gboolean
156 h264_bitstream_write_uint(
157     H264Bitstream *bitstream,
158     guint32 value,
159     guint32 bit_size
160 );
161
162 static gboolean
163 h264_bitstream_align(H264Bitstream *bitstream, guint32 value);
164
165 static gboolean
166 h264_bitstream_write_ue(H264Bitstream *bitstream, guint32 value);
167
168 static gboolean
169 h264_bitstream_write_se(H264Bitstream *bitstream, gint32 value);
170
171 static gboolean
172 h264_bitstream_write_trailing_bits(H264Bitstream *bitstream);
173
174 static gboolean
175 h264_bitstream_write_byte_array(
176     H264Bitstream *bitstream,
177     const guint8 *buf,
178     guint32 byte_size
179 );
180
181 static void
182 h264_bitstream_destroy(H264Bitstream *bitstream, gboolean free_flag);
183
184 static gboolean
185 h264_bitstream_auto_grow(H264Bitstream *bitstream, guint32 extra_bit_size);
186
187 static gboolean
188 h264_bitstream_write_sps(
189     H264Bitstream *bitstream,
190     VAEncSequenceParameterBufferH264 *seq,
191     H264_Profile profile
192 );
193 static gboolean
194 h264_bitstream_write_pps(
195     H264Bitstream *bitstream,
196     VAEncPictureParameterBufferH264 *pic
197 );
198
199 static gboolean
200 h264_bitstream_write_nal_header(
201     H264Bitstream *bitstream,
202     guint nal_ref_idc,
203     guint nal_unit_type
204 );
205
206 static VAProfile
207 h264_get_va_profile(guint32 profile)
208 {
209   switch (profile) {
210     case H264_PROFILE_BASELINE:
211       return VAProfileH264Baseline;
212
213     case H264_PROFILE_MAIN:
214       return VAProfileH264Main;
215
216     case H264_PROFILE_HIGH:
217       return VAProfileH264High;
218
219     default:
220       break;
221   }
222   return (-1);
223 }
224
225 GstVaapiEncoderH264 *
226 gst_vaapi_encoder_h264_new(void)
227 {
228   return GST_VAAPI_ENCODER_H264_CAST(
229              g_object_new(GST_TYPE_VAAPI_ENCODER_H264, NULL));
230 }
231
232 static void
233 gst_vaapi_encoder_h264_init_public_values(GstVaapiEncoderH264* encoder)
234 {
235   encoder->profile = 0;
236   encoder->level = 0;
237   encoder->bitrate = 0;
238   encoder->intra_period = 0;
239   encoder->init_qp = -1;
240   encoder->min_qp = -1;
241   encoder->slice_num = 0;
242   encoder->b_frame_num = 0;
243 }
244
245 void
246 gst_vaapi_encoder_h264_set_avc_flag(GstVaapiEncoderH264* encoder, gboolean avc)
247 {
248   GstVaapiEncoderH264Private *priv = encoder->priv;
249   priv->avc_flag = avc;
250 }
251
252 gboolean
253 gst_vaapi_encoder_h264_get_avc_flag(GstVaapiEncoderH264* encoder)
254 {
255   GstVaapiEncoderH264Private *priv = encoder->priv;
256   return priv->avc_flag;
257 }
258
259 gboolean
260 gst_vaapi_encoder_h264_validate_attributes(GstVaapiBaseEncoder *base)
261 {
262   GstVaapiEncoderH264 *encoder = GST_VAAPI_ENCODER_H264_CAST(base);
263   GstVaapiEncoderH264Private *priv = encoder->priv;
264   if (!ENCODER_WIDTH(encoder) ||
265       !ENCODER_HEIGHT(encoder) ||
266       !ENCODER_FPS(encoder)) {
267     return FALSE;
268   }
269   if (!encoder->profile) {
270     encoder->profile = H264_DEFAULT_PROFILE;
271   }
272   gst_vaapi_base_encoder_set_va_profile(base, h264_get_va_profile(encoder->profile));
273   if (!encoder->level) {
274     if (encoder->profile <= H264_PROFILE_BASELINE)
275       encoder->level = H264_LEVEL_30;
276     else
277       encoder->level = H264_LEVEL_41;
278   }
279   if (!encoder->intra_period) {
280     encoder->intra_period = H264_DEFAULT_INTRA_PERIOD;
281   }
282   if (-1 == encoder->init_qp) {
283     encoder->init_qp = H264_DEFAULT_INIT_QP;
284   }
285   if (-1 == encoder->min_qp) {
286     encoder->min_qp = H264_DEFAULT_MIN_QP;
287   }
288
289   if (encoder->min_qp > encoder->init_qp) {
290     encoder->min_qp = encoder->init_qp;
291   }
292
293   /* default compress ratio 1: (4*8*1.5) */
294   if (!encoder->bitrate) {
295     encoder->bitrate = 0; //ENCODER_WIDTH(encoder)*ENCODER_HEIGHT(encoder)*ENCODER_FPS(encoder)/4;
296   }
297
298   if (!encoder->slice_num) {
299     encoder->slice_num = H264_DEFAULT_SLICE_NUM;
300   }
301
302   /* need  calculate slice-num and each slice-height
303         suppose:  ((encoder->height+15)/16) = 13, slice_num = 8
304         then: slice_1_height = 2
305                  slice_2_height = 2
306                  slice_3_height = 2
307                  slice_4_height = 2
308                  slice_5_height = 2
309                  slice_6_height = 1
310                  slice_7_height = 1
311                  slice_8_height = 1
312    */
313   priv->default_slice_height = (ENCODER_HEIGHT(encoder)+15)/16/encoder->slice_num;
314   if (0 == priv->default_slice_height) { /* special value */
315     priv->default_slice_height = 1;
316     priv->slice_mod_mb_num = 0;
317     encoder->slice_num = (ENCODER_HEIGHT(encoder)+15)/16;
318   } else {
319     priv->slice_mod_mb_num = ((ENCODER_HEIGHT(encoder)+15)/16)%encoder->slice_num;
320   }
321
322   if (encoder->b_frame_num) {
323     priv->default_cts_offset = GST_SECOND/ENCODER_FPS(encoder);
324   } else {
325     priv->default_cts_offset = 0;
326   }
327   return TRUE;
328 }
329
330
331 static gboolean
332 h264_encoder_release_parameters(GstVaapiEncoderH264 *encoder)
333 {
334   VAStatus va_status = VA_STATUS_SUCCESS;
335   GstVaapiEncoderH264Private *priv = encoder->priv;
336   GstVaapiDisplay *display = ENCODER_DISPLAY(encoder);
337   GstVaapiContext *context = ENCODER_CONTEXT(encoder);
338
339   gboolean is_locked = FALSE;
340
341   ENCODER_ASSERT(display);
342   ENCODER_ASSERT(context);
343   VAAPI_UNUSED_ARG(va_status);
344   VADisplay va_dpy = gst_vaapi_display_get_display(display);
345
346   ENCODER_ACQUIRE_DISPLAY_LOCK(display);
347   if (VA_INVALID_ID != priv->seq_param_id) {
348     va_status = vaDestroyBuffer(va_dpy, priv->seq_param_id);
349     priv->seq_param_id = VA_INVALID_ID;
350   }
351   if (VA_INVALID_ID != priv->pic_param_id) {
352     va_status = vaDestroyBuffer(va_dpy, priv->pic_param_id);
353     priv->pic_param_id = VA_INVALID_ID;
354   }
355   if (VA_INVALID_ID != priv->slice_param_id) {
356     va_status = vaDestroyBuffer(va_dpy, priv->slice_param_id);
357     priv->slice_param_id = VA_INVALID_ID;
358   }
359
360   if (VA_INVALID_ID != priv->packed_seq_param_id) {
361     va_status = vaDestroyBuffer(va_dpy, priv->packed_seq_param_id);
362     priv->packed_seq_param_id = VA_INVALID_ID;
363   }
364   if (VA_INVALID_ID != priv->packed_seq_data_id) {
365     va_status = vaDestroyBuffer(va_dpy, priv->packed_seq_data_id);
366     priv->packed_seq_data_id = VA_INVALID_ID;
367   }
368   if (VA_INVALID_ID != priv->packed_pic_param_id) {
369     va_status = vaDestroyBuffer(va_dpy, priv->packed_pic_param_id);
370     priv->packed_pic_param_id = VA_INVALID_ID;
371   }
372   if (VA_INVALID_ID != priv->packed_pic_data_id) {
373     va_status = vaDestroyBuffer(va_dpy, priv->packed_pic_data_id);
374     priv->packed_pic_data_id = VA_INVALID_ID;
375   }
376
377   ENCODER_RELEASE_DISPLAY_LOCK(display);
378
379   if (priv->slice_param_buffers) {
380     g_free(priv->slice_param_buffers);
381     priv->slice_param_buffers = NULL;
382   }
383
384   if (priv->sps_data) {
385     gst_buffer_unref(priv->sps_data);
386     priv->sps_data = NULL;
387   }
388   if (priv->pps_data) {
389     gst_buffer_unref(priv->pps_data);
390     priv->pps_data = NULL;
391   }
392
393   return TRUE;
394 }
395
396 static void
397 h264_release_queued_buffers(GstVaapiEncoderH264Private *priv)
398 {
399     while (!g_queue_is_empty(priv->queued_buffers)) {
400     GstBuffer* tmp = g_queue_pop_head(priv->queued_buffers);
401     if (tmp)
402       gst_buffer_unref(tmp);
403   }
404 }
405
406
407 static gboolean
408 gst_vaapi_encoder_h264_release_resource(
409     GstVaapiBaseEncoder* base
410 )
411 {
412   GstVaapiEncoderH264* encoder = GST_VAAPI_ENCODER_H264_CAST(base);
413   gboolean ret = TRUE;
414   GstVaapiEncoderH264Private *priv = encoder->priv;
415   GstVaapiContext *context = ENCODER_CONTEXT(base);
416
417   /* release buffers first */
418   h264_encoder_release_parameters(encoder);
419   h264_release_queued_buffers(priv);
420   priv->cur_display_num = 0;
421   priv->cur_decode_num = 0;
422   priv->cur_slice_type = SLICE_TYPE_I;
423   priv->gop_count = 0;
424   priv->last_decode_time = 0LL;
425   priv->default_cts_offset = 0;
426
427   /*remove ref_surface1*/
428   if (priv->ref_surface1) {
429     if (context) {
430       gst_vaapi_context_put_surface(context, priv->ref_surface1);
431     } else {
432       g_object_unref(priv->ref_surface1);
433     }
434     priv->ref_surface1 = NULL;
435   }
436
437   if (priv->ref_surface2) {
438     if (context) {
439       gst_vaapi_context_put_surface(context, priv->ref_surface2);
440     } else {
441       g_object_unref(priv->ref_surface2);
442     }
443     priv->ref_surface2 = NULL;
444   }
445
446   /*remove recon_surface*/
447   if (priv->recon_surface) {
448     if (context) {
449       gst_vaapi_context_put_surface(context, priv->recon_surface);
450     } else {
451       g_object_unref(priv->recon_surface);
452     }
453     priv->recon_surface = NULL;
454   }
455
456   return ret;
457 }
458
459 static gboolean
460 gst_vaapi_encoder_h264_alloc_slices(
461     GstVaapiBaseEncoder *base,
462     GstVaapiContext *context
463 )
464 {
465   gboolean ret = TRUE;
466   GstVaapiEncoderH264 *encoder = GST_VAAPI_ENCODER_H264_CAST(base);
467   GstVaapiEncoderH264Private *priv = encoder->priv;
468
469   priv->slice_param_buffers =
470 #if HAVE_OLD_H264_ENCODER
471   (VAEncSliceParameterBuffer*)
472 #else
473   (VAEncSliceParameterBufferH264*)
474 #endif
475           g_malloc0_n(encoder->slice_num,
476               sizeof(priv->slice_param_buffers[0]));
477
478   return ret;
479 }
480
481 static void
482 gst_vaapi_encoder_h264_frame_failed(
483     GstVaapiBaseEncoder *base,
484     GstVaapiVideoBuffer* buffer
485 )
486 {
487   GstVaapiEncoderH264 *encoder = GST_VAAPI_ENCODER_H264_CAST(base);
488   GstVaapiEncoderH264Private *priv = encoder->priv;
489
490   h264_release_queued_buffers(priv);
491   priv->cur_display_num = 0;
492   priv->cur_decode_num = 0;
493   priv->cur_slice_type = SLICE_TYPE_I;
494   priv->gop_count = 0;
495   priv->last_decode_time = 0LL;
496 }
497
498 static EncoderStatus
499 gst_vaapi_encoder_h264_prepare_next_buffer(
500     GstVaapiBaseEncoder* base,
501     GstVaapiVideoBuffer *display_buf,
502     gboolean need_flush,
503     GstVaapiVideoBuffer **out_buf
504 )
505 {
506   EncoderStatus ret = ENCODER_NO_ERROR;
507   GstVaapiEncoderH264 *encoder = GST_VAAPI_ENCODER_H264_CAST(base);
508   GstVaapiEncoderH264Private *priv = encoder->priv;
509   GstVaapiVideoBuffer  *return_buf = NULL;
510   //guint64 pts = 0;
511
512   if (NULL == display_buf && g_queue_is_empty(priv->queued_buffers)) {
513     ret = ENCODER_BUFFER_EMPTY;
514     if (priv->gop_count >= encoder->intra_period || need_flush)
515       priv->gop_count = 0;
516     goto end;
517   }
518
519   if (display_buf) {
520     ++priv->gop_count;
521     gst_buffer_ref(GST_BUFFER_CAST(display_buf));
522     priv->last_decode_time = GST_BUFFER_TIMESTAMP(display_buf);
523   }
524
525   /* first frame */
526   if (priv->gop_count == 1) {
527     ENCODER_ASSERT(display_buf);
528     priv->cur_display_num = 0;
529     priv->cur_decode_num = 0;
530     priv->cur_slice_type = SLICE_TYPE_I;
531     ++priv->idr_num;
532     return_buf = display_buf;
533     goto end;
534   }
535
536   if (display_buf) {
537     if (encoder->b_frame_num &&
538         priv->gop_count < encoder->intra_period &&
539         g_queue_get_length(priv->queued_buffers) < encoder->b_frame_num
540         )
541     {
542       g_queue_push_tail(priv->queued_buffers, display_buf);
543       ret = ENCODER_BUFFER_WAITING;
544       goto end;
545     }
546     priv->cur_slice_type = SLICE_TYPE_P;
547     priv->cur_display_num = priv->gop_count-1;
548     ++priv->cur_decode_num;
549     return_buf = display_buf;
550   } else {
551     if (need_flush) {
552       return_buf = (GstVaapiVideoBuffer*)g_queue_pop_tail(priv->queued_buffers);
553       priv->cur_slice_type = SLICE_TYPE_P;
554       priv->cur_display_num = priv->gop_count - 1;
555       ++priv->cur_decode_num;
556     } else {
557       return_buf = (GstVaapiVideoBuffer*)g_queue_pop_head(priv->queued_buffers);
558       priv->cur_slice_type = SLICE_TYPE_B;
559       priv->cur_display_num =
560         priv->gop_count - 2 - g_queue_get_length(priv->queued_buffers);
561     }
562   }
563
564 end:
565   *out_buf = return_buf;
566
567   return ret;
568 }
569
570 static inline void
571 h264_swap_surface(GstVaapiSurface **s1, GstVaapiSurface **s2)
572 {
573   GstVaapiSurface *tmp;
574
575   g_return_if_fail(s1 && s2);
576   tmp = *s1;
577   *s1 = *s2;
578   *s2 = tmp;
579 }
580
581 static inline const char *
582 get_slice_type(H264_SLICE_TYPE type)
583 {
584     switch (type) {
585     case SLICE_TYPE_I:
586         return "I";
587     case SLICE_TYPE_P:
588         return "P";
589     case SLICE_TYPE_B:
590         return "B";
591     default:
592         return "Unknown";
593     }
594 }
595
596 #if HAVE_OLD_H264_ENCODER
597
598 static gboolean
599 set_sequence_parameters(
600     GstVaapiEncoderH264 *encoder,
601     VAEncSequenceParameterBufferH264 *seq_param
602 )
603 {
604   seq_param->seq_parameter_set_id = 0;
605   seq_param->level_idc = encoder->level; /* 3.0 */
606   seq_param->intra_period = encoder->intra_period;
607   seq_param->intra_idr_period = encoder->intra_period;
608   seq_param->max_num_ref_frames = 1; /*Only I, P frames*/
609   seq_param->picture_width_in_mbs = (ENCODER_WIDTH(encoder)+15)/16;
610   seq_param->picture_height_in_mbs = (ENCODER_HEIGHT(encoder)+15)/16;
611
612   seq_param->bits_per_second = encoder->bitrate;
613   seq_param->frame_rate = ENCODER_FPS(encoder);
614   seq_param->initial_qp = encoder->init_qp; /*qp_value; 15, 24, 26?*/
615   seq_param->min_qp = encoder->min_qp;     /*1, 6, 10*/
616   seq_param->basic_unit_size = 0;
617   seq_param->vui_flag = FALSE;
618
619   return TRUE;
620 }
621
622 static gboolean
623 set_picture_parameters(
624     GstVaapiEncoderH264 *encoder,
625     VAEncPictureParameterBufferH264 *pic_param,
626     VABufferID coded_buf
627 )
628 {
629   GstVaapiEncoderH264Private *priv = encoder->priv;
630
631   pic_param->reference_picture = GST_VAAPI_OBJECT_ID(priv->ref_surface1);
632   pic_param->reconstructed_picture = GST_VAAPI_OBJECT_ID(priv->recon_surface);
633   pic_param->coded_buf = coded_buf;
634   pic_param->picture_width = ENCODER_WIDTH(encoder);
635   pic_param->picture_height = ENCODER_HEIGHT(encoder);
636   pic_param->last_picture = 0; // last pic or not
637
638   return TRUE;
639 }
640
641 static gboolean
642 set_slices_parameters(
643     GstVaapiEncoderH264 *encoder,
644     VAEncSliceParameterBuffer *slices,
645     guint slice_num
646 )
647 {
648   GstVaapiEncoderH264Private *priv = encoder->priv;
649   VAEncSliceParameterBuffer *slice_param;
650
651   int i = 0;
652   guint32 last_row_num = 0;
653   guint32 slice_mod_num = priv->slice_mod_mb_num;
654
655   for (i = 0; i < slice_num; ++i) {
656     slice_param = &slices[i];
657     slice_param->start_row_number = last_row_num;               /* unit MB*/
658     slice_param->slice_height = priv->default_slice_height; /* unit MB */
659     if (slice_mod_num) {
660       ++slice_param->slice_height;
661       --slice_mod_num;
662     }
663     last_row_num += slice_param->slice_height;
664     slice_param->slice_flags.bits.is_intra =
665         (priv->cur_slice_type == SLICE_TYPE_I);
666     slice_param->slice_flags.bits.disable_deblocking_filter_idc = FALSE;
667     slice_param->slice_flags.bits.uses_long_term_ref = FALSE;
668     slice_param->slice_flags.bits.is_long_term_ref = FALSE;
669   }
670
671   ENCODER_ASSERT(last_row_num == (ENCODER_HEIGHT(encoder)+15)/16);
672   return TRUE;
673 }
674
675 #else  /* extended libva, new parameter structures*/
676
677 static gboolean
678 set_sequence_parameters(
679     GstVaapiEncoderH264 *encoder,
680     VAEncSequenceParameterBufferH264 *seq_param
681 )
682 {
683   GstVaapiEncoderH264Private *priv = encoder->priv;
684   guint width_in_mbs, height_in_mbs;
685
686   width_in_mbs = (ENCODER_WIDTH(encoder)+15)/16;
687   height_in_mbs = (ENCODER_HEIGHT(encoder)+15)/16;
688
689   seq_param->seq_parameter_set_id = 0;
690   seq_param->level_idc = encoder->level; /* 3.0 */
691   seq_param->intra_period = encoder->intra_period;
692   seq_param->ip_period = 0;           // ?
693   if (encoder->bitrate> 0)
694       seq_param->bits_per_second = encoder->bitrate; /* use kbps as input */
695   else
696       seq_param->bits_per_second = 0;
697
698   seq_param->max_num_ref_frames =
699       (encoder->b_frame_num < 2 ? 3 : encoder->b_frame_num+1);  // ?, why 4
700   seq_param->picture_width_in_mbs = width_in_mbs;
701   seq_param->picture_height_in_mbs = height_in_mbs;
702
703   /*sequence field values*/
704   seq_param->seq_fields.value = 0;
705   seq_param->seq_fields.bits.chroma_format_idc = 1;
706   seq_param->seq_fields.bits.frame_mbs_only_flag = 1;
707   seq_param->seq_fields.bits.mb_adaptive_frame_field_flag = FALSE;
708   seq_param->seq_fields.bits.seq_scaling_matrix_present_flag = FALSE;
709   /* direct_8x8_inference_flag default false */
710   seq_param->seq_fields.bits.direct_8x8_inference_flag = FALSE;
711   seq_param->seq_fields.bits.log2_max_frame_num_minus4 = 4; // log2(seq.intra_period)-3 : 0
712   /* picture order count */
713   seq_param->seq_fields.bits.pic_order_cnt_type = 0;
714   seq_param->seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4 =
715         seq_param->seq_fields.bits.log2_max_frame_num_minus4 + 2;
716   seq_param->seq_fields.bits.delta_pic_order_always_zero_flag = TRUE;
717
718   priv->max_frame_num =
719       1<<(seq_param->seq_fields.bits.log2_max_frame_num_minus4 + 4);
720   priv->max_pic_order_cnt =
721       1 <<(seq_param->seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4 + 4);
722
723   seq_param->bit_depth_luma_minus8 = 0;
724   seq_param->bit_depth_chroma_minus8 = 0;
725
726   /* not used if pic_order_cnt_type == 0 */
727   seq_param->num_ref_frames_in_pic_order_cnt_cycle = 0;
728   seq_param->offset_for_non_ref_pic = 0;
729   seq_param->offset_for_top_to_bottom_field = 0;
730   memset(seq_param->offset_for_ref_frame,
731          0,
732          sizeof(seq_param->offset_for_ref_frame));
733
734   if (height_in_mbs*16 - ENCODER_HEIGHT(encoder)) {
735     seq_param->frame_cropping_flag = 1;
736     seq_param->frame_crop_left_offset = 0;
737     seq_param->frame_crop_right_offset = 0;
738     seq_param->frame_crop_top_offset = 0;
739     seq_param->frame_crop_bottom_offset =
740         ((height_in_mbs * 16 - ENCODER_HEIGHT(encoder))/
741          (2 * (!seq_param->seq_fields.bits.frame_mbs_only_flag + 1)));
742   }
743 #if 0
744   if (h264_encoder->init_qp == -1)
745       seq.rate_control_method = BR_CBR;
746   else if (h264_encoder->init_qp == -2)
747       seq.rate_control_method = BR_VBR;
748   else {
749       ENCODER_ASSERT(h264_encoder->init_qp >= 0 && h264_encoder->init_qp <= 51);
750       seq.rate_control_method = BR_CQP;
751   }
752 #endif
753
754   /*vui not set*/
755   seq_param->vui_parameters_present_flag = FALSE;
756   return TRUE;
757 }
758
759 static gboolean
760 ensure_packed_sps_data(
761     GstVaapiEncoderH264 *encoder,
762     VAEncSequenceParameterBufferH264 *seq_param
763 )
764 {
765   GstVaapiEncoderH264Private *priv = encoder->priv;
766   VAEncPackedHeaderParameterBuffer packed_header_param_buffer = { 0 };
767   VADisplay va_dpy = ENCODER_VA_DISPLAY(encoder);
768   VAContextID context_id = ENCODER_VA_CONTEXT(encoder);
769   guint32 length_in_bits;
770   guint8 *packed_seq_buffer = NULL;
771   H264Bitstream bitstream;
772   gboolean ret = TRUE;
773   VAStatus va_status = VA_STATUS_SUCCESS;
774
775   if (priv->sps_data) 
776     return TRUE;
777
778   h264_bitstream_init(&bitstream, 128*8);
779   h264_bitstream_write_uint(&bitstream, 0x00000001, 32); /* start code*/
780   h264_bitstream_write_nal_header(&bitstream, NAL_REF_IDC_HIGH, NAL_SPS);
781   h264_bitstream_write_sps(&bitstream, seq_param, encoder->profile);
782   ENCODER_ASSERT(BIT_STREAM_BIT_SIZE(&bitstream)%8 == 0);
783   length_in_bits = BIT_STREAM_BIT_SIZE(&bitstream);
784   packed_seq_buffer = BIT_STREAM_BUFFER(&bitstream);
785
786   /* set codec data sps */
787   priv->sps_data = gst_buffer_new_and_alloc((length_in_bits+7)/8);
788   GST_BUFFER_SIZE(priv->sps_data) = (length_in_bits+7)/8-4; /* start code size == 4*/
789   memcpy(GST_BUFFER_DATA(priv->sps_data),
790          packed_seq_buffer+4,
791          GST_BUFFER_SIZE(priv->sps_data));
792
793   packed_header_param_buffer.type = VAEncPackedHeaderSequence;
794   packed_header_param_buffer.bit_length = length_in_bits;
795   packed_header_param_buffer.has_emulation_bytes = 0;
796   va_status = vaCreateBuffer(va_dpy,
797                              context_id,
798                              VAEncPackedHeaderParameterBufferType,
799                              sizeof(packed_header_param_buffer), 1,
800                              &packed_header_param_buffer,
801                              &priv->packed_seq_param_id);
802   ENCODER_CHECK_STATUS(VA_STATUS_SUCCESS == va_status,
803                        FALSE,
804                        "EncPackedSeqHeaderParameterBuffer failed");
805   va_status = vaCreateBuffer(va_dpy,
806                              context_id,
807                              VAEncPackedHeaderDataBufferType,
808                              (length_in_bits + 7) / 8, 1,
809                              packed_seq_buffer,
810                              &priv->packed_seq_data_id);
811   h264_bitstream_destroy(&bitstream, TRUE);
812   ENCODER_CHECK_STATUS(VA_STATUS_SUCCESS == va_status,
813                        FALSE,
814                        "EncPackedSeqHeaderDataBuffer failed");
815 end:
816     return ret;
817
818 }
819
820 static gboolean
821 set_picture_parameters(
822     GstVaapiEncoderH264 *encoder,
823     VAEncPictureParameterBufferH264 *pic_param,
824     VABufferID coded_buf
825 )
826 {
827   GstVaapiEncoderH264Private *priv = encoder->priv;
828
829   pic_param->pic_parameter_set_id = 0;
830   pic_param->seq_parameter_set_id = 0;
831   pic_param->last_picture = 0; /* means last encoding picture */
832   pic_param->frame_num = (priv->cur_slice_type == SLICE_TYPE_B ?
833                        (priv->cur_decode_num + 1) : priv->cur_decode_num);
834   //pic_param.coding_type = 0;
835   pic_param->pic_init_qp = (encoder->init_qp >= 0 ? encoder->init_qp : 26);
836   pic_param->num_ref_idx_l0_active_minus1 = 0; /* only 1 reference */
837   pic_param->num_ref_idx_l1_active_minus1 = 0; /* B frames only have 1 backward and 1 forward reference*/
838   pic_param->chroma_qp_index_offset = 0;
839   pic_param->second_chroma_qp_index_offset = 0;
840
841   /* set picture fields */
842   pic_param->pic_fields.value = 0;
843   pic_param->pic_fields.bits.idr_pic_flag = (priv->cur_slice_type == SLICE_TYPE_I);
844   pic_param->pic_fields.bits.reference_pic_flag = (priv->cur_slice_type != SLICE_TYPE_B);
845   pic_param->pic_fields.bits.entropy_coding_mode_flag = ENTROPY_MODE_CABAC;
846   pic_param->pic_fields.bits.weighted_pred_flag = FALSE;
847   pic_param->pic_fields.bits.weighted_bipred_idc = 0;
848   pic_param->pic_fields.bits.constrained_intra_pred_flag = 0;
849   pic_param->pic_fields.bits.transform_8x8_mode_flag = TRUE; /* enable 8x8 */
850   pic_param->pic_fields.bits.deblocking_filter_control_present_flag = TRUE; /* enable debloking */
851   pic_param->pic_fields.bits.redundant_pic_cnt_present_flag = FALSE;
852   /* bottom_field_pic_order_in_frame_present_flag */
853   pic_param->pic_fields.bits.pic_order_present_flag = FALSE;
854   pic_param->pic_fields.bits.pic_scaling_matrix_present_flag = FALSE;
855
856   /* reference list,  */
857   pic_param->CurrPic.picture_id = GST_VAAPI_OBJECT_ID(priv->recon_surface);
858   pic_param->CurrPic.TopFieldOrderCnt = priv->cur_display_num * 2;   // ??? /**/
859   pic_param->ReferenceFrames[0].picture_id = GST_VAAPI_OBJECT_ID(priv->ref_surface1);
860   pic_param->ReferenceFrames[1].picture_id = GST_VAAPI_OBJECT_ID(priv->ref_surface2);
861   pic_param->ReferenceFrames[2].picture_id = VA_INVALID_ID;
862   pic_param->coded_buf = coded_buf;
863
864   ENCODER_LOG_INFO("type:%s, frame_num:%d, display_num:%d",
865     get_slice_type(priv->cur_slice_type),
866     pic_param->frame_num,
867     pic_param->CurrPic.TopFieldOrderCnt);
868   return TRUE;
869 }
870
871 static gboolean
872 ensure_packed_pps_data(
873     GstVaapiEncoderH264 *encoder,
874     VAEncPictureParameterBufferH264 *pic_param
875 )
876 {
877   GstVaapiEncoderH264Private *priv = encoder->priv;
878   VAEncPackedHeaderParameterBuffer packed_header_param_buffer = { 0 };
879   H264Bitstream bitstream;
880   VADisplay va_dpy = ENCODER_VA_DISPLAY(encoder);
881   VAContextID context_id = ENCODER_VA_CONTEXT(encoder);
882   guint32 length_in_bits;
883   guint8 *packed_pic_buffer = NULL;
884   gboolean ret = TRUE;
885   VAStatus va_status = VA_STATUS_SUCCESS;
886
887   if (VA_INVALID_ID != priv->packed_pic_data_id)
888       return TRUE;
889
890   h264_bitstream_init(&bitstream, 128*8);
891   h264_bitstream_write_uint(&bitstream, 0x00000001, 32); /* start code*/
892   h264_bitstream_write_nal_header(&bitstream, NAL_REF_IDC_HIGH, NAL_PPS);
893   h264_bitstream_write_pps(&bitstream, pic_param);
894   ENCODER_ASSERT(BIT_STREAM_BIT_SIZE(&bitstream)%8 == 0);
895   length_in_bits = BIT_STREAM_BIT_SIZE(&bitstream);
896   packed_pic_buffer = BIT_STREAM_BUFFER(&bitstream);
897
898   /*set codec data pps*/
899   priv->pps_data = gst_buffer_new_and_alloc((length_in_bits+7)/8);
900   GST_BUFFER_SIZE(priv->pps_data) = (length_in_bits+7)/8-4;
901   memcpy(GST_BUFFER_DATA(priv->pps_data),
902          packed_pic_buffer+4,
903          GST_BUFFER_SIZE(priv->pps_data));
904
905   packed_header_param_buffer.type = VAEncPackedHeaderPicture;
906   packed_header_param_buffer.bit_length = length_in_bits;
907   packed_header_param_buffer.has_emulation_bytes = 0;
908
909   va_status = vaCreateBuffer(va_dpy,
910                              context_id,
911                              VAEncPackedHeaderParameterBufferType,
912                              sizeof(packed_header_param_buffer), 1,
913                              &packed_header_param_buffer,
914                              &priv->packed_pic_param_id);
915   ENCODER_CHECK_STATUS(VA_STATUS_SUCCESS == va_status,
916                        FALSE,
917                        "EncPackedPicHeaderParameterBuffer failed");
918
919   va_status = vaCreateBuffer(va_dpy,
920                              context_id,
921                              VAEncPackedHeaderDataBufferType,
922                              (length_in_bits + 7) / 8, 1,
923                              packed_pic_buffer,
924                              &priv->packed_pic_data_id);
925   h264_bitstream_destroy(&bitstream, TRUE);
926   ENCODER_CHECK_STATUS(VA_STATUS_SUCCESS == va_status,
927                        FALSE,
928                        "EncPackedPicHeaderDataBuffer failed");
929 end:
930   return ret;
931 }
932
933 static gboolean
934 set_slices_parameters(
935     GstVaapiEncoderH264 *encoder,
936     VAEncSliceParameterBufferH264 *slices,
937     guint slice_num
938 )
939 {
940   GstVaapiEncoderH264Private *priv = encoder->priv;
941   VAEncSliceParameterBufferH264 *slice_param;
942
943   guint width_in_mbs = (ENCODER_WIDTH(encoder)+15)/16;
944   int i = 0;
945   guint32 last_row_num = 0;
946   guint32 slice_mod_num = priv->slice_mod_mb_num;
947
948   for (i = 0; i < slice_num; ++i) {
949     int i_pic = 0;
950     slice_param = slices + i;
951
952     slice_param->macroblock_address = last_row_num*width_in_mbs;
953     slice_param->num_macroblocks = width_in_mbs*priv->default_slice_height;
954     last_row_num += priv->default_slice_height;
955     if (slice_mod_num) {
956       slice_param->num_macroblocks += width_in_mbs;
957       ++last_row_num;
958       --slice_mod_num;
959     }
960     slice_param->macroblock_info = VA_INVALID_ID;
961     slice_param->slice_type = priv->cur_slice_type;
962     slice_param->pic_parameter_set_id = 0;
963     slice_param->idr_pic_id = priv->idr_num;
964     slice_param->pic_order_cnt_lsb =
965         (priv->cur_display_num*2) % priv->max_pic_order_cnt;
966
967     /* not used if pic_order_cnt_type = 0 */
968     slice_param->delta_pic_order_cnt_bottom = 0;
969     memset(slice_param->delta_pic_order_cnt,
970            0,
971            sizeof(slice_param->delta_pic_order_cnt));
972
973     /*only works for B frames*/
974     slice_param->direct_spatial_mv_pred_flag = FALSE;
975     /* default equal to picture parameters */
976     slice_param->num_ref_idx_active_override_flag = FALSE;
977     slice_param->num_ref_idx_l0_active_minus1 = 0;
978     slice_param->num_ref_idx_l1_active_minus1 = 0;
979
980     slice_param->RefPicList0[0].picture_id =
981         GST_VAAPI_OBJECT_ID(priv->ref_surface1);
982     for (i_pic = 1;
983          i_pic < sizeof(slice_param->RefPicList0)/sizeof(slice_param->RefPicList0[0]);
984          i_pic++) {
985       slice_param->RefPicList0[i_pic].picture_id = VA_INVALID_ID;
986     }
987
988     if (SLICE_TYPE_B == priv->cur_slice_type) {
989       slice_param->RefPicList1[0].picture_id =
990           GST_VAAPI_OBJECT_ID(priv->ref_surface2);
991       i_pic = 1;
992     } else
993       i_pic = 0;
994     for (;
995          i_pic < sizeof(slice_param->RefPicList1)/sizeof(slice_param->RefPicList1[0]);
996          i_pic++)
997       slice_param->RefPicList1[i_pic].picture_id = VA_INVALID_ID;
998
999     /* not used if  pic_param.pic_fields.bits.weighted_pred_flag == FALSE */
1000     slice_param->luma_log2_weight_denom = 0;
1001     slice_param->chroma_log2_weight_denom = 0;
1002     slice_param->luma_weight_l0_flag = FALSE;
1003     memset(slice_param->luma_weight_l0, 0, sizeof(slice_param->luma_weight_l0));
1004     memset(slice_param->luma_offset_l0, 0, sizeof(slice_param->luma_offset_l0));
1005     slice_param->chroma_weight_l0_flag = FALSE;
1006     memset(slice_param->chroma_weight_l0, 0, sizeof(slice_param->chroma_weight_l0));
1007     memset(slice_param->chroma_offset_l0, 0, sizeof(slice_param->chroma_offset_l0));
1008     slice_param->luma_weight_l1_flag = FALSE;
1009     memset(slice_param->luma_weight_l1, 0, sizeof(slice_param->luma_weight_l1));
1010     memset(slice_param->luma_offset_l1, 0, sizeof(slice_param->luma_offset_l1));
1011     slice_param->chroma_weight_l1_flag = FALSE;
1012     memset(slice_param->chroma_weight_l1, 0, sizeof(slice_param->chroma_weight_l1));
1013     memset(slice_param->chroma_offset_l1, 0, sizeof(slice_param->chroma_offset_l1));
1014
1015     slice_param->cabac_init_idc = 0;
1016     slice_param->slice_qp_delta = 0;
1017     slice_param->disable_deblocking_filter_idc = 0;
1018     slice_param->slice_alpha_c0_offset_div2 = 2;
1019     slice_param->slice_beta_offset_div2 = 2;
1020
1021   }
1022   ENCODER_ASSERT(last_row_num == (ENCODER_HEIGHT(encoder)+15)/16);
1023   return TRUE;
1024 }
1025
1026 #endif
1027
1028 static gboolean
1029 h264_fill_sequence_buffer(GstVaapiEncoderH264 *encoder)
1030 {
1031   GstVaapiEncoderH264Private *priv = encoder->priv;
1032   VAEncSequenceParameterBufferH264 seq_param = { 0 };
1033   VADisplay va_dpy       = ENCODER_VA_DISPLAY(encoder);
1034   VAContextID context_id = ENCODER_VA_CONTEXT(encoder);
1035   gboolean ret = TRUE;
1036   VAStatus va_status = VA_STATUS_SUCCESS;
1037
1038   /* only once */
1039   if (VA_INVALID_ID != priv->seq_param_id)
1040     return TRUE;
1041
1042   set_sequence_parameters(encoder, &seq_param);
1043   va_status = vaCreateBuffer(va_dpy, context_id,
1044                              VAEncSequenceParameterBufferType,
1045                              sizeof(seq_param), 1,
1046                              &seq_param, &priv->seq_param_id);
1047   ENCODER_CHECK_STATUS(VA_STATUS_SUCCESS == va_status,
1048                        FALSE,
1049                        "alloc seq-buffer failed.");
1050
1051 #if !HAVE_OLD_H264_ENCODER
1052   ensure_packed_sps_data(encoder, &seq_param);
1053 #endif
1054
1055 end:
1056   return ret;
1057 }
1058
1059 static gboolean
1060 h264_fill_picture_buffer(
1061     GstVaapiEncoderH264 *encoder,
1062     VABufferID coded_buf
1063 )
1064 {
1065   GstVaapiEncoderH264Private *priv = encoder->priv;
1066   VAEncPictureParameterBufferH264 pic_param;
1067   VADisplay va_dpy = ENCODER_VA_DISPLAY(encoder);
1068   VAContextID context_id = ENCODER_VA_CONTEXT(encoder);
1069   gboolean ret = TRUE;
1070   VAStatus va_status = VA_STATUS_SUCCESS;
1071
1072   VAAPI_UNUSED_ARG(va_status);
1073   memset(&pic_param, 0, sizeof(pic_param));
1074   set_picture_parameters(encoder, &pic_param, coded_buf);
1075
1076   if (VA_INVALID_ID != priv->pic_param_id) { /* share the same pic_param_id*/
1077     vaDestroyBuffer(va_dpy, priv->pic_param_id);
1078     priv->pic_param_id = VA_INVALID_ID;
1079   }
1080   va_status = vaCreateBuffer(va_dpy,
1081                              context_id,
1082                              VAEncPictureParameterBufferType,
1083                              sizeof(pic_param), 1,
1084                              &pic_param,
1085                              &priv->pic_param_id);
1086
1087   ENCODER_CHECK_STATUS(VA_STATUS_SUCCESS == va_status,
1088                        FALSE,
1089                        "creating pic-param buffer failed.");
1090 #if !HAVE_OLD_H264_ENCODER
1091   ensure_packed_pps_data(encoder, &pic_param);
1092 #endif
1093
1094 end:
1095   return ret;
1096 }
1097
1098 static gboolean
1099 h264_fill_slice_buffers(
1100     GstVaapiEncoderH264 *encoder
1101 )
1102 {
1103   GstVaapiEncoderH264Private *priv = encoder->priv;
1104   VADisplay va_dpy       = ENCODER_VA_DISPLAY(encoder);
1105   VAContextID context_id = ENCODER_VA_CONTEXT(encoder);
1106   gboolean ret = TRUE;
1107   VAStatus va_status = VA_STATUS_SUCCESS;
1108
1109   memset(priv->slice_param_buffers,
1110          0,
1111          encoder->slice_num * sizeof(priv->slice_param_buffers[0]));
1112   set_slices_parameters(encoder,
1113                         priv->slice_param_buffers,
1114                         encoder->slice_num);
1115
1116   if (VA_INVALID_ID != priv->slice_param_id) {
1117     vaDestroyBuffer(va_dpy, priv->slice_param_id);
1118     priv->slice_param_id = VA_INVALID_ID;
1119   }
1120   va_status = vaCreateBuffer(va_dpy,
1121                              context_id,
1122                              VAEncSliceParameterBufferType,
1123                              sizeof(priv->slice_param_buffers[0]),
1124                              encoder->slice_num,
1125                              priv->slice_param_buffers,
1126                              &priv->slice_param_id);
1127   ENCODER_CHECK_STATUS(VA_STATUS_SUCCESS == va_status,
1128                        FALSE,
1129                        "creating slice-parameters buffer failed.");
1130
1131 end:
1132   return ret;
1133 }
1134
1135 static EncoderStatus
1136 gst_vaapi_encoder_h264_rendering(
1137     GstVaapiBaseEncoder *base,
1138     GstVaapiSurface *surface,
1139     guint frame_index,
1140     VABufferID coded_buf,
1141     gboolean *is_key
1142 )
1143 {
1144   EncoderStatus ret = ENCODER_NO_ERROR;
1145   VAStatus va_status = VA_STATUS_SUCCESS;
1146   GstVaapiEncoderH264 *encoder = GST_VAAPI_ENCODER_H264_CAST(base);
1147   GstVaapiEncoderH264Private *priv = encoder->priv;
1148   GstVaapiDisplay *display = ENCODER_DISPLAY(base);
1149   GstVaapiContext *context = ENCODER_CONTEXT(base);
1150   VADisplay va_dpy;
1151   VAContextID context_id;
1152   VABufferID va_buffers[64];
1153   guint32    va_buffers_count = 0;
1154   gboolean is_params_ok = TRUE;
1155   gboolean is_locked = FALSE;
1156
1157   ENCODER_ASSERT(context);
1158
1159   va_dpy = ENCODER_VA_DISPLAY(base);
1160   context_id = ENCODER_VA_CONTEXT(base);
1161
1162   *is_key = (priv->cur_slice_type == SLICE_TYPE_I);
1163
1164   if (!priv->ref_surface1) {
1165     priv->ref_surface1 = gst_vaapi_context_get_surface(context);
1166     ENCODER_CHECK_STATUS(priv->ref_surface1,
1167                          ENCODER_SURFACE_ERR,
1168                          "reference surface, h264_pop_free_surface failed.");
1169   }
1170   if (!priv->ref_surface2) {
1171     priv->ref_surface2 = gst_vaapi_context_get_surface(context);
1172     ENCODER_CHECK_STATUS(priv->ref_surface2,
1173                          ENCODER_SURFACE_ERR,
1174                          "reference surface, h264_pop_free_surface failed.");
1175   }
1176   if (!priv->recon_surface) {
1177     priv->recon_surface = gst_vaapi_context_get_surface(context);
1178     ENCODER_CHECK_STATUS(priv->recon_surface,
1179                          ENCODER_SURFACE_ERR,
1180                          "reconstructed surface, h264_pop_free_surface failed.");
1181   }
1182
1183   if (SLICE_TYPE_P == priv->cur_slice_type) {
1184     h264_swap_surface(&priv->ref_surface1, &priv->ref_surface2);
1185   }
1186
1187   /* fill sequence parameters, need set every time */
1188   is_params_ok = h264_fill_sequence_buffer(encoder);
1189   ENCODER_CHECK_STATUS(is_params_ok,
1190                        ENCODER_PARAMETER_ERR,
1191                        "h264_recreate_seq_param failed");
1192   /* set pic_parameters*/
1193   is_params_ok = h264_fill_picture_buffer(encoder, coded_buf);
1194   ENCODER_CHECK_STATUS(is_params_ok,
1195                        ENCODER_PARAMETER_ERR,
1196                        "h264_recreate_pic_param failed");
1197   /* set slice parameters, support multiple slices */
1198   is_params_ok = h264_fill_slice_buffers(encoder);
1199   ENCODER_CHECK_STATUS(is_params_ok,
1200                        ENCODER_PARAMETER_ERR,
1201                        "h264_recreate_slice_param failed");
1202
1203   /* lock display */
1204   ENCODER_ACQUIRE_DISPLAY_LOCK(display);
1205
1206   /*render all buffers*/
1207   if (VA_INVALID_ID != priv->seq_param_id) {
1208     va_buffers[va_buffers_count++] = priv->seq_param_id;
1209   }
1210   if (VA_INVALID_ID != priv->pic_param_id) {
1211     va_buffers[va_buffers_count++] = priv->pic_param_id;
1212   }
1213   if (VA_INVALID_ID != priv->slice_param_id) {
1214     va_buffers[va_buffers_count++] = priv->slice_param_id;
1215   }
1216   if (SLICE_TYPE_I == priv->cur_slice_type) {
1217     if (VA_INVALID_ID != priv->packed_seq_param_id) {
1218       va_buffers[va_buffers_count++] = priv->packed_seq_param_id;
1219     }
1220     if (VA_INVALID_ID != priv->packed_seq_data_id) {
1221       va_buffers[va_buffers_count++] = priv->packed_seq_data_id;
1222     }
1223     if (VA_INVALID_ID != priv->packed_pic_param_id) {
1224       va_buffers[va_buffers_count++] = priv->packed_pic_param_id;
1225     }
1226     if (VA_INVALID_ID != priv->packed_pic_data_id) {
1227       va_buffers[va_buffers_count++] = priv->packed_pic_data_id;
1228     }
1229   }
1230
1231   va_status = vaRenderPicture(va_dpy,
1232                               context_id,
1233                               va_buffers,
1234                               va_buffers_count);
1235   ENCODER_CHECK_STATUS(VA_STATUS_SUCCESS == va_status,
1236                        ENCODER_PICTURE_ERR,
1237                        "vaRenderH264Picture failed.");
1238
1239   /*after finished,  swap  recon and surface2*/
1240   if (SLICE_TYPE_P == priv->cur_slice_type ||
1241       SLICE_TYPE_I == priv->cur_slice_type) {
1242     h264_swap_surface(&priv->recon_surface, &priv->ref_surface2);
1243   }
1244
1245   end:
1246   ENCODER_RELEASE_DISPLAY_LOCK(display);
1247   return ret;
1248 }
1249
1250 static const guint8 *
1251 h264_next_nal(const guint8 *buffer, guint32 len, guint32 *nal_size)
1252 {
1253     const guint8 *cur = buffer;
1254     const guint8 *end = buffer + len;
1255     const guint8 *nal_start = NULL;
1256     guint32 flag = 0xFFFFFFFF;
1257     guint32 nal_start_len = 0;
1258
1259     ENCODER_ASSERT(len >= 0 && buffer && nal_size);
1260     if (len < 3) {
1261         *nal_size = len;
1262         nal_start = (len ? buffer : NULL);
1263         return nal_start;
1264     }
1265
1266     /*locate head postion*/
1267     if (!buffer[0] && !buffer[1]) {
1268         if (buffer[2] == 1) { // 0x000001
1269             nal_start_len = 3;
1270         } else if (!buffer[2] && len >=4 && buffer[3] == 1) { //0x00000001
1271             nal_start_len = 4;
1272         }
1273     }
1274     nal_start = buffer + nal_start_len;
1275     cur = nal_start;
1276
1277     /*find next nal start position*/
1278     while (cur < end) {
1279         flag = ((flag<<8) | ((*cur++)&0xFF));
1280         if (flag == 0x00000001) {
1281             *nal_size = cur - 4 - nal_start;
1282             break;
1283         } else if ((flag&0x00FFFFFF) == 0x00000001) {
1284             *nal_size = cur - 3 - nal_start;
1285             break;
1286         }
1287     }
1288     if (cur >= end) {
1289       *nal_size = end - nal_start;
1290       if (nal_start >= end) {
1291         nal_start = NULL;
1292       }
1293     }
1294     return nal_start;
1295 }
1296
1297 static GstBuffer *
1298 gst_vaapi_encoder_h264_copy_coded_buffer(
1299     GstVaapiBaseEncoder *base,
1300     guint8 *frame,
1301     guint32 frame_size,
1302     VABufferID *coded_buf
1303 )
1304 {
1305   GstVaapiEncoderH264 *encoder = GST_VAAPI_ENCODER_H264_CAST(base);
1306   GstVaapiEncoderH264Private *priv = encoder->priv;
1307   GstBuffer *ret_buffer;
1308   guint32   nal_size;
1309   const guint8   *nal_start;
1310   guint8  *frame_end;
1311
1312   ret_buffer = gst_buffer_new();
1313   ENCODER_ASSERT(ret_buffer);
1314   H264Bitstream bitstream;
1315   h264_bitstream_init(&bitstream, (frame_size+32)*8);
1316   h264_bitstream_align(&bitstream, 0);
1317   ENCODER_ASSERT(bitstream.bit_size == 0);
1318
1319   if (!priv->avc_flag) { /*nal format*/
1320     h264_bitstream_write_byte_array(&bitstream, frame, frame_size);
1321     ENCODER_ASSERT(bitstream.bit_size == frame_size*8);
1322   } else { /* elementary format */
1323     frame_end = frame + frame_size;
1324     nal_start = frame;
1325     nal_size = 0;
1326     while(NULL !=
1327           (nal_start = h264_next_nal(nal_start, frame_end-nal_start, &nal_size))
1328          ) {
1329       ENCODER_ASSERT(nal_size);
1330       if (!nal_size) {
1331         nal_start += nal_size;
1332         continue;
1333       }
1334       h264_bitstream_write_uint(&bitstream, nal_size, 32);
1335       h264_bitstream_write_byte_array(&bitstream, nal_start, nal_size);
1336       nal_start += nal_size;
1337     }
1338   }
1339   h264_bitstream_align(&bitstream, 0);
1340
1341   GST_BUFFER_MALLOCDATA(ret_buffer) =
1342         GST_BUFFER_DATA(ret_buffer) = BIT_STREAM_BUFFER(&bitstream);
1343   GST_BUFFER_SIZE(ret_buffer) = BIT_STREAM_BIT_SIZE(&bitstream)/8;
1344   h264_bitstream_destroy(&bitstream, FALSE);
1345
1346   return ret_buffer;
1347 }
1348
1349 static EncoderStatus
1350 read_sps_pps(GstVaapiEncoderH264Private *priv, const guint8 *buf, guint32 size)
1351 {
1352   const guint8 *end = buf + size;
1353   const guint8 *nal_start = buf;
1354   guint32 nal_size = 0;
1355   guint8 nal_type;
1356   GstBuffer *sps = NULL, *pps = NULL;
1357
1358   while((!sps || !pps) &&
1359         (nal_start = h264_next_nal(nal_start, end-nal_start, &nal_size)) != NULL
1360        ) {
1361     if (!nal_size) {
1362       nal_start += nal_size;
1363       continue;
1364     }
1365
1366     nal_type = (*nal_start)&0x1F;
1367     switch (nal_type) {
1368       case NAL_SPS: {
1369         sps = gst_buffer_new_and_alloc(nal_size);
1370         memcpy(GST_BUFFER_DATA(sps), nal_start, nal_size);
1371         gst_buffer_replace(&priv->sps_data, sps);
1372         gst_buffer_unref(sps); /*don't set to NULL*/
1373         break;
1374       }
1375
1376       case NAL_PPS: {
1377         pps = gst_buffer_new_and_alloc(nal_size);
1378         memcpy(GST_BUFFER_DATA(pps), nal_start, nal_size);
1379         gst_buffer_replace(&priv->pps_data, pps);
1380         gst_buffer_unref(pps);
1381         break;
1382       }
1383
1384       default:
1385         break;
1386     }
1387     nal_start += nal_size;
1388   }
1389
1390   if (!sps || !pps) {
1391     return ENCODER_DATA_NOT_READY;
1392   }
1393
1394   return ENCODER_NO_ERROR;
1395 }
1396
1397 static void
1398 gst_vaapi_encoder_h264_notify_frame(
1399     GstVaapiBaseEncoder *base,
1400     guint8 *buf,
1401     guint32 size
1402 )
1403 {
1404   GstVaapiEncoderH264 *encoder = GST_VAAPI_ENCODER_H264_CAST(base);
1405   GstVaapiEncoderH264Private *priv = encoder->priv;
1406   if (!priv->sps_data || !priv->pps_data) {
1407     read_sps_pps(priv, buf, size);
1408   }
1409   if (priv->sps_data && priv->pps_data) {
1410     gst_vaapi_base_encoder_set_frame_notify(base, FALSE);
1411   }
1412 }
1413
1414 static gboolean
1415 read_sps_attributes(
1416     const guint8 *sps_data,
1417     guint32 sps_size,
1418     guint32 *profile_idc,
1419     guint32 *profile_comp,
1420     guint32 *level_idc
1421 )
1422 {
1423   ENCODER_ASSERT(profile_idc && profile_comp && level_idc);
1424   ENCODER_ASSERT(sps_size >= 4);
1425   if (sps_size < 4) {
1426     return FALSE;
1427   }
1428   /*skip sps_data[0], nal_type*/
1429   *profile_idc = sps_data[1];
1430   *profile_comp = sps_data[2];
1431   *level_idc = sps_data[3];
1432   return TRUE;
1433 }
1434
1435 static EncoderStatus
1436 gst_vaapi_encoder_h264_flush(
1437     GstVaapiEncoder* base,
1438     GList **coded_pics
1439 )
1440 {
1441   GstVaapiEncoderH264* encoder = GST_VAAPI_ENCODER_H264_CAST(base);
1442   EncoderStatus ret = ENCODER_NO_ERROR;
1443   GstVaapiEncoderH264Private *priv = encoder->priv;
1444
1445   //priv->frame_count = 0;
1446   priv->cur_display_num = 0;
1447   priv->cur_decode_num = 0;
1448   priv->cur_slice_type = SLICE_TYPE_I;
1449   priv->gop_count = g_queue_get_length(priv->queued_buffers);
1450   //gst_vaapi_base_encoder_set_frame_notify((GST_VAAPI_BASE_ENCODER)encoder, TRUE);
1451
1452   //end:
1453   return ret;
1454 }
1455
1456 static EncoderStatus
1457 gst_vaapi_encoder_h264_get_avcC_codec_data(
1458     GstVaapiEncoderH264 *encoder,
1459     GstBuffer **buffer
1460 )
1461 {
1462   GstVaapiEncoderH264Private *priv = encoder->priv;
1463   GstBuffer *avc_codec;
1464   const guint32 configuration_version = 0x01;
1465   const guint32 length_size_minus_one = 0x03;
1466   guint32 profile, profile_comp, level_idc;
1467
1468   ENCODER_ASSERT(buffer);
1469   if (!priv->sps_data || !priv->pps_data) {
1470     return ENCODER_DATA_NOT_READY;
1471   }
1472
1473   if (FALSE == read_sps_attributes(GST_BUFFER_DATA(priv->sps_data),
1474                                    GST_BUFFER_SIZE(priv->sps_data),
1475                                    &profile, &profile_comp, &level_idc))
1476   {
1477     ENCODER_ASSERT(0);
1478     return ENCODER_DATA_ERR;
1479   }
1480
1481   H264Bitstream bitstream;
1482   h264_bitstream_init(&bitstream,
1483                       (GST_BUFFER_SIZE(priv->sps_data) +
1484                        GST_BUFFER_SIZE(priv->pps_data) + 32)*8);
1485
1486   /*codec_data*/
1487   h264_bitstream_write_uint(&bitstream, configuration_version, 8);
1488   h264_bitstream_write_uint(&bitstream, profile, 8);
1489   h264_bitstream_write_uint(&bitstream, profile_comp, 8);
1490   h264_bitstream_write_uint(&bitstream, level_idc, 8);
1491   h264_bitstream_write_uint(&bitstream, h264_bit_mask[6], 6); /*111111*/
1492   h264_bitstream_write_uint(&bitstream, length_size_minus_one, 2);
1493   h264_bitstream_write_uint(&bitstream, h264_bit_mask[3], 3); /*111*/
1494
1495   /*write sps*/
1496   h264_bitstream_write_uint(&bitstream, 1, 5);   /* sps count = 1*/
1497   ENCODER_ASSERT( BIT_STREAM_BIT_SIZE(&bitstream)%8 == 0);
1498   h264_bitstream_write_uint(&bitstream, GST_BUFFER_SIZE(priv->sps_data), 16);
1499   h264_bitstream_write_byte_array(&bitstream,
1500                                   GST_BUFFER_DATA(priv->sps_data),
1501                                   GST_BUFFER_SIZE(priv->sps_data));
1502
1503   /*write pps*/
1504   h264_bitstream_write_uint(&bitstream, 1, 8); /*pps count = 1*/
1505   h264_bitstream_write_uint(&bitstream, GST_BUFFER_SIZE(priv->pps_data), 16);
1506   h264_bitstream_write_byte_array(&bitstream,
1507                                   GST_BUFFER_DATA(priv->pps_data),
1508                                   GST_BUFFER_SIZE(priv->pps_data));
1509
1510   avc_codec = gst_buffer_new();
1511   GST_BUFFER_MALLOCDATA(avc_codec) =
1512          GST_BUFFER_DATA(avc_codec) =
1513          BIT_STREAM_BUFFER(&bitstream);
1514   GST_BUFFER_SIZE(avc_codec) = BIT_STREAM_BIT_SIZE(&bitstream)/8;
1515   h264_bitstream_destroy(&bitstream, FALSE);
1516   *buffer = avc_codec;
1517
1518   return ENCODER_NO_ERROR;
1519 }
1520
1521 static EncoderStatus
1522 gst_vaapi_encoder_h264_get_codec_data(
1523     GstVaapiEncoder* base,
1524     GstBuffer **buffer)
1525 {
1526   GstVaapiEncoderH264 *encoder = GST_VAAPI_ENCODER_H264_CAST(base);
1527   GstVaapiEncoderH264Private *priv = encoder->priv;
1528
1529   if (priv->avc_flag)
1530     return gst_vaapi_encoder_h264_get_avcC_codec_data(encoder, buffer);
1531   return ENCODER_NO_DATA;
1532 }
1533
1534 static void
1535 gst_vaapi_encoder_h264_init(GstVaapiEncoderH264 *encoder)
1536 {
1537   GstVaapiEncoderH264Private *priv = GST_VAAPI_ENCODER_H264_GET_PRIVATE(encoder);
1538   ENCODER_ASSERT(priv);
1539   priv->public = encoder;
1540   encoder->priv = priv;
1541
1542   /* init public attributes */
1543   gst_vaapi_encoder_h264_init_public_values(encoder);
1544   gst_vaapi_base_encoder_set_frame_notify(GST_VAAPI_BASE_ENCODER(encoder), TRUE);
1545
1546   /* init private values*/
1547   priv->format = GST_MAKE_FOURCC('N','V','1','2');
1548   priv->avc_flag = FALSE;
1549
1550   priv->ref_surface1 = NULL;
1551   priv->ref_surface2 = NULL;
1552   priv->recon_surface = NULL;
1553
1554   priv->seq_param_id = VA_INVALID_ID;
1555   priv->pic_param_id = VA_INVALID_ID;
1556   priv->slice_param_id = VA_INVALID_ID;
1557   priv->packed_seq_param_id = VA_INVALID_ID;
1558   priv->packed_seq_data_id = VA_INVALID_ID;
1559   priv->packed_pic_param_id = VA_INVALID_ID;
1560   priv->packed_pic_data_id = VA_INVALID_ID;
1561   priv->slice_param_buffers = NULL;
1562   priv->default_slice_height = 0;
1563   priv->slice_mod_mb_num = 0;
1564
1565   priv->sps_data = NULL;
1566   priv->pps_data = NULL;
1567
1568   priv->queued_buffers = g_queue_new();
1569   priv->gop_count = 0;
1570   priv->cur_display_num = 0;
1571   priv->cur_decode_num = 0;
1572   priv->cur_slice_type = SLICE_TYPE_I;
1573   priv->last_decode_time = 0LL;
1574   priv->default_cts_offset = 0;
1575
1576   priv->max_frame_num = 0;
1577   priv->max_pic_order_cnt = 0;
1578   priv->idr_num = 0;
1579 }
1580
1581 static void
1582 gst_vaapi_encoder_h264_finalize(GObject *object)
1583 {
1584   /*free private buffers*/
1585   GstVaapiEncoder *encoder = GST_VAAPI_ENCODER(object);
1586   GstVaapiEncoderH264Private *priv = GST_VAAPI_ENCODER_H264_GET_PRIVATE(object);
1587
1588   if (gst_vaapi_encoder_get_state(encoder) != VAAPI_ENC_NULL) {
1589     gst_vaapi_encoder_uninitialize(encoder);
1590   }
1591
1592   if (priv->sps_data) {
1593     gst_buffer_unref(priv->sps_data);
1594     priv->sps_data = NULL;
1595   }
1596   if (priv->pps_data) {
1597     gst_buffer_unref(priv->pps_data);
1598     priv->pps_data = NULL;
1599   }
1600   if (priv->slice_param_buffers) {
1601     g_free(priv->slice_param_buffers);
1602     priv->slice_param_buffers = NULL;
1603   }
1604
1605   if (priv->queued_buffers) {
1606     ENCODER_ASSERT(g_queue_is_empty(priv->queued_buffers));
1607     g_queue_free(priv->queued_buffers);
1608     priv->queued_buffers = NULL;
1609   }
1610
1611   G_OBJECT_CLASS(gst_vaapi_encoder_h264_parent_class)->finalize(object);
1612 }
1613
1614 static void
1615 gst_vaapi_encoder_h264_class_init(GstVaapiEncoderH264Class *klass)
1616 {
1617   GObjectClass * const object_class = G_OBJECT_CLASS(klass);
1618   GstVaapiEncoderClass * const encoder_class = GST_VAAPI_ENCODER_CLASS(klass);
1619   GstVaapiBaseEncoderClass * const base_class = GST_VAAPI_BASE_ENCODER_CLASS(klass);
1620
1621   g_type_class_add_private(klass, sizeof(GstVaapiEncoderH264Private));
1622
1623   GST_DEBUG_CATEGORY_INIT (gst_vaapi_h264_encoder_debug,
1624                            "gst_va_h264_encoder",
1625                            0,
1626                            "gst_va_h264_encoder element");
1627
1628   object_class->finalize = gst_vaapi_encoder_h264_finalize;
1629
1630   base_class->validate_attributes = gst_vaapi_encoder_h264_validate_attributes;
1631   base_class->pre_alloc_resource  = gst_vaapi_encoder_h264_alloc_slices;
1632   base_class->release_resource    = gst_vaapi_encoder_h264_release_resource;
1633   base_class->prepare_next_input_buffer =
1634       gst_vaapi_encoder_h264_prepare_next_buffer;
1635   base_class->render_frame = gst_vaapi_encoder_h264_rendering;
1636   base_class->notify_frame = gst_vaapi_encoder_h264_notify_frame;
1637   base_class->copy_coded_frame = gst_vaapi_encoder_h264_copy_coded_buffer;
1638   base_class->encode_frame_failed = gst_vaapi_encoder_h264_frame_failed;
1639
1640   encoder_class->flush = gst_vaapi_encoder_h264_flush;
1641   encoder_class->get_codec_data = gst_vaapi_encoder_h264_get_codec_data;
1642 }
1643
1644 static void
1645 h264_bitstream_init(H264Bitstream *bitstream, guint32 bit_capability)
1646 {
1647   bitstream->bit_size = 0;
1648   bitstream->buffer = NULL;
1649   bitstream->max_bit_capability = 0;
1650   if (bit_capability) {
1651     h264_bitstream_auto_grow(bitstream, bit_capability);
1652   }
1653 }
1654
1655 static gboolean
1656 h264_bitstream_write_uint(
1657     H264Bitstream *bitstream,
1658     guint32 value,
1659     guint32 bit_size
1660 )
1661 {
1662   gboolean ret = TRUE;
1663   guint32 byte_pos, bit_offset;
1664   guint8  *cur_byte;
1665   guint32 fill_bits;
1666
1667   if(!bit_size) {
1668     return TRUE;
1669   }
1670
1671   VAAPI_UNUSED_ARG(ret);
1672   ENCODER_CHECK_STATUS(h264_bitstream_auto_grow(bitstream, bit_size),
1673                        FALSE,
1674                        "h264_bitstream_auto_grow failed.");
1675   byte_pos = (bitstream->bit_size>>3);
1676   bit_offset = (bitstream->bit_size&0x07);
1677   cur_byte = bitstream->buffer + byte_pos;
1678   ENCODER_ASSERT(bit_offset < 8 &&
1679                  bitstream->bit_size <= bitstream->max_bit_capability);
1680
1681   while (bit_size) {
1682     fill_bits = ((8-bit_offset) < bit_size ? (8-bit_offset) : bit_size);
1683     bit_size -= fill_bits;
1684     bitstream->bit_size += fill_bits;
1685
1686     *cur_byte |=
1687         ((value>>bit_size) & h264_bit_mask[fill_bits])<<(8-bit_offset-fill_bits);
1688     ++cur_byte;
1689     bit_offset = 0;
1690   }
1691   ENCODER_ASSERT(cur_byte <=
1692                  (bitstream->buffer + bitstream->max_bit_capability/8));
1693
1694   end:
1695   return ret;
1696 }
1697
1698 static gboolean
1699 h264_bitstream_align(H264Bitstream *bitstream, guint32 value)
1700 {
1701   guint32 bit_offset, bit_left;
1702
1703   bit_offset = (bitstream->bit_size&0x07);
1704   if (!bit_offset) {
1705     return TRUE;
1706   }
1707   bit_left = 8 - bit_offset;
1708   if (value) value = h264_bit_mask[bit_left];
1709   return h264_bitstream_write_uint(bitstream, value, bit_left);
1710 }
1711
1712
1713 static gboolean
1714 h264_bitstream_write_byte_array(
1715     H264Bitstream *bitstream,
1716     const guint8 *buf,
1717     guint32 byte_size
1718 )
1719 {
1720   gboolean ret = TRUE;
1721   if (!byte_size) {
1722     return 0;
1723   }
1724
1725   VAAPI_UNUSED_ARG(ret);
1726   ENCODER_CHECK_STATUS(h264_bitstream_auto_grow(bitstream, byte_size<<3),
1727                        FALSE,
1728                        "h264_bitstream_auto_grow failed.");
1729   if (0 == (bitstream->bit_size&0x07)) {
1730     memcpy(&bitstream->buffer[bitstream->bit_size>>3], buf, byte_size);
1731     bitstream->bit_size += (byte_size<<3);
1732   } else {
1733     ENCODER_ASSERT(0);
1734     while(byte_size) {
1735       h264_bitstream_write_uint(bitstream, *buf, 8);
1736       --byte_size;
1737       ++buf;
1738     }
1739   }
1740
1741 end:
1742   return ret;
1743 }
1744
1745 static gboolean
1746 h264_bitstream_write_ue(H264Bitstream *bitstream, guint32 value)
1747 {
1748   gboolean ret = TRUE;
1749   guint32  size_in_bits = 0;
1750   guint32  tmp_value = ++value;
1751   while (tmp_value) {
1752     ++size_in_bits;
1753     tmp_value >>= 1;
1754   }
1755   ENCODER_CHECK_STATUS(h264_bitstream_write_uint(bitstream, 0, size_in_bits-1),
1756                        FALSE,
1757                        "h264_bitstream_write_ue failed.");
1758   ENCODER_CHECK_STATUS(h264_bitstream_write_uint(bitstream, value, size_in_bits),
1759                        FALSE,
1760                        "h264_bitstream_write_ue failed.");
1761
1762 end:
1763   return ret;
1764 }
1765
1766 static gboolean
1767 h264_bitstream_write_se(H264Bitstream *bitstream, gint32 value)
1768 {
1769   gboolean ret = TRUE;
1770   guint32 new_val;
1771
1772   if (value <= 0) {
1773     new_val = -(value<<1);
1774   } else {
1775     new_val = (value<<1) - 1;
1776   }
1777
1778   ENCODER_CHECK_STATUS(h264_bitstream_write_ue(bitstream, new_val),
1779                        FALSE,
1780                        "h264_bitstream_write_se failed.");
1781
1782 end:
1783   return ret;
1784 }
1785
1786 static gboolean
1787 h264_bitstream_write_trailing_bits(H264Bitstream *bitstream)
1788 {
1789     h264_bitstream_write_uint(bitstream, 1, 1);
1790     h264_bitstream_align(bitstream, 0);
1791     return TRUE;
1792 }
1793
1794 static void
1795 h264_bitstream_destroy(H264Bitstream *bitstream, gboolean free_flag)
1796 {
1797   if (bitstream->buffer && free_flag) {
1798     free (bitstream->buffer);
1799   }
1800   bitstream->buffer = NULL;
1801   bitstream->bit_size = 0;
1802   bitstream->max_bit_capability = 0;
1803 }
1804
1805 static gboolean
1806 h264_bitstream_auto_grow(H264Bitstream *bitstream, guint32 extra_bit_size)
1807 {
1808   guint32 new_bit_size = extra_bit_size + bitstream->bit_size;
1809   guint32 clear_pos;
1810
1811   ENCODER_ASSERT(bitstream->bit_size <= bitstream->max_bit_capability);
1812   if (new_bit_size <= bitstream->max_bit_capability) {
1813     return TRUE;
1814   }
1815
1816   new_bit_size = ((new_bit_size + H264_BITSTREAM_ALLOC_ALIGN_MASK)
1817                 &(~H264_BITSTREAM_ALLOC_ALIGN_MASK));
1818   ENCODER_ASSERT(new_bit_size%(H264_BITSTREAM_ALLOC_ALIGN_MASK+1) == 0);
1819   clear_pos = ((bitstream->bit_size+7)>>3);
1820   bitstream->buffer = realloc(bitstream->buffer, new_bit_size>>3);
1821   memset(bitstream->buffer+clear_pos, 0, (new_bit_size>>3)-clear_pos);
1822   bitstream->max_bit_capability = new_bit_size;
1823   return TRUE;
1824 }
1825
1826 static gboolean
1827 h264_bitstream_write_nal_header(
1828     H264Bitstream *bitstream,
1829     guint nal_ref_idc,
1830     guint nal_unit_type
1831 )
1832 {
1833   h264_bitstream_write_uint(bitstream, 0, 1);
1834   h264_bitstream_write_uint(bitstream, nal_ref_idc, 2);
1835   h264_bitstream_write_uint(bitstream, nal_unit_type, 5);
1836   return TRUE;
1837 }
1838
1839 #if !HAVE_OLD_H264_ENCODER
1840
1841 static gboolean
1842 h264_bitstream_write_sps(
1843     H264Bitstream *bitstream,
1844     VAEncSequenceParameterBufferH264 *seq,
1845     H264_Profile profile
1846 )
1847 {
1848   guint32 constraint_set0_flag, constraint_set1_flag;
1849   guint32 constraint_set2_flag, constraint_set3_flag;
1850   guint32 gaps_in_frame_num_value_allowed_flag = 0; // ??
1851
1852   guint32 b_qpprime_y_zero_transform_bypass = 0;
1853   guint32 residual_color_transform_flag = 0;
1854   guint32 pic_height_in_map_units =
1855     (seq->seq_fields.bits.frame_mbs_only_flag ?
1856      seq->picture_height_in_mbs :
1857      seq->picture_height_in_mbs/2);
1858   guint32 mb_adaptive_frame_field = !seq->seq_fields.bits.frame_mbs_only_flag;
1859   guint32 i = 0;
1860
1861   constraint_set0_flag = profile == H264_PROFILE_BASELINE;
1862   constraint_set1_flag = profile <= H264_PROFILE_MAIN;
1863   constraint_set2_flag = 0;
1864   constraint_set3_flag = 0;
1865
1866   /* profile_idc */
1867   h264_bitstream_write_uint(bitstream, profile, 8);
1868   /* constraint_set0_flag */
1869   h264_bitstream_write_uint(bitstream, constraint_set0_flag, 1);
1870   /* constraint_set1_flag */
1871   h264_bitstream_write_uint(bitstream, constraint_set1_flag, 1);
1872   /* constraint_set2_flag */
1873   h264_bitstream_write_uint(bitstream, constraint_set2_flag, 1);
1874   /* constraint_set3_flag */
1875   h264_bitstream_write_uint(bitstream, constraint_set3_flag, 1);
1876   /* reserved_zero_4bits */
1877   h264_bitstream_write_uint(bitstream, 0, 4);
1878   /* level_idc */
1879   h264_bitstream_write_uint(bitstream, seq->level_idc, 8);
1880   /* seq_parameter_set_id */
1881   h264_bitstream_write_ue(bitstream, seq->seq_parameter_set_id);
1882
1883   if (profile >= H264_PROFILE_HIGH) {
1884     /* for high profile */
1885     ENCODER_ASSERT(0);
1886     /* chroma_format_idc  = 1, 4:2:0*/
1887     h264_bitstream_write_ue(bitstream, seq->seq_fields.bits.chroma_format_idc);
1888     if (3 == seq->seq_fields.bits.chroma_format_idc) {
1889       h264_bitstream_write_uint(bitstream, residual_color_transform_flag, 1);
1890     }
1891     /* bit_depth_luma_minus8 */
1892     h264_bitstream_write_ue(bitstream, seq->bit_depth_luma_minus8);
1893     /* bit_depth_chroma_minus8 */
1894     h264_bitstream_write_ue(bitstream, seq->bit_depth_chroma_minus8);
1895     /* b_qpprime_y_zero_transform_bypass */
1896     h264_bitstream_write_uint(bitstream, b_qpprime_y_zero_transform_bypass, 1);
1897     ENCODER_ASSERT(seq->seq_fields.bits.seq_scaling_matrix_present_flag == 0);
1898     /*seq_scaling_matrix_present_flag  */
1899     h264_bitstream_write_uint(bitstream,
1900                               seq->seq_fields.bits.seq_scaling_matrix_present_flag,
1901                               1);
1902
1903     #if 0
1904     if (seq->seq_fields.bits.seq_scaling_matrix_present_flag) {
1905       for (i = 0; i < (seq->seq_fields.bits.chroma_format_idc != 3 ? 8 : 12); i++) {
1906         h264_bitstream_write_uint(bitstream, seq->seq_fields.bits.seq_scaling_list_present_flag, 1);
1907         if (seq->seq_fields.bits.seq_scaling_list_present_flag) {
1908           ENCODER_ASSERT(0);
1909           /* FIXME, need write scaling list if seq_scaling_matrix_present_flag ==1*/
1910         }
1911       }
1912     }
1913     #endif
1914   }
1915
1916   /* log2_max_frame_num_minus4 */
1917   h264_bitstream_write_ue(bitstream,
1918                           seq->seq_fields.bits.log2_max_frame_num_minus4);
1919   /* pic_order_cnt_type */
1920   h264_bitstream_write_ue(bitstream, seq->seq_fields.bits.pic_order_cnt_type);
1921
1922   if (seq->seq_fields.bits.pic_order_cnt_type == 0) {
1923     /* log2_max_pic_order_cnt_lsb_minus4 */
1924     h264_bitstream_write_ue(bitstream,
1925                             seq->seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4);
1926   } else if (seq->seq_fields.bits.pic_order_cnt_type == 1) {
1927     ENCODER_ASSERT(0);
1928     h264_bitstream_write_uint(bitstream,
1929                               seq->seq_fields.bits.delta_pic_order_always_zero_flag,
1930                               1);
1931     h264_bitstream_write_se(bitstream, seq->offset_for_non_ref_pic);
1932     h264_bitstream_write_se(bitstream,
1933                             seq->offset_for_top_to_bottom_field);
1934     h264_bitstream_write_ue(bitstream,
1935                             seq->num_ref_frames_in_pic_order_cnt_cycle);
1936     for ( i = 0; i < seq->num_ref_frames_in_pic_order_cnt_cycle; i++) {
1937       h264_bitstream_write_se(bitstream, seq->offset_for_ref_frame[i]);
1938     }
1939   }
1940
1941   /* num_ref_frames */
1942   h264_bitstream_write_ue(bitstream, seq->max_num_ref_frames);
1943   /* gaps_in_frame_num_value_allowed_flag */
1944   h264_bitstream_write_uint(bitstream,
1945                             gaps_in_frame_num_value_allowed_flag,
1946                             1);
1947
1948   /* pic_width_in_mbs_minus1 */
1949   h264_bitstream_write_ue(bitstream, seq->picture_width_in_mbs - 1);
1950   /* pic_height_in_map_units_minus1 */
1951   h264_bitstream_write_ue(bitstream, pic_height_in_map_units - 1);
1952   /* frame_mbs_only_flag */
1953   h264_bitstream_write_uint(bitstream,
1954                             seq->seq_fields.bits.frame_mbs_only_flag,
1955                             1);
1956
1957   if (!seq->seq_fields.bits.frame_mbs_only_flag) { //ONLY mbs
1958       ENCODER_ASSERT(0);
1959       h264_bitstream_write_uint(bitstream, mb_adaptive_frame_field, 1);
1960   }
1961
1962   /* direct_8x8_inference_flag */
1963   h264_bitstream_write_uint(bitstream, 0, 1);
1964   /* frame_cropping_flag */
1965   h264_bitstream_write_uint(bitstream, seq->frame_cropping_flag, 1);
1966
1967   if (seq->frame_cropping_flag) {
1968       /* frame_crop_left_offset */
1969       h264_bitstream_write_ue(bitstream, seq->frame_crop_left_offset);
1970       /* frame_crop_right_offset */
1971       h264_bitstream_write_ue(bitstream, seq->frame_crop_right_offset);
1972       /* frame_crop_top_offset */
1973       h264_bitstream_write_ue(bitstream, seq->frame_crop_top_offset);
1974       /* frame_crop_bottom_offset */
1975       h264_bitstream_write_ue(bitstream, seq->frame_crop_bottom_offset);
1976   }
1977   ENCODER_ASSERT(seq->vui_parameters_present_flag == FALSE);
1978
1979   /* vui_parameters_present_flag */
1980   h264_bitstream_write_uint(bitstream, seq->vui_parameters_present_flag, 1);
1981   if (seq->vui_parameters_present_flag) {
1982     /*FIXME, to write vui parameters*/
1983   }
1984   /* rbsp_trailing_bits */
1985   h264_bitstream_write_trailing_bits(bitstream);
1986   return TRUE;
1987 }
1988
1989 static gboolean
1990 h264_bitstream_write_pps(
1991     H264Bitstream *bitstream,
1992     VAEncPictureParameterBufferH264 *pic
1993 )
1994 {
1995   guint32 num_slice_groups_minus1 = 0;
1996   guint32 pic_init_qs_minus26 = 0;
1997   guint32 redundant_pic_cnt_present_flag = 0;
1998
1999   /* pic_parameter_set_id */
2000   h264_bitstream_write_ue(bitstream, pic->pic_parameter_set_id);
2001   /* seq_parameter_set_id */
2002   h264_bitstream_write_ue(bitstream, pic->seq_parameter_set_id);
2003   /* entropy_coding_mode_flag */
2004   h264_bitstream_write_uint(bitstream,
2005                             pic->pic_fields.bits.entropy_coding_mode_flag,
2006                             1);
2007   /* pic_order_present_flag */
2008   h264_bitstream_write_uint(bitstream,
2009                             pic->pic_fields.bits.pic_order_present_flag,
2010                             1);
2011   /*slice_groups-1*/
2012   h264_bitstream_write_ue(bitstream, num_slice_groups_minus1);
2013
2014   if (num_slice_groups_minus1 > 0) {
2015     /*FIXME*/
2016     ENCODER_ASSERT(0);
2017   }
2018   h264_bitstream_write_ue(bitstream, pic->num_ref_idx_l0_active_minus1);
2019   h264_bitstream_write_ue(bitstream, pic->num_ref_idx_l1_active_minus1);
2020   h264_bitstream_write_uint(bitstream,
2021                             pic->pic_fields.bits.weighted_pred_flag,
2022                             1);
2023   h264_bitstream_write_uint(bitstream,
2024                             pic->pic_fields.bits.weighted_bipred_idc,
2025                             2);
2026   /* pic_init_qp_minus26 */
2027   h264_bitstream_write_se(bitstream, pic->pic_init_qp-26);
2028   /* pic_init_qs_minus26 */
2029   h264_bitstream_write_se(bitstream, pic_init_qs_minus26);
2030   /*chroma_qp_index_offset*/
2031   h264_bitstream_write_se(bitstream, pic->chroma_qp_index_offset);
2032
2033   h264_bitstream_write_uint(bitstream,
2034                             pic->pic_fields.bits.deblocking_filter_control_present_flag,
2035                             1);
2036   h264_bitstream_write_uint(bitstream,
2037                             pic->pic_fields.bits.constrained_intra_pred_flag,
2038                             1);
2039   h264_bitstream_write_uint(bitstream, redundant_pic_cnt_present_flag, 1);
2040
2041   /*more_rbsp_data*/
2042   h264_bitstream_write_uint(bitstream,
2043                             pic->pic_fields.bits.transform_8x8_mode_flag,
2044                             1);
2045   h264_bitstream_write_uint(bitstream,
2046                             pic->pic_fields.bits.pic_scaling_matrix_present_flag,
2047                             1);
2048   if (pic->pic_fields.bits.pic_scaling_matrix_present_flag) {
2049     ENCODER_ASSERT(0);
2050     /* FIXME */
2051     /*
2052     for (i = 0; i <
2053       (6+(-( (chroma_format_idc ! = 3) ? 2 : 6) * -pic->pic_fields.bits.transform_8x8_mode_flag));
2054       i++) {
2055       h264_bitstream_write_uint(bitstream, pic->pic_fields.bits.pic_scaling_list_present_flag, 1);
2056     }
2057     */
2058   }
2059
2060   h264_bitstream_write_se(bitstream, pic->second_chroma_qp_index_offset);
2061   h264_bitstream_write_trailing_bits(bitstream);
2062   return TRUE;
2063 }
2064 #endif