encoder: fix issue of encoded data truncated
[vaapi:windyuan-gstreamer-vaapi.git] / gst-libs / gst / vaapi / gstvaapiencoder.c
1 /*
2  *  gstvaapiencoder.c - VA-API encoder interface
3  *
4  *  Copyright (C) 2013 Intel Corporation
5  *
6  *  This library is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU Lesser General Public License
8  *  as published by the Free Software Foundation; either version 2.1
9  *  of the License, or (at your option) any later version.
10  *
11  *  This library is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  *  Lesser General Public License for more details.
15  *
16  *  You should have received a copy of the GNU Lesser General Public
17  *  License along with this library; if not, write to the Free
18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19  *  Boston, MA 02110-1301 USA
20  */
21 #include "sysdeps.h"
22 #include "gstvaapicompat.h"
23 #include "gstvaapiencoder.h"
24 #include "gstvaapiencoder_priv.h"
25 #include "gstvaapicontext.h"
26 #include "gstvaapidisplay_priv.h"
27
28 #define DEBUG 1
29 #include "gstvaapidebug.h"
30
31 typedef struct _GstVaapiCodedBufferProxyClass GstVaapiCodedBufferProxyClass;
32
33 #define GST_VAAPI_ENCODER_LOCK(encoder)                        \
34     G_STMT_START {                                             \
35       g_mutex_lock(&(GST_VAAPI_ENCODER_CAST(encoder))->lock); \
36     } G_STMT_END
37
38 #define GST_VAAPI_ENCODER_UNLOCK(encoder)                        \
39     G_STMT_START {                                               \
40       g_mutex_unlock(&(GST_VAAPI_ENCODER_CAST(encoder))->lock); \
41     } G_STMT_END
42
43 #define GST_VAAPI_ENCODER_BUF_FREE_WAIT(encoder)                    \
44     G_STMT_START {                                                  \
45         g_cond_wait(&(GST_VAAPI_ENCODER_CAST(encoder))->codedbuf_free,  \
46                     &(GST_VAAPI_ENCODER_CAST(encoder))->lock);     \
47     } G_STMT_END
48
49 #define GST_VAAPI_ENCODER_BUF_FREE_SIGNAL(encoder)                    \
50     G_STMT_START {                                                    \
51         g_cond_signal(&(GST_VAAPI_ENCODER_CAST(encoder))->codedbuf_free); \
52     } G_STMT_END
53
54 #define GST_VAAPI_ENCODER_FREE_SURFACE_WAIT(encoder)                \
55     G_STMT_START {                                                  \
56         g_cond_wait(&(GST_VAAPI_ENCODER_CAST(encoder))->surface_free,  \
57                     &(GST_VAAPI_ENCODER_CAST(encoder))->lock);     \
58     } G_STMT_END
59
60 #define GST_VAAPI_ENCODER_FREE_SURFACE_SIGNAL(encoder)                \
61     G_STMT_START {                                                    \
62         g_cond_signal(&(GST_VAAPI_ENCODER_CAST(encoder))->surface_free); \
63     } G_STMT_END
64
65 #define GST_VAAPI_ENCODER_SYNC_SIGNAL(encoder)                              \
66         G_STMT_START {                                                      \
67             g_cond_signal(&(GST_VAAPI_ENCODER_CAST(encoder))->sync_ready); \
68         } G_STMT_END
69
70 static inline gboolean
71 GST_VAAPI_ENCODER_SYNC_WAIT_TIMEOUT(GstVaapiEncoder *encoder, gint64 timeout)
72 {
73     gint64 end_time = g_get_monotonic_time() + timeout;
74     return g_cond_wait_until(&encoder->sync_ready,
75             &encoder->lock, end_time);
76 }
77
78 static GstVaapiCodedBuffer *
79 gst_vaapi_encoder_dequeue_coded_buffer(GstVaapiEncoder *encoder);
80
81 static void
82 gst_vaapi_encoder_queue_coded_buffer(
83     GstVaapiEncoder *encoder,
84     GstVaapiCodedBuffer *buf
85 );
86
87 typedef struct {
88     GstVaapiEncPicture       *picture;
89     GstVaapiCodedBufferProxy *buf;
90 } GstVaapiEncoderSyncPic;
91
92 static void
93 gst_vaapi_coded_buffer_proxy_finalize(GstVaapiCodedBufferProxy *proxy)
94 {
95     if (proxy->buffer) {
96         gst_vaapi_coded_buffer_unmap(proxy->buffer);
97         if (proxy->encoder)
98             gst_vaapi_encoder_queue_coded_buffer(proxy->encoder, proxy->buffer);
99         else {
100             g_assert(FALSE);
101             gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(proxy->buffer));
102         }
103         proxy->buffer = NULL;
104     }
105     gst_vaapi_encoder_replace(&proxy->encoder, NULL);
106 }
107
108 static void
109 gst_vaapi_coded_buffer_proxy_class_init(GstVaapiCodedBufferProxyClass *klass)
110 {
111     GstVaapiMiniObjectClass * const object_class =
112         GST_VAAPI_MINI_OBJECT_CLASS(klass);
113
114     object_class->size = sizeof(GstVaapiCodedBufferProxy);
115     object_class->finalize =
116         (GDestroyNotify)gst_vaapi_coded_buffer_proxy_finalize;
117 }
118
119 static inline const GstVaapiCodedBufferProxyClass *
120 gst_vaapi_coded_buffer_proxy_class(void)
121 {
122     static GstVaapiCodedBufferProxyClass g_class;
123     static gsize g_class_init = FALSE;
124
125     if (g_once_init_enter(&g_class_init)) {
126         gst_vaapi_coded_buffer_proxy_class_init(&g_class);
127         g_once_init_leave(&g_class_init, TRUE);
128     }
129     return (&g_class);
130 }
131
132 GstVaapiCodedBufferProxy *
133 gst_vaapi_coded_buffer_proxy_new(GstVaapiEncoder *encoder)
134 {
135     GstVaapiCodedBuffer *buf;
136     GstVaapiCodedBufferProxy *ret;
137
138     g_assert(encoder);
139     buf = gst_vaapi_encoder_dequeue_coded_buffer(encoder);
140     if (!buf)
141         return NULL;
142
143     ret = (GstVaapiCodedBufferProxy*)gst_vaapi_mini_object_new0(
144              GST_VAAPI_MINI_OBJECT_CLASS(gst_vaapi_coded_buffer_proxy_class()));
145     g_assert(ret);
146     ret->encoder = gst_vaapi_encoder_ref(encoder);
147     ret->buffer = buf;
148     return ret;
149 }
150
151 GstVaapiCodedBufferProxy *
152 gst_vaapi_coded_buffer_proxy_ref(GstVaapiCodedBufferProxy *proxy)
153 {
154     return (GstVaapiCodedBufferProxy *)(gst_vaapi_mini_object_ref(
155                                         GST_VAAPI_MINI_OBJECT(proxy)));
156 }
157
158 void
159 gst_vaapi_coded_buffer_proxy_unref(GstVaapiCodedBufferProxy *proxy)
160 {
161     gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(proxy));
162 }
163
164 void
165 gst_vaapi_coded_buffer_proxy_replace(
166     GstVaapiCodedBufferProxy **old_proxy_ptr,
167     GstVaapiCodedBufferProxy *new_proxy
168 )
169 {
170     gst_vaapi_mini_object_replace((GstVaapiMiniObject**)old_proxy_ptr,
171         GST_VAAPI_MINI_OBJECT(new_proxy));
172 }
173
174 GstVaapiEncoder *
175 gst_vaapi_encoder_ref(GstVaapiEncoder *encoder)
176 {
177     return gst_vaapi_object_ref(encoder);
178 }
179
180 void
181 gst_vaapi_encoder_unref(GstVaapiEncoder *encoder)
182 {
183     gst_vaapi_object_unref(encoder);
184 }
185
186 void
187 gst_vaapi_encoder_replace(
188     GstVaapiEncoder **old_encoder_ptr,
189     GstVaapiEncoder *new_encoder
190 )
191 {
192     gst_vaapi_object_replace(old_encoder_ptr, new_encoder);
193 }
194
195 static gboolean
196 gst_vaapi_encoder_init_coded_buffer_queue(
197     GstVaapiEncoder *encoder,
198     guint count)
199 {
200     GstVaapiCodedBuffer *buf;
201     guint i = 0;
202
203     GST_VAAPI_ENCODER_LOCK(encoder);
204     if (count > encoder->max_buf_num)
205         count = encoder->max_buf_num;
206
207     g_assert(encoder->buf_size);
208     for (i = 0; i < count; ++i) {
209         buf = GST_VAAPI_CODED_BUFFER_NEW(encoder, encoder->buf_size);
210         g_queue_push_tail(&encoder->coded_buffers, buf);
211         ++encoder->buf_count;
212     }
213     g_assert(encoder->buf_count <= encoder->max_buf_num);
214
215     GST_VAAPI_ENCODER_UNLOCK(encoder);
216     return TRUE;
217 }
218
219 static GstVaapiCodedBuffer *
220 gst_vaapi_encoder_dequeue_coded_buffer(GstVaapiEncoder *encoder)
221 {
222     GstVaapiCodedBuffer *ret = NULL;
223
224     GST_VAAPI_ENCODER_LOCK(encoder);
225     while(encoder->buf_count >= encoder->max_buf_num &&
226           g_queue_is_empty(&encoder->coded_buffers)) {
227         GST_VAAPI_ENCODER_BUF_FREE_WAIT(encoder);
228     }
229     if (!g_queue_is_empty(&encoder->coded_buffers)) {
230         ret = (GstVaapiCodedBuffer*)g_queue_pop_head(&encoder->coded_buffers);
231         goto end;
232     }
233
234     g_assert(encoder->buf_size);
235     ret = GST_VAAPI_CODED_BUFFER_NEW(encoder, encoder->buf_size);
236     if (ret)
237         ++encoder->buf_count;
238
239 end:
240     GST_VAAPI_ENCODER_UNLOCK(encoder);
241     return ret;
242 }
243
244 static void
245 gst_vaapi_encoder_queue_coded_buffer(
246     GstVaapiEncoder *encoder,
247     GstVaapiCodedBuffer *buf
248 )
249 {
250     g_assert(buf);
251     g_return_if_fail(buf);
252
253     GST_VAAPI_ENCODER_LOCK(encoder);
254     g_queue_push_tail(&encoder->coded_buffers, buf);
255     GST_VAAPI_ENCODER_BUF_FREE_SIGNAL(encoder);
256     GST_VAAPI_ENCODER_UNLOCK(encoder);
257 }
258
259 static gboolean
260 gst_vaapi_encoder_free_coded_buffers(GstVaapiEncoder *encoder)
261 {
262     GstVaapiCodedBuffer *buf;
263     guint count = 0;
264     gboolean ret;
265
266     GST_VAAPI_ENCODER_LOCK(encoder);
267     while(!g_queue_is_empty(&encoder->coded_buffers)) {
268         buf = (GstVaapiCodedBuffer *)g_queue_pop_head(&encoder->coded_buffers);
269         g_assert(buf);
270         gst_vaapi_mini_object_unref(GST_VAAPI_MINI_OBJECT(buf));
271         ++count;
272     }
273     ret = (count == encoder->buf_count);
274     GST_VAAPI_ENCODER_UNLOCK(encoder);
275
276     if (!ret) {
277         GST_ERROR("coded buffer leak, freed count:%d, total buf:%d",
278                   count, encoder->buf_count);
279     }
280
281     return ret;
282 }
283
284 static void
285 _surface_proxy_released_notify(GstVaapiEncoder *encoder)
286 {
287     GST_VAAPI_ENCODER_FREE_SURFACE_SIGNAL(encoder);
288 }
289
290 GstVaapiSurfaceProxy*
291 gst_vaapi_encoder_create_surface(GstVaapiEncoder *encoder)
292 {
293     GstVaapiSurfaceProxy *proxy;
294
295     g_assert(encoder && encoder->context);
296     g_return_val_if_fail(encoder->context, NULL);
297
298     GST_VAAPI_ENCODER_LOCK(encoder);
299     while(!gst_vaapi_context_get_surface_count(encoder->context)) {
300         GST_VAAPI_ENCODER_FREE_SURFACE_WAIT(encoder);
301     }
302     proxy = gst_vaapi_context_get_surface_proxy(encoder->context);
303     GST_VAAPI_ENCODER_UNLOCK(encoder);
304
305     gst_vaapi_surface_proxy_set_destroy_notify(proxy,
306             (GDestroyNotify)_surface_proxy_released_notify, encoder);
307
308     return proxy;
309 }
310
311 void
312 gst_vaapi_encoder_release_surface(
313     GstVaapiEncoder *encoder,
314     GstVaapiSurfaceProxy *surface
315 )
316 {
317     GST_VAAPI_ENCODER_LOCK(encoder);
318     gst_vaapi_surface_proxy_unref(surface);
319     GST_VAAPI_ENCODER_UNLOCK(encoder);
320 }
321
322 static GstVaapiEncoderSyncPic *
323 _create_sync_picture(
324     GstVaapiEncPicture *picture,
325     GstVaapiCodedBufferProxy *coded_buf
326 )
327 {
328     GstVaapiEncoderSyncPic *sync = g_slice_new0(GstVaapiEncoderSyncPic);
329
330     g_assert(picture && coded_buf);
331     sync->picture = gst_vaapi_enc_picture_ref(picture);
332     sync->buf = gst_vaapi_coded_buffer_proxy_ref(coded_buf);
333     return sync;
334 }
335 static void
336 _free_sync_picture(
337     GstVaapiEncoder *encoder,
338     GstVaapiEncoderSyncPic *sync_pic
339 )
340 {
341     g_assert(sync_pic);
342
343     if (sync_pic->picture)
344         gst_vaapi_enc_picture_unref(sync_pic->picture);
345     if (sync_pic->buf)
346         gst_vaapi_coded_buffer_proxy_unref(sync_pic->buf);
347     g_slice_free(GstVaapiEncoderSyncPic, sync_pic);
348 }
349
350 static void
351 gst_vaapi_encoder_free_sync_pictures(GstVaapiEncoder *encoder)
352 {
353     GstVaapiEncoderSyncPic* sync;
354
355     GST_VAAPI_ENCODER_LOCK(encoder);
356     while(!g_queue_is_empty(&encoder->sync_pictures)) {
357         sync = (GstVaapiEncoderSyncPic*)g_queue_pop_head(&encoder->sync_pictures);
358         _free_sync_picture(encoder, sync);
359     }
360     GST_VAAPI_ENCODER_UNLOCK(encoder);
361 }
362
363 static gboolean
364 gst_vaapi_encoder_push_sync_picture(
365     GstVaapiEncoder *encoder,
366     GstVaapiEncoderSyncPic *sync_pic
367 )
368 {
369     GST_VAAPI_ENCODER_LOCK(encoder);
370     g_queue_push_tail(&encoder->sync_pictures, sync_pic);
371     GST_VAAPI_ENCODER_SYNC_SIGNAL(encoder);
372     GST_VAAPI_ENCODER_UNLOCK(encoder);
373     return TRUE;
374 }
375
376 static GstVaapiEncoderStatus
377 gst_vaapi_encoder_pop_sync_picture(
378     GstVaapiEncoder *encoder,
379     GstVaapiEncoderSyncPic **sync_pic,
380     guint64 timeout
381 )
382 {
383     GstVaapiEncoderStatus ret = GST_VAAPI_ENCODER_STATUS_SUCCESS;
384
385     *sync_pic = NULL;
386
387     GST_VAAPI_ENCODER_LOCK(encoder);
388     if(g_queue_is_empty(&encoder->sync_pictures) &&
389         !GST_VAAPI_ENCODER_SYNC_WAIT_TIMEOUT(encoder, timeout))
390         goto timeout;
391
392     if (g_queue_is_empty(&encoder->sync_pictures)) {
393         ret = GST_VAAPI_ENCODER_STATUS_UNKNOWN_ERR;
394         goto end;
395     }
396
397     *sync_pic = (GstVaapiEncoderSyncPic*)g_queue_pop_head(
398                                           &encoder->sync_pictures);
399     g_assert(*sync_pic);
400     ret = GST_VAAPI_ENCODER_STATUS_SUCCESS;
401     goto end;
402
403 timeout:
404     ret = GST_VAAPI_ENCODER_STATUS_TIMEOUT;
405
406 end:
407     GST_VAAPI_ENCODER_UNLOCK(encoder);
408     return ret;
409 }
410
411
412 GstVaapiEncoderStatus
413 gst_vaapi_encoder_encode(
414     GstVaapiEncoder *encoder,
415     GstVideoCodecFrame *frame
416 )
417 {
418     GstVaapiEncoderStatus ret = GST_VAAPI_ENCODER_STATUS_SUCCESS;
419     GstVaapiEncoderClass *klass =
420         GST_VAAPI_ENCODER_GET_CLASS(encoder);
421     GstVaapiEncPicture *picture = NULL;
422     GstVaapiCodedBufferProxy *coded_buf = NULL;
423     GstVaapiEncoderSyncPic *sync_pic = NULL;
424
425     if (!klass->reordering || !klass->encode)
426         goto error;
427
428 again:
429     picture = NULL;
430     sync_pic = NULL;
431     ret = klass->reordering(encoder, frame, FALSE, &picture);
432
433     if (ret == GST_VAAPI_ENCODER_STATUS_FRAME_NOT_READY)
434         return GST_VAAPI_ENCODER_STATUS_SUCCESS;
435
436     g_assert(picture);
437     if (ret != GST_VAAPI_ENCODER_STATUS_SUCCESS)
438         goto error;
439     if (!picture) {
440         ret = GST_VAAPI_ENCODER_STATUS_PICTURE_ERR;
441         goto error;
442     }
443
444     coded_buf = gst_vaapi_coded_buffer_proxy_new(encoder);
445     if (!coded_buf) {
446         ret = GST_VAAPI_ENCODER_STATUS_OBJECT_ERR;
447         goto error;
448     }
449
450     ret = klass->encode(encoder, picture, coded_buf);
451     if (ret != GST_VAAPI_ENCODER_STATUS_SUCCESS)
452         goto error;
453
454     /* another thread would sync and get coded buffer */
455     sync_pic = _create_sync_picture(picture, coded_buf);
456     gst_vaapi_coded_buffer_proxy_replace(&coded_buf, NULL);
457     gst_vaapi_enc_picture_replace(&picture, NULL);
458
459     if (!gst_vaapi_encoder_push_sync_picture(encoder, sync_pic)) {
460         ret = GST_VAAPI_ENCODER_STATUS_THREAD_ERR;
461         goto error;
462     }
463
464     frame = NULL;
465     goto again;
466
467 error:
468     gst_vaapi_enc_picture_replace(&picture, NULL);
469     gst_vaapi_coded_buffer_proxy_replace(&coded_buf, NULL);
470     if (sync_pic)
471         _free_sync_picture(encoder, sync_pic);
472     GST_ERROR("encoding failed, error:%d", ret);
473     return ret;
474 }
475
476 GstVaapiEncoderStatus
477 gst_vaapi_encoder_get_frame(
478     GstVaapiEncoder *encoder,
479     GstVideoCodecFrame **frame,
480     GstVaapiCodedBufferProxy **codedbuf,
481     gint64 us_of_timeout
482 )
483 {
484     GstVaapiEncoderStatus ret = GST_VAAPI_ENCODER_STATUS_SUCCESS;
485     GstVaapiEncoderSyncPic *sync_pic = NULL;
486     GstVaapiSurfaceStatus surface_status;
487     GstVaapiEncPicture *picture;
488
489     ret = gst_vaapi_encoder_pop_sync_picture(encoder, &sync_pic, us_of_timeout);
490     if (ret != GST_VAAPI_ENCODER_STATUS_SUCCESS)
491         goto end;
492
493     picture = sync_pic->picture;
494
495     if (!picture->surface || !gst_vaapi_surface_sync(picture->surface)) {
496         ret = GST_VAAPI_ENCODER_STATUS_PARAM_ERR;
497         goto end;
498     }
499     if (!gst_vaapi_surface_query_status(picture->surface, &surface_status)) {
500         ret = GST_VAAPI_ENCODER_STATUS_PICTURE_ERR;
501         goto end;
502     }
503     if (frame)
504         *frame = gst_video_codec_frame_ref(picture->frame);
505     if (codedbuf)
506         *codedbuf = gst_vaapi_coded_buffer_proxy_ref(sync_pic->buf);
507
508 end:
509     if (sync_pic)
510         _free_sync_picture(encoder, sync_pic);
511     return ret;
512 }
513
514 GstVaapiEncoderStatus
515 gst_vaapi_encoder_convert_frame(
516     GstVaapiEncoder *encoder,
517     GstVideoCodecFrame *frame
518 )
519 {
520     GstVaapiEncoderClass * const klass =
521         GST_VAAPI_ENCODER_GET_CLASS(encoder);
522
523     if (!klass->convert_buf)
524         return GST_VAAPI_ENCODER_STATUS_SUCCESS;
525
526     return klass->convert_buf(encoder, frame);
527 }
528
529 GstVaapiEncoderStatus
530 gst_vaapi_encoder_flush(
531     GstVaapiEncoder *encoder
532 )
533 {
534     GstVaapiEncoderStatus ret = GST_VAAPI_ENCODER_STATUS_SUCCESS;
535     GstVaapiEncoderClass * const klass =
536         GST_VAAPI_ENCODER_GET_CLASS(encoder);
537
538     if (!klass->flush)
539         goto error;
540
541     ret = klass->flush(encoder);
542     return ret;
543
544 error:
545     GST_ERROR("flush failed");
546     return GST_VAAPI_ENCODER_STATUS_FUNC_PTR_ERR;
547 }
548
549 GstVaapiEncoderStatus
550 gst_vaapi_encoder_get_codec_data(
551     GstVaapiEncoder *encoder,
552     GstBuffer **codec_data
553 )
554 {
555     GstVaapiEncoderStatus ret = GST_VAAPI_ENCODER_STATUS_SUCCESS;
556     GstVaapiEncoderClass * const klass =
557         GST_VAAPI_ENCODER_GET_CLASS(encoder);
558
559     *codec_data = NULL;
560     if (!klass->get_codec_data)
561         return GST_VAAPI_ENCODER_STATUS_SUCCESS;
562
563     ret = klass->get_codec_data(encoder, codec_data);
564     return ret;
565 }
566
567 static gboolean
568 gst_vaapi_encoder_ensure_context(GstVaapiEncoder *encoder)
569 {
570     GstVaapiEncoderClass * const klass =
571         GST_VAAPI_ENCODER_GET_CLASS(encoder);
572     GstVaapiContextInfo info;
573     GstVaapiContext *context;
574
575     if (GST_VAAPI_ENCODER_CONTEXT(encoder))
576         return TRUE;
577
578     memset(&info, 0, sizeof(info));
579     if (!klass->get_context_info ||
580         !klass->get_context_info(encoder, &info)) {
581         return FALSE;
582     }
583
584     context = gst_vaapi_context_new_full(
585                   GST_VAAPI_ENCODER_DISPLAY(encoder),
586                   &info);
587     if (!context)
588         return FALSE;
589
590     GST_VAAPI_ENCODER_CONTEXT(encoder) = context;
591     GST_VAAPI_ENCODER_VA_CONTEXT(encoder) = gst_vaapi_context_get_id(context);
592     return TRUE;
593 }
594
595 gboolean
596 gst_vaapi_encoder_reset_display(
597     GstVaapiEncoder *encoder,
598     GstVaapiDisplay *display
599 )
600 {
601     g_assert(display);
602     g_assert(GST_VAAPI_ENCODER_CONTEXT(encoder) == NULL);
603     g_return_val_if_fail(GST_VAAPI_ENCODER_CONTEXT(encoder) == NULL, FALSE);
604
605     gst_vaapi_display_replace(&encoder->display, display);
606     if (display)
607         encoder->va_display = gst_vaapi_display_get_display(display);
608
609     return TRUE;
610 }
611
612 GstCaps *
613 gst_vaapi_encoder_set_format(
614     GstVaapiEncoder *encoder,
615     GstVideoCodecState *in_state,
616     GstCaps *ref_caps
617 )
618 {
619     GstVaapiEncoderClass * const klass =
620         GST_VAAPI_ENCODER_GET_CLASS(encoder);
621     GstCaps *out_caps = NULL;
622
623     if (!GST_VIDEO_INFO_WIDTH(&in_state->info) ||
624         !GST_VIDEO_INFO_HEIGHT(&in_state->info)) {
625         GST_WARNING("encoder set format failed, width or height equal to 0.");
626         return NULL;
627     }
628     GST_VAAPI_ENCODER_VIDEO_INFO(encoder) = in_state->info;
629
630     if (!klass->set_format)
631         goto error;
632
633     out_caps = klass->set_format(encoder, in_state, ref_caps);
634     if (!out_caps)
635         goto error;
636
637     if (GST_VAAPI_ENCODER_CAPS(encoder) &&
638         gst_caps_is_equal(out_caps,GST_VAAPI_ENCODER_CAPS(encoder))) {
639         gst_caps_unref(out_caps);
640         return GST_VAAPI_ENCODER_CAPS(encoder);
641     }
642     gst_caps_replace(&GST_VAAPI_ENCODER_CAPS(encoder), out_caps);
643     g_assert(GST_VAAPI_ENCODER_CONTEXT(encoder) == NULL);
644     gst_vaapi_object_replace(&GST_VAAPI_ENCODER_CONTEXT(encoder), NULL);
645
646     if (!gst_vaapi_encoder_ensure_context(encoder))
647         goto error;
648
649     encoder->buf_size = (GST_VAAPI_ENCODER_WIDTH(encoder) *
650                          GST_VAAPI_ENCODER_HEIGHT(encoder) * 400) /(16*16);
651
652     if (!gst_vaapi_encoder_init_coded_buffer_queue(encoder, 5)) {
653         GST_ERROR("encoder init coded buffer failed");
654         goto error;
655     }
656
657     return out_caps;
658
659 error:
660     gst_caps_replace(&GST_VAAPI_ENCODER_CAPS(encoder), NULL);
661     gst_caps_replace(&out_caps, NULL);
662     GST_ERROR("encoder set format failed");
663     return NULL;
664 }
665
666 char *
667 gst_vaapi_encoder_dump_bytes(const guint8 *buf, guint32 num)
668 {
669   static char tmp[1024];
670   guint32 i = 0;
671   memset(tmp, 0, sizeof(tmp));
672
673   char *p = tmp;
674   for (i = 0; i < num; i++) {
675     snprintf(p, 1024-(p-tmp), "%02x", (guint8)buf[i]);
676     p += strlen(p);
677   }
678   return tmp;
679 }
680
681 static gboolean
682 gst_vaapi_encoder_init(GstVaapiEncoder *encoder, GstVaapiDisplay *display)
683 {
684     GstVaapiEncoderClass *kclass = GST_VAAPI_ENCODER_GET_CLASS(encoder);
685
686     g_assert(kclass);
687
688     if (display)
689         encoder->display = gst_vaapi_display_ref(display);
690     else
691         encoder->display = NULL;
692     encoder->context = NULL;
693     encoder->caps = NULL;
694
695     if (display)
696         encoder->va_display = gst_vaapi_display_get_display(display);
697     else
698         encoder->va_display = NULL;
699     encoder->va_context = VA_INVALID_ID;
700     //priv->rate_control = ;
701
702     gst_video_info_init(&encoder->video_info);
703
704     encoder->buf_count = 0;
705     encoder->max_buf_num = 10;
706     encoder->buf_size = 0;
707
708     g_mutex_init(&encoder->lock);
709     g_cond_init(&encoder->codedbuf_free);
710     g_cond_init(&encoder->surface_free);
711     g_queue_init(&encoder->coded_buffers);
712     g_queue_init(&encoder->sync_pictures);
713     g_cond_init(&encoder->sync_ready);
714
715     if (kclass->init)
716         return kclass->init(encoder);
717     return TRUE;
718 }
719
720 static void
721 gst_vaapi_encoder_destroy(GstVaapiEncoder *encoder)
722 {
723     GstVaapiEncoderClass * const klass =
724         GST_VAAPI_ENCODER_GET_CLASS(encoder);
725
726     if (klass->destroy)
727         klass->destroy(encoder);
728
729     gst_vaapi_encoder_free_coded_buffers(encoder);
730     gst_vaapi_encoder_free_sync_pictures(encoder);
731
732     gst_vaapi_object_replace(&encoder->context, NULL);
733     gst_vaapi_display_replace(&encoder->display, NULL);
734     encoder->va_display = NULL;
735     g_mutex_clear(&encoder->lock);
736     g_cond_clear(&encoder->codedbuf_free);
737     g_cond_clear(&encoder->surface_free);
738     g_queue_clear(&encoder->coded_buffers);
739     g_queue_clear(&encoder->sync_pictures);
740     g_cond_clear(&encoder->sync_ready);
741 }
742
743 void
744 gst_vaapi_encoder_finalize(GstVaapiEncoder *encoder)
745 {
746   gst_vaapi_encoder_destroy(encoder);
747 }
748
749 void
750 gst_vaapi_encoder_class_init(GstVaapiEncoderClass *klass)
751 {
752     GstVaapiMiniObjectClass * const object_class =
753         GST_VAAPI_MINI_OBJECT_CLASS(klass);
754
755     object_class->size = sizeof (GstVaapiEncoder);
756     object_class->finalize = (GDestroyNotify)gst_vaapi_encoder_finalize;
757 }
758
759 GstVaapiEncoder *
760 gst_vaapi_encoder_new(
761     const GstVaapiEncoderClass *klass,
762     GstVaapiDisplay *display
763 )
764 {
765     GstVaapiEncoder *encoder;
766
767     encoder = (GstVaapiEncoder *)
768         gst_vaapi_mini_object_new0(GST_VAAPI_MINI_OBJECT_CLASS(klass));
769     if (!encoder)
770         return NULL;
771
772     if (!gst_vaapi_encoder_init(encoder, display))
773         goto error;
774     return encoder;
775
776 error:
777     gst_vaapi_encoder_unref(encoder);
778     return NULL;
779 }