Merge branch '0.10_basevideodec' into 1.0
[vaapi:sree-gstreamer-vaapi.git] / gst / vaapi / gstvaapidownload.c
1 /*
2  *  gstvaapidownload.c - VA-API video downloader
3  *
4  *  Copyright (C) 2010-2011 Splitted-Desktop Systems
5  *  Copyright (C) 2011-2012 Intel Corporation
6  *
7  *  This library is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU Lesser General Public License
9  *  as published by the Free Software Foundation; either version 2.1
10  *  of the License, or (at your option) any later version.
11  *
12  *  This library is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  *  Lesser General Public License for more details.
16  *
17  *  You should have received a copy of the GNU Lesser General Public
18  *  License along with this library; if not, write to the Free
19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20  *  Boston, MA 02110-1301 USA
21  */
22
23 /**
24  * SECTION:gstvaapidownload
25  * @short_description: A VA to video flow filter
26  *
27  * vaapidownload converts from VA surfaces to raw YUV pixels.
28  */
29
30 #include "config.h"
31
32 #include <gst/gst.h>
33 #include <gst/video/video.h>
34 #include <gst/video/videocontext.h>
35 #include <gst/vaapi/gstvaapivideobuffer.h>
36
37 #include "gstvaapidownload.h"
38 #include "gstvaapipluginutil.h"
39 #include "gstvaapipluginbuffer.h"
40
41 #define GST_PLUGIN_NAME "vaapidownload"
42 #define GST_PLUGIN_DESC "A VA to video flow filter"
43
44 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapidownload);
45 #define GST_CAT_DEFAULT gst_debug_vaapidownload
46
47 /* ElementFactory information */
48 static const GstElementDetails gst_vaapidownload_details =
49     GST_ELEMENT_DETAILS(
50         "VA-API colorspace converter",
51         "Filter/Converter/Video",
52         GST_PLUGIN_DESC,
53         "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
54
55 /* Default templates */
56 static const char gst_vaapidownload_yuv_caps_str[] =
57     "video/x-raw-yuv, "
58     "width  = (int) [ 1, MAX ], "
59     "height = (int) [ 1, MAX ]; ";
60
61 static const char gst_vaapidownload_vaapi_caps_str[] =
62     GST_VAAPI_SURFACE_CAPS;
63
64 static GstStaticPadTemplate gst_vaapidownload_sink_factory =
65     GST_STATIC_PAD_TEMPLATE(
66         "sink",
67         GST_PAD_SINK,
68         GST_PAD_ALWAYS,
69         GST_STATIC_CAPS(gst_vaapidownload_vaapi_caps_str));
70
71 static GstStaticPadTemplate gst_vaapidownload_src_factory =
72     GST_STATIC_PAD_TEMPLATE(
73         "src",
74         GST_PAD_SRC,
75         GST_PAD_ALWAYS,
76         GST_STATIC_CAPS(gst_vaapidownload_yuv_caps_str));
77
78 typedef struct _TransformSizeCache TransformSizeCache;
79 struct _TransformSizeCache {
80     GstCaps            *caps;
81     guint               size;
82 };
83
84 struct _GstVaapiDownload {
85     /*< private >*/
86     GstBaseTransform    parent_instance;
87
88     GstVaapiDisplay    *display;
89     GstCaps            *allowed_caps;
90     TransformSizeCache  transform_size_cache[2];
91     GstVaapiVideoPool  *images;
92     GstVaapiImageFormat image_format;
93     guint               image_width;
94     guint               image_height;
95     unsigned int        images_reset    : 1;
96 };
97
98 struct _GstVaapiDownloadClass {
99     /*< private >*/
100     GstBaseTransformClass parent_class;
101 };
102
103 static void
104 gst_vaapidownload_implements_iface_init(GstImplementsInterfaceClass *iface);
105
106 static void
107 gst_video_context_interface_init(GstVideoContextInterface *iface);
108
109 #define GstVideoContextClass GstVideoContextInterface
110 G_DEFINE_TYPE_WITH_CODE(
111     GstVaapiDownload,
112     gst_vaapidownload,
113     GST_TYPE_BASE_TRANSFORM,
114     G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
115                           gst_vaapidownload_implements_iface_init);
116     G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
117                           gst_video_context_interface_init))
118
119 static gboolean
120 gst_vaapidownload_start(GstBaseTransform *trans);
121
122 static gboolean
123 gst_vaapidownload_stop(GstBaseTransform *trans);
124
125 static void
126 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer);
127
128 static GstFlowReturn
129 gst_vaapidownload_transform(
130     GstBaseTransform *trans,
131     GstBuffer        *inbuf,
132     GstBuffer        *outbuf
133 );
134
135 static GstCaps *
136 gst_vaapidownload_transform_caps(
137     GstBaseTransform *trans,
138     GstPadDirection   direction,
139     GstCaps          *caps
140 );
141
142 static gboolean
143 gst_vaapidownload_transform_size(
144     GstBaseTransform *trans,
145     GstPadDirection   direction,
146     GstCaps          *caps,
147     guint             size,
148     GstCaps          *othercaps,
149     guint            *othersize
150 );
151
152 static gboolean
153 gst_vaapidownload_set_caps(
154     GstBaseTransform *trans,
155     GstCaps          *incaps,
156     GstCaps          *outcaps
157 );
158
159 static gboolean
160 gst_vaapidownload_query(
161     GstPad   *pad,
162     GstQuery *query
163 );
164
165 /* GstImplementsInterface interface */
166
167 static gboolean
168 gst_vaapidownload_implements_interface_supported(
169     GstImplementsInterface *iface,
170     GType                   type
171 )
172 {
173     return (type == GST_TYPE_VIDEO_CONTEXT);
174 }
175
176 static void
177 gst_vaapidownload_implements_iface_init(GstImplementsInterfaceClass *iface)
178 {
179     iface->supported = gst_vaapidownload_implements_interface_supported;
180 }
181
182 /* GstVideoContext interface */
183
184 static void
185 gst_vaapidownload_set_video_context(GstVideoContext *context, const gchar *type,
186     const GValue *value)
187 {
188   GstVaapiDownload *download = GST_VAAPIDOWNLOAD (context);
189   gst_vaapi_set_display (type, value, &download->display);
190 }
191
192 static void
193 gst_video_context_interface_init(GstVideoContextInterface *iface)
194 {
195     iface->set_context = gst_vaapidownload_set_video_context;
196 }
197
198 static void
199 gst_vaapidownload_destroy(GstVaapiDownload *download)
200 {
201     guint i;
202
203     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
204         TransformSizeCache * const tsc = &download->transform_size_cache[i];
205         if (tsc->caps) {
206             gst_caps_unref(tsc->caps);
207             tsc->caps = NULL;
208             tsc->size = 0;
209         }
210     }
211
212     if (download->allowed_caps) {
213         gst_caps_unref(download->allowed_caps);
214         download->allowed_caps = NULL;
215     }
216
217     g_clear_object(&download->images);
218     g_clear_object(&download->display);
219 }
220
221 static void
222 gst_vaapidownload_finalize(GObject *object)
223 {
224     gst_vaapidownload_destroy(GST_VAAPIDOWNLOAD(object));
225
226     G_OBJECT_CLASS(gst_vaapidownload_parent_class)->finalize(object);
227 }
228
229 static void
230 gst_vaapidownload_class_init(GstVaapiDownloadClass *klass)
231 {
232     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
233     GstBaseTransformClass * const trans_class = GST_BASE_TRANSFORM_CLASS(klass);
234     GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
235     GstPadTemplate *pad_template;
236
237     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapidownload,
238                             GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
239
240     object_class->finalize        = gst_vaapidownload_finalize;
241     trans_class->start            = gst_vaapidownload_start;
242     trans_class->stop             = gst_vaapidownload_stop;
243     trans_class->before_transform = gst_vaapidownload_before_transform;
244     trans_class->transform        = gst_vaapidownload_transform;
245     trans_class->transform_caps   = gst_vaapidownload_transform_caps;
246     trans_class->transform_size   = gst_vaapidownload_transform_size;
247     trans_class->set_caps         = gst_vaapidownload_set_caps;
248
249     gst_element_class_set_details_simple(
250         element_class,
251         gst_vaapidownload_details.longname,
252         gst_vaapidownload_details.klass,
253         gst_vaapidownload_details.description,
254         gst_vaapidownload_details.author
255     );
256
257     /* sink pad */
258     pad_template = gst_static_pad_template_get(&gst_vaapidownload_sink_factory);
259     gst_element_class_add_pad_template(element_class, pad_template);
260     gst_object_unref(pad_template);
261
262     /* src pad */
263     pad_template = gst_static_pad_template_get(&gst_vaapidownload_src_factory);
264     gst_element_class_add_pad_template(element_class, pad_template);
265     gst_object_unref(pad_template);
266 }
267
268 static void
269 gst_vaapidownload_init(GstVaapiDownload *download)
270 {
271     GstPad *sinkpad, *srcpad;
272
273     download->display           = NULL;
274     download->allowed_caps      = NULL;
275     download->images            = NULL;
276     download->images_reset      = FALSE;
277     download->image_format      = (GstVaapiImageFormat)0;
278     download->image_width       = 0;
279     download->image_height      = 0;
280
281     /* Override buffer allocator on sink pad */
282     sinkpad = gst_element_get_static_pad(GST_ELEMENT(download), "sink");
283     gst_pad_set_query_function(sinkpad, gst_vaapidownload_query);
284     gst_object_unref(sinkpad);
285
286     /* Override query on src pad */
287     srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
288     gst_pad_set_query_function(srcpad, gst_vaapidownload_query);
289     gst_object_unref(srcpad);
290 }
291
292 static inline gboolean
293 gst_vaapidownload_ensure_display(GstVaapiDownload *download)
294 {
295     return gst_vaapi_ensure_display(download, GST_VAAPI_DISPLAY_TYPE_ANY,
296         &download->display);
297 }
298
299 static gboolean
300 gst_vaapidownload_start(GstBaseTransform *trans)
301 {
302     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
303
304     if (!gst_vaapidownload_ensure_display(download))
305         return FALSE;
306     return TRUE;
307 }
308
309 static gboolean
310 gst_vaapidownload_stop(GstBaseTransform *trans)
311 {
312     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
313
314     g_clear_object(&download->display);
315
316     return TRUE;
317 }
318
319 static GstVaapiImageFormat
320 get_surface_format(GstVaapiSurface *surface)
321 {
322     GstVaapiImage *image;
323     GstVaapiImageFormat format = GST_VAAPI_IMAGE_NV12;
324
325     /* XXX: NV12 is assumed by default */
326     image = gst_vaapi_surface_derive_image(surface);
327     if (image) {
328         format = gst_vaapi_image_get_format(image);
329         g_object_unref(image);
330     }
331     return format;
332 }
333
334 static gboolean
335 gst_vaapidownload_update_src_caps(GstVaapiDownload *download, GstBuffer *buffer)
336 {
337     GstVaapiVideoBuffer *vbuffer;
338     GstVaapiSurface *surface;
339     GstVaapiImageFormat format;
340     GstPad *srcpad;
341     GstCaps *in_caps, *out_caps;
342
343     vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
344     surface = gst_vaapi_video_buffer_get_surface(vbuffer);
345     if (!surface) {
346         GST_WARNING("failed to retrieve VA surface from buffer");
347         return FALSE;
348     }
349
350     format = get_surface_format(surface);
351     if (format == download->image_format)
352         return TRUE;
353
354     in_caps = GST_BUFFER_CAPS(buffer);
355     if (!in_caps) {
356         GST_WARNING("failed to retrieve caps from buffer");
357         return FALSE;
358     }
359
360     out_caps = gst_vaapi_image_format_get_caps(format);
361     if (!out_caps) {
362         GST_WARNING("failed to create caps from format %" GST_FOURCC_FORMAT,
363                     GST_FOURCC_ARGS(format));
364         return FALSE;
365     }
366
367     if (!gst_vaapi_append_surface_caps(out_caps, in_caps)) {
368         gst_caps_unref(out_caps);
369         return FALSE;
370     }
371
372     /* Try to renegotiate downstream caps */
373     srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
374     gst_pad_set_caps(srcpad, out_caps);
375     gst_object_unref(srcpad);
376
377     gst_vaapidownload_set_caps(GST_BASE_TRANSFORM(download), in_caps, out_caps);
378     gst_caps_replace(&download->allowed_caps, out_caps);
379     gst_caps_unref(out_caps);
380     return TRUE;
381 }
382
383 static void
384 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer)
385 {
386     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
387
388     gst_vaapidownload_update_src_caps(download, buffer);
389 }
390
391 static GstFlowReturn
392 gst_vaapidownload_transform(
393     GstBaseTransform *trans,
394     GstBuffer        *inbuf,
395     GstBuffer        *outbuf
396 )
397 {
398     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
399     GstVaapiVideoBuffer *vbuffer;
400     GstVaapiSurface *surface;
401     GstVaapiImage *image = NULL;
402     gboolean success;
403
404     vbuffer = GST_VAAPI_VIDEO_BUFFER(inbuf);
405     surface = gst_vaapi_video_buffer_get_surface(vbuffer);
406     if (!surface)
407         return GST_FLOW_UNEXPECTED;
408
409     image = gst_vaapi_video_pool_get_object(download->images);
410     if (!image)
411         return GST_FLOW_UNEXPECTED;
412     if (!gst_vaapi_surface_get_image(surface, image))
413         goto error_get_image;
414
415     success = gst_vaapi_image_get_buffer(image, outbuf, NULL);
416     gst_vaapi_video_pool_put_object(download->images, image);
417     if (!success)
418         goto error_get_buffer;
419     return GST_FLOW_OK;
420
421 error_get_image:
422     {
423         GST_WARNING("failed to download %" GST_FOURCC_FORMAT " image "
424                     "from surface 0x%08x",
425                     GST_FOURCC_ARGS(gst_vaapi_image_get_format(image)),
426                     gst_vaapi_surface_get_id(surface));
427         gst_vaapi_video_pool_put_object(download->images, image);
428         return GST_FLOW_UNEXPECTED;
429     }
430
431 error_get_buffer:
432     {
433         GST_WARNING("failed to transfer image to output video buffer");
434         return GST_FLOW_UNEXPECTED;
435     }
436 }
437
438 static GstCaps *
439 gst_vaapidownload_transform_caps(
440     GstBaseTransform *trans,
441     GstPadDirection   direction,
442     GstCaps          *caps
443 )
444 {
445     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
446     GstPad *srcpad;
447     GstCaps *allowed_caps, *inter_caps, *out_caps = NULL;
448     GstStructure *structure;
449
450     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
451
452     structure = gst_caps_get_structure(caps, 0);
453
454     if (direction == GST_PAD_SINK) {
455         if (!gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
456             return NULL;
457         if (!gst_vaapidownload_ensure_display(download))
458             return NULL;
459         out_caps = gst_caps_from_string(gst_vaapidownload_yuv_caps_str);
460
461         /* Build up allowed caps */
462         /* XXX: we don't know the decoded surface format yet so we
463            expose whatever VA images we support */
464         if (download->allowed_caps)
465             allowed_caps = gst_caps_ref(download->allowed_caps);
466         else {
467             allowed_caps = gst_vaapi_display_get_image_caps(download->display);
468             if (!allowed_caps)
469                 return NULL;
470         }
471         inter_caps = gst_caps_intersect(out_caps, allowed_caps);
472         gst_caps_unref(allowed_caps);
473         gst_caps_unref(out_caps);
474         out_caps = inter_caps;
475
476         /* Intersect with allowed caps from the peer, if any */
477         srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
478         allowed_caps = gst_pad_peer_get_caps(srcpad);
479         if (allowed_caps) {
480             inter_caps = gst_caps_intersect(out_caps, allowed_caps);
481             gst_caps_unref(allowed_caps);
482             gst_caps_unref(out_caps);
483             out_caps = inter_caps;
484         }
485     }
486     else {
487         if (!gst_structure_has_name(structure, "video/x-raw-yuv"))
488             return NULL;
489         out_caps = gst_caps_from_string(gst_vaapidownload_vaapi_caps_str);
490
491         structure = gst_caps_get_structure(out_caps, 0);
492         gst_structure_set(
493             structure,
494             "type", G_TYPE_STRING, "vaapi",
495             "opengl", G_TYPE_BOOLEAN, USE_GLX,
496             NULL
497         );
498     }
499
500     if (!gst_vaapi_append_surface_caps(out_caps, caps)) {
501         gst_caps_unref(out_caps);
502         return NULL;
503     }
504     return out_caps;
505 }
506
507 static gboolean
508 gst_vaapidownload_ensure_image_pool(GstVaapiDownload *download, GstCaps *caps)
509 {
510     GstStructure * const structure = gst_caps_get_structure(caps, 0);
511     GstVaapiImageFormat format;
512     gint width, height;
513
514     format = gst_vaapi_image_format_from_caps(caps);
515     gst_structure_get_int(structure, "width",  &width);
516     gst_structure_get_int(structure, "height", &height);
517
518     if (format != download->image_format ||
519         width  != download->image_width  ||
520         height != download->image_height) {
521         download->image_format = format;
522         download->image_width  = width;
523         download->image_height = height;
524         g_clear_object(&download->images);
525         download->images = gst_vaapi_image_pool_new(download->display, caps);
526         if (!download->images)
527             return FALSE;
528         download->images_reset = TRUE;
529     }
530     return TRUE;
531 }
532
533 static inline gboolean
534 gst_vaapidownload_negotiate_buffers(
535     GstVaapiDownload  *download,
536     GstCaps          *incaps,
537     GstCaps          *outcaps
538 )
539 {
540     if (!gst_vaapidownload_ensure_image_pool(download, outcaps))
541         return FALSE;
542     return TRUE;
543 }
544
545 static gboolean
546 gst_vaapidownload_set_caps(
547     GstBaseTransform *trans,
548     GstCaps          *incaps,
549     GstCaps          *outcaps
550 )
551 {
552     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
553
554     if (!gst_vaapidownload_negotiate_buffers(download, incaps, outcaps))
555         return FALSE;
556     return TRUE;
557 }
558
559 static gboolean
560 gst_vaapidownload_transform_size(
561     GstBaseTransform *trans,
562     GstPadDirection   direction,
563     GstCaps          *caps,
564     guint             size,
565     GstCaps          *othercaps,
566     guint            *othersize
567 )
568 {
569     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
570     GstStructure * const structure = gst_caps_get_structure(othercaps, 0);
571     GstVideoFormat format;
572     gint width, height;
573     guint i;
574
575     /* Lookup in cache */
576     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
577         TransformSizeCache * const tsc = &download->transform_size_cache[i];
578         if (tsc->caps && tsc->caps == othercaps) {
579             *othersize = tsc->size;
580             return TRUE;
581         }
582     }
583
584     /* Compute requested buffer size */
585     if (gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
586         *othersize = 0;
587     else {
588         if (!gst_video_format_parse_caps(othercaps, &format, &width, &height))
589             return FALSE;
590         *othersize = gst_video_format_get_size(format, width, height);
591     }
592
593     /* Update cache */
594     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
595         TransformSizeCache * const tsc = &download->transform_size_cache[i];
596         if (!tsc->caps) {
597             gst_caps_replace(&tsc->caps, othercaps);
598             tsc->size = *othersize;
599         }
600     }
601     return TRUE;
602 }
603
604 static gboolean
605 gst_vaapidownload_query(GstPad *pad, GstQuery *query)
606 {
607     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(gst_pad_get_parent_element(pad));
608     gboolean res;
609
610     GST_DEBUG("sharing display %p", download->display);
611
612     if (gst_vaapi_reply_to_query(query, download->display))
613         res = TRUE;
614     else
615         res = gst_pad_query_default(pad, query);
616
617     g_object_unref(download);
618     return res;
619
620 }