vaapidownload: add new plugin to download pixels from VA surfaces.
[vaapi:sree-gstreamer-vaapi.git] / gst / vaapi / gstvaapidownload.c
1 /*
2  *  gstvaapidownload.c - VA-API video downloader
3  *
4  *  gstreamer-vaapi (C) 2010-2011 Splitted-Desktop Systems
5  *  Copyright (C) 2011 Intel Corporation
6  *
7  *  This library is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU Lesser General Public License
9  *  as published by the Free Software Foundation; either version 2.1
10  *  of the License, or (at your option) any later version.
11  *
12  *  This library is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  *  Lesser General Public License for more details.
16  *
17  *  You should have received a copy of the GNU Lesser General Public
18  *  License along with this library; if not, write to the Free
19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20  *  Boston, MA 02110-1301 USA
21  */
22
23 /**
24  * SECTION:gstvaapidownload
25  * @short_description: A VA to video flow filter
26  *
27  * vaapidownload converts from VA surfaces to raw YUV pixels.
28  */
29
30 #include "config.h"
31
32 #include <gst/gst.h>
33 #include <gst/video/video.h>
34 #include <gst/video/videocontext.h>
35 #include <gst/vaapi/gstvaapivideosink.h>
36 #include <gst/vaapi/gstvaapivideobuffer.h>
37
38 #if USE_VAAPI_GLX
39 #include <gst/vaapi/gstvaapivideobuffer_glx.h>
40 #define gst_vaapi_video_buffer_new_from_pool(pool) \
41     gst_vaapi_video_buffer_glx_new_from_pool(pool)
42 #define gst_vaapi_video_buffer_new_from_buffer(buffer) \
43     gst_vaapi_video_buffer_glx_new_from_buffer(buffer)
44 #endif
45
46 #include "gstvaapipluginutil.h"
47 #include "gstvaapidownload.h"
48
49 #define GST_PLUGIN_NAME "vaapidownload"
50 #define GST_PLUGIN_DESC "A VA to video flow filter"
51
52 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapidownload);
53 #define GST_CAT_DEFAULT gst_debug_vaapidownload
54
55 /* ElementFactory information */
56 static const GstElementDetails gst_vaapidownload_details =
57     GST_ELEMENT_DETAILS(
58         "VA-API colorspace converter",
59         "Filter/Converter/Video",
60         GST_PLUGIN_DESC,
61         "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
62
63 /* Default templates */
64 static const char gst_vaapidownload_yuv_caps_str[] =
65     "video/x-raw-yuv, "
66     "width  = (int) [ 1, MAX ], "
67     "height = (int) [ 1, MAX ]; ";
68
69 static const char gst_vaapidownload_vaapi_caps_str[] =
70     GST_VAAPI_SURFACE_CAPS;
71
72 static GstStaticPadTemplate gst_vaapidownload_sink_factory =
73     GST_STATIC_PAD_TEMPLATE(
74         "sink",
75         GST_PAD_SINK,
76         GST_PAD_ALWAYS,
77         GST_STATIC_CAPS(gst_vaapidownload_vaapi_caps_str));
78
79 static GstStaticPadTemplate gst_vaapidownload_src_factory =
80     GST_STATIC_PAD_TEMPLATE(
81         "src",
82         GST_PAD_SRC,
83         GST_PAD_ALWAYS,
84         GST_STATIC_CAPS(gst_vaapidownload_yuv_caps_str));
85
86 typedef struct _TransformSizeCache TransformSizeCache;
87 struct _TransformSizeCache {
88     GstCaps            *caps;
89     guint               size;
90 };
91
92 struct _GstVaapiDownload {
93     /*< private >*/
94     GstBaseTransform    parent_instance;
95
96     GstVaapiDisplay    *display;
97     GstCaps            *allowed_caps;
98     TransformSizeCache  transform_size_cache[2];
99     GstVaapiVideoPool  *images;
100     GstVaapiImageFormat image_format;
101     guint               image_width;
102     guint               image_height;
103     unsigned int        images_reset    : 1;
104 };
105
106 struct _GstVaapiDownloadClass {
107     /*< private >*/
108     GstBaseTransformClass parent_class;
109 };
110
111 #define GstVideoContextClass GstVideoContextInterface
112 GST_BOILERPLATE_WITH_INTERFACE(
113     GstVaapiDownload,
114     gst_vaapidownload,
115     GstBaseTransform,
116     GST_TYPE_BASE_TRANSFORM,
117     GstVideoContext,
118     GST_TYPE_VIDEO_CONTEXT,
119     gst_video_context);
120
121 static gboolean
122 gst_vaapidownload_start(GstBaseTransform *trans);
123
124 static gboolean
125 gst_vaapidownload_stop(GstBaseTransform *trans);
126
127 static void
128 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer);
129
130 static GstFlowReturn
131 gst_vaapidownload_transform(
132     GstBaseTransform *trans,
133     GstBuffer        *inbuf,
134     GstBuffer        *outbuf
135 );
136
137 static GstCaps *
138 gst_vaapidownload_transform_caps(
139     GstBaseTransform *trans,
140     GstPadDirection   direction,
141     GstCaps          *caps
142 );
143
144 static gboolean
145 gst_vaapidownload_transform_size(
146     GstBaseTransform *trans,
147     GstPadDirection   direction,
148     GstCaps          *caps,
149     guint             size,
150     GstCaps          *othercaps,
151     guint            *othersize
152 );
153
154 static gboolean
155 gst_vaapidownload_set_caps(
156     GstBaseTransform *trans,
157     GstCaps          *incaps,
158     GstCaps          *outcaps
159 );
160
161 static gboolean
162 gst_vaapidownload_query(
163     GstPad   *pad,
164     GstQuery *query
165 );
166
167 /* GstVideoContext interface */
168
169 static void
170 gst_vaapidownload_set_video_context(GstVideoContext *context, const gchar *type,
171     const GValue *value)
172 {
173   GstVaapiDownload *download = GST_VAAPIDOWNLOAD (context);
174   gst_vaapi_set_display (type, value, &download->display);
175 }
176
177 static gboolean
178 gst_video_context_supported (GstVaapiDownload *download, GType iface_type)
179 {
180   return (iface_type == GST_TYPE_VIDEO_CONTEXT);
181 }
182
183 static void
184 gst_video_context_interface_init(GstVideoContextInterface *iface)
185 {
186     iface->set_context = gst_vaapidownload_set_video_context;
187 }
188
189 static void
190 gst_vaapidownload_destroy(GstVaapiDownload *download)
191 {
192     guint i;
193
194     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
195         TransformSizeCache * const tsc = &download->transform_size_cache[i];
196         if (tsc->caps) {
197             gst_caps_unref(tsc->caps);
198             tsc->caps = NULL;
199             tsc->size = 0;
200         }
201     }
202
203     if (download->allowed_caps) {
204         gst_caps_unref(download->allowed_caps);
205         download->allowed_caps = NULL;
206     }
207
208     if (download->images) {
209         g_object_unref(download->images);
210         download->images = NULL;
211     }
212
213     if (download->display) {
214         g_object_unref(download->display);
215         download->display = NULL;
216     }
217 }
218
219 static void
220 gst_vaapidownload_base_init(gpointer klass)
221 {
222     GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
223
224     gst_element_class_set_details(element_class, &gst_vaapidownload_details);
225
226     /* sink pad */
227     gst_element_class_add_pad_template(
228         element_class,
229         gst_static_pad_template_get(&gst_vaapidownload_sink_factory)
230     );
231
232     /* src pad */
233     gst_element_class_add_pad_template(
234         element_class,
235         gst_static_pad_template_get(&gst_vaapidownload_src_factory)
236     );
237 }
238
239 static void
240 gst_vaapidownload_finalize(GObject *object)
241 {
242     gst_vaapidownload_destroy(GST_VAAPIDOWNLOAD(object));
243
244     G_OBJECT_CLASS(parent_class)->finalize(object);
245 }
246
247 static void
248 gst_vaapidownload_class_init(GstVaapiDownloadClass *klass)
249 {
250     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
251     GstBaseTransformClass * const trans_class = GST_BASE_TRANSFORM_CLASS(klass);
252
253     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapidownload,
254                             GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
255
256     object_class->finalize        = gst_vaapidownload_finalize;
257     trans_class->start            = gst_vaapidownload_start;
258     trans_class->stop             = gst_vaapidownload_stop;
259     trans_class->before_transform = gst_vaapidownload_before_transform;
260     trans_class->transform        = gst_vaapidownload_transform;
261     trans_class->transform_caps   = gst_vaapidownload_transform_caps;
262     trans_class->transform_size   = gst_vaapidownload_transform_size;
263     trans_class->set_caps         = gst_vaapidownload_set_caps;
264 }
265
266 static void
267 gst_vaapidownload_init(GstVaapiDownload *download, GstVaapiDownloadClass *klass)
268 {
269     GstPad *sinkpad, *srcpad;
270
271     download->display           = NULL;
272     download->allowed_caps      = NULL;
273     download->images            = NULL;
274     download->images_reset      = FALSE;
275     download->image_format      = (GstVaapiImageFormat)0;
276     download->image_width       = 0;
277     download->image_height      = 0;
278
279     /* Override buffer allocator on sink pad */
280     sinkpad = gst_element_get_static_pad(GST_ELEMENT(download), "sink");
281     gst_pad_set_query_function(sinkpad, gst_vaapidownload_query);
282     gst_object_unref(sinkpad);
283
284     /* Override query on src pad */
285     srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
286     gst_pad_set_query_function(srcpad, gst_vaapidownload_query);
287     gst_object_unref(srcpad);
288 }
289
290 static gboolean
291 gst_vaapidownload_start(GstBaseTransform *trans)
292 {
293     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
294
295     if (!gst_vaapi_ensure_display(download, &download->display))
296         return FALSE;
297     return TRUE;
298 }
299
300 static gboolean
301 gst_vaapidownload_stop(GstBaseTransform *trans)
302 {
303     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
304
305     if (download->display) {
306         g_object_unref(download->display);
307         download->display = NULL;
308     }
309     return TRUE;
310 }
311
312 static GstVaapiImageFormat
313 get_surface_format(GstVaapiSurface *surface)
314 {
315     GstVaapiImage *image;
316     GstVaapiImageFormat format = GST_VAAPI_IMAGE_NV12;
317
318     /* XXX: NV12 is assumed by default */
319     image = gst_vaapi_surface_derive_image(surface);
320     if (image) {
321         format = gst_vaapi_image_get_format(image);
322         g_object_unref(image);
323     }
324     return format;
325 }
326
327 static gboolean
328 gst_vaapidownload_update_src_caps(GstVaapiDownload *download, GstBuffer *buffer)
329 {
330     GstVaapiVideoBuffer *vbuffer;
331     GstVaapiSurface *surface;
332     GstVaapiImageFormat format;
333     GstPad *srcpad;
334     GstCaps *in_caps, *out_caps;
335
336     vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
337     surface = gst_vaapi_video_buffer_get_surface(vbuffer);
338     if (!surface) {
339         GST_WARNING("failed to retrieve VA surface from buffer");
340         return FALSE;
341     }
342
343     format = get_surface_format(surface);
344     if (format == download->image_format)
345         return TRUE;
346
347     in_caps = GST_BUFFER_CAPS(buffer);
348     if (!in_caps) {
349         GST_WARNING("failed to retrieve caps from buffer");
350         return FALSE;
351     }
352
353     out_caps = gst_vaapi_image_format_get_caps(format);
354     if (!out_caps) {
355         GST_WARNING("failed to create caps from format %" GST_FOURCC_FORMAT,
356                     GST_FOURCC_ARGS(format));
357         return FALSE;
358     }
359
360     if (!gst_vaapi_append_surface_caps(out_caps, in_caps)) {
361         gst_caps_unref(out_caps);
362         return FALSE;
363     }
364
365     /* Try to renegotiate downstream caps */
366     srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
367     gst_pad_set_caps(srcpad, out_caps);
368     gst_object_unref(srcpad);
369
370     gst_vaapidownload_set_caps(GST_BASE_TRANSFORM(download), in_caps, out_caps);
371     gst_caps_replace(&download->allowed_caps, out_caps);
372     gst_caps_unref(out_caps);
373     return TRUE;
374 }
375
376 static void
377 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer)
378 {
379     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
380
381     gst_vaapidownload_update_src_caps(download, buffer);
382 }
383
384 static GstFlowReturn
385 gst_vaapidownload_transform(
386     GstBaseTransform *trans,
387     GstBuffer        *inbuf,
388     GstBuffer        *outbuf
389 )
390 {
391     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
392     GstVaapiVideoBuffer *vbuffer;
393     GstVaapiSurface *surface;
394     GstVaapiImage *image = NULL;
395     gboolean success;
396
397     vbuffer = GST_VAAPI_VIDEO_BUFFER(inbuf);
398     surface = gst_vaapi_video_buffer_get_surface(vbuffer);
399     if (!surface)
400         return GST_FLOW_UNEXPECTED;
401
402     image = gst_vaapi_video_pool_get_object(download->images);
403     if (!image)
404         return GST_FLOW_UNEXPECTED;
405     if (!gst_vaapi_surface_get_image(surface, image))
406         goto error_get_image;
407
408     success = gst_vaapi_image_get_buffer(image, outbuf, NULL);
409     gst_vaapi_video_pool_put_object(download->images, image);
410     if (!success)
411         goto error_get_buffer;
412     return GST_FLOW_OK;
413
414 error_get_image:
415     {
416         GST_WARNING("failed to download %" GST_FOURCC_FORMAT " image "
417                     "from surface 0x%08x",
418                     GST_FOURCC_ARGS(gst_vaapi_image_get_format(image)),
419                     gst_vaapi_surface_get_id(surface));
420         gst_vaapi_video_pool_put_object(download->images, image);
421         return GST_FLOW_UNEXPECTED;
422     }
423
424 error_get_buffer:
425     {
426         GST_WARNING("failed to transfer image to output video buffer");
427         return GST_FLOW_UNEXPECTED;
428     }
429 }
430
431 static GstCaps *
432 gst_vaapidownload_transform_caps(
433     GstBaseTransform *trans,
434     GstPadDirection   direction,
435     GstCaps          *caps
436 )
437 {
438     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
439     GstPad *srcpad;
440     GstCaps *allowed_caps, *inter_caps, *out_caps = NULL;
441     GstStructure *structure;
442
443     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
444
445     structure = gst_caps_get_structure(caps, 0);
446
447     if (direction == GST_PAD_SINK) {
448         if (!gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
449             return NULL;
450         if (!gst_vaapi_ensure_display(download, &download->display))
451             return NULL;
452         out_caps = gst_caps_from_string(gst_vaapidownload_yuv_caps_str);
453
454         /* Build up allowed caps */
455         /* XXX: we don't know the decoded surface format yet so we
456            expose whatever VA images we support */
457         if (download->allowed_caps)
458             allowed_caps = gst_caps_ref(download->allowed_caps);
459         else {
460             allowed_caps = gst_vaapi_display_get_image_caps(download->display);
461             if (!allowed_caps)
462                 return NULL;
463         }
464         inter_caps = gst_caps_intersect(out_caps, allowed_caps);
465         gst_caps_unref(allowed_caps);
466         gst_caps_unref(out_caps);
467         out_caps = inter_caps;
468
469         /* Intersect with allowed caps from the peer, if any */
470         srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
471         allowed_caps = gst_pad_peer_get_caps(srcpad);
472         if (allowed_caps) {
473             inter_caps = gst_caps_intersect(out_caps, allowed_caps);
474             gst_caps_unref(allowed_caps);
475             gst_caps_unref(out_caps);
476             out_caps = inter_caps;
477         }
478     }
479     else {
480         if (!gst_structure_has_name(structure, "video/x-raw-yuv"))
481             return NULL;
482         out_caps = gst_caps_from_string(gst_vaapidownload_vaapi_caps_str);
483
484         structure = gst_caps_get_structure(out_caps, 0);
485         gst_structure_set(
486             structure,
487             "type", G_TYPE_STRING, "vaapi",
488             "opengl", G_TYPE_BOOLEAN, USE_VAAPI_GLX,
489             NULL
490         );
491     }
492
493     if (!gst_vaapi_append_surface_caps(out_caps, caps)) {
494         gst_caps_unref(out_caps);
495         return NULL;
496     }
497     return out_caps;
498 }
499
500 static gboolean
501 gst_vaapidownload_ensure_image_pool(GstVaapiDownload *download, GstCaps *caps)
502 {
503     GstStructure * const structure = gst_caps_get_structure(caps, 0);
504     GstVaapiImageFormat format;
505     gint width, height;
506
507     format = gst_vaapi_image_format_from_caps(caps);
508     gst_structure_get_int(structure, "width",  &width);
509     gst_structure_get_int(structure, "height", &height);
510
511     if (format != download->image_format ||
512         width  != download->image_width  ||
513         height != download->image_height) {
514         download->image_format = format;
515         download->image_width  = width;
516         download->image_height = height;
517         if (download->images)
518             g_object_unref(download->images);
519         download->images = gst_vaapi_image_pool_new(download->display, caps);
520         if (!download->images)
521             return FALSE;
522         download->images_reset = TRUE;
523     }
524     return TRUE;
525 }
526
527 static inline gboolean
528 gst_vaapidownload_negotiate_buffers(
529     GstVaapiDownload  *download,
530     GstCaps          *incaps,
531     GstCaps          *outcaps
532 )
533 {
534     if (!gst_vaapidownload_ensure_image_pool(download, outcaps))
535         return FALSE;
536     return TRUE;
537 }
538
539 static gboolean
540 gst_vaapidownload_set_caps(
541     GstBaseTransform *trans,
542     GstCaps          *incaps,
543     GstCaps          *outcaps
544 )
545 {
546     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
547
548     if (!gst_vaapidownload_negotiate_buffers(download, incaps, outcaps))
549         return FALSE;
550     return TRUE;
551 }
552
553 static gboolean
554 gst_vaapidownload_transform_size(
555     GstBaseTransform *trans,
556     GstPadDirection   direction,
557     GstCaps          *caps,
558     guint             size,
559     GstCaps          *othercaps,
560     guint            *othersize
561 )
562 {
563     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
564     GstStructure * const structure = gst_caps_get_structure(othercaps, 0);
565     GstVideoFormat format;
566     gint width, height;
567     guint i;
568
569     /* Lookup in cache */
570     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
571         TransformSizeCache * const tsc = &download->transform_size_cache[i];
572         if (tsc->caps && tsc->caps == othercaps) {
573             *othersize = tsc->size;
574             return TRUE;
575         }
576     }
577
578     /* Compute requested buffer size */
579     if (gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
580         *othersize = 0;
581     else {
582         if (!gst_video_format_parse_caps(othercaps, &format, &width, &height))
583             return FALSE;
584         *othersize = gst_video_format_get_size(format, width, height);
585     }
586
587     /* Update cache */
588     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
589         TransformSizeCache * const tsc = &download->transform_size_cache[i];
590         if (!tsc->caps) {
591             gst_caps_replace(&tsc->caps, othercaps);
592             tsc->size = *othersize;
593         }
594     }
595     return TRUE;
596 }
597
598 static gboolean
599 gst_vaapidownload_query(GstPad *pad, GstQuery *query)
600 {
601     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(gst_pad_get_parent_element(pad));
602     gboolean res;
603
604     GST_DEBUG("sharing display %p", download->display);
605
606     if (gst_vaapi_reply_to_query(query, download->display))
607         res = TRUE;
608     else
609         res = gst_pad_query_default(pad, query);
610
611     g_object_unref(download);
612     return res;
613
614 }