2 * gstvaapidownload.c - VA-API video downloader
4 * Copyright (C) 2010-2011 Splitted-Desktop Systems
5 * Copyright (C) 2011-2012 Intel Corporation
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public License
9 * as published by the Free Software Foundation; either version 2.1
10 * of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free
19 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301 USA
24 * SECTION:gstvaapidownload
25 * @short_description: A VA to video flow filter
27 * vaapidownload converts from VA surfaces to raw YUV pixels.
30 #include "gst/vaapi/sysdeps.h"
32 #include <gst/video/video.h>
33 #include <gst/video/videocontext.h>
35 #include "gstvaapidownload.h"
36 #include "gstvaapipluginutil.h"
38 #define GST_PLUGIN_NAME "vaapidownload"
39 #define GST_PLUGIN_DESC "A VA to video flow filter"
41 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapidownload);
42 #define GST_CAT_DEFAULT gst_debug_vaapidownload
44 /*Fixme: format is not only NV12 ?*/
46 /* Default templates */
47 static const char gst_vaapidownload_yuv_caps_str[] =
49 "format=(string)NV12, "
50 "width = (int) [ 1, MAX ], "
51 "height = (int) [ 1, MAX ]; ";
53 static const char gst_vaapidownload_vaapi_caps_str[] =
56 static GstStaticPadTemplate gst_vaapidownload_sink_factory =
57 GST_STATIC_PAD_TEMPLATE(
61 GST_STATIC_CAPS(gst_vaapidownload_vaapi_caps_str));
63 static GstStaticPadTemplate gst_vaapidownload_src_factory =
64 GST_STATIC_PAD_TEMPLATE(
68 GST_STATIC_CAPS(gst_vaapidownload_yuv_caps_str));
70 typedef struct _TransformSizeCache TransformSizeCache;
71 struct _TransformSizeCache {
76 struct _GstVaapiDownload {
78 GstBaseTransform parent_instance;
80 GstVaapiDisplay *display;
81 GstCaps *allowed_caps;
82 TransformSizeCache transform_size_cache[2];
83 GstVaapiImagePool *images;
84 GstVaapiImageFormat image_format;
87 unsigned int images_reset : 1;
90 struct _GstVaapiDownloadClass {
92 GstBaseTransformClass parent_class;
96 gst_video_context_interface_init(GstVideoContextInterface *iface);
98 #define GstVideoContextClass GstVideoContextInterface
99 G_DEFINE_TYPE_WITH_CODE(
102 GST_TYPE_BASE_TRANSFORM,
103 G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
104 gst_video_context_interface_init))
107 gst_vaapidownload_start(GstBaseTransform *trans);
110 gst_vaapidownload_stop(GstBaseTransform *trans);
114 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer);
118 gst_vaapidownload_transform(
119 GstBaseTransform *trans,
125 gst_vaapidownload_transform_caps(
126 GstBaseTransform *trans,
127 GstPadDirection direction,
133 gst_vaapidownload_transform_size(
134 GstBaseTransform *trans,
135 GstPadDirection direction,
143 gst_vaapidownload_set_caps(
144 GstBaseTransform *trans,
150 gst_vaapidownload_query(
156 /* GstVideoContext interface */
159 gst_vaapidownload_set_video_context(GstVideoContext *context, const gchar *type,
162 GstVaapiDownload *download = GST_VAAPIDOWNLOAD (context);
163 gst_vaapi_set_display (type, value, &download->display);
167 gst_video_context_interface_init(GstVideoContextInterface *iface)
169 iface->set_context = gst_vaapidownload_set_video_context;
173 gst_vaapidownload_destroy(GstVaapiDownload *download)
177 for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
178 TransformSizeCache * const tsc = &download->transform_size_cache[i];
180 gst_caps_unref(tsc->caps);
186 if (download->allowed_caps) {
187 gst_caps_unref(download->allowed_caps);
188 download->allowed_caps = NULL;
191 g_clear_object(&download->images);
192 g_clear_object(&download->display);
196 gst_vaapidownload_finalize(GObject *object)
198 gst_vaapidownload_destroy(GST_VAAPIDOWNLOAD(object));
200 G_OBJECT_CLASS(gst_vaapidownload_parent_class)->finalize(object);
204 gst_vaapidownload_class_init(GstVaapiDownloadClass *klass)
206 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
207 GstBaseTransformClass * const trans_class = GST_BASE_TRANSFORM_CLASS(klass);
208 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
209 GstPadTemplate *pad_template;
211 GST_DEBUG_CATEGORY_INIT(gst_debug_vaapidownload,
212 GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
214 object_class->finalize = gst_vaapidownload_finalize;
215 trans_class->start = gst_vaapidownload_start;
216 trans_class->stop = gst_vaapidownload_stop;
218 trans_class->before_transform = gst_vaapidownload_before_transform;
220 trans_class->transform = gst_vaapidownload_transform;
221 trans_class->transform_caps = gst_vaapidownload_transform_caps;
222 trans_class->transform_size = gst_vaapidownload_transform_size;
223 trans_class->set_caps = gst_vaapidownload_set_caps;
225 gst_element_class_set_static_metadata (element_class,
226 "VA-API colorspace converter",
227 "Filter/Converter/Video",
229 "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
232 pad_template = gst_static_pad_template_get(&gst_vaapidownload_sink_factory);
233 gst_element_class_add_pad_template(element_class, pad_template);
236 pad_template = gst_static_pad_template_get(&gst_vaapidownload_src_factory);
237 gst_element_class_add_pad_template(element_class, pad_template);
241 gst_vaapidownload_init(GstVaapiDownload *download)
243 GstPad *sinkpad, *srcpad;
245 download->display = NULL;
246 download->allowed_caps = NULL;
247 download->images = NULL;
248 download->images_reset = FALSE;
249 download->image_format = (GstVaapiImageFormat)0;
250 download->image_width = 0;
251 download->image_height = 0;
253 /* Override buffer allocator on sink pad */
254 sinkpad = gst_element_get_static_pad(GST_ELEMENT(download), "sink");
255 gst_pad_set_query_function(sinkpad, gst_vaapidownload_query);
256 gst_object_unref(sinkpad);
258 /* Override query on src pad */
259 srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
260 gst_pad_set_query_function(srcpad, gst_vaapidownload_query);
261 gst_object_unref(srcpad);
264 static inline gboolean
265 gst_vaapidownload_ensure_display(GstVaapiDownload *download)
267 return gst_vaapi_ensure_display(download, GST_VAAPI_DISPLAY_TYPE_ANY,
272 gst_vaapidownload_start(GstBaseTransform *trans)
274 GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
276 if (!gst_vaapidownload_ensure_display(download))
282 gst_vaapidownload_stop(GstBaseTransform *trans)
284 GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
286 g_clear_object(&download->display);
291 static GstVaapiImageFormat
292 get_surface_format(GstVaapiSurface *surface)
294 GstVaapiImage *image;
295 GstVaapiImageFormat format = GST_VAAPI_IMAGE_NV12;
297 /* XXX: NV12 is assumed by default */
298 image = gst_vaapi_surface_derive_image(surface);
300 format = gst_vaapi_image_get_format(image);
301 g_object_unref(image);
308 gst_vaapidownload_update_src_caps(GstVaapiDownload *download, GstBuffer *buffer)
310 GstVaapiSurface *surface;
311 GstVaapiImageFormat format;
313 GstCaps *in_caps, *out_caps;
316 gst_buffer_map (buf, &map_info, GST_MAP_READ);
317 surface = map_info.data;
319 GST_DEBUG_OBJECT (sink, "Failed to retrieve the VA surface from buffer");
323 format = get_surface_format(surface);
324 if (format == download->image_format)
327 in_caps = GST_BUFFER_CAPS(buffer);
329 GST_WARNING("failed to retrieve caps from buffer");
333 out_caps = gst_vaapi_image_format_get_caps(format);
335 GST_WARNING("failed to create caps from format %" GST_FOURCC_FORMAT,
336 GST_FOURCC_ARGS(format));
340 if (!gst_vaapi_append_surface_caps(out_caps, in_caps)) {
341 gst_caps_unref(out_caps);
345 /* Try to renegotiate downstream caps */
346 srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
347 gst_pad_set_caps(srcpad, out_caps);
348 gst_object_unref(srcpad);
350 gst_vaapidownload_set_caps(GST_BASE_TRANSFORM(download), in_caps, out_caps);
351 gst_caps_replace(&download->allowed_caps, out_caps);
352 gst_caps_unref(out_caps);
357 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer)
359 GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
361 gst_vaapidownload_update_src_caps(download, buffer);
365 gst_vaapidownload_transform(
366 GstBaseTransform *trans,
371 GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
372 GstVaapiSurface *surface;
373 GstVaapiImage *image = NULL;
379 GstVaapiSurfaceMeta *meta;
381 meta = (GstVaapiSurfaceMeta *)gst_buffer_get_meta((inbuf),GST_VAAPI_SURFACE_META_API_TYPE);
382 surface = (GstVaapiSurface *)meta->surface;
384 GST_DEBUG_OBJECT (download, "Failed to retrieve the VA surface from buffer");
388 if (gst_buffer_pool_acquire_buffer((GstBufferPool *)download->images, &buf, NULL) != GST_FLOW_OK){
389 GST_ERROR("Failed to acquire buffer");
393 gst_buffer_map(buf, &info, GST_MAP_WRITE);
394 image = (GstVaapiImage *)info.data;
397 GST_DEBUG_OBJECT (download, "Failed to retrieve the VA image from buffer");
401 if (!gst_vaapi_surface_get_image(surface, image))
402 goto error_get_image;
404 success = gst_vaapi_image_get_buffer(image, outbuf, NULL);
405 gst_buffer_pool_release_buffer((GstBufferPool *)download->images, buf);
407 goto error_get_buffer;
412 GST_WARNING("failed to download %" GST_FOURCC_FORMAT " image "
413 "from surface 0x%08x",
414 GST_FOURCC_ARGS(gst_vaapi_image_get_format(image)),
415 gst_vaapi_surface_get_id(surface));
416 gst_buffer_pool_release_buffer((GstBufferPool *)download->images, buf);
422 GST_WARNING("failed to transfer image to output video buffer");
428 gst_vaapidownload_transform_caps(
429 GstBaseTransform *trans,
430 GstPadDirection direction,
435 GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
437 GstCaps *allowed_caps, *inter_caps, *out_caps = NULL;
438 GstStructure *structure;
440 g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
442 structure = gst_caps_get_structure(caps, 0);
444 if (direction == GST_PAD_SINK) {
445 if (!gst_structure_has_name(structure, "video/x-raw"))
447 if (!gst_vaapidownload_ensure_display(download))
449 out_caps = gst_caps_from_string(gst_vaapidownload_yuv_caps_str);
451 /* Build up allowed caps */
452 /* XXX: we don't know the decoded surface format yet so we
453 expose whatever VA images we support */
454 if (download->allowed_caps)
455 allowed_caps = gst_caps_ref(download->allowed_caps);
457 allowed_caps = gst_vaapi_display_get_image_caps(download->display);
461 inter_caps = gst_caps_intersect(out_caps, allowed_caps);
462 gst_caps_unref(allowed_caps);
463 gst_caps_unref(out_caps);
464 out_caps = inter_caps;
466 /* Intersect with allowed caps from the peer, if any */
467 srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
468 allowed_caps = gst_pad_peer_query_caps(srcpad, NULL);
470 inter_caps = gst_caps_intersect(out_caps, allowed_caps);
471 gst_caps_unref(allowed_caps);
472 gst_caps_unref(out_caps);
473 out_caps = inter_caps;
477 if (!gst_structure_has_name(structure, "video/x-raw"))
479 out_caps = gst_caps_from_string(gst_vaapidownload_vaapi_caps_str);
481 structure = gst_caps_get_structure(out_caps, 0);
484 "type", G_TYPE_STRING, "vaapi",
485 "opengl", G_TYPE_BOOLEAN, USE_GLX,
490 if (!gst_vaapi_append_surface_caps(out_caps, caps)) {
491 gst_caps_unref(out_caps);
498 gst_vaapidownload_ensure_image_pool(GstVaapiDownload *download, GstCaps *caps)
500 GstStructure * const structure = gst_caps_get_structure(caps, 0);
501 GstVaapiImageFormat format;
503 GstStructure *config;
505 format = gst_vaapi_image_format_from_caps(caps);
506 gst_structure_get_int(structure, "width", &width);
507 gst_structure_get_int(structure, "height", &height);
509 if (format != download->image_format ||
510 width != download->image_width ||
511 height != download->image_height) {
512 download->image_format = format;
513 download->image_width = width;
514 download->image_height = height;
515 g_clear_object(&download->images);
516 download->images = (GstVaapiImagePool *)gst_vaapi_image_pool_new(download->display, caps);
517 if (!download->images)
519 /*Fixme: remove hard-coded min and max values*/
520 config = gst_buffer_pool_get_config ((GstBufferPool *)download->images);
521 gst_buffer_pool_config_set_params (config, caps, width*height, 6, 24);
523 gst_buffer_pool_set_config ((GstBufferPool *)download->images, config);
524 gst_buffer_pool_set_active ((GstBufferPool *)download->images, TRUE);
526 download->images_reset = TRUE;
531 static inline gboolean
532 gst_vaapidownload_negotiate_buffers(
533 GstVaapiDownload *download,
538 if (!gst_vaapidownload_ensure_image_pool(download, outcaps))
544 gst_vaapidownload_set_caps(
545 GstBaseTransform *trans,
550 GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
552 if (!gst_vaapidownload_negotiate_buffers(download, incaps, outcaps))
558 gst_vaapidownload_transform_size(
559 GstBaseTransform *trans,
560 GstPadDirection direction,
567 GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
568 GstStructure * const structure = gst_caps_get_structure(othercaps, 0);
569 GstVideoFormat format;
574 /* Lookup in cache */
575 for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
576 TransformSizeCache * const tsc = &download->transform_size_cache[i];
577 if (tsc->caps && tsc->caps == othercaps) {
578 *othersize = tsc->size;
583 /* Compute requested buffer size */
584 if (gst_structure_has_name(structure, "video/x-raw"))
587 if (!gst_video_info_from_caps(&info, othercaps))
591 height = info.height;
592 *othersize = info.size;
594 /*if (!gst_video_format_parse_caps(othercaps, &format, &width, &height))
596 *othersize = gst_video_format_get_size(format, width, height);*/
600 for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
601 TransformSizeCache * const tsc = &download->transform_size_cache[i];
603 gst_caps_replace(&tsc->caps, othercaps);
604 tsc->size = *othersize;
611 gst_vaapidownload_query(GstPad *pad, GstObject *parent, GstQuery *query)
613 GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(parent);
616 GST_DEBUG("sharing display %p", download->display);
618 if (gst_vaapi_reply_to_query(query, download->display))
621 res = gst_pad_query_default(pad, parent, query);