Fix new video format API.
[vaapi:zhongcongs-gstreamer-vaapi.git] / gst / vaapi / gstvaapidownload.c
1 /*
2  *  gstvaapidownload.c - VA-API video downloader
3  *
4  *  Copyright (C) 2010-2011 Splitted-Desktop Systems
5  *  Copyright (C) 2011-2013 Intel Corporation
6  *
7  *  This library is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU Lesser General Public License
9  *  as published by the Free Software Foundation; either version 2.1
10  *  of the License, or (at your option) any later version.
11  *
12  *  This library is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  *  Lesser General Public License for more details.
16  *
17  *  You should have received a copy of the GNU Lesser General Public
18  *  License along with this library; if not, write to the Free
19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20  *  Boston, MA 02110-1301 USA
21  */
22
23 /**
24  * SECTION:gstvaapidownload
25  * @short_description: A VA to video flow filter
26  *
27  * vaapidownload converts from VA surfaces to raw YUV pixels.
28  */
29
30 #include "gst/vaapi/sysdeps.h"
31 #include <gst/gst.h>
32 #include <gst/video/video.h>
33 #include <gst/video/videocontext.h>
34
35 #include "gstvaapidownload.h"
36 #include "gstvaapipluginutil.h"
37 #include "gstvaapivideobuffer.h"
38
39 #define GST_PLUGIN_NAME "vaapidownload"
40 #define GST_PLUGIN_DESC "A VA to video flow filter"
41
42 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapidownload);
43 #define GST_CAT_DEFAULT gst_debug_vaapidownload
44
45 /* Default templates */
46 static const char gst_vaapidownload_yuv_caps_str[] =
47     "video/x-raw-yuv, "
48     "width  = (int) [ 1, MAX ], "
49     "height = (int) [ 1, MAX ]; ";
50
51 static const char gst_vaapidownload_vaapi_caps_str[] =
52     GST_VAAPI_SURFACE_CAPS;
53
54 static GstStaticPadTemplate gst_vaapidownload_sink_factory =
55     GST_STATIC_PAD_TEMPLATE(
56         "sink",
57         GST_PAD_SINK,
58         GST_PAD_ALWAYS,
59         GST_STATIC_CAPS(gst_vaapidownload_vaapi_caps_str));
60
61 static GstStaticPadTemplate gst_vaapidownload_src_factory =
62     GST_STATIC_PAD_TEMPLATE(
63         "src",
64         GST_PAD_SRC,
65         GST_PAD_ALWAYS,
66         GST_STATIC_CAPS(gst_vaapidownload_yuv_caps_str));
67
68 typedef struct _TransformSizeCache TransformSizeCache;
69 struct _TransformSizeCache {
70     GstCaps            *caps;
71     guint               size;
72 };
73
74 struct _GstVaapiDownload {
75     /*< private >*/
76     GstBaseTransform    parent_instance;
77
78     GstVaapiDisplay    *display;
79     GstCaps            *allowed_caps;
80     TransformSizeCache  transform_size_cache[2];
81     GstVaapiVideoPool  *images;
82     GstVideoFormat      image_format;
83     guint               image_width;
84     guint               image_height;
85     unsigned int        images_reset    : 1;
86 };
87
88 struct _GstVaapiDownloadClass {
89     /*< private >*/
90     GstBaseTransformClass parent_class;
91 };
92
93 /* GstImplementsInterface interface */
94 static gboolean
95 gst_vaapidownload_implements_interface_supported(
96     GstImplementsInterface *iface,
97     GType                   type
98 )
99 {
100     return (type == GST_TYPE_VIDEO_CONTEXT);
101 }
102
103 static void
104 gst_vaapidownload_implements_iface_init(GstImplementsInterfaceClass *iface)
105 {
106     iface->supported = gst_vaapidownload_implements_interface_supported;
107 }
108
109 /* GstVideoContext interface */
110 static void
111 gst_vaapidownload_set_video_context(GstVideoContext *context, const gchar *type,
112     const GValue *value)
113 {
114   GstVaapiDownload *download = GST_VAAPIDOWNLOAD (context);
115   gst_vaapi_set_display (type, value, &download->display);
116 }
117
118 static void
119 gst_video_context_interface_init(GstVideoContextInterface *iface)
120 {
121     iface->set_context = gst_vaapidownload_set_video_context;
122 }
123
124 #define GstVideoContextClass GstVideoContextInterface
125 G_DEFINE_TYPE_WITH_CODE(
126     GstVaapiDownload,
127     gst_vaapidownload,
128     GST_TYPE_BASE_TRANSFORM,
129     G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
130                           gst_vaapidownload_implements_iface_init);
131     G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
132                           gst_video_context_interface_init))
133
134 static gboolean
135 gst_vaapidownload_start(GstBaseTransform *trans);
136
137 static gboolean
138 gst_vaapidownload_stop(GstBaseTransform *trans);
139
140 static void
141 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer);
142
143 static GstFlowReturn
144 gst_vaapidownload_transform(
145     GstBaseTransform *trans,
146     GstBuffer        *inbuf,
147     GstBuffer        *outbuf
148 );
149
150 static GstCaps *
151 gst_vaapidownload_transform_caps(
152     GstBaseTransform *trans,
153     GstPadDirection   direction,
154     GstCaps          *caps
155 );
156
157 static gboolean
158 gst_vaapidownload_transform_size(
159     GstBaseTransform *trans,
160     GstPadDirection   direction,
161     GstCaps          *caps,
162     guint             size,
163     GstCaps          *othercaps,
164     guint            *othersize
165 );
166
167 static gboolean
168 gst_vaapidownload_set_caps(
169     GstBaseTransform *trans,
170     GstCaps          *incaps,
171     GstCaps          *outcaps
172 );
173
174 static gboolean
175 gst_vaapidownload_query(
176     GstPad   *pad,
177     GstQuery *query
178 );
179
180 static void
181 gst_vaapidownload_destroy(GstVaapiDownload *download)
182 {
183     guint i;
184
185     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
186         TransformSizeCache * const tsc = &download->transform_size_cache[i];
187         if (tsc->caps) {
188             gst_caps_unref(tsc->caps);
189             tsc->caps = NULL;
190             tsc->size = 0;
191         }
192     }
193
194     if (download->allowed_caps) {
195         gst_caps_unref(download->allowed_caps);
196         download->allowed_caps = NULL;
197     }
198
199     gst_vaapi_video_pool_replace(&download->images, NULL);
200     gst_vaapi_display_replace(&download->display, NULL);
201 }
202
203 static void
204 gst_vaapidownload_finalize(GObject *object)
205 {
206     gst_vaapidownload_destroy(GST_VAAPIDOWNLOAD(object));
207
208     G_OBJECT_CLASS(gst_vaapidownload_parent_class)->finalize(object);
209 }
210
211 static void
212 gst_vaapidownload_class_init(GstVaapiDownloadClass *klass)
213 {
214     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
215     GstBaseTransformClass * const trans_class = GST_BASE_TRANSFORM_CLASS(klass);
216     GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
217     GstPadTemplate *pad_template;
218
219     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapidownload,
220                             GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
221
222     object_class->finalize        = gst_vaapidownload_finalize;
223     trans_class->start            = gst_vaapidownload_start;
224     trans_class->stop             = gst_vaapidownload_stop;
225     trans_class->before_transform = gst_vaapidownload_before_transform;
226     trans_class->transform        = gst_vaapidownload_transform;
227     trans_class->transform_caps   = gst_vaapidownload_transform_caps;
228     trans_class->transform_size   = gst_vaapidownload_transform_size;
229     trans_class->set_caps         = gst_vaapidownload_set_caps;
230
231     gst_element_class_set_static_metadata(element_class,
232         "VA-API colorspace converter",
233         "Filter/Converter/Video",
234         GST_PLUGIN_DESC,
235         "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
236
237     /* sink pad */
238     pad_template = gst_static_pad_template_get(&gst_vaapidownload_sink_factory);
239     gst_element_class_add_pad_template(element_class, pad_template);
240
241     /* src pad */
242     pad_template = gst_static_pad_template_get(&gst_vaapidownload_src_factory);
243     gst_element_class_add_pad_template(element_class, pad_template);
244 }
245
246 static void
247 gst_vaapidownload_init(GstVaapiDownload *download)
248 {
249     GstPad *sinkpad, *srcpad;
250
251     download->display           = NULL;
252     download->allowed_caps      = NULL;
253     download->images            = NULL;
254     download->images_reset      = FALSE;
255     download->image_format      = GST_VIDEO_FORMAT_UNKNOWN;
256     download->image_width       = 0;
257     download->image_height      = 0;
258
259     /* Override buffer allocator on sink pad */
260     sinkpad = gst_element_get_static_pad(GST_ELEMENT(download), "sink");
261     gst_pad_set_query_function(sinkpad, gst_vaapidownload_query);
262     gst_object_unref(sinkpad);
263
264     /* Override query on src pad */
265     srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
266     gst_pad_set_query_function(srcpad, gst_vaapidownload_query);
267     gst_object_unref(srcpad);
268 }
269
270 static inline gboolean
271 gst_vaapidownload_ensure_display(GstVaapiDownload *download)
272 {
273     return gst_vaapi_ensure_display(download, GST_VAAPI_DISPLAY_TYPE_ANY,
274         &download->display);
275 }
276
277 static gboolean
278 gst_vaapidownload_start(GstBaseTransform *trans)
279 {
280     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
281
282     if (!gst_vaapidownload_ensure_display(download))
283         return FALSE;
284     return TRUE;
285 }
286
287 static gboolean
288 gst_vaapidownload_stop(GstBaseTransform *trans)
289 {
290     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
291
292     gst_vaapi_display_replace(&download->display, NULL);
293     return TRUE;
294 }
295
296 static GstVideoFormat
297 get_surface_format(GstVaapiSurface *surface)
298 {
299     GstVaapiImage *image;
300     GstVideoFormat format = GST_VIDEO_FORMAT_NV12;
301
302     /* XXX: NV12 is assumed by default */
303     image = gst_vaapi_surface_derive_image(surface);
304     if (image) {
305         format = gst_vaapi_image_get_format(image);
306         gst_vaapi_object_unref(image);
307     }
308     return format;
309 }
310
311 static gboolean
312 gst_vaapidownload_update_src_caps(GstVaapiDownload *download, GstBuffer *buffer)
313 {
314     GstVaapiVideoMeta *meta;
315     GstVaapiSurface *surface;
316     GstVideoFormat format;
317     GstPad *srcpad;
318     GstCaps *in_caps, *out_caps;
319
320     meta = gst_buffer_get_vaapi_video_meta(buffer);
321     surface = gst_vaapi_video_meta_get_surface(meta);
322     if (!surface) {
323         GST_WARNING("failed to retrieve VA surface from buffer");
324         return FALSE;
325     }
326
327     format = get_surface_format(surface);
328     if (format == download->image_format)
329         return TRUE;
330
331     in_caps = GST_BUFFER_CAPS(buffer);
332     if (!in_caps) {
333         GST_WARNING("failed to retrieve caps from buffer");
334         return FALSE;
335     }
336
337     out_caps = gst_vaapi_video_format_to_caps(download->image_format);
338     if (!out_caps) {
339         GST_WARNING("failed to create caps from format %s",
340                     gst_video_format_to_string(download->image_format));
341         return FALSE;
342     }
343
344     if (!gst_vaapi_append_surface_caps(out_caps, in_caps)) {
345         gst_caps_unref(out_caps);
346         return FALSE;
347     }
348
349     /* Try to renegotiate downstream caps */
350     srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
351     gst_pad_set_caps(srcpad, out_caps);
352     gst_object_unref(srcpad);
353
354     gst_vaapidownload_set_caps(GST_BASE_TRANSFORM(download), in_caps, out_caps);
355     gst_caps_replace(&download->allowed_caps, out_caps);
356     gst_caps_unref(out_caps);
357     return TRUE;
358 }
359
360 static void
361 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer)
362 {
363     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
364
365     gst_vaapidownload_update_src_caps(download, buffer);
366 }
367
368 static GstFlowReturn
369 gst_vaapidownload_transform(
370     GstBaseTransform *trans,
371     GstBuffer        *inbuf,
372     GstBuffer        *outbuf
373 )
374 {
375     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
376     GstVaapiVideoMeta *meta;
377     GstVaapiSurface *surface;
378     GstVaapiImage *image = NULL;
379     gboolean success;
380
381     meta = gst_buffer_get_vaapi_video_meta(inbuf);
382     surface = gst_vaapi_video_meta_get_surface(meta);
383     if (!surface)
384         return GST_FLOW_UNEXPECTED;
385
386     image = gst_vaapi_video_pool_get_object(download->images);
387     if (!image)
388         return GST_FLOW_UNEXPECTED;
389     if (!gst_vaapi_surface_get_image(surface, image))
390         goto error_get_image;
391
392     success = gst_vaapi_image_get_buffer(image, outbuf, NULL);
393     gst_vaapi_video_pool_put_object(download->images, image);
394     if (!success)
395         goto error_get_buffer;
396     return GST_FLOW_OK;
397
398 error_get_image:
399     {
400         const GstVideoFormat format = gst_vaapi_image_get_format(image);
401         GST_WARNING("failed to download %s image from surface 0x%08x",
402                     gst_video_format_to_string(format),
403                     gst_vaapi_surface_get_id(surface));
404         gst_vaapi_video_pool_put_object(download->images, image);
405         return GST_FLOW_UNEXPECTED;
406     }
407
408 error_get_buffer:
409     {
410         GST_WARNING("failed to transfer image to output video buffer");
411         return GST_FLOW_UNEXPECTED;
412     }
413 }
414
415 static GstCaps *
416 gst_vaapidownload_transform_caps(
417     GstBaseTransform *trans,
418     GstPadDirection   direction,
419     GstCaps          *caps
420 )
421 {
422     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
423     GstPad *srcpad;
424     GstCaps *allowed_caps, *inter_caps, *out_caps = NULL;
425     GstStructure *structure;
426
427     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
428
429     structure = gst_caps_get_structure(caps, 0);
430
431     if (direction == GST_PAD_SINK) {
432         if (!gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
433             return NULL;
434         if (!gst_vaapidownload_ensure_display(download))
435             return NULL;
436         out_caps = gst_caps_from_string(gst_vaapidownload_yuv_caps_str);
437
438         /* Build up allowed caps */
439         /* XXX: we don't know the decoded surface format yet so we
440            expose whatever VA images we support */
441         if (download->allowed_caps)
442             allowed_caps = gst_caps_ref(download->allowed_caps);
443         else {
444             allowed_caps = gst_vaapi_display_get_image_caps(download->display);
445             if (!allowed_caps)
446                 return NULL;
447         }
448         inter_caps = gst_caps_intersect(out_caps, allowed_caps);
449         gst_caps_unref(allowed_caps);
450         gst_caps_unref(out_caps);
451         out_caps = inter_caps;
452
453         /* Intersect with allowed caps from the peer, if any */
454         srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
455         allowed_caps = gst_pad_peer_get_caps(srcpad);
456         if (allowed_caps) {
457             inter_caps = gst_caps_intersect(out_caps, allowed_caps);
458             gst_caps_unref(allowed_caps);
459             gst_caps_unref(out_caps);
460             out_caps = inter_caps;
461         }
462     }
463     else {
464         if (!gst_structure_has_name(structure, "video/x-raw-yuv"))
465             return NULL;
466         out_caps = gst_caps_from_string(gst_vaapidownload_vaapi_caps_str);
467
468         structure = gst_caps_get_structure(out_caps, 0);
469         gst_structure_set(
470             structure,
471             "type", G_TYPE_STRING, "vaapi",
472             "opengl", G_TYPE_BOOLEAN, USE_GLX,
473             NULL
474         );
475     }
476
477     if (!gst_vaapi_append_surface_caps(out_caps, caps)) {
478         gst_caps_unref(out_caps);
479         return NULL;
480     }
481     return out_caps;
482 }
483
484 static gboolean
485 gst_vaapidownload_ensure_image_pool(GstVaapiDownload *download, GstCaps *caps)
486 {
487     GstVideoInfo vi;
488     GstVideoFormat format;
489     guint width, height;
490
491     if (!gst_video_info_from_caps(&vi, caps))
492         return FALSE;
493
494     format = GST_VIDEO_INFO_FORMAT(&vi);
495     width  = GST_VIDEO_INFO_WIDTH(&vi);
496     height = GST_VIDEO_INFO_HEIGHT(&vi);
497
498     if (format != download->image_format ||
499         width  != download->image_width  ||
500         height != download->image_height) {
501         download->image_format = format;
502         download->image_width  = width;
503         download->image_height = height;
504         gst_vaapi_video_pool_replace(&download->images, NULL);
505         download->images = gst_vaapi_image_pool_new(download->display, &vi);
506         if (!download->images)
507             return FALSE;
508         download->images_reset = TRUE;
509     }
510     return TRUE;
511 }
512
513 static inline gboolean
514 gst_vaapidownload_negotiate_buffers(
515     GstVaapiDownload  *download,
516     GstCaps          *incaps,
517     GstCaps          *outcaps
518 )
519 {
520     if (!gst_vaapidownload_ensure_image_pool(download, outcaps))
521         return FALSE;
522     return TRUE;
523 }
524
525 static gboolean
526 gst_vaapidownload_set_caps(
527     GstBaseTransform *trans,
528     GstCaps          *incaps,
529     GstCaps          *outcaps
530 )
531 {
532     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
533
534     if (!gst_vaapidownload_negotiate_buffers(download, incaps, outcaps))
535         return FALSE;
536     return TRUE;
537 }
538
539 static gboolean
540 gst_vaapidownload_transform_size(
541     GstBaseTransform *trans,
542     GstPadDirection   direction,
543     GstCaps          *caps,
544     guint             size,
545     GstCaps          *othercaps,
546     guint            *othersize
547 )
548 {
549     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
550     GstStructure * const structure = gst_caps_get_structure(othercaps, 0);
551     GstVideoFormat format;
552     gint width, height;
553     guint i;
554
555     /* Lookup in cache */
556     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
557         TransformSizeCache * const tsc = &download->transform_size_cache[i];
558         if (tsc->caps && tsc->caps == othercaps) {
559             *othersize = tsc->size;
560             return TRUE;
561         }
562     }
563
564     /* Compute requested buffer size */
565     if (gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
566         *othersize = 0;
567     else {
568         if (!gst_video_format_parse_caps(othercaps, &format, &width, &height))
569             return FALSE;
570         *othersize = gst_video_format_get_size(format, width, height);
571     }
572
573     /* Update cache */
574     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
575         TransformSizeCache * const tsc = &download->transform_size_cache[i];
576         if (!tsc->caps) {
577             gst_caps_replace(&tsc->caps, othercaps);
578             tsc->size = *othersize;
579         }
580     }
581     return TRUE;
582 }
583
584 static gboolean
585 gst_vaapidownload_query(GstPad *pad, GstQuery *query)
586 {
587     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(gst_pad_get_parent_element(pad));
588     gboolean res;
589
590     GST_DEBUG("sharing display %p", download->display);
591
592     if (gst_vaapi_reply_to_query(query, download->display))
593         res = TRUE;
594     else
595         res = gst_pad_query_default(pad, query);
596
597     gst_object_unref(download);
598     return res;
599
600 }