Make vaapiupload and vaapidownload to work with new vaapisurfacememory and vaapisurfa...
[vaapi:sree-gstreamer-vaapi.git] / gst / vaapi / gstvaapidownload.c
1 /*
2  *  gstvaapidownload.c - VA-API video downloader
3  *
4  *  Copyright (C) 2010-2011 Splitted-Desktop Systems
5  *  Copyright (C) 2011-2012 Intel Corporation
6  *
7  *  This library is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU Lesser General Public License
9  *  as published by the Free Software Foundation; either version 2.1
10  *  of the License, or (at your option) any later version.
11  *
12  *  This library is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  *  Lesser General Public License for more details.
16  *
17  *  You should have received a copy of the GNU Lesser General Public
18  *  License along with this library; if not, write to the Free
19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20  *  Boston, MA 02110-1301 USA
21  */
22
23 /**
24  * SECTION:gstvaapidownload
25  * @short_description: A VA to video flow filter
26  *
27  * vaapidownload converts from VA surfaces to raw YUV pixels.
28  */
29
30 #include "gst/vaapi/sysdeps.h"
31 #include <gst/gst.h>
32 #include <gst/video/video.h>
33 #include <gst/video/videocontext.h>
34
35 #include "gstvaapidownload.h"
36 #include "gstvaapipluginutil.h"
37
38 #define GST_PLUGIN_NAME "vaapidownload"
39 #define GST_PLUGIN_DESC "A VA to video flow filter"
40
41 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapidownload);
42 #define GST_CAT_DEFAULT gst_debug_vaapidownload
43
44 /*Fixme: format is not only NV12 ?*/
45
46 /* Default templates */
47 static const char gst_vaapidownload_yuv_caps_str[] =
48     "video/x-raw, "
49     "format=(string)NV12, "
50     "width  = (int) [ 1, MAX ], "
51     "height = (int) [ 1, MAX ]; ";
52
53 static const char gst_vaapidownload_vaapi_caps_str[] =
54     "video/x-raw";
55
56 static GstStaticPadTemplate gst_vaapidownload_sink_factory =
57     GST_STATIC_PAD_TEMPLATE(
58         "sink",
59         GST_PAD_SINK,
60         GST_PAD_ALWAYS,
61         GST_STATIC_CAPS(gst_vaapidownload_vaapi_caps_str));
62
63 static GstStaticPadTemplate gst_vaapidownload_src_factory =
64     GST_STATIC_PAD_TEMPLATE(
65         "src",
66         GST_PAD_SRC,
67         GST_PAD_ALWAYS,
68         GST_STATIC_CAPS(gst_vaapidownload_yuv_caps_str));
69
70 typedef struct _TransformSizeCache TransformSizeCache;
71 struct _TransformSizeCache {
72     GstCaps            *caps;
73     guint               size;
74 };
75
76 struct _GstVaapiDownload {
77     /*< private >*/
78     GstBaseTransform    parent_instance;
79
80     GstVaapiDisplay    *display;
81     GstCaps            *allowed_caps;
82     TransformSizeCache  transform_size_cache[2];
83     GstVaapiVideoPool  *images;
84     GstVaapiImageFormat image_format;
85     guint               image_width;
86     guint               image_height;
87     unsigned int        images_reset    : 1;
88 };
89
90 struct _GstVaapiDownloadClass {
91     /*< private >*/
92     GstBaseTransformClass parent_class;
93 };
94
95 static void
96 gst_video_context_interface_init(GstVideoContextInterface *iface);
97
98 #define GstVideoContextClass GstVideoContextInterface
99 G_DEFINE_TYPE_WITH_CODE(
100     GstVaapiDownload,
101     gst_vaapidownload,
102     GST_TYPE_BASE_TRANSFORM,
103     G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
104                           gst_video_context_interface_init))
105
106 static gboolean
107 gst_vaapidownload_start(GstBaseTransform *trans);
108
109 static gboolean
110 gst_vaapidownload_stop(GstBaseTransform *trans);
111
112 #if 0
113 static void
114 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer);
115 #endif
116
117 static GstFlowReturn
118 gst_vaapidownload_transform(
119     GstBaseTransform *trans,
120     GstBuffer        *inbuf,
121     GstBuffer        *outbuf
122 );
123
124 static GstCaps *
125 gst_vaapidownload_transform_caps(
126     GstBaseTransform *trans,
127     GstPadDirection   direction,
128     GstCaps          *caps,
129     GstCaps          *filter
130 );
131
132 static gboolean
133 gst_vaapidownload_transform_size(
134     GstBaseTransform *trans,
135     GstPadDirection   direction,
136     GstCaps          *caps,
137     gsize             size,
138     GstCaps          *othercaps,
139     gsize            *othersize
140 );
141
142 static gboolean
143 gst_vaapidownload_set_caps(
144     GstBaseTransform *trans,
145     GstCaps          *incaps,
146     GstCaps          *outcaps
147 );
148
149 static gboolean
150 gst_vaapidownload_query(
151     GstPad   *pad,
152     GstObject *parent,
153     GstQuery *query
154 );
155
156 /* GstVideoContext interface */
157
158 static void
159 gst_vaapidownload_set_video_context(GstVideoContext *context, const gchar *type,
160     const GValue *value)
161 {
162   GstVaapiDownload *download = GST_VAAPIDOWNLOAD (context);
163   gst_vaapi_set_display (type, value, &download->display);
164 }
165
166 static void
167 gst_video_context_interface_init(GstVideoContextInterface *iface)
168 {
169     iface->set_context = gst_vaapidownload_set_video_context;
170 }
171
172 static void
173 gst_vaapidownload_destroy(GstVaapiDownload *download)
174 {
175     guint i;
176
177     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
178         TransformSizeCache * const tsc = &download->transform_size_cache[i];
179         if (tsc->caps) {
180             gst_caps_unref(tsc->caps);
181             tsc->caps = NULL;
182             tsc->size = 0;
183         }
184     }
185
186     if (download->allowed_caps) {
187         gst_caps_unref(download->allowed_caps);
188         download->allowed_caps = NULL;
189     }
190
191     g_clear_object(&download->images);
192     g_clear_object(&download->display);
193 }
194
195 static void
196 gst_vaapidownload_finalize(GObject *object)
197 {
198     gst_vaapidownload_destroy(GST_VAAPIDOWNLOAD(object));
199
200     G_OBJECT_CLASS(gst_vaapidownload_parent_class)->finalize(object);
201 }
202
203 static void
204 gst_vaapidownload_class_init(GstVaapiDownloadClass *klass)
205 {
206     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
207     GstBaseTransformClass * const trans_class = GST_BASE_TRANSFORM_CLASS(klass);
208     GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
209     GstPadTemplate *pad_template;
210
211     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapidownload,
212                             GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
213
214     object_class->finalize        = gst_vaapidownload_finalize;
215     trans_class->start            = gst_vaapidownload_start;
216     trans_class->stop             = gst_vaapidownload_stop;
217 #if 0
218     trans_class->before_transform = gst_vaapidownload_before_transform;
219 #endif
220     trans_class->transform        = gst_vaapidownload_transform;
221     trans_class->transform_caps   = gst_vaapidownload_transform_caps;
222     trans_class->transform_size   = gst_vaapidownload_transform_size;
223     trans_class->set_caps         = gst_vaapidownload_set_caps;
224
225     gst_element_class_set_static_metadata (element_class,
226         "VA-API colorspace converter",
227         "Filter/Converter/Video",
228         GST_PLUGIN_DESC,
229         "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
230
231     /* sink pad */
232     pad_template = gst_static_pad_template_get(&gst_vaapidownload_sink_factory);
233     gst_element_class_add_pad_template(element_class, pad_template);
234
235     /* src pad */
236     pad_template = gst_static_pad_template_get(&gst_vaapidownload_src_factory);
237     gst_element_class_add_pad_template(element_class, pad_template);
238 }
239
240 static void
241 gst_vaapidownload_init(GstVaapiDownload *download)
242 {
243     GstPad *sinkpad, *srcpad;
244
245     download->display           = NULL;
246     download->allowed_caps      = NULL;
247     download->images            = NULL;
248     download->images_reset      = FALSE;
249     download->image_format      = (GstVaapiImageFormat)0;
250     download->image_width       = 0;
251     download->image_height      = 0;
252
253     /* Override buffer allocator on sink pad */
254     sinkpad = gst_element_get_static_pad(GST_ELEMENT(download), "sink");
255     gst_pad_set_query_function(sinkpad, gst_vaapidownload_query);
256     gst_object_unref(sinkpad);
257
258     /* Override query on src pad */
259     srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
260     gst_pad_set_query_function(srcpad, gst_vaapidownload_query);
261     gst_object_unref(srcpad);
262 }
263
264 static inline gboolean
265 gst_vaapidownload_ensure_display(GstVaapiDownload *download)
266 {
267     return gst_vaapi_ensure_display(download, GST_VAAPI_DISPLAY_TYPE_ANY,
268         &download->display);
269 }
270
271 static gboolean
272 gst_vaapidownload_start(GstBaseTransform *trans)
273 {
274     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
275
276     if (!gst_vaapidownload_ensure_display(download))
277         return FALSE;
278     return TRUE;
279 }
280
281 static gboolean
282 gst_vaapidownload_stop(GstBaseTransform *trans)
283 {
284     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
285
286     g_clear_object(&download->display);
287
288     return TRUE;
289 }
290
291 static GstVaapiImageFormat
292 get_surface_format(GstVaapiSurface *surface)
293 {
294     GstVaapiImage *image;
295     GstVaapiImageFormat format = GST_VAAPI_IMAGE_NV12;
296
297     /* XXX: NV12 is assumed by default */
298     image = gst_vaapi_surface_derive_image(surface);
299     if (image) {
300         format = gst_vaapi_image_get_format(image);
301         g_object_unref(image);
302     }
303     return format;
304 }
305
306 #if 0
307 static gboolean
308 gst_vaapidownload_update_src_caps(GstVaapiDownload *download, GstBuffer *buffer)
309 {
310     GstVaapiSurface *surface;
311     GstVaapiImageFormat format;
312     GstPad *srcpad;
313     GstCaps *in_caps, *out_caps;
314     GstMapInfo info;
315   
316     gst_buffer_map (buf, &map_info, GST_MAP_READ);
317     surface = map_info.data;
318     if (!surface){
319         GST_DEBUG_OBJECT (sink, "Failed to retrieve the VA surface from buffer");
320         return FALSE;
321     }
322
323     format = get_surface_format(surface);
324     if (format == download->image_format)
325         return TRUE;
326
327     in_caps = GST_BUFFER_CAPS(buffer);
328     if (!in_caps) {
329         GST_WARNING("failed to retrieve caps from buffer");
330         return FALSE;
331     }
332
333     out_caps = gst_vaapi_image_format_get_caps(format);
334     if (!out_caps) {
335         GST_WARNING("failed to create caps from format %" GST_FOURCC_FORMAT,
336                     GST_FOURCC_ARGS(format));
337         return FALSE;
338     }
339
340     if (!gst_vaapi_append_surface_caps(out_caps, in_caps)) {
341         gst_caps_unref(out_caps);
342         return FALSE;
343     }
344
345     /* Try to renegotiate downstream caps */
346     srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
347     gst_pad_set_caps(srcpad, out_caps);
348     gst_object_unref(srcpad);
349
350     gst_vaapidownload_set_caps(GST_BASE_TRANSFORM(download), in_caps, out_caps);
351     gst_caps_replace(&download->allowed_caps, out_caps);
352     gst_caps_unref(out_caps);
353     return TRUE;
354 }
355
356 static void
357 gst_vaapidownload_before_transform(GstBaseTransform *trans, GstBuffer *buffer)
358 {
359     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
360
361     gst_vaapidownload_update_src_caps(download, buffer);
362 }
363 #endif
364 static GstFlowReturn
365 gst_vaapidownload_transform(
366     GstBaseTransform *trans,
367     GstBuffer        *inbuf,
368     GstBuffer        *outbuf
369 )
370 {
371     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
372     GstVaapiSurface *surface;
373     GstVaapiImage *image = NULL;
374     gboolean success;
375     GstBuffer *buf;
376     GstMapInfo in_info;
377     GstMapInfo out_info;
378     GstMapInfo info;
379     GstVaapiSurfaceMeta *meta;
380  
381     meta =  gst_buffer_get_meta((inbuf),GST_VAAPI_SURFACE_META_API_TYPE); 
382     surface = (GstVaapiSurface *)meta->surface;
383     if (!surface){
384         GST_DEBUG_OBJECT (download, "Failed to retrieve the VA surface from buffer");
385         return GST_FLOW_EOS;
386     }
387
388     if (gst_buffer_pool_acquire_buffer((GstBufferPool *)download->images, &buf, NULL) != GST_FLOW_OK){
389         GST_ERROR("Failed to acquire buffer");
390         buf = NULL;
391     }
392     if (buf) {
393         gst_buffer_map(buf, &info, GST_MAP_WRITE);
394         image = (GstVaapiImage *)info.data;
395     }
396     if (!image) { 
397         GST_DEBUG_OBJECT (download, "Failed to retrieve the VA image from buffer");
398         return GST_FLOW_EOS;
399     }
400
401     if (!gst_vaapi_surface_get_image(surface, image))
402         goto error_get_image;
403
404     success = gst_vaapi_image_get_buffer(image, outbuf, NULL);
405     gst_buffer_pool_release_buffer((GstBufferPool *)download->images, buf);
406     if (!success)
407         goto error_get_buffer;
408     return GST_FLOW_OK;
409
410 error_get_image:
411     {
412         GST_WARNING("failed to download %" GST_FOURCC_FORMAT " image "
413                     "from surface 0x%08x",
414                     GST_FOURCC_ARGS(gst_vaapi_image_get_format(image)),
415                     gst_vaapi_surface_get_id(surface));
416         gst_buffer_pool_release_buffer((GstBufferPool *)download->images, buf);
417         return GST_FLOW_EOS;
418     }
419
420 error_get_buffer:
421     {
422         GST_WARNING("failed to transfer image to output video buffer");
423         return GST_FLOW_EOS;
424     }
425 }
426
427 static GstCaps *
428 gst_vaapidownload_transform_caps(
429     GstBaseTransform *trans,
430     GstPadDirection   direction,
431     GstCaps          *caps,
432     GstCaps          *filter
433 )
434 {
435     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
436     GstPad *srcpad;
437     GstCaps *allowed_caps, *inter_caps, *out_caps = NULL;
438     GstStructure *structure;
439
440     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
441
442     structure = gst_caps_get_structure(caps, 0);
443
444     if (direction == GST_PAD_SINK) {
445         if (!gst_structure_has_name(structure, "video/x-raw"))
446             return NULL;
447         if (!gst_vaapidownload_ensure_display(download))
448             return NULL;
449         out_caps = gst_caps_from_string(gst_vaapidownload_yuv_caps_str);
450
451         /* Build up allowed caps */
452         /* XXX: we don't know the decoded surface format yet so we
453            expose whatever VA images we support */
454         if (download->allowed_caps)
455             allowed_caps = gst_caps_ref(download->allowed_caps);
456         else {
457             allowed_caps = gst_vaapi_display_get_image_caps(download->display);
458             if (!allowed_caps)
459                 return NULL;
460         }
461         inter_caps = gst_caps_intersect(out_caps, allowed_caps);
462         gst_caps_unref(allowed_caps);
463         gst_caps_unref(out_caps);
464         out_caps = inter_caps;
465
466         /* Intersect with allowed caps from the peer, if any */
467         srcpad = gst_element_get_static_pad(GST_ELEMENT(download), "src");
468         allowed_caps = gst_pad_peer_query_caps(srcpad, NULL);
469         if (allowed_caps) {
470             inter_caps = gst_caps_intersect(out_caps, allowed_caps);
471             gst_caps_unref(allowed_caps);
472             gst_caps_unref(out_caps);
473             out_caps = inter_caps;
474         }
475     }
476     else {
477         if (!gst_structure_has_name(structure, "video/x-raw"))
478             return NULL;
479         out_caps = gst_caps_from_string(gst_vaapidownload_vaapi_caps_str);
480
481         structure = gst_caps_get_structure(out_caps, 0);
482         gst_structure_set(
483             structure,
484             "type", G_TYPE_STRING, "vaapi",
485             "opengl", G_TYPE_BOOLEAN, USE_GLX,
486             NULL
487         );
488     }
489
490     if (!gst_vaapi_append_surface_caps(out_caps, caps)) {
491         gst_caps_unref(out_caps);
492         return NULL;
493     }
494     return out_caps;
495 }
496
497 static gboolean
498 gst_vaapidownload_ensure_image_pool(GstVaapiDownload *download, GstCaps *caps)
499 {
500     GstStructure * const structure = gst_caps_get_structure(caps, 0);
501     GstVaapiImageFormat format;
502     gint width, height;
503     GstStructure *config;
504
505     format = gst_vaapi_image_format_from_caps(caps);
506     gst_structure_get_int(structure, "width",  &width);
507     gst_structure_get_int(structure, "height", &height);
508
509     if (format != download->image_format ||
510         width  != download->image_width  ||
511         height != download->image_height) {
512         download->image_format = format;
513         download->image_width  = width;
514         download->image_height = height;
515         g_clear_object(&download->images);
516         download->images = gst_vaapi_image_pool_new(download->display, caps);
517         if (!download->images)
518             return FALSE;
519         /*Fixme: remove hard-coded min and max values*/ 
520         config = gst_buffer_pool_get_config ((GstBufferPool *)download->images);
521         gst_buffer_pool_config_set_params (config, caps, width*height, 6, 24);
522
523         gst_buffer_pool_set_config ((GstBufferPool *)download->images, config);
524         gst_buffer_pool_set_active ((GstBufferPool *)download->images, TRUE);
525
526         download->images_reset = TRUE;
527     }
528     return TRUE;
529 }
530
531 static inline gboolean
532 gst_vaapidownload_negotiate_buffers(
533     GstVaapiDownload  *download,
534     GstCaps          *incaps,
535     GstCaps          *outcaps
536 )
537 {
538     if (!gst_vaapidownload_ensure_image_pool(download, outcaps))
539         return FALSE;
540     return TRUE;
541 }
542
543 static gboolean
544 gst_vaapidownload_set_caps(
545     GstBaseTransform *trans,
546     GstCaps          *incaps,
547     GstCaps          *outcaps
548 )
549 {
550     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
551
552     if (!gst_vaapidownload_negotiate_buffers(download, incaps, outcaps))
553         return FALSE;
554     return TRUE;
555 }
556
557 static gboolean
558 gst_vaapidownload_transform_size(
559     GstBaseTransform *trans,
560     GstPadDirection   direction,
561     GstCaps          *caps,
562     gsize             size,
563     GstCaps          *othercaps,
564     gsize            *othersize
565 )
566 {
567     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
568     GstStructure * const structure = gst_caps_get_structure(othercaps, 0);
569     GstVideoFormat format;
570     GstVideoInfo info;
571     gint width, height;
572     guint i;
573
574     /* Lookup in cache */
575     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
576         TransformSizeCache * const tsc = &download->transform_size_cache[i];
577         if (tsc->caps && tsc->caps == othercaps) {
578             *othersize = tsc->size;
579             return TRUE;
580         }
581     }
582
583     /* Compute requested buffer size */
584     if (gst_structure_has_name(structure, "video/x-raw"))
585         *othersize = 0;
586     else {
587         if (!gst_video_info_from_caps(&info, othercaps))
588             return FALSE;
589
590         width      = info.width;
591         height     = info.height; 
592         *othersize = info.size;
593
594         /*if (!gst_video_format_parse_caps(othercaps, &format, &width, &height))
595             return FALSE;
596         *othersize = gst_video_format_get_size(format, width, height);*/
597     }
598
599     /* Update cache */
600     for (i = 0; i < G_N_ELEMENTS(download->transform_size_cache); i++) {
601         TransformSizeCache * const tsc = &download->transform_size_cache[i];
602         if (!tsc->caps) {
603             gst_caps_replace(&tsc->caps, othercaps);
604             tsc->size = *othersize;
605         }
606     }
607     return TRUE;
608 }
609
610 static gboolean
611 gst_vaapidownload_query(GstPad *pad, GstObject *parent, GstQuery *query)
612 {
613     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(parent);
614     gboolean res;
615
616     GST_DEBUG("sharing display %p", download->display);
617
618     if (gst_vaapi_reply_to_query(query, download->display))
619         res = TRUE;
620     else
621         res = gst_pad_query_default(pad, parent, query);
622
623     return res;
624 }