vaapi: add support for global-alpha subpictures.
[hwdecode-demos:vj-hwdecode-demos.git] / src / vaapi.c
1 /*
2  *  vaapi.c - VA API common code
3  *
4  *  hwdecode-demos (C) 2009-2010 Splitted-Desktop Systems
5  *
6  *  This program is free software; you can redistribute it and/or modify
7  *  it under the terms of the GNU General Public License as published by
8  *  the Free Software Foundation; either version 2 of the License, or
9  *  (at your option) any later version.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  *  GNU General Public License for more details.
15  *
16  *  You should have received a copy of the GNU General Public License
17  *  along with this program; if not, write to the Free Software
18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19  */
20
21 #include "sysdeps.h"
22 #include "vaapi.h"
23 #include "vaapi_compat.h"
24 #include "common.h"
25 #include "utils.h"
26 #include "x11.h"
27
28 #if USE_GLX
29 #include "glx.h"
30 #endif
31
32 #if USE_VAAPI_GLX
33 #include <va/va_glx.h>
34 #endif
35
36 #define DEBUG 1
37 #include "debug.h"
38
39
40 static VAAPIContext *vaapi_context;
41
42 static inline const char *string_of_VAImageFormat(VAImageFormat *imgfmt)
43 {
44     return string_of_FOURCC(imgfmt->fourcc);
45 }
46
47 static const char *string_of_VAProfile(VAProfile profile)
48 {
49     switch (profile) {
50 #define PROFILE(profile) \
51         case VAProfile##profile: return "VAProfile" #profile
52         PROFILE(MPEG2Simple);
53         PROFILE(MPEG2Main);
54         PROFILE(MPEG4Simple);
55         PROFILE(MPEG4AdvancedSimple);
56         PROFILE(MPEG4Main);
57 #if VA_CHECK_VERSION(0,32,0)
58         PROFILE(JPEGBaseline);
59         PROFILE(H263Baseline);
60         PROFILE(H264ConstrainedBaseline);
61 #endif
62         PROFILE(H264Baseline);
63         PROFILE(H264Main);
64         PROFILE(H264High);
65         PROFILE(VC1Simple);
66         PROFILE(VC1Main);
67         PROFILE(VC1Advanced);
68 #undef PROFILE
69     default: break;
70     }
71     return "<unknown>";
72 }
73
74 static const char *string_of_VAEntrypoint(VAEntrypoint entrypoint)
75 {
76     switch (entrypoint) {
77 #define ENTRYPOINT(entrypoint) \
78         case VAEntrypoint##entrypoint: return "VAEntrypoint" #entrypoint
79         ENTRYPOINT(VLD);
80         ENTRYPOINT(IZZ);
81         ENTRYPOINT(IDCT);
82         ENTRYPOINT(MoComp);
83         ENTRYPOINT(Deblocking);
84 #if VA_CHECK_VERSION(0,32,0)
85         ENTRYPOINT(EncSlice);
86         ENTRYPOINT(EncPicture);
87 #endif
88 #undef ENTRYPOINT
89     default: break;
90     }
91     return "<unknown>";
92 }
93
94 static const char *string_of_VADisplayAttribType(VADisplayAttribType type)
95 {
96     switch (type) {
97 #define TYPE(type) \
98         case VADisplayAttrib##type: return "VADisplayAttrib" #type
99         TYPE(Brightness);
100         TYPE(Contrast);
101         TYPE(Hue);
102         TYPE(Saturation);
103         TYPE(BackgroundColor);
104 #if !VA_CHECK_VERSION(0,34,0)
105         TYPE(DirectSurface);
106 #endif
107 #if VA_CHECK_VERSION(0,32,0)
108         TYPE(Rotation);
109 #endif
110 #undef TYPE
111     default: break;
112     }
113     return "<unknown>";
114 }
115
116 static void destroy_buffers(VADisplay display, VABufferID *buffers, unsigned int n_buffers)
117 {
118     unsigned int i;
119     for (i = 0; i < n_buffers; i++) {
120         if (buffers[i]) {
121             vaDestroyBuffer(display, buffers[i]);
122             buffers[i] = 0;
123         }
124     }
125 }
126
127 static bool
128 has_display_attribute(VADisplayAttribType type)
129 {
130     VAAPIContext * const vaapi = vaapi_get_context();
131     int i;
132
133     if (vaapi->display_attrs) {
134         for (i = 0; i < vaapi->n_display_attrs; i++) {
135             if (vaapi->display_attrs[i].type == type)
136                 return true;
137         }
138     }
139     return false;
140 }
141
142 #if 0
143 static bool
144 get_display_attribute(VADisplayAttribType type, int *pvalue)
145 {
146     VAAPIContext * const vaapi = vaapi_get_context();
147     VADisplayAttribute attr;
148     VAStatus status;
149
150     attr.type  = type;
151     attr.flags = VA_DISPLAY_ATTRIB_GETTABLE;
152     status = vaGetDisplayAttributes(vaapi->display, &attr, 1);
153     if (!vaapi_check_status(status, "vaGetDisplayAttributes()"))
154         return false;
155
156     if (pvalue)
157         *pvalue = attr.value;
158     return true;
159 }
160 #endif
161
162 static bool
163 set_display_attribute(VADisplayAttribType type, int value)
164 {
165     VAAPIContext * const vaapi = vaapi_get_context();
166     VADisplayAttribute attr;
167     VAStatus status;
168
169     attr.type  = type;
170     attr.value = value;
171     attr.flags = VA_DISPLAY_ATTRIB_SETTABLE;
172     status = vaSetDisplayAttributes(vaapi->display, &attr, 1);
173     if (!vaapi_check_status(status, "vaSetDisplayAttributes()"))
174         return false;
175     return true;
176 }
177
178 int vaapi_init(VADisplay display)
179 {
180     CommonContext * common = common_get_context();
181     VAAPIContext *vaapi;
182     int major_version, minor_version;
183     int i, num_display_attrs, max_display_attrs;
184     VADisplayAttribute *display_attrs = NULL;
185     VAStatus status;
186
187     if (vaapi_context)
188         return 0;
189
190     if (!display)
191         goto error;
192     D(bug("VA display %p\n", display));
193
194     status = vaInitialize(display, &major_version, &minor_version);
195     if (!vaapi_check_status(status, "vaInitialize()"))
196         goto error;
197     D(bug("VA API version %d.%d\n", major_version, minor_version));
198
199     max_display_attrs = vaMaxNumDisplayAttributes(display);
200     display_attrs = malloc(max_display_attrs * sizeof(display_attrs[0]));
201     if (!display_attrs)
202         goto error;
203
204     num_display_attrs = 0; /* XXX: workaround old GMA500 bug */
205     status = vaQueryDisplayAttributes(display, display_attrs, &num_display_attrs);
206     if (!vaapi_check_status(status, "vaQueryDisplayAttributes()"))
207         goto error;
208     D(bug("%d display attributes available\n", num_display_attrs));
209     for (i = 0; i < num_display_attrs; i++) {
210         VADisplayAttribute * const display_attr = &display_attrs[i];
211         D(bug("  %-32s (%s/%s) min %d max %d value 0x%x\n",
212               string_of_VADisplayAttribType(display_attr->type),
213               (display_attr->flags & VA_DISPLAY_ATTRIB_GETTABLE) ? "get" : "---",
214               (display_attr->flags & VA_DISPLAY_ATTRIB_SETTABLE) ? "set" : "---",
215               display_attr->min_value,
216               display_attr->max_value,
217               display_attr->value));
218     }
219
220     if (common->use_vaapi_background_color) {
221         VADisplayAttribute attr;
222         attr.type  = VADisplayAttribBackgroundColor;
223         attr.value = common->vaapi_background_color;
224         status = vaSetDisplayAttributes(display, &attr, 1);
225         if (!vaapi_check_status(status, "vaSetDisplayAttributes()"))
226             goto error;
227     }
228
229     if ((vaapi = calloc(1, sizeof(*vaapi))) == NULL)
230         goto error;
231     vaapi->display               = display;
232     vaapi->subpic_image.image_id = VA_INVALID_ID;
233     for (i = 0; i < ARRAY_ELEMS(vaapi->subpic_ids); i++)
234         vaapi->subpic_ids[i]     = VA_INVALID_ID;
235     vaapi->display_attrs         = display_attrs;
236     vaapi->n_display_attrs       = num_display_attrs;
237
238     vaapi_context = vaapi;
239
240     if (common->rotation != ROTATION_NONE) {
241         if (!has_display_attribute(VADisplayAttribRotation))
242             printf("VAAPI: display rotation attribute is not supported\n");
243         else {
244             int rotation;
245             switch (common->rotation) {
246             case ROTATION_NONE: rotation = VA_ROTATION_NONE; break;
247             case ROTATION_90:   rotation = VA_ROTATION_90;   break;
248             case ROTATION_180:  rotation = VA_ROTATION_180;  break;
249             case ROTATION_270:  rotation = VA_ROTATION_270;  break;
250             default:            ASSERT(0 && "unsupported rotation mode");
251             }
252             if (!set_display_attribute(VADisplayAttribRotation, rotation))
253                 return -1;
254         }
255     }
256     return 0;
257
258 error:
259     free(display_attrs);
260     return -1;
261 }
262
263 int vaapi_exit(void)
264 {
265     VAAPIContext * const vaapi = vaapi_get_context();
266     unsigned int i;
267
268     if (!vaapi)
269         return 0;
270
271 #if USE_GLX
272     if (display_type() == DISPLAY_GLX)
273         vaapi_glx_destroy_surface();
274 #endif
275
276     destroy_buffers(vaapi->display, &vaapi->pic_param_buf_id, 1);
277     destroy_buffers(vaapi->display, &vaapi->iq_matrix_buf_id, 1);
278     destroy_buffers(vaapi->display, &vaapi->bitplane_buf_id, 1);
279     destroy_buffers(vaapi->display, vaapi->slice_buf_ids, vaapi->n_slice_buf_ids);
280
281     if (vaapi->subpic_flags) {
282         free(vaapi->subpic_flags);
283         vaapi->subpic_flags = NULL;
284     }
285
286     if (vaapi->subpic_formats) {
287         free(vaapi->subpic_formats);
288         vaapi->subpic_formats = NULL;
289         vaapi->n_subpic_formats = 0;
290     }
291
292     if (vaapi->image_formats) {
293         free(vaapi->image_formats);
294         vaapi->image_formats = NULL;
295         vaapi->n_image_formats = 0;
296     }
297
298     if (vaapi->entrypoints) {
299         free(vaapi->entrypoints);
300         vaapi->entrypoints = NULL;
301         vaapi->n_entrypoints = 0;
302     }
303
304     if (vaapi->profiles) {
305         free(vaapi->profiles);
306         vaapi->profiles = NULL;
307         vaapi->n_profiles = 0;
308     }
309
310     if (vaapi->slice_params) {
311         free(vaapi->slice_params);
312         vaapi->slice_params = NULL;
313         vaapi->slice_params_alloc = 0;
314         vaapi->n_slice_params = 0;
315     }
316
317     if (vaapi->slice_buf_ids) {
318         free(vaapi->slice_buf_ids);
319         vaapi->slice_buf_ids = NULL;
320         vaapi->n_slice_buf_ids = 0;
321     }
322
323     if (vaapi->subpic_image.image_id != VA_INVALID_ID) {
324         vaDestroyImage(vaapi->display, vaapi->subpic_image.image_id);
325         vaapi->subpic_image.image_id = VA_INVALID_ID;
326     }
327
328     for (i = 0; i < ARRAY_ELEMS(vaapi->subpic_ids); i++) {
329         if (vaapi->subpic_ids[i] != VA_INVALID_ID) {
330             vaDestroySubpicture(vaapi->display, vaapi->subpic_ids[i]);
331             vaapi->subpic_ids[i] = VA_INVALID_ID;
332         }
333     }
334
335     if (vaapi->context_id) {
336         vaDestroyContext(vaapi->display, vaapi->context_id);
337         vaapi->context_id = 0;
338     }
339
340     if (vaapi->surface_id) {
341         vaDestroySurfaces(vaapi->display, &vaapi->surface_id, 1);
342         vaapi->surface_id = 0;
343     }
344
345     if (vaapi->config_id) {
346         vaDestroyConfig(vaapi->display, vaapi->config_id);
347         vaapi->config_id = 0;
348     }
349
350     if (vaapi->display_attrs) {
351         free(vaapi->display_attrs);
352         vaapi->display_attrs = NULL;
353         vaapi->n_display_attrs = 0;
354     }
355
356     if (vaapi->display) {
357         vaTerminate(vaapi->display);
358         vaapi->display = NULL;
359     }
360
361     free(vaapi_context);
362     return 0;
363 }
364
365 VAAPIContext *vaapi_get_context(void)
366 {
367     return vaapi_context;
368 }
369
370 int vaapi_check_status(VAStatus status, const char *msg)
371 {
372     if (status != VA_STATUS_SUCCESS) {
373         fprintf(stderr, "[%s] %s: %s\n", PACKAGE_NAME, msg, vaErrorStr(status));
374         return 0;
375     }
376     return 1;
377 }
378
379 static void *alloc_buffer(VAAPIContext *vaapi, int type, unsigned int size, VABufferID *buf_id)
380 {
381     VAStatus status;
382     void *data = NULL;
383
384     *buf_id = 0;
385     status = vaCreateBuffer(vaapi->display, vaapi->context_id,
386                             type, size, 1, NULL, buf_id);
387     if (!vaapi_check_status(status, "vaCreateBuffer()"))
388         return NULL;
389
390     status = vaMapBuffer(vaapi->display, *buf_id, &data);
391     if (!vaapi_check_status(status, "vaMapBuffer()"))
392         return NULL;
393
394     return data;
395 }
396
397 void *vaapi_alloc_picture(unsigned int size)
398 {
399     VAAPIContext *vaapi = vaapi_get_context();
400     if (!vaapi)
401         return NULL;
402     return alloc_buffer(vaapi, VAPictureParameterBufferType, size, &vaapi->pic_param_buf_id);
403 }
404
405 void *vaapi_alloc_iq_matrix(unsigned int size)
406 {
407     VAAPIContext *vaapi = vaapi_get_context();
408     if (!vaapi)
409         return NULL;
410     return alloc_buffer(vaapi, VAIQMatrixBufferType, size, &vaapi->iq_matrix_buf_id);
411 }
412
413 void *vaapi_alloc_bitplane(unsigned int size)
414 {
415     VAAPIContext *vaapi = vaapi_get_context();
416     if (!vaapi)
417         return NULL;
418     return alloc_buffer(vaapi, VABitPlaneBufferType, size, &vaapi->bitplane_buf_id);
419 }
420
421 static int commit_slices(VAAPIContext *vaapi)
422 {
423     VAStatus status;
424     VABufferID *slice_buf_ids;
425     VABufferID slice_param_buf_id, slice_data_buf_id;
426
427     if (vaapi->n_slice_params == 0)
428         return 0;
429
430     slice_buf_ids =
431         fast_realloc(vaapi->slice_buf_ids,
432                      &vaapi->slice_buf_ids_alloc,
433                      (vaapi->n_slice_buf_ids + 2) * sizeof(slice_buf_ids[0]));
434     if (!slice_buf_ids)
435         return -1;
436     vaapi->slice_buf_ids = slice_buf_ids;
437
438     slice_param_buf_id = 0;
439     status = vaCreateBuffer(vaapi->display, vaapi->context_id,
440                             VASliceParameterBufferType,
441                             vaapi->slice_param_size,
442                             vaapi->n_slice_params, vaapi->slice_params,
443                             &slice_param_buf_id);
444     if (!vaapi_check_status(status, "vaCreateBuffer() for slice params"))
445         return -1;
446     vaapi->n_slice_params = 0;
447
448     slice_data_buf_id = 0;
449     status = vaCreateBuffer(vaapi->display, vaapi->context_id,
450                             VASliceDataBufferType,
451                             vaapi->slice_data_size,
452                             1, (void *)vaapi->slice_data,
453                             &slice_data_buf_id);
454     if (!vaapi_check_status(status, "vaCreateBuffer() for slice data"))
455         return -1;
456     vaapi->slice_data = NULL;
457     vaapi->slice_data_size = 0;
458
459     slice_buf_ids[vaapi->n_slice_buf_ids++] = slice_param_buf_id;
460     slice_buf_ids[vaapi->n_slice_buf_ids++] = slice_data_buf_id;
461     return 0;
462 }
463
464 void *vaapi_alloc_slice(unsigned int size, const uint8_t *buf, unsigned int buf_size)
465 {
466     VAAPIContext *vaapi = vaapi_get_context();
467     uint8_t *slice_params;
468     VASliceParameterBufferBase *slice_param;
469
470     if (!vaapi)
471         return NULL;
472
473     if (vaapi->slice_param_size == 0)
474         vaapi->slice_param_size = size;
475     else if (vaapi->slice_param_size != size)
476         return NULL;
477
478     if (!vaapi->slice_data)
479         vaapi->slice_data = buf;
480     if (vaapi->slice_data + vaapi->slice_data_size != buf) {
481         if (commit_slices(vaapi) < 0)
482             return NULL;
483         vaapi->slice_data = buf;
484     }
485
486     slice_params =
487         fast_realloc(vaapi->slice_params,
488                      &vaapi->slice_params_alloc,
489                      (vaapi->n_slice_params + 1) * vaapi->slice_param_size);
490     if (!slice_params)
491         return NULL;
492     vaapi->slice_params = slice_params;
493
494     slice_param = (VASliceParameterBufferBase *)(slice_params + vaapi->n_slice_params * vaapi->slice_param_size);
495     slice_param->slice_data_size   = buf_size;
496     slice_param->slice_data_offset = vaapi->slice_data_size;
497     slice_param->slice_data_flag   = VA_SLICE_DATA_FLAG_ALL;
498
499     vaapi->n_slice_params++;
500     vaapi->slice_data_size += buf_size;
501     return slice_param;
502 }
503
504 static int has_profile(VAAPIContext *vaapi, VAProfile profile)
505 {
506     VAStatus status;
507     int i;
508
509     if (!vaapi->profiles || vaapi->n_profiles == 0) {
510         vaapi->profiles = calloc(vaMaxNumProfiles(vaapi->display), sizeof(vaapi->profiles[0]));
511
512         status = vaQueryConfigProfiles(vaapi->display,
513                                        vaapi->profiles,
514                                        &vaapi->n_profiles);
515         if (!vaapi_check_status(status, "vaQueryConfigProfiles()"))
516             return 0;
517
518         D(bug("%d profiles available\n", vaapi->n_profiles));
519         for (i = 0; i < vaapi->n_profiles; i++)
520             D(bug("  %s\n", string_of_VAProfile(vaapi->profiles[i])));
521     }
522
523     for (i = 0; i < vaapi->n_profiles; i++) {
524         if (vaapi->profiles[i] == profile)
525             return 1;
526     }
527     return 0;
528 }
529
530 static int has_entrypoint(VAAPIContext *vaapi, VAProfile profile, VAEntrypoint entrypoint)
531 {
532     VAStatus status;
533     int i;
534
535     if (!vaapi->entrypoints || vaapi->n_entrypoints == 0) {
536         vaapi->entrypoints = calloc(vaMaxNumEntrypoints(vaapi->display), sizeof(vaapi->entrypoints[0]));
537
538         status = vaQueryConfigEntrypoints(vaapi->display, profile,
539                                           vaapi->entrypoints,
540                                           &vaapi->n_entrypoints);
541         if (!vaapi_check_status(status, "vaQueryConfigEntrypoints()"))
542             return 0;
543
544         D(bug("%d entrypoints available for %s\n", vaapi->n_entrypoints,
545               string_of_VAProfile(profile)));
546         for (i = 0; i < vaapi->n_entrypoints; i++)
547             D(bug("  %s\n", string_of_VAEntrypoint(vaapi->entrypoints[i])));
548     }
549
550     for (i = 0; i < vaapi->n_entrypoints; i++) {
551         if (vaapi->entrypoints[i] == entrypoint)
552             return 1;
553     }
554     return 0;
555 }
556
557 int vaapi_init_decoder(VAProfile    profile,
558                        VAEntrypoint entrypoint,
559                        unsigned int picture_width,
560                        unsigned int picture_height)
561 {
562     VAAPIContext * const vaapi = vaapi_get_context();
563     VAConfigAttrib attrib;
564     VAConfigID config_id = 0;
565     VAContextID context_id = 0;
566     VASurfaceID surface_id = 0;
567     VAStatus status;
568
569     if (!vaapi)
570         return -1;
571
572     if (common_init_decoder(picture_width, picture_height) < 0)
573         return -1;
574
575     if (!has_profile(vaapi, profile))
576         return -1;
577     if (!has_entrypoint(vaapi, profile, entrypoint))
578         return -1;
579
580 #if USE_GLX
581     if (display_type() == DISPLAY_GLX) {
582         GLXContext * const glx = glx_get_context();
583
584         if (!glx)
585             return -1;
586
587         if (glx_init_texture(picture_width, picture_height) < 0)
588             return -1;
589
590         if (vaapi_glx_create_surface(glx->texture_target, glx->texture) < 0)
591             return -1;
592     }
593 #endif
594
595     if (vaapi->profile != profile || vaapi->entrypoint != entrypoint) {
596         if (vaapi->config_id)
597             vaDestroyConfig(vaapi->display, vaapi->config_id);
598
599         attrib.type = VAConfigAttribRTFormat;
600         status = vaGetConfigAttributes(vaapi->display, profile, entrypoint,
601                                        &attrib, 1);
602         if (!vaapi_check_status(status, "vaGetConfigAttributes()"))
603             return -1;
604         if ((attrib.value & VA_RT_FORMAT_YUV420) == 0)
605             return -1;
606
607         status = vaCreateConfig(vaapi->display, profile, entrypoint,
608                                 &attrib, 1, &config_id);
609         if (!vaapi_check_status(status, "vaCreateConfig()"))
610             return -1;
611     }
612     else
613         config_id = vaapi->config_id;
614
615     if (vaapi->picture_width != picture_width || vaapi->picture_height != picture_height) {
616         if (vaapi->surface_id)
617             vaDestroySurfaces(vaapi->display, &vaapi->surface_id, 1);
618
619         status = vaCreateSurfaces(vaapi->display, picture_width, picture_height,
620                                   VA_RT_FORMAT_YUV420, 1, &surface_id);
621         if (!vaapi_check_status(status, "vaCreateSurfaces()"))
622             return -1;
623
624         if (vaapi->context_id)
625             vaDestroyContext(vaapi->display, vaapi->context_id);
626
627         status = vaCreateContext(vaapi->display, config_id,
628                                  picture_width, picture_height,
629                                  VA_PROGRESSIVE,
630                                  &surface_id, 1,
631                                  &context_id);
632         if (!vaapi_check_status(status, "vaCreateContext()"))
633             return -1;
634     }
635     else {
636         context_id = vaapi->context_id;
637         surface_id = vaapi->surface_id;
638     }
639
640     vaapi->config_id      = config_id;
641     vaapi->context_id     = context_id;
642     vaapi->surface_id     = surface_id;
643     vaapi->profile        = profile;
644     vaapi->entrypoint     = entrypoint;
645     vaapi->picture_width  = picture_width;
646     vaapi->picture_height = picture_height;
647     return 0;
648 }
649
650 static int
651 get_image_format(
652     VAAPIContext   *vaapi,
653     uint32_t        fourcc,
654     VAImageFormat **image_format
655 )
656 {
657     VAStatus status;
658     int i;
659
660     if (image_format)
661         *image_format = NULL;
662
663     if (!vaapi->image_formats || vaapi->n_image_formats == 0) {
664         vaapi->image_formats = calloc(vaMaxNumImageFormats(vaapi->display),
665                                       sizeof(vaapi->image_formats[0]));
666         if (!vaapi->image_formats)
667             return 0;
668
669         status = vaQueryImageFormats(vaapi->display,
670                                      vaapi->image_formats,
671                                      &vaapi->n_image_formats);
672         if (!vaapi_check_status(status, "vaQueryImageFormats()"))
673             return 0;
674
675         D(bug("%d image formats\n", vaapi->n_image_formats));
676         for (i = 0; i < vaapi->n_image_formats; i++)
677             D(bug("  %s\n", string_of_VAImageFormat(&vaapi->image_formats[i])));
678     }
679
680     for (i = 0; i < vaapi->n_image_formats; i++) {
681         if (vaapi->image_formats[i].fourcc == fourcc) {
682             if (image_format)
683                 *image_format = &vaapi->image_formats[i];
684             return 1;
685         }
686     }
687     return 0;
688 }
689
690 /** Checks whether VAAPI format is RGB */
691 static int is_vaapi_rgb_format(const VAImageFormat *image_format)
692 {
693     switch (image_format->fourcc) {
694     case VA_FOURCC('A','R','G','B'):
695     case VA_FOURCC('A','B','G','R'):
696     case VA_FOURCC('B','G','R','A'):
697     case VA_FOURCC('R','G','B','A'):
698         return 1;
699     }
700     return 0;
701 }
702
703 static int bind_image(VAImage *va_image, Image *image)
704 {
705     VAAPIContext * const vaapi = vaapi_get_context();
706     VAImageFormat * const va_format = &va_image->format;
707     VAStatus status;
708     void *va_image_data;
709     unsigned int i;
710
711     if (va_image->num_planes > MAX_IMAGE_PLANES)
712         return -1;
713
714     status = vaMapBuffer(vaapi->display, va_image->buf, &va_image_data);
715     if (!vaapi_check_status(status, "vaMapBuffer()"))
716         return -1;
717
718     memset(image, 0, sizeof(*image));
719     image->format = va_format->fourcc;
720     if (is_vaapi_rgb_format(va_format)) {
721         image->format = image_rgba_format(
722             va_format->bits_per_pixel,
723             va_format->byte_order == VA_MSB_FIRST,
724             va_format->red_mask,
725             va_format->green_mask,
726             va_format->blue_mask,
727             va_format->alpha_mask
728         );
729         if (!image->format)
730             return -1;
731     }
732
733     image->width      = va_image->width;
734     image->height     = va_image->height;
735     image->num_planes = va_image->num_planes;
736     for (i = 0; i < va_image->num_planes; i++) {
737         image->pixels[i]  = (uint8_t *)va_image_data + va_image->offsets[i];
738         image->pitches[i] = va_image->pitches[i];
739     }
740     return 0;
741 }
742
743 static int release_image(VAImage *va_image)
744 {
745     VAAPIContext * const vaapi = vaapi_get_context();
746     VAStatus status;
747
748     status = vaUnmapBuffer(vaapi->display, va_image->buf);
749     if (!vaapi_check_status(status, "vaUnmapBuffer()"))
750         return -1;
751     return 0;
752 }
753
754 static const uint32_t image_formats[] = {
755     VA_FOURCC('Y','V','1','2'),
756     VA_FOURCC('N','V','1','2'),
757     VA_FOURCC('U','Y','V','Y'),
758     VA_FOURCC('Y','U','Y','V'),
759     VA_FOURCC('A','R','G','B'),
760     VA_FOURCC('A','B','G','R'),
761     VA_FOURCC('B','G','R','A'),
762     VA_FOURCC('R','G','B','A'),
763     0
764 };
765
766 /** Converts Image format to VAAPI format */
767 static uint32_t get_vaapi_format(uint32_t format)
768 {
769     uint32_t fourcc;
770
771     /* Only translate image formats we support */
772     switch (format) {
773     case IMAGE_NV12:
774     case IMAGE_YV12:
775     case IMAGE_IYUV:
776     case IMAGE_I420:
777     case IMAGE_AYUV:
778     case IMAGE_UYVY:
779     case IMAGE_YUY2:
780     case IMAGE_YUYV:
781     case IMAGE_ARGB:
782     case IMAGE_BGRA:
783     case IMAGE_RGBA:
784     case IMAGE_ABGR:
785         fourcc = format;
786         break;
787     default:
788         fourcc = 0;
789         break;
790     }
791     return fourcc;
792 }
793
794 static int get_image(VASurfaceID surface, Image *dst_img)
795 {
796     CommonContext * const common = common_get_context();
797     VAAPIContext * const vaapi = vaapi_get_context();
798     VAImage image;
799     VAImageFormat *image_format = NULL;
800     VAStatus status;
801     Image bound_image;
802     int i, is_bound_image = 0, is_derived_image = 0, error = -1;
803
804     image.image_id = VA_INVALID_ID;
805     image.buf      = VA_INVALID_ID;
806
807     if (common->getimage_format) {
808         uint32_t fourcc = get_vaapi_format(common->getimage_format);
809         if (!fourcc || !get_image_format(vaapi, fourcc, &image_format))
810             goto end;
811     }
812
813     if (!image_format && common->vaapi_derive_image) {
814         status = vaDeriveImage(vaapi->display, surface, &image);
815         if (vaapi_check_status(status, "vaDeriveImage()")) {
816             if (image.image_id != VA_INVALID_ID && image.buf != VA_INVALID_ID) {
817                 D(bug("using vaDeriveImage()\n"));
818                 is_derived_image = 1;
819                 image_format = &image.format;
820             }
821             else {
822                 D(bug("vaDeriveImage() returned success but VA image is invalid. Trying vaGetImage()\n"));
823             }
824         }
825     }
826
827     if (!image_format) {
828         for (i = 0; image_formats[i] != 0; i++) {
829             if (get_image_format(vaapi, image_formats[i], &image_format))
830                 break;
831         }
832     }
833
834     if (!image_format)
835         goto end;
836     D(bug("selected %s image format for getimage\n",
837           string_of_VAImageFormat(image_format)));
838
839     if (!is_derived_image) {
840         status = vaCreateImage(vaapi->display, image_format,
841                                vaapi->picture_width, vaapi->picture_height,
842                                &image);
843         if (!vaapi_check_status(status, "vaCreateImage()"))
844             goto end;
845         D(bug("created image with id 0x%08x and buffer id 0x%08x\n",
846               image.image_id, image.buf));
847
848         VARectangle src_rect;
849         if (common->use_getimage_rect) {
850             Rectangle * const img_rect = &common->getimage_rect;
851             if (img_rect->x < 0 || img_rect->y >= vaapi->picture_width)
852                 goto end;
853             if (img_rect->y < 0 || img_rect->y >= vaapi->picture_height)
854                 goto end;
855             if (img_rect->x + img_rect->width > vaapi->picture_width)
856                 goto end;
857             if (img_rect->y + img_rect->height > vaapi->picture_height)
858                 goto end;
859             src_rect.x      = img_rect->x;
860             src_rect.y      = img_rect->y;
861             src_rect.width  = img_rect->width;
862             src_rect.height = img_rect->height;
863         }
864         else {
865             src_rect.x      = 0;
866             src_rect.y      = 0;
867             src_rect.width  = vaapi->picture_width;
868             src_rect.height = vaapi->picture_height;
869         }
870         D(bug("src rect (%d,%d):%ux%u\n",
871               src_rect.x, src_rect.y, src_rect.width, src_rect.height));
872
873         status = vaGetImage(
874             vaapi->display, vaapi->surface_id,
875             src_rect.x, src_rect.y, src_rect.width, src_rect.height,
876             image.image_id
877         );
878         if (!vaapi_check_status(status, "vaGetImage()")) {
879             vaDestroyImage(vaapi->display, image.image_id);
880             goto end;
881         }
882     }
883
884     if (bind_image(&image, &bound_image) < 0)
885         goto end;
886     is_bound_image = 1;
887
888     if (image_convert(dst_img, &bound_image) < 0)
889         goto end;
890
891     error = 0;
892 end:
893     if (is_bound_image) {
894         if (release_image(&image) < 0)
895             error = -1;
896     }
897
898     if (image.image_id != VA_INVALID_ID) {
899         status = vaDestroyImage(vaapi->display, image.image_id);
900         if (!vaapi_check_status(status, "vaDestroyImage()"))
901             error = -1;
902     }
903     return error;
904 }
905
906 static inline int vaapi_decode_to_image(void)
907 {
908     CommonContext * const common = common_get_context();
909     VAAPIContext * const vaapi = vaapi_get_context();
910
911     return get_image(vaapi->surface_id, common->image);
912 }
913
914 static int put_image(VASurfaceID surface, Image *img)
915 {
916     CommonContext * const common = common_get_context();
917     VAAPIContext * const vaapi = vaapi_get_context();
918     VAImageFormat *va_image_format = NULL;
919     VAImage va_image;
920     VAStatus status;
921     Image bound_image;
922     int i, w, h, is_bound_image = 0, is_derived_image = 0, error = -1;
923
924     va_image.image_id = VA_INVALID_ID;
925     va_image.buf      = VA_INVALID_ID;
926
927     if (common->putimage_format) {
928         uint32_t fourcc = get_vaapi_format(common->putimage_format);
929         if (!fourcc || !get_image_format(vaapi, fourcc, &va_image_format))
930             goto end;
931     }
932
933     if (!va_image_format && common->vaapi_derive_image) {
934         status = vaDeriveImage(vaapi->display, surface, &va_image);
935         if (vaapi_check_status(status, "vaDeriveImage()")) {
936             if (va_image.image_id != VA_INVALID_ID && va_image.buf != VA_INVALID_ID) {
937                 D(bug("using vaDeriveImage()\n"));
938                 is_derived_image = 1;
939                 va_image_format = &va_image.format;
940             }
941             else {
942                 D(bug("vaDeriveImage() returned success but VA image is invalid. Trying vaPutImage()\n"));
943             }
944         }
945     }
946
947     if (!va_image_format) {
948         for (i = 0; image_formats[i] != 0; i++) {
949             if (get_image_format(vaapi, image_formats[i], &va_image_format))
950                 break;
951         }
952     }
953
954     if (!va_image_format)
955         goto end;
956     D(bug("selected %s image format for putimage in override mode\n",
957           string_of_VAImageFormat(va_image_format)));
958
959     if (!is_derived_image) {
960         if (common->vaapi_putimage_scaled) {
961             /* Let vaPutImage() scale the image, though only a very few
962                drivers support that */
963             w = img->width;
964             h = img->height;
965         }
966         else {
967             w = vaapi->picture_width;
968             h = vaapi->picture_height;
969         }
970
971         status = vaCreateImage(vaapi->display, va_image_format, w, h, &va_image);
972         if (!vaapi_check_status(status, "vaCreateImage()"))
973             goto end;
974         D(bug("created image with id 0x%08x and buffer id 0x%08x\n",
975               va_image.image_id, va_image.buf));
976     }
977
978     if (bind_image(&va_image, &bound_image) < 0)
979         goto end;
980     is_bound_image = 1;
981     if (image_convert(&bound_image, img) < 0)
982         goto end;
983     is_bound_image = 0;
984     if (release_image(&va_image) < 0)
985         goto end;
986
987     if (!is_derived_image) {
988         status = vaPutImage2(vaapi->display, surface, va_image.image_id,
989                              0, 0, va_image.width, va_image.height,
990                              0, 0, vaapi->picture_width, vaapi->picture_height);
991         if (!vaapi_check_status(status, "vaPutImage()"))
992             goto end;
993     }
994     error = 0;
995 end:
996     if (is_bound_image) {
997         if (release_image(&va_image) < 0)
998             error = -1;
999     }
1000
1001     if (va_image.image_id != VA_INVALID_ID) {
1002         status = vaDestroyImage(vaapi->display, va_image.image_id);
1003         if (!vaapi_check_status(status, "vaDestroyImage()"))
1004             error = -1;
1005     }
1006     return error;
1007 }
1008
1009 static const uint32_t subpic_formats[] = {
1010     VA_FOURCC('A','R','G','B'),
1011     VA_FOURCC('A','B','G','R'),
1012     VA_FOURCC('B','G','R','A'),
1013     VA_FOURCC('R','G','B','A'),
1014     0
1015 };
1016
1017 static int
1018 get_subpic_format(
1019     VAAPIContext   *vaapi,
1020     uint32_t        fourcc,
1021     VAImageFormat **format,
1022     unsigned int   *flags
1023 )
1024 {
1025     VAStatus status;
1026     unsigned int i;
1027
1028     if (format)
1029         *format = NULL;
1030
1031     if (flags)
1032         *flags = 0;
1033
1034     if (!vaapi->subpic_formats) {
1035         vaapi->n_subpic_formats = vaMaxNumSubpictureFormats(vaapi->display);
1036
1037         vaapi->subpic_formats = calloc(vaapi->n_subpic_formats,
1038                                        sizeof(vaapi->subpic_formats[0]));
1039         if (!vaapi->subpic_formats)
1040             return 0;
1041
1042         vaapi->subpic_flags = calloc(vaapi->n_subpic_formats,
1043                                      sizeof(vaapi->subpic_flags[0]));
1044         if (!vaapi->subpic_flags)
1045             return 0;
1046
1047         status = vaQuerySubpictureFormats(vaapi->display,
1048                                           vaapi->subpic_formats,
1049                                           vaapi->subpic_flags,
1050                                           &vaapi->n_subpic_formats);
1051         if (!vaapi_check_status(status, "vaQuerySubpictureFormats()"))
1052             return 0;
1053
1054         D(bug("%d subpicture formats\n", vaapi->n_subpic_formats));
1055         for (i = 0; i < vaapi->n_subpic_formats; i++) {
1056             VAImageFormat *const image_format = &vaapi->subpic_formats[i];
1057             D(bug("  %s, flags 0x%x\n",
1058                   string_of_VAImageFormat(image_format),
1059                   vaapi->subpic_flags[i]));
1060             if (is_vaapi_rgb_format(image_format))
1061                 D(bug("    byte-order %s, %d-bit, depth %d, r/g/b/a 0x%08x/0x%08x/0x%08x/0x%08x\n",
1062                       image_format->byte_order == VA_MSB_FIRST ? "MSB" : "LSB",
1063                       image_format->bits_per_pixel,
1064                       image_format->depth,
1065                       image_format->red_mask,
1066                       image_format->green_mask,
1067                       image_format->blue_mask,
1068                       image_format->alpha_mask));
1069         }
1070     }
1071
1072     for (i = 0; i < vaapi->n_subpic_formats; i++) {
1073         if (vaapi->subpic_formats[i].fourcc == fourcc) {
1074             if (format)
1075                 *format = &vaapi->subpic_formats[i];
1076             if (flags)
1077                 *flags = vaapi->subpic_flags[i];
1078             return 1;
1079         }
1080     }
1081     return 0;
1082 }
1083
1084 static int blend_image(VASurfaceID surface, Image *img)
1085 {
1086     CommonContext * const common = common_get_context();
1087     VAAPIContext * const vaapi = vaapi_get_context();
1088     unsigned int subpic_count, subpic_count_x, subpic_count_y, i, j;
1089     unsigned int subpic_flags = 0;
1090     VAImageFormat *subpic_format = NULL;
1091     VAStatus status;
1092     Image bound_image;
1093     int is_bound_image = 0, error = -1;
1094     Rectangle src_rect, dst_rect, srect, drect;
1095
1096     vaapi->subpic_image.image_id = VA_INVALID_ID;
1097     vaapi->subpic_image.buf      = VA_INVALID_ID;
1098
1099     if (common->putimage_format) {
1100         uint32_t fourcc = get_vaapi_format(common->putimage_format);
1101         if (!fourcc || !get_subpic_format(vaapi, fourcc, &subpic_format, &subpic_flags))
1102             goto end;
1103     }
1104
1105     if (!subpic_format) {
1106         for (i = 0; subpic_formats[i] != 0; i++) {
1107             if (get_subpic_format(vaapi, subpic_formats[i], &subpic_format, &subpic_flags))
1108                 break;
1109         }
1110     }
1111     if (!subpic_format)
1112         goto end;
1113
1114     if (common->use_vaapi_subpicture_flags) {
1115         static const struct {
1116             unsigned int flags;
1117             const char *name;
1118         } g_flags[] = {
1119             { VA_SUBPICTURE_GLOBAL_ALPHA,
1120               "VA_SUBPICTURE_GLOBAL_ALPHA" },
1121 #ifdef VA_SUBPICTURE_DESTINATION_IS_SCREEN_COORD
1122             { VA_SUBPICTURE_DESTINATION_IS_SCREEN_COORD,
1123               "VA_SUBPICTURE_DESTINATION_IS_SCREEN_COORD" },
1124 #endif
1125             { 0, }
1126         };
1127         const unsigned int flags =
1128             common->vaapi_subpicture_flags ^ subpic_flags;
1129
1130         for (i = 0; g_flags[i].flags != 0; i++) {
1131             if (flags & g_flags[i].flags) {
1132                 D(bug("driver does not support %s subpicture flag\n",
1133                       g_flags[i].name));
1134                 goto end;
1135             }
1136         }
1137     }
1138
1139     D(bug("selected %s subpicture format for putimage in blend mode\n",
1140           string_of_VAImageFormat(subpic_format)));
1141
1142     status = vaCreateImage(vaapi->display, subpic_format,
1143                            img->width, img->height, &vaapi->subpic_image);
1144     if (!vaapi_check_status(status, "vaCreateImage()"))
1145         goto end;
1146     D(bug("created image with id 0x%08x and buffer id 0x%08x\n",
1147           vaapi->subpic_image.image_id, vaapi->subpic_image.buf));
1148
1149     if (bind_image(&vaapi->subpic_image, &bound_image) < 0)
1150         goto end;
1151     is_bound_image = 1;
1152     if (image_convert(&bound_image, img) < 0)
1153         goto end;
1154     is_bound_image = 0;
1155     if (release_image(&vaapi->subpic_image) < 0)
1156         goto end;
1157
1158     subpic_count = 1;
1159     if (common->vaapi_multi_subpictures)
1160         subpic_count = ARRAY_ELEMS(vaapi->subpic_ids);
1161
1162     for (i = 0; i < subpic_count; i++) {
1163         status = vaCreateSubpicture(
1164             vaapi->display,
1165             vaapi->subpic_image.image_id,
1166             &vaapi->subpic_ids[i]
1167         );
1168         if (!vaapi_check_status(status, "vaCreateSubpicture()"))
1169             goto end;
1170         D(bug("created subpicture with id 0x%08x\n", vaapi->subpic_ids[i]));
1171
1172         if (common->use_vaapi_subpicture_flags) {
1173
1174             if (common->vaapi_subpicture_flags & VA_SUBPICTURE_GLOBAL_ALPHA) {
1175                 status = vaSetSubpictureGlobalAlpha(
1176                     vaapi->display,
1177                     vaapi->subpic_ids[i],
1178                     common->vaapi_subpicture_alpha
1179                 );
1180                 if (!vaapi_check_status(status, "vaSetSubpictureGlobalAlpha()"))
1181                     goto end;
1182             }
1183         }
1184     }
1185
1186     if (common->use_vaapi_subpicture_source_rect)
1187         src_rect = common->vaapi_subpicture_source_rect;
1188     else {
1189         src_rect.x      = 0;
1190         src_rect.y      = 0;
1191         src_rect.width  = img->width;
1192         src_rect.height = img->height;
1193     }
1194
1195     if (common->use_vaapi_subpicture_target_rect)
1196         dst_rect = common->vaapi_subpicture_target_rect;
1197     else {
1198         dst_rect.x      = 0;
1199         dst_rect.y      = 0;
1200         dst_rect.width  = vaapi->picture_width;
1201         dst_rect.height = vaapi->picture_height;
1202     }
1203
1204     D(bug("render %d subpicture%s from (%d,%d):%ux%u to (%d,%d):%ux%u\n",
1205           subpic_count, subpic_count > 1 ? "s" : "",
1206           src_rect.x, src_rect.y, src_rect.width, src_rect.height,
1207           dst_rect.x, dst_rect.y, dst_rect.width, dst_rect.height));
1208
1209     subpic_count_y = subpic_count / 2;
1210     if (subpic_count_y == 0)
1211         subpic_count_y = 1;
1212
1213     subpic_flags = 0;
1214     if (common->use_vaapi_subpicture_flags)
1215         subpic_flags = common->vaapi_subpicture_flags;
1216
1217     for (j = 0; j < subpic_count_y; j++) {
1218         subpic_count_x = subpic_count / subpic_count_y;
1219         if (j == subpic_count_y - 1)
1220             subpic_count_x += subpic_count % subpic_count_y; /* 0/1 */
1221
1222         for (i = 0; i < subpic_count_x; i++) {
1223             srect.x      = src_rect.x + (i * src_rect.width) / subpic_count_x;
1224             srect.y      = src_rect.y + (j * src_rect.height) / subpic_count_y;
1225             srect.width  = src_rect.width / subpic_count_x;
1226             srect.height = src_rect.height / subpic_count_y;
1227             drect.x      = dst_rect.x + (i * dst_rect.width) / subpic_count_x;
1228             drect.y      = dst_rect.y + (j * dst_rect.height) / subpic_count_y;
1229             drect.width  = dst_rect.width / subpic_count_x;
1230             drect.height = dst_rect.height / subpic_count_y;
1231
1232             if (i == subpic_count_x - 1) {
1233                 srect.width  = (src_rect.x + src_rect.width) - srect.x;
1234                 drect.width  = (dst_rect.x + dst_rect.width) - drect.x;
1235             }
1236
1237             if (j == subpic_count_y - 1) {
1238                 srect.height = (src_rect.y + src_rect.height) - srect.y;
1239                 drect.height = (dst_rect.y + dst_rect.height) - drect.y;
1240             }
1241
1242             if (subpic_count > 1)
1243                 D(bug("  id 0x%08x: src (%d,%d):%ux%u, dst (%d,%d):%ux%u\n",
1244                       vaapi->subpic_ids[j * subpic_count_y + i],
1245                       srect.x, srect.y, srect.width, srect.height,
1246                       drect.x, drect.y, drect.width, drect.height));
1247
1248             status = vaAssociateSubpicture2(
1249                 vaapi->display,
1250                 vaapi->subpic_ids[j * subpic_count_y + i],
1251                 &surface, 1,
1252                 srect.x, srect.y, srect.width, srect.height,
1253                 drect.x, drect.y, drect.width, drect.height,
1254                 subpic_flags
1255             );
1256             if (!vaapi_check_status(status, "vaAssociateSubpicture()"))
1257                 goto end;
1258         }
1259     }
1260     error = 0;
1261 end:
1262     if (is_bound_image) {
1263         if (release_image(&vaapi->subpic_image) < 0)
1264             error = -1;
1265     }
1266     return error;
1267 }
1268
1269 int vaapi_decode(void)
1270 {
1271     VAAPIContext * const vaapi = vaapi_get_context();
1272     VABufferID va_buffers[3];
1273     unsigned int n_va_buffers = 0;
1274     VAStatus status;
1275
1276     if (!vaapi || vaapi->context_id == 0 || vaapi->surface_id == 0)
1277         return -1;
1278
1279     if (commit_slices(vaapi) < 0)
1280         return -1;
1281
1282     vaUnmapBuffer(vaapi->display, vaapi->pic_param_buf_id);
1283     va_buffers[n_va_buffers++] = vaapi->pic_param_buf_id;
1284
1285     if (vaapi->iq_matrix_buf_id) {
1286         vaUnmapBuffer(vaapi->display, vaapi->iq_matrix_buf_id);
1287         va_buffers[n_va_buffers++] = vaapi->iq_matrix_buf_id;
1288     }
1289
1290     if (vaapi->bitplane_buf_id) {
1291         vaUnmapBuffer(vaapi->display, vaapi->bitplane_buf_id);
1292         va_buffers[n_va_buffers++] = vaapi->bitplane_buf_id;
1293     }
1294
1295     status = vaBeginPicture(vaapi->display, vaapi->context_id,
1296                             vaapi->surface_id);
1297     if (!vaapi_check_status(status, "vaBeginPicture()"))
1298         return -1;
1299
1300     status = vaRenderPicture(vaapi->display, vaapi->context_id,
1301                              va_buffers, n_va_buffers);
1302     if (!vaapi_check_status(status, "vaRenderPicture()"))
1303         return -1;
1304
1305     status = vaRenderPicture(vaapi->display, vaapi->context_id,
1306                              vaapi->slice_buf_ids,
1307                              vaapi->n_slice_buf_ids);
1308     if (!vaapi_check_status(status, "vaRenderPicture()"))
1309         return -1;
1310
1311     status = vaEndPicture(vaapi->display, vaapi->context_id);
1312     if (!vaapi_check_status(status, "vaEndPicture()"))
1313         return -1;
1314
1315     if (getimage_mode() == GETIMAGE_FROM_VIDEO)
1316         return vaapi_decode_to_image();
1317
1318     return 0;
1319 }
1320
1321 static int vaapi_display_cliprects(void)
1322 {
1323     CommonContext * const common = common_get_context();
1324     X11Context * const x11 = x11_get_context();
1325     VAAPIContext * const vaapi = vaapi_get_context();
1326     VASurfaceID clip_surface_id = VA_INVALID_ID;
1327     VARectangle *cliprects = NULL;
1328     VAStatus status;
1329     int i, error = 1;
1330
1331     if (!common->use_clipping)
1332         return 0;
1333
1334     status = vaCreateSurfaces(vaapi->display,
1335                               vaapi->picture_width,
1336                               vaapi->picture_height,
1337                               VA_RT_FORMAT_YUV420, 1, &clip_surface_id);
1338     if (!vaapi_check_status(status, "vaCreateSurfaces()"))
1339         goto end;
1340
1341     if (put_image(clip_surface_id, common->cliprects_image) < 0)
1342         goto end;
1343
1344     cliprects = malloc(common->cliprects_count * sizeof(cliprects[0]));
1345     if (!cliprects)
1346         goto end;
1347     for (i = 0; i < common->cliprects_count; i++) {
1348         cliprects[i].x      = common->cliprects[i].x;
1349         cliprects[i].y      = common->cliprects[i].y;
1350         cliprects[i].width  = common->cliprects[i].width;
1351         cliprects[i].height = common->cliprects[i].height;
1352     }
1353
1354     status = vaPutSurface(vaapi->display, clip_surface_id, x11->window,
1355                           0, 0, vaapi->picture_width, vaapi->picture_height,
1356                           0, 0, x11->window_width, x11->window_height,
1357                           cliprects, common->cliprects_count,
1358                           VA_FRAME_PICTURE);
1359     if (!vaapi_check_status(status, "vaPutSurface()"))
1360         goto end;
1361
1362     error = 0;
1363 end:
1364     if (cliprects)
1365         free(cliprects);
1366     if (clip_surface_id != VA_INVALID_ID)
1367         vaDestroySurfaces(vaapi->display, &clip_surface_id, 1);
1368     if (error)
1369         return -1;
1370     return 0;
1371 }
1372
1373 int vaapi_display(void)
1374 {
1375     CommonContext * const common = common_get_context();
1376     X11Context * const x11 = x11_get_context();
1377     VAAPIContext * const vaapi = vaapi_get_context();
1378     unsigned int vaPutSurface_count = 0;
1379     unsigned int flags = VA_FRAME_PICTURE;
1380     VAStatus status;
1381     Drawable drawable;
1382
1383     if (!common || !x11 || !vaapi)
1384         return -1;
1385
1386     if (putimage_mode() != PUTIMAGE_NONE) {
1387         Image *img;
1388         img = image_generate(
1389             common->putimage_size.width,
1390             common->putimage_size.height
1391         );
1392         if (img) {
1393             switch (putimage_mode()) {
1394             case PUTIMAGE_OVERRIDE:
1395                 if (put_image(vaapi->surface_id, img) < 0)
1396                     return -1;
1397                 break;
1398             case PUTIMAGE_BLEND:
1399                 if (blend_image(vaapi->surface_id, img) < 0)
1400                     return -1;
1401                 break;
1402             default:
1403                 break;
1404             }
1405             image_destroy(img);
1406         }
1407     }
1408
1409     /* XXX: video and output surfaces are the same for VA API */
1410     if (getimage_mode() == GETIMAGE_FROM_OUTPUT) {
1411         if (vaapi_decode_to_image() < 0)
1412             return -1;
1413         return 0;
1414     }
1415
1416     if (getimage_mode() == GETIMAGE_FROM_VIDEO)
1417         return 0;
1418
1419     if (getimage_mode() == GETIMAGE_FROM_PIXMAP)
1420         drawable = x11->pixmap;
1421     else
1422         drawable = x11->window;
1423
1424     if (common->use_vaapi_putsurface_flags)
1425         flags = common->vaapi_putsurface_flags;
1426
1427 #if USE_VAAPI_GLX
1428     if (display_type() == DISPLAY_GLX) {
1429         vaapi->use_glx_copy = common->vaapi_glx_use_copy;
1430         if (vaapi->use_glx_copy) {
1431             status = vaCopySurfaceGLX(vaapi->display,
1432                                       vaapi->glx_surface,
1433                                       vaapi->surface_id,
1434                                       flags);
1435
1436             if (status == VA_STATUS_ERROR_UNIMPLEMENTED) {
1437                 printf("VAAPI: vaCopySurfaceGLX() is not implemented\n");
1438                 vaapi->use_glx_copy = 0;
1439             }
1440             else {
1441                 printf("VAAPI: use vaCopySurfaceGLX()\n");
1442                 if (!vaapi_check_status(status, "vaCopySurfaceGLX()"))
1443                     return -1;
1444             }
1445         }
1446     }
1447     else
1448 #endif
1449     {
1450         Rectangle src_rect, dst_rect;
1451
1452         if (common->use_vaapi_putsurface_source_rect)
1453             src_rect = common->vaapi_putsurface_source_rect;
1454         else {
1455             src_rect.x      = 0;
1456             src_rect.y      = 0;
1457             src_rect.width  = vaapi->picture_width;
1458             src_rect.height = vaapi->picture_height;
1459         }
1460
1461         if (common->use_vaapi_putsurface_target_rect)
1462             dst_rect = common->vaapi_putsurface_target_rect;
1463         else {
1464             dst_rect.x      = 0;
1465             dst_rect.y      = 0;
1466             dst_rect.width  = x11->window_width;
1467             dst_rect.height = x11->window_height;
1468         }
1469
1470         if (common->use_vaapi_background_color)
1471             flags |= VA_CLEAR_DRAWABLE;
1472
1473         status = vaPutSurface(vaapi->display, vaapi->surface_id, drawable,
1474                               src_rect.x, src_rect.y,
1475                               src_rect.width, src_rect.height,
1476                               dst_rect.x, dst_rect.y,
1477                               dst_rect.width, dst_rect.height,
1478                               NULL, 0, flags);
1479         if (!vaapi_check_status(status, "vaPutSurface()"))
1480             return -1;
1481         ++vaPutSurface_count;
1482
1483         /* Clear the drawable only the first time */
1484         if (common->use_vaapi_background_color && !common->use_vaapi_putsurface_flags)
1485             flags &= ~VA_CLEAR_DRAWABLE;
1486
1487         if (common->multi_rendering) {
1488             status = vaPutSurface(vaapi->display, vaapi->surface_id, drawable,
1489                                   0, 0,
1490                                   vaapi->picture_width, vaapi->picture_height,
1491                                   x11->window_width/2, 0,
1492                                   x11->window_width/2, x11->window_height/2,
1493                                   NULL, 0, flags);
1494             if (!vaapi_check_status(status, "vaPutSurface() for multirendering"))
1495                 return -1;
1496             ++vaPutSurface_count;
1497
1498             status = vaPutSurface(vaapi->display, vaapi->surface_id, drawable,
1499                                   0, 0,
1500                                   vaapi->picture_width, vaapi->picture_height,
1501                                   0, x11->window_height/2,
1502                                   x11->window_width/2, x11->window_height/2,
1503                                   NULL, 0, flags);
1504             if (!vaapi_check_status(status, "vaPutSurface() for multirendering"))
1505                 return -1;
1506             ++vaPutSurface_count;
1507         }
1508     }
1509
1510     if (vaapi_display_cliprects() < 0)
1511         return -1;
1512
1513     if (common->use_subwindow_rect) {
1514         status = vaPutSurface(vaapi->display, vaapi->surface_id,
1515                               x11->subwindow,
1516                               0, 0, vaapi->picture_width, vaapi->picture_height,
1517                               0, 0,
1518                               common->subwindow_rect.width,
1519                               common->subwindow_rect.height,
1520                               NULL, 0, 0);
1521         if (!vaapi_check_status(status, "vaPutSurface()"))
1522             return -1;
1523         ++vaPutSurface_count;
1524     }
1525
1526     if (vaPutSurface_count > 1) {
1527         /* We don't have to call vaSyncSurface() explicitly. However,
1528            if we use multiple vaPutSurface() and subwindows, we probably
1529            want the surfaces to be presented at the same time */
1530         status = vaSyncSurface(vaapi->display, vaapi->context_id,
1531                                vaapi->surface_id);
1532         if (!vaapi_check_status(status, "vaSyncSurface() for display"))
1533             return -1;
1534     }
1535     return 0;
1536 }
1537
1538 #ifndef USE_FFMPEG
1539 int pre(void)
1540 {
1541     X11Context *x11;
1542     VADisplay dpy;
1543
1544     if (hwaccel_type() != HWACCEL_VAAPI)
1545         return -1;
1546
1547     if (x11_init() < 0)
1548         return -1;
1549     if ((x11 = x11_get_context()) == NULL)
1550         return -1;
1551 #if USE_GLX
1552     if (display_type() == DISPLAY_GLX) {
1553         if (glx_init() < 0)
1554             return -1;
1555     }
1556 #endif
1557 #if USE_VAAPI_GLX
1558     if (display_type() == DISPLAY_GLX)
1559         dpy = vaGetDisplayGLX(x11->display);
1560     else
1561 #endif
1562         dpy = vaGetDisplay(x11->display);
1563     return vaapi_init(dpy);
1564 }
1565
1566 int post(void)
1567 {
1568     if (vaapi_exit() < 0)
1569         return -1;
1570 #if USE_GLX
1571     if (display_type() == DISPLAY_GLX) {
1572         if (glx_exit() < 0)
1573             return -1;
1574     }
1575 #endif
1576     return x11_init();
1577 }
1578
1579 int display(void)
1580 {
1581     if (vaapi_display() < 0)
1582         return -1;
1583 #if USE_GLX
1584     if (display_type() == DISPLAY_GLX)
1585         return glx_display();
1586 #endif
1587     return x11_display();
1588 }
1589 #endif
1590
1591 int vaapi_glx_create_surface(unsigned int target, unsigned int texture)
1592 {
1593 #if USE_VAAPI_GLX
1594     VAAPIContext * const vaapi = vaapi_get_context();
1595     VAStatus status;
1596
1597     status = vaCreateSurfaceGLX(vaapi->display, target, texture,
1598                                 &vaapi->glx_surface);
1599     if (vaapi_check_status(status, "vaCreateSurfaceGLX()"))
1600         return 0;
1601 #endif
1602     return -1;
1603 }
1604
1605 void vaapi_glx_destroy_surface(void)
1606 {
1607 #if USE_VAAPI_GLX
1608     VAAPIContext * const vaapi = vaapi_get_context();
1609     VAStatus status;
1610
1611     status = vaDestroySurfaceGLX(vaapi->display, vaapi->glx_surface);
1612     if (vaapi_check_status(status, "vaDestroySurfaceGLX()"))
1613         return;
1614 #endif
1615 }
1616
1617 int vaapi_glx_begin_render_surface(void)
1618 {
1619 #if USE_VAAPI_GLX
1620     VAAPIContext * const vaapi = vaapi_get_context();
1621
1622     if (vaapi->use_glx_copy)
1623         return 0;
1624
1625     printf("VAAPI: use vaBeginRenderSurfaceGLX()\n");
1626
1627 #if HAS_VAAPI_GLX_BIND
1628     VAStatus status = vaAssociateSurfaceGLX(vaapi->display,
1629                                             vaapi->glx_surface,
1630                                             vaapi->surface_id,
1631                                             VA_FRAME_PICTURE);
1632     if (!vaapi_check_status(status, "vaAssociateSurfaceGLX()"))
1633         return -1;
1634
1635     status = vaBeginRenderSurfaceGLX(vaapi->display, vaapi->glx_surface);
1636     if (!vaapi_check_status(status, "vaBeginRenderSurfaceGLX()"))
1637         return -1;
1638     return 0;
1639 #endif
1640     fprintf(stderr, "ERROR: not implemented\n");
1641 #endif
1642     return -1;
1643 }
1644
1645 int vaapi_glx_end_render_surface(void)
1646 {
1647 #if USE_VAAPI_GLX
1648     VAAPIContext * const vaapi = vaapi_get_context();
1649
1650     if (vaapi->use_glx_copy)
1651         return 0;
1652
1653 #if HAS_VAAPI_GLX_BIND
1654     VAStatus status = vaEndRenderSurfaceGLX(vaapi->display, vaapi->glx_surface);
1655     if (!vaapi_check_status(status, "vaEndRenderSurfaceGLX()"))
1656         return -1;
1657
1658     status = vaDeassociateSurfaceGLX(vaapi->display, vaapi->glx_surface);
1659     if (!vaapi_check_status(status, "vaDeassociateSurfaceGLX()"))
1660         return -1;
1661     return 0;
1662 #endif
1663     fprintf(stderr, "ERROR: not implemented\n");
1664 #endif
1665     return -1;
1666 }