vda_h264.c
Go to the documentation of this file.
1 /*
2  * VDA H.264 hardware acceleration
3  *
4  * copyright (c) 2011 Sebastien Zwickert
5  *
6  * This file is part of Libav.
7  *
8  * Libav is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2.1 of the License, or (at your option) any later version.
12  *
13  * Libav is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with Libav; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21  */
22 
23 #include <CoreFoundation/CFNumber.h>
24 #include <CoreFoundation/CFData.h>
25 #include <CoreFoundation/CFString.h>
26 
27 #include "libavutil/avutil.h"
28 #include "h264.h"
29 #include "vda.h"
30 
31 #if FF_API_VDA_ASYNC
32 #include <CoreFoundation/CFDictionary.h>
33 
34 /* helper to create a dictionary according to the given pts */
35 static CFDictionaryRef vda_dictionary_with_pts(int64_t i_pts)
36 {
37  CFStringRef key = CFSTR("FF_VDA_DECODER_PTS_KEY");
38  CFNumberRef value = CFNumberCreate(kCFAllocatorDefault,
39  kCFNumberSInt64Type, &i_pts);
40  CFDictionaryRef user_info = CFDictionaryCreate(kCFAllocatorDefault,
41  (const void **)&key,
42  (const void **)&value,
43  1,
44  &kCFTypeDictionaryKeyCallBacks,
45  &kCFTypeDictionaryValueCallBacks);
46  CFRelease(value);
47  return user_info;
48 }
49 
50 /* helper to retrieve the pts from the given dictionary */
51 static int64_t vda_pts_from_dictionary(CFDictionaryRef user_info)
52 {
53  CFNumberRef pts;
54  int64_t outValue = 0;
55 
56  if (!user_info)
57  return 0;
58 
59  pts = CFDictionaryGetValue(user_info, CFSTR("FF_VDA_DECODER_PTS_KEY"));
60 
61  if (pts)
62  CFNumberGetValue(pts, kCFNumberSInt64Type, &outValue);
63 
64  return outValue;
65 }
66 
67 /* Remove and release all frames from the queue. */
68 static void vda_clear_queue(struct vda_context *vda_ctx)
69 {
70  vda_frame *top_frame;
71 
72  pthread_mutex_lock(&vda_ctx->queue_mutex);
73 
74  while (vda_ctx->queue) {
75  top_frame = vda_ctx->queue;
76  vda_ctx->queue = top_frame->next_frame;
77  ff_vda_release_vda_frame(top_frame);
78  }
79 
80  pthread_mutex_unlock(&vda_ctx->queue_mutex);
81 }
82 
83 static int vda_decoder_decode(struct vda_context *vda_ctx,
84  uint8_t *bitstream,
85  int bitstream_size,
86  int64_t frame_pts)
87 {
88  OSStatus status = kVDADecoderNoErr;
89  CFDictionaryRef user_info;
90  CFDataRef coded_frame;
91 
92  coded_frame = CFDataCreate(kCFAllocatorDefault, bitstream, bitstream_size);
93  user_info = vda_dictionary_with_pts(frame_pts);
94  status = VDADecoderDecode(vda_ctx->decoder, 0, coded_frame, user_info);
95 
96  CFRelease(user_info);
97  CFRelease(coded_frame);
98 
99  return status;
100 }
101 
102 vda_frame *ff_vda_queue_pop(struct vda_context *vda_ctx)
103 {
104  vda_frame *top_frame;
105 
106  if (!vda_ctx->queue)
107  return NULL;
108 
109  pthread_mutex_lock(&vda_ctx->queue_mutex);
110  top_frame = vda_ctx->queue;
111  vda_ctx->queue = top_frame->next_frame;
112  pthread_mutex_unlock(&vda_ctx->queue_mutex);
113 
114  return top_frame;
115 }
116 
117 void ff_vda_release_vda_frame(vda_frame *frame)
118 {
119  if (frame) {
120  CVPixelBufferRelease(frame->cv_buffer);
121  av_freep(&frame);
122  }
123 }
124 #endif
125 
126 /* Decoder callback that adds the VDA frame to the queue in display order. */
127 static void vda_decoder_callback(void *vda_hw_ctx,
128  CFDictionaryRef user_info,
129  OSStatus status,
130  uint32_t infoFlags,
131  CVImageBufferRef image_buffer)
132 {
133  struct vda_context *vda_ctx = vda_hw_ctx;
134 
135  if (!image_buffer)
136  return;
137 
138  if (vda_ctx->cv_pix_fmt_type != CVPixelBufferGetPixelFormatType(image_buffer))
139  return;
140 
141  if (vda_ctx->use_sync_decoding) {
142  vda_ctx->cv_buffer = CVPixelBufferRetain(image_buffer);
143  } else {
144  vda_frame *new_frame;
145  vda_frame *queue_walker;
146 
147  if (!(new_frame = av_mallocz(sizeof(*new_frame))))
148  return;
149  new_frame->next_frame = NULL;
150  new_frame->cv_buffer = CVPixelBufferRetain(image_buffer);
151  new_frame->pts = vda_pts_from_dictionary(user_info);
152 
153  pthread_mutex_lock(&vda_ctx->queue_mutex);
154 
155  queue_walker = vda_ctx->queue;
156 
157  if (!queue_walker || new_frame->pts < queue_walker->pts) {
158  /* we have an empty queue, or this frame earlier than the current queue head */
159  new_frame->next_frame = queue_walker;
160  vda_ctx->queue = new_frame;
161  } else {
162  /* walk the queue and insert this frame where it belongs in display order */
163  vda_frame *next_frame;
164  while (1) {
165  next_frame = queue_walker->next_frame;
166  if (!next_frame || new_frame->pts < next_frame->pts) {
167  new_frame->next_frame = next_frame;
168  queue_walker->next_frame = new_frame;
169  break;
170  }
171  queue_walker = next_frame;
172  }
173  }
174 
175  pthread_mutex_unlock(&vda_ctx->queue_mutex);
176  }
177 }
178 
179 static int vda_sync_decode(struct vda_context *vda_ctx)
180 {
181  OSStatus status;
182  CFDataRef coded_frame;
183  uint32_t flush_flags = 1 << 0;
184 
185  coded_frame = CFDataCreate(kCFAllocatorDefault,
186  vda_ctx->priv_bitstream,
187  vda_ctx->priv_bitstream_size);
188 
189  status = VDADecoderDecode(vda_ctx->decoder, 0, coded_frame, NULL);
190 
191  if (kVDADecoderNoErr == status)
192  status = VDADecoderFlush(vda_ctx->decoder, flush_flags);
193 
194  CFRelease(coded_frame);
195 
196  return status;
197 }
198 
199 
200 static int start_frame(AVCodecContext *avctx,
201  av_unused const uint8_t *buffer,
202  av_unused uint32_t size)
203 {
204  struct vda_context *vda_ctx = avctx->hwaccel_context;
205 
206  if (!vda_ctx->decoder)
207  return -1;
208 
209  vda_ctx->priv_bitstream_size = 0;
210 
211  return 0;
212 }
213 
214 static int decode_slice(AVCodecContext *avctx,
215  const uint8_t *buffer,
216  uint32_t size)
217 {
218  struct vda_context *vda_ctx = avctx->hwaccel_context;
219  void *tmp;
220 
221  if (!vda_ctx->decoder)
222  return -1;
223 
224  tmp = av_fast_realloc(vda_ctx->priv_bitstream,
225  &vda_ctx->priv_allocated_size,
226  vda_ctx->priv_bitstream_size + size + 4);
227  if (!tmp)
228  return AVERROR(ENOMEM);
229 
230  vda_ctx->priv_bitstream = tmp;
231 
232  AV_WB32(vda_ctx->priv_bitstream + vda_ctx->priv_bitstream_size, size);
233  memcpy(vda_ctx->priv_bitstream + vda_ctx->priv_bitstream_size + 4, buffer, size);
234 
235  vda_ctx->priv_bitstream_size += size + 4;
236 
237  return 0;
238 }
239 
240 static int end_frame(AVCodecContext *avctx)
241 {
242  H264Context *h = avctx->priv_data;
243  struct vda_context *vda_ctx = avctx->hwaccel_context;
244  AVFrame *frame = &h->s.current_picture_ptr->f;
245  int status;
246 
247  if (!vda_ctx->decoder || !vda_ctx->priv_bitstream)
248  return -1;
249 
250  if (vda_ctx->use_sync_decoding) {
251  status = vda_sync_decode(vda_ctx);
252  frame->data[3] = (void*)vda_ctx->cv_buffer;
253  } else {
254  status = vda_decoder_decode(vda_ctx, vda_ctx->priv_bitstream,
255  vda_ctx->priv_bitstream_size,
256  frame->reordered_opaque);
257  }
258 
259  if (status)
260  av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
261 
262  return status;
263 }
264 
265 int ff_vda_create_decoder(struct vda_context *vda_ctx,
266  uint8_t *extradata,
267  int extradata_size)
268 {
269  OSStatus status = kVDADecoderNoErr;
270  CFNumberRef height;
271  CFNumberRef width;
272  CFNumberRef format;
273  CFDataRef avc_data;
274  CFMutableDictionaryRef config_info;
275  CFMutableDictionaryRef buffer_attributes;
276  CFMutableDictionaryRef io_surface_properties;
277  CFNumberRef cv_pix_fmt;
278 
279 #if FF_API_VDA_ASYNC
280  pthread_mutex_init(&vda_ctx->queue_mutex, NULL);
281 #endif
282 
283  /* Each VCL NAL in the bistream sent to the decoder
284  * is preceded by a 4 bytes length header.
285  * Change the avcC atom header if needed, to signal headers of 4 bytes. */
286  if (extradata_size >= 4 && (extradata[4] & 0x03) != 0x03) {
287  uint8_t *rw_extradata;
288 
289  if (!(rw_extradata = av_malloc(extradata_size)))
290  return AVERROR(ENOMEM);
291 
292  memcpy(rw_extradata, extradata, extradata_size);
293 
294  rw_extradata[4] |= 0x03;
295 
296  avc_data = CFDataCreate(kCFAllocatorDefault, rw_extradata, extradata_size);
297 
298  av_freep(&rw_extradata);
299  } else {
300  avc_data = CFDataCreate(kCFAllocatorDefault, extradata, extradata_size);
301  }
302 
303  config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
304  4,
305  &kCFTypeDictionaryKeyCallBacks,
306  &kCFTypeDictionaryValueCallBacks);
307 
308  height = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->height);
309  width = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->width);
310  format = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->format);
311 
312  CFDictionarySetValue(config_info, kVDADecoderConfiguration_Height, height);
313  CFDictionarySetValue(config_info, kVDADecoderConfiguration_Width, width);
314  CFDictionarySetValue(config_info, kVDADecoderConfiguration_SourceFormat, format);
315  CFDictionarySetValue(config_info, kVDADecoderConfiguration_avcCData, avc_data);
316 
317  buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
318  2,
319  &kCFTypeDictionaryKeyCallBacks,
320  &kCFTypeDictionaryValueCallBacks);
321  io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
322  0,
323  &kCFTypeDictionaryKeyCallBacks,
324  &kCFTypeDictionaryValueCallBacks);
325  cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault,
326  kCFNumberSInt32Type,
327  &vda_ctx->cv_pix_fmt_type);
328  CFDictionarySetValue(buffer_attributes,
329  kCVPixelBufferPixelFormatTypeKey,
330  cv_pix_fmt);
331  CFDictionarySetValue(buffer_attributes,
332  kCVPixelBufferIOSurfacePropertiesKey,
333  io_surface_properties);
334 
335  status = VDADecoderCreate(config_info,
336  buffer_attributes,
338  vda_ctx,
339  &vda_ctx->decoder);
340 
341  CFRelease(height);
342  CFRelease(width);
343  CFRelease(format);
344  CFRelease(avc_data);
345  CFRelease(config_info);
346  CFRelease(io_surface_properties);
347  CFRelease(cv_pix_fmt);
348  CFRelease(buffer_attributes);
349 
350  return status;
351 }
352 
354 {
355  OSStatus status = kVDADecoderNoErr;
356 
357  if (vda_ctx->decoder)
358  status = VDADecoderDestroy(vda_ctx->decoder);
359 
360 #if FF_API_VDA_ASYNC
361  vda_clear_queue(vda_ctx);
362  pthread_mutex_destroy(&vda_ctx->queue_mutex);
363 #endif
364 
365  av_freep(&vda_ctx->priv_bitstream);
366 
367  return status;
368 }
369 
371  .name = "h264_vda",
372  .type = AVMEDIA_TYPE_VIDEO,
373  .id = AV_CODEC_ID_H264,
374  .pix_fmt = AV_PIX_FMT_VDA_VLD,
375  .start_frame = start_frame,
376  .decode_slice = decode_slice,
377  .end_frame = end_frame,
378 };
void * av_malloc(size_t size)
Allocate a block of size bytes with alignment suitable for all memory accesses (including vectors if ...
Definition: mem.c:61
int size
This structure describes decoded (raw) audio or video data.
Definition: avcodec.h:989
static int vda_sync_decode(struct vda_context *vda_ctx)
Definition: vda_h264.c:179
external API header
H264Context.
Definition: h264.h:254
int format
The frame format.
Definition: vda.h:163
void av_freep(void *arg)
Free a memory block which has been allocated with av_malloc(z)() or av_realloc() and set the pointer ...
Definition: mem.c:151
OSType cv_pix_fmt_type
The pixel format for output image buffers.
Definition: vda.h:171
uint8_t
void * hwaccel_context
Hardware accelerator context.
Definition: avcodec.h:2634
#define AV_WB32(p, d)
Definition: intreadwrite.h:239
int width
The frame width.
Definition: vda.h:147
CVPixelBufferRef cv_buffer
The Core Video pixel buffer that contains the current image data.
Definition: vda.h:109
H.264 / AVC / MPEG4 part10 codec.
void * av_fast_realloc(void *ptr, unsigned int *size, size_t min_size)
Reallocate the given block if it is not large enough, otherwise do nothing.
Definition: utils.c:53
VDADecoder decoder
VDA decoder object.
Definition: vda.h:101
static int pthread_mutex_init(pthread_mutex_t *m, void *attr)
Definition: w32pthreads.h:97
static int pthread_mutex_unlock(pthread_mutex_t *m)
Definition: w32pthreads.h:112
void av_log(void *avcl, int level, const char *fmt,...)
Definition: log.c:146
static int decode_slice(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size)
Definition: vda_h264.c:214
This structure is used to provide the necessary configurations and data to the VDA Libav HWAccel impl...
Definition: vda.h:94
static int pthread_mutex_destroy(pthread_mutex_t *m)
Definition: w32pthreads.h:102
const char * name
Name of the hardware accelerated codec.
Definition: avcodec.h:3061
Picture * current_picture_ptr
pointer to the current picture
Definition: mpegvideo.h:317
static char buffer[20]
Definition: seek-test.c:31
NULL
Definition: eval.c:52
static int pthread_mutex_lock(pthread_mutex_t *m)
Definition: w32pthreads.h:107
static int width
Definition: utils.c:156
MpegEncContext s
Definition: h264.h:255
AVHWAccel.
Definition: avcodec.h:3055
Public libavcodec VDA header.
int ff_vda_destroy_decoder(struct vda_context *vda_ctx)
Destroy the video decoder.
Definition: vda_h264.c:353
main external API structure.
Definition: avcodec.h:1339
static void vda_decoder_callback(void *vda_hw_ctx, CFDictionaryRef user_info, OSStatus status, uint32_t infoFlags, CVImageBufferRef image_buffer)
Definition: vda_h264.c:127
int64_t reordered_opaque
reordered opaque 64bit (generally an integer or a double precision float PTS but can be anything)...
Definition: avcodec.h:1273
AVHWAccel ff_h264_vda_hwaccel
Definition: vda_h264.c:370
int use_sync_decoding
Use the hardware decoder in synchronous mode.
Definition: vda.h:117
uint8_t * priv_bitstream
The current bitstream buffer.
Definition: vda.h:176
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: avcodec.h:997
int height
Definition: gxfenc.c:72
hardware decoding through VDA
Definition: pixfmt.h:153
void * priv_data
Definition: avcodec.h:1382
int ff_vda_create_decoder(struct vda_context *vda_ctx, uint8_t *extradata, int extradata_size)
Create the video decoder.
Definition: vda_h264.c:265
struct AVFrame f
Definition: mpegvideo.h:95
int priv_allocated_size
The reference size used for fast reallocation.
Definition: vda.h:186
static int end_frame(AVCodecContext *avctx)
Definition: vda_h264.c:240
static int start_frame(AVCodecContext *avctx, av_unused const uint8_t *buffer, av_unused uint32_t size)
Definition: vda_h264.c:200
int height
The frame height.
Definition: vda.h:155
void * av_mallocz(size_t size)
Allocate a block of size bytes with alignment suitable for all memory accesses (including vectors if ...
Definition: mem.c:158
int priv_bitstream_size
The current size of the bitstream.
Definition: vda.h:181