2 * producer_avformat.c -- avformat producer
3 * Copyright (C) 2003-2004 Ushodaya Enterprises Limited
4 * Author: Charles Yates <charles.yates@pandora.be>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software Foundation,
18 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 #include "producer_avformat.h"
25 #include <framework/mlt_frame.h>
27 // ffmpeg Header files
28 #include <ffmpeg/avformat.h>
30 // System header files
34 // Forward references.
35 static int producer_open( mlt_producer
this, char *file
);
36 static int producer_get_frame( mlt_producer
this, mlt_frame_ptr frame
, int index
);
38 // A static flag used to determine if avformat has been initialised
39 static int avformat_initialised
= 0;
41 /** Constructor for libavformat.
44 mlt_producer
producer_avformat_init( char *file
)
46 mlt_producer
this = NULL
;
48 // Check that we have a non-NULL argument
51 // Construct the producer
52 this = calloc( 1, sizeof( struct mlt_producer_s
) );
55 if ( mlt_producer_init( this, NULL
) == 0 )
58 mlt_properties properties
= mlt_producer_properties( this );
60 // Set the resource property (required for all producers)
61 mlt_properties_set( properties
, "resource", file
);
63 // TEST: audio sync tweaking
64 mlt_properties_set_double( properties
, "discrepancy", 1 );
66 // Register our get_frame implementation
67 this->get_frame
= producer_get_frame
;
69 // Initialise avformat if necessary
70 if ( avformat_initialised
== 0 )
72 avformat_initialised
= 1;
77 if ( producer_open( this, file
) != 0 )
80 mlt_producer_close( this );
89 /** Find the default streams.
92 void find_default_streams( AVFormatContext
*context
, int *audio_index
, int *video_index
)
96 // Allow for multiple audio and video streams in the file and select first of each (if available)
97 for( i
= 0; i
< context
->nb_streams
; i
++ )
99 // Get the codec context
100 AVCodecContext
*codec_context
= &context
->streams
[ i
]->codec
;
102 // Determine the type and obtain the first index of each type
103 switch( codec_context
->codec_type
)
105 case CODEC_TYPE_VIDEO
:
106 if ( *video_index
< 0 )
109 case CODEC_TYPE_AUDIO
:
110 if ( *audio_index
< 0 )
121 NOTE: We need to have a valid [PAL or NTSC] frame rate before we can determine the
122 number of frames in the file. However, this is at odds with the way things work - the
123 constructor needs to provide in/out points before the user of the producer is able
124 to specify properties :-/. However, the PAL/NTSC distinction applies to all producers
125 and while we currently accept whatever the producer provides, this will not work in
126 the more general case. Plans are afoot... and this one will work without modification
127 (in theory anyway ;-)).
130 static int producer_open( mlt_producer
this, char *file
)
132 // Return an error code (0 == no error)
135 // Context for avformat
136 AVFormatContext
*context
= NULL
;
138 // Get the properties
139 mlt_properties properties
= mlt_producer_properties( this );
141 // We will treat everything with the producer fps
142 double fps
= mlt_properties_get_double( properties
, "fps" );
144 // Now attempt to open the file
145 error
= av_open_input_file( &context
, file
, NULL
, 0, NULL
) < 0;
147 // If successful, then try to get additional info
150 // Get the stream info
151 error
= av_find_stream_info( context
) < 0;
153 // Continue if no error
156 // We will default to the first audio and video streams found
157 int audio_index
= -1;
158 int video_index
= -1;
160 // Now set properties where we can (use default unknowns if required)
161 if ( context
->duration
!= AV_NOPTS_VALUE
)
163 // This isn't going to be accurate for all formats
164 mlt_position frames
= ( mlt_position
)( ( ( double )context
->duration
/ ( double )AV_TIME_BASE
) * fps
);
165 mlt_properties_set_position( properties
, "out", frames
- 1 );
166 mlt_properties_set_position( properties
, "length", frames
);
169 // Find default audio and video streams
170 find_default_streams( context
, &audio_index
, &video_index
);
172 // Store selected audio and video indexes on properties
173 mlt_properties_set_int( properties
, "audio_index", audio_index
);
174 mlt_properties_set_int( properties
, "video_index", video_index
);
176 // We're going to cheat here - for a/v files, we will have two contexts (reasoning will be clear later)
177 if ( audio_index
!= -1 && video_index
!= -1 )
179 // We'll use the open one as our video_context
180 mlt_properties_set_data( properties
, "video_context", context
, 0, ( mlt_destructor
)av_close_input_file
, NULL
);
182 // And open again for our audio context
183 av_open_input_file( &context
, file
, NULL
, 0, NULL
);
184 av_find_stream_info( context
);
187 mlt_properties_set_data( properties
, "audio_context", context
, 0, ( mlt_destructor
)av_close_input_file
, NULL
);
189 else if ( video_index
!= -1 )
191 // We only have a video context
192 mlt_properties_set_data( properties
, "video_context", context
, 0, ( mlt_destructor
)av_close_input_file
, NULL
);
194 else if ( audio_index
!= -1 )
196 // We only have an audio context
197 mlt_properties_set_data( properties
, "audio_context", context
, 0, ( mlt_destructor
)av_close_input_file
, NULL
);
201 // Something has gone wrong
210 /** Convert a frame position to a time code.
213 static double producer_time_of_frame( mlt_producer
this, mlt_position position
)
215 // Get the properties
216 mlt_properties properties
= mlt_producer_properties( this );
219 double fps
= mlt_properties_get_double( properties
, "fps" );
222 return ( double )position
/ fps
;
225 /** Get an image from a frame.
228 static int producer_get_image( mlt_frame frame
, uint8_t **buffer
, mlt_image_format
*format
, int *width
, int *height
, int writable
)
230 // Get the properties from the frame
231 mlt_properties frame_properties
= mlt_frame_properties( frame
);
233 // Obtain the frame number of this frame
234 mlt_position position
= mlt_properties_get_position( frame_properties
, "avformat_position" );
237 mlt_producer
this = mlt_properties_get_data( frame_properties
, "avformat_producer", NULL
);
239 // Get the producer properties
240 mlt_properties properties
= mlt_producer_properties( this );
242 // Fetch the video_context
243 AVFormatContext
*context
= mlt_properties_get_data( properties
, "video_context", NULL
);
245 // Get the video_index
246 int index
= mlt_properties_get_int( properties
, "video_index" );
248 // Obtain the expected frame numer
249 mlt_position expected
= mlt_properties_get_position( properties
, "video_expected" );
251 // Calculate the real time code
252 double real_timecode
= producer_time_of_frame( this, position
);
254 // Get the video stream
255 AVStream
*stream
= context
->streams
[ index
];
258 AVCodecContext
*codec_context
= &stream
->codec
;
263 // Get the conversion frame
264 AVPicture
*output
= mlt_properties_get_data( properties
, "video_output_frame", NULL
);
266 // Special case pause handling flag
269 // Special case ffwd handling
272 // Current time calcs
273 double current_time
= 0;
275 // Set the result arguments that we know here (only *buffer is now required)
276 *format
= mlt_image_yuv422
;
277 *width
= codec_context
->width
;
278 *height
= codec_context
->height
;
280 // Set this on the frame properties
281 mlt_properties_set_int( frame_properties
, "width", *width
);
282 mlt_properties_set_int( frame_properties
, "height", *height
);
284 // Construct an AVFrame for YUV422 conversion
285 if ( output
== NULL
)
287 int size
= avpicture_get_size( PIX_FMT_YUV422
, *width
, *height
);
288 uint8_t *buf
= malloc( size
);
289 output
= malloc( sizeof( AVPicture
) );
290 avpicture_fill( output
, buf
, PIX_FMT_YUV422
, *width
, *height
);
291 mlt_properties_set_data( properties
, "video_output_frame", output
, 0, av_free
, NULL
);
292 mlt_properties_set_data( properties
, "video_output_buffer", buf
, 0, free
, NULL
);
296 if ( position
!= expected
)
298 if ( position
+ 1 == expected
)
300 // We're paused - use last image
303 else if ( position
> expected
&& ( position
- expected
) < 250 )
305 // Fast forward - seeking is inefficient for small distances - just ignore following frames
306 ignore
= position
- expected
;
310 // Set to the real timecode
311 av_seek_frame( context
, -1, real_timecode
* 1000000.0 );
313 // Remove the cached info relating to the previous position
314 mlt_properties_set_double( properties
, "current_time", 0 );
315 mlt_properties_set_data( properties
, "current_image", NULL
, 0, NULL
, NULL
);
319 // Duplicate the last image if necessary
320 if ( mlt_properties_get_data( properties
, "current_image", NULL
) != NULL
&&
321 ( paused
|| mlt_properties_get_double( properties
, "current_time" ) > real_timecode
) )
323 // Get current image and size
325 uint8_t *image
= mlt_properties_get_data( properties
, "current_image", &size
);
328 *buffer
= malloc( size
);
329 memcpy( *buffer
, image
, size
);
331 // Set this on the frame properties
332 mlt_properties_set_data( frame_properties
, "image", *buffer
, size
, free
, NULL
);
340 memset( &pkt
, 0, sizeof( pkt
) );
341 memset( &frame
, 0, sizeof( frame
) );
343 while( ret
>= 0 && !got_picture
)
346 ret
= av_read_frame( context
, &pkt
);
348 // We only deal with video from the selected video_index
349 if ( ret
>= 0 && pkt
.stream_index
== index
&& pkt
.size
> 0 )
352 // Wouldn't it be great if I could use this...
353 //if ( (float)pkt.pts / 1000000.0 >= real_timecode )
354 ret
= avcodec_decode_video( codec_context
, &frame
, &got_picture
, pkt
.data
, pkt
.size
);
357 if ( (float)pkt
.pts
/ 1000000.0 < real_timecode
)
362 else if ( (float)pkt
.pts
/ 1000000.0 >= real_timecode
)
366 else if ( got_picture
&& ignore
-- )
371 current_time
= ( double )pkt
.pts
/ 1000000.0;
374 // We're finished with this packet regardless
375 av_free_packet( &pkt
);
378 // Now handle the picture if we have one
381 // Get current image and size
383 uint8_t *image
= mlt_properties_get_data( properties
, "current_image", &size
);
385 if ( image
== NULL
|| size
!= *width
* *height
* 2 )
387 size
= *width
* *height
* 2;
388 image
= malloc( size
);
389 mlt_properties_set_data( properties
, "current_image", image
, size
, free
, NULL
);
392 *buffer
= malloc( size
);
393 img_convert( output
, PIX_FMT_YUV422
, (AVPicture
*)&frame
, codec_context
->pix_fmt
, *width
, *height
);
394 memcpy( image
, output
->data
[ 0 ], size
);
395 memcpy( *buffer
, output
->data
[ 0 ], size
);
396 mlt_properties_set_data( frame_properties
, "image", *buffer
, size
, free
, NULL
);
397 mlt_properties_set_double( properties
, "current_time", current_time
);
401 // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
402 mlt_properties_set_position( properties
, "video_expected", position
+ 1 );
407 /** Set up video handling.
410 static void producer_set_up_video( mlt_producer
this, mlt_frame frame
)
412 // Get the properties
413 mlt_properties properties
= mlt_producer_properties( this );
415 // Fetch the video_context
416 AVFormatContext
*context
= mlt_properties_get_data( properties
, "video_context", NULL
);
418 // Get the video_index
419 int index
= mlt_properties_get_int( properties
, "video_index" );
421 if ( context
!= NULL
&& index
!= -1 )
423 // Get the frame properties
424 mlt_properties frame_properties
= mlt_frame_properties( frame
);
426 // Get the video stream
427 AVStream
*stream
= context
->streams
[ index
];
430 AVCodecContext
*codec_context
= &stream
->codec
;
433 AVCodec
*codec
= mlt_properties_get_data( properties
, "video_codec", NULL
);
435 // Initialise the codec if necessary
439 codec
= avcodec_find_decoder( codec_context
->codec_id
);
441 // If we don't have a codec and we can't initialise it, we can't do much more...
442 if ( codec
!= NULL
&& avcodec_open( codec_context
, codec
) >= 0 )
444 double aspect_ratio
= 0;
447 if ( codec_context
->sample_aspect_ratio
.num
== 0)
450 aspect_ratio
= av_q2d( codec_context
->sample_aspect_ratio
) * codec_context
->width
/ codec_context
->height
;
452 if (aspect_ratio
<= 0.0)
453 aspect_ratio
= ( double )codec_context
->width
/ ( double )codec_context
->height
;
455 mlt_properties_set_double( properties
, "aspect_ratio", aspect_ratio
);
457 // Now store the codec with its destructor
458 mlt_properties_set_data( properties
, "video_codec", codec
, 0, ( mlt_destructor
)avcodec_close
, NULL
);
460 // Set to the real timecode
461 av_seek_frame( context
, -1, 0 );
465 // Remember that we can't use this later
466 mlt_properties_set_int( properties
, "video_index", -1 );
470 // No codec, no show...
473 mlt_frame_push_get_image( frame
, producer_get_image
);
474 mlt_properties_set_data( frame_properties
, "avformat_producer", this, 0, NULL
, NULL
);
479 /** Get the audio from a frame.
482 static int producer_get_audio( mlt_frame frame
, int16_t **buffer
, mlt_audio_format
*format
, int *frequency
, int *channels
, int *samples
)
484 // Get the properties from the frame
485 mlt_properties frame_properties
= mlt_frame_properties( frame
);
487 // Obtain the frame number of this frame
488 mlt_position position
= mlt_properties_get_position( frame_properties
, "avformat_position" );
491 mlt_producer
this = mlt_properties_get_data( frame_properties
, "avformat_producer", NULL
);
493 // Get the producer properties
494 mlt_properties properties
= mlt_producer_properties( this );
496 // Fetch the audio_context
497 AVFormatContext
*context
= mlt_properties_get_data( properties
, "audio_context", NULL
);
499 // Get the audio_index
500 int index
= mlt_properties_get_int( properties
, "audio_index" );
502 // Obtain the expected frame numer
503 mlt_position expected
= mlt_properties_get_position( properties
, "audio_expected" );
505 // Obtain the resample context if it exists (not always needed)
506 ReSampleContext
*resample
= mlt_properties_get_data( properties
, "audio_resample", NULL
);
508 // Obtain the audio buffer
509 int16_t *audio_buffer
= mlt_properties_get_data( properties
, "audio_buffer", NULL
);
511 // Get amount of audio used
512 int audio_used
= mlt_properties_get_int( properties
, "audio_used" );
514 // Calculate the real time code
515 double real_timecode
= producer_time_of_frame( this, position
);
517 // Get the audio stream
518 AVStream
*stream
= context
->streams
[ index
];
521 AVCodecContext
*codec_context
= &stream
->codec
;
526 // Number of frames to ignore (for ffwd)
529 // Flag for paused (silence)
532 // Check for resample and create if necessary
533 if ( resample
== NULL
)
535 // Create the resampler
536 resample
= audio_resample_init( *channels
, codec_context
->channels
, *frequency
, codec_context
->sample_rate
);
538 // And store it on properties
539 mlt_properties_set_data( properties
, "audio_resample", resample
, 0, ( mlt_destructor
) audio_resample_close
, NULL
);
542 // Check for audio buffer and create if necessary
543 if ( audio_buffer
== NULL
)
545 // Allocate the audio buffer
546 audio_buffer
= malloc( AVCODEC_MAX_AUDIO_FRAME_SIZE
* sizeof( int16_t ) );
548 // And store it on properties for reuse
549 mlt_properties_set_data( properties
, "audio_buffer", audio_buffer
, 0, free
, NULL
);
553 if ( position
!= expected
)
555 if ( position
+ 1 == expected
)
557 // We're paused - silence required
560 else if ( position
> expected
&& ( position
- expected
) < 250 )
562 // Fast forward - seeking is inefficient for small distances - just ignore following frames
563 ignore
= position
- expected
;
567 // Set to the real timecode
568 av_seek_frame( context
, -1, real_timecode
* 1000000.0 );
570 // Clear the usage in the audio buffer
575 // Get the audio if required
580 int16_t temp
[ AVCODEC_MAX_AUDIO_FRAME_SIZE
/ 2 ];
582 memset( &pkt
, 0, sizeof( pkt
) );
584 while( ret
>= 0 && !got_audio
)
586 // Check if the buffer already contains the samples required
587 if ( audio_used
>= *samples
&& ignore
== 0 )
594 ret
= av_read_frame( context
, &pkt
);
597 uint8_t *ptr
= pkt
.data
;
600 if ( ptr
== NULL
|| len
== 0 )
603 // We only deal with video from the selected video_index
604 while ( ret
>= 0 && pkt
.stream_index
== index
&& len
> 0 )
607 ret
= avcodec_decode_audio( codec_context
, temp
, &data_size
, ptr
, len
);
617 int size_out
= audio_resample( resample
, &audio_buffer
[ audio_used
* *channels
], temp
, data_size
/ ( codec_context
->channels
* sizeof( int16_t ) ) );
619 audio_used
+= size_out
;
622 while ( ignore
&& audio_used
> *samples
)
625 audio_used
-= *samples
;
626 memmove( audio_buffer
, &audio_buffer
[ *samples
* *channels
], audio_used
* sizeof( int16_t ) );
630 // If we're behind, ignore this packet
631 float current_pts
= (float)pkt
.pts
/ 1000000.0;
632 double discrepancy
= mlt_properties_get_double( properties
, "discrepancy" );
633 if ( discrepancy
* current_pts
< real_timecode
)
637 // We're finished with this packet regardless
638 av_free_packet( &pkt
);
641 // Now handle the audio if we have enough
642 if ( audio_used
>= *samples
)
644 *buffer
= malloc( *samples
* *channels
* sizeof( int16_t ) );
645 memcpy( *buffer
, audio_buffer
, *samples
* *channels
* sizeof( int16_t ) );
646 audio_used
-= *samples
;
647 memmove( audio_buffer
, &audio_buffer
[ *samples
* *channels
], audio_used
* *channels
* sizeof( int16_t ) );
648 mlt_properties_set_data( frame_properties
, "audio", *buffer
, 0, free
, NULL
);
652 frame
->get_audio
= NULL
;
653 mlt_frame_get_audio( frame
, buffer
, format
, frequency
, channels
, samples
);
657 // Store the number of audio samples still available
658 mlt_properties_set_int( properties
, "audio_used", audio_used
);
662 // Get silence and don't touch the context
663 frame
->get_audio
= NULL
;
664 mlt_frame_get_audio( frame
, buffer
, format
, frequency
, channels
, samples
);
667 // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
668 mlt_properties_set_position( properties
, "audio_expected", position
+ 1 );
673 /** Set up audio handling.
676 static void producer_set_up_audio( mlt_producer
this, mlt_frame frame
)
678 // Get the properties
679 mlt_properties properties
= mlt_producer_properties( this );
681 // Fetch the audio_context
682 AVFormatContext
*context
= mlt_properties_get_data( properties
, "audio_context", NULL
);
684 // Get the audio_index
685 int index
= mlt_properties_get_int( properties
, "audio_index" );
687 // Deal with audio context
688 if ( context
!= NULL
&& index
!= -1 )
690 // Get the frame properties
691 mlt_properties frame_properties
= mlt_frame_properties( frame
);
693 // Get the audio stream
694 AVStream
*stream
= context
->streams
[ index
];
697 AVCodecContext
*codec_context
= &stream
->codec
;
700 AVCodec
*codec
= mlt_properties_get_data( properties
, "audio_codec", NULL
);
702 // Initialise the codec if necessary
706 codec
= avcodec_find_decoder( codec_context
->codec_id
);
708 // If we don't have a codec and we can't initialise it, we can't do much more...
709 if ( codec
!= NULL
&& avcodec_open( codec_context
, codec
) >= 0 )
711 // Now store the codec with its destructor
712 mlt_properties_set_data( properties
, "audio_codec", codec
, 0, ( mlt_destructor
)avcodec_close
, NULL
);
716 // Remember that we can't use this later
717 mlt_properties_set_int( properties
, "audio_index", -1 );
721 // No codec, no show...
724 frame
->get_audio
= producer_get_audio
;
725 mlt_properties_set_data( frame_properties
, "avformat_producer", this, 0, NULL
, NULL
);
730 /** Our get frame implementation.
733 static int producer_get_frame( mlt_producer
this, mlt_frame_ptr frame
, int index
)
735 // Create an empty frame
736 *frame
= mlt_frame_init( );
738 // Update timecode on the frame we're creating
739 mlt_frame_set_position( *frame
, mlt_producer_position( this ) );
741 // Set the position of this producer
742 mlt_properties_set_position( mlt_frame_properties( *frame
), "avformat_position", mlt_producer_get_in( this ) + mlt_producer_position( this ) );
745 producer_set_up_video( this, *frame
);
748 producer_set_up_audio( this, *frame
);
750 // Set the aspect_ratio
751 mlt_properties_set_double( mlt_frame_properties( *frame
), "aspect_ratio", mlt_properties_get_double( mlt_producer_properties( this ), "aspect_ratio" ) );
753 // Calculate the next timecode
754 mlt_producer_prepare_next( this );