2 * producer_avformat.c -- avformat producer
3 * Copyright (C) 2003-2004 Ushodaya Enterprises Limited
4 * Author: Charles Yates <charles.yates@pandora.be>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software Foundation,
18 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 #include "producer_avformat.h"
25 #include <framework/mlt_frame.h>
27 // ffmpeg Header files
28 #include <ffmpeg/avformat.h>
30 // System header files
36 // Forward references.
37 static int producer_open( mlt_producer
this, char *file
);
38 static int producer_get_frame( mlt_producer
this, mlt_frame_ptr frame
, int index
);
40 // A static flag used to determine if avformat has been initialised
41 static int avformat_initialised
= 0;
42 static pthread_mutex_t avformat_mutex
;
44 /** Constructor for libavformat.
47 mlt_producer
producer_avformat_init( char *file
)
49 mlt_producer
this = NULL
;
51 // Check that we have a non-NULL argument
54 // Construct the producer
55 this = calloc( 1, sizeof( struct mlt_producer_s
) );
58 if ( mlt_producer_init( this, NULL
) == 0 )
61 mlt_properties properties
= mlt_producer_properties( this );
63 // Set the resource property (required for all producers)
64 mlt_properties_set( properties
, "resource", file
);
66 // TEST: audio sync tweaking
67 mlt_properties_set_double( properties
, "discrepancy", 1 );
69 // Register our get_frame implementation
70 this->get_frame
= producer_get_frame
;
72 // Initialise avformat if necessary
73 if ( avformat_initialised
== 0 )
75 pthread_mutex_init( &avformat_mutex
, NULL
);
76 avformat_initialised
= 1;
81 if ( producer_open( this, file
) != 0 )
84 mlt_producer_close( this );
93 /** Find the default streams.
96 static void find_default_streams( AVFormatContext
*context
, int *audio_index
, int *video_index
)
100 // Allow for multiple audio and video streams in the file and select first of each (if available)
101 for( i
= 0; i
< context
->nb_streams
; i
++ )
103 // Get the codec context
104 AVCodecContext
*codec_context
= &context
->streams
[ i
]->codec
;
106 // Determine the type and obtain the first index of each type
107 switch( codec_context
->codec_type
)
109 case CODEC_TYPE_VIDEO
:
110 if ( *video_index
< 0 )
113 case CODEC_TYPE_AUDIO
:
114 if ( *audio_index
< 0 )
123 /** Producer file destructor.
126 static void producer_file_close( void *context
)
128 if ( context
!= NULL
)
130 // Lock the mutex now
131 pthread_mutex_lock( &avformat_mutex
);
134 av_close_input_file( context
);
136 // Unlock the mutex now
137 pthread_mutex_unlock( &avformat_mutex
);
141 /** Producer file destructor.
144 static void producer_codec_close( void *codec
)
148 // Lock the mutex now
149 pthread_mutex_lock( &avformat_mutex
);
152 avcodec_close( codec
);
154 // Unlock the mutex now
155 pthread_mutex_unlock( &avformat_mutex
);
161 NOTE: We need to have a valid [PAL or NTSC] frame rate before we can determine the
162 number of frames in the file. However, this is at odds with the way things work - the
163 constructor needs to provide in/out points before the user of the producer is able
164 to specify properties :-/. However, the PAL/NTSC distinction applies to all producers
165 and while we currently accept whatever the producer provides, this will not work in
166 the more general case. Plans are afoot... and this one will work without modification
167 (in theory anyway ;-)).
170 static int producer_open( mlt_producer
this, char *file
)
172 // Return an error code (0 == no error)
175 // Context for avformat
176 AVFormatContext
*context
= NULL
;
178 // Get the properties
179 mlt_properties properties
= mlt_producer_properties( this );
181 // We will treat everything with the producer fps
182 double fps
= mlt_properties_get_double( properties
, "fps" );
184 // Lock the mutex now
185 pthread_mutex_lock( &avformat_mutex
);
187 // Now attempt to open the file
188 error
= av_open_input_file( &context
, file
, NULL
, 0, NULL
);
189 // fprintf( stderr, "AVFORMAT: open %d %s\n", error, file );
192 // If successful, then try to get additional info
195 // Get the stream info
196 error
= av_find_stream_info( context
) < 0;
198 // Continue if no error
201 // We will default to the first audio and video streams found
202 int audio_index
= -1;
203 int video_index
= -1;
205 // Now set properties where we can (use default unknowns if required)
206 if ( context
->duration
!= AV_NOPTS_VALUE
)
208 // This isn't going to be accurate for all formats
209 mlt_position frames
= ( mlt_position
)( ( ( double )context
->duration
/ ( double )AV_TIME_BASE
) * fps
);
210 mlt_properties_set_position( properties
, "out", frames
- 2 );
211 mlt_properties_set_position( properties
, "length", frames
- 1 );
214 // Find default audio and video streams
215 find_default_streams( context
, &audio_index
, &video_index
);
217 // Store selected audio and video indexes on properties
218 mlt_properties_set_int( properties
, "audio_index", audio_index
);
219 mlt_properties_set_int( properties
, "video_index", video_index
);
221 // We're going to cheat here - for a/v files, we will have two contexts (reasoning will be clear later)
222 if ( audio_index
!= -1 && video_index
!= -1 )
224 // We'll use the open one as our video_context
225 mlt_properties_set_data( properties
, "video_context", context
, 0, producer_file_close
, NULL
);
227 // And open again for our audio context
228 av_open_input_file( &context
, file
, NULL
, 0, NULL
);
229 av_find_stream_info( context
);
232 mlt_properties_set_data( properties
, "audio_context", context
, 0, producer_file_close
, NULL
);
234 else if ( video_index
!= -1 )
236 // We only have a video context
237 mlt_properties_set_data( properties
, "video_context", context
, 0, producer_file_close
, NULL
);
239 else if ( audio_index
!= -1 )
241 // We only have an audio context
242 mlt_properties_set_data( properties
, "audio_context", context
, 0, producer_file_close
, NULL
);
246 // Something has gone wrong
252 // Unlock the mutex now
253 pthread_mutex_unlock( &avformat_mutex
);
258 /** Convert a frame position to a time code.
261 static double producer_time_of_frame( mlt_producer
this, mlt_position position
)
263 // Get the properties
264 mlt_properties properties
= mlt_producer_properties( this );
267 double fps
= mlt_properties_get_double( properties
, "fps" );
270 return ( double )position
/ fps
;
273 /** Get an image from a frame.
276 static int producer_get_image( mlt_frame frame
, uint8_t **buffer
, mlt_image_format
*format
, int *width
, int *height
, int writable
)
278 // Get the properties from the frame
279 mlt_properties frame_properties
= mlt_frame_properties( frame
);
281 // Obtain the frame number of this frame
282 mlt_position position
= mlt_properties_get_position( frame_properties
, "avformat_position" );
285 mlt_producer
this = mlt_properties_get_data( frame_properties
, "avformat_producer", NULL
);
287 // Get the producer properties
288 mlt_properties properties
= mlt_producer_properties( this );
290 // Fetch the video_context
291 AVFormatContext
*context
= mlt_properties_get_data( properties
, "video_context", NULL
);
293 // Get the video_index
294 int index
= mlt_properties_get_int( properties
, "video_index" );
296 // Obtain the expected frame numer
297 mlt_position expected
= mlt_properties_get_position( properties
, "video_expected" );
299 // Calculate the real time code
300 double real_timecode
= producer_time_of_frame( this, position
);
302 // Get the video stream
303 AVStream
*stream
= context
->streams
[ index
];
306 AVCodecContext
*codec_context
= &stream
->codec
;
311 // Get the conversion frame
312 AVPicture
*output
= mlt_properties_get_data( properties
, "video_output_frame", NULL
);
314 // Special case pause handling flag
317 // Special case ffwd handling
320 // Current time calcs
321 double current_time
= 0;
323 // We may want to use the source fps if available
324 double source_fps
= mlt_properties_get_double( properties
, "source_fps" );
326 // Set the result arguments that we know here (only *buffer is now required)
327 *format
= mlt_image_yuv422
;
328 *width
= codec_context
->width
;
329 *height
= codec_context
->height
;
331 // Set this on the frame properties
332 mlt_properties_set_int( frame_properties
, "width", *width
);
333 mlt_properties_set_int( frame_properties
, "height", *height
);
335 // Lock the mutex now
336 pthread_mutex_lock( &avformat_mutex
);
338 // Construct an AVFrame for YUV422 conversion
339 if ( output
== NULL
)
341 int size
= avpicture_get_size( PIX_FMT_YUV422
, *width
, *height
);
344 uint8_t *buf
= malloc( size
);
345 output
= malloc( sizeof( AVPicture
) );
346 avpicture_fill( output
, buf
, PIX_FMT_YUV422
, *width
, *height
);
347 mlt_properties_set_data( properties
, "video_output_frame", output
, 0, av_free
, NULL
);
348 mlt_properties_set_data( properties
, "video_output_buffer", buf
, 0, free
, NULL
);
352 if ( position
!= expected
)
354 if ( position
+ 1 == expected
)
356 // We're paused - use last image
359 else if ( position
> expected
&& ( position
- expected
) < 250 )
361 // Fast forward - seeking is inefficient for small distances - just ignore following frames
362 ignore
= position
- expected
;
366 // Set to the real timecode
367 av_seek_frame( context
, -1, real_timecode
* 1000000.0 );
369 // Remove the cached info relating to the previous position
370 mlt_properties_set_double( properties
, "current_time", 0 );
371 mlt_properties_set_data( properties
, "current_image", NULL
, 0, NULL
, NULL
);
375 // Duplicate the last image if necessary
376 if ( mlt_properties_get_data( properties
, "current_image", NULL
) != NULL
&&
377 ( paused
|| mlt_properties_get_double( properties
, "current_time" ) >= real_timecode
) )
379 // Get current image and size
381 uint8_t *image
= mlt_properties_get_data( properties
, "current_image", &size
);
385 *buffer
= malloc( size
);
386 memcpy( *buffer
, image
, size
);
388 // Set this on the frame properties
389 mlt_properties_set_data( frame_properties
, "image", *buffer
, size
, free
, NULL
);
397 memset( &pkt
, 0, sizeof( pkt
) );
398 memset( &frame
, 0, sizeof( frame
) );
400 while( ret
>= 0 && !got_picture
)
403 ret
= av_read_frame( context
, &pkt
);
405 // We only deal with video from the selected video_index
406 if ( ret
>= 0 && pkt
.stream_index
== index
&& pkt
.size
> 0 )
408 current_time
= ( double )pkt
.pts
/ 1000000.0;
411 // Wouldn't it be great if I could use this...
412 //if ( (float)pkt.pts / 1000000.0 >= real_timecode )
413 ret
= avcodec_decode_video( codec_context
, &frame
, &got_picture
, pkt
.data
, pkt
.size
);
418 if ( current_time
< real_timecode
)
423 else if ( current_time
>= real_timecode
)
427 else if ( got_picture
&& ignore
-- )
434 // We're finished with this packet regardless
435 av_free_packet( &pkt
);
438 // Now handle the picture if we have one
441 // Get current image and size
443 uint8_t *image
= mlt_properties_get_data( properties
, "current_image", &size
);
445 if ( image
== NULL
|| size
!= *width
* *height
* 2 )
447 size
= *width
* ( *height
+ 1 ) * 2;
449 image
= malloc( size
);
450 mlt_properties_set_data( properties
, "current_image", image
, size
, free
, NULL
);
454 *buffer
= malloc( size
);
455 img_convert( output
, PIX_FMT_YUV422
, (AVPicture
*)&frame
, codec_context
->pix_fmt
, *width
, *height
);
456 memcpy( image
, output
->data
[ 0 ], size
);
457 memcpy( *buffer
, output
->data
[ 0 ], size
);
458 mlt_properties_set_data( frame_properties
, "image", *buffer
, size
, free
, NULL
);
460 if ( current_time
== 0 && source_fps
!= 0 )
462 double fps
= mlt_properties_get_double( properties
, "fps" );
463 current_time
= ceil( source_fps
* ( double )position
/ fps
) * ( 1 / source_fps
);
464 mlt_properties_set_double( properties
, "current_time", current_time
);
468 mlt_properties_set_double( properties
, "current_time", current_time
);
473 // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
474 mlt_properties_set_position( properties
, "video_expected", position
+ 1 );
476 // Unlock the mutex now
477 pthread_mutex_unlock( &avformat_mutex
);
482 /** Set up video handling.
485 static void producer_set_up_video( mlt_producer
this, mlt_frame frame
)
487 // Get the properties
488 mlt_properties properties
= mlt_producer_properties( this );
490 // Fetch the video_context
491 AVFormatContext
*context
= mlt_properties_get_data( properties
, "video_context", NULL
);
493 // Get the video_index
494 int index
= mlt_properties_get_int( properties
, "video_index" );
496 // Get the frame properties
497 mlt_properties frame_properties
= mlt_frame_properties( frame
);
499 // Lock the mutex now
500 pthread_mutex_lock( &avformat_mutex
);
502 if ( context
!= NULL
&& index
!= -1 )
504 // Get the video stream
505 AVStream
*stream
= context
->streams
[ index
];
508 AVCodecContext
*codec_context
= &stream
->codec
;
511 AVCodec
*codec
= mlt_properties_get_data( properties
, "video_codec", NULL
);
513 // Initialise the codec if necessary
517 codec
= avcodec_find_decoder( codec_context
->codec_id
);
519 // If we don't have a codec and we can't initialise it, we can't do much more...
520 if ( codec
!= NULL
&& avcodec_open( codec_context
, codec
) >= 0 )
522 double aspect_ratio
= 0;
523 double source_fps
= 0;
526 if ( codec_context
->sample_aspect_ratio
.num
== 0)
529 aspect_ratio
= av_q2d( codec_context
->sample_aspect_ratio
) * codec_context
->width
/ codec_context
->height
;
531 if (aspect_ratio
<= 0.0)
532 aspect_ratio
= ( double )codec_context
->width
/ ( double )codec_context
->height
;
534 mlt_properties_set_double( properties
, "aspect_ratio", aspect_ratio
);
535 fprintf( stderr
, "AVFORMAT: sample aspect %f\n", aspect_ratio
);
538 source_fps
= ( double )codec_context
->frame_rate
/ codec_context
->frame_rate_base
;
540 // We'll use fps if it's available
541 if ( source_fps
> 0 && source_fps
< 30 )
542 mlt_properties_set_double( properties
, "source_fps", source_fps
);
544 // Now store the codec with its destructor
545 mlt_properties_set_data( properties
, "video_codec", codec_context
, 0, producer_codec_close
, NULL
);
547 // Set to the real timecode
548 av_seek_frame( context
, -1, 0 );
552 // Remember that we can't use this later
553 mlt_properties_set_int( properties
, "video_index", -1 );
557 // No codec, no show...
560 mlt_frame_push_get_image( frame
, producer_get_image
);
561 mlt_properties_set_data( frame_properties
, "avformat_producer", this, 0, NULL
, NULL
);
565 mlt_properties_set_int( frame_properties
, "test_image", 1 );
570 mlt_properties_set_int( frame_properties
, "test_image", 1 );
573 // Unlock the mutex now
574 pthread_mutex_unlock( &avformat_mutex
);
577 /** Get the audio from a frame.
580 static int producer_get_audio( mlt_frame frame
, int16_t **buffer
, mlt_audio_format
*format
, int *frequency
, int *channels
, int *samples
)
582 // Get the properties from the frame
583 mlt_properties frame_properties
= mlt_frame_properties( frame
);
585 // Obtain the frame number of this frame
586 mlt_position position
= mlt_properties_get_position( frame_properties
, "avformat_position" );
589 mlt_producer
this = mlt_properties_get_data( frame_properties
, "avformat_producer", NULL
);
591 // Get the producer properties
592 mlt_properties properties
= mlt_producer_properties( this );
594 // Fetch the audio_context
595 AVFormatContext
*context
= mlt_properties_get_data( properties
, "audio_context", NULL
);
597 // Get the audio_index
598 int index
= mlt_properties_get_int( properties
, "audio_index" );
600 // Obtain the expected frame numer
601 mlt_position expected
= mlt_properties_get_position( properties
, "audio_expected" );
603 // Obtain the resample context if it exists (not always needed)
604 ReSampleContext
*resample
= mlt_properties_get_data( properties
, "audio_resample", NULL
);
606 // Obtain the audio buffer
607 int16_t *audio_buffer
= mlt_properties_get_data( properties
, "audio_buffer", NULL
);
609 // Get amount of audio used
610 int audio_used
= mlt_properties_get_int( properties
, "audio_used" );
612 // Calculate the real time code
613 double real_timecode
= producer_time_of_frame( this, position
);
615 // Get the audio stream
616 AVStream
*stream
= context
->streams
[ index
];
619 AVCodecContext
*codec_context
= &stream
->codec
;
624 // Number of frames to ignore (for ffwd)
627 // Flag for paused (silence)
631 // Lock the mutex now
632 pthread_mutex_lock( &avformat_mutex
);
634 // Check for resample and create if necessary
635 if ( resample
== NULL
)
637 // Create the resampler
638 resample
= audio_resample_init( *channels
, codec_context
->channels
, *frequency
, codec_context
->sample_rate
);
640 // And store it on properties
641 mlt_properties_set_data( properties
, "audio_resample", resample
, 0, ( mlt_destructor
)audio_resample_close
, NULL
);
644 // Check for audio buffer and create if necessary
645 if ( audio_buffer
== NULL
)
647 // Allocate the audio buffer
648 audio_buffer
= malloc( AVCODEC_MAX_AUDIO_FRAME_SIZE
* sizeof( int16_t ) );
650 // And store it on properties for reuse
651 mlt_properties_set_data( properties
, "audio_buffer", audio_buffer
, 0, free
, NULL
);
655 if ( position
!= expected
)
657 if ( position
+ 1 == expected
)
659 // We're paused - silence required
662 else if ( position
> expected
&& ( position
- expected
) < 250 )
664 // Fast forward - seeking is inefficient for small distances - just ignore following frames
665 ignore
= position
- expected
;
669 // Set to the real timecode
670 av_seek_frame( context
, -1, real_timecode
* 1000000.0 );
672 // Clear the usage in the audio buffer
679 // Get the audio if required
684 int16_t temp
[ AVCODEC_MAX_AUDIO_FRAME_SIZE
/ 2 ];
686 memset( &pkt
, 0, sizeof( pkt
) );
688 while( ret
>= 0 && !got_audio
)
690 // Check if the buffer already contains the samples required
691 if ( audio_used
>= *samples
&& ignore
== 0 )
698 ret
= av_read_frame( context
, &pkt
);
701 uint8_t *ptr
= pkt
.data
;
704 // We only deal with video from the selected video_index
705 while ( ptr
!= NULL
&& ret
>= 0 && pkt
.stream_index
== index
&& len
> 0 )
708 ret
= avcodec_decode_audio( codec_context
, temp
, &data_size
, ptr
, len
);
721 int size_out
= audio_resample( resample
, &audio_buffer
[ audio_used
* *channels
], temp
, data_size
/ ( codec_context
->channels
* sizeof( int16_t ) ) );
723 audio_used
+= size_out
;
726 while ( ignore
&& audio_used
> *samples
)
729 audio_used
-= *samples
;
730 memmove( audio_buffer
, &audio_buffer
[ *samples
* *channels
], audio_used
* sizeof( int16_t ) );
734 // If we're behind, ignore this packet
735 float current_pts
= (float)pkt
.pts
/ 1000000.0;
736 double discrepancy
= mlt_properties_get_double( properties
, "discrepancy" );
737 if ( current_pts
!= 0 && real_timecode
!= 0 )
739 if ( discrepancy
!= 1 )
740 discrepancy
= ( discrepancy
+ ( real_timecode
/ current_pts
) ) / 2;
742 discrepancy
= real_timecode
/ current_pts
;
743 if ( discrepancy
> 0.9 && discrepancy
< 1.1 )
746 discrepancy
= floor( discrepancy
+ 0.5 );
748 if ( discrepancy
== 0 )
751 mlt_properties_set_double( properties
, "discrepancy", discrepancy
);
754 if ( discrepancy
* current_pts
<= ( real_timecode
- 0.02 ) )
758 // We're finished with this packet regardless
759 av_free_packet( &pkt
);
762 // Now handle the audio if we have enough
763 if ( audio_used
>= *samples
)
765 *buffer
= malloc( *samples
* *channels
* sizeof( int16_t ) );
766 memcpy( *buffer
, audio_buffer
, *samples
* *channels
* sizeof( int16_t ) );
767 audio_used
-= *samples
;
768 memmove( audio_buffer
, &audio_buffer
[ *samples
* *channels
], audio_used
* *channels
* sizeof( int16_t ) );
769 mlt_properties_set_data( frame_properties
, "audio", *buffer
, 0, free
, NULL
);
773 frame
->get_audio
= NULL
;
774 mlt_frame_get_audio( frame
, buffer
, format
, frequency
, channels
, samples
);
778 // Store the number of audio samples still available
779 mlt_properties_set_int( properties
, "audio_used", audio_used
);
783 // Get silence and don't touch the context
784 frame
->get_audio
= NULL
;
785 mlt_frame_get_audio( frame
, buffer
, format
, frequency
, channels
, samples
);
788 // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
789 mlt_properties_set_position( properties
, "audio_expected", position
+ 1 );
791 // Unlock the mutex now
792 pthread_mutex_unlock( &avformat_mutex
);
797 /** Set up audio handling.
800 static void producer_set_up_audio( mlt_producer
this, mlt_frame frame
)
802 // Get the properties
803 mlt_properties properties
= mlt_producer_properties( this );
805 // Fetch the audio_context
806 AVFormatContext
*context
= mlt_properties_get_data( properties
, "audio_context", NULL
);
808 // Get the audio_index
809 int index
= mlt_properties_get_int( properties
, "audio_index" );
811 // Lock the mutex now
812 pthread_mutex_lock( &avformat_mutex
);
814 // Deal with audio context
815 if ( context
!= NULL
&& index
!= -1 )
817 // Get the frame properties
818 mlt_properties frame_properties
= mlt_frame_properties( frame
);
820 // Get the audio stream
821 AVStream
*stream
= context
->streams
[ index
];
824 AVCodecContext
*codec_context
= &stream
->codec
;
827 AVCodec
*codec
= mlt_properties_get_data( properties
, "audio_codec", NULL
);
829 // Initialise the codec if necessary
833 codec
= avcodec_find_decoder( codec_context
->codec_id
);
835 // If we don't have a codec and we can't initialise it, we can't do much more...
836 if ( codec
!= NULL
&& avcodec_open( codec_context
, codec
) >= 0 )
838 // Now store the codec with its destructor
839 mlt_properties_set_data( properties
, "audio_codec", codec_context
, 0, producer_codec_close
, NULL
);
844 // Remember that we can't use this later
845 mlt_properties_set_int( properties
, "audio_index", -1 );
849 // No codec, no show...
852 frame
->get_audio
= producer_get_audio
;
853 mlt_properties_set_data( frame_properties
, "avformat_producer", this, 0, NULL
, NULL
);
857 // Unlock the mutex now
858 pthread_mutex_unlock( &avformat_mutex
);
861 /** Our get frame implementation.
864 static int producer_get_frame( mlt_producer
this, mlt_frame_ptr frame
, int index
)
866 // Create an empty frame
867 *frame
= mlt_frame_init( );
869 // Update timecode on the frame we're creating
870 mlt_frame_set_position( *frame
, mlt_producer_position( this ) );
872 // Set the position of this producer
873 mlt_properties_set_position( mlt_frame_properties( *frame
), "avformat_position", mlt_producer_get_in( this ) + mlt_producer_position( this ) );
876 producer_set_up_video( this, *frame
);
879 producer_set_up_audio( this, *frame
);
881 // Set the aspect_ratio
882 mlt_properties_set_double( mlt_frame_properties( *frame
), "aspect_ratio", mlt_properties_get_double( mlt_producer_properties( this ), "aspect_ratio" ) );
884 // Calculate the next timecode
885 mlt_producer_prepare_next( this );