2 * producer_avformat.c -- avformat producer
3 * Copyright (C) 2003-2004 Ushodaya Enterprises Limited
4 * Author: Charles Yates <charles.yates@pandora.be>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software Foundation,
18 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 #include "producer_avformat.h"
25 #include <framework/mlt_frame.h>
27 // ffmpeg Header files
28 #include <ffmpeg/avformat.h>
30 // System header files
35 // Forward references.
36 static int producer_open( mlt_producer
this, char *file
);
37 static int producer_get_frame( mlt_producer
this, mlt_frame_ptr frame
, int index
);
39 // A static flag used to determine if avformat has been initialised
40 static int avformat_initialised
= 0;
41 static pthread_mutex_t avformat_mutex
;
43 /** Constructor for libavformat.
46 mlt_producer
producer_avformat_init( char *file
)
48 mlt_producer
this = NULL
;
50 // Check that we have a non-NULL argument
53 // Construct the producer
54 this = calloc( 1, sizeof( struct mlt_producer_s
) );
57 if ( mlt_producer_init( this, NULL
) == 0 )
60 mlt_properties properties
= mlt_producer_properties( this );
62 // Set the resource property (required for all producers)
63 mlt_properties_set( properties
, "resource", file
);
65 // TEST: audio sync tweaking
66 mlt_properties_set_double( properties
, "discrepancy", 1 );
68 // Register our get_frame implementation
69 this->get_frame
= producer_get_frame
;
71 // Initialise avformat if necessary
72 if ( avformat_initialised
== 0 )
74 pthread_mutex_init( &avformat_mutex
, NULL
);
75 avformat_initialised
= 1;
80 if ( producer_open( this, file
) != 0 )
83 mlt_producer_close( this );
92 /** Find the default streams.
95 static void find_default_streams( AVFormatContext
*context
, int *audio_index
, int *video_index
)
99 // Allow for multiple audio and video streams in the file and select first of each (if available)
100 for( i
= 0; i
< context
->nb_streams
; i
++ )
102 // Get the codec context
103 AVCodecContext
*codec_context
= &context
->streams
[ i
]->codec
;
105 // Determine the type and obtain the first index of each type
106 switch( codec_context
->codec_type
)
108 case CODEC_TYPE_VIDEO
:
109 if ( *video_index
< 0 )
112 case CODEC_TYPE_AUDIO
:
113 if ( *audio_index
< 0 )
122 /** Producer file destructor.
125 static void producer_file_close( void *context
)
127 if ( context
!= NULL
)
129 // Lock the mutex now
130 pthread_mutex_lock( &avformat_mutex
);
133 av_close_input_file( context
);
135 // Unlock the mutex now
136 pthread_mutex_unlock( &avformat_mutex
);
140 /** Producer file destructor.
143 static void producer_codec_close( void *codec
)
147 // Lock the mutex now
148 pthread_mutex_lock( &avformat_mutex
);
151 avcodec_close( codec
);
153 // Unlock the mutex now
154 pthread_mutex_unlock( &avformat_mutex
);
160 NOTE: We need to have a valid [PAL or NTSC] frame rate before we can determine the
161 number of frames in the file. However, this is at odds with the way things work - the
162 constructor needs to provide in/out points before the user of the producer is able
163 to specify properties :-/. However, the PAL/NTSC distinction applies to all producers
164 and while we currently accept whatever the producer provides, this will not work in
165 the more general case. Plans are afoot... and this one will work without modification
166 (in theory anyway ;-)).
169 static int producer_open( mlt_producer
this, char *file
)
171 // Return an error code (0 == no error)
174 // Context for avformat
175 AVFormatContext
*context
= NULL
;
177 // Get the properties
178 mlt_properties properties
= mlt_producer_properties( this );
180 // We will treat everything with the producer fps
181 double fps
= mlt_properties_get_double( properties
, "fps" );
183 // Lock the mutex now
184 pthread_mutex_lock( &avformat_mutex
);
186 // Now attempt to open the file
187 error
= av_open_input_file( &context
, file
, NULL
, 0, NULL
) < 0;
189 // If successful, then try to get additional info
192 // Get the stream info
193 error
= av_find_stream_info( context
) < 0;
195 // Continue if no error
198 // We will default to the first audio and video streams found
199 int audio_index
= -1;
200 int video_index
= -1;
202 // Now set properties where we can (use default unknowns if required)
203 if ( context
->duration
!= AV_NOPTS_VALUE
)
205 // This isn't going to be accurate for all formats
206 mlt_position frames
= ( mlt_position
)( ( ( double )context
->duration
/ ( double )AV_TIME_BASE
) * fps
);
207 mlt_properties_set_position( properties
, "out", frames
- 2 );
208 mlt_properties_set_position( properties
, "length", frames
- 1 );
211 // Find default audio and video streams
212 find_default_streams( context
, &audio_index
, &video_index
);
214 // Store selected audio and video indexes on properties
215 mlt_properties_set_int( properties
, "audio_index", audio_index
);
216 mlt_properties_set_int( properties
, "video_index", video_index
);
218 // We're going to cheat here - for a/v files, we will have two contexts (reasoning will be clear later)
219 if ( audio_index
!= -1 && video_index
!= -1 )
221 // We'll use the open one as our video_context
222 mlt_properties_set_data( properties
, "video_context", context
, 0, producer_file_close
, NULL
);
224 // And open again for our audio context
225 av_open_input_file( &context
, file
, NULL
, 0, NULL
);
226 av_find_stream_info( context
);
229 mlt_properties_set_data( properties
, "audio_context", context
, 0, producer_file_close
, NULL
);
231 else if ( video_index
!= -1 )
233 // We only have a video context
234 mlt_properties_set_data( properties
, "video_context", context
, 0, producer_file_close
, NULL
);
236 else if ( audio_index
!= -1 )
238 // We only have an audio context
239 mlt_properties_set_data( properties
, "audio_context", context
, 0, producer_file_close
, NULL
);
243 // Something has gone wrong
249 // Unlock the mutex now
250 pthread_mutex_unlock( &avformat_mutex
);
255 /** Convert a frame position to a time code.
258 static double producer_time_of_frame( mlt_producer
this, mlt_position position
)
260 // Get the properties
261 mlt_properties properties
= mlt_producer_properties( this );
264 double fps
= mlt_properties_get_double( properties
, "fps" );
267 return ( double )position
/ fps
;
270 /** Get an image from a frame.
273 static int producer_get_image( mlt_frame frame
, uint8_t **buffer
, mlt_image_format
*format
, int *width
, int *height
, int writable
)
275 // Get the properties from the frame
276 mlt_properties frame_properties
= mlt_frame_properties( frame
);
278 // Obtain the frame number of this frame
279 mlt_position position
= mlt_properties_get_position( frame_properties
, "avformat_position" );
282 mlt_producer
this = mlt_properties_get_data( frame_properties
, "avformat_producer", NULL
);
284 // Get the producer properties
285 mlt_properties properties
= mlt_producer_properties( this );
287 // Fetch the video_context
288 AVFormatContext
*context
= mlt_properties_get_data( properties
, "video_context", NULL
);
290 // Get the video_index
291 int index
= mlt_properties_get_int( properties
, "video_index" );
293 // Obtain the expected frame numer
294 mlt_position expected
= mlt_properties_get_position( properties
, "video_expected" );
296 // Calculate the real time code
297 double real_timecode
= producer_time_of_frame( this, position
);
299 // Get the video stream
300 AVStream
*stream
= context
->streams
[ index
];
303 AVCodecContext
*codec_context
= &stream
->codec
;
308 // Get the conversion frame
309 AVPicture
*output
= mlt_properties_get_data( properties
, "video_output_frame", NULL
);
311 // Special case pause handling flag
314 // Special case ffwd handling
317 // Current time calcs
318 double current_time
= 0;
320 // Set the result arguments that we know here (only *buffer is now required)
321 *format
= mlt_image_yuv422
;
322 *width
= codec_context
->width
;
323 *height
= codec_context
->height
;
325 // Set this on the frame properties
326 mlt_properties_set_int( frame_properties
, "width", *width
);
327 mlt_properties_set_int( frame_properties
, "height", *height
);
329 // Lock the mutex now
330 pthread_mutex_lock( &avformat_mutex
);
332 // Construct an AVFrame for YUV422 conversion
333 if ( output
== NULL
)
335 int size
= avpicture_get_size( PIX_FMT_YUV422
, *width
, *height
);
336 uint8_t *buf
= malloc( size
);
337 output
= malloc( sizeof( AVPicture
) );
338 avpicture_fill( output
, buf
, PIX_FMT_YUV422
, *width
, *height
);
339 mlt_properties_set_data( properties
, "video_output_frame", output
, 0, av_free
, NULL
);
340 mlt_properties_set_data( properties
, "video_output_buffer", buf
, 0, free
, NULL
);
344 if ( position
!= expected
)
346 if ( position
+ 1 == expected
)
348 // We're paused - use last image
351 else if ( position
> expected
&& ( position
- expected
) < 250 )
353 // Fast forward - seeking is inefficient for small distances - just ignore following frames
354 ignore
= position
- expected
;
358 // Set to the real timecode
359 av_seek_frame( context
, -1, real_timecode
* 1000000.0 );
361 // Remove the cached info relating to the previous position
362 mlt_properties_set_double( properties
, "current_time", 0 );
363 mlt_properties_set_data( properties
, "current_image", NULL
, 0, NULL
, NULL
);
367 // Duplicate the last image if necessary
368 if ( mlt_properties_get_data( properties
, "current_image", NULL
) != NULL
&&
369 ( paused
|| mlt_properties_get_double( properties
, "current_time" ) > real_timecode
) )
371 // Get current image and size
373 uint8_t *image
= mlt_properties_get_data( properties
, "current_image", &size
);
376 *buffer
= malloc( size
);
377 memcpy( *buffer
, image
, size
);
379 // Set this on the frame properties
380 mlt_properties_set_data( frame_properties
, "image", *buffer
, size
, free
, NULL
);
388 memset( &pkt
, 0, sizeof( pkt
) );
389 memset( &frame
, 0, sizeof( frame
) );
391 while( ret
>= 0 && !got_picture
)
394 ret
= av_read_frame( context
, &pkt
);
396 // We only deal with video from the selected video_index
397 if ( ret
>= 0 && pkt
.stream_index
== index
&& pkt
.size
> 0 )
400 // Wouldn't it be great if I could use this...
401 //if ( (float)pkt.pts / 1000000.0 >= real_timecode )
402 ret
= avcodec_decode_video( codec_context
, &frame
, &got_picture
, pkt
.data
, pkt
.size
);
405 if ( (float)pkt
.pts
/ 1000000.0 < real_timecode
)
410 else if ( (float)pkt
.pts
/ 1000000.0 >= real_timecode
)
414 else if ( got_picture
&& ignore
-- )
419 current_time
= ( double )pkt
.pts
/ 1000000.0;
422 // We're finished with this packet regardless
423 av_free_packet( &pkt
);
426 // Now handle the picture if we have one
429 // Get current image and size
431 uint8_t *image
= mlt_properties_get_data( properties
, "current_image", &size
);
433 if ( image
== NULL
|| size
!= *width
* *height
* 2 )
435 size
= *width
* *height
* 2;
436 image
= malloc( size
);
437 mlt_properties_set_data( properties
, "current_image", image
, size
, free
, NULL
);
440 *buffer
= malloc( size
);
441 img_convert( output
, PIX_FMT_YUV422
, (AVPicture
*)&frame
, codec_context
->pix_fmt
, *width
, *height
);
442 memcpy( image
, output
->data
[ 0 ], size
);
443 memcpy( *buffer
, output
->data
[ 0 ], size
);
444 mlt_properties_set_data( frame_properties
, "image", *buffer
, size
, free
, NULL
);
445 mlt_properties_set_double( properties
, "current_time", current_time
);
449 // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
450 mlt_properties_set_position( properties
, "video_expected", position
+ 1 );
452 // Unlock the mutex now
453 pthread_mutex_unlock( &avformat_mutex
);
458 /** Set up video handling.
461 static void producer_set_up_video( mlt_producer
this, mlt_frame frame
)
463 // Get the properties
464 mlt_properties properties
= mlt_producer_properties( this );
466 // Fetch the video_context
467 AVFormatContext
*context
= mlt_properties_get_data( properties
, "video_context", NULL
);
469 // Get the video_index
470 int index
= mlt_properties_get_int( properties
, "video_index" );
472 // Lock the mutex now
473 pthread_mutex_lock( &avformat_mutex
);
475 if ( context
!= NULL
&& index
!= -1 )
477 // Get the frame properties
478 mlt_properties frame_properties
= mlt_frame_properties( frame
);
480 // Get the video stream
481 AVStream
*stream
= context
->streams
[ index
];
484 AVCodecContext
*codec_context
= &stream
->codec
;
487 AVCodec
*codec
= mlt_properties_get_data( properties
, "video_codec", NULL
);
489 // Initialise the codec if necessary
493 codec
= avcodec_find_decoder( codec_context
->codec_id
);
495 // If we don't have a codec and we can't initialise it, we can't do much more...
496 if ( codec
!= NULL
&& avcodec_open( codec_context
, codec
) >= 0 )
498 double aspect_ratio
= 0;
501 if ( codec_context
->sample_aspect_ratio
.num
== 0)
504 aspect_ratio
= av_q2d( codec_context
->sample_aspect_ratio
) * codec_context
->width
/ codec_context
->height
;
506 if (aspect_ratio
<= 0.0)
507 aspect_ratio
= ( double )codec_context
->width
/ ( double )codec_context
->height
;
509 mlt_properties_set_double( properties
, "aspect_ratio", aspect_ratio
);
511 // Now store the codec with its destructor
512 mlt_properties_set_data( properties
, "video_codec", codec_context
, 0, producer_codec_close
, NULL
);
514 // Set to the real timecode
515 av_seek_frame( context
, -1, 0 );
519 // Remember that we can't use this later
520 mlt_properties_set_int( properties
, "video_index", -1 );
524 // No codec, no show...
527 mlt_frame_push_get_image( frame
, producer_get_image
);
528 mlt_properties_set_data( frame_properties
, "avformat_producer", this, 0, NULL
, NULL
);
532 // Unlock the mutex now
533 pthread_mutex_unlock( &avformat_mutex
);
536 /** Get the audio from a frame.
539 static int producer_get_audio( mlt_frame frame
, int16_t **buffer
, mlt_audio_format
*format
, int *frequency
, int *channels
, int *samples
)
541 // Get the properties from the frame
542 mlt_properties frame_properties
= mlt_frame_properties( frame
);
544 // Obtain the frame number of this frame
545 mlt_position position
= mlt_properties_get_position( frame_properties
, "avformat_position" );
548 mlt_producer
this = mlt_properties_get_data( frame_properties
, "avformat_producer", NULL
);
550 // Get the producer properties
551 mlt_properties properties
= mlt_producer_properties( this );
553 // Fetch the audio_context
554 AVFormatContext
*context
= mlt_properties_get_data( properties
, "audio_context", NULL
);
556 // Get the audio_index
557 int index
= mlt_properties_get_int( properties
, "audio_index" );
559 // Obtain the expected frame numer
560 mlt_position expected
= mlt_properties_get_position( properties
, "audio_expected" );
562 // Obtain the resample context if it exists (not always needed)
563 ReSampleContext
*resample
= mlt_properties_get_data( properties
, "audio_resample", NULL
);
565 // Obtain the audio buffer
566 int16_t *audio_buffer
= mlt_properties_get_data( properties
, "audio_buffer", NULL
);
568 // Get amount of audio used
569 int audio_used
= mlt_properties_get_int( properties
, "audio_used" );
571 // Calculate the real time code
572 double real_timecode
= producer_time_of_frame( this, position
);
574 // Get the audio stream
575 AVStream
*stream
= context
->streams
[ index
];
578 AVCodecContext
*codec_context
= &stream
->codec
;
583 // Number of frames to ignore (for ffwd)
586 // Flag for paused (silence)
589 // Lock the mutex now
590 pthread_mutex_lock( &avformat_mutex
);
592 // Check for resample and create if necessary
593 if ( resample
== NULL
)
595 // Create the resampler
596 resample
= audio_resample_init( *channels
, codec_context
->channels
, *frequency
, codec_context
->sample_rate
);
598 // And store it on properties
599 mlt_properties_set_data( properties
, "audio_resample", resample
, 0, ( mlt_destructor
)audio_resample_close
, NULL
);
602 // Check for audio buffer and create if necessary
603 if ( audio_buffer
== NULL
)
605 // Allocate the audio buffer
606 audio_buffer
= malloc( AVCODEC_MAX_AUDIO_FRAME_SIZE
* sizeof( int16_t ) );
608 // And store it on properties for reuse
609 mlt_properties_set_data( properties
, "audio_buffer", audio_buffer
, 0, free
, NULL
);
613 if ( position
!= expected
)
615 if ( position
+ 1 == expected
)
617 // We're paused - silence required
620 else if ( position
> expected
&& ( position
- expected
) < 250 )
622 // Fast forward - seeking is inefficient for small distances - just ignore following frames
623 ignore
= position
- expected
;
627 // Set to the real timecode
628 av_seek_frame( context
, -1, real_timecode
* 1000000.0 );
630 // Clear the usage in the audio buffer
635 // Get the audio if required
640 int16_t temp
[ AVCODEC_MAX_AUDIO_FRAME_SIZE
/ 2 ];
642 memset( &pkt
, 0, sizeof( pkt
) );
644 while( ret
>= 0 && !got_audio
)
646 // Check if the buffer already contains the samples required
647 if ( audio_used
>= *samples
&& ignore
== 0 )
654 ret
= av_read_frame( context
, &pkt
);
657 uint8_t *ptr
= pkt
.data
;
660 if ( ptr
== NULL
|| len
== 0 )
663 // We only deal with video from the selected video_index
664 while ( ret
>= 0 && pkt
.stream_index
== index
&& len
> 0 )
667 ret
= avcodec_decode_audio( codec_context
, temp
, &data_size
, ptr
, len
);
677 int size_out
= audio_resample( resample
, &audio_buffer
[ audio_used
* *channels
], temp
, data_size
/ ( codec_context
->channels
* sizeof( int16_t ) ) );
679 audio_used
+= size_out
;
682 while ( ignore
&& audio_used
> *samples
)
685 audio_used
-= *samples
;
686 memmove( audio_buffer
, &audio_buffer
[ *samples
* *channels
], audio_used
* sizeof( int16_t ) );
690 // If we're behind, ignore this packet
691 float current_pts
= (float)pkt
.pts
/ 1000000.0;
692 double discrepancy
= mlt_properties_get_double( properties
, "discrepancy" );
693 if ( discrepancy
* current_pts
< real_timecode
)
697 // We're finished with this packet regardless
698 av_free_packet( &pkt
);
701 // Now handle the audio if we have enough
703 if ( audio_used
>= *samples
)
705 *buffer
= malloc( *samples
* *channels
* sizeof( int16_t ) );
706 memcpy( *buffer
, audio_buffer
, *samples
* *channels
* sizeof( int16_t ) );
707 audio_used
-= *samples
;
708 memmove( audio_buffer
, &audio_buffer
[ *samples
* *channels
], audio_used
* *channels
* sizeof( int16_t ) );
709 mlt_properties_set_data( frame_properties
, "audio", *buffer
, 0, free
, NULL
);
713 frame
->get_audio
= NULL
;
714 mlt_frame_get_audio( frame
, buffer
, format
, frequency
, channels
, samples
);
718 // Store the number of audio samples still available
719 mlt_properties_set_int( properties
, "audio_used", audio_used
);
723 // Get silence and don't touch the context
724 frame
->get_audio
= NULL
;
725 mlt_frame_get_audio( frame
, buffer
, format
, frequency
, channels
, samples
);
728 // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
729 mlt_properties_set_position( properties
, "audio_expected", position
+ 1 );
731 // Unlock the mutex now
732 pthread_mutex_unlock( &avformat_mutex
);
737 /** Set up audio handling.
740 static void producer_set_up_audio( mlt_producer
this, mlt_frame frame
)
742 // Get the properties
743 mlt_properties properties
= mlt_producer_properties( this );
745 // Fetch the audio_context
746 AVFormatContext
*context
= mlt_properties_get_data( properties
, "audio_context", NULL
);
748 // Get the audio_index
749 int index
= mlt_properties_get_int( properties
, "audio_index" );
751 // Lock the mutex now
752 pthread_mutex_lock( &avformat_mutex
);
754 // Deal with audio context
755 if ( context
!= NULL
&& index
!= -1 )
757 // Get the frame properties
758 mlt_properties frame_properties
= mlt_frame_properties( frame
);
760 // Get the audio stream
761 AVStream
*stream
= context
->streams
[ index
];
764 AVCodecContext
*codec_context
= &stream
->codec
;
767 AVCodec
*codec
= mlt_properties_get_data( properties
, "audio_codec", NULL
);
769 // Initialise the codec if necessary
773 codec
= avcodec_find_decoder( codec_context
->codec_id
);
775 // If we don't have a codec and we can't initialise it, we can't do much more...
776 if ( codec
!= NULL
&& avcodec_open( codec_context
, codec
) >= 0 )
778 // Now store the codec with its destructor
779 mlt_properties_set_data( properties
, "audio_codec", codec_context
, 0, producer_codec_close
, NULL
);
783 // Remember that we can't use this later
784 mlt_properties_set_int( properties
, "audio_index", -1 );
788 // No codec, no show...
791 frame
->get_audio
= producer_get_audio
;
792 mlt_properties_set_data( frame_properties
, "avformat_producer", this, 0, NULL
, NULL
);
796 // Unlock the mutex now
797 pthread_mutex_unlock( &avformat_mutex
);
800 /** Our get frame implementation.
803 static int producer_get_frame( mlt_producer
this, mlt_frame_ptr frame
, int index
)
805 // Create an empty frame
806 *frame
= mlt_frame_init( );
808 // Update timecode on the frame we're creating
809 mlt_frame_set_position( *frame
, mlt_producer_position( this ) );
811 // Set the position of this producer
812 mlt_properties_set_position( mlt_frame_properties( *frame
), "avformat_position", mlt_producer_get_in( this ) + mlt_producer_position( this ) );
815 producer_set_up_video( this, *frame
);
818 producer_set_up_audio( this, *frame
);
820 // Set the aspect_ratio
821 mlt_properties_set_double( mlt_frame_properties( *frame
), "aspect_ratio", mlt_properties_get_double( mlt_producer_properties( this ), "aspect_ratio" ) );
823 // Calculate the next timecode
824 mlt_producer_prepare_next( this );