2 * producer_avformat.c -- avformat producer
3 * Copyright (C) 2003-2004 Ushodaya Enterprises Limited
4 * Author: Charles Yates <charles.yates@pandora.be>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software Foundation,
18 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 #include "producer_avformat.h"
25 #include <framework/mlt_frame.h>
27 // ffmpeg Header files
28 #include <ffmpeg/avformat.h>
30 // System header files
35 // Forward references.
36 static int producer_open( mlt_producer
this, char *file
);
37 static int producer_get_frame( mlt_producer
this, mlt_frame_ptr frame
, int index
);
39 // A static flag used to determine if avformat has been initialised
40 static int avformat_initialised
= 0;
41 static pthread_mutex_t avformat_mutex
;
43 /** Constructor for libavformat.
46 mlt_producer
producer_avformat_init( char *file
)
48 mlt_producer
this = NULL
;
50 // Check that we have a non-NULL argument
53 // Construct the producer
54 this = calloc( 1, sizeof( struct mlt_producer_s
) );
57 if ( mlt_producer_init( this, NULL
) == 0 )
60 mlt_properties properties
= mlt_producer_properties( this );
62 // Set the resource property (required for all producers)
63 mlt_properties_set( properties
, "resource", file
);
65 // TEST: audio sync tweaking
66 mlt_properties_set_double( properties
, "discrepancy", 1 );
68 // Register our get_frame implementation
69 this->get_frame
= producer_get_frame
;
71 // Initialise avformat if necessary
72 if ( avformat_initialised
== 0 )
74 pthread_mutex_init( &avformat_mutex
, NULL
);
75 avformat_initialised
= 1;
80 if ( producer_open( this, file
) != 0 )
83 mlt_producer_close( this );
92 /** Find the default streams.
95 static void find_default_streams( AVFormatContext
*context
, int *audio_index
, int *video_index
)
99 // Allow for multiple audio and video streams in the file and select first of each (if available)
100 for( i
= 0; i
< context
->nb_streams
; i
++ )
102 // Get the codec context
103 AVCodecContext
*codec_context
= &context
->streams
[ i
]->codec
;
105 // Determine the type and obtain the first index of each type
106 switch( codec_context
->codec_type
)
108 case CODEC_TYPE_VIDEO
:
109 if ( *video_index
< 0 )
112 case CODEC_TYPE_AUDIO
:
113 if ( *audio_index
< 0 )
122 /** Producer file destructor.
125 static void producer_file_close( void *context
)
127 if ( context
!= NULL
)
129 // Lock the mutex now
130 pthread_mutex_lock( &avformat_mutex
);
133 av_close_input_file( context
);
135 // Unlock the mutex now
136 pthread_mutex_unlock( &avformat_mutex
);
140 /** Producer file destructor.
143 static void producer_codec_close( void *codec
)
147 // Lock the mutex now
148 pthread_mutex_lock( &avformat_mutex
);
151 avcodec_close( codec
);
153 // Unlock the mutex now
154 pthread_mutex_unlock( &avformat_mutex
);
160 NOTE: We need to have a valid [PAL or NTSC] frame rate before we can determine the
161 number of frames in the file. However, this is at odds with the way things work - the
162 constructor needs to provide in/out points before the user of the producer is able
163 to specify properties :-/. However, the PAL/NTSC distinction applies to all producers
164 and while we currently accept whatever the producer provides, this will not work in
165 the more general case. Plans are afoot... and this one will work without modification
166 (in theory anyway ;-)).
169 static int producer_open( mlt_producer
this, char *file
)
171 // Return an error code (0 == no error)
174 // Context for avformat
175 AVFormatContext
*context
= NULL
;
177 // Get the properties
178 mlt_properties properties
= mlt_producer_properties( this );
180 // We will treat everything with the producer fps
181 double fps
= mlt_properties_get_double( properties
, "fps" );
183 // Lock the mutex now
184 pthread_mutex_lock( &avformat_mutex
);
186 // Now attempt to open the file
187 error
= av_open_input_file( &context
, file
, NULL
, 0, NULL
) < 0;
189 // If successful, then try to get additional info
192 // Get the stream info
193 error
= av_find_stream_info( context
) < 0;
195 // Continue if no error
198 // We will default to the first audio and video streams found
199 int audio_index
= -1;
200 int video_index
= -1;
202 // Now set properties where we can (use default unknowns if required)
203 if ( context
->duration
!= AV_NOPTS_VALUE
)
205 // This isn't going to be accurate for all formats
206 mlt_position frames
= ( mlt_position
)( ( ( double )context
->duration
/ ( double )AV_TIME_BASE
) * fps
);
207 mlt_properties_set_position( properties
, "out", frames
- 2 );
208 mlt_properties_set_position( properties
, "length", frames
- 1 );
211 // Find default audio and video streams
212 find_default_streams( context
, &audio_index
, &video_index
);
214 // Store selected audio and video indexes on properties
215 mlt_properties_set_int( properties
, "audio_index", audio_index
);
216 mlt_properties_set_int( properties
, "video_index", video_index
);
218 // We're going to cheat here - for a/v files, we will have two contexts (reasoning will be clear later)
219 if ( audio_index
!= -1 && video_index
!= -1 )
221 // We'll use the open one as our video_context
222 mlt_properties_set_data( properties
, "video_context", context
, 0, producer_file_close
, NULL
);
224 // And open again for our audio context
225 av_open_input_file( &context
, file
, NULL
, 0, NULL
);
226 av_find_stream_info( context
);
229 mlt_properties_set_data( properties
, "audio_context", context
, 0, producer_file_close
, NULL
);
231 else if ( video_index
!= -1 )
233 // We only have a video context
234 mlt_properties_set_data( properties
, "video_context", context
, 0, producer_file_close
, NULL
);
236 else if ( audio_index
!= -1 )
238 // We only have an audio context
239 mlt_properties_set_data( properties
, "audio_context", context
, 0, producer_file_close
, NULL
);
243 // Something has gone wrong
249 // Unlock the mutex now
250 pthread_mutex_unlock( &avformat_mutex
);
255 /** Convert a frame position to a time code.
258 static double producer_time_of_frame( mlt_producer
this, mlt_position position
)
260 // Get the properties
261 mlt_properties properties
= mlt_producer_properties( this );
264 double fps
= mlt_properties_get_double( properties
, "fps" );
267 return ( double )position
/ fps
;
270 /** Get an image from a frame.
273 static int producer_get_image( mlt_frame frame
, uint8_t **buffer
, mlt_image_format
*format
, int *width
, int *height
, int writable
)
275 // Get the properties from the frame
276 mlt_properties frame_properties
= mlt_frame_properties( frame
);
278 // Obtain the frame number of this frame
279 mlt_position position
= mlt_properties_get_position( frame_properties
, "avformat_position" );
282 mlt_producer
this = mlt_properties_get_data( frame_properties
, "avformat_producer", NULL
);
284 // Get the producer properties
285 mlt_properties properties
= mlt_producer_properties( this );
287 // Fetch the video_context
288 AVFormatContext
*context
= mlt_properties_get_data( properties
, "video_context", NULL
);
290 // Get the video_index
291 int index
= mlt_properties_get_int( properties
, "video_index" );
293 // Obtain the expected frame numer
294 mlt_position expected
= mlt_properties_get_position( properties
, "video_expected" );
296 // Calculate the real time code
297 double real_timecode
= producer_time_of_frame( this, position
);
299 // Get the video stream
300 AVStream
*stream
= context
->streams
[ index
];
303 AVCodecContext
*codec_context
= &stream
->codec
;
308 // Get the conversion frame
309 AVPicture
*output
= mlt_properties_get_data( properties
, "video_output_frame", NULL
);
311 // Special case pause handling flag
314 // Special case ffwd handling
317 // Current time calcs
318 double current_time
= 0;
320 // Set the result arguments that we know here (only *buffer is now required)
321 *format
= mlt_image_yuv422
;
322 *width
= codec_context
->width
;
323 *height
= codec_context
->height
;
325 // Set this on the frame properties
326 mlt_properties_set_int( frame_properties
, "width", *width
);
327 mlt_properties_set_int( frame_properties
, "height", *height
);
329 // Lock the mutex now
330 pthread_mutex_lock( &avformat_mutex
);
332 // Construct an AVFrame for YUV422 conversion
333 if ( output
== NULL
)
335 int size
= avpicture_get_size( PIX_FMT_YUV422
, *width
, *height
);
336 uint8_t *buf
= malloc( size
);
337 output
= malloc( sizeof( AVPicture
) );
338 avpicture_fill( output
, buf
, PIX_FMT_YUV422
, *width
, *height
);
339 mlt_properties_set_data( properties
, "video_output_frame", output
, 0, av_free
, NULL
);
340 mlt_properties_set_data( properties
, "video_output_buffer", buf
, 0, free
, NULL
);
344 if ( position
!= expected
)
346 if ( position
+ 1 == expected
)
348 // We're paused - use last image
351 else if ( position
> expected
&& ( position
- expected
) < 250 )
353 // Fast forward - seeking is inefficient for small distances - just ignore following frames
354 ignore
= position
- expected
;
358 // Set to the real timecode
359 av_seek_frame( context
, -1, real_timecode
* 1000000.0 );
361 // Remove the cached info relating to the previous position
362 mlt_properties_set_double( properties
, "current_time", 0 );
363 mlt_properties_set_data( properties
, "current_image", NULL
, 0, NULL
, NULL
);
367 // Duplicate the last image if necessary
368 if ( mlt_properties_get_data( properties
, "current_image", NULL
) != NULL
&&
369 ( paused
|| mlt_properties_get_double( properties
, "current_time" ) > real_timecode
) )
371 // Get current image and size
373 uint8_t *image
= mlt_properties_get_data( properties
, "current_image", &size
);
376 *buffer
= malloc( size
);
377 memcpy( *buffer
, image
, size
);
379 // Set this on the frame properties
380 mlt_properties_set_data( frame_properties
, "image", *buffer
, size
, free
, NULL
);
388 memset( &pkt
, 0, sizeof( pkt
) );
389 memset( &frame
, 0, sizeof( frame
) );
391 while( ret
>= 0 && !got_picture
)
394 ret
= av_read_frame( context
, &pkt
);
396 // We only deal with video from the selected video_index
397 if ( ret
>= 0 && pkt
.stream_index
== index
&& pkt
.size
> 0 )
400 // Wouldn't it be great if I could use this...
401 //if ( (float)pkt.pts / 1000000.0 >= real_timecode )
402 ret
= avcodec_decode_video( codec_context
, &frame
, &got_picture
, pkt
.data
, pkt
.size
);
405 if ( (float)pkt
.pts
/ 1000000.0 < real_timecode
)
410 else if ( (float)pkt
.pts
/ 1000000.0 >= real_timecode
)
414 else if ( got_picture
&& ignore
-- )
419 current_time
= ( double )pkt
.pts
/ 1000000.0;
422 // We're finished with this packet regardless
423 av_free_packet( &pkt
);
426 // Now handle the picture if we have one
429 // Get current image and size
431 uint8_t *image
= mlt_properties_get_data( properties
, "current_image", &size
);
433 if ( image
== NULL
|| size
!= *width
* *height
* 2 )
435 size
= *width
* *height
* 2;
436 image
= malloc( size
);
437 mlt_properties_set_data( properties
, "current_image", image
, size
, free
, NULL
);
440 *buffer
= malloc( size
);
441 img_convert( output
, PIX_FMT_YUV422
, (AVPicture
*)&frame
, codec_context
->pix_fmt
, *width
, *height
);
442 memcpy( image
, output
->data
[ 0 ], size
);
443 memcpy( *buffer
, output
->data
[ 0 ], size
);
444 mlt_properties_set_data( frame_properties
, "image", *buffer
, size
, free
, NULL
);
445 mlt_properties_set_double( properties
, "current_time", current_time
);
449 // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
450 mlt_properties_set_position( properties
, "video_expected", position
+ 1 );
452 // Unlock the mutex now
453 pthread_mutex_unlock( &avformat_mutex
);
458 /** Set up video handling.
461 static void producer_set_up_video( mlt_producer
this, mlt_frame frame
)
463 // Get the properties
464 mlt_properties properties
= mlt_producer_properties( this );
466 // Fetch the video_context
467 AVFormatContext
*context
= mlt_properties_get_data( properties
, "video_context", NULL
);
469 // Get the video_index
470 int index
= mlt_properties_get_int( properties
, "video_index" );
472 // Lock the mutex now
473 pthread_mutex_lock( &avformat_mutex
);
475 if ( context
!= NULL
&& index
!= -1 )
477 // Get the frame properties
478 mlt_properties frame_properties
= mlt_frame_properties( frame
);
480 // Get the video stream
481 AVStream
*stream
= context
->streams
[ index
];
484 AVCodecContext
*codec_context
= &stream
->codec
;
487 AVCodec
*codec
= mlt_properties_get_data( properties
, "video_codec", NULL
);
489 // Initialise the codec if necessary
493 codec
= avcodec_find_decoder( codec_context
->codec_id
);
495 // If we don't have a codec and we can't initialise it, we can't do much more...
496 if ( codec
!= NULL
&& avcodec_open( codec_context
, codec
) >= 0 )
498 double aspect_ratio
= 0;
501 if ( codec_context
->sample_aspect_ratio
.num
== 0)
504 aspect_ratio
= av_q2d( codec_context
->sample_aspect_ratio
) * codec_context
->width
/ codec_context
->height
;
506 if (aspect_ratio
<= 0.0)
507 aspect_ratio
= ( double )codec_context
->width
/ ( double )codec_context
->height
;
509 mlt_properties_set_double( properties
, "aspect_ratio", aspect_ratio
);
510 fprintf( stderr
, "avformat: %dx%d (%f)\n", codec_context
->width
, codec_context
->height
, av_q2d( codec_context
->sample_aspect_ratio
) );
512 // Now store the codec with its destructor
513 mlt_properties_set_data( properties
, "video_codec", codec_context
, 0, producer_codec_close
, NULL
);
515 // Set to the real timecode
516 av_seek_frame( context
, -1, 0 );
520 // Remember that we can't use this later
521 mlt_properties_set_int( properties
, "video_index", -1 );
525 // No codec, no show...
528 mlt_frame_push_get_image( frame
, producer_get_image
);
529 mlt_properties_set_data( frame_properties
, "avformat_producer", this, 0, NULL
, NULL
);
533 // Unlock the mutex now
534 pthread_mutex_unlock( &avformat_mutex
);
537 /** Get the audio from a frame.
540 static int producer_get_audio( mlt_frame frame
, int16_t **buffer
, mlt_audio_format
*format
, int *frequency
, int *channels
, int *samples
)
542 // Get the properties from the frame
543 mlt_properties frame_properties
= mlt_frame_properties( frame
);
545 // Obtain the frame number of this frame
546 mlt_position position
= mlt_properties_get_position( frame_properties
, "avformat_position" );
549 mlt_producer
this = mlt_properties_get_data( frame_properties
, "avformat_producer", NULL
);
551 // Get the producer properties
552 mlt_properties properties
= mlt_producer_properties( this );
554 // Fetch the audio_context
555 AVFormatContext
*context
= mlt_properties_get_data( properties
, "audio_context", NULL
);
557 // Get the audio_index
558 int index
= mlt_properties_get_int( properties
, "audio_index" );
560 // Obtain the expected frame numer
561 mlt_position expected
= mlt_properties_get_position( properties
, "audio_expected" );
563 // Obtain the resample context if it exists (not always needed)
564 ReSampleContext
*resample
= mlt_properties_get_data( properties
, "audio_resample", NULL
);
566 // Obtain the audio buffer
567 int16_t *audio_buffer
= mlt_properties_get_data( properties
, "audio_buffer", NULL
);
569 // Get amount of audio used
570 int audio_used
= mlt_properties_get_int( properties
, "audio_used" );
572 // Calculate the real time code
573 double real_timecode
= producer_time_of_frame( this, position
);
575 // Get the audio stream
576 AVStream
*stream
= context
->streams
[ index
];
579 AVCodecContext
*codec_context
= &stream
->codec
;
584 // Number of frames to ignore (for ffwd)
587 // Flag for paused (silence)
590 // Lock the mutex now
591 pthread_mutex_lock( &avformat_mutex
);
593 // Check for resample and create if necessary
594 if ( resample
== NULL
)
596 // Create the resampler
597 resample
= audio_resample_init( *channels
, codec_context
->channels
, *frequency
, codec_context
->sample_rate
);
599 // And store it on properties
600 mlt_properties_set_data( properties
, "audio_resample", resample
, 0, ( mlt_destructor
)audio_resample_close
, NULL
);
603 // Check for audio buffer and create if necessary
604 if ( audio_buffer
== NULL
)
606 // Allocate the audio buffer
607 audio_buffer
= malloc( AVCODEC_MAX_AUDIO_FRAME_SIZE
* sizeof( int16_t ) );
609 // And store it on properties for reuse
610 mlt_properties_set_data( properties
, "audio_buffer", audio_buffer
, 0, free
, NULL
);
614 if ( position
!= expected
)
616 if ( position
+ 1 == expected
)
618 // We're paused - silence required
621 else if ( position
> expected
&& ( position
- expected
) < 250 )
623 // Fast forward - seeking is inefficient for small distances - just ignore following frames
624 ignore
= position
- expected
;
628 // Set to the real timecode
629 av_seek_frame( context
, -1, real_timecode
* 1000000.0 );
631 // Clear the usage in the audio buffer
636 // Get the audio if required
641 int16_t temp
[ AVCODEC_MAX_AUDIO_FRAME_SIZE
/ 2 ];
643 memset( &pkt
, 0, sizeof( pkt
) );
645 while( ret
>= 0 && !got_audio
)
647 // Check if the buffer already contains the samples required
648 if ( audio_used
>= *samples
&& ignore
== 0 )
655 ret
= av_read_frame( context
, &pkt
);
658 uint8_t *ptr
= pkt
.data
;
661 if ( ptr
== NULL
|| len
== 0 )
664 // We only deal with video from the selected video_index
665 while ( ret
>= 0 && pkt
.stream_index
== index
&& len
> 0 )
668 ret
= avcodec_decode_audio( codec_context
, temp
, &data_size
, ptr
, len
);
678 int size_out
= audio_resample( resample
, &audio_buffer
[ audio_used
* *channels
], temp
, data_size
/ ( codec_context
->channels
* sizeof( int16_t ) ) );
680 audio_used
+= size_out
;
683 while ( ignore
&& audio_used
> *samples
)
686 audio_used
-= *samples
;
687 memmove( audio_buffer
, &audio_buffer
[ *samples
* *channels
], audio_used
* sizeof( int16_t ) );
691 // If we're behind, ignore this packet
692 float current_pts
= (float)pkt
.pts
/ 1000000.0;
693 double discrepancy
= mlt_properties_get_double( properties
, "discrepancy" );
694 if ( discrepancy
* current_pts
< real_timecode
)
698 // We're finished with this packet regardless
699 av_free_packet( &pkt
);
702 // Now handle the audio if we have enough
704 if ( audio_used
>= *samples
)
706 *buffer
= malloc( *samples
* *channels
* sizeof( int16_t ) );
707 memcpy( *buffer
, audio_buffer
, *samples
* *channels
* sizeof( int16_t ) );
708 audio_used
-= *samples
;
709 memmove( audio_buffer
, &audio_buffer
[ *samples
* *channels
], audio_used
* *channels
* sizeof( int16_t ) );
710 mlt_properties_set_data( frame_properties
, "audio", *buffer
, 0, free
, NULL
);
714 frame
->get_audio
= NULL
;
715 mlt_frame_get_audio( frame
, buffer
, format
, frequency
, channels
, samples
);
719 // Store the number of audio samples still available
720 mlt_properties_set_int( properties
, "audio_used", audio_used
);
724 // Get silence and don't touch the context
725 frame
->get_audio
= NULL
;
726 mlt_frame_get_audio( frame
, buffer
, format
, frequency
, channels
, samples
);
729 // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
730 mlt_properties_set_position( properties
, "audio_expected", position
+ 1 );
732 // Unlock the mutex now
733 pthread_mutex_unlock( &avformat_mutex
);
738 /** Set up audio handling.
741 static void producer_set_up_audio( mlt_producer
this, mlt_frame frame
)
743 // Get the properties
744 mlt_properties properties
= mlt_producer_properties( this );
746 // Fetch the audio_context
747 AVFormatContext
*context
= mlt_properties_get_data( properties
, "audio_context", NULL
);
749 // Get the audio_index
750 int index
= mlt_properties_get_int( properties
, "audio_index" );
752 // Lock the mutex now
753 pthread_mutex_lock( &avformat_mutex
);
755 // Deal with audio context
756 if ( context
!= NULL
&& index
!= -1 )
758 // Get the frame properties
759 mlt_properties frame_properties
= mlt_frame_properties( frame
);
761 // Get the audio stream
762 AVStream
*stream
= context
->streams
[ index
];
765 AVCodecContext
*codec_context
= &stream
->codec
;
768 AVCodec
*codec
= mlt_properties_get_data( properties
, "audio_codec", NULL
);
770 // Initialise the codec if necessary
774 codec
= avcodec_find_decoder( codec_context
->codec_id
);
776 // If we don't have a codec and we can't initialise it, we can't do much more...
777 if ( codec
!= NULL
&& avcodec_open( codec_context
, codec
) >= 0 )
779 // Now store the codec with its destructor
780 mlt_properties_set_data( properties
, "audio_codec", codec_context
, 0, producer_codec_close
, NULL
);
784 // Remember that we can't use this later
785 mlt_properties_set_int( properties
, "audio_index", -1 );
789 // No codec, no show...
792 frame
->get_audio
= producer_get_audio
;
793 mlt_properties_set_data( frame_properties
, "avformat_producer", this, 0, NULL
, NULL
);
797 // Unlock the mutex now
798 pthread_mutex_unlock( &avformat_mutex
);
801 /** Our get frame implementation.
804 static int producer_get_frame( mlt_producer
this, mlt_frame_ptr frame
, int index
)
806 // Create an empty frame
807 *frame
= mlt_frame_init( );
809 // Update timecode on the frame we're creating
810 mlt_frame_set_position( *frame
, mlt_producer_position( this ) );
812 // Set the position of this producer
813 mlt_properties_set_position( mlt_frame_properties( *frame
), "avformat_position", mlt_producer_get_in( this ) + mlt_producer_position( this ) );
816 producer_set_up_video( this, *frame
);
819 producer_set_up_audio( this, *frame
);
821 // Set the aspect_ratio
822 mlt_properties_set_double( mlt_frame_properties( *frame
), "aspect_ratio", mlt_properties_get_double( mlt_producer_properties( this ), "aspect_ratio" ) );
824 // Calculate the next timecode
825 mlt_producer_prepare_next( this );