2 * /brief fast motion estimation filter
3 * /author Zachary Drew, Copyright 2005
5 * Currently only uses Gamma data for comparisonon (bug or feature?)
6 * SSE optimized where available.
8 * Vector orientation: The vector data that is generated for the current frame specifies
9 * the motion from the previous frame to the current frame. To know how a macroblock
10 * in the current frame will move in the future, the next frame is needed.
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software Foundation,
24 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
28 #include "filter_motion_est.h"
29 #include <framework/mlt.h>
49 #define DIAMOND_SEARCH 0x0
50 #define FULL_SEARCH 0x1
52 #define MIN(a,b) ((a) > (b) ? (b) : (a))
53 #define ABS(a) ((a) >= 0 ? (a) : (-(a)))
56 struct motion_est_context_s
58 int initialized
; // true if filter has been initialized
64 /* same as mlt_frame's parameters */
67 /* Operational details */
70 uint8_t *cache_image
; // Copy of current frame
71 uint8_t *former_image
; // Copy of former frame
75 int limit_x
, limit_y
; // max x and y of a motion vector
77 int check_chroma
; // if check_chroma == 1 then compare chroma
80 int show_reconstruction
;
81 int toggle_when_paused
;
85 struct mlt_geometry_item_s bounds
; // Current bounds (from filters crop_detect, autotrack rectangle, or other)
87 /* bounds in macroblock units; macroblocks are completely contained within the boundry */
88 int left_mb
, prev_left_mb
, right_mb
, prev_right_mb
;
89 int top_mb
, prev_top_mb
, bottom_mb
, prev_bottom_mb
;
91 /* size of our vector buffers */
92 int mv_buffer_height
, mv_buffer_width
, mv_size
;
95 int former_vectors_valid
; //<! true if the previous frame's buffered motion vector data is valid
96 motion_vector
*former_vectors
;
97 motion_vector
*current_vectors
;
98 motion_vector
*denoise_vectors
;
99 mlt_position former_frame_position
, current_frame_position
;
101 /* diagnostic metrics */
102 float predictive_misses
; // How often do the prediction motion vectors fail?
103 int comparison_average
; // How far does the best estimation deviate from a perfect comparison?
106 int average_x
, average_y
;
108 /* run-time configurable comparison functions */
109 int (*compare_reference
)(uint8_t *, uint8_t *, int, int, int, int);
110 int (*compare_optimized
)(uint8_t *, uint8_t *, int, int, int, int);
114 // This is used to constrains pixel operations between two blocks to be within the image boundry
115 inline static int constrain( int *x
, int *y
, int *w
, int *h
,
116 const int dx
, const int dy
,
117 const int left
, const int right
,
118 const int top
, const int bottom
)
120 uint32_t penalty
= 1 << SHIFT
; // Retain a few extra bits of precision
126 // Origin of macroblock moves left of image boundy
127 if( *x
< left
|| x2
< left
) {
128 w_remains
= *w
- left
+ ((*x
< x2
) ?
*x
: x2
);
129 *x
+= *w
- w_remains
;
131 // Portion of macroblock moves right of image boundry
132 else if( *x
+ *w
> right
|| x2
+ *w
> right
)
133 w_remains
= right
- ((*x
> x2
) ?
*x
: x2
);
135 // Origin of macroblock moves above image boundy
136 if( *y
< top
|| y2
< top
) {
137 h_remains
= *h
- top
+ ((*y
< y2
) ?
*y
: y2
);
138 *y
+= *h
- h_remains
;
140 // Portion of macroblock moves bellow image boundry
141 else if( *y
+ *h
> bottom
|| y2
+ *h
> bottom
)
142 h_remains
= bottom
- ((*y
> y2
) ?
*y
: y2
);
144 if( w_remains
== *w
&& h_remains
== *h
) return penalty
;
145 if( w_remains
<= 0 || h_remains
<= 0) return 0; // Block is clipped out of existance
146 penalty
= (*w
* *h
* penalty
)
147 / ( w_remains
* h_remains
); // Recipricol of the fraction of the block that remains
149 assert(*x
>= left
); assert(x2
+ *w
- w_remains
>= left
);
150 assert(*y
>= top
); assert(y2
+ *h
- h_remains
>= top
);
151 assert(*x
+ w_remains
<= right
); assert(x2
+ w_remains
<= right
);
152 assert(*y
+ h_remains
<= bottom
); assert(y2
+ h_remains
<= bottom
);
154 *w
= w_remains
; // Update the width and height
160 /** /brief Reference Sum of Absolute Differences comparison function
163 inline static int sad_reference( uint8_t *block1
, uint8_t *block2
, const int xstride
, const int ystride
, const int w
, const int h
)
166 for ( j
= 0; j
< h
; j
++ ){
167 for ( i
= 0; i
< w
; i
++ ){
168 score
+= ABS( block1
[i
*xstride
] - block2
[i
*xstride
] );
178 /** /brief Abstracted block comparison function
180 inline static int block_compare( uint8_t *block1
,
186 struct motion_est_context_s
*c
)
189 #ifdef COUNT_COMPARES
195 // Default comparison may be overridden by the slower, more capable reference comparison
196 int (*cmp
)(uint8_t *, uint8_t *, int, int, int, int) = c
->compare_optimized
;
198 // vector displacement limited has been exceeded
199 if( ABS( dx
) >= c
->limit_x
|| ABS( dy
) >= c
->limit_y
)
202 int mb_w
= c
->mb_w
; // Some writeable local copies
205 // Determine if either macroblock got clipped
206 int penalty
= constrain( &x
, &y
, &mb_w
, &mb_h
, dx
, dy
, 0, c
->width
, 0, c
->height
);
209 if( penalty
== 0 ) // Clipped out of existance: Return worst score
211 else if( penalty
!= 1<<SHIFT
) // Nonstandard macroblock dimensions: Disable SIMD optimizizations.
212 cmp
= c
->compare_reference
;
214 // Calculate the memory locations of the macroblocks
215 block1
+= x
* c
->xstride
+ y
* c
->ystride
;
216 block2
+= (x
+dx
) * c
->xstride
+ (y
+dy
) * c
->ystride
;
219 if( penalty
== 1<<SHIFT
){
220 score
= c
->compare_reference( block1
, block2
, c
->xstride
, c
->ystride
, mb_w
, mb_h
);
221 int score2
= c
->compare_optimized( block1
, block2
, c
->xstride
, c
->ystride
, mb_w
, mb_h
);
222 if ( score
!= score2
)
223 fprintf(stderr
, "Your assembly doesn't work! Reference: %d Asm: %d\n", score
, score2
);
228 score
= cmp( block1
, block2
, c
->xstride
, c
->ystride
, mb_w
, mb_h
);
230 return ( score
* penalty
) >> SHIFT
; // Ditch the extra precision
233 static inline void check_candidates ( uint8_t *ref
,
234 uint8_t *candidate_base
,
237 const motion_vector
*candidates
,// Contains to_x & to_y
238 const int count
, // Number of candidates
239 const int unique
, // Sometimes we know the candidates are unique
240 motion_vector
*result
,
241 struct motion_est_context_s
*c
)
244 /* Scan for the best candidate */
245 for ( i
= 0; i
< count
; i
++ )
247 // this little dohicky ignores duplicate candidates, if they are possible
252 if ( candidates
[j
].dx
== candidates
[i
].dx
&&
253 candidates
[j
].dy
== candidates
[i
].dy
)
261 score
= block_compare( ref
, candidate_base
,
263 candidates
[i
].dx
, // from
267 if ( score
< result
->msad
) { // New minimum
268 result
->dx
= candidates
[i
].dx
;
269 result
->dy
= candidates
[i
].dy
;
270 result
->msad
= score
;
276 /* /brief Diamond search
277 * Operates on a single macroblock
279 static inline void diamond_search(
280 uint8_t *ref
, //<! Image data from previous frame
281 uint8_t *candidate_base
, //<! Image data in current frame
282 const int x
, //<! X upper left corner of macroblock
283 const int y
, //<! U upper left corner of macroblock
284 struct motion_vector_s
*result
, //<! Best predicted mv and eventual result
285 struct motion_est_context_s
*c
) //<! motion estimation context
288 // diamond search pattern
289 motion_vector candidates
[4];
291 // Keep track of best and former best candidates
292 motion_vector best
, former
;
296 // The direction of the refinement needs to be known
297 motion_vector current
;
301 // Loop through the search pattern
304 current
.dx
= result
->dx
;
305 current
.dy
= result
->dy
;
307 if ( first
== 1 ) // Set the initial pattern
309 candidates
[0].dx
= result
->dx
+ 1; candidates
[0].dy
= result
->dy
+ 0;
310 candidates
[1].dx
= result
->dx
+ 0; candidates
[1].dy
= result
->dy
+ 1;
311 candidates
[2].dx
= result
->dx
- 1; candidates
[2].dy
= result
->dy
+ 0;
312 candidates
[3].dx
= result
->dx
+ 0; candidates
[3].dy
= result
->dy
- 1;
315 else // Construct the next portion of the search pattern
317 candidates
[0].dx
= result
->dx
+ best
.dx
;
318 candidates
[0].dy
= result
->dy
+ best
.dy
;
319 if (best
.dx
== former
.dx
&& best
.dy
== former
.dy
) {
320 candidates
[1].dx
= result
->dx
+ best
.dy
;
321 candidates
[1].dy
= result
->dy
+ best
.dx
; // Yes, the wires
322 candidates
[2].dx
= result
->dx
- best
.dy
; // are crossed
323 candidates
[2].dy
= result
->dy
- best
.dx
;
326 candidates
[1].dx
= result
->dx
+ former
.dx
;
327 candidates
[1].dy
= result
->dy
+ former
.dy
;
331 former
.dx
= best
.dx
; former
.dy
= best
.dy
; // Keep track of new former best
334 check_candidates ( ref
, candidate_base
, x
, y
, candidates
, i
, 1, result
, c
);
336 // Which candidate was the best?
337 best
.dx
= result
->dx
- current
.dx
;
338 best
.dy
= result
->dy
- current
.dy
;
340 // A better canidate was not found
341 if ( best
.dx
== 0 && best
.dy
== 0 )
346 former
.dx
= best
.dx
; former
.dy
= best
.dy
; // First iteration, sensible value for former.d*
351 /* /brief Full (brute) search
352 * Operates on a single macroblock
354 __attribute__((used
))
355 static void full_search(
356 uint8_t *ref
, //<! Image data from previous frame
357 uint8_t *candidate_base
, //<! Image data in current frame
358 int x
, //<! X upper left corner of macroblock
359 int y
, //<! U upper left corner of macroblock
360 struct motion_vector_s
*result
, //<! Best predicted mv and eventual result
361 struct motion_est_context_s
*c
) //<! motion estimation context
363 // Keep track of best candidate
367 for( i
= -c
->mb_w
; i
<= c
->mb_w
; i
++ ){
368 for( j
= -c
->mb_h
; j
<= c
->mb_h
; j
++ ){
370 score
= block_compare( ref
, candidate_base
,
377 if ( score
< result
->msad
) {
380 result
->msad
= score
;
386 // Macros for pointer calculations
387 #define CURRENT(i,j) ( c->current_vectors + (j)*c->mv_buffer_width + (i) )
388 #define FORMER(i,j) ( c->former_vectors + (j)*c->mv_buffer_width + (i) )
389 #define DENOISE(i,j) ( c->denoise_vectors + (j)*c->mv_buffer_width + (i) )
391 int ncompare (const void * a
, const void * b
)
393 return ( *(int*)a
- *(int*)b
);
396 // motion vector denoising
397 // for x and y components seperately,
398 // change the vector to be the median value of the 9 adjacent vectors
399 static void median_denoise( motion_vector
*v
, struct motion_est_context_s
*c
)
401 int xvalues
[9], yvalues
[9];
404 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ )
405 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
409 xvalues
[n
] = CURRENT(i
,j
)->dx
; // Center
410 yvalues
[n
++] = CURRENT(i
,j
)->dy
;
412 if( i
> c
->left_mb
) // Not in First Column
414 xvalues
[n
] = CURRENT(i
-1,j
)->dx
; // Left
415 yvalues
[n
++] = CURRENT(i
-1,j
)->dy
;
417 if( j
> c
->top_mb
) {
418 xvalues
[n
] = CURRENT(i
-1,j
-1)->dx
; // Upper Left
419 yvalues
[n
++] = CURRENT(i
-1,j
-1)->dy
;
422 if( j
< c
->bottom_mb
) {
423 xvalues
[n
] = CURRENT(i
-1,j
+1)->dx
; // Bottom Left
424 yvalues
[n
++] = CURRENT(i
-1,j
+1)->dy
;
427 if( i
< c
->right_mb
) // Not in Last Column
429 xvalues
[n
] = CURRENT(i
+1,j
)->dx
; // Right
430 yvalues
[n
++] = CURRENT(i
+1,j
)->dy
;
433 if( j
> c
->top_mb
) {
434 xvalues
[n
] = CURRENT(i
+1,j
-1)->dx
; // Upper Right
435 yvalues
[n
++] = CURRENT(i
+1,j
-1)->dy
;
438 if( j
< c
->bottom_mb
) {
439 xvalues
[n
] = CURRENT(i
+1,j
+1)->dx
; // Bottom Right
440 yvalues
[n
++] = CURRENT(i
+1,j
+1)->dy
;
443 if( j
> c
->top_mb
) // Not in First Row
445 xvalues
[n
] = CURRENT(i
,j
-1)->dx
; // Top
446 yvalues
[n
++] = CURRENT(i
,j
-1)->dy
;
449 if( j
< c
->bottom_mb
) // Not in Last Row
451 xvalues
[n
] = CURRENT(i
,j
+1)->dx
; // Bottom
452 yvalues
[n
++] = CURRENT(i
,j
+1)->dy
;
455 qsort (xvalues
, n
, sizeof(int), ncompare
);
456 qsort (yvalues
, n
, sizeof(int), ncompare
);
459 DENOISE(i
,j
)->dx
= xvalues
[n
/2];
460 DENOISE(i
,j
)->dy
= yvalues
[n
/2];
463 DENOISE(i
,j
)->dx
= (xvalues
[n
/2] + xvalues
[n
/2+1])/2;
464 DENOISE(i
,j
)->dy
= (yvalues
[n
/2] + yvalues
[n
/2+1])/2;
469 motion_vector
*t
= c
->current_vectors
;
470 c
->current_vectors
= c
->denoise_vectors
;
471 c
->denoise_vectors
= t
;
477 static inline int median_predictor(int a
, int b
, int c
) {
493 /** /brief Motion search
495 * For each macroblock in the current frame, estimate the block from the last frame that
498 * Vocab: Colocated - the pixel in the previous frame at the current position
500 * Based on enhanced predictive zonal search. [Tourapis 2002]
502 static void motion_search( uint8_t *from
, //<! Image data.
503 uint8_t *to
, //<! Image data. Rigid grid.
504 struct motion_est_context_s
*c
) //<! The context
507 #ifdef COUNT_COMPARES
511 motion_vector candidates
[10];
512 motion_vector
*here
; // This one gets used alot (about 30 times per macroblock)
517 // For every macroblock, perform motion vector estimation
518 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
519 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
524 here
->msad
= MAX_MSAD
;
529 /* Stack the predictors [i.e. checked in reverse order] */
531 /* Adjacent to collocated */
532 if( c
->former_vectors_valid
)
535 if( j
> c
->prev_top_mb
){// && COL_TOP->valid ){
536 candidates
[n
].dx
= FORMER(i
,j
-1)->dx
;
537 candidates
[n
++].dy
= FORMER(i
,j
-1)->dy
;
541 if( i
> c
->prev_left_mb
){// && COL_LEFT->valid ){
542 candidates
[n
].dx
= FORMER(i
-1,j
)->dx
;
543 candidates
[n
++].dy
= FORMER(i
-1,j
)->dy
;
546 // Right of colocated
547 if( i
< c
->prev_right_mb
){// && COL_RIGHT->valid ){
548 candidates
[n
].dx
= FORMER(i
+1,j
)->dx
;
549 candidates
[n
++].dy
= FORMER(i
+1,j
)->dy
;
552 // Bottom of colocated
553 if( j
< c
->prev_bottom_mb
){// && COL_BOTTOM->valid ){
554 candidates
[n
].dx
= FORMER(i
,j
+1)->dx
;
555 candidates
[n
++].dy
= FORMER(i
,j
+1)->dy
;
558 // And finally, colocated
559 candidates
[n
].dx
= FORMER(i
,j
)->dx
;
560 candidates
[n
++].dy
= FORMER(i
,j
)->dy
;
563 // For macroblocks not in the top row
564 if ( j
> c
->top_mb
) {
566 // Top if ( TOP->valid ) {
567 candidates
[n
].dx
= CURRENT(i
,j
-1)->dx
;
568 candidates
[n
++].dy
= CURRENT(i
,j
-1)->dy
;
571 // Top-Right, macroblocks not in the right row
572 if ( i
< c
->right_mb
){// && TOP_RIGHT->valid ) {
573 candidates
[n
].dx
= CURRENT(i
+1,j
-1)->dx
;
574 candidates
[n
++].dy
= CURRENT(i
+1,j
-1)->dy
;
578 // Left, Macroblocks not in the left column
579 if ( i
> c
->left_mb
){// && LEFT->valid ) {
580 candidates
[n
].dx
= CURRENT(i
-1,j
)->dx
;
581 candidates
[n
++].dy
= CURRENT(i
-1,j
)->dy
;
584 /* Median predictor vector (median of left, top, and top right adjacent vectors) */
585 if ( i
> c
->left_mb
&& j
> c
->top_mb
&& i
< c
->right_mb
586 )//&& LEFT->valid && TOP->valid && TOP_RIGHT->valid )
588 candidates
[n
].dx
= median_predictor( CURRENT(i
-1,j
)->dx
, CURRENT(i
,j
-1)->dx
, CURRENT(i
+1,j
-1)->dx
);
589 candidates
[n
++].dy
= median_predictor( CURRENT(i
-1,j
)->dy
, CURRENT(i
,j
-1)->dy
, CURRENT(i
+1,j
-1)->dy
);
593 candidates
[n
].dx
= 0;
594 candidates
[n
++].dy
= 0;
598 check_candidates ( to
, from
, x
, y
, candidates
, n
, 0, here
, c
);
602 diamond_search( to
, from
, x
, y
, here
, c
);
604 full_search( to
, from
, x
, y
, here
, c
);
607 assert( x
+ c
->mb_w
+ here
->dx
> 0 ); // All macroblocks must have area > 0
608 assert( y
+ c
->mb_h
+ here
->dy
> 0 );
609 assert( x
+ here
->dx
< c
->width
);
610 assert( y
+ here
->dy
< c
->height
);
612 } /* End column loop */
616 asm volatile ( "emms" );
619 #ifdef COUNT_COMPARES
620 fprintf(stderr
, "%d comparisons per block were made", compares
/count
);
625 void collect_post_statistics( struct motion_est_context_s
*c
) {
627 c
->comparison_average
= 0;
628 c
->average_length
= 0;
634 for ( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
635 for ( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
638 c
->comparison_average
+= CURRENT(i
,j
)->msad
;
639 c
->average_x
+= CURRENT(i
,j
)->dx
;
640 c
->average_y
+= CURRENT(i
,j
)->dy
;
648 c
->comparison_average
/= count
;
649 c
->average_x
/= count
;
650 c
->average_y
/= count
;
651 c
->average_length
= sqrt( c
->average_x
* c
->average_x
+ c
->average_y
* c
->average_y
);
656 static void init_optimizations( struct motion_est_context_s
*c
)
660 case 4: if(c
->mb_h
== 4) c
->compare_optimized
= sad_sse_422_luma_4x4
;
661 else c
->compare_optimized
= sad_sse_422_luma_4w
;
663 case 8: if(c
->mb_h
== 8) c
->compare_optimized
= sad_sse_422_luma_8x8
;
664 else c
->compare_optimized
= sad_sse_422_luma_8w
;
666 case 16: if(c
->mb_h
== 16) c
->compare_optimized
= sad_sse_422_luma_16x16
;
667 else c
->compare_optimized
= sad_sse_422_luma_16w
;
669 case 32: if(c
->mb_h
== 32) c
->compare_optimized
= sad_sse_422_luma_32x32
;
670 else c
->compare_optimized
= sad_sse_422_luma_32w
;
672 case 64: c
->compare_optimized
= sad_sse_422_luma_64w
;
675 default: c
->compare_optimized
= sad_reference
;
680 inline static void set_red(uint8_t *image
, struct motion_est_context_s
*c
)
683 for( n
= 0; n
< c
->width
* c
->height
* 2; n
+=4 )
693 static void show_residual( uint8_t *result
, struct motion_est_context_s
*c
)
701 // set_red(result,c);
703 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
704 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
706 dx
= CURRENT(i
,j
)->dx
;
707 dy
= CURRENT(i
,j
)->dy
;
713 // Denoise function caused some blocks to be completely clipped, ignore them
714 if (constrain( &x
, &y
, &w
, &h
, dx
, dy
, 0, c
->width
, 0, c
->height
) == 0 )
717 for( ty
= y
; ty
< y
+ h
; ty
++ ){
718 for( tx
= x
; tx
< x
+ w
; tx
++ ){
720 b
= c
->former_image
+ (tx
+dx
)*c
->xstride
+ (ty
+dy
)*c
->ystride
;
721 r
= result
+ tx
*c
->xstride
+ ty
*c
->ystride
;
723 r
[0] = 16 + ABS( r
[0] - b
[0] );
726 r
[1] = 128 + ABS( r
[1] - b
[1] );
728 // FIXME: may exceed boundies
729 r
[1] = 128 + ABS( r
[1] - ( *(b
-1) + b
[3] ) /2 );
736 static void show_reconstruction( uint8_t *result
, struct motion_est_context_s
*c
)
744 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
745 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
747 dx
= CURRENT(i
,j
)->dx
;
748 dy
= CURRENT(i
,j
)->dy
;
754 // Denoise function caused some blocks to be completely clipped, ignore them
755 if (constrain( &x
, &y
, &w
, &h
, dx
, dy
, 0, c
->width
, 0, c
->height
) == 0 )
758 for( ty
= y
; ty
< y
+ h
; ty
++ ){
759 for( tx
= x
; tx
< x
+ w
; tx
++ ){
761 r
= result
+ tx
*c
->xstride
+ ty
*c
->ystride
;
762 s
= c
->former_image
+ (tx
+dx
)*c
->xstride
+ (ty
+dy
)*c
->ystride
;
769 // FIXME: may exceed boundies
770 r
[1] = ( *(s
-1) + s
[3] ) /2;
777 // Image stack(able) method
778 static int filter_get_image( mlt_frame frame
, uint8_t **image
, mlt_image_format
*format
, int *width
, int *height
, int writable
)
781 mlt_filter filter
= mlt_frame_pop_service( frame
);
783 // Get the motion_est context object
784 struct motion_est_context_s
*c
= mlt_properties_get_data( MLT_FILTER_PROPERTIES( filter
), "context", NULL
);
787 // Get the new image and frame number
788 int error
= mlt_frame_get_image( frame
, image
, format
, width
, height
, 1 );
791 struct timeval start
; gettimeofday(&start
, NULL
);
796 mlt_properties_debug( MLT_FRAME_PROPERTIES(frame
), "error after mlt_frame_get_image() in motion_est", stderr
);
798 c
->current_frame_position
= mlt_frame_get_position( frame
);
800 /* Context Initialization */
801 if ( c
->initialized
== 0 ) {
803 // Get the filter properties object
804 mlt_properties properties
= mlt_filter_properties( filter
);
809 /* Get parameters that may have been overridden */
810 if( mlt_properties_get( properties
, "macroblock_width") != NULL
)
811 c
->mb_w
= mlt_properties_get_int( properties
, "macroblock_width");
813 if( mlt_properties_get( properties
, "macroblock_height") != NULL
)
814 c
->mb_h
= mlt_properties_get_int( properties
, "macroblock_height");
816 if( mlt_properties_get( properties
, "prediction_thresh") != NULL
)
817 c
->initial_thresh
= mlt_properties_get_int( properties
, "prediction_thresh" );
819 c
->initial_thresh
= c
->mb_w
* c
->mb_h
;
821 if( mlt_properties_get( properties
, "search_method") != NULL
)
822 c
->search_method
= mlt_properties_get_int( properties
, "search_method");
824 if( mlt_properties_get( properties
, "skip_prediction") != NULL
)
825 c
->skip_prediction
= mlt_properties_get_int( properties
, "skip_prediction");
827 if( mlt_properties_get( properties
, "limit_x") != NULL
)
828 c
->limit_x
= mlt_properties_get_int( properties
, "limit_x");
830 if( mlt_properties_get( properties
, "limit_y") != NULL
)
831 c
->limit_y
= mlt_properties_get_int( properties
, "limit_y");
833 if( mlt_properties_get( properties
, "check_chroma" ) != NULL
)
834 c
->check_chroma
= mlt_properties_get_int( properties
, "check_chroma" );
836 if( mlt_properties_get( properties
, "denoise" ) != NULL
)
837 c
->denoise
= mlt_properties_get_int( properties
, "denoise" );
839 if( mlt_properties_get( properties
, "show_reconstruction" ) != NULL
)
840 c
->show_reconstruction
= mlt_properties_get_int( properties
, "show_reconstruction" );
842 if( mlt_properties_get( properties
, "show_residual" ) != NULL
)
843 c
->show_residual
= mlt_properties_get_int( properties
, "show_residual" );
845 if( mlt_properties_get( properties
, "toggle_when_paused" ) != NULL
)
846 c
->toggle_when_paused
= mlt_properties_get_int( properties
, "toggle_when_paused" );
848 init_optimizations( c
);
850 // Calculate the dimensions in macroblock units
851 c
->mv_buffer_width
= (*width
/ c
->mb_w
);
852 c
->mv_buffer_height
= (*height
/ c
->mb_h
);
854 // Size of the motion vector buffer
855 c
->mv_size
= c
->mv_buffer_width
* c
->mv_buffer_height
* sizeof(struct motion_vector_s
);
857 // Allocate the motion vector buffers
858 c
->former_vectors
= mlt_pool_alloc( c
->mv_size
);
859 c
->current_vectors
= mlt_pool_alloc( c
->mv_size
);
860 c
->denoise_vectors
= mlt_pool_alloc( c
->mv_size
);
862 // Register motion buffers for destruction
863 mlt_properties_set_data( properties
, "current_motion_vectors", (void *)c
->current_vectors
, 0, mlt_pool_release
, NULL
);
864 mlt_properties_set_data( properties
, "former_motion_vectors", (void *)c
->former_vectors
, 0, mlt_pool_release
, NULL
);
865 mlt_properties_set_data( properties
, "denoise_motion_vectors", (void *)c
->denoise_vectors
, 0, mlt_pool_release
, NULL
);
867 c
->former_vectors_valid
= 0;
868 memset( c
->former_vectors
, 0, c
->mv_size
);
870 // Calculate the size of our steps (the number of bytes that seperate adjacent pixels in X and Y direction)
872 case mlt_image_yuv422
:
874 c
->ystride
= c
->xstride
* *width
;
878 fprintf(stderr
, "\"I am unfamiliar with your new fangled pixel format!\" -filter_motion_est\n");
882 // Allocate a cache for the previous frame's image
883 c
->former_image
= mlt_pool_alloc( *width
* *height
* 2 );
884 c
->cache_image
= mlt_pool_alloc( *width
* *height
* 2 );
886 // Register for destruction
887 mlt_properties_set_data( properties
, "cache_image", (void *)c
->cache_image
, 0, mlt_pool_release
, NULL
);
888 mlt_properties_set_data( properties
, "former_image", (void *)c
->former_image
, 0, mlt_pool_release
, NULL
);
890 c
->former_frame_position
= c
->current_frame_position
;
891 c
->previous_msad
= 0;
896 /* Check to see if somebody else has given us bounds */
897 struct mlt_geometry_item_s
*bounds
= mlt_properties_get_data( MLT_FRAME_PROPERTIES( frame
), "bounds", NULL
);
899 if( bounds
!= NULL
) {
900 // translate pixel units (from bounds) to macroblock units
901 // make sure whole macroblock stays within bounds
902 c
->left_mb
= ( bounds
->x
+ c
->mb_w
- 1 ) / c
->mb_w
;
903 c
->top_mb
= ( bounds
->y
+ c
->mb_h
- 1 ) / c
->mb_h
;
904 c
->right_mb
= ( bounds
->x
+ bounds
->w
) / c
->mb_w
- 1;
905 c
->bottom_mb
= ( bounds
->y
+ bounds
->h
) / c
->mb_h
- 1;
906 c
->bounds
.x
= bounds
->x
;
907 c
->bounds
.y
= bounds
->y
;
908 c
->bounds
.w
= bounds
->w
;
909 c
->bounds
.h
= bounds
->h
;
911 c
->left_mb
= c
->prev_left_mb
= 0;
912 c
->top_mb
= c
->prev_top_mb
= 0;
913 c
->right_mb
= c
->prev_right_mb
= c
->mv_buffer_width
- 1; // Zero indexed
914 c
->bottom_mb
= c
->prev_bottom_mb
= c
->mv_buffer_height
- 1;
917 c
->bounds
.w
= *width
;
918 c
->bounds
.h
= *height
;
921 // If video is advancing, run motion vector algorithm and etc...
922 if( c
->former_frame_position
+ 1 == c
->current_frame_position
)
925 // Swap the motion vector buffers and reuse allocated memory
926 struct motion_vector_s
*temp
= c
->current_vectors
;
927 c
->current_vectors
= c
->former_vectors
;
928 c
->former_vectors
= temp
;
930 // This is done because filter_vismv doesn't pay attention to frame boundry
931 memset( c
->current_vectors
, 0, c
->mv_size
);
933 // Perform the motion search
934 motion_search( c
->cache_image
, *image
, c
);
936 collect_post_statistics( c
);
939 // Detect shot changes
940 if( c
->comparison_average
> 10 * c
->mb_w
* c
->mb_h
&&
941 c
->comparison_average
> c
->previous_msad
* 2 )
943 fprintf(stderr
, " - SAD: %d <<Shot change>>\n", c
->comparison_average
);
944 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "shot_change", 1);
945 // c->former_vectors_valid = 0; // Invalidate the previous frame's predictors
949 c
->former_vectors_valid
= 1;
951 //fprintf(stderr, " - SAD: %d\n", c->comparison_average);
954 c
->previous_msad
= c
->comparison_average
;
956 if( c
->comparison_average
!= 0 ) { // If the frame is not a duplicate of the previous frame
958 // denoise the vector buffer
960 median_denoise( c
->current_vectors
, c
);
962 // Pass the new vector data into the frame
963 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame
), "motion_est.vectors",
964 (void*)c
->current_vectors
, c
->mv_size
, NULL
, NULL
);
966 // Cache the frame's image. Save the old cache. Reuse memory.
967 // After this block, exactly two unique frames will be cached
968 uint8_t *timg
= c
->cache_image
;
969 c
->cache_image
= c
->former_image
;
970 c
->former_image
= timg
;
971 memcpy( c
->cache_image
, *image
, *width
* *height
* c
->xstride
);
976 // Undo the Swap, This fixes the ugliness caused by a duplicate frame
977 temp
= c
->current_vectors
;
978 c
->current_vectors
= c
->former_vectors
;
979 c
->former_vectors
= temp
;
980 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame
), "motion_est.vectors",
981 (void*)c
->former_vectors
, c
->mv_size
, NULL
, NULL
);
985 if( c
->shot_change
== 1)
987 else if( c
->show_reconstruction
)
988 show_reconstruction( *image
, c
);
989 else if( c
->show_residual
)
990 show_residual( *image
, c
);
994 else if( c
->former_frame_position
== c
->current_frame_position
)
996 // Pass the old vector data into the frame if it's valid
997 if( c
->former_vectors_valid
== 1 ) {
998 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame
), "motion_est.vectors",
999 (void*)c
->current_vectors
, c
->mv_size
, NULL
, NULL
);
1001 if( c
->shot_change
== 1)
1003 else if( c
->toggle_when_paused
== 1 ) {
1004 if( c
->show_reconstruction
)
1005 show_reconstruction( *image
, c
);
1006 else if( c
->show_residual
)
1007 show_residual( *image
, c
);
1008 c
->toggle_when_paused
= 2;
1010 else if( c
->toggle_when_paused
== 2 )
1011 c
->toggle_when_paused
= 1;
1013 if( c
->show_reconstruction
)
1014 show_reconstruction( *image
, c
);
1015 else if( c
->show_residual
)
1016 show_residual( *image
, c
);
1021 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "shot_change", c
->shot_change
);
1023 // there was jump in frame number
1025 // fprintf(stderr, "Warning: there was a frame number jumped from %d to %d.\n", c->former_frame_position, c->current_frame_position);
1026 c
->former_vectors_valid
= 0;
1030 // Cache our bounding geometry for the next frame's processing
1031 c
->prev_left_mb
= c
->left_mb
;
1032 c
->prev_top_mb
= c
->top_mb
;
1033 c
->prev_right_mb
= c
->right_mb
;
1034 c
->prev_bottom_mb
= c
->bottom_mb
;
1036 // Remember which frame this is
1037 c
->former_frame_position
= c
->current_frame_position
;
1040 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.macroblock_width", c
->mb_w
);
1041 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.macroblock_height", c
->mb_h
);
1042 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.left_mb", c
->left_mb
);
1043 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.right_mb", c
->right_mb
);
1044 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.top_mb", c
->top_mb
);
1045 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.bottom_mb", c
->bottom_mb
);
1048 struct timeval finish
; gettimeofday(&finish
, NULL
); int difference
= (finish
.tv_sec
- start
.tv_sec
) * 1000000 + (finish
.tv_usec
- start
.tv_usec
);
1049 fprintf(stderr
, " in frame %d:%d usec\n", c
->current_frame_position
, difference
);
1058 /** filter processing.
1061 static mlt_frame
filter_process( mlt_filter
this, mlt_frame frame
)
1064 // Keeps tabs on the filter object
1065 mlt_frame_push_service( frame
, this);
1067 // Push the frame filter
1068 mlt_frame_push_get_image( frame
, filter_get_image
);
1073 /** Constructor for the filter.
1075 mlt_filter
filter_motion_est_init( char *arg
)
1077 mlt_filter
this = mlt_filter_new( );
1080 // Get the properties object
1081 mlt_properties properties
= MLT_FILTER_PROPERTIES( this );
1083 // Initialize the motion estimation context
1084 struct motion_est_context_s
*context
;
1085 context
= mlt_pool_alloc( sizeof(struct motion_est_context_s
) );
1086 mlt_properties_set_data( properties
, "context", (void *)context
, sizeof( struct motion_est_context_s
),
1087 mlt_pool_release
, NULL
);
1090 // Register the filter
1091 this->process
= filter_process
;
1093 /* defaults that may be overridden */
1096 context
->skip_prediction
= 0;
1097 context
->limit_x
= 64;
1098 context
->limit_y
= 64;
1099 context
->search_method
= DIAMOND_SEARCH
; // FIXME: not used
1100 context
->check_chroma
= 0;
1101 context
->denoise
= 1;
1102 context
->show_reconstruction
= 0;
1103 context
->show_residual
= 0;
1104 context
->toggle_when_paused
= 0;
1106 /* reference functions that may have optimized versions */
1107 context
->compare_reference
= sad_reference
;
1109 // The rest of the buffers will be initialized when the filter is first processed
1110 context
->initialized
= 0;