2 * /brief fast motion estimation filter
3 * /author Zachary Drew, Copyright 2005
5 * Currently only uses Gamma data for comparisonon (bug or feature?)
6 * Vector optimization coming soon.
8 * Vector orientation: The vector data that is generated for the current frame specifies
9 * the motion from the previous frame to the current frame. To know how a macroblock
10 * in the current frame will move in the future, the next frame is needed.
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software Foundation,
24 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
28 #include "filter_motion_est.h"
29 #include <framework/mlt.h>
47 #define DIAMOND_SEARCH 0x0
48 #define FULL_SEARCH 0x1
50 #define MIN(a,b) ((a) > (b) ? (b) : (a))
51 #define ABS(a) ((a) >= 0 ? (a) : (-(a)))
54 struct motion_est_context_s
56 int initialized
; // true if filter has been initialized
62 /* same as mlt_frame's parameters */
65 /* Operational details */
68 uint8_t *cache_image
; // Copy of current frame
69 uint8_t *former_image
; // Copy of former frame
73 int limit_x
, limit_y
; // max x and y of a motion vector
75 int check_chroma
; // if check_chroma == 1 then compare chroma
78 int show_reconstruction
;
79 int toggle_when_paused
;
83 struct mlt_geometry_item_s bounds
; // Current bounds (from filters crop_detect, autotrack rectangle, or other)
85 /* bounds in macroblock units; macroblocks are completely contained within the boundry */
86 int left_mb
, prev_left_mb
, right_mb
, prev_right_mb
;
87 int top_mb
, prev_top_mb
, bottom_mb
, prev_bottom_mb
;
89 /* size of our vector buffers */
90 int mv_buffer_height
, mv_buffer_width
, mv_size
;
93 int former_vectors_valid
; //<! true if the previous frame's buffered motion vector data is valid
94 motion_vector
*former_vectors
;
95 motion_vector
*current_vectors
;
96 motion_vector
*denoise_vectors
;
97 mlt_position former_frame_position
, current_frame_position
;
99 /* diagnostic metrics */
100 float predictive_misses
; // How often do the prediction motion vectors fail?
101 int comparison_average
; // How far does the best estimation deviate from a perfect comparison?
104 int average_x
, average_y
;
106 /* run-time configurable comparison functions */
107 int (*compare_reference
)(uint8_t *, uint8_t *, int, int, int, int);
108 int (*compare_optimized
)(uint8_t *, uint8_t *, int, int, int, int);
112 // This is used to constrains pixel operations between two blocks to be within the image boundry
113 inline static int constrain( int *x
, int *y
, int *w
, int *h
,
114 const int dx
, const int dy
,
115 const int left
, const int right
,
116 const int top
, const int bottom
)
118 uint32_t penalty
= 1 << SHIFT
; // Retain a few extra bits of precision
124 // Origin of macroblock moves left of image boundy
125 if( *x
< left
|| x2
< left
) {
126 w_remains
= *w
- left
+ ((*x
< x2
) ?
*x
: x2
);
127 *x
+= *w
- w_remains
;
129 // Portion of macroblock moves right of image boundry
130 else if( *x
+ *w
> right
|| x2
+ *w
> right
)
131 w_remains
= right
- ((*x
> x2
) ?
*x
: x2
);
133 // Origin of macroblock moves above image boundy
134 if( *y
< top
|| y2
< top
) {
135 h_remains
= *h
- top
+ ((*y
< y2
) ?
*y
: y2
);
136 *y
+= *h
- h_remains
;
138 // Portion of macroblock moves bellow image boundry
139 else if( *y
+ *h
> bottom
|| y2
+ *h
> bottom
)
140 h_remains
= bottom
- ((*y
> y2
) ?
*y
: y2
);
142 if( w_remains
== *w
&& h_remains
== *h
) return penalty
;
143 if( w_remains
<= 0 || h_remains
<= 0) return 0; // Block is clipped out of existance
144 penalty
= (*w
* *h
* penalty
)
145 / ( w_remains
* h_remains
); // Recipricol of the fraction of the block that remains
147 assert(*x
>= left
); assert(x2
+ *w
- w_remains
>= left
);
148 assert(*y
>= top
); assert(y2
+ *h
- h_remains
>= top
);
149 assert(*x
+ w_remains
<= right
); assert(x2
+ w_remains
<= right
);
150 assert(*y
+ h_remains
<= bottom
); assert(y2
+ h_remains
<= bottom
);
152 *w
= w_remains
; // Update the width and height
158 /** /brief Reference Sum of Absolute Differences comparison function
161 inline static int sad_reference( uint8_t *block1
, uint8_t *block2
, const int xstride
, const int ystride
, const int w
, const int h
)
164 for ( j
= 0; j
< h
; j
++ ){
165 for ( i
= 0; i
< w
; i
++ ){
166 score
+= ABS( block1
[i
*xstride
] - block2
[i
*xstride
] );
176 /** /brief Abstracted block comparison function
178 inline static int block_compare( uint8_t *block1
,
184 struct motion_est_context_s
*c
)
187 #ifdef COUNT_COMPARES
193 // Default comparison may be overridden by the slower, more capable reference comparison
194 int (*cmp
)(uint8_t *, uint8_t *, int, int, int, int) = c
->compare_optimized
;
196 // vector displacement limited has been exceeded
197 if( ABS( dx
) >= c
->limit_x
|| ABS( dy
) >= c
->limit_y
)
200 int mb_w
= c
->mb_w
; // Some writeable local copies
203 // Determine if either macroblock got clipped
204 int penalty
= constrain( &x
, &y
, &mb_w
, &mb_h
, dx
, dy
, 0, c
->width
, 0, c
->height
);
207 if( penalty
== 0 ) // Clipped out of existance: Return worst score
209 else if( penalty
!= 1<<SHIFT
) // Nonstandard macroblock dimensions: Disable SIMD optimizizations.
210 cmp
= c
->compare_reference
;
212 // Calculate the memory locations of the macroblocks
213 block1
+= x
* c
->xstride
+ y
* c
->ystride
;
214 block2
+= (x
+dx
) * c
->xstride
+ (y
+dy
) * c
->ystride
;
217 if( penalty
== 1<<SHIFT
){
218 score
= c
->compare_reference( block1
, block2
, c
->xstride
, c
->ystride
, mb_w
, mb_h
);
219 int score2
= c
->compare_optimized( block1
, block2
, c
->xstride
, c
->ystride
, mb_w
, mb_h
);
220 if ( score
!= score2
)
221 fprintf(stderr
, "Your assembly doesn't work! Reference: %d Asm: %d\n", score
, score2
);
226 score
= cmp( block1
, block2
, c
->xstride
, c
->ystride
, mb_w
, mb_h
);
228 return ( score
* penalty
) >> SHIFT
; // Ditch the extra precision
231 static inline void check_candidates ( uint8_t *ref
,
232 uint8_t *candidate_base
,
235 const motion_vector
*candidates
,// Contains to_x & to_y
236 const int count
, // Number of candidates
237 const int unique
, // Sometimes we know the candidates are unique
238 motion_vector
*result
,
239 struct motion_est_context_s
*c
)
242 /* Scan for the best candidate */
243 for ( i
= 0; i
< count
; i
++ )
245 // this little dohicky ignores duplicate candidates, if they are possible
250 if ( candidates
[j
].dx
== candidates
[i
].dx
&&
251 candidates
[j
].dy
== candidates
[i
].dy
)
259 score
= block_compare( ref
, candidate_base
,
261 candidates
[i
].dx
, // from
265 if ( score
< result
->msad
) { // New minimum
266 result
->dx
= candidates
[i
].dx
;
267 result
->dy
= candidates
[i
].dy
;
268 result
->msad
= score
;
274 /* /brief Diamond search
275 * Operates on a single macroblock
277 static inline void diamond_search(
278 uint8_t *ref
, //<! Image data from previous frame
279 uint8_t *candidate_base
, //<! Image data in current frame
280 const int x
, //<! X upper left corner of macroblock
281 const int y
, //<! U upper left corner of macroblock
282 struct motion_vector_s
*result
, //<! Best predicted mv and eventual result
283 struct motion_est_context_s
*c
) //<! motion estimation context
286 // diamond search pattern
287 motion_vector candidates
[4];
289 // Keep track of best and former best candidates
290 motion_vector best
, former
;
292 // The direction of the refinement needs to be known
293 motion_vector current
;
297 // Loop through the search pattern
300 current
.dx
= result
->dx
;
301 current
.dy
= result
->dy
;
303 if ( first
== 1 ) // Set the initial pattern
305 candidates
[0].dx
= result
->dx
+ 1; candidates
[0].dy
= result
->dy
+ 0;
306 candidates
[1].dx
= result
->dx
+ 0; candidates
[1].dy
= result
->dy
+ 1;
307 candidates
[2].dx
= result
->dx
- 1; candidates
[2].dy
= result
->dy
+ 0;
308 candidates
[3].dx
= result
->dx
+ 0; candidates
[3].dy
= result
->dy
- 1;
311 else // Construct the next portion of the search pattern
313 candidates
[0].dx
= result
->dx
+ best
.dx
;
314 candidates
[0].dy
= result
->dy
+ best
.dy
;
315 if (best
.dx
== former
.dx
&& best
.dy
== former
.dy
) {
316 candidates
[1].dx
= result
->dx
+ best
.dy
;
317 candidates
[1].dy
= result
->dy
+ best
.dx
; // Yes, the wires
318 candidates
[2].dx
= result
->dx
- best
.dy
; // are crossed
319 candidates
[2].dy
= result
->dy
- best
.dx
;
322 candidates
[1].dx
= result
->dx
+ former
.dx
;
323 candidates
[1].dy
= result
->dy
+ former
.dy
;
327 former
.dx
= best
.dx
; former
.dy
= best
.dy
; // Keep track of new former best
330 check_candidates ( ref
, candidate_base
, x
, y
, candidates
, i
, 1, result
, c
);
332 // Which candidate was the best?
333 best
.dx
= result
->dx
- current
.dx
;
334 best
.dy
= result
->dy
- current
.dy
;
336 // A better canidate was not found
337 if ( best
.dx
== 0 && best
.dy
== 0 )
342 former
.dx
= best
.dx
; former
.dy
= best
.dy
; // First iteration, sensible value for former.d*
347 /* /brief Full (brute) search
348 * Operates on a single macroblock
350 __attribute__((used
))
351 static void full_search(
352 uint8_t *ref
, //<! Image data from previous frame
353 uint8_t *candidate_base
, //<! Image data in current frame
354 int x
, //<! X upper left corner of macroblock
355 int y
, //<! U upper left corner of macroblock
356 struct motion_vector_s
*result
, //<! Best predicted mv and eventual result
357 struct motion_est_context_s
*c
) //<! motion estimation context
359 // Keep track of best candidate
363 for( i
= -c
->mb_w
; i
<= c
->mb_w
; i
++ ){
364 for( j
= -c
->mb_h
; j
<= c
->mb_h
; j
++ ){
366 score
= block_compare( ref
, candidate_base
,
373 if ( score
< result
->msad
) {
376 result
->msad
= score
;
382 // Macros for pointer calculations
383 #define CURRENT(i,j) ( c->current_vectors + (j)*c->mv_buffer_width + (i) )
384 #define FORMER(i,j) ( c->former_vectors + (j)*c->mv_buffer_width + (i) )
385 #define DENOISE(i,j) ( c->denoise_vectors + (j)*c->mv_buffer_width + (i) )
387 int ncompare (const void * a
, const void * b
)
389 return ( *(int*)a
- *(int*)b
);
392 // motion vector denoising
393 // for x and y components seperately,
394 // change the vector to be the median value of the 9 adjacent vectors
395 static void median_denoise( motion_vector
*v
, struct motion_est_context_s
*c
)
397 int xvalues
[9], yvalues
[9];
400 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ )
401 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
405 xvalues
[n
] = CURRENT(i
,j
)->dx
; // Center
406 yvalues
[n
++] = CURRENT(i
,j
)->dy
;
408 if( i
> c
->left_mb
) // Not in First Column
410 xvalues
[n
] = CURRENT(i
-1,j
)->dx
; // Left
411 yvalues
[n
++] = CURRENT(i
-1,j
)->dy
;
413 if( j
> c
->top_mb
) {
414 xvalues
[n
] = CURRENT(i
-1,j
-1)->dx
; // Upper Left
415 yvalues
[n
++] = CURRENT(i
-1,j
-1)->dy
;
418 if( j
< c
->bottom_mb
) {
419 xvalues
[n
] = CURRENT(i
-1,j
+1)->dx
; // Bottom Left
420 yvalues
[n
++] = CURRENT(i
-1,j
+1)->dy
;
423 if( i
< c
->right_mb
) // Not in Last Column
425 xvalues
[n
] = CURRENT(i
+1,j
)->dx
; // Right
426 yvalues
[n
++] = CURRENT(i
+1,j
)->dy
;
429 if( j
> c
->top_mb
) {
430 xvalues
[n
] = CURRENT(i
+1,j
-1)->dx
; // Upper Right
431 yvalues
[n
++] = CURRENT(i
+1,j
-1)->dy
;
434 if( j
< c
->bottom_mb
) {
435 xvalues
[n
] = CURRENT(i
+1,j
+1)->dx
; // Bottom Right
436 yvalues
[n
++] = CURRENT(i
+1,j
+1)->dy
;
439 if( j
> c
->top_mb
) // Not in First Row
441 xvalues
[n
] = CURRENT(i
,j
-1)->dx
; // Top
442 yvalues
[n
++] = CURRENT(i
,j
-1)->dy
;
445 if( j
< c
->bottom_mb
) // Not in Last Row
447 xvalues
[n
] = CURRENT(i
,j
+1)->dx
; // Bottom
448 yvalues
[n
++] = CURRENT(i
,j
+1)->dy
;
451 qsort (xvalues
, n
, sizeof(int), ncompare
);
452 qsort (yvalues
, n
, sizeof(int), ncompare
);
455 DENOISE(i
,j
)->dx
= xvalues
[n
/2];
456 DENOISE(i
,j
)->dy
= yvalues
[n
/2];
459 DENOISE(i
,j
)->dx
= (xvalues
[n
/2] + xvalues
[n
/2+1])/2;
460 DENOISE(i
,j
)->dy
= (yvalues
[n
/2] + yvalues
[n
/2+1])/2;
465 motion_vector
*t
= c
->current_vectors
;
466 c
->current_vectors
= c
->denoise_vectors
;
467 c
->denoise_vectors
= t
;
473 static inline int median_predictor(int a
, int b
, int c
) {
489 /** /brief Motion search
491 * For each macroblock in the current frame, estimate the block from the last frame that
494 * Vocab: Colocated - the pixel in the previous frame at the current position
496 * Based on enhanced predictive zonal search. [Tourapis 2002]
498 static void motion_search( uint8_t *from
, //<! Image data.
499 uint8_t *to
, //<! Image data. Rigid grid.
500 struct motion_est_context_s
*c
) //<! The context
503 #ifdef COUNT_COMPARES
507 motion_vector candidates
[10];
508 motion_vector
*here
; // This one gets used alot (about 30 times per macroblock)
513 // For every macroblock, perform motion vector estimation
514 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
515 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
520 here
->msad
= MAX_MSAD
;
525 /* Stack the predictors [i.e. checked in reverse order] */
527 /* Adjacent to collocated */
528 if( c
->former_vectors_valid
)
531 if( j
> c
->prev_top_mb
){// && COL_TOP->valid ){
532 candidates
[n
].dx
= FORMER(i
,j
-1)->dx
;
533 candidates
[n
++].dy
= FORMER(i
,j
-1)->dy
;
537 if( i
> c
->prev_left_mb
){// && COL_LEFT->valid ){
538 candidates
[n
].dx
= FORMER(i
-1,j
)->dx
;
539 candidates
[n
++].dy
= FORMER(i
-1,j
)->dy
;
542 // Right of colocated
543 if( i
< c
->prev_right_mb
){// && COL_RIGHT->valid ){
544 candidates
[n
].dx
= FORMER(i
+1,j
)->dx
;
545 candidates
[n
++].dy
= FORMER(i
+1,j
)->dy
;
548 // Bottom of colocated
549 if( j
< c
->prev_bottom_mb
){// && COL_BOTTOM->valid ){
550 candidates
[n
].dx
= FORMER(i
,j
+1)->dx
;
551 candidates
[n
++].dy
= FORMER(i
,j
+1)->dy
;
554 // And finally, colocated
555 candidates
[n
].dx
= FORMER(i
,j
)->dx
;
556 candidates
[n
++].dy
= FORMER(i
,j
)->dy
;
559 // For macroblocks not in the top row
560 if ( j
> c
->top_mb
) {
562 // Top if ( TOP->valid ) {
563 candidates
[n
].dx
= CURRENT(i
,j
-1)->dx
;
564 candidates
[n
++].dy
= CURRENT(i
,j
-1)->dy
;
567 // Top-Right, macroblocks not in the right row
568 if ( i
< c
->right_mb
){// && TOP_RIGHT->valid ) {
569 candidates
[n
].dx
= CURRENT(i
+1,j
-1)->dx
;
570 candidates
[n
++].dy
= CURRENT(i
+1,j
-1)->dy
;
574 // Left, Macroblocks not in the left column
575 if ( i
> c
->left_mb
){// && LEFT->valid ) {
576 candidates
[n
].dx
= CURRENT(i
-1,j
)->dx
;
577 candidates
[n
++].dy
= CURRENT(i
-1,j
)->dy
;
580 /* Median predictor vector (median of left, top, and top right adjacent vectors) */
581 if ( i
> c
->left_mb
&& j
> c
->top_mb
&& i
< c
->right_mb
582 )//&& LEFT->valid && TOP->valid && TOP_RIGHT->valid )
584 candidates
[n
].dx
= median_predictor( CURRENT(i
-1,j
)->dx
, CURRENT(i
,j
-1)->dx
, CURRENT(i
+1,j
-1)->dx
);
585 candidates
[n
++].dy
= median_predictor( CURRENT(i
-1,j
)->dy
, CURRENT(i
,j
-1)->dy
, CURRENT(i
+1,j
-1)->dy
);
589 candidates
[n
].dx
= 0;
590 candidates
[n
++].dy
= 0;
594 check_candidates ( to
, from
, x
, y
, candidates
, n
, 0, here
, c
);
598 diamond_search( to
, from
, x
, y
, here
, c
);
600 full_search( to
, from
, x
, y
, here
, c
);
603 assert( x
+ c
->mb_w
+ here
->dx
> 0 ); // All macroblocks must have area > 0
604 assert( y
+ c
->mb_h
+ here
->dy
> 0 );
605 assert( x
+ here
->dx
< c
->width
);
606 assert( y
+ here
->dy
< c
->height
);
608 } /* End column loop */
611 asm volatile ( "emms" );
613 #ifdef COUNT_COMPARES
614 fprintf(stderr
, "%d comparisons per block were made", compares
/count
);
619 void collect_post_statistics( struct motion_est_context_s
*c
) {
621 c
->comparison_average
= 0;
622 c
->average_length
= 0;
628 for ( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
629 for ( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
632 c
->comparison_average
+= CURRENT(i
,j
)->msad
;
633 c
->average_x
+= CURRENT(i
,j
)->dx
;
634 c
->average_y
+= CURRENT(i
,j
)->dy
;
642 c
->comparison_average
/= count
;
643 c
->average_x
/= count
;
644 c
->average_y
/= count
;
645 c
->average_length
= sqrt( c
->average_x
* c
->average_x
+ c
->average_y
* c
->average_y
);
650 static void init_optimizations( struct motion_est_context_s
*c
)
653 case 4: if(c
->mb_h
== 4) c
->compare_optimized
= sad_sse_422_luma_4x4
;
654 else c
->compare_optimized
= sad_sse_422_luma_4w
;
656 case 8: if(c
->mb_h
== 8) c
->compare_optimized
= sad_sse_422_luma_8x8
;
657 else c
->compare_optimized
= sad_sse_422_luma_8w
;
659 case 16: if(c
->mb_h
== 16) c
->compare_optimized
= sad_sse_422_luma_16x16
;
660 else c
->compare_optimized
= sad_sse_422_luma_16w
;
662 case 32: if(c
->mb_h
== 32) c
->compare_optimized
= sad_sse_422_luma_32x32
;
663 else c
->compare_optimized
= sad_sse_422_luma_32w
;
665 case 64: c
->compare_optimized
= sad_sse_422_luma_64w
;
667 default: c
->compare_optimized
= sad_reference
;
672 inline static void set_red(uint8_t *image
, struct motion_est_context_s
*c
)
675 for( n
= 0; n
< c
->width
* c
->height
* 2; n
+=4 )
685 static void show_residual( uint8_t *result
, struct motion_est_context_s
*c
)
693 // set_red(result,c);
695 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
696 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
698 dx
= CURRENT(i
,j
)->dx
;
699 dy
= CURRENT(i
,j
)->dy
;
705 // Denoise function caused some blocks to be completely clipped, ignore them
706 if (constrain( &x
, &y
, &w
, &h
, dx
, dy
, 0, c
->width
, 0, c
->height
) == 0 )
709 for( ty
= y
; ty
< y
+ h
; ty
++ ){
710 for( tx
= x
; tx
< x
+ w
; tx
++ ){
712 b
= c
->former_image
+ (tx
+dx
)*c
->xstride
+ (ty
+dy
)*c
->ystride
;
713 r
= result
+ tx
*c
->xstride
+ ty
*c
->ystride
;
715 r
[0] = 16 + ABS( r
[0] - b
[0] );
718 r
[1] = 128 + ABS( r
[1] - b
[1] );
720 // FIXME: may exceed boundies
721 r
[1] = 128 + ABS( r
[1] - ( *(b
-1) + b
[3] ) /2 );
728 static void show_reconstruction( uint8_t *result
, struct motion_est_context_s
*c
)
736 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
737 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
739 dx
= CURRENT(i
,j
)->dx
;
740 dy
= CURRENT(i
,j
)->dy
;
746 // Denoise function caused some blocks to be completely clipped, ignore them
747 if (constrain( &x
, &y
, &w
, &h
, dx
, dy
, 0, c
->width
, 0, c
->height
) == 0 )
750 for( ty
= y
; ty
< y
+ h
; ty
++ ){
751 for( tx
= x
; tx
< x
+ w
; tx
++ ){
753 r
= result
+ tx
*c
->xstride
+ ty
*c
->ystride
;
754 s
= c
->former_image
+ (tx
+dx
)*c
->xstride
+ (ty
+dy
)*c
->ystride
;
761 // FIXME: may exceed boundies
762 r
[1] = ( *(s
-1) + s
[3] ) /2;
769 // Image stack(able) method
770 static int filter_get_image( mlt_frame frame
, uint8_t **image
, mlt_image_format
*format
, int *width
, int *height
, int writable
)
773 mlt_filter filter
= mlt_frame_pop_service( frame
);
775 // Get the motion_est context object
776 struct motion_est_context_s
*c
= mlt_properties_get_data( MLT_FILTER_PROPERTIES( filter
), "context", NULL
);
779 // Get the new image and frame number
780 int error
= mlt_frame_get_image( frame
, image
, format
, width
, height
, 1 );
783 struct timeval start
; gettimeofday(&start
, NULL
);
788 mlt_properties_debug( MLT_FRAME_PROPERTIES(frame
), "error after mlt_frame_get_image() in motion_est", stderr
);
790 c
->current_frame_position
= mlt_frame_get_position( frame
);
792 /* Context Initialization */
793 if ( c
->initialized
== 0 ) {
795 // Get the filter properties object
796 mlt_properties properties
= mlt_filter_properties( filter
);
801 /* Get parameters that may have been overridden */
802 if( mlt_properties_get( properties
, "macroblock_width") != NULL
)
803 c
->mb_w
= mlt_properties_get_int( properties
, "macroblock_width");
805 if( mlt_properties_get( properties
, "macroblock_height") != NULL
)
806 c
->mb_h
= mlt_properties_get_int( properties
, "macroblock_height");
808 if( mlt_properties_get( properties
, "prediction_thresh") != NULL
)
809 c
->initial_thresh
= mlt_properties_get_int( properties
, "prediction_thresh" );
811 c
->initial_thresh
= c
->mb_w
* c
->mb_h
;
813 if( mlt_properties_get( properties
, "search_method") != NULL
)
814 c
->search_method
= mlt_properties_get_int( properties
, "search_method");
816 if( mlt_properties_get( properties
, "skip_prediction") != NULL
)
817 c
->skip_prediction
= mlt_properties_get_int( properties
, "skip_prediction");
819 if( mlt_properties_get( properties
, "limit_x") != NULL
)
820 c
->limit_x
= mlt_properties_get_int( properties
, "limit_x");
822 if( mlt_properties_get( properties
, "limit_y") != NULL
)
823 c
->limit_y
= mlt_properties_get_int( properties
, "limit_y");
825 if( mlt_properties_get( properties
, "check_chroma" ) != NULL
)
826 c
->check_chroma
= mlt_properties_get_int( properties
, "check_chroma" );
828 if( mlt_properties_get( properties
, "denoise" ) != NULL
)
829 c
->denoise
= mlt_properties_get_int( properties
, "denoise" );
831 if( mlt_properties_get( properties
, "show_reconstruction" ) != NULL
)
832 c
->show_reconstruction
= mlt_properties_get_int( properties
, "show_reconstruction" );
834 if( mlt_properties_get( properties
, "show_residual" ) != NULL
)
835 c
->show_residual
= mlt_properties_get_int( properties
, "show_residual" );
837 if( mlt_properties_get( properties
, "toggle_when_paused" ) != NULL
)
838 c
->toggle_when_paused
= mlt_properties_get_int( properties
, "toggle_when_paused" );
840 init_optimizations( c
);
842 // Calculate the dimensions in macroblock units
843 c
->mv_buffer_width
= (*width
/ c
->mb_w
);
844 c
->mv_buffer_height
= (*height
/ c
->mb_h
);
846 // Size of the motion vector buffer
847 c
->mv_size
= c
->mv_buffer_width
* c
->mv_buffer_height
* sizeof(struct motion_vector_s
);
849 // Allocate the motion vector buffers
850 c
->former_vectors
= mlt_pool_alloc( c
->mv_size
);
851 c
->current_vectors
= mlt_pool_alloc( c
->mv_size
);
852 c
->denoise_vectors
= mlt_pool_alloc( c
->mv_size
);
854 // Register motion buffers for destruction
855 mlt_properties_set_data( properties
, "current_motion_vectors", (void *)c
->current_vectors
, 0, mlt_pool_release
, NULL
);
856 mlt_properties_set_data( properties
, "former_motion_vectors", (void *)c
->former_vectors
, 0, mlt_pool_release
, NULL
);
857 mlt_properties_set_data( properties
, "denoise_motion_vectors", (void *)c
->denoise_vectors
, 0, mlt_pool_release
, NULL
);
859 c
->former_vectors_valid
= 0;
860 memset( c
->former_vectors
, 0, c
->mv_size
);
862 // Calculate the size of our steps (the number of bytes that seperate adjacent pixels in X and Y direction)
864 case mlt_image_yuv422
:
866 c
->ystride
= c
->xstride
* *width
;
870 fprintf(stderr
, "\"I am unfamiliar with your new fangled pixel format!\" -filter_motion_est\n");
874 // Allocate a cache for the previous frame's image
875 c
->former_image
= mlt_pool_alloc( *width
* *height
* 2 );
876 c
->cache_image
= mlt_pool_alloc( *width
* *height
* 2 );
878 // Register for destruction
879 mlt_properties_set_data( properties
, "cache_image", (void *)c
->cache_image
, 0, mlt_pool_release
, NULL
);
880 mlt_properties_set_data( properties
, "former_image", (void *)c
->former_image
, 0, mlt_pool_release
, NULL
);
882 c
->former_frame_position
= c
->current_frame_position
;
883 c
->previous_msad
= 0;
888 /* Check to see if somebody else has given us bounds */
889 struct mlt_geometry_item_s
*bounds
= mlt_properties_get_data( MLT_FRAME_PROPERTIES( frame
), "bounds", NULL
);
891 if( bounds
!= NULL
) {
892 // translate pixel units (from bounds) to macroblock units
893 // make sure whole macroblock stays within bounds
894 c
->left_mb
= ( bounds
->x
+ c
->mb_w
- 1 ) / c
->mb_w
;
895 c
->top_mb
= ( bounds
->y
+ c
->mb_h
- 1 ) / c
->mb_h
;
896 c
->right_mb
= ( bounds
->x
+ bounds
->w
) / c
->mb_w
- 1;
897 c
->bottom_mb
= ( bounds
->y
+ bounds
->h
) / c
->mb_h
- 1;
898 c
->bounds
.x
= bounds
->x
;
899 c
->bounds
.y
= bounds
->y
;
900 c
->bounds
.w
= bounds
->w
;
901 c
->bounds
.h
= bounds
->h
;
903 c
->left_mb
= c
->prev_left_mb
= 0;
904 c
->top_mb
= c
->prev_top_mb
= 0;
905 c
->right_mb
= c
->prev_right_mb
= c
->mv_buffer_width
- 1; // Zero indexed
906 c
->bottom_mb
= c
->prev_bottom_mb
= c
->mv_buffer_height
- 1;
909 c
->bounds
.w
= *width
;
910 c
->bounds
.h
= *height
;
913 // If video is advancing, run motion vector algorithm and etc...
914 if( c
->former_frame_position
+ 1 == c
->current_frame_position
)
917 // Swap the motion vector buffers and reuse allocated memory
918 struct motion_vector_s
*temp
= c
->current_vectors
;
919 c
->current_vectors
= c
->former_vectors
;
920 c
->former_vectors
= temp
;
922 // This is done because filter_vismv doesn't pay attention to frame boundry
923 memset( c
->current_vectors
, 0, c
->mv_size
);
925 // Perform the motion search
926 motion_search( c
->cache_image
, *image
, c
);
928 collect_post_statistics( c
);
931 // Detect shot changes
932 if( c
->comparison_average
> 10 * c
->mb_w
* c
->mb_h
&&
933 c
->comparison_average
> c
->previous_msad
* 2 )
935 fprintf(stderr
, " - SAD: %d <<Shot change>>\n", c
->comparison_average
);
936 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "shot_change", 1);
937 // c->former_vectors_valid = 0; // Invalidate the previous frame's predictors
941 c
->former_vectors_valid
= 1;
943 //fprintf(stderr, " - SAD: %d\n", c->comparison_average);
946 c
->previous_msad
= c
->comparison_average
;
948 if( c
->comparison_average
!= 0 ) { // If the frame is not a duplicate of the previous frame
950 // denoise the vector buffer
952 median_denoise( c
->current_vectors
, c
);
954 // Pass the new vector data into the frame
955 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame
), "motion_est.vectors",
956 (void*)c
->current_vectors
, c
->mv_size
, NULL
, NULL
);
958 // Cache the frame's image. Save the old cache. Reuse memory.
959 // After this block, exactly two unique frames will be cached
960 uint8_t *timg
= c
->cache_image
;
961 c
->cache_image
= c
->former_image
;
962 c
->former_image
= timg
;
963 memcpy( c
->cache_image
, *image
, *width
* *height
* c
->xstride
);
968 // Undo the Swap, This fixes the ugliness caused by a duplicate frame
969 temp
= c
->current_vectors
;
970 c
->current_vectors
= c
->former_vectors
;
971 c
->former_vectors
= temp
;
972 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame
), "motion_est.vectors",
973 (void*)c
->former_vectors
, c
->mv_size
, NULL
, NULL
);
977 if( c
->shot_change
== 1)
979 else if( c
->show_reconstruction
)
980 show_reconstruction( *image
, c
);
981 else if( c
->show_residual
)
982 show_residual( *image
, c
);
986 else if( c
->former_frame_position
== c
->current_frame_position
)
988 // Pass the old vector data into the frame if it's valid
989 if( c
->former_vectors_valid
== 1 ) {
990 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame
), "motion_est.vectors",
991 (void*)c
->current_vectors
, c
->mv_size
, NULL
, NULL
);
993 if( c
->shot_change
== 1)
995 else if( c
->toggle_when_paused
== 1 ) {
996 if( c
->show_reconstruction
)
997 show_reconstruction( *image
, c
);
998 else if( c
->show_residual
)
999 show_residual( *image
, c
);
1000 c
->toggle_when_paused
= 2;
1002 else if( c
->toggle_when_paused
== 2 )
1003 c
->toggle_when_paused
= 1;
1005 if( c
->show_reconstruction
)
1006 show_reconstruction( *image
, c
);
1007 else if( c
->show_residual
)
1008 show_residual( *image
, c
);
1013 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "shot_change", c
->shot_change
);
1015 // there was jump in frame number
1017 // fprintf(stderr, "Warning: there was a frame number jumped from %d to %d.\n", c->former_frame_position, c->current_frame_position);
1018 c
->former_vectors_valid
= 0;
1022 // Cache our bounding geometry for the next frame's processing
1023 c
->prev_left_mb
= c
->left_mb
;
1024 c
->prev_top_mb
= c
->top_mb
;
1025 c
->prev_right_mb
= c
->right_mb
;
1026 c
->prev_bottom_mb
= c
->bottom_mb
;
1028 // Remember which frame this is
1029 c
->former_frame_position
= c
->current_frame_position
;
1032 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.macroblock_width", c
->mb_w
);
1033 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.macroblock_height", c
->mb_h
);
1034 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.left_mb", c
->left_mb
);
1035 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.right_mb", c
->right_mb
);
1036 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.top_mb", c
->top_mb
);
1037 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.bottom_mb", c
->bottom_mb
);
1040 struct timeval finish
; gettimeofday(&finish
, NULL
); int difference
= (finish
.tv_sec
- start
.tv_sec
) * 1000000 + (finish
.tv_usec
- start
.tv_usec
);
1041 fprintf(stderr
, " in frame %d:%d usec\n", c
->current_frame_position
, difference
);
1050 /** filter processing.
1053 static mlt_frame
filter_process( mlt_filter
this, mlt_frame frame
)
1056 // Keeps tabs on the filter object
1057 mlt_frame_push_service( frame
, this);
1059 // Push the frame filter
1060 mlt_frame_push_get_image( frame
, filter_get_image
);
1065 /** Constructor for the filter.
1067 mlt_filter
filter_motion_est_init( char *arg
)
1069 mlt_filter
this = mlt_filter_new( );
1072 // Get the properties object
1073 mlt_properties properties
= MLT_FILTER_PROPERTIES( this );
1075 // Initialize the motion estimation context
1076 struct motion_est_context_s
*context
;
1077 context
= mlt_pool_alloc( sizeof(struct motion_est_context_s
) );
1078 mlt_properties_set_data( properties
, "context", (void *)context
, sizeof( struct motion_est_context_s
),
1079 mlt_pool_release
, NULL
);
1082 // Register the filter
1083 this->process
= filter_process
;
1085 /* defaults that may be overridden */
1088 context
->skip_prediction
= 0;
1089 context
->limit_x
= 64;
1090 context
->limit_y
= 64;
1091 context
->search_method
= DIAMOND_SEARCH
; // FIXME: not used
1092 context
->check_chroma
= 0;
1093 context
->denoise
= 1;
1094 context
->show_reconstruction
= 0;
1095 context
->show_residual
= 0;
1096 context
->toggle_when_paused
= 0;
1098 /* reference functions that may have optimized versions */
1099 context
->compare_reference
= sad_reference
;
1101 // The rest of the buffers will be initialized when the filter is first processed
1102 context
->initialized
= 0;