2 * /brief fast motion estimation filter
3 * /author Zachary Drew, Copyright 2005
5 * Currently only uses Gamma data for comparisonon (bug or feature?)
6 * Vector optimization coming soon.
8 * Vector orientation: The vector data that is generated for the current frame specifies
9 * the motion from the previous frame to the current frame. To know how a macroblock
10 * in the current frame will move in the future, the next frame is needed.
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software Foundation,
24 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
28 #include "filter_motion_est.h"
29 #include <framework/mlt.h>
47 #define DIAMOND_SEARCH 0x0
48 #define FULL_SEARCH 0x1
50 #define MIN(a,b) ((a) > (b) ? (b) : (a))
51 #define ABS(a) ((a) >= 0 ? (a) : (-(a)))
54 typedef struct motion_vector_s motion_vector
;
56 struct motion_est_context_s
58 int initialized
; // true if filter has been initialized
64 /* same as mlt_frame's parameters */
67 /* Operational details */
70 uint8_t *cache_image
; // Copy of current frame
71 uint8_t *former_image
; // Copy of former frame
75 int limit_x
, limit_y
; // max x and y of a motion vector
77 int check_chroma
; // if check_chroma == 1 then compare chroma
80 int show_reconstruction
;
81 int toggle_when_paused
;
85 struct mlt_geometry_item_s bounds
; // Current bounds (from filters crop_detect, autotrack rectangle, or other)
87 /* bounds in macroblock units; macroblocks are completely contained within the boundry */
88 int left_mb
, prev_left_mb
, right_mb
, prev_right_mb
;
89 int top_mb
, prev_top_mb
, bottom_mb
, prev_bottom_mb
;
91 /* size of our vector buffers */
92 int mv_buffer_height
, mv_buffer_width
, mv_size
;
95 int former_vectors_valid
; //<! true if the previous frame's buffered motion vector data is valid
96 motion_vector
*former_vectors
;
97 motion_vector
*current_vectors
;
98 motion_vector
*denoise_vectors
;
99 mlt_position former_frame_position
, current_frame_position
;
101 /* diagnostic metrics */
102 float predictive_misses
; // How often do the prediction motion vectors fail?
103 int comparison_average
; // How far does the best estimation deviate from a perfect comparison?
106 int average_x
, average_y
;
108 /* run-time configurable comparison functions */
109 int (*compare_reference
)(uint8_t *, uint8_t *, int, int, int, int);
110 int (*compare_optimized
)(uint8_t *, uint8_t *, int, int, int, int);
114 // This is used to constrains pixel operations between two blocks to be within the image boundry
115 inline static int constrain( int *x
, int *y
, int *w
, int *h
,
116 const int dx
, const int dy
,
117 const int left
, const int right
,
118 const int top
, const int bottom
)
120 uint32_t penalty
= 1 << SHIFT
; // Retain a few extra bits of precision
126 // Origin of macroblock moves left of image boundy
127 if( *x
< left
|| x2
< left
) {
128 w_remains
= *w
- left
+ ((*x
< x2
) ?
*x
: x2
);
129 *x
+= *w
- w_remains
;
131 // Portion of macroblock moves right of image boundry
132 else if( *x
+ *w
> right
|| x2
+ *w
> right
)
133 w_remains
= right
- ((*x
> x2
) ?
*x
: x2
);
135 // Origin of macroblock moves above image boundy
136 if( *y
< top
|| y2
< top
) {
137 h_remains
= *h
- top
+ ((*y
< y2
) ?
*y
: y2
);
138 *y
+= *h
- h_remains
;
140 // Portion of macroblock moves bellow image boundry
141 else if( *y
+ *h
> bottom
|| y2
+ *h
> bottom
)
142 h_remains
= bottom
- ((*y
> y2
) ?
*y
: y2
);
144 if( w_remains
== *w
&& h_remains
== *h
) return penalty
;
145 if( w_remains
<= 0 || h_remains
<= 0) return 0; // Block is clipped out of existance
146 penalty
= (*w
* *h
* penalty
)
147 / ( w_remains
* h_remains
); // Recipricol of the fraction of the block that remains
149 assert(*x
>= left
); assert(x2
+ *w
- w_remains
>= left
);
150 assert(*y
>= top
); assert(y2
+ *h
- h_remains
>= top
);
151 assert(*x
+ w_remains
<= right
); assert(x2
+ w_remains
<= right
);
152 assert(*y
+ h_remains
<= bottom
); assert(y2
+ h_remains
<= bottom
);
154 *w
= w_remains
; // Update the width and height
160 /** /brief Reference Sum of Absolute Differences comparison function
163 inline static int sad_reference( uint8_t *block1
, uint8_t *block2
, const int xstride
, const int ystride
, const int w
, const int h
)
166 for ( j
= 0; j
< h
; j
++ ){
167 for ( i
= 0; i
< w
; i
++ ){
168 score
+= ABS( block1
[i
*xstride
] - block2
[i
*xstride
] );
178 /** /brief Abstracted block comparison function
180 inline static int block_compare( uint8_t *block1
,
186 struct motion_est_context_s
*c
)
189 #ifdef COUNT_COMPARES
195 // Default comparison may be overridden by the slower, more capable reference comparison
196 int (*cmp
)(uint8_t *, uint8_t *, int, int, int, int) = c
->compare_optimized
;
198 // vector displacement limited has been exceeded
199 if( ABS( dx
) >= c
->limit_x
|| ABS( dy
) >= c
->limit_y
)
202 int mb_w
= c
->mb_w
; // Some writeable local copies
205 // Determine if either macroblock got clipped
206 int penalty
= constrain( &x
, &y
, &mb_w
, &mb_h
, dx
, dy
, 0, c
->width
, 0, c
->height
);
209 if( penalty
== 0 ) // Clipped out of existance: Return worst score
211 else if( penalty
!= 1<<SHIFT
) // Nonstandard macroblock dimensions: Disable SIMD optimizizations.
212 cmp
= c
->compare_reference
;
214 // Calculate the memory locations of the macroblocks
215 block1
+= x
* c
->xstride
+ y
* c
->ystride
;
216 block2
+= (x
+dx
) * c
->xstride
+ (y
+dy
) * c
->ystride
;
219 if( penalty
== 1<<SHIFT
){
220 score
= c
->compare_reference( block1
, block2
, c
->xstride
, c
->ystride
, mb_w
, mb_h
);
221 int score2
= c
->compare_optimized( block1
, block2
, c
->xstride
, c
->ystride
, mb_w
, mb_h
);
222 if ( score
!= score2
)
223 fprintf(stderr
, "Your assembly doesn't work! Reference: %d Asm: %d\n", score
, score2
);
228 score
= cmp( block1
, block2
, c
->xstride
, c
->ystride
, mb_w
, mb_h
);
230 return ( score
* penalty
) >> SHIFT
; // Ditch the extra precision
233 static inline void check_candidates ( uint8_t *ref
,
234 uint8_t *candidate_base
,
237 const motion_vector
*candidates
,// Contains to_x & to_y
238 const int count
, // Number of candidates
239 const int unique
, // Sometimes we know the candidates are unique
240 motion_vector
*result
,
241 struct motion_est_context_s
*c
)
244 /* Scan for the best candidate */
245 for ( i
= 0; i
< count
; i
++ )
247 // this little dohicky ignores duplicate candidates, if they are possible
252 if ( candidates
[j
].dx
== candidates
[i
].dx
&&
253 candidates
[j
].dy
== candidates
[i
].dy
)
261 score
= block_compare( ref
, candidate_base
,
263 candidates
[i
].dx
, // from
267 if ( score
< result
->msad
) { // New minimum
268 result
->dx
= candidates
[i
].dx
;
269 result
->dy
= candidates
[i
].dy
;
270 result
->msad
= score
;
276 /* /brief Diamond search
277 * Operates on a single macroblock
279 static inline void diamond_search(
280 uint8_t *ref
, //<! Image data from previous frame
281 uint8_t *candidate_base
, //<! Image data in current frame
282 const int x
, //<! X upper left corner of macroblock
283 const int y
, //<! U upper left corner of macroblock
284 struct motion_vector_s
*result
, //<! Best predicted mv and eventual result
285 struct motion_est_context_s
*c
) //<! motion estimation context
288 // diamond search pattern
289 motion_vector candidates
[4];
291 // Keep track of best and former best candidates
292 motion_vector best
, former
;
294 // The direction of the refinement needs to be known
295 motion_vector current
;
299 // Loop through the search pattern
302 current
.dx
= result
->dx
;
303 current
.dy
= result
->dy
;
305 if ( first
== 1 ) // Set the initial pattern
307 candidates
[0].dx
= result
->dx
+ 1; candidates
[0].dy
= result
->dy
+ 0;
308 candidates
[1].dx
= result
->dx
+ 0; candidates
[1].dy
= result
->dy
+ 1;
309 candidates
[2].dx
= result
->dx
- 1; candidates
[2].dy
= result
->dy
+ 0;
310 candidates
[3].dx
= result
->dx
+ 0; candidates
[3].dy
= result
->dy
- 1;
313 else // Construct the next portion of the search pattern
315 candidates
[0].dx
= result
->dx
+ best
.dx
;
316 candidates
[0].dy
= result
->dy
+ best
.dy
;
317 if (best
.dx
== former
.dx
&& best
.dy
== former
.dy
) {
318 candidates
[1].dx
= result
->dx
+ best
.dy
;
319 candidates
[1].dy
= result
->dy
+ best
.dx
; // Yes, the wires
320 candidates
[2].dx
= result
->dx
- best
.dy
; // are crossed
321 candidates
[2].dy
= result
->dy
- best
.dx
;
324 candidates
[1].dx
= result
->dx
+ former
.dx
;
325 candidates
[1].dy
= result
->dy
+ former
.dy
;
329 former
.dx
= best
.dx
; former
.dy
= best
.dy
; // Keep track of new former best
332 check_candidates ( ref
, candidate_base
, x
, y
, candidates
, i
, 1, result
, c
);
334 // Which candidate was the best?
335 best
.dx
= result
->dx
- current
.dx
;
336 best
.dy
= result
->dy
- current
.dy
;
338 // A better canidate was not found
339 if ( best
.dx
== 0 && best
.dy
== 0 )
344 former
.dx
= best
.dx
; former
.dy
= best
.dy
; // First iteration, sensible value for former.d*
349 /* /brief Full (brute) search
350 * Operates on a single macroblock
352 __attribute__((used
))
353 static void full_search(
354 uint8_t *ref
, //<! Image data from previous frame
355 uint8_t *candidate_base
, //<! Image data in current frame
356 int x
, //<! X upper left corner of macroblock
357 int y
, //<! U upper left corner of macroblock
358 struct motion_vector_s
*result
, //<! Best predicted mv and eventual result
359 struct motion_est_context_s
*c
) //<! motion estimation context
361 // Keep track of best candidate
365 for( i
= -c
->mb_w
; i
<= c
->mb_w
; i
++ ){
366 for( j
= -c
->mb_h
; j
<= c
->mb_h
; j
++ ){
368 score
= block_compare( ref
, candidate_base
,
375 if ( score
< result
->msad
) {
378 result
->msad
= score
;
384 // Macros for pointer calculations
385 #define CURRENT(i,j) ( c->current_vectors + (j)*c->mv_buffer_width + (i) )
386 #define FORMER(i,j) ( c->former_vectors + (j)*c->mv_buffer_width + (i) )
387 #define DENOISE(i,j) ( c->denoise_vectors + (j)*c->mv_buffer_width + (i) )
389 int ncompare (const void * a
, const void * b
)
391 return ( *(int*)a
- *(int*)b
);
394 // motion vector denoising
395 // for x and y components seperately,
396 // change the vector to be the median value of the 9 adjacent vectors
397 static void median_denoise( motion_vector
*v
, struct motion_est_context_s
*c
)
399 int xvalues
[9], yvalues
[9];
402 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ )
403 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
407 xvalues
[n
] = CURRENT(i
,j
)->dx
; // Center
408 yvalues
[n
++] = CURRENT(i
,j
)->dy
;
410 if( i
> c
->left_mb
) // Not in First Column
412 xvalues
[n
] = CURRENT(i
-1,j
)->dx
; // Left
413 yvalues
[n
++] = CURRENT(i
-1,j
)->dy
;
415 if( j
> c
->top_mb
) {
416 xvalues
[n
] = CURRENT(i
-1,j
-1)->dx
; // Upper Left
417 yvalues
[n
++] = CURRENT(i
-1,j
-1)->dy
;
420 if( j
< c
->bottom_mb
) {
421 xvalues
[n
] = CURRENT(i
-1,j
+1)->dx
; // Bottom Left
422 yvalues
[n
++] = CURRENT(i
-1,j
+1)->dy
;
425 if( i
< c
->right_mb
) // Not in Last Column
427 xvalues
[n
] = CURRENT(i
+1,j
)->dx
; // Right
428 yvalues
[n
++] = CURRENT(i
+1,j
)->dy
;
431 if( j
> c
->top_mb
) {
432 xvalues
[n
] = CURRENT(i
+1,j
-1)->dx
; // Upper Right
433 yvalues
[n
++] = CURRENT(i
+1,j
-1)->dy
;
436 if( j
< c
->bottom_mb
) {
437 xvalues
[n
] = CURRENT(i
+1,j
+1)->dx
; // Bottom Right
438 yvalues
[n
++] = CURRENT(i
+1,j
+1)->dy
;
441 if( j
> c
->top_mb
) // Not in First Row
443 xvalues
[n
] = CURRENT(i
,j
-1)->dx
; // Top
444 yvalues
[n
++] = CURRENT(i
,j
-1)->dy
;
447 if( j
< c
->bottom_mb
) // Not in Last Row
449 xvalues
[n
] = CURRENT(i
,j
+1)->dx
; // Bottom
450 yvalues
[n
++] = CURRENT(i
,j
+1)->dy
;
453 qsort (xvalues
, n
, sizeof(int), ncompare
);
454 qsort (yvalues
, n
, sizeof(int), ncompare
);
457 DENOISE(i
,j
)->dx
= xvalues
[n
/2];
458 DENOISE(i
,j
)->dy
= yvalues
[n
/2];
461 DENOISE(i
,j
)->dx
= (xvalues
[n
/2] + xvalues
[n
/2+1])/2;
462 DENOISE(i
,j
)->dy
= (yvalues
[n
/2] + yvalues
[n
/2+1])/2;
467 motion_vector
*t
= c
->current_vectors
;
468 c
->current_vectors
= c
->denoise_vectors
;
469 c
->denoise_vectors
= t
;
475 static inline int median_predictor(int a
, int b
, int c
) {
491 /** /brief Motion search
493 * For each macroblock in the current frame, estimate the block from the last frame that
496 * Vocab: Colocated - the pixel in the previous frame at the current position
498 * Based on enhanced predictive zonal search. [Tourapis 2002]
500 static void motion_search( uint8_t *from
, //<! Image data.
501 uint8_t *to
, //<! Image data. Rigid grid.
502 struct motion_est_context_s
*c
) //<! The context
505 #ifdef COUNT_COMPARES
509 motion_vector candidates
[10];
510 motion_vector
*here
; // This one gets used alot (about 30 times per macroblock)
515 // For every macroblock, perform motion vector estimation
516 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
517 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
522 here
->msad
= MAX_MSAD
;
527 /* Stack the predictors [i.e. checked in reverse order] */
529 /* Adjacent to collocated */
530 if( c
->former_vectors_valid
)
533 if( j
> c
->prev_top_mb
){// && COL_TOP->valid ){
534 candidates
[n
].dx
= FORMER(i
,j
-1)->dx
;
535 candidates
[n
++].dy
= FORMER(i
,j
-1)->dy
;
539 if( i
> c
->prev_left_mb
){// && COL_LEFT->valid ){
540 candidates
[n
].dx
= FORMER(i
-1,j
)->dx
;
541 candidates
[n
++].dy
= FORMER(i
-1,j
)->dy
;
544 // Right of colocated
545 if( i
< c
->prev_right_mb
){// && COL_RIGHT->valid ){
546 candidates
[n
].dx
= FORMER(i
+1,j
)->dx
;
547 candidates
[n
++].dy
= FORMER(i
+1,j
)->dy
;
550 // Bottom of colocated
551 if( j
< c
->prev_bottom_mb
){// && COL_BOTTOM->valid ){
552 candidates
[n
].dx
= FORMER(i
,j
+1)->dx
;
553 candidates
[n
++].dy
= FORMER(i
,j
+1)->dy
;
556 // And finally, colocated
557 candidates
[n
].dx
= FORMER(i
,j
)->dx
;
558 candidates
[n
++].dy
= FORMER(i
,j
)->dy
;
561 // For macroblocks not in the top row
562 if ( j
> c
->top_mb
) {
564 // Top if ( TOP->valid ) {
565 candidates
[n
].dx
= CURRENT(i
,j
-1)->dx
;
566 candidates
[n
++].dy
= CURRENT(i
,j
-1)->dy
;
569 // Top-Right, macroblocks not in the right row
570 if ( i
< c
->right_mb
){// && TOP_RIGHT->valid ) {
571 candidates
[n
].dx
= CURRENT(i
+1,j
-1)->dx
;
572 candidates
[n
++].dy
= CURRENT(i
+1,j
-1)->dy
;
576 // Left, Macroblocks not in the left column
577 if ( i
> c
->left_mb
){// && LEFT->valid ) {
578 candidates
[n
].dx
= CURRENT(i
-1,j
)->dx
;
579 candidates
[n
++].dy
= CURRENT(i
-1,j
)->dy
;
582 /* Median predictor vector (median of left, top, and top right adjacent vectors) */
583 if ( i
> c
->left_mb
&& j
> c
->top_mb
&& i
< c
->right_mb
584 )//&& LEFT->valid && TOP->valid && TOP_RIGHT->valid )
586 candidates
[n
].dx
= median_predictor( CURRENT(i
-1,j
)->dx
, CURRENT(i
,j
-1)->dx
, CURRENT(i
+1,j
-1)->dx
);
587 candidates
[n
++].dy
= median_predictor( CURRENT(i
-1,j
)->dy
, CURRENT(i
,j
-1)->dy
, CURRENT(i
+1,j
-1)->dy
);
591 candidates
[n
].dx
= 0;
592 candidates
[n
++].dy
= 0;
596 check_candidates ( to
, from
, x
, y
, candidates
, n
, 0, here
, c
);
600 diamond_search( to
, from
, x
, y
, here
, c
);
602 full_search( to
, from
, x
, y
, here
, c
);
605 assert( x
+ c
->mb_w
+ here
->dx
> 0 ); // All macroblocks must have area > 0
606 assert( y
+ c
->mb_h
+ here
->dy
> 0 );
607 assert( x
+ here
->dx
< c
->width
);
608 assert( y
+ here
->dy
< c
->height
);
610 } /* End column loop */
613 asm volatile ( "emms" );
615 #ifdef COUNT_COMPARES
616 fprintf(stderr
, "%d comparisons per block were made", compares
/count
);
621 void collect_post_statistics( struct motion_est_context_s
*c
) {
623 c
->comparison_average
= 0;
624 c
->average_length
= 0;
630 for ( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
631 for ( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
634 c
->comparison_average
+= CURRENT(i
,j
)->msad
;
635 c
->average_x
+= CURRENT(i
,j
)->dx
;
636 c
->average_y
+= CURRENT(i
,j
)->dy
;
644 c
->comparison_average
/= count
;
645 c
->average_x
/= count
;
646 c
->average_y
/= count
;
647 c
->average_length
= sqrt( c
->average_x
* c
->average_x
+ c
->average_y
* c
->average_y
);
652 static void init_optimizations( struct motion_est_context_s
*c
)
655 case 4: if(c
->mb_h
== 4) c
->compare_optimized
= sad_sse_422_luma_4x4
;
656 else c
->compare_optimized
= sad_sse_422_luma_4w
;
658 case 8: if(c
->mb_h
== 8) c
->compare_optimized
= sad_sse_422_luma_8x8
;
659 else c
->compare_optimized
= sad_sse_422_luma_8w
;
661 case 16: if(c
->mb_h
== 16) c
->compare_optimized
= sad_sse_422_luma_16x16
;
662 else c
->compare_optimized
= sad_sse_422_luma_16w
;
664 case 32: if(c
->mb_h
== 32) c
->compare_optimized
= sad_sse_422_luma_32x32
;
665 else c
->compare_optimized
= sad_sse_422_luma_32w
;
667 case 64: c
->compare_optimized
= sad_sse_422_luma_64w
;
669 default: c
->compare_optimized
= sad_reference
;
674 inline static void set_red(uint8_t *image
, struct motion_est_context_s
*c
)
677 for( n
= 0; n
< c
->width
* c
->height
* 2; n
+=4 )
687 static void show_residual( uint8_t *result
, struct motion_est_context_s
*c
)
695 // set_red(result,c);
697 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
698 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
700 dx
= CURRENT(i
,j
)->dx
;
701 dy
= CURRENT(i
,j
)->dy
;
707 // Denoise function caused some blocks to be completely clipped, ignore them
708 if (constrain( &x
, &y
, &w
, &h
, dx
, dy
, 0, c
->width
, 0, c
->height
) == 0 )
711 for( ty
= y
; ty
< y
+ h
; ty
++ ){
712 for( tx
= x
; tx
< x
+ w
; tx
++ ){
714 b
= c
->former_image
+ (tx
+dx
)*c
->xstride
+ (ty
+dy
)*c
->ystride
;
715 r
= result
+ tx
*c
->xstride
+ ty
*c
->ystride
;
717 r
[0] = 16 + ABS( r
[0] - b
[0] );
720 r
[1] = 128 + ABS( r
[1] - b
[1] );
722 // FIXME: may exceed boundies
723 r
[1] = 128 + ABS( r
[1] - ( *(b
-1) + b
[3] ) /2 );
730 static void show_reconstruction( uint8_t *result
, struct motion_est_context_s
*c
)
738 for( i
= c
->left_mb
; i
<= c
->right_mb
; i
++ ){
739 for( j
= c
->top_mb
; j
<= c
->bottom_mb
; j
++ ){
741 dx
= CURRENT(i
,j
)->dx
;
742 dy
= CURRENT(i
,j
)->dy
;
748 // Denoise function caused some blocks to be completely clipped, ignore them
749 if (constrain( &x
, &y
, &w
, &h
, dx
, dy
, 0, c
->width
, 0, c
->height
) == 0 )
752 for( ty
= y
; ty
< y
+ h
; ty
++ ){
753 for( tx
= x
; tx
< x
+ w
; tx
++ ){
755 r
= result
+ tx
*c
->xstride
+ ty
*c
->ystride
;
756 s
= c
->former_image
+ (tx
+dx
)*c
->xstride
+ (ty
+dy
)*c
->ystride
;
763 // FIXME: may exceed boundies
764 r
[1] = ( *(s
-1) + s
[3] ) /2;
771 // Image stack(able) method
772 static int filter_get_image( mlt_frame frame
, uint8_t **image
, mlt_image_format
*format
, int *width
, int *height
, int writable
)
775 mlt_filter filter
= mlt_frame_pop_service( frame
);
777 // Get the motion_est context object
778 struct motion_est_context_s
*c
= mlt_properties_get_data( MLT_FILTER_PROPERTIES( filter
), "context", NULL
);
781 // Get the new image and frame number
782 int error
= mlt_frame_get_image( frame
, image
, format
, width
, height
, 1 );
785 struct timeval start
; gettimeofday(&start
, NULL
);
790 mlt_properties_debug( MLT_FRAME_PROPERTIES(frame
), "error after mlt_frame_get_image() in motion_est", stderr
);
792 c
->current_frame_position
= mlt_frame_get_position( frame
);
794 /* Context Initialization */
795 if ( c
->initialized
== 0 ) {
797 // Get the filter properties object
798 mlt_properties properties
= mlt_filter_properties( filter
);
803 /* Get parameters that may have been overridden */
804 if( mlt_properties_get( properties
, "macroblock_width") != NULL
)
805 c
->mb_w
= mlt_properties_get_int( properties
, "macroblock_width");
807 if( mlt_properties_get( properties
, "macroblock_height") != NULL
)
808 c
->mb_h
= mlt_properties_get_int( properties
, "macroblock_height");
810 if( mlt_properties_get( properties
, "prediction_thresh") != NULL
)
811 c
->initial_thresh
= mlt_properties_get_int( properties
, "prediction_thresh" );
813 c
->initial_thresh
= c
->mb_w
* c
->mb_h
;
815 if( mlt_properties_get( properties
, "search_method") != NULL
)
816 c
->search_method
= mlt_properties_get_int( properties
, "search_method");
818 if( mlt_properties_get( properties
, "skip_prediction") != NULL
)
819 c
->skip_prediction
= mlt_properties_get_int( properties
, "skip_prediction");
821 if( mlt_properties_get( properties
, "limit_x") != NULL
)
822 c
->limit_x
= mlt_properties_get_int( properties
, "limit_x");
824 if( mlt_properties_get( properties
, "limit_y") != NULL
)
825 c
->limit_y
= mlt_properties_get_int( properties
, "limit_y");
827 if( mlt_properties_get( properties
, "check_chroma" ) != NULL
)
828 c
->check_chroma
= mlt_properties_get_int( properties
, "check_chroma" );
830 if( mlt_properties_get( properties
, "denoise" ) != NULL
)
831 c
->denoise
= mlt_properties_get_int( properties
, "denoise" );
833 if( mlt_properties_get( properties
, "show_reconstruction" ) != NULL
)
834 c
->show_reconstruction
= mlt_properties_get_int( properties
, "show_reconstruction" );
836 if( mlt_properties_get( properties
, "show_residual" ) != NULL
)
837 c
->show_residual
= mlt_properties_get_int( properties
, "show_residual" );
839 if( mlt_properties_get( properties
, "toggle_when_paused" ) != NULL
)
840 c
->toggle_when_paused
= mlt_properties_get_int( properties
, "toggle_when_paused" );
842 init_optimizations( c
);
844 // Calculate the dimensions in macroblock units
845 c
->mv_buffer_width
= (*width
/ c
->mb_w
);
846 c
->mv_buffer_height
= (*height
/ c
->mb_h
);
848 // Size of the motion vector buffer
849 c
->mv_size
= c
->mv_buffer_width
* c
->mv_buffer_height
* sizeof(struct motion_vector_s
);
851 // Allocate the motion vector buffers
852 c
->former_vectors
= mlt_pool_alloc( c
->mv_size
);
853 c
->current_vectors
= mlt_pool_alloc( c
->mv_size
);
854 c
->denoise_vectors
= mlt_pool_alloc( c
->mv_size
);
856 // Register motion buffers for destruction
857 mlt_properties_set_data( properties
, "current_motion_vectors", (void *)c
->current_vectors
, 0, mlt_pool_release
, NULL
);
858 mlt_properties_set_data( properties
, "former_motion_vectors", (void *)c
->former_vectors
, 0, mlt_pool_release
, NULL
);
859 mlt_properties_set_data( properties
, "denoise_motion_vectors", (void *)c
->denoise_vectors
, 0, mlt_pool_release
, NULL
);
861 c
->former_vectors_valid
= 0;
862 memset( c
->former_vectors
, 0, c
->mv_size
);
864 // Calculate the size of our steps (the number of bytes that seperate adjacent pixels in X and Y direction)
866 case mlt_image_yuv422
:
868 c
->ystride
= c
->xstride
* *width
;
872 fprintf(stderr
, "\"I am unfamiliar with your new fangled pixel format!\" -filter_motion_est\n");
876 // Allocate a cache for the previous frame's image
877 c
->former_image
= mlt_pool_alloc( *width
* *height
* 2 );
878 c
->cache_image
= mlt_pool_alloc( *width
* *height
* 2 );
880 // Register for destruction
881 mlt_properties_set_data( properties
, "cache_image", (void *)c
->cache_image
, 0, mlt_pool_release
, NULL
);
882 mlt_properties_set_data( properties
, "former_image", (void *)c
->former_image
, 0, mlt_pool_release
, NULL
);
884 c
->former_frame_position
= c
->current_frame_position
;
885 c
->previous_msad
= 0;
890 /* Check to see if somebody else has given us bounds */
891 struct mlt_geometry_item_s
*bounds
= mlt_properties_get_data( MLT_FRAME_PROPERTIES( frame
), "bounds", NULL
);
893 if( bounds
!= NULL
) {
894 // translate pixel units (from bounds) to macroblock units
895 // make sure whole macroblock stays within bounds
896 c
->left_mb
= ( bounds
->x
+ c
->mb_w
- 1 ) / c
->mb_w
;
897 c
->top_mb
= ( bounds
->y
+ c
->mb_h
- 1 ) / c
->mb_h
;
898 c
->right_mb
= ( bounds
->x
+ bounds
->w
) / c
->mb_w
- 1;
899 c
->bottom_mb
= ( bounds
->y
+ bounds
->h
) / c
->mb_h
- 1;
900 c
->bounds
.x
= bounds
->x
;
901 c
->bounds
.y
= bounds
->y
;
902 c
->bounds
.w
= bounds
->w
;
903 c
->bounds
.h
= bounds
->h
;
905 c
->left_mb
= c
->prev_left_mb
= 0;
906 c
->top_mb
= c
->prev_top_mb
= 0;
907 c
->right_mb
= c
->prev_right_mb
= c
->mv_buffer_width
- 1; // Zero indexed
908 c
->bottom_mb
= c
->prev_bottom_mb
= c
->mv_buffer_height
- 1;
911 c
->bounds
.w
= *width
;
912 c
->bounds
.h
= *height
;
915 // If video is advancing, run motion vector algorithm and etc...
916 if( c
->former_frame_position
+ 1 == c
->current_frame_position
)
919 // Swap the motion vector buffers and reuse allocated memory
920 struct motion_vector_s
*temp
= c
->current_vectors
;
921 c
->current_vectors
= c
->former_vectors
;
922 c
->former_vectors
= temp
;
924 // This is done because filter_vismv doesn't pay attention to frame boundry
925 memset( c
->current_vectors
, 0, c
->mv_size
);
927 // Perform the motion search
928 motion_search( c
->cache_image
, *image
, c
);
930 collect_post_statistics( c
);
933 // Detect shot changes
934 if( c
->comparison_average
> 10 * c
->mb_w
* c
->mb_h
&&
935 c
->comparison_average
> c
->previous_msad
* 2 )
937 fprintf(stderr
, " - SAD: %d <<Shot change>>\n", c
->comparison_average
);
938 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "shot_change", 1);
939 // c->former_vectors_valid = 0; // Invalidate the previous frame's predictors
943 c
->former_vectors_valid
= 1;
945 //fprintf(stderr, " - SAD: %d\n", c->comparison_average);
948 c
->previous_msad
= c
->comparison_average
;
950 if( c
->comparison_average
!= 0 ) { // If the frame is not a duplicate of the previous frame
952 // denoise the vector buffer
954 median_denoise( c
->current_vectors
, c
);
956 // Pass the new vector data into the frame
957 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame
), "motion_est.vectors",
958 (void*)c
->current_vectors
, c
->mv_size
, NULL
, NULL
);
960 // Cache the frame's image. Save the old cache. Reuse memory.
961 // After this block, exactly two unique frames will be cached
962 uint8_t *timg
= c
->cache_image
;
963 c
->cache_image
= c
->former_image
;
964 c
->former_image
= timg
;
965 memcpy( c
->cache_image
, *image
, *width
* *height
* c
->xstride
);
970 // Undo the Swap, This fixes the ugliness caused by a duplicate frame
971 temp
= c
->current_vectors
;
972 c
->current_vectors
= c
->former_vectors
;
973 c
->former_vectors
= temp
;
974 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame
), "motion_est.vectors",
975 (void*)c
->former_vectors
, c
->mv_size
, NULL
, NULL
);
979 if( c
->shot_change
== 1)
981 else if( c
->show_reconstruction
)
982 show_reconstruction( *image
, c
);
983 else if( c
->show_residual
)
984 show_residual( *image
, c
);
988 else if( c
->former_frame_position
== c
->current_frame_position
)
990 // Pass the old vector data into the frame if it's valid
991 if( c
->former_vectors_valid
== 1 ) {
992 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame
), "motion_est.vectors",
993 (void*)c
->current_vectors
, c
->mv_size
, NULL
, NULL
);
995 if( c
->shot_change
== 1)
997 else if( c
->toggle_when_paused
== 1 ) {
998 if( c
->show_reconstruction
)
999 show_reconstruction( *image
, c
);
1000 else if( c
->show_residual
)
1001 show_residual( *image
, c
);
1002 c
->toggle_when_paused
= 2;
1004 else if( c
->toggle_when_paused
== 2 )
1005 c
->toggle_when_paused
= 1;
1007 if( c
->show_reconstruction
)
1008 show_reconstruction( *image
, c
);
1009 else if( c
->show_residual
)
1010 show_residual( *image
, c
);
1015 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "shot_change", c
->shot_change
);
1017 // there was jump in frame number
1019 // fprintf(stderr, "Warning: there was a frame number jumped from %d to %d.\n", c->former_frame_position, c->current_frame_position);
1020 c
->former_vectors_valid
= 0;
1024 // Cache our bounding geometry for the next frame's processing
1025 c
->prev_left_mb
= c
->left_mb
;
1026 c
->prev_top_mb
= c
->top_mb
;
1027 c
->prev_right_mb
= c
->right_mb
;
1028 c
->prev_bottom_mb
= c
->bottom_mb
;
1030 // Remember which frame this is
1031 c
->former_frame_position
= c
->current_frame_position
;
1034 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.macroblock_width", c
->mb_w
);
1035 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame
), "motion_est.macroblock_height", c
->mb_h
);
1038 struct timeval finish
; gettimeofday(&finish
, NULL
); int difference
= (finish
.tv_sec
- start
.tv_sec
) * 1000000 + (finish
.tv_usec
- start
.tv_usec
);
1039 fprintf(stderr
, " in frame %d:%d usec\n", c
->current_frame_position
, difference
);
1048 /** filter processing.
1051 static mlt_frame
filter_process( mlt_filter
this, mlt_frame frame
)
1054 // Keeps tabs on the filter object
1055 mlt_frame_push_service( frame
, this);
1057 // Push the frame filter
1058 mlt_frame_push_get_image( frame
, filter_get_image
);
1063 /** Constructor for the filter.
1065 mlt_filter
filter_motion_est_init( char *arg
)
1067 mlt_filter
this = mlt_filter_new( );
1070 // Get the properties object
1071 mlt_properties properties
= MLT_FILTER_PROPERTIES( this );
1073 // Initialize the motion estimation context
1074 struct motion_est_context_s
*context
;
1075 context
= mlt_pool_alloc( sizeof(struct motion_est_context_s
) );
1076 mlt_properties_set_data( properties
, "context", (void *)context
, sizeof( struct motion_est_context_s
),
1077 mlt_pool_release
, NULL
);
1080 // Register the filter
1081 this->process
= filter_process
;
1083 /* defaults that may be overridden */
1086 context
->skip_prediction
= 0;
1087 context
->limit_x
= 64;
1088 context
->limit_y
= 64;
1089 context
->search_method
= DIAMOND_SEARCH
; // FIXME: not used
1090 context
->check_chroma
= 0;
1091 context
->denoise
= 1;
1092 context
->show_reconstruction
= 0;
1093 context
->show_residual
= 0;
1094 context
->toggle_when_paused
= 0;
1096 /* reference functions that may have optimized versions */
1097 context
->compare_reference
= sad_reference
;
1099 // The rest of the buffers will be initialized when the filter is first processed
1100 context
->initialized
= 0;