Update EX_SIMPLE_IMAGE_STABILIZATION.hpp

Author: lauriebose

EXAMPLES/EX_SIMPLE_IMAGE_STABILIZATION.hpp

@@ -33,11 +33,15 @@ int main()
     vs_handle display_00 = vs_gui_add_display("Stabilized Image",0,0,display_size);
     vs_handle display_01 = vs_gui_add_display("Captured Image",0,display_size,display_size);
 
+    //The variables used to store how the current image should be shifted to align it with the stored key-frame
+    //Note this is a very simple method in that it only aligns via translation, and does not account for rotation & scaling
 	int alignment_x = 0; int alignment_y = 0;
-	int vel_x = 0; int vel_y = 0;
+	
+	//Varibles used to keep track of alignment "velocity", allows more robust alignment under motion by estimating the new alignment each frame using this "velocity"
+	int alignment_vel_x = 0; int alignment_vel_y = 0;
 
-	int use_inertia = 1;
-	vs_gui_add_switch("use_inertia",1,&use_inertia);
+	int use_velocity = 1;
+	vs_gui_add_switch("use_velocity",1,&use_velocity);
 
 	bool set_anchor_image = false;
 	auto gui_btn_set_anchor_image = vs_gui_add_button("set_anchor_image");
@@ -49,6 +53,12 @@ int main()
     int edge_threshold = 12;
     vs_gui_add_slider("edge_threshold x",0,127,edge_threshold,&edge_threshold);
 
+    int alignment_strength = 0;
+    int minimum_alignment_strength = 2000;
+    vs_gui_add_slider("minimum_alignment_strength",0,4000,minimum_alignment_strength,&minimum_alignment_strength);
+
+	int show_current_edge_image = 1;
+	vs_gui_add_switch("show_captured_image",1,&show_current_edge_image);
 
     // Frame Loop
     while(1)
@@ -58,6 +68,13 @@ int main()
        	vs_disable_frame_trigger();
         vs_frame_loop_control();
 
+        if(alignment_strength < minimum_alignment_strength)
+        {
+        	set_anchor_image = true;
+        	vs_post_text("insufficient edges");
+        }
+        alignment_strength = 0;
+
         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
         //CAPTURE FRAME
 
@@ -120,19 +137,13 @@ int main()
 				OR(DREG_combined_edges,DREG_horizontal_edges);//OR WITH HORIZONTAL EDGE RESULT
 			scamp5_kernel_end();
 
-
-			int position_threshold = 48;
-			if(abs(alignment_x) > position_threshold || abs(alignment_y) > position_threshold)
-			{
-				set_anchor_image = true;
-			}
 			if(set_anchor_image)
 			{
 				set_anchor_image = false;
 				alignment_x = 0;
 				alignment_y = 0;
-				vel_x = 0;
-				vel_y = 0;
+				alignment_vel_x = 0;
+				alignment_vel_y = 0;
 
 				scamp5_kernel_begin();
 					MOV(DREG_keyframe_edges,DREG_combined_edges);
@@ -147,26 +158,26 @@ int main()
         //IMAGE ALIGNMENT
 		//This code updates the "position" at which the captured image aligns with the stored key-frame
 
-			if(use_inertia)
+			if(use_velocity)
 			{
-				alignment_x += vel_x;
-				alignment_y += vel_y;
+				alignment_x += alignment_vel_x;
+				alignment_y += alignment_vel_y;
 			}
 
-			//Copy the latest edge frame and shift it to the previous position determined to align with the edge key-frame
+			//Copy the latest edge frame and shift it to the position determined to align with the edge key-frame
 			scamp5_kernel_begin();
 				MOV(S4,S1);
 			scamp5_kernel_end();
 			scamp5_shift(S4,alignment_x,alignment_y);
 
-			//The  directions to shift and test for better alignment (i.e. noshift,left,right,down,up)
+			//The shifts to test and see if they improve image alignment with the keyframe
 			int test_shift_directions[][5] =
 			{
-			    {0,0},
-				{1,0},
-				{-1, 0},
-				{0,1},
-				{0, -1},
+			    {0,0},//no shift
+				{1,0},//left
+				{-1, 0},//right
+				{0,1},//down
+				{0, -1},//up
 			};
 
 			//Array to store the global summations for each tested shift
@@ -175,14 +186,14 @@ int main()
 			//Test each shift
 			for (int n = 0; n < 5; n++)
 			{
-				int x = test_shift_directions[n][0];
-				int y = test_shift_directions[n][1];
+				int shift_x = test_shift_directions[n][0];
+				int shift_y = test_shift_directions[n][1];
 
 				//Copy the shifted edge frame & apply the test shift to it
 				scamp5_kernel_begin();
 					MOV(S6,S4);
 				scamp5_kernel_end();
-				scamp5_shift(S6,x,y);
+				scamp5_shift(S6,shift_x,shift_y);
 
 				scamp5_in(F,127);
 				scamp5_kernel_begin();
@@ -193,38 +204,57 @@ int main()
 						mov(D,F);//D = 127 in PEs where the edges of
 					ALL();
 				scamp5_kernel_end();
+
 				direction_global_sums[n] = scamp5_global_sum_64(D);
+
+				//The highest value sum will have the best alignment, ie. the most overlap between the edge from the current frame and the key-frame
+				if(alignment_strength < direction_global_sums[n])
+				{
+					alignment_strength = direction_global_sums[n];
+				}
 			}
 
 			if(direction_global_sums[0] > direction_global_sums[1] &&
-					direction_global_sums[0] > direction_global_sums[2] &&
-					direction_global_sums[0] > direction_global_sums[3] &&
-					direction_global_sums[0] > direction_global_sums[4])
+				direction_global_sums[0] > direction_global_sums[2] &&
+				direction_global_sums[0] > direction_global_sums[3] &&
+				direction_global_sums[0] > direction_global_sums[4])
 			{
-
+				//The best alignment was when no shift was applied at all so nothing need be updated
 			}
 			else
 			{
-				if(direction_global_sums[1] < direction_global_sums[2])
-				{
-					alignment_x--;
-					vel_x--;
-				}
-				else
-				{
-					alignment_x++;
-					vel_x++;
-				}
+				//Determine which horizontal and vertical shift directions that improved alignment, and update the alignment & "velocity"
 
-				if(direction_global_sums[3] < direction_global_sums[4])
+				//Did either shifting left or right result in a better alignment than not shifting at all?
+				if(direction_global_sums[0] < direction_global_sums[1] || direction_global_sums[0] < direction_global_sums[2])
 				{
-					alignment_y--;
-					vel_y--;
+					//Compare the summation from shifting left and right
+					if(direction_global_sums[1] < direction_global_sums[2])
+					{
+						alignment_x--;
+						alignment_vel_x--;
+					}
+					else
+					{
+						alignment_x++;
+						alignment_vel_x++;
+					}
 				}
-				else
+
+				//Did either shifting left or right result in a better alignment than not shifting at all?
+				if(direction_global_sums[0] < direction_global_sums[3] || direction_global_sums[0] < direction_global_sums[4])
 				{
-					alignment_y++;
-					vel_y++;
+					//Compare the summation from shifting up and down
+					if(direction_global_sums[3] < direction_global_sums[4])
+					{
+						alignment_y--;
+						alignment_vel_y--;
+					}
+					else
+					{
+						alignment_y++;
+						alignment_vel_y++;
+					}
 				}
 			}
 
@@ -245,11 +275,11 @@ int main()
 				scamp5_kernel_end();
 				scamp5_output_image(S6,display_00);
 
-				scamp5_output_image(S1,display_01);
-
-
-
-//				scamp5_output_image(DREG_keyframe_edges,display_02);
+				if(show_current_edge_image)
+				{
+					scamp5_output_image(S1,display_01);
+				}
+				vs_post_text("alignment (%d,%d), velocity (%d,%d), strength %d \n",alignment_x,alignment_y,alignment_vel_x,alignment_vel_y,alignment_strength);
 			}
 			int image_output_time_microseconds = output_timer.get_usec();//get the time taken for image output