Skip to content

Commit

Permalink
MAVLinkParam: fix rounding to 7 digits
Browse files Browse the repository at this point in the history
Rounding to fixed number of decimal places led to over rounding small
numbers and under-rounding big numbers.
  • Loading branch information
robertlong13 committed Aug 11, 2024
1 parent 12a7456 commit 1c11902
Showing 1 changed file with 10 additions and 1 deletion.
11 changes: 10 additions & 1 deletion ExtLibs/Mavlink/MAVLinkParam.cs
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,21 @@ public double GetValue()
(double)item.float_value
0.800000011920929
*/
return Math.Round((double)float_value, 7);
return RoundToSignificantDigits(float_value, 7);
}

throw new FormatException("invalid type");
}

private double RoundToSignificantDigits(double d, int digits)
{
if (d == 0)
return 0;

double scale = Math.Pow(10, Math.Floor(Math.Log10(Math.Abs(d))) + 1);
return scale * Math.Round(d / scale, digits);
}

public void SetValue(double input)
{
switch (Type)
Expand Down

0 comments on commit 1c11902

Please sign in to comment.