inverse_beta_cdf hits signed integer overflow with very large values

[kevinwilfong]

Description

inverse_beta_cdf hits a signed integer overflow and crashes (with UBSan enabled)

in Presto Java, running with the same arguments causes it to throw a system exception illegal state: Continued fraction diverged to NaN for value 0.5

Error Reproduction

select inverse_beta_cdf(6.76044e+170, 6.76044e+170, 0.0756564);

Relevant logs

boost/include/boost/math/special_functions/beta.hpp:686:24: runtime error: signed integer overflow: 2 * 1073741824 cannot be represented in type 'int'
    #0 0x32be2a1 in boost::math::detail::ibeta_fraction2_t<long double>::operator()() boost/include/boost/math/special_functions/beta.hpp:686
    #1 0x32be2a1 in boost::math::tools::detail::fraction_traits<boost::math::detail::ibeta_fraction2_t<long double>>::result_type boost::math::tools::continued_fraction_b<boost::math::detail::ibeta_fraction2_t<long double>, long double>(boost::math::detail::ibeta_fraction2_t<long double>&, long double const&, unsigned long&) boost/include/boost/math/tools/fraction.hpp:134
    #2 0x329c954 in boost::math::tools::detail::fraction_traits<boost::math::detail::ibeta_fraction2_t<long double>>::result_type boost::math::tools::continued_fraction_b<boost::math::detail::ibeta_fraction2_t<long double>, long double>(boost::math::detail::ibeta_fraction2_t<long double>&, long double const&) boost/include/boost/math/tools/fraction.hpp:156
    #3 0x329c954 in long double boost::math::detail::ibeta_fraction2<long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>>(long double, long double, long double, long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy> const&, bool, long double*) boost/include/boost/math/special_functions/beta.hpp:720
    #4 0x329586c in long double boost::math::detail::ibeta_imp<long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>>(long double, long double, long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy> const&, bool, bool, long double*) boost/include/boost/math/special_functions/beta.hpp
    #5 0x33fc532 in boost::math::detail::ibeta_roots<long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>>::operator()(long double) boost/include/boost/math/special_functions/detail/ibeta_inverse.hpp:427
    #6 0x33fa636 in long double boost::math::tools::detail::second_order_root_finder<boost::math::tools::detail::halley_step, boost::math::detail::ibeta_roots<long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>>, long double>(boost::math::detail::ibeta_roots<long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>>, long double, long double, long double, int, unsigned long&) boost/include/boost/math/tools/roots.hpp:530
    #7 0x33e4d4e in long double boost::math::tools::halley_iterate<boost::math::detail::ibeta_roots<long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>>, long double>(boost::math::detail::ibeta_roots<long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>>, long double, long double, long double, int, unsigned long&) boost/include/boost/math/tools/roots.hpp:677
    #8 0x33e4d4e in long double boost::math::detail::ibeta_inv_imp<long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>>(long double, long double, long double, long double, boost::math::policies::policy<boost::math::policies::promote_float<false>, boost::math::policies::promote_double<false>, boost::math::policie
s::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy> const&, long double*) boost/include/boost/math/special_functions/detail/ibeta_inverse.hpp:876
    #9 0x33e3ef5 in boost::math::tools::promote_args<double, double, double, double, float, float>::type boost::math::ibeta_inv<double, double, double, double, boost::math::policies::policy<boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>>(double, double, double, double*, boost::math::policies::policy<boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy> const&) boost/include/boost/math/special_functions/detail/ibeta_inverse.hpp:921
    #10 0x3411b9f in double boost::math::quantile<double, boost::math::policies::policy<boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>>(boost::math::beta_distribution<double, boost::math::policies::policy<boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy, boost::math::policies::default_policy>> const&, double const&) boost/include/boost/math/distributions/beta.hpp:494
    #11 0x3411b9f in facebook::velox::functions::(anonymous namespace)::InverseBetaCDFFunction<facebook::velox::exec::VectorExec>::call(double&, double, double, double) velox/functions/prestosql/Probability.h:185

[kevinwilfong]

We might need some upper/lower bounds on the arguments a, b