diff --git a/ops/clean.sh b/ops/clean.sh index eb57906..d3f769c 100644 --- a/ops/clean.sh +++ b/ops/clean.sh @@ -7,5 +7,6 @@ python setup.py clean --all cd .. cd pybind11 +python setup.py clean --all rm -f *.so diff --git a/ops/make.sh b/ops/make.sh index d0d052a..aa0029f 100644 --- a/ops/make.sh +++ b/ops/make.sh @@ -6,5 +6,7 @@ python setup.py build_ext --inplace cd .. cd pybind11 -include=`python -m pybind11 --includes` -g++ -std=c++11 -shared -o box_ops_cc.so box_ops.cc -fPIC -O3 ${include} \ No newline at end of file +python setup.py build_ext --inplace + +# include=`python -m pybind11 --includes` +# g++ -std=c++11 -shared -o box_ops_cc.so box_ops.cc -fPIC -O3 ${include} \ No newline at end of file diff --git a/ops/pybind11/compile.sh b/ops/pybind11/compile.sh index 93447cc..48fad24 100644 --- a/ops/pybind11/compile.sh +++ b/ops/pybind11/compile.sh @@ -5,4 +5,4 @@ set -e include=`python -m pybind11 --includes` g++ -std=c++11 -shared -o box_ops_cc.so box_ops.cc -fPIC -O3 ${include} - +g++ -std=c++11 -shared -o nms.so nms.cc -fPIC -O3 ${include} diff --git a/ops/pybind11/nms.cc b/ops/pybind11/nms.cc new file mode 100644 index 0000000..a671d89 --- /dev/null +++ b/ops/pybind11/nms.cc @@ -0,0 +1,20 @@ +#include "nms_cpu.h" +PYBIND11_MODULE(nms, m) +{ + m.doc() = "non_max_suppression asd"; + + m.def("non_max_suppression_cpu", &non_max_suppression_cpu, py::return_value_policy::reference_internal, "bbox iou", + "boxes"_a = 1, "order"_a = 2, "nms_overlap_thresh"_a = 3, "eps"_a = 4); + m.def("non_max_suppression_cpu", &non_max_suppression_cpu, py::return_value_policy::reference_internal, "bbox iou", + "boxes"_a = 1, "order"_a = 2, "nms_overlap_thresh"_a = 3, "eps"_a = 4); + + m.def("rotate_non_max_suppression_cpu", &rotate_non_max_suppression_cpu, py::return_value_policy::reference_internal, "bbox iou", + "box_corners"_a = 1, "order"_a = 2, "standup_iou"_a = 3, "thresh"_a = 4); + m.def("rotate_non_max_suppression_cpu", &rotate_non_max_suppression_cpu, py::return_value_policy::reference_internal, "bbox iou", + "box_corners"_a = 1, "order"_a = 2, "standup_iou"_a = 3, "thresh"_a = 4); + + m.def("rotate_non_max_suppression_3d_cpu", &rotate_non_max_suppression_3d_cpu, py::return_value_policy::reference_internal, "bbox iou", + "box_corners"_a = 1, "order"_a = 2, "standup_iou"_a = 3, "thresh"_a = 4); + m.def("rotate_non_max_suppression_3d_cpu", &rotate_non_max_suppression_3d_cpu, py::return_value_policy::reference_internal, "bbox iou", + "box_corners"_a = 1, "order"_a = 2, "standup_iou"_a = 3, "thresh"_a = 4); +} \ No newline at end of file diff --git a/ops/pybind11/nms_cpu.h b/ops/pybind11/nms_cpu.h new file mode 100644 index 0000000..ecd5dbc --- /dev/null +++ b/ops/pybind11/nms_cpu.h @@ -0,0 +1,243 @@ +#ifndef NMS_CPU_H +#define NMS_CPU_H +#include +// must include pybind11/stl.h if using containers in STL in arguments. +#include +#include +#include +#include +#include + +namespace py = pybind11; +using namespace pybind11::literals; + + +template +inline py::array_t constant(ShapeContainer shape, DType value){ + // create ROWMAJOR array. + py::array_t array(shape); + std::fill(array.mutable_data(), array.mutable_data() + array.size(), value); + return array; +} + +template +inline py::array_t zeros(std::vector shape){ + return constant>(shape, 0); +} + +template +std::vector non_max_suppression_cpu( + py::array_t boxes, + py::array_t order, + DType thresh, + DType eps=0) +{ + auto ndets = boxes.shape(0); + auto boxes_r = boxes.template unchecked<2>(); + auto order_r = order.template unchecked<1>(); + auto suppressed = zeros({ndets}); + auto suppressed_rw = suppressed.template mutable_unchecked<1>(); + auto area = zeros({ndets}); + auto area_rw = area.template mutable_unchecked<1>(); + // get areas + for(int i = 0; i < ndets; ++i){ + area_rw(i) = (boxes_r(i, 2) - boxes_r(i, 0) + eps) * (boxes_r(i, 3) - boxes_r(i, 1) + eps); + } + std::vector keep; + int i, j; + DType xx1, xx2, w, h, inter, ovr; + for(int _i = 0; _i < ndets; ++_i){ + i = order_r(_i); + if(suppressed_rw(i) == 1) + continue; + keep.push_back(i); + for(int _j = _i + 1; _j < ndets; ++_j){ + j = order_r(_j); + if(suppressed_rw(j) == 1) + continue; + xx2 = std::min(boxes_r(i, 2), boxes_r(j, 2)); + xx1 = std::max(boxes_r(i, 0), boxes_r(j, 0)); + w = xx2 - xx1 + eps; + if (w > 0){ + xx2 = std::min(boxes_r(i, 3), boxes_r(j, 3)); + xx1 = std::max(boxes_r(i, 1), boxes_r(j, 1)); + h = xx2 - xx1 + eps; + if (h > 0){ + inter = w * h; + ovr = inter / (area_rw(i) + area_rw(j) - inter); + if(ovr >= thresh) + suppressed_rw(j) = 1; + } + } + } + } + return keep; +} + +template +std::vector rotate_non_max_suppression_cpu( + py::array_t box_corners, + py::array_t order, + py::array_t standup_iou, + DType thresh) +{ + auto ndets = box_corners.shape(0); + auto box_corners_r = box_corners.template unchecked<3>(); + auto order_r = order.template unchecked<1>(); + auto suppressed = zeros({ndets}); + auto suppressed_rw = suppressed.template mutable_unchecked<1>(); + auto standup_iou_r = standup_iou.template unchecked<2>(); + std::vector keep; + int i, j; + + namespace bg = boost::geometry; + typedef bg::model::point point_t; + typedef bg::model::polygon polygon_t; + polygon_t poly, qpoly; + std::vector poly_inter, poly_union; + DType inter_area, union_area, overlap; + + for(int _i = 0; _i < ndets; ++_i){ + i = order_r(_i); + if(suppressed_rw(i) == 1) + continue; + keep.push_back(i); + for(int _j = _i + 1; _j < ndets; ++_j){ + j = order_r(_j); + if(suppressed_rw(j) == 1) + continue; + if (standup_iou_r(i, j) <= 0.0) + continue; + + bg::append(poly, point_t(box_corners_r(i, 0, 0), box_corners_r(i, 0, 1))); + bg::append(poly, point_t(box_corners_r(i, 1, 0), box_corners_r(i, 1, 1))); + bg::append(poly, point_t(box_corners_r(i, 2, 0), box_corners_r(i, 2, 1))); + bg::append(poly, point_t(box_corners_r(i, 3, 0), box_corners_r(i, 3, 1))); + bg::append(poly, point_t(box_corners_r(i, 0, 0), box_corners_r(i, 0, 1))); + bg::append(qpoly, point_t(box_corners_r(j, 0, 0), box_corners_r(j, 0, 1))); + bg::append(qpoly, point_t(box_corners_r(j, 1, 0), box_corners_r(j, 1, 1))); + bg::append(qpoly, point_t(box_corners_r(j, 2, 0), box_corners_r(j, 2, 1))); + bg::append(qpoly, point_t(box_corners_r(j, 3, 0), box_corners_r(j, 3, 1))); + bg::append(qpoly, point_t(box_corners_r(j, 0, 0), box_corners_r(j, 0, 1))); + bg::intersection(poly, qpoly, poly_inter); + + if (!poly_inter.empty()) + { + inter_area = bg::area(poly_inter.front()); + bg::union_(poly, qpoly, poly_union); + + if (!poly_union.empty()){ // ignore invalid box + union_area = bg::area(poly_union.front()); + // std::cout << "post union area" << std::endl; + // std::cout << union_area << "debug" << std::endl; + overlap = inter_area / union_area; + if(overlap >= thresh) + suppressed_rw(j) = 1; + poly_union.clear(); + } + } + poly.clear(); + qpoly.clear(); + poly_inter.clear(); + + } + } + return keep; +} + +template +std::vector rotate_non_max_suppression_3d_cpu( + py::array_t box_corners, + py::array_t order, + py::array_t standup_iou, + DType thresh) + /* + camera coordinate + 7 -------- 4 + /| /| + 6 -------- 5 . + | | | | + . 3 -------- 0 + |/ |/ + 2 -------- 1 + */ +{ + auto ndets = box_corners.shape(0); + auto box_corners_r = box_corners.template unchecked<3>(); + auto order_r = order.template unchecked<1>(); + auto suppressed = zeros({ndets}); + auto suppressed_rw = suppressed.template mutable_unchecked<1>(); + auto standup_iou_r = standup_iou.template unchecked<2>(); + std::vector keep; + int i, j; + + namespace bg = boost::geometry; + typedef bg::model::point point_t; + typedef bg::model::polygon polygon_t; + polygon_t poly, qpoly; + std::vector poly_inter, poly_union; + DType inter_area, inter_vol, overlap; + DType ymin, ymax; + DType area, qarea, h, qh; + DType vol, qvol; + DType zero = 0.0; + for(int _i = 0; _i < ndets; ++_i){ + i = order_r(_i); + if(suppressed_rw(i) == 1) + continue; + keep.push_back(i); + for(int _j = _i + 1; _j < ndets; ++_j){ + j = order_r(_j); + if(suppressed_rw(j) == 1) + continue; + if (standup_iou_r(i, j) <= 0.0) + continue; + bg::append(poly, point_t(box_corners_r(i, 6, 0), box_corners_r(i, 6, 2))); + bg::append(poly, point_t(box_corners_r(i, 7, 0), box_corners_r(i, 7, 2))); + bg::append(poly, point_t(box_corners_r(i, 4, 0), box_corners_r(i, 4, 2))); + bg::append(poly, point_t(box_corners_r(i, 5, 0), box_corners_r(i, 5, 2))); + bg::append(poly, point_t(box_corners_r(i, 6, 0), box_corners_r(i, 6, 2))); + bg::append(qpoly, point_t(box_corners_r(j, 6, 0), box_corners_r(j, 6, 2))); + bg::append(qpoly, point_t(box_corners_r(j, 7, 0), box_corners_r(j, 7, 2))); + bg::append(qpoly, point_t(box_corners_r(j, 4, 0), box_corners_r(j, 4, 2))); + bg::append(qpoly, point_t(box_corners_r(j, 5, 0), box_corners_r(j, 5, 2))); + bg::append(qpoly, point_t(box_corners_r(j, 6, 0), box_corners_r(j, 6, 2))); + bg::intersection(poly, qpoly, poly_inter); + + if (!poly_inter.empty()) + { + inter_area = bg::area(poly_inter.front()); + bg::union_(poly, qpoly, poly_union); + if (!poly_union.empty()){ + ymax = std::min(box_corners_r(i, 0, 1), box_corners_r(j, 0, 1)); + ymin = std::max(box_corners_r(i, 4, 1), box_corners_r(j, 4, 1)); + + h = box_corners_r(i, 0, 1) - box_corners_r(i, 4, 1); + qh = box_corners_r(j, 0, 1) - box_corners_r(j, 4, 1); + + area = bg::area(poly); + qarea = bg::area(qpoly); + + inter_vol = inter_area * std::max(zero, ymax - ymin);; + + vol = std::max(zero, area * h); + qvol = std::max(zero, qarea * qh); + + overlap = inter_vol / (vol + qvol - inter_vol); + + if(overlap >= thresh) + suppressed_rw(j) = 1; + poly_union.clear(); + } + } + poly.clear(); + qpoly.clear(); + poly_inter.clear(); + + } + } + return keep; +} + + +#endif diff --git a/ops/pybind11/rbbox_iou.py b/ops/pybind11/rbbox_iou.py index 7f44e75..ae59c4d 100644 --- a/ops/pybind11/rbbox_iou.py +++ b/ops/pybind11/rbbox_iou.py @@ -1,10 +1,7 @@ import numpy as np from . import box_ops_cc - - -# import nms - +from . import nms def bbox_overlaps_1d(ex, gt): ''' @@ -275,6 +272,45 @@ def bev_nms_np(dets, nms_thresh, top_k=300): return keep +def rotate_nms_bev_cc(dets, thresh, top_k=300): + ''' + dets: [[cx, cz, l, w, ry, score]] n,6 + ''' + assert dets.shape[1] == 6 + scores = dets[:, 5] + + order = scores.argsort()[::-1].astype(np.int32) # highest->lowest + + boxes_corners = rbbox2corner(dets[:, :5]) + boxes_standup = corner2standup(boxes_corners) + + standup_iou = bbox_overlaps_2d(boxes_standup, boxes_standup) + + keep = nms.rotate_non_max_suppression_cpu(boxes_corners, order, standup_iou, thresh) + + return keep[:top_k] + + +def rotate_nms_3d_cc(dets, thresh, top_k=300): + ''' + dets: [[cx, cy, cz, l, w, h, ry, score]] n,8 + + ''' + assert dets.shape[1] == 8 + scores = dets[:, 7] + + order = scores.argsort()[::-1].astype(np.int32) # highest->lowest + + bbox_corner_3d = boxes3d2corners(dets) # n, 8, 3 + bbox_standup = np.concatenate([np.min(bbox_corner_3d, 1), np.max(bbox_corner_3d, 1)], 1) # n, 6 + + standup_iou = bbox_overlaps_3d(bbox_standup, bbox_standup) + + keep = nms.rotate_non_max_suppression_3d_cpu(bbox_corner_3d, order, standup_iou, thresh) + + return keep[:top_k] + + if __name__ == '__main__': # import matplotlib.pyplot as plt diff --git a/ops/pybind11/setup.py b/ops/pybind11/setup.py new file mode 100644 index 0000000..2c3f56b --- /dev/null +++ b/ops/pybind11/setup.py @@ -0,0 +1,126 @@ +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext +import sys +import setuptools + +__version__ = '0.0.1' + +# modified from https://github.com/pybind/python_example + +class get_pybind_include(object): + """Helper class to determine the pybind11 include path + + The purpose of this class is to postpone importing pybind11 + until it is actually installed, so that the ``get_include()`` + method can be invoked. """ + + def __init__(self, user=False): + self.user = user + + def __str__(self): + import pybind11 + return pybind11.get_include(self.user) + + +ext_modules = [ + Extension( + 'box_ops_cc', + ['box_ops.cc'], + include_dirs=[ + # Path to pybind11 headers + get_pybind_include(), + # get_pybind_include(user=True) + ], + language='c++' + ), + Extension( + 'nms', + ['nms.cc'], + include_dirs=[ + # Path to pybind11 headers + get_pybind_include(), + # get_pybind_include(user=True) + ], + language='c++' + ), +] + + +# As of Python 3.6, CCompiler has a `has_flag` method. +# cf http://bugs.python.org/issue26689 +def has_flag(compiler, flagname): + """Return a boolean indicating whether a flag name is supported on + the specified compiler. + """ + import tempfile + with tempfile.NamedTemporaryFile('w', suffix='.cpp') as f: + f.write('int main (int argc, char **argv) { return 0; }') + try: + compiler.compile([f.name], extra_postargs=[flagname]) + except setuptools.distutils.errors.CompileError: + return False + return True + + +def cpp_flag(compiler): + """Return the -std=c++[11/14/17] compiler flag. + + The newer version is prefered over c++11 (when it is available). + """ + flags = ['-std=c++11', '-std=c++14', '-std=c++17'] + + for flag in flags: + if has_flag(compiler, flag): + return flag + + raise RuntimeError('Unsupported compiler -- at least C++11 support ' + 'is needed!') + + +class BuildExt(build_ext): + """A custom build extension for adding compiler-specific options.""" + c_opts = { + 'msvc': ['/EHsc'], + 'unix': [], + } + l_opts = { + 'msvc': [], + 'unix': [], + } + + if sys.platform == 'darwin': + darwin_opts = ['-stdlib=libc++', '-mmacosx-version-min=10.7'] + c_opts['unix'] += darwin_opts + l_opts['unix'] += darwin_opts + + def build_extensions(self): + compile_so_command = self.compiler.compiler_so + if "-Wstrict-prototypes" in compile_so_command: + compile_so_command.remove("-Wstrict-prototypes") + + ct = self.compiler.compiler_type + opts = self.c_opts.get(ct, []) + link_opts = self.l_opts.get(ct, []) + if ct == 'unix': + opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version()) + opts.append(cpp_flag(self.compiler)) + if has_flag(self.compiler, '-fvisibility=hidden'): + opts.append('-fvisibility=hidden') + elif ct == 'msvc': + opts.append('/DVERSION_INFO=\\"%s\\"' % self.distribution.get_version()) + + for ext in self.extensions: + ext.extra_compile_args = opts + ext.extra_link_args = link_opts + build_ext.build_extensions(self) + + +setup( + name='box_ops_cc', + version=__version__, + ext_modules=ext_modules, + install_requires=['pybind11>=2.4'], + setup_requires=['pybind11>=2.4'], + cmdclass={'build_ext': BuildExt}, + zip_safe=False, +)