diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..6e77b26e --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +*.exe +*.obj +*.ilk +*.pdb +*.gcno +*.gcda +*.html +*.css +*.swp +*.swo +.vscode diff --git a/include/boost/token_functions.hpp b/include/boost/token_functions.hpp index e6b8bef5..93c02b47 100644 --- a/include/boost/token_functions.hpp +++ b/include/boost/token_functions.hpp @@ -31,19 +31,37 @@ #ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_ #define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_ -#include -#include -#include -#include -#include // for find_if #include #include -#include +#include #include +#include #include +#include #include +#include + +#include +#include +#include +#include +#include +#include + +#if !defined(BOOST_NO_CXX11_NOEXCEPT) +# define BOOST_TOKENIZER_NOEXCEPT BOOST_NOEXCEPT +# define BOOST_TOKENIZER_NOEXCEPT_EXPR(expr) BOOST_NOEXCEPT_EXPR(expr) +#else +# define BOOST_TOKENIZER_NOEXCEPT BOOST_NOEXCEPT_OR_NOTHROW +# define BOOST_TOKENIZER_NOEXCEPT_EXPR(expr) BOOST_NOEXCEPT_OR_NOTHROW +#endif + #if !defined(BOOST_NO_CWCTYPE) -#include +# include +#endif + +#if !defined(BOOST_NO_CXX17_HDR_STRING_VIEW) +# include #endif // @@ -88,64 +106,159 @@ namespace boost{ // character (backslash \), can be assigned to other characters. struct escaped_list_error : public std::runtime_error{ - escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { } + explicit escaped_list_error(const char* what_arg) BOOST_TOKENIZER_NOEXCEPT + : std::runtime_error(what_arg) { } + }; + + namespace tokenizer_detail { + //=========================================================================== + // Tokenizer was broken for wide character separators, at least on Windows, since + // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts + // if higher values are passed in. The traits extension class should take care of this. + // Assuming that the conditional will always get optimized out in the function + // implementations, argument types are not a problem since both forms of character classifiers + // expect an int. + +#if !defined(BOOST_NO_CWCTYPE) + template + struct traits_extension_details : public traits { + typedef typename traits::char_type char_type; + static bool isspace(char_type c) BOOST_TOKENIZER_NOEXCEPT + { + return std::iswspace(c) != 0; + } + static bool ispunct(char_type c) BOOST_TOKENIZER_NOEXCEPT + { + return std::iswpunct(c) != 0; + } + }; + + template + struct traits_extension_details : public traits { + typedef typename traits::char_type char_type; + static bool isspace(char_type c) BOOST_TOKENIZER_NOEXCEPT + { + return std::isspace(c) != 0; + } + static bool ispunct(char_type c) BOOST_TOKENIZER_NOEXCEPT + { + return std::ispunct(c) != 0; + } + }; +#endif + + // In case there is no cwctype header, we implement the checks manually. + // We make use of the fact that the tested categories should fit in ASCII. + template + struct traits_extension : public traits { + typedef typename traits::char_type char_type; + static bool isspace(char_type c) BOOST_TOKENIZER_NOEXCEPT + { +#if !defined(BOOST_NO_CWCTYPE) + return traits_extension_details::isspace(c); +#else + return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0; +#endif + } + + static bool ispunct(char_type c) BOOST_TOKENIZER_NOEXCEPT + { +#if !defined(BOOST_NO_CWCTYPE) + return traits_extension_details::ispunct(c); +#else + return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0; +#endif + } }; + template + void assign(std::basic_string& str, Iterator first, Sentinel last) { + str.assign(first, last); + } + + template + void append(std::basic_string& str, CharT ch) { + str.push_back(ch); + } + + template + void clear(std::basic_string& str) BOOST_TOKENIZER_NOEXCEPT { + str.clear(); + } + + template + void assign(boost::basic_string_view& sv, Iterator first, Sentinel last) BOOST_TOKENIZER_NOEXCEPT { + boost::basic_string_view( + boost::addressof(*first), boost::distance(first, last) + ).swap(sv); + } + + template + void clear(boost::basic_string_view& sv) BOOST_TOKENIZER_NOEXCEPT { + sv.clear(); + } + +#if !defined(BOOST_NO_CXX17_HDR_STRING_VIEW) + template + void assign(std::basic_string_view& sv, Iterator first, Sentinel last) BOOST_TOKENIZER_NOEXCEPT { + sv = std::basic_string_view( + std::addressof(*first), std::distance(first, last) + ); + } + + template + void clear(std::basic_string_view& sv) BOOST_TOKENIZER_NOEXCEPT { + sv = std::basic_string_view(); + } +#endif + } // namespace tokenizer_detail -// The out of the box GCC 2.95 on cygwin does not have a char_traits class. -// MSVC does not like the following typename + // The out of the box GCC 2.95 on cygwin does not have a char_traits class. + // MSVC does not like the following typename template ::traits_type > class escaped_list_separator { - private: typedef std::basic_string string_type; - struct char_eq { - Char e_; - char_eq(Char e):e_(e) { } - bool operator()(Char c) { - return Traits::eq(e_,c); - } - }; + string_type escape_; string_type c_; string_type quote_; bool last_; - bool is_escape(Char e) { - char_eq f(e); - return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end(); + bool is_escape(Char e) const BOOST_TOKENIZER_NOEXCEPT { + return escape_.find(e) != string_type::npos; } - bool is_c(Char e) { - char_eq f(e); - return std::find_if(c_.begin(),c_.end(),f)!=c_.end(); + bool is_c(Char e) const BOOST_TOKENIZER_NOEXCEPT { + return c_.find(e) != string_type::npos; } - bool is_quote(Char e) { - char_eq f(e); - return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end(); + bool is_quote(Char e) const BOOST_TOKENIZER_NOEXCEPT { + return quote_.find(e) != string_type::npos; } template void do_escape(iterator& next,iterator end,Token& tok) { + using namespace tokenizer_detail; + if (++next == end) - BOOST_THROW_EXCEPTION(escaped_list_error(std::string("cannot end with escape"))); + BOOST_THROW_EXCEPTION(escaped_list_error("cannot end with escape")); if (Traits::eq(*next,'n')) { - tok+='\n'; + append(tok, '\n'); return; } else if (is_quote(*next)) { - tok+=*next; + append(tok, *next); return; } else if (is_c(*next)) { - tok+=*next; + append(tok, *next); return; } else if (is_escape(*next)) { - tok+=*next; + append(tok, *next); return; } else - BOOST_THROW_EXCEPTION(escaped_list_error(std::string("unknown escape sequence"))); + BOOST_THROW_EXCEPTION(escaped_list_error("unknown escape sequence")); } public: @@ -155,14 +268,16 @@ namespace boost{ : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { } escaped_list_separator(string_type e, string_type c, string_type q) - : escape_(e), c_(c), quote_(q), last_(false) { } + : escape_(boost::move(e)), c_(boost::move(c)), quote_(boost::move(q)), last_(false) { } - void reset() {last_=false;} + void reset() BOOST_TOKENIZER_NOEXCEPT { last_ = false; } template bool operator()(InputIterator& next,InputIterator end,Token& tok) { + using namespace tokenizer_detail; + bool bInQuote = false; - tok = Token(); + clear(tok); if (next == end) { if (last_) { @@ -175,7 +290,7 @@ namespace boost{ last_ = false; for (;next != end;++next) { if (is_escape(*next)) { - do_escape(next,end,tok); + do_escape(next, end, tok); } else if (is_c(*next)) { if (!bInQuote) { @@ -186,156 +301,21 @@ namespace boost{ last_ = true; return true; } - else tok+=*next; + else { + append(tok, *next); + } } else if (is_quote(*next)) { bInQuote=!bInQuote; } else { - tok += *next; + append(tok, *next); } } return true; } }; - //=========================================================================== - // The classes here are used by offset_separator and char_separator to implement - // faster assigning of tokens using assign instead of += - - namespace tokenizer_detail { - //=========================================================================== - // Tokenizer was broken for wide character separators, at least on Windows, since - // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts - // if higher values are passed in. The traits extension class should take care of this. - // Assuming that the conditional will always get optimized out in the function - // implementations, argument types are not a problem since both forms of character classifiers - // expect an int. - -#if !defined(BOOST_NO_CWCTYPE) - template - struct traits_extension_details : public traits { - typedef typename traits::char_type char_type; - static bool isspace(char_type c) - { - return std::iswspace(c) != 0; - } - static bool ispunct(char_type c) - { - return std::iswpunct(c) != 0; - } - }; - - template - struct traits_extension_details : public traits { - typedef typename traits::char_type char_type; - static bool isspace(char_type c) - { - return std::isspace(c) != 0; - } - static bool ispunct(char_type c) - { - return std::ispunct(c) != 0; - } - }; -#endif - - - // In case there is no cwctype header, we implement the checks manually. - // We make use of the fact that the tested categories should fit in ASCII. - template - struct traits_extension : public traits { - typedef typename traits::char_type char_type; - static bool isspace(char_type c) - { -#if !defined(BOOST_NO_CWCTYPE) - return traits_extension_details::isspace(c); -#else - return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0; -#endif - } - - static bool ispunct(char_type c) - { -#if !defined(BOOST_NO_CWCTYPE) - return traits_extension_details::ispunct(c); -#else - return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0; -#endif - } - }; - - // The assign_or_plus_equal struct contains functions that implement - // assign, +=, and clearing based on the iterator type. The - // generic case does nothing for plus_equal and clearing, while - // passing through the call for assign. - // - // When an input iterator is being used, the situation is reversed. - // The assign method does nothing, plus_equal invokes operator +=, - // and the clearing method sets the supplied token to the default - // token constructor's result. - // - - template - struct assign_or_plus_equal { - template - static void assign(Iterator b, Iterator e, Token &t) { - t.assign(b, e); - } - - template - static void plus_equal(Token &, const Value &) { } - - // If we are doing an assign, there is no need for the - // the clear. - // - template - static void clear(Token &) { } - }; - - template <> - struct assign_or_plus_equal { - template - static void assign(Iterator , Iterator , Token &) { } - template - static void plus_equal(Token &t, const Value &v) { - t += v; - } - template - static void clear(Token &t) { - t = Token(); - } - }; - - - template - struct pointer_iterator_category{ - typedef std::random_access_iterator_tag type; - }; - - - template - struct class_iterator_category{ - typedef typename Iterator::iterator_category type; - }; - - - - // This portably gets the iterator_tag without partial template specialization - template - struct get_iterator_category{ - typedef typename mpl::if_, - pointer_iterator_category, - class_iterator_category - >::type cat; - - typedef typename cat::type iterator_category; - }; - - - } // namespace tokenizer_detail - - //=========================================================================== // The offset_separator class, which is a model of TokenizerFunction. // Offset breaks a string into tokens based on a range of offsets @@ -357,25 +337,21 @@ namespace boost{ return_partial_last_(return_partial_last) { } offset_separator() - : offsets_(1,1), current_offset_(), + : offsets_(1,1), current_offset_(0), wrap_offsets_(true), return_partial_last_(true) { } - void reset() { + void reset() BOOST_TOKENIZER_NOEXCEPT { current_offset_ = 0; } template bool operator()(InputIterator& next, InputIterator end, Token& tok) { - typedef tokenizer_detail::assign_or_plus_equal< - BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category< - InputIterator - >::iterator_category - > assigner; + using namespace tokenizer_detail; BOOST_ASSERT(!offsets_.empty()); - assigner::clear(tok); + clear(tok); InputIterator start(next); if (next == end) @@ -393,9 +369,9 @@ namespace boost{ int i = 0; for (; i < c; ++i) { if (next == end)break; - assigner::plus_equal(tok,*next++); + ++next; } - assigner::assign(start,next,tok); + assign(tok, start, next); if (!return_partial_last_) if (i < (c-1) ) @@ -435,16 +411,13 @@ namespace boost{ char_separator(const Char* dropped_delims, const Char* kept_delims = 0, empty_token_policy empty_tokens = drop_empty_tokens) - : m_dropped_delims(dropped_delims), + : m_kept_delims(kept_delims, (kept_delims ? Traits::length(kept_delims) : 0)), + m_dropped_delims(dropped_delims), m_use_ispunct(false), m_use_isspace(false), m_empty_tokens(empty_tokens), m_output_done(false) - { - // Borland workaround - if (kept_delims) - m_kept_delims = kept_delims; - } + { } // use ispunct() for kept delimiters and isspace for dropped. explicit @@ -454,18 +427,14 @@ namespace boost{ m_empty_tokens(drop_empty_tokens), m_output_done(false) { } - void reset() { } + void reset() BOOST_TOKENIZER_NOEXCEPT { } template bool operator()(InputIterator& next, InputIterator end, Token& tok) { - typedef tokenizer_detail::assign_or_plus_equal< - BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category< - InputIterator - >::iterator_category - > assigner; + using namespace tokenizer_detail; - assigner::clear(tok); + clear(tok); // skip past all dropped_delims if (m_empty_tokens == drop_empty_tokens) @@ -475,40 +444,37 @@ namespace boost{ InputIterator start(next); if (m_empty_tokens == drop_empty_tokens) { - - if (next == end) + if (next == end) { return false; - + } // if we are on a kept_delims move past it and stop if (is_kept(*next)) { - assigner::plus_equal(tok,*next); ++next; - } else + } else { // append all the non delim characters - for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next) - assigner::plus_equal(tok,*next); + while (next != end && !is_dropped(*next) && !is_kept(*next)) { + ++next; + } + } } else { // m_empty_tokens == keep_empty_tokens - // Handle empty token at the end - if (next == end) - { - if (m_output_done == false) - { + if (next == end) { + if (m_output_done == false) { m_output_done = true; - assigner::assign(start,next,tok); + assign(tok, start, next); return true; } - else + else { return false; + } } if (is_kept(*next)) { - if (m_output_done == false) + if (m_output_done == false) { m_output_done = true; - else { - assigner::plus_equal(tok,*next); + } else { ++next; m_output_done = false; } @@ -517,14 +483,16 @@ namespace boost{ m_output_done = true; } else { - if (is_dropped(*next)) - start=++next; - for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next) - assigner::plus_equal(tok,*next); + if (is_dropped(*next)) { + start = ++next; + } + while (next != end && !is_dropped(*next) && !is_kept(*next)) { + ++next; + } m_output_done = true; } } - assigner::assign(start,next,tok); + assign(tok, start, next); return true; } @@ -536,7 +504,7 @@ namespace boost{ empty_token_policy m_empty_tokens; bool m_output_done; - bool is_kept(Char E) const + bool is_kept(Char E) const BOOST_TOKENIZER_NOEXCEPT { if (m_kept_delims.length()) return m_kept_delims.find(E) != string_type::npos; @@ -545,7 +513,7 @@ namespace boost{ } else return false; } - bool is_dropped(Char E) const + bool is_dropped(Char E) const BOOST_TOKENIZER_NOEXCEPT { if (m_dropped_delims.length()) return m_dropped_delims.find(E) != string_type::npos; @@ -580,7 +548,7 @@ namespace boost{ bool no_ispunct_; bool no_isspace_; - bool is_ret(Char E)const + bool is_ret(Char E) const BOOST_TOKENIZER_NOEXCEPT { if (returnable_.length()) return returnable_.find(E) != string_type::npos; @@ -592,7 +560,7 @@ namespace boost{ } } } - bool is_nonret(Char E)const + bool is_nonret(Char E) const BOOST_TOKENIZER_NOEXCEPT { if (nonreturnable_.length()) return nonreturnable_.find(E) != string_type::npos; @@ -609,45 +577,47 @@ namespace boost{ explicit char_delimiters_separator(bool return_delims = false, const Char* returnable = 0, const Char* nonreturnable = 0) - : returnable_(returnable ? returnable : string_type().c_str()), - nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()), + : returnable_(returnable, (returnable ? Traits::length(returnable) : 0)), + nonreturnable_(nonreturnable, (returnable ? Traits::length(nonreturnable) : 0)), return_delims_(return_delims), no_ispunct_(returnable!=0), no_isspace_(nonreturnable!=0) { } - void reset() { } + void reset() BOOST_TOKENIZER_NOEXCEPT { } public: - template - bool operator()(InputIterator& next, InputIterator end,Token& tok) { - tok = Token(); - - // skip past all nonreturnable delims - // skip past the returnable only if we are not returning delims - for (;next!=end && ( is_nonret(*next) || (is_ret(*next) - && !return_delims_ ) );++next) { } - - if (next == end) { - return false; - } - - // if we are to return delims and we are one a returnable one - // move past it and stop - if (is_ret(*next) && return_delims_) { - tok+=*next; - ++next; - } - else - // append all the non delim characters - for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next) - tok+=*next; - - - return true; - } - }; + template + bool operator()(InputIterator& next, InputIterator end,Token& tok) { + using namespace tokenizer_detail; + + clear(tok); + + // skip past all nonreturnable delims + // skip past the returnable only if we are not returning delims + for (;next!=end && ( is_nonret(*next) || (is_ret(*next) + && !return_delims_ ) );++next) { } + if (next == end) { + return false; + } + + const InputIterator start(next); + // if we are to return delims and we are one a returnable one + // move past it and stop + if (is_ret(*next)) { + ++next; + } + else { + // append all the non delim characters + while (next != end && !is_nonret(*next) && !is_ret(*next)) { + ++next; + } + } + assign(tok, start, next); + return true; + } + }; } //namespace boost -#endif +#endif \ No newline at end of file diff --git a/include/boost/token_iterator.hpp b/include/boost/token_iterator.hpp index 42945d7e..93c83ad1 100644 --- a/include/boost/token_iterator.hpp +++ b/include/boost/token_iterator.hpp @@ -22,8 +22,13 @@ #include #include #include + +#if !defined(BOOST_NO_CXX11_HDR_TYPE_TRAITS) +# include +#endif #include + namespace boost { template @@ -68,7 +73,6 @@ namespace boost } void initialize(){ - if(valid_) return; f_.reset(); valid_ = (begin_ != end_)? f_(begin_,end_,tok_):false; @@ -95,13 +99,9 @@ namespace boost TokenizerFunc tokenizer_function()const{return f_;} - Type current_token()const{return tok_;} - - bool at_end()const{return !valid_;} - - - + Type current_token() const BOOST_TOKENIZER_NOEXCEPT_EXPR(std::is_copy_constructible::value) { return tok_; } + bool at_end() const BOOST_TOKENIZER_NOEXCEPT { return !valid_; } }; template < class TokenizerFunc = char_delimiters_separator, diff --git a/test/examples.cpp b/test/examples.cpp index 1e0b69b8..8081f949 100644 --- a/test/examples.cpp +++ b/test/examples.cpp @@ -8,15 +8,45 @@ // See http://www.boost.org for updates, documentation, and revision history. +#include #include #include +#include #include -#include -#include + +#if defined(BOOST_ENABLE_ASSERT_HANDLER) +# error "already defined BOOST_ENABLE_ASSERT_HANDLER somewhere" +#else +# define BOOST_ENABLE_ASSERT_HANDLER +#endif + #include +#include +#include +#include +#include +#include #include +#define PP_REQUIRE_EXCEPTION(statement, exception_type, error_message) \ + do { \ + bool caught_exception__ = false; \ + try { \ + { statement } \ + } catch (exception_type& e) { \ + BOOST_REQUIRE(std::strcmp(e.what(), error_message) == 0); \ + caught_exception__ = true; \ + } \ + BOOST_REQUIRE(caught_exception__); \ + } while (false) + +namespace boost { + void assertion_failed(char const* expr, char const *function, char const* file, long line) { + BOOST_THROW_EXCEPTION(std::runtime_error("assertion failed")); + } +} // namespace boost + int test_main( int /*argc*/, char* /*argv*/[] ) { using namespace boost; @@ -39,13 +69,6 @@ int test_main( int /*argc*/, char* /*argv*/[] ) Tok t(test_string, sep); BOOST_REQUIRE(std::equal(t.begin(), t.end(), answer)); } - { - const std::string test_string = "This,,is, a.test.."; - std::string answer[] = {"This","is","a","test"}; - typedef tokenizer<> Tok; - Tok t(test_string); - BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer)); - } { const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\"; @@ -137,12 +160,420 @@ int test_main( int /*argc*/, char* /*argv*/[] ) BOOST_REQUIRE(*other=="a"); } + // Test token_iterator + { + // compare equality of two iterators: valid & valid (same target string) + { + const std::string test_string = "abc"; + typedef char_separator separator_type; + typedef token_iterator_generator::type token_iterator_type; + const std::string::const_iterator first = boost::begin(test_string); + const std::string::const_iterator last = boost::end(test_string); + const separator_type separator; + const token_iterator_type a = make_token_iterator(first, last, separator); + const token_iterator_type b = make_token_iterator(first, last, separator); + BOOST_REQUIRE(a == b); + } + + // compare equality of two iterators: valid & valid (same but partial target string) + { + const std::string test_string = "abc"; + typedef char_separator separator_type; + typedef token_iterator_generator::type token_iterator_type; + const std::string::const_iterator first = boost::begin(test_string); + std::string::const_iterator last = boost::end(test_string); + const token_iterator_type a = make_token_iterator(first, last, separator_type()); + const token_iterator_type b = make_token_iterator(first, --last, separator_type()); + BOOST_REQUIRE(a != b); + } + + // compare equality of two iterators: valid & valid (same but partial target string) + { + const std::string test_string = "abc,def"; + typedef char_delimiters_separator separator_type; + typedef token_iterator_generator::type token_iterator_type; + const std::string::const_iterator first = boost::begin(test_string); + std::string::const_iterator last = boost::end(test_string); + const token_iterator_type a = make_token_iterator(first, last, separator_type()); + const token_iterator_type b = make_token_iterator(first, --last, separator_type()); + BOOST_REQUIRE(a != b); + } + + // compare equality of two iterators: invalid & valid + { + const std::string empty_string = ""; + const std::string non_empty_string = "abc"; + typedef char_delimiters_separator separator_type; + typedef token_iterator_generator::type token_iterator_type; + const token_iterator_type a = make_token_iterator(boost::begin(empty_string), boost::end(empty_string), separator_type()); + const token_iterator_type b = make_token_iterator(boost::begin(non_empty_string), boost::end(non_empty_string), separator_type()); + BOOST_REQUIRE(a != b); + } + } + + // Test escaped_list_separator + { + // input contains "\n" + { + const std::string test_string = "\\n"; + const std::string answer[] = {"\n"}; + tokenizer > tokenizer(test_string); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // input ends with escape + { + const std::string test_string = "\\"; + tokenizer > t(test_string); + + PP_REQUIRE_EXCEPTION( + { std::distance(boost::begin(t), boost::end(t)); }, + boost::escaped_list_error, + "cannot end with escape" + ); + } + + // input contains unknown escape sequence + { + const std::string test_string = "\\q"; + tokenizer > t(test_string); + + PP_REQUIRE_EXCEPTION( + { std::distance(boost::begin(t), boost::end(t)); }, + boost::escaped_list_error, + "unknown escape sequence" + ); + } + } + + // Test default constructed offset_separator + { + // use std::string content + { + typedef std::string string_type; + const string_type test_string = "1234567"; + const string_type answer[] = {"1", "2", "3", "4", "5", "6", "7"}; + tokenizer tokenizer(test_string); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // use boost::string_view content + { + typedef boost::string_view string_type; + const string_type test_string = "1234567"; + const string_type answer[] = {"1", "2", "3", "4", "5", "6", "7"}; + tokenizer tokenizer(test_string); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + #if !defined(BOOST_NO_CXX17_HDR_STRING_VIEW) + // use std::string_view content + { + typedef std::string_view string_type; + const string_type test_string = "1234567"; + const string_type answer[] = {"1", "2", "3", "4", "5", "6", "7"}; + tokenizer tokenizer(test_string); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + #endif + } + + // Test non-default constructed offset_separator + { + const std::string test_string = "1234567"; + + // empty offsets, wrap_offsets = false, return_partial_last = false + { + const int offset = 0; + typedef offset_separator separator_type; + offset_separator separator(&offset, &offset, false, false); + typedef token_iterator_generator::type token_iterator_type; + + PP_REQUIRE_EXCEPTION( + { make_token_iterator(boost::begin(test_string), boost::end(test_string), separator); }, + std::runtime_error, + "assertion failed" + ); + } + + // wrap_offsets = false, return_partial_last = false + { + const std::string answer[] = {"1", "234"}; + + boost::array offsets = {{1, 3, 5}}; + typedef offset_separator separator_type; + offset_separator separator(offsets.begin(), offsets.end(), false, false); + tokenizer tokenizer(test_string, separator); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // wrap_offsets = false, return_partial_last = true + { + { + const std::string answer[] = {"1", "234", "567"}; + + boost::array offsets = {{1, 3, 5}}; + typedef offset_separator separator_type; + offset_separator separator(offsets.begin(), offsets.end(), false, true); + tokenizer tokenizer(test_string, separator); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + { + const std::string answer[] = {"12345"}; + + boost::array offsets = {{5}}; + typedef offset_separator separator_type; + offset_separator separator(offsets.begin(), offsets.end(), false, true); + tokenizer tokenizer(test_string, separator); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + } + + // wrap_offsets = true, return_partial_last = false + { + const std::string answer[] = {"1", "234"}; + + boost::array offsets = {{1, 3, 5}}; + typedef offset_separator separator_type; + offset_separator separator(offsets.begin(), offsets.end(), true, false); + tokenizer tokenizer(test_string, separator); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + } + + // Test default constructed char_separator + { + // use std::string content + { + typedef std::string string_type; + typedef tokenizer< + char_separator, + string_type::const_iterator, + string_type + > tokenizer_type; + const string_type test_string = ";Hello|world-"; + const string_type answer[] = {";", "Hello", "|", "world", "-"}; + tokenizer_type tokenizer(test_string); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // use boost::string_view content + { + typedef boost::string_view string_type; + typedef tokenizer< + char_separator, + string_type::const_iterator, + string_type + > tokenizer_type; + const string_type test_string = ";Hello|world-"; + const string_type answer[] = {";", "Hello", "|", "world", "-"}; + tokenizer_type tokenizer(test_string); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + #if !defined(BOOST_NO_CXX17_HDR_STRING_VIEW) + // use std::string_view content + { + typedef std::string_view string_type; + typedef tokenizer< + char_separator, + string_type::const_iterator, + string_type + > tokenizer_type; + const string_type test_string = ";Hello|world-"; + const string_type answer[] = {";", "Hello", "|", "world", "-"}; + tokenizer_type tokenizer(test_string); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + #endif + } + + // Test non-default contstructed char_separator + { + const std::string test_string = ";Hello||world-"; + + // dropped_delims = non-null, kept_delims = null, empty_tokens = drop_empty_tokens + { + const std::string answer[] = {"Hello||world"}; + typedef char_separator separator_type; + separator_type separator("-;", 0, boost::drop_empty_tokens); + tokenizer tokenizer(test_string, separator); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // dropped_delims = non-null, kept_delims = null, empty_tokens = keep_empty_tokens + { + const std::string answer[] = {"", "Hello", "", "world", ""}; + typedef char_separator separator_type; + separator_type separator("-;|", 0, boost::keep_empty_tokens); + tokenizer tokenizer(test_string, separator); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // dropped_delims = non-null, kept_delims = non-null, empty_tokens = drop_empty_tokens + { + const std::string answer[] = {"Hello", "|", "|", "world"}; + typedef char_separator separator_type; + separator_type separator("-;", "|", boost::drop_empty_tokens); + tokenizer tokenizer(test_string, separator); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // dropped_delims = non-null, kept_delims = non-null, empty_tokens = keep_empty_tokens + { + const std::string answer[] = {"", "Hello", "|", "", "|", "world", ""}; + typedef char_separator separator_type; + separator_type separator("-;", "|", boost::keep_empty_tokens); + tokenizer tokenizer(test_string, separator); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // dropped_delims = non-null (empty), kept_delims = null, empty_tokens = drop_empty_tokens + { + const std::string answer[] = {";Hello||world-"}; + typedef char_separator separator_type; + separator_type separator("", 0, boost::keep_empty_tokens); + tokenizer tokenizer(test_string, separator); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + } + + // Test default constructed char_delimiters_separator + { + // use std::string content + { + typedef std::string string_type; + typedef tokenizer< + char_delimiters_separator, + string_type::const_iterator, + string_type + > tokenizer_type; + const string_type test_string = "This,,is, a.test.."; + const string_type answer[] = {"This","is","a","test"}; + tokenizer_type tokenizer(test_string); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // use boost::string_view content + { + typedef boost::string_view string_type; + typedef tokenizer< + char_delimiters_separator, + string_type::const_iterator, + string_type + > tokenizer_type; + const string_type test_string = "This,,is, a.test.."; + const string_type answer[] = {"This","is","a","test"}; + tokenizer_type tokenizer(test_string); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + #if !defined(BOOST_NO_CXX17_HDR_STRING_VIEW) + // use std::string_view content + { + typedef std::string_view string_type; + typedef tokenizer< + char_delimiters_separator, + string_type::const_iterator, + string_type + > tokenizer_type; + const string_type test_string = "This,,is, a.test.."; + const string_type answer[] = {"This","is","a","test"}; + tokenizer_type tokenizer(test_string); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + #endif + } + // Test non-default constructed char_delimiters_separator { - const std::string test_string = "how,are you, doing"; - std::string answer[] = {"how",",","are you",","," doing"}; - tokenizer<> t(test_string,char_delimiters_separator(true,",","")); - BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer)); + const std::string test_string = "how,are you, doing?"; + + // return_delims = true, returnable = non-null, nonreturnable = non-null + { + const std::string answer[] = {"how",",","are you",","," doing"}; + tokenizer<> tokenizer(test_string,char_delimiters_separator(true,",","?")); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // return_delims = true, returnable = non-null, nonreturnable = non-null (empty) + { + const std::string answer[] = {"how",",","are you",","," doing?"}; + tokenizer<> tokenizer(test_string,char_delimiters_separator(true,",","")); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // return_delims = true, returnable = non-null (empty), nonreturnable = non-null + { + const std::string answer[] = {"how,are you, doing"}; + tokenizer<> tokenizer(test_string,char_delimiters_separator(true,"","?")); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // return_delims = true, returnable = non-null (empty), nonreturnable = non-null (empty) + { + const std::string answer[] = {"how,are you, doing?"}; + tokenizer<> tokenizer(test_string,char_delimiters_separator(true,"","")); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // return_delims = false, returnable = non-null, nonreturnable = non-null + { + const std::string answer[] = {"how","are you"," doing"}; + tokenizer<> tokenizer(test_string,char_delimiters_separator(false,",","?")); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // return_delims = false, returnable = non-null, nonreturnable = non-null (empty) + { + const std::string answer[] = {"how","are you"," doing?"}; + tokenizer<> tokenizer(test_string,char_delimiters_separator(false,",","")); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // return_delims = false, returnable = non-null (empty), nonreturnable = non-null + { + const std::string answer[] = {"how,are you, doing"}; + tokenizer<> tokenizer(test_string,char_delimiters_separator(false,"","?")); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + + // return_delims = false, returnable = non-null (empty), nonreturnable = non-null (emppty) + { + const std::string answer[] = {"how,are you, doing?"}; + tokenizer<> tokenizer(test_string,char_delimiters_separator(false,"","")); + BOOST_REQUIRE(boost::range::equal(tokenizer, answer)); + } + } + + // Test iterator operations + { + const std::string test_string; + + { + // increment invalid iterator + typedef tokenizer<> tokenizer_type; + tokenizer_type tokenizer(test_string); + tokenizer_type::iterator first = boost::begin(tokenizer); + PP_REQUIRE_EXCEPTION( + { std::advance(first, 1); }, + std::runtime_error, + "assertion failed" + ); + } + + { + // dereference invalid iterator + typedef tokenizer<> tokenizer_type; + tokenizer_type tokenizer(test_string); + const tokenizer_type::iterator first = boost::begin(tokenizer); + PP_REQUIRE_EXCEPTION( + { *first; }, + std::runtime_error, + "assertion failed" + ); + } } return 0;