[3428] | 1 | // Boost token_functions.hpp ------------------------------------------------// |
---|
| 2 | |
---|
| 3 | // Copyright John R. Bandela 2001. |
---|
| 4 | |
---|
| 5 | // Distributed under the Boost Software License, Version 1.0. (See |
---|
| 6 | // accompanying file LICENSE_1_0.txt or copy at |
---|
| 7 | // http://www.boost.org/LICENSE_1_0.txt) |
---|
| 8 | |
---|
| 9 | // See http://www.boost.org/libs/tokenizer/ for documentation. |
---|
| 10 | |
---|
| 11 | // Revision History: |
---|
| 12 | // 01 Oct 2004 Joaquin M Lopez Munoz |
---|
| 13 | // Workaround for a problem with string::assign in msvc-stlport |
---|
| 14 | // 06 Apr 2004 John Bandela |
---|
| 15 | // Fixed a bug involving using char_delimiter with a true input iterator |
---|
| 16 | // 28 Nov 2003 Robert Zeh and John Bandela |
---|
| 17 | // Converted into "fast" functions that avoid using += when |
---|
| 18 | // the supplied iterator isn't an input_iterator; based on |
---|
| 19 | // some work done at Archelon and a version that was checked into |
---|
| 20 | // the boost CVS for a short period of time. |
---|
| 21 | // 20 Feb 2002 John Maddock |
---|
| 22 | // Removed using namespace std declarations and added |
---|
| 23 | // workaround for BOOST_NO_STDC_NAMESPACE (the library |
---|
| 24 | // can be safely mixed with regex). |
---|
| 25 | // 06 Feb 2002 Jeremy Siek |
---|
| 26 | // Added char_separator. |
---|
| 27 | // 02 Feb 2002 Jeremy Siek |
---|
| 28 | // Removed tabs and a little cleanup. |
---|
| 29 | |
---|
| 30 | |
---|
| 31 | #ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_ |
---|
| 32 | #define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_ |
---|
| 33 | |
---|
| 34 | #include <vector> |
---|
| 35 | #include <stdexcept> |
---|
| 36 | #include <string> |
---|
| 37 | #include <cctype> |
---|
| 38 | #include <algorithm> // for find_if |
---|
| 39 | #include <boost/config.hpp> |
---|
| 40 | #include <boost/assert.hpp> |
---|
| 41 | #include <boost/detail/workaround.hpp> |
---|
| 42 | #include <boost/mpl/if.hpp> |
---|
| 43 | #if !defined(BOOST_NO_CWCTYPE) |
---|
| 44 | #include <cwctype> |
---|
| 45 | #endif |
---|
| 46 | |
---|
| 47 | // |
---|
| 48 | // the following must not be macros if we are to prefix them |
---|
| 49 | // with std:: (they shouldn't be macros anyway...) |
---|
| 50 | // |
---|
| 51 | #ifdef ispunct |
---|
| 52 | # undef ispunct |
---|
| 53 | #endif |
---|
| 54 | #ifdef iswpunct |
---|
| 55 | # undef iswpunct |
---|
| 56 | #endif |
---|
| 57 | #ifdef isspace |
---|
| 58 | # undef isspace |
---|
| 59 | #endif |
---|
| 60 | #ifdef iswspace |
---|
| 61 | # undef iswspace |
---|
| 62 | #endif |
---|
| 63 | // |
---|
| 64 | // fix namespace problems: |
---|
| 65 | // |
---|
| 66 | #ifdef BOOST_NO_STDC_NAMESPACE |
---|
| 67 | namespace std{ |
---|
| 68 | using ::ispunct; |
---|
| 69 | using ::isspace; |
---|
| 70 | #if !defined(BOOST_NO_CWCTYPE) |
---|
| 71 | using ::iswpunct; |
---|
| 72 | using ::iswspace; |
---|
| 73 | #endif |
---|
| 74 | } |
---|
| 75 | #endif |
---|
| 76 | |
---|
| 77 | namespace boost{ |
---|
| 78 | //=========================================================================== |
---|
| 79 | // The escaped_list_separator class. Which is a model of TokenizerFunction |
---|
| 80 | // An escaped list is a super-set of what is commonly known as a comma |
---|
| 81 | // separated value (csv) list.It is separated into fields by a comma or |
---|
| 82 | // other character. If the delimiting character is inside quotes, then it is |
---|
| 83 | // counted as a regular character.To allow for embedded quotes in a field, |
---|
| 84 | // there can be escape sequences using the \ much like C. |
---|
| 85 | // The role of the comma, the quotation mark, and the escape |
---|
| 86 | // character (backslash \), can be assigned to other characters. |
---|
| 87 | |
---|
| 88 | struct escaped_list_error : public std::runtime_error{ |
---|
| 89 | escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { } |
---|
| 90 | }; |
---|
| 91 | |
---|
| 92 | |
---|
| 93 | // The out of the box GCC 2.95 on cygwin does not have a char_traits class. |
---|
| 94 | // MSVC does not like the following typename |
---|
| 95 | template <class Char, |
---|
| 96 | class Traits = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type > |
---|
| 97 | class escaped_list_separator { |
---|
| 98 | |
---|
| 99 | private: |
---|
| 100 | typedef std::basic_string<Char,Traits> string_type; |
---|
| 101 | struct char_eq { |
---|
| 102 | Char e_; |
---|
| 103 | char_eq(Char e):e_(e) { } |
---|
| 104 | bool operator()(Char c) { |
---|
| 105 | return Traits::eq(e_,c); |
---|
| 106 | } |
---|
| 107 | }; |
---|
| 108 | string_type escape_; |
---|
| 109 | string_type c_; |
---|
| 110 | string_type quote_; |
---|
| 111 | bool last_; |
---|
| 112 | |
---|
| 113 | bool is_escape(Char e) { |
---|
| 114 | char_eq f(e); |
---|
| 115 | return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end(); |
---|
| 116 | } |
---|
| 117 | bool is_c(Char e) { |
---|
| 118 | char_eq f(e); |
---|
| 119 | return std::find_if(c_.begin(),c_.end(),f)!=c_.end(); |
---|
| 120 | } |
---|
| 121 | bool is_quote(Char e) { |
---|
| 122 | char_eq f(e); |
---|
| 123 | return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end(); |
---|
| 124 | } |
---|
| 125 | template <typename iterator, typename Token> |
---|
| 126 | void do_escape(iterator& next,iterator end,Token& tok) { |
---|
| 127 | if (++next == end) |
---|
| 128 | throw escaped_list_error(std::string("cannot end with escape")); |
---|
| 129 | if (Traits::eq(*next,'n')) { |
---|
| 130 | tok+='\n'; |
---|
| 131 | return; |
---|
| 132 | } |
---|
| 133 | else if (is_quote(*next)) { |
---|
| 134 | tok+=*next; |
---|
| 135 | return; |
---|
| 136 | } |
---|
| 137 | else if (is_c(*next)) { |
---|
| 138 | tok+=*next; |
---|
| 139 | return; |
---|
| 140 | } |
---|
| 141 | else if (is_escape(*next)) { |
---|
| 142 | tok+=*next; |
---|
| 143 | return; |
---|
| 144 | } |
---|
| 145 | else |
---|
| 146 | throw escaped_list_error(std::string("unknown escape sequence")); |
---|
| 147 | } |
---|
| 148 | |
---|
| 149 | public: |
---|
| 150 | |
---|
| 151 | explicit escaped_list_separator(Char e = '\\', |
---|
| 152 | Char c = ',',Char q = '\"') |
---|
| 153 | : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { } |
---|
| 154 | |
---|
| 155 | escaped_list_separator(string_type e, string_type c, string_type q) |
---|
| 156 | : escape_(e), c_(c), quote_(q), last_(false) { } |
---|
| 157 | |
---|
| 158 | void reset() {last_=false;} |
---|
| 159 | |
---|
| 160 | template <typename InputIterator, typename Token> |
---|
| 161 | bool operator()(InputIterator& next,InputIterator end,Token& tok) { |
---|
| 162 | bool bInQuote = false; |
---|
| 163 | tok = Token(); |
---|
| 164 | |
---|
| 165 | if (next == end) { |
---|
| 166 | if (last_) { |
---|
| 167 | last_ = false; |
---|
| 168 | return true; |
---|
| 169 | } |
---|
| 170 | else |
---|
| 171 | return false; |
---|
| 172 | } |
---|
| 173 | last_ = false; |
---|
| 174 | for (;next != end;++next) { |
---|
| 175 | if (is_escape(*next)) { |
---|
| 176 | do_escape(next,end,tok); |
---|
| 177 | } |
---|
| 178 | else if (is_c(*next)) { |
---|
| 179 | if (!bInQuote) { |
---|
| 180 | // If we are not in quote, then we are done |
---|
| 181 | ++next; |
---|
| 182 | // The last character was a c, that means there is |
---|
| 183 | // 1 more blank field |
---|
| 184 | last_ = true; |
---|
| 185 | return true; |
---|
| 186 | } |
---|
| 187 | else tok+=*next; |
---|
| 188 | } |
---|
| 189 | else if (is_quote(*next)) { |
---|
| 190 | bInQuote=!bInQuote; |
---|
| 191 | } |
---|
| 192 | else { |
---|
| 193 | tok += *next; |
---|
| 194 | } |
---|
| 195 | } |
---|
| 196 | return true; |
---|
| 197 | } |
---|
| 198 | }; |
---|
| 199 | |
---|
| 200 | //=========================================================================== |
---|
| 201 | // The classes here are used by offset_separator and char_separator to implement |
---|
| 202 | // faster assigning of tokens using assign instead of += |
---|
| 203 | |
---|
| 204 | namespace tokenizer_detail { |
---|
| 205 | //=========================================================================== |
---|
| 206 | // Tokenizer was broken for wide character separators, at least on Windows, since |
---|
| 207 | // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts |
---|
| 208 | // if higher values are passed in. The traits extension class should take care of this. |
---|
| 209 | // Assuming that the conditional will always get optimized out in the function |
---|
| 210 | // implementations, argument types are not a problem since both forms of character classifiers |
---|
| 211 | // expect an int. |
---|
| 212 | // In case there is no cwctype header, we implement the checks manually. |
---|
| 213 | // We make use of the fact that the tested categories should fit in ASCII. |
---|
| 214 | template<typename traits> |
---|
| 215 | struct traits_extension : public traits { |
---|
| 216 | typedef typename traits::char_type char_type; |
---|
| 217 | static bool isspace(char_type c) |
---|
| 218 | { |
---|
| 219 | #if !defined(BOOST_NO_CWCTYPE) |
---|
| 220 | if (sizeof(char_type) == 1) |
---|
| 221 | return std::isspace(c) != 0; |
---|
| 222 | else |
---|
| 223 | return std::iswspace(c) != 0; |
---|
| 224 | #else |
---|
| 225 | return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0; |
---|
| 226 | #endif |
---|
| 227 | } |
---|
| 228 | |
---|
| 229 | static bool ispunct(char_type c) |
---|
| 230 | { |
---|
| 231 | #if !defined(BOOST_NO_CWCTYPE) |
---|
| 232 | if (sizeof(char_type) == 1) |
---|
| 233 | return std::ispunct(c) != 0; |
---|
| 234 | else |
---|
| 235 | return std::iswpunct(c) != 0; |
---|
| 236 | #else |
---|
| 237 | return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0; |
---|
| 238 | #endif |
---|
| 239 | } |
---|
| 240 | }; |
---|
| 241 | |
---|
| 242 | // The assign_or_plus_equal struct contains functions that implement |
---|
| 243 | // assign, +=, and clearing based on the iterator type. The |
---|
| 244 | // generic case does nothing for plus_equal and clearing, while |
---|
| 245 | // passing through the call for assign. |
---|
| 246 | // |
---|
| 247 | // When an input iterator is being used, the situation is reversed. |
---|
| 248 | // The assign method does nothing, plus_equal invokes operator +=, |
---|
| 249 | // and the clearing method sets the supplied token to the default |
---|
| 250 | // token constructor's result. |
---|
| 251 | // |
---|
| 252 | |
---|
| 253 | template<class IteratorTag> |
---|
| 254 | struct assign_or_plus_equal { |
---|
| 255 | template<class Iterator, class Token> |
---|
| 256 | static void assign(Iterator b, Iterator e, Token &t) { |
---|
| 257 | |
---|
| 258 | #if BOOST_WORKAROUND(BOOST_MSVC, < 1300) &&\ |
---|
| 259 | BOOST_WORKAROUND(__SGI_STL_PORT, < 0x500) &&\ |
---|
| 260 | defined(_STLP_DEBUG) &&\ |
---|
| 261 | (defined(_STLP_USE_DYNAMIC_LIB) || defined(_DLL)) |
---|
| 262 | // Problem with string::assign for msvc-stlport in debug mode: the |
---|
| 263 | // linker tries to import the templatized version of this memfun, |
---|
| 264 | // which is obviously not exported. |
---|
| 265 | // See http://www.stlport.com/dcforum/DCForumID6/1763.html for details. |
---|
| 266 | |
---|
| 267 | t = Token(); |
---|
| 268 | while(b != e) t += *b++; |
---|
| 269 | #else |
---|
| 270 | t.assign(b, e); |
---|
| 271 | #endif |
---|
| 272 | |
---|
| 273 | } |
---|
| 274 | |
---|
| 275 | template<class Token, class Value> |
---|
| 276 | static void plus_equal(Token &, const Value &) { } |
---|
| 277 | |
---|
| 278 | // If we are doing an assign, there is no need for the |
---|
| 279 | // the clear. |
---|
| 280 | // |
---|
| 281 | template<class Token> |
---|
| 282 | static void clear(Token &) { } |
---|
| 283 | }; |
---|
| 284 | |
---|
| 285 | template <> |
---|
| 286 | struct assign_or_plus_equal<std::input_iterator_tag> { |
---|
| 287 | template<class Iterator, class Token> |
---|
| 288 | static void assign(Iterator b, Iterator e, Token &t) { } |
---|
| 289 | template<class Token, class Value> |
---|
| 290 | static void plus_equal(Token &t, const Value &v) { |
---|
| 291 | t += v; |
---|
| 292 | } |
---|
| 293 | template<class Token> |
---|
| 294 | static void clear(Token &t) { |
---|
| 295 | t = Token(); |
---|
| 296 | } |
---|
| 297 | }; |
---|
| 298 | |
---|
| 299 | |
---|
| 300 | template<class Iterator> |
---|
| 301 | struct pointer_iterator_category{ |
---|
| 302 | typedef std::random_access_iterator_tag type; |
---|
| 303 | }; |
---|
| 304 | |
---|
| 305 | |
---|
| 306 | template<class Iterator> |
---|
| 307 | struct class_iterator_category{ |
---|
| 308 | typedef typename Iterator::iterator_category type; |
---|
| 309 | }; |
---|
| 310 | |
---|
| 311 | |
---|
| 312 | |
---|
| 313 | // This portably gets the iterator_tag without partial template specialization |
---|
| 314 | template<class Iterator> |
---|
| 315 | struct get_iterator_category{ |
---|
| 316 | typedef typename mpl::if_<is_pointer<Iterator>, |
---|
| 317 | pointer_iterator_category<Iterator>, |
---|
| 318 | class_iterator_category<Iterator> |
---|
| 319 | >::type cat; |
---|
| 320 | |
---|
| 321 | typedef typename cat::type iterator_category; |
---|
| 322 | }; |
---|
| 323 | |
---|
| 324 | |
---|
| 325 | } // namespace tokenizer_detail |
---|
| 326 | |
---|
| 327 | |
---|
| 328 | //=========================================================================== |
---|
| 329 | // The offset_separator class, which is a model of TokenizerFunction. |
---|
| 330 | // Offset breaks a string into tokens based on a range of offsets |
---|
| 331 | |
---|
| 332 | class offset_separator { |
---|
| 333 | private: |
---|
| 334 | |
---|
| 335 | std::vector<int> offsets_; |
---|
| 336 | unsigned int current_offset_; |
---|
| 337 | bool wrap_offsets_; |
---|
| 338 | bool return_partial_last_; |
---|
| 339 | |
---|
| 340 | public: |
---|
| 341 | template <typename Iter> |
---|
| 342 | offset_separator(Iter begin, Iter end, bool wrap_offsets = true, |
---|
| 343 | bool return_partial_last = true) |
---|
| 344 | : offsets_(begin,end), current_offset_(0), |
---|
| 345 | wrap_offsets_(wrap_offsets), |
---|
| 346 | return_partial_last_(return_partial_last) { } |
---|
| 347 | |
---|
| 348 | offset_separator() |
---|
| 349 | : offsets_(1,1), current_offset_(), |
---|
| 350 | wrap_offsets_(true), return_partial_last_(true) { } |
---|
| 351 | |
---|
| 352 | void reset() { |
---|
| 353 | current_offset_ = 0; |
---|
| 354 | } |
---|
| 355 | |
---|
| 356 | template <typename InputIterator, typename Token> |
---|
| 357 | bool operator()(InputIterator& next, InputIterator end, Token& tok) |
---|
| 358 | { |
---|
| 359 | typedef tokenizer_detail::assign_or_plus_equal< |
---|
| 360 | BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category< |
---|
| 361 | InputIterator |
---|
| 362 | >::iterator_category |
---|
| 363 | > assigner; |
---|
| 364 | |
---|
| 365 | BOOST_ASSERT(!offsets_.empty()); |
---|
| 366 | |
---|
| 367 | assigner::clear(tok); |
---|
| 368 | InputIterator start(next); |
---|
| 369 | |
---|
| 370 | if (next == end) |
---|
| 371 | return false; |
---|
| 372 | |
---|
| 373 | if (current_offset_ == offsets_.size()) |
---|
| 374 | { |
---|
| 375 | if (wrap_offsets_) |
---|
| 376 | current_offset_=0; |
---|
| 377 | else |
---|
| 378 | return false; |
---|
| 379 | } |
---|
| 380 | |
---|
| 381 | int c = offsets_[current_offset_]; |
---|
| 382 | int i = 0; |
---|
| 383 | for (; i < c; ++i) { |
---|
| 384 | if (next == end)break; |
---|
| 385 | assigner::plus_equal(tok,*next++); |
---|
| 386 | } |
---|
| 387 | assigner::assign(start,next,tok); |
---|
| 388 | |
---|
| 389 | if (!return_partial_last_) |
---|
| 390 | if (i < (c-1) ) |
---|
| 391 | return false; |
---|
| 392 | |
---|
| 393 | ++current_offset_; |
---|
| 394 | return true; |
---|
| 395 | } |
---|
| 396 | }; |
---|
| 397 | |
---|
| 398 | |
---|
| 399 | //=========================================================================== |
---|
| 400 | // The char_separator class breaks a sequence of characters into |
---|
| 401 | // tokens based on the character delimiters (very much like bad old |
---|
| 402 | // strtok). A delimiter character can either be kept or dropped. A |
---|
| 403 | // kept delimiter shows up as an output token, whereas a dropped |
---|
| 404 | // delimiter does not. |
---|
| 405 | |
---|
| 406 | // This class replaces the char_delimiters_separator class. The |
---|
| 407 | // constructor for the char_delimiters_separator class was too |
---|
| 408 | // confusing and needed to be deprecated. However, because of the |
---|
| 409 | // default arguments to the constructor, adding the new constructor |
---|
| 410 | // would cause ambiguity, so instead I deprecated the whole class. |
---|
| 411 | // The implementation of the class was also simplified considerably. |
---|
| 412 | |
---|
| 413 | enum empty_token_policy { drop_empty_tokens, keep_empty_tokens }; |
---|
| 414 | |
---|
| 415 | // The out of the box GCC 2.95 on cygwin does not have a char_traits class. |
---|
| 416 | template <typename Char, |
---|
| 417 | typename Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type > |
---|
| 418 | class char_separator |
---|
| 419 | { |
---|
| 420 | typedef tokenizer_detail::traits_extension<Tr> Traits; |
---|
| 421 | typedef std::basic_string<Char,Traits> string_type; |
---|
| 422 | public: |
---|
| 423 | explicit |
---|
| 424 | char_separator(const Char* dropped_delims, |
---|
| 425 | const Char* kept_delims = 0, |
---|
| 426 | empty_token_policy empty_tokens = drop_empty_tokens) |
---|
| 427 | : m_dropped_delims(dropped_delims), |
---|
| 428 | m_use_ispunct(false), |
---|
| 429 | m_use_isspace(false), |
---|
| 430 | m_empty_tokens(empty_tokens), |
---|
| 431 | m_output_done(false) |
---|
| 432 | { |
---|
| 433 | // Borland workaround |
---|
| 434 | if (kept_delims) |
---|
| 435 | m_kept_delims = kept_delims; |
---|
| 436 | } |
---|
| 437 | |
---|
| 438 | // use ispunct() for kept delimiters and isspace for dropped. |
---|
| 439 | explicit |
---|
| 440 | char_separator() |
---|
| 441 | : m_use_ispunct(true), |
---|
| 442 | m_use_isspace(true), |
---|
| 443 | m_empty_tokens(drop_empty_tokens) { } |
---|
| 444 | |
---|
| 445 | void reset() { } |
---|
| 446 | |
---|
| 447 | template <typename InputIterator, typename Token> |
---|
| 448 | bool operator()(InputIterator& next, InputIterator end, Token& tok) |
---|
| 449 | { |
---|
| 450 | typedef tokenizer_detail::assign_or_plus_equal< |
---|
| 451 | BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category< |
---|
| 452 | InputIterator |
---|
| 453 | >::iterator_category |
---|
| 454 | > assigner; |
---|
| 455 | |
---|
| 456 | assigner::clear(tok); |
---|
| 457 | |
---|
| 458 | // skip past all dropped_delims |
---|
| 459 | if (m_empty_tokens == drop_empty_tokens) |
---|
| 460 | for (; next != end && is_dropped(*next); ++next) |
---|
| 461 | { } |
---|
| 462 | |
---|
| 463 | InputIterator start(next); |
---|
| 464 | |
---|
| 465 | if (m_empty_tokens == drop_empty_tokens) { |
---|
| 466 | |
---|
| 467 | if (next == end) |
---|
| 468 | return false; |
---|
| 469 | |
---|
| 470 | |
---|
| 471 | // if we are on a kept_delims move past it and stop |
---|
| 472 | if (is_kept(*next)) { |
---|
| 473 | assigner::plus_equal(tok,*next); |
---|
| 474 | ++next; |
---|
| 475 | } else |
---|
| 476 | // append all the non delim characters |
---|
| 477 | for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next) |
---|
| 478 | assigner::plus_equal(tok,*next); |
---|
| 479 | } |
---|
| 480 | else { // m_empty_tokens == keep_empty_tokens |
---|
| 481 | |
---|
| 482 | // Handle empty token at the end |
---|
| 483 | if (next == end) |
---|
| 484 | { |
---|
| 485 | if (m_output_done == false) |
---|
| 486 | { |
---|
| 487 | m_output_done = true; |
---|
| 488 | assigner::assign(start,next,tok); |
---|
| 489 | return true; |
---|
| 490 | } |
---|
| 491 | else |
---|
| 492 | return false; |
---|
| 493 | } |
---|
| 494 | |
---|
| 495 | if (is_kept(*next)) { |
---|
| 496 | if (m_output_done == false) |
---|
| 497 | m_output_done = true; |
---|
| 498 | else { |
---|
| 499 | assigner::plus_equal(tok,*next); |
---|
| 500 | ++next; |
---|
| 501 | m_output_done = false; |
---|
| 502 | } |
---|
| 503 | } |
---|
| 504 | else if (m_output_done == false && is_dropped(*next)) { |
---|
| 505 | m_output_done = true; |
---|
| 506 | } |
---|
| 507 | else { |
---|
| 508 | if (is_dropped(*next)) |
---|
| 509 | start=++next; |
---|
| 510 | for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next) |
---|
| 511 | assigner::plus_equal(tok,*next); |
---|
| 512 | m_output_done = true; |
---|
| 513 | } |
---|
| 514 | } |
---|
| 515 | assigner::assign(start,next,tok); |
---|
| 516 | return true; |
---|
| 517 | } |
---|
| 518 | |
---|
| 519 | private: |
---|
| 520 | string_type m_kept_delims; |
---|
| 521 | string_type m_dropped_delims; |
---|
| 522 | bool m_use_ispunct; |
---|
| 523 | bool m_use_isspace; |
---|
| 524 | empty_token_policy m_empty_tokens; |
---|
| 525 | bool m_output_done; |
---|
| 526 | |
---|
| 527 | bool is_kept(Char E) const |
---|
| 528 | { |
---|
| 529 | if (m_kept_delims.length()) |
---|
| 530 | return m_kept_delims.find(E) != string_type::npos; |
---|
| 531 | else if (m_use_ispunct) { |
---|
| 532 | return Traits::ispunct(E) != 0; |
---|
| 533 | } else |
---|
| 534 | return false; |
---|
| 535 | } |
---|
| 536 | bool is_dropped(Char E) const |
---|
| 537 | { |
---|
| 538 | if (m_dropped_delims.length()) |
---|
| 539 | return m_dropped_delims.find(E) != string_type::npos; |
---|
| 540 | else if (m_use_isspace) { |
---|
| 541 | return Traits::isspace(E) != 0; |
---|
| 542 | } else |
---|
| 543 | return false; |
---|
| 544 | } |
---|
| 545 | }; |
---|
| 546 | |
---|
| 547 | //=========================================================================== |
---|
| 548 | // The following class is DEPRECATED, use class char_separators instead. |
---|
| 549 | // |
---|
| 550 | // The char_delimiters_separator class, which is a model of |
---|
| 551 | // TokenizerFunction. char_delimiters_separator breaks a string |
---|
| 552 | // into tokens based on character delimiters. There are 2 types of |
---|
| 553 | // delimiters. returnable delimiters can be returned as |
---|
| 554 | // tokens. These are often punctuation. nonreturnable delimiters |
---|
| 555 | // cannot be returned as tokens. These are often whitespace |
---|
| 556 | |
---|
| 557 | // The out of the box GCC 2.95 on cygwin does not have a char_traits class. |
---|
| 558 | template <class Char, |
---|
| 559 | class Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type > |
---|
| 560 | class char_delimiters_separator { |
---|
| 561 | private: |
---|
| 562 | |
---|
| 563 | typedef tokenizer_detail::traits_extension<Tr> Traits; |
---|
| 564 | typedef std::basic_string<Char,Traits> string_type; |
---|
| 565 | string_type returnable_; |
---|
| 566 | string_type nonreturnable_; |
---|
| 567 | bool return_delims_; |
---|
| 568 | bool no_ispunct_; |
---|
| 569 | bool no_isspace_; |
---|
| 570 | |
---|
| 571 | bool is_ret(Char E)const |
---|
| 572 | { |
---|
| 573 | if (returnable_.length()) |
---|
| 574 | return returnable_.find(E) != string_type::npos; |
---|
| 575 | else{ |
---|
| 576 | if (no_ispunct_) {return false;} |
---|
| 577 | else{ |
---|
| 578 | int r = Traits::ispunct(E); |
---|
| 579 | return r != 0; |
---|
| 580 | } |
---|
| 581 | } |
---|
| 582 | } |
---|
| 583 | bool is_nonret(Char E)const |
---|
| 584 | { |
---|
| 585 | if (nonreturnable_.length()) |
---|
| 586 | return nonreturnable_.find(E) != string_type::npos; |
---|
| 587 | else{ |
---|
| 588 | if (no_isspace_) {return false;} |
---|
| 589 | else{ |
---|
| 590 | int r = Traits::isspace(E); |
---|
| 591 | return r != 0; |
---|
| 592 | } |
---|
| 593 | } |
---|
| 594 | } |
---|
| 595 | |
---|
| 596 | public: |
---|
| 597 | explicit char_delimiters_separator(bool return_delims = false, |
---|
| 598 | const Char* returnable = 0, |
---|
| 599 | const Char* nonreturnable = 0) |
---|
| 600 | : returnable_(returnable ? returnable : string_type().c_str()), |
---|
| 601 | nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()), |
---|
| 602 | return_delims_(return_delims), no_ispunct_(returnable!=0), |
---|
| 603 | no_isspace_(nonreturnable!=0) { } |
---|
| 604 | |
---|
| 605 | void reset() { } |
---|
| 606 | |
---|
| 607 | public: |
---|
| 608 | |
---|
| 609 | template <typename InputIterator, typename Token> |
---|
| 610 | bool operator()(InputIterator& next, InputIterator end,Token& tok) { |
---|
| 611 | tok = Token(); |
---|
| 612 | |
---|
| 613 | // skip past all nonreturnable delims |
---|
| 614 | // skip past the returnable only if we are not returning delims |
---|
| 615 | for (;next!=end && ( is_nonret(*next) || (is_ret(*next) |
---|
| 616 | && !return_delims_ ) );++next) { } |
---|
| 617 | |
---|
| 618 | if (next == end) { |
---|
| 619 | return false; |
---|
| 620 | } |
---|
| 621 | |
---|
| 622 | // if we are to return delims and we are one a returnable one |
---|
| 623 | // move past it and stop |
---|
| 624 | if (is_ret(*next) && return_delims_) { |
---|
| 625 | tok+=*next; |
---|
| 626 | ++next; |
---|
| 627 | } |
---|
| 628 | else |
---|
| 629 | // append all the non delim characters |
---|
| 630 | for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next) |
---|
| 631 | tok+=*next; |
---|
| 632 | |
---|
| 633 | |
---|
| 634 | return true; |
---|
| 635 | } |
---|
| 636 | }; |
---|
| 637 | |
---|
| 638 | |
---|
| 639 | } //namespace boost |
---|
| 640 | |
---|
| 641 | #endif |
---|