1 | // Boost token_functions.hpp ------------------------------------------------// |
---|
2 | |
---|
3 | // Copyright John R. Bandela 2001. |
---|
4 | |
---|
5 | // Distributed under the Boost Software License, Version 1.0. (See |
---|
6 | // accompanying file LICENSE_1_0.txt or copy at |
---|
7 | // http://www.boost.org/LICENSE_1_0.txt) |
---|
8 | |
---|
9 | // See http://www.boost.org/libs/tokenizer/ for documentation. |
---|
10 | |
---|
11 | // Revision History: |
---|
12 | // 01 Oct 2004 Joaquin M Lopez Munoz |
---|
13 | // Workaround for a problem with string::assign in msvc-stlport |
---|
14 | // 06 Apr 2004 John Bandela |
---|
15 | // Fixed a bug involving using char_delimiter with a true input iterator |
---|
16 | // 28 Nov 2003 Robert Zeh and John Bandela |
---|
17 | // Converted into "fast" functions that avoid using += when |
---|
18 | // the supplied iterator isn't an input_iterator; based on |
---|
19 | // some work done at Archelon and a version that was checked into |
---|
20 | // the boost CVS for a short period of time. |
---|
21 | // 20 Feb 2002 John Maddock |
---|
22 | // Removed using namespace std declarations and added |
---|
23 | // workaround for BOOST_NO_STDC_NAMESPACE (the library |
---|
24 | // can be safely mixed with regex). |
---|
25 | // 06 Feb 2002 Jeremy Siek |
---|
26 | // Added char_separator. |
---|
27 | // 02 Feb 2002 Jeremy Siek |
---|
28 | // Removed tabs and a little cleanup. |
---|
29 | |
---|
30 | |
---|
31 | #ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_ |
---|
32 | #define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_ |
---|
33 | |
---|
34 | #include <vector> |
---|
35 | #include <stdexcept> |
---|
36 | #include <string> |
---|
37 | #include <cctype> |
---|
38 | #include <algorithm> // for find_if |
---|
39 | #include <boost/config.hpp> |
---|
40 | #include <boost/assert.hpp> |
---|
41 | #include <boost/detail/workaround.hpp> |
---|
42 | #include <boost/mpl/if.hpp> |
---|
43 | #if !defined(BOOST_NO_CWCTYPE) |
---|
44 | #include <cwctype> |
---|
45 | #endif |
---|
46 | |
---|
47 | // |
---|
48 | // the following must not be macros if we are to prefix them |
---|
49 | // with std:: (they shouldn't be macros anyway...) |
---|
50 | // |
---|
51 | #ifdef ispunct |
---|
52 | # undef ispunct |
---|
53 | #endif |
---|
54 | #ifdef iswpunct |
---|
55 | # undef iswpunct |
---|
56 | #endif |
---|
57 | #ifdef isspace |
---|
58 | # undef isspace |
---|
59 | #endif |
---|
60 | #ifdef iswspace |
---|
61 | # undef iswspace |
---|
62 | #endif |
---|
63 | // |
---|
64 | // fix namespace problems: |
---|
65 | // |
---|
66 | #ifdef BOOST_NO_STDC_NAMESPACE |
---|
67 | namespace std{ |
---|
68 | using ::ispunct; |
---|
69 | using ::isspace; |
---|
70 | #if !defined(BOOST_NO_CWCTYPE) |
---|
71 | using ::iswpunct; |
---|
72 | using ::iswspace; |
---|
73 | #endif |
---|
74 | } |
---|
75 | #endif |
---|
76 | |
---|
77 | namespace boost{ |
---|
78 | //=========================================================================== |
---|
79 | // The escaped_list_separator class. Which is a model of TokenizerFunction |
---|
80 | // An escaped list is a super-set of what is commonly known as a comma |
---|
81 | // separated value (csv) list.It is separated into fields by a comma or |
---|
82 | // other character. If the delimiting character is inside quotes, then it is |
---|
83 | // counted as a regular character.To allow for embedded quotes in a field, |
---|
84 | // there can be escape sequences using the \ much like C. |
---|
85 | // The role of the comma, the quotation mark, and the escape |
---|
86 | // character (backslash \), can be assigned to other characters. |
---|
87 | |
---|
88 | struct escaped_list_error : public std::runtime_error{ |
---|
89 | escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { } |
---|
90 | }; |
---|
91 | |
---|
92 | |
---|
93 | // The out of the box GCC 2.95 on cygwin does not have a char_traits class. |
---|
94 | // MSVC does not like the following typename |
---|
95 | template <class Char, |
---|
96 | class Traits = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type > |
---|
97 | class escaped_list_separator { |
---|
98 | |
---|
99 | private: |
---|
100 | typedef std::basic_string<Char,Traits> string_type; |
---|
101 | struct char_eq { |
---|
102 | Char e_; |
---|
103 | char_eq(Char e):e_(e) { } |
---|
104 | bool operator()(Char c) { |
---|
105 | return Traits::eq(e_,c); |
---|
106 | } |
---|
107 | }; |
---|
108 | string_type escape_; |
---|
109 | string_type c_; |
---|
110 | string_type quote_; |
---|
111 | bool last_; |
---|
112 | |
---|
113 | bool is_escape(Char e) { |
---|
114 | char_eq f(e); |
---|
115 | return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end(); |
---|
116 | } |
---|
117 | bool is_c(Char e) { |
---|
118 | char_eq f(e); |
---|
119 | return std::find_if(c_.begin(),c_.end(),f)!=c_.end(); |
---|
120 | } |
---|
121 | bool is_quote(Char e) { |
---|
122 | char_eq f(e); |
---|
123 | return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end(); |
---|
124 | } |
---|
125 | template <typename iterator, typename Token> |
---|
126 | void do_escape(iterator& next,iterator end,Token& tok) { |
---|
127 | if (++next == end) |
---|
128 | throw escaped_list_error(std::string("cannot end with escape")); |
---|
129 | if (Traits::eq(*next,'n')) { |
---|
130 | tok+='\n'; |
---|
131 | return; |
---|
132 | } |
---|
133 | else if (is_quote(*next)) { |
---|
134 | tok+=*next; |
---|
135 | return; |
---|
136 | } |
---|
137 | else if (is_c(*next)) { |
---|
138 | tok+=*next; |
---|
139 | return; |
---|
140 | } |
---|
141 | else if (is_escape(*next)) { |
---|
142 | tok+=*next; |
---|
143 | return; |
---|
144 | } |
---|
145 | else |
---|
146 | throw escaped_list_error(std::string("unknown escape sequence")); |
---|
147 | } |
---|
148 | |
---|
149 | public: |
---|
150 | |
---|
151 | explicit escaped_list_separator(Char e = '\\', |
---|
152 | Char c = ',',Char q = '\"') |
---|
153 | : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { } |
---|
154 | |
---|
155 | escaped_list_separator(string_type e, string_type c, string_type q) |
---|
156 | : escape_(e), c_(c), quote_(q), last_(false) { } |
---|
157 | |
---|
158 | void reset() {last_=false;} |
---|
159 | |
---|
160 | template <typename InputIterator, typename Token> |
---|
161 | bool operator()(InputIterator& next,InputIterator end,Token& tok) { |
---|
162 | bool bInQuote = false; |
---|
163 | tok = Token(); |
---|
164 | |
---|
165 | if (next == end) { |
---|
166 | if (last_) { |
---|
167 | last_ = false; |
---|
168 | return true; |
---|
169 | } |
---|
170 | else |
---|
171 | return false; |
---|
172 | } |
---|
173 | last_ = false; |
---|
174 | for (;next != end;++next) { |
---|
175 | if (is_escape(*next)) { |
---|
176 | do_escape(next,end,tok); |
---|
177 | } |
---|
178 | else if (is_c(*next)) { |
---|
179 | if (!bInQuote) { |
---|
180 | // If we are not in quote, then we are done |
---|
181 | ++next; |
---|
182 | // The last character was a c, that means there is |
---|
183 | // 1 more blank field |
---|
184 | last_ = true; |
---|
185 | return true; |
---|
186 | } |
---|
187 | else tok+=*next; |
---|
188 | } |
---|
189 | else if (is_quote(*next)) { |
---|
190 | bInQuote=!bInQuote; |
---|
191 | } |
---|
192 | else { |
---|
193 | tok += *next; |
---|
194 | } |
---|
195 | } |
---|
196 | return true; |
---|
197 | } |
---|
198 | }; |
---|
199 | |
---|
200 | //=========================================================================== |
---|
201 | // The classes here are used by offset_separator and char_separator to implement |
---|
202 | // faster assigning of tokens using assign instead of += |
---|
203 | |
---|
204 | namespace tokenizer_detail { |
---|
205 | //=========================================================================== |
---|
206 | // Tokenizer was broken for wide character separators, at least on Windows, since |
---|
207 | // CRT functions isspace etc only expect values in [0, 0xFF]. Debug build asserts |
---|
208 | // if higher values are passed in. The traits extension class should take care of this. |
---|
209 | // Assuming that the conditional will always get optimized out in the function |
---|
210 | // implementations, argument types are not a problem since both forms of character classifiers |
---|
211 | // expect an int. |
---|
212 | // In case there is no cwctype header, we implement the checks manually. |
---|
213 | // We make use of the fact that the tested categories should fit in ASCII. |
---|
214 | template<typename traits> |
---|
215 | struct traits_extension : public traits { |
---|
216 | typedef typename traits::char_type char_type; |
---|
217 | static bool isspace(char_type c) |
---|
218 | { |
---|
219 | #if !defined(BOOST_NO_CWCTYPE) |
---|
220 | if (sizeof(char_type) == 1) |
---|
221 | return std::isspace(c) != 0; |
---|
222 | else |
---|
223 | return std::iswspace(c) != 0; |
---|
224 | #else |
---|
225 | return static_cast< unsigned >(c) <= 255 && std::isspace(c) != 0; |
---|
226 | #endif |
---|
227 | } |
---|
228 | |
---|
229 | static bool ispunct(char_type c) |
---|
230 | { |
---|
231 | #if !defined(BOOST_NO_CWCTYPE) |
---|
232 | if (sizeof(char_type) == 1) |
---|
233 | return std::ispunct(c) != 0; |
---|
234 | else |
---|
235 | return std::iswpunct(c) != 0; |
---|
236 | #else |
---|
237 | return static_cast< unsigned >(c) <= 255 && std::ispunct(c) != 0; |
---|
238 | #endif |
---|
239 | } |
---|
240 | }; |
---|
241 | |
---|
242 | // The assign_or_plus_equal struct contains functions that implement |
---|
243 | // assign, +=, and clearing based on the iterator type. The |
---|
244 | // generic case does nothing for plus_equal and clearing, while |
---|
245 | // passing through the call for assign. |
---|
246 | // |
---|
247 | // When an input iterator is being used, the situation is reversed. |
---|
248 | // The assign method does nothing, plus_equal invokes operator +=, |
---|
249 | // and the clearing method sets the supplied token to the default |
---|
250 | // token constructor's result. |
---|
251 | // |
---|
252 | |
---|
253 | template<class IteratorTag> |
---|
254 | struct assign_or_plus_equal { |
---|
255 | template<class Iterator, class Token> |
---|
256 | static void assign(Iterator b, Iterator e, Token &t) { |
---|
257 | |
---|
258 | #if BOOST_WORKAROUND(BOOST_MSVC, < 1300) &&\ |
---|
259 | BOOST_WORKAROUND(__SGI_STL_PORT, < 0x500) &&\ |
---|
260 | defined(_STLP_DEBUG) &&\ |
---|
261 | (defined(_STLP_USE_DYNAMIC_LIB) || defined(_DLL)) |
---|
262 | // Problem with string::assign for msvc-stlport in debug mode: the |
---|
263 | // linker tries to import the templatized version of this memfun, |
---|
264 | // which is obviously not exported. |
---|
265 | // See http://www.stlport.com/dcforum/DCForumID6/1763.html for details. |
---|
266 | |
---|
267 | t = Token(); |
---|
268 | while(b != e) t += *b++; |
---|
269 | #else |
---|
270 | t.assign(b, e); |
---|
271 | #endif |
---|
272 | |
---|
273 | } |
---|
274 | |
---|
275 | template<class Token, class Value> |
---|
276 | static void plus_equal(Token &, const Value &) { } |
---|
277 | |
---|
278 | // If we are doing an assign, there is no need for the |
---|
279 | // the clear. |
---|
280 | // |
---|
281 | template<class Token> |
---|
282 | static void clear(Token &) { } |
---|
283 | }; |
---|
284 | |
---|
285 | template <> |
---|
286 | struct assign_or_plus_equal<std::input_iterator_tag> { |
---|
287 | template<class Iterator, class Token> |
---|
288 | static void assign(Iterator b, Iterator e, Token &t) { } |
---|
289 | template<class Token, class Value> |
---|
290 | static void plus_equal(Token &t, const Value &v) { |
---|
291 | t += v; |
---|
292 | } |
---|
293 | template<class Token> |
---|
294 | static void clear(Token &t) { |
---|
295 | t = Token(); |
---|
296 | } |
---|
297 | }; |
---|
298 | |
---|
299 | |
---|
300 | template<class Iterator> |
---|
301 | struct pointer_iterator_category{ |
---|
302 | typedef std::random_access_iterator_tag type; |
---|
303 | }; |
---|
304 | |
---|
305 | |
---|
306 | template<class Iterator> |
---|
307 | struct class_iterator_category{ |
---|
308 | typedef typename Iterator::iterator_category type; |
---|
309 | }; |
---|
310 | |
---|
311 | |
---|
312 | |
---|
313 | // This portably gets the iterator_tag without partial template specialization |
---|
314 | template<class Iterator> |
---|
315 | struct get_iterator_category{ |
---|
316 | typedef typename mpl::if_<is_pointer<Iterator>, |
---|
317 | pointer_iterator_category<Iterator>, |
---|
318 | class_iterator_category<Iterator> |
---|
319 | >::type cat; |
---|
320 | |
---|
321 | typedef typename cat::type iterator_category; |
---|
322 | }; |
---|
323 | |
---|
324 | |
---|
325 | } // namespace tokenizer_detail |
---|
326 | |
---|
327 | |
---|
328 | //=========================================================================== |
---|
329 | // The offset_separator class, which is a model of TokenizerFunction. |
---|
330 | // Offset breaks a string into tokens based on a range of offsets |
---|
331 | |
---|
332 | class offset_separator { |
---|
333 | private: |
---|
334 | |
---|
335 | std::vector<int> offsets_; |
---|
336 | unsigned int current_offset_; |
---|
337 | bool wrap_offsets_; |
---|
338 | bool return_partial_last_; |
---|
339 | |
---|
340 | public: |
---|
341 | template <typename Iter> |
---|
342 | offset_separator(Iter begin, Iter end, bool wrap_offsets = true, |
---|
343 | bool return_partial_last = true) |
---|
344 | : offsets_(begin,end), current_offset_(0), |
---|
345 | wrap_offsets_(wrap_offsets), |
---|
346 | return_partial_last_(return_partial_last) { } |
---|
347 | |
---|
348 | offset_separator() |
---|
349 | : offsets_(1,1), current_offset_(), |
---|
350 | wrap_offsets_(true), return_partial_last_(true) { } |
---|
351 | |
---|
352 | void reset() { |
---|
353 | current_offset_ = 0; |
---|
354 | } |
---|
355 | |
---|
356 | template <typename InputIterator, typename Token> |
---|
357 | bool operator()(InputIterator& next, InputIterator end, Token& tok) |
---|
358 | { |
---|
359 | typedef tokenizer_detail::assign_or_plus_equal< |
---|
360 | BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category< |
---|
361 | InputIterator |
---|
362 | >::iterator_category |
---|
363 | > assigner; |
---|
364 | |
---|
365 | BOOST_ASSERT(!offsets_.empty()); |
---|
366 | |
---|
367 | assigner::clear(tok); |
---|
368 | InputIterator start(next); |
---|
369 | |
---|
370 | if (next == end) |
---|
371 | return false; |
---|
372 | |
---|
373 | if (current_offset_ == offsets_.size()) |
---|
374 | { |
---|
375 | if (wrap_offsets_) |
---|
376 | current_offset_=0; |
---|
377 | else |
---|
378 | return false; |
---|
379 | } |
---|
380 | |
---|
381 | int c = offsets_[current_offset_]; |
---|
382 | int i = 0; |
---|
383 | for (; i < c; ++i) { |
---|
384 | if (next == end)break; |
---|
385 | assigner::plus_equal(tok,*next++); |
---|
386 | } |
---|
387 | assigner::assign(start,next,tok); |
---|
388 | |
---|
389 | if (!return_partial_last_) |
---|
390 | if (i < (c-1) ) |
---|
391 | return false; |
---|
392 | |
---|
393 | ++current_offset_; |
---|
394 | return true; |
---|
395 | } |
---|
396 | }; |
---|
397 | |
---|
398 | |
---|
399 | //=========================================================================== |
---|
400 | // The char_separator class breaks a sequence of characters into |
---|
401 | // tokens based on the character delimiters (very much like bad old |
---|
402 | // strtok). A delimiter character can either be kept or dropped. A |
---|
403 | // kept delimiter shows up as an output token, whereas a dropped |
---|
404 | // delimiter does not. |
---|
405 | |
---|
406 | // This class replaces the char_delimiters_separator class. The |
---|
407 | // constructor for the char_delimiters_separator class was too |
---|
408 | // confusing and needed to be deprecated. However, because of the |
---|
409 | // default arguments to the constructor, adding the new constructor |
---|
410 | // would cause ambiguity, so instead I deprecated the whole class. |
---|
411 | // The implementation of the class was also simplified considerably. |
---|
412 | |
---|
413 | enum empty_token_policy { drop_empty_tokens, keep_empty_tokens }; |
---|
414 | |
---|
415 | // The out of the box GCC 2.95 on cygwin does not have a char_traits class. |
---|
416 | template <typename Char, |
---|
417 | typename Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type > |
---|
418 | class char_separator |
---|
419 | { |
---|
420 | typedef tokenizer_detail::traits_extension<Tr> Traits; |
---|
421 | typedef std::basic_string<Char,Traits> string_type; |
---|
422 | public: |
---|
423 | explicit |
---|
424 | char_separator(const Char* dropped_delims, |
---|
425 | const Char* kept_delims = 0, |
---|
426 | empty_token_policy empty_tokens = drop_empty_tokens) |
---|
427 | : m_dropped_delims(dropped_delims), |
---|
428 | m_use_ispunct(false), |
---|
429 | m_use_isspace(false), |
---|
430 | m_empty_tokens(empty_tokens), |
---|
431 | m_output_done(false) |
---|
432 | { |
---|
433 | // Borland workaround |
---|
434 | if (kept_delims) |
---|
435 | m_kept_delims = kept_delims; |
---|
436 | } |
---|
437 | |
---|
438 | // use ispunct() for kept delimiters and isspace for dropped. |
---|
439 | explicit |
---|
440 | char_separator() |
---|
441 | : m_use_ispunct(true), |
---|
442 | m_use_isspace(true), |
---|
443 | m_empty_tokens(drop_empty_tokens) { } |
---|
444 | |
---|
445 | void reset() { } |
---|
446 | |
---|
447 | template <typename InputIterator, typename Token> |
---|
448 | bool operator()(InputIterator& next, InputIterator end, Token& tok) |
---|
449 | { |
---|
450 | typedef tokenizer_detail::assign_or_plus_equal< |
---|
451 | BOOST_DEDUCED_TYPENAME tokenizer_detail::get_iterator_category< |
---|
452 | InputIterator |
---|
453 | >::iterator_category |
---|
454 | > assigner; |
---|
455 | |
---|
456 | assigner::clear(tok); |
---|
457 | |
---|
458 | // skip past all dropped_delims |
---|
459 | if (m_empty_tokens == drop_empty_tokens) |
---|
460 | for (; next != end && is_dropped(*next); ++next) |
---|
461 | { } |
---|
462 | |
---|
463 | InputIterator start(next); |
---|
464 | |
---|
465 | if (m_empty_tokens == drop_empty_tokens) { |
---|
466 | |
---|
467 | if (next == end) |
---|
468 | return false; |
---|
469 | |
---|
470 | |
---|
471 | // if we are on a kept_delims move past it and stop |
---|
472 | if (is_kept(*next)) { |
---|
473 | assigner::plus_equal(tok,*next); |
---|
474 | ++next; |
---|
475 | } else |
---|
476 | // append all the non delim characters |
---|
477 | for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next) |
---|
478 | assigner::plus_equal(tok,*next); |
---|
479 | } |
---|
480 | else { // m_empty_tokens == keep_empty_tokens |
---|
481 | |
---|
482 | // Handle empty token at the end |
---|
483 | if (next == end) |
---|
484 | { |
---|
485 | if (m_output_done == false) |
---|
486 | { |
---|
487 | m_output_done = true; |
---|
488 | assigner::assign(start,next,tok); |
---|
489 | return true; |
---|
490 | } |
---|
491 | else |
---|
492 | return false; |
---|
493 | } |
---|
494 | |
---|
495 | if (is_kept(*next)) { |
---|
496 | if (m_output_done == false) |
---|
497 | m_output_done = true; |
---|
498 | else { |
---|
499 | assigner::plus_equal(tok,*next); |
---|
500 | ++next; |
---|
501 | m_output_done = false; |
---|
502 | } |
---|
503 | } |
---|
504 | else if (m_output_done == false && is_dropped(*next)) { |
---|
505 | m_output_done = true; |
---|
506 | } |
---|
507 | else { |
---|
508 | if (is_dropped(*next)) |
---|
509 | start=++next; |
---|
510 | for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next) |
---|
511 | assigner::plus_equal(tok,*next); |
---|
512 | m_output_done = true; |
---|
513 | } |
---|
514 | } |
---|
515 | assigner::assign(start,next,tok); |
---|
516 | return true; |
---|
517 | } |
---|
518 | |
---|
519 | private: |
---|
520 | string_type m_kept_delims; |
---|
521 | string_type m_dropped_delims; |
---|
522 | bool m_use_ispunct; |
---|
523 | bool m_use_isspace; |
---|
524 | empty_token_policy m_empty_tokens; |
---|
525 | bool m_output_done; |
---|
526 | |
---|
527 | bool is_kept(Char E) const |
---|
528 | { |
---|
529 | if (m_kept_delims.length()) |
---|
530 | return m_kept_delims.find(E) != string_type::npos; |
---|
531 | else if (m_use_ispunct) { |
---|
532 | return Traits::ispunct(E) != 0; |
---|
533 | } else |
---|
534 | return false; |
---|
535 | } |
---|
536 | bool is_dropped(Char E) const |
---|
537 | { |
---|
538 | if (m_dropped_delims.length()) |
---|
539 | return m_dropped_delims.find(E) != string_type::npos; |
---|
540 | else if (m_use_isspace) { |
---|
541 | return Traits::isspace(E) != 0; |
---|
542 | } else |
---|
543 | return false; |
---|
544 | } |
---|
545 | }; |
---|
546 | |
---|
547 | //=========================================================================== |
---|
548 | // The following class is DEPRECATED, use class char_separators instead. |
---|
549 | // |
---|
550 | // The char_delimiters_separator class, which is a model of |
---|
551 | // TokenizerFunction. char_delimiters_separator breaks a string |
---|
552 | // into tokens based on character delimiters. There are 2 types of |
---|
553 | // delimiters. returnable delimiters can be returned as |
---|
554 | // tokens. These are often punctuation. nonreturnable delimiters |
---|
555 | // cannot be returned as tokens. These are often whitespace |
---|
556 | |
---|
557 | // The out of the box GCC 2.95 on cygwin does not have a char_traits class. |
---|
558 | template <class Char, |
---|
559 | class Tr = BOOST_DEDUCED_TYPENAME std::basic_string<Char>::traits_type > |
---|
560 | class char_delimiters_separator { |
---|
561 | private: |
---|
562 | |
---|
563 | typedef tokenizer_detail::traits_extension<Tr> Traits; |
---|
564 | typedef std::basic_string<Char,Traits> string_type; |
---|
565 | string_type returnable_; |
---|
566 | string_type nonreturnable_; |
---|
567 | bool return_delims_; |
---|
568 | bool no_ispunct_; |
---|
569 | bool no_isspace_; |
---|
570 | |
---|
571 | bool is_ret(Char E)const |
---|
572 | { |
---|
573 | if (returnable_.length()) |
---|
574 | return returnable_.find(E) != string_type::npos; |
---|
575 | else{ |
---|
576 | if (no_ispunct_) {return false;} |
---|
577 | else{ |
---|
578 | int r = Traits::ispunct(E); |
---|
579 | return r != 0; |
---|
580 | } |
---|
581 | } |
---|
582 | } |
---|
583 | bool is_nonret(Char E)const |
---|
584 | { |
---|
585 | if (nonreturnable_.length()) |
---|
586 | return nonreturnable_.find(E) != string_type::npos; |
---|
587 | else{ |
---|
588 | if (no_isspace_) {return false;} |
---|
589 | else{ |
---|
590 | int r = Traits::isspace(E); |
---|
591 | return r != 0; |
---|
592 | } |
---|
593 | } |
---|
594 | } |
---|
595 | |
---|
596 | public: |
---|
597 | explicit char_delimiters_separator(bool return_delims = false, |
---|
598 | const Char* returnable = 0, |
---|
599 | const Char* nonreturnable = 0) |
---|
600 | : returnable_(returnable ? returnable : string_type().c_str()), |
---|
601 | nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()), |
---|
602 | return_delims_(return_delims), no_ispunct_(returnable!=0), |
---|
603 | no_isspace_(nonreturnable!=0) { } |
---|
604 | |
---|
605 | void reset() { } |
---|
606 | |
---|
607 | public: |
---|
608 | |
---|
609 | template <typename InputIterator, typename Token> |
---|
610 | bool operator()(InputIterator& next, InputIterator end,Token& tok) { |
---|
611 | tok = Token(); |
---|
612 | |
---|
613 | // skip past all nonreturnable delims |
---|
614 | // skip past the returnable only if we are not returning delims |
---|
615 | for (;next!=end && ( is_nonret(*next) || (is_ret(*next) |
---|
616 | && !return_delims_ ) );++next) { } |
---|
617 | |
---|
618 | if (next == end) { |
---|
619 | return false; |
---|
620 | } |
---|
621 | |
---|
622 | // if we are to return delims and we are one a returnable one |
---|
623 | // move past it and stop |
---|
624 | if (is_ret(*next) && return_delims_) { |
---|
625 | tok+=*next; |
---|
626 | ++next; |
---|
627 | } |
---|
628 | else |
---|
629 | // append all the non delim characters |
---|
630 | for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next) |
---|
631 | tok+=*next; |
---|
632 | |
---|
633 | |
---|
634 | return true; |
---|
635 | } |
---|
636 | }; |
---|
637 | |
---|
638 | |
---|
639 | } //namespace boost |
---|
640 | |
---|
641 | #endif |
---|