utils/iterator/token_iterator.hpp

0001 //  (C) Copyright Gennadiy Rozental 2001.
0002 //  Distributed under the Boost Software License, Version 1.0.
0003 //  (See accompanying file LICENSE_1_0.txt or copy at
0004 //  http://www.boost.org/LICENSE_1_0.txt)
0005
0006 //  See http://www.boost.org/libs/test for the library home page.
0007 //
0008 //  File        : $RCSfile$
0009 //
0010 //  Version     : $Revision$
0011 //
0012 //  Description : token iterator for string and range tokenization
0013 // ***************************************************************************
0014
0015 #ifndef BOOST_TEST_UTILS_TOKEN_ITERATOR_HPP
0016 #define BOOST_TEST_UTILS_TOKEN_ITERATOR_HPP
0017
0018 // Boost
0019 #include <boost/config.hpp>
0020 #include <boost/detail/workaround.hpp>
0021
0022 #include <boost/iterator/iterator_categories.hpp>
0023 #include <boost/iterator/iterator_traits.hpp>
0024
0025 #include <boost/test/utils/iterator/input_iterator_facade.hpp>
0026 #include <boost/test/utils/basic_cstring/basic_cstring.hpp>
0027 #include <boost/test/utils/named_params.hpp>
0028 #include <boost/test/utils/foreach.hpp>
0029
0030 // STL
0031 #include <iosfwd>
0032 #include <cctype>
0033
0034 #include <boost/test/detail/suppress_warnings.hpp>
0035
0036 //____________________________________________________________________________//
0037
0038 #ifdef BOOST_NO_STDC_NAMESPACE
0039 namespace std{ using ::ispunct; using ::isspace; }
0040 #endif
0041
0042 namespace boost {
0043 namespace unit_test {
0044 namespace utils {
0045
0046 // ************************************************************************** //
0047 // **************               ti_delimeter_type              ************** //
0048 // ************************************************************************** //
0049
0050 enum ti_delimeter_type {
0051     dt_char,        // character is delimeter if it among explicit list of some characters
0052     dt_ispunct,     // character is delimeter if it satisfies ispunct functor
0053     dt_isspace,     // character is delimeter if it satisfies isspace functor
0054     dt_none         // no character is delimeter
0055 };
0056
0057 namespace ut_detail {
0058
0059 // ************************************************************************** //
0060 // **************             default_char_compare             ************** //
0061 // ************************************************************************** //
0062
0063 template<typename CharT>
0064 class default_char_compare {
0065 public:
0066     bool operator()( CharT c1, CharT c2 )
0067     {
0068 #ifdef BOOST_CLASSIC_IOSTREAMS
0069         return std::string_char_traits<CharT>::eq( c1, c2 );
0070 #else
0071         return std::char_traits<CharT>::eq( c1, c2 );
0072 #endif
0073     }
0074 };
0075
0076 // ************************************************************************** //
0077 // **************                 delim_policy                 ************** //
0078 // ************************************************************************** //
0079
0080 template<typename CharT,typename CharCompare>
0081 class delim_policy {
0082     typedef basic_cstring<CharT const> cstring;
0083 public:
0084     // Constructor
0085     explicit    delim_policy( ti_delimeter_type type_ = dt_char, cstring delimeters_ = cstring() )
0086     : m_type( type_ )
0087     {
0088         set_delimeters( delimeters_ );
0089     }
0090
0091     void        set_delimeters( ti_delimeter_type type_ ) { m_type = type_; }
0092     void        set_delimeters( cstring delimeters_ )
0093     {
0094         m_delimeters = delimeters_;
0095
0096         if( !m_delimeters.is_empty() )
0097             m_type = dt_char;
0098     }
0099     void        set_delimeters( nfp::nil ) {}
0100     bool        operator()( CharT c )
0101     {
0102         switch( m_type ) {
0103         case dt_char: {
0104             BOOST_TEST_FOREACH( CharT, delim, m_delimeters )
0105                 if( CharCompare()( delim, c ) )
0106                     return true;
0107
0108             return false;
0109         }
0110         case dt_ispunct:
0111             return (std::ispunct)( c ) != 0;
0112         case dt_isspace:
0113             return (std::isspace)( c ) != 0;
0114         case dt_none:
0115             return false;
0116         }
0117
0118         return false;
0119     }
0120
0121 private:
0122     // Data members
0123     cstring             m_delimeters;
0124     ti_delimeter_type   m_type;
0125 };
0126
0127 // ************************************************************************** //
0128 // **************                 token_assigner               ************** //
0129 // ************************************************************************** //
0130
0131 template<typename TraversalTag>
0132 struct token_assigner {
0133 #if BOOST_WORKAROUND( BOOST_DINKUMWARE_STDLIB, < 306 )
0134     template<typename Iterator, typename C, typename T>
0135     static void assign( Iterator b, Iterator e, std::basic_string<C,T>& t )
0136     { for( ; b != e; ++b ) t += *b; }
0137
0138     template<typename Iterator, typename C>
0139     static void assign( Iterator b, Iterator e, basic_cstring<C>& t )  { t.assign( b, e ); }
0140 #else
0141     template<typename Iterator, typename Token>
0142     static void assign( Iterator b, Iterator e, Token& t )  { t.assign( b, e ); }
0143 #endif
0144     template<typename Iterator, typename Token>
0145     static void append_move( Iterator& b, Token& )          { ++b; }
0146 };
0147
0148 //____________________________________________________________________________//
0149
0150 template<>
0151 struct token_assigner<single_pass_traversal_tag> {
0152     template<typename Iterator, typename Token>
0153     static void assign( Iterator /*b*/, Iterator /*e*/, Token& /*t*/ )  {}
0154
0155     template<typename Iterator, typename Token>
0156     static void append_move( Iterator& b, Token& t )        { t += *b; ++b; }
0157 };
0158
0159 } // namespace ut_detail
0160
0161 // ************************************************************************** //
0162 // **************                  modifiers                   ************** //
0163 // ************************************************************************** //
0164
0165 namespace {
0166 nfp::keyword<struct dropped_delimeters_t >           dropped_delimeters;
0167 nfp::keyword<struct kept_delimeters_t >              kept_delimeters;
0168 nfp::typed_keyword<bool,struct keep_empty_tokens_t > keep_empty_tokens;
0169 nfp::typed_keyword<std::size_t,struct max_tokens_t > max_tokens;
0170 }
0171
0172 // ************************************************************************** //
0173 // **************             token_iterator_base              ************** //
0174 // ************************************************************************** //
0175
0176 template<typename Derived,
0177          typename CharT,
0178          typename CharCompare   = ut_detail::default_char_compare<CharT>,
0179          typename ValueType     = basic_cstring<CharT const>,
0180          typename Reference     = basic_cstring<CharT const>,
0181          typename Traversal     = forward_traversal_tag>
0182 class token_iterator_base
0183 : public input_iterator_facade<Derived,ValueType,Reference,Traversal> {
0184     typedef basic_cstring<CharT const>                                      cstring;
0185     typedef ut_detail::delim_policy<CharT,CharCompare>                      delim_policy;
0186     typedef input_iterator_facade<Derived,ValueType,Reference,Traversal>    base;
0187
0188 protected:
0189     // Constructor
0190     explicit    token_iterator_base()
0191     : m_is_dropped( dt_isspace )
0192     , m_is_kept( dt_ispunct )
0193     , m_keep_empty_tokens( false )
0194     , m_tokens_left( static_cast<std::size_t>(-1) )
0195     , m_token_produced( false )
0196     {
0197     }
0198
0199     template<typename Modifier>
0200     void
0201     apply_modifier( Modifier const& m )
0202     {
0203         if( m.has( dropped_delimeters ) )
0204             m_is_dropped.set_delimeters( m[dropped_delimeters] );
0205
0206         if( m.has( kept_delimeters ) )
0207             m_is_kept.set_delimeters( m[kept_delimeters] );
0208
0209         if( m.has( keep_empty_tokens ) )
0210             m_keep_empty_tokens = true;
0211
0212         nfp::opt_assign( m_tokens_left, m, max_tokens );
0213     }
0214
0215     template<typename Iter>
0216     bool                    get( Iter& begin, Iter end )
0217     {
0218         typedef ut_detail::token_assigner<BOOST_DEDUCED_TYPENAME iterator_traversal<Iter>::type> Assigner;
0219         Iter check_point;
0220
0221         this->m_value.clear();
0222
0223         if( !m_keep_empty_tokens ) {
0224             while( begin != end && m_is_dropped( *begin ) )
0225                 ++begin;
0226
0227             if( begin == end )
0228                 return false;
0229
0230             check_point = begin;
0231
0232             if( m_tokens_left == 1 )
0233                 while( begin != end )
0234                     Assigner::append_move( begin, this->m_value );
0235             else if( m_is_kept( *begin ) )
0236                 Assigner::append_move( begin, this->m_value );
0237             else
0238                 while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
0239                     Assigner::append_move( begin, this->m_value );
0240
0241             --m_tokens_left;
0242         }
0243         else { // m_keep_empty_tokens is true
0244             check_point = begin;
0245
0246             if( begin == end ) {
0247                 if( m_token_produced )
0248                     return false;
0249
0250                 m_token_produced = true;
0251             }
0252             if( m_is_kept( *begin ) ) {
0253                 if( m_token_produced )
0254                     Assigner::append_move( begin, this->m_value );
0255
0256                 m_token_produced = !m_token_produced;
0257             }
0258             else if( !m_token_produced && m_is_dropped( *begin ) )
0259                 m_token_produced = true;
0260             else {
0261                 if( m_is_dropped( *begin ) )
0262                     check_point = ++begin;
0263
0264                 while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
0265                     Assigner::append_move( begin, this->m_value );
0266
0267                 m_token_produced = true;
0268             }
0269         }
0270
0271         Assigner::assign( check_point, begin, this->m_value );
0272
0273         return true;
0274     }
0275
0276 private:
0277     // Data members
0278     delim_policy            m_is_dropped;
0279     delim_policy            m_is_kept;
0280     bool                    m_keep_empty_tokens;
0281     std::size_t             m_tokens_left;
0282     bool                    m_token_produced;
0283 };
0284
0285 // ************************************************************************** //
0286 // **************          basic_string_token_iterator         ************** //
0287 // ************************************************************************** //
0288
0289 template<typename CharT,
0290          typename CharCompare = ut_detail::default_char_compare<CharT> >
0291 class basic_string_token_iterator
0292 : public token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> {
0293     typedef basic_cstring<CharT const> cstring;
0294     typedef token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> base;
0295 public:
0296     explicit    basic_string_token_iterator() {}
0297     explicit    basic_string_token_iterator( cstring src )
0298     : m_src( src )
0299     {
0300         this->init();
0301     }
0302
0303     // warning: making the constructor accept anything else than a cstring should
0304     // ensure that no temporary object is created during string creation (previous
0305     // definition was "template<typename Src, typename Modifier> basic_string_token_iterator( Src src ..."
0306     // which may create a temporary string copy when called with an std::string.
0307     template<typename Modifier>
0308     basic_string_token_iterator( cstring src, Modifier const& m )
0309     : m_src( src )
0310     {
0311         this->apply_modifier( m );
0312
0313         this->init();
0314     }
0315
0316 private:
0317     friend class input_iterator_core_access;
0318
0319     // input iterator implementation
0320     bool        get()
0321     {
0322         typename cstring::iterator begin = m_src.begin();
0323         bool res = base::get( begin, m_src.end() );
0324
0325         m_src.assign( begin, m_src.end() );
0326
0327         return res;
0328     }
0329
0330     // Data members
0331     cstring     m_src;
0332 };
0333
0334 typedef basic_string_token_iterator<char>       string_token_iterator;
0335 typedef basic_string_token_iterator<wchar_t>    wstring_token_iterator;
0336
0337 // ************************************************************************** //
0338 // **************              range_token_iterator            ************** //
0339 // ************************************************************************** //
0340
0341 template<typename Iter,
0342          typename CharCompare = ut_detail::default_char_compare<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
0343          typename ValueType   = std::basic_string<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
0344          typename Reference   = ValueType const&>
0345 class range_token_iterator
0346 : public token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
0347                              typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> {
0348     typedef basic_cstring<typename ValueType::value_type> cstring;
0349     typedef token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
0350                                 typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> base;
0351 public:
0352     explicit    range_token_iterator() {}
0353     explicit    range_token_iterator( Iter begin, Iter end = Iter() )
0354     : m_begin( begin ), m_end( end )
0355     {
0356         this->init();
0357     }
0358     range_token_iterator( range_token_iterator const& rhs )
0359     : base( rhs )
0360     {
0361         if( this->m_valid ) {
0362             m_begin = rhs.m_begin;
0363             m_end   = rhs.m_end;
0364         }
0365     }
0366
0367     template<typename Modifier>
0368     range_token_iterator( Iter begin, Iter end, Modifier const& m )
0369     : m_begin( begin ), m_end( end )
0370     {
0371         this->apply_modifier( m );
0372
0373         this->init();
0374     }
0375
0376 private:
0377     friend class input_iterator_core_access;
0378
0379     // input iterator implementation
0380     bool        get()
0381     {
0382         return base::get( m_begin, m_end );
0383     }
0384
0385     // Data members
0386     Iter m_begin;
0387     Iter m_end;
0388 };
0389
0390 // ************************************************************************** //
0391 // **************            make_range_token_iterator         ************** //
0392 // ************************************************************************** //
0393
0394 template<typename Iter>
0395 inline range_token_iterator<Iter>
0396 make_range_token_iterator( Iter begin, Iter end = Iter() )
0397 {
0398     return range_token_iterator<Iter>( begin, end );
0399 }
0400
0401 //____________________________________________________________________________//
0402
0403 template<typename Iter,typename Modifier>
0404 inline range_token_iterator<Iter>
0405 make_range_token_iterator( Iter begin, Iter end, Modifier const& m )
0406 {
0407     return range_token_iterator<Iter>( begin, end, m );
0408 }
0409
0410 //____________________________________________________________________________//
0411
0412 } // namespace utils
0413 } // namespace unit_test
0414 } // namespace boost
0415
0416 //____________________________________________________________________________//
0417
0418 #include <boost/test/detail/enable_warnings.hpp>
0419
0420 #endif // BOOST_TEST_UTILS_TOKEN_ITERATOR_HPP
0421