include/boost/token_iterator.hpp

0001 // Boost token_iterator.hpp  -------------------------------------------------//
0002
0003 // Copyright John R. Bandela 2001
0004 // Distributed under the Boost Software License, Version 1.0. (See
0005 // accompanying file LICENSE_1_0.txt or copy at
0006 // http://www.boost.org/LICENSE_1_0.txt)
0007
0008 // See http://www.boost.org/libs/tokenizer for documentation.
0009
0010 // Revision History:
0011 // 16 Jul 2003   John Bandela
0012 //      Allowed conversions from convertible base iterators
0013 // 03 Jul 2003   John Bandela
0014 //      Converted to new iterator adapter
0015
0016
0017
0018 #ifndef BOOST_TOKENIZER_POLICY_JRB070303_HPP_
0019 #define BOOST_TOKENIZER_POLICY_JRB070303_HPP_
0020
0021 #include <boost/assert.hpp>
0022 #include <boost/iterator/iterator_adaptor.hpp>
0023 #include <boost/iterator/minimum_category.hpp>
0024 #include <boost/token_functions.hpp>
0025 #include <utility>
0026
0027 namespace boost
0028 {
0029   template <class TokenizerFunc, class Iterator, class Type>
0030   class token_iterator
0031       : public iterator_facade<
0032             token_iterator<TokenizerFunc, Iterator, Type>
0033           , Type
0034           , typename iterators::minimum_category<
0035                 forward_traversal_tag
0036               , typename iterator_traversal<Iterator>::type
0037             >::type
0038           , const Type&
0039         >
0040   {
0041
0042 #ifdef __DCC__
0043       friend class boost::iterator_core_access;
0044 #else
0045       friend class iterator_core_access;
0046 #endif
0047       TokenizerFunc f_;
0048       Iterator begin_;
0049       Iterator end_;
0050       bool valid_;
0051       Type tok_;
0052
0053       void increment(){
0054           BOOST_ASSERT(valid_);
0055           valid_ = f_(begin_,end_,tok_);
0056       }
0057
0058       const Type&  dereference() const {
0059           BOOST_ASSERT(valid_);
0060           return tok_;
0061       }
0062       template<class Other>
0063       bool equal(const Other& a) const{
0064           return (a.valid_ && valid_)
0065               ?( (a.begin_==begin_) && (a.end_ == end_) )
0066               :(a.valid_==valid_);
0067
0068       }
0069
0070       void initialize(){
0071           if(valid_) return;
0072           f_.reset();
0073           valid_ = (begin_ != end_)?
0074               f_(begin_,end_,tok_):false;
0075       }
0076   public:
0077       token_iterator():begin_(),end_(),valid_(false),tok_() { }
0078
0079       token_iterator(TokenizerFunc f, Iterator begin, Iterator e = Iterator())
0080           : f_(f),begin_(begin),end_(e),valid_(false),tok_(){ initialize(); }
0081
0082       token_iterator(Iterator begin, Iterator e = Iterator())
0083             : f_(),begin_(begin),end_(e),valid_(false),tok_() {initialize();}
0084
0085       template<class OtherIter>
0086       token_iterator(
0087             token_iterator<TokenizerFunc, OtherIter,Type> const& t
0088             , typename enable_if_convertible<OtherIter, Iterator>::type* = 0)
0089             : f_(t.tokenizer_function()),begin_(t.base())
0090             ,end_(t.end()),valid_(!t.at_end()),tok_(t.current_token()) {}
0091
0092       Iterator base()const{return begin_;}
0093
0094       Iterator end()const{return end_;}
0095
0096       TokenizerFunc tokenizer_function()const{return f_;}
0097
0098       Type current_token()const{return tok_;}
0099
0100       bool at_end()const{return !valid_;}
0101
0102
0103
0104
0105   };
0106     template <
0107         class TokenizerFunc = char_delimiters_separator<char>,
0108         class Iterator = std::string::const_iterator,
0109         class Type = std::string
0110     >
0111     class token_iterator_generator {
0112
0113     private:
0114     public:
0115         typedef token_iterator<TokenizerFunc,Iterator,Type> type;
0116     };
0117
0118
0119     // Type has to be first because it needs to be explicitly specified
0120     // because there is no way the function can deduce it.
0121     template<class Type, class Iterator, class TokenizerFunc>
0122         typename token_iterator_generator<TokenizerFunc,Iterator,Type>::type
0123     make_token_iterator(Iterator begin, Iterator end,const TokenizerFunc& fun){
0124         typedef typename
0125             token_iterator_generator<TokenizerFunc,Iterator,Type>::type ret_type;
0126         return ret_type(fun,begin,end);
0127     }
0128
0129 } // namespace boost
0130
0131 #endif