lexer/lexertl/iterator_tokenizer.hpp

0001 //  Copyright (c) 2001-2011 Hartmut Kaiser
0002 //
0003 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
0004 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
0005
0006 #if !defined(BOOST_SPIRIT_LEXERTL_ITERATOR_TOKENISER_MARCH_22_2007_0859AM)
0007 #define BOOST_SPIRIT_LEXERTL_ITERATOR_TOKENISER_MARCH_22_2007_0859AM
0008
0009 #if defined(_MSC_VER)
0010 #pragma once
0011 #endif
0012
0013 #include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
0014 #include <boost/spirit/home/support/detail/lexer/consts.hpp>
0015 #include <boost/spirit/home/support/detail/lexer/size_t.hpp>
0016 #include <boost/spirit/home/support/detail/lexer/char_traits.hpp>
0017 #include <iterator> // for std::iterator_traits
0018 #include <vector>
0019
0020 namespace boost { namespace spirit { namespace lex { namespace lexertl
0021 {
0022     ///////////////////////////////////////////////////////////////////////////
0023     template<typename Iterator>
0024     class basic_iterator_tokeniser
0025     {
0026     public:
0027         typedef std::vector<std::size_t> size_t_vector;
0028         typedef typename std::iterator_traits<Iterator>::value_type char_type;
0029
0030         static std::size_t next (
0031             boost::lexer::basic_state_machine<char_type> const& state_machine_
0032           , std::size_t &dfa_state_, bool& bol_, Iterator &start_token_
0033           , Iterator const& end_, std::size_t& unique_id_)
0034         {
0035             if (start_token_ == end_)
0036             {
0037                 unique_id_ = boost::lexer::npos;
0038                 return 0;
0039             }
0040
0041             bool bol = bol_;
0042             boost::lexer::detail::internals const& internals_ =
0043                 state_machine_.data();
0044
0045         again:
0046             std::size_t const* lookup_ = &internals_._lookup[dfa_state_]->
0047                 front ();
0048             std::size_t dfa_alphabet_ = internals_._dfa_alphabet[dfa_state_];
0049             std::size_t const* dfa_ = &internals_._dfa[dfa_state_]->front ();
0050
0051             std::size_t const* ptr_ = dfa_ + dfa_alphabet_;
0052             Iterator curr_ = start_token_;
0053             bool end_state_ = *ptr_ != 0;
0054             std::size_t id_ = *(ptr_ + boost::lexer::id_index);
0055             std::size_t uid_ = *(ptr_ + boost::lexer::unique_id_index);
0056             std::size_t end_start_state_ = dfa_state_;
0057             bool end_bol_ = bol_;
0058             Iterator end_token_ = start_token_;
0059
0060             while (curr_ != end_)
0061             {
0062                 std::size_t const BOL_state_ = ptr_[boost::lexer::bol_index];
0063                 std::size_t const EOL_state_ = ptr_[boost::lexer::eol_index];
0064
0065                 if (BOL_state_ && bol)
0066                 {
0067                     ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
0068                 }
0069                 else if (EOL_state_ && *curr_ == '\n')
0070                 {
0071                     ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
0072                 }
0073                 else
0074                 {
0075                     typedef typename
0076                         std::iterator_traits<Iterator>::value_type
0077                     value_type;
0078                     typedef typename
0079                         boost::lexer::char_traits<value_type>::index_type
0080                     index_type;
0081
0082                     index_type index =
0083                         boost::lexer::char_traits<value_type>::call(*curr_++);
0084                     bol = (index == '\n') ? true : false;
0085                     std::size_t const state_ = ptr_[
0086                         lookup_[static_cast<std::size_t>(index)]];
0087
0088                     if (state_ == 0)
0089                     {
0090                         break;
0091                     }
0092
0093                     ptr_ = &dfa_[state_ * dfa_alphabet_];
0094                 }
0095
0096                 if (*ptr_)
0097                 {
0098                     end_state_ = true;
0099                     id_ = *(ptr_ + boost::lexer::id_index);
0100                     uid_ = *(ptr_ + boost::lexer::unique_id_index);
0101                     end_start_state_ = *(ptr_ + boost::lexer::state_index);
0102                     end_bol_ = bol;
0103                     end_token_ = curr_;
0104                 }
0105             }
0106
0107             std::size_t const EOL_state_ = ptr_[boost::lexer::eol_index];
0108
0109             if (EOL_state_ && curr_ == end_)
0110             {
0111                 ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
0112
0113                 if (*ptr_)
0114                 {
0115                     end_state_ = true;
0116                     id_ = *(ptr_ + boost::lexer::id_index);
0117                     uid_ = *(ptr_ + boost::lexer::unique_id_index);
0118                     end_start_state_ = *(ptr_ + boost::lexer::state_index);
0119                     end_bol_ = bol;
0120                     end_token_ = curr_;
0121                 }
0122             }
0123
0124             if (end_state_) {
0125                 // return longest match
0126                 dfa_state_ = end_start_state_;
0127                 start_token_ = end_token_;
0128
0129                 if (id_ == 0)
0130                 {
0131                     bol = end_bol_;
0132                     goto again;
0133                 }
0134                 else
0135                 {
0136                     bol_ = end_bol_;
0137                 }
0138             }
0139             else {
0140                 bol_ = (*start_token_ == '\n') ? true : false;
0141                 id_ = boost::lexer::npos;
0142                 uid_ = boost::lexer::npos;
0143             }
0144
0145             unique_id_ = uid_;
0146             return id_;
0147         }
0148
0149         ///////////////////////////////////////////////////////////////////////
0150         static std::size_t next (
0151             boost::lexer::basic_state_machine<char_type> const& state_machine_
0152           , bool& bol_, Iterator &start_token_, Iterator const& end_
0153           , std::size_t& unique_id_)
0154         {
0155             if (start_token_ == end_)
0156             {
0157                 unique_id_ = boost::lexer::npos;
0158                 return 0;
0159             }
0160
0161             bool bol = bol_;
0162             std::size_t const* lookup_ = &state_machine_.data()._lookup[0]->front();
0163             std::size_t dfa_alphabet_ = state_machine_.data()._dfa_alphabet[0];
0164             std::size_t const* dfa_ = &state_machine_.data()._dfa[0]->front ();
0165             std::size_t const* ptr_ = dfa_ + dfa_alphabet_;
0166
0167             Iterator curr_ = start_token_;
0168             bool end_state_ = *ptr_ != 0;
0169             std::size_t id_ = *(ptr_ + boost::lexer::id_index);
0170             std::size_t uid_ = *(ptr_ + boost::lexer::unique_id_index);
0171             bool end_bol_ = bol_;
0172             Iterator end_token_ = start_token_;
0173
0174             while (curr_ != end_)
0175             {
0176                 std::size_t const BOL_state_ = ptr_[boost::lexer::bol_index];
0177                 std::size_t const EOL_state_ = ptr_[boost::lexer::eol_index];
0178
0179                 if (BOL_state_ && bol)
0180                 {
0181                     ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
0182                 }
0183                 else if (EOL_state_ && *curr_ == '\n')
0184                 {
0185                     ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
0186                 }
0187                 else
0188                 {
0189                     typedef typename
0190                         std::iterator_traits<Iterator>::value_type
0191                     value_type;
0192                     typedef typename
0193                         boost::lexer::char_traits<value_type>::index_type
0194                     index_type;
0195
0196                     index_type index =
0197                         boost::lexer::char_traits<value_type>::call(*curr_++);
0198                     bol = (index == '\n') ? true : false;
0199                     std::size_t const state_ = ptr_[
0200                         lookup_[static_cast<std::size_t>(index)]];
0201
0202                     if (state_ == 0)
0203                     {
0204                         break;
0205                     }
0206
0207                     ptr_ = &dfa_[state_ * dfa_alphabet_];
0208                 }
0209
0210                 if (*ptr_)
0211                 {
0212                     end_state_ = true;
0213                     id_ = *(ptr_ + boost::lexer::id_index);
0214                     uid_ = *(ptr_ + boost::lexer::unique_id_index);
0215                     end_bol_ = bol;
0216                     end_token_ = curr_;
0217                 }
0218             }
0219
0220             std::size_t const EOL_state_ = ptr_[boost::lexer::eol_index];
0221
0222             if (EOL_state_ && curr_ == end_)
0223             {
0224                 ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
0225
0226                 if (*ptr_)
0227                 {
0228                     end_state_ = true;
0229                     id_ = *(ptr_ + boost::lexer::id_index);
0230                     uid_ = *(ptr_ + boost::lexer::unique_id_index);
0231                     end_bol_ = bol;
0232                     end_token_ = curr_;
0233                 }
0234             }
0235
0236             if (end_state_) {
0237                 // return longest match
0238                 bol_ = end_bol_;
0239                 start_token_ = end_token_;
0240             }
0241             else {
0242                 bol_ = *start_token_ == '\n';
0243                 id_ = boost::lexer::npos;
0244                 uid_ = boost::lexer::npos;
0245             }
0246
0247             unique_id_ = uid_;
0248             return id_;
0249         }
0250     };
0251
0252 }}}}
0253
0254 #endif