lex/lexer/token_def.hpp

0001 //  Copyright (c) 2001-2011 Hartmut Kaiser
0002 //
0003 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
0004 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
0005
0006 #if !defined(BOOST_SPIRIT_LEX_TOKEN_DEF_MAR_13_2007_0145PM)
0007 #define BOOST_SPIRIT_LEX_TOKEN_DEF_MAR_13_2007_0145PM
0008
0009 #if defined(_MSC_VER)
0010 #pragma once
0011 #endif
0012
0013 #include <boost/spirit/home/support/unused.hpp>
0014 #include <boost/spirit/home/support/argument.hpp>
0015 #include <boost/spirit/home/support/info.hpp>
0016 #include <boost/spirit/home/support/handles_container.hpp>
0017 #include <boost/spirit/home/qi/parser.hpp>
0018 #include <boost/spirit/home/qi/skip_over.hpp>
0019 #include <boost/spirit/home/qi/detail/construct.hpp>
0020 #include <boost/spirit/home/qi/detail/assign_to.hpp>
0021 #include <boost/spirit/home/lex/reference.hpp>
0022 #include <boost/spirit/home/lex/lexer_type.hpp>
0023 #include <boost/spirit/home/lex/lexer/terminals.hpp>
0024
0025 #include <boost/fusion/include/vector.hpp>
0026 #include <boost/mpl/if.hpp>
0027 #include <boost/proto/extends.hpp>
0028 #include <boost/proto/traits.hpp>
0029 #include <boost/type_traits/is_same.hpp>
0030 #include <boost/variant.hpp>
0031
0032 #include <iterator> // for std::iterator_traits
0033 #include <string>
0034 #include <cstdlib>
0035
0036 #if defined(BOOST_MSVC)
0037 # pragma warning(push)
0038 # pragma warning(disable: 4355) // 'this' : used in base member initializer list warning
0039 #endif
0040
0041 namespace boost { namespace spirit { namespace lex
0042 {
0043     ///////////////////////////////////////////////////////////////////////////
0044     //  This component represents a token definition
0045     ///////////////////////////////////////////////////////////////////////////
0046     template<typename Attribute = unused_type
0047       , typename Char = char
0048       , typename Idtype = std::size_t>
0049     struct token_def
0050       : proto::extends<
0051             typename proto::terminal<
0052                 lex::reference<token_def<Attribute, Char, Idtype> const, Idtype>
0053             >::type
0054           , token_def<Attribute, Char, Idtype> >
0055       , qi::parser<token_def<Attribute, Char, Idtype> >
0056       , lex::lexer_type<token_def<Attribute, Char, Idtype> >
0057     {
0058     private:
0059         // initialize proto base class
0060         typedef lex::reference<token_def const, Idtype> reference_;
0061         typedef typename proto::terminal<reference_>::type terminal_type;
0062         typedef proto::extends<terminal_type, token_def> proto_base_type;
0063
0064         static std::size_t const all_states_id = static_cast<std::size_t>(-2);
0065
0066     public:
0067         // Qi interface: meta-function calculating parser return type
0068         template <typename Context, typename Iterator>
0069         struct attribute
0070         {
0071             //  The return value of the token_def is either the specified
0072             //  attribute type, or the pair of iterators from the match of the
0073             //  corresponding token (if no attribute type has been specified),
0074             //  or unused_type (if omit has been specified).
0075             typedef typename Iterator::base_iterator_type iterator_type;
0076             typedef typename mpl::if_<
0077                 traits::not_is_unused<Attribute>
0078               , typename mpl::if_<
0079                     is_same<Attribute, lex::omit>, unused_type, Attribute
0080                 >::type
0081               , iterator_range<iterator_type>
0082             >::type type;
0083         };
0084
0085     public:
0086         // Qi interface: parse functionality
0087         template <typename Iterator, typename Context
0088           , typename Skipper, typename Attribute_>
0089         bool parse(Iterator& first, Iterator const& last
0090           , Context& /*context*/, Skipper const& skipper
0091           , Attribute_& attr) const
0092         {
0093             qi::skip_over(first, last, skipper);   // always do a pre-skip
0094
0095             if (first != last) {
0096                 typedef typename
0097                     std::iterator_traits<Iterator>::value_type
0098                 token_type;
0099
0100                 //  If the following assertion fires you probably forgot to
0101                 //  associate this token definition with a lexer instance.
0102                 BOOST_ASSERT(std::size_t(~0) != token_state_);
0103
0104                 token_type const& t = *first;
0105                 if (token_id_ == t.id() &&
0106                     (all_states_id == token_state_ || token_state_ == t.state()))
0107                 {
0108                     spirit::traits::assign_to(t, attr);
0109                     ++first;
0110                     return true;
0111                 }
0112             }
0113             return false;
0114         }
0115
0116         template <typename Context>
0117         info what(Context& /*context*/) const
0118         {
0119             if (0 == def_.which())
0120                 return info("token_def", boost::get<string_type>(def_));
0121
0122             return info("token_def", boost::get<char_type>(def_));
0123         }
0124
0125         ///////////////////////////////////////////////////////////////////////
0126         // Lex interface: collect token definitions and put it into the
0127         // provided lexer def
0128         template <typename LexerDef, typename String>
0129         void collect(LexerDef& lexdef, String const& state
0130           , String const& targetstate) const
0131         {
0132             std::size_t state_id = lexdef.add_state(state.c_str());
0133
0134             // If the following assertion fires you are probably trying to use
0135             // a single token_def instance in more than one lexer state. This
0136             // is not possible. Please create a separate token_def instance
0137             // from the same regular expression for each lexer state it needs
0138             // to be associated with.
0139             BOOST_ASSERT(
0140                 (std::size_t(~0) == token_state_ || state_id == token_state_) &&
0141                 "Can't use single token_def with more than one lexer state");
0142
0143             char_type const* target = targetstate.empty() ? 0 : targetstate.c_str();
0144             if (target)
0145                 lexdef.add_state(target);
0146
0147             token_state_ = state_id;
0148             if (0 == token_id_)
0149                 token_id_ = lexdef.get_next_id();
0150
0151             if (0 == def_.which()) {
0152                 unique_id_ = lexdef.add_token(state.c_str()
0153                   , boost::get<string_type>(def_), token_id_, target);
0154             }
0155             else {
0156                 unique_id_ = lexdef.add_token(state.c_str()
0157                   , boost::get<char_type>(def_), token_id_, target);
0158             }
0159         }
0160
0161         template <typename LexerDef>
0162         void add_actions(LexerDef&) const {}
0163
0164     public:
0165         typedef Char char_type;
0166         typedef Idtype id_type;
0167         typedef std::basic_string<char_type> string_type;
0168
0169         // Lex interface: constructing token definitions
0170         token_def()
0171           : proto_base_type(terminal_type::make(reference_(*this)))
0172           , def_('\0'), token_id_()
0173           , unique_id_(std::size_t(~0)), token_state_(std::size_t(~0)) {}
0174
0175         token_def(token_def const& rhs)
0176           : proto_base_type(terminal_type::make(reference_(*this)))
0177           , def_(rhs.def_), token_id_(rhs.token_id_)
0178           , unique_id_(rhs.unique_id_), token_state_(rhs.token_state_) {}
0179
0180         explicit token_def(char_type def_, Idtype id_ = Idtype())
0181           : proto_base_type(terminal_type::make(reference_(*this)))
0182           , def_(def_)
0183           , token_id_(Idtype() == id_ ? Idtype(def_) : id_)
0184           , unique_id_(std::size_t(~0)), token_state_(std::size_t(~0)) {}
0185
0186         explicit token_def(string_type const& def_, Idtype id_ = Idtype())
0187           : proto_base_type(terminal_type::make(reference_(*this)))
0188           , def_(def_), token_id_(id_)
0189           , unique_id_(std::size_t(~0)), token_state_(std::size_t(~0)) {}
0190
0191         template <typename String>
0192         token_def& operator= (String const& definition)
0193         {
0194             def_ = definition;
0195             token_id_ = Idtype();
0196             unique_id_ = std::size_t(~0);
0197             token_state_ = std::size_t(~0);
0198             return *this;
0199         }
0200         token_def& operator= (token_def const& rhs)
0201         {
0202             def_ = rhs.def_;
0203             token_id_ = rhs.token_id_;
0204             unique_id_ = rhs.unique_id_;
0205             token_state_ = rhs.token_state_;
0206             return *this;
0207         }
0208
0209         // general accessors
0210         Idtype const& id() const { return token_id_; }
0211         void id(Idtype const& id) { token_id_ = id; }
0212         std::size_t unique_id() const { return unique_id_; }
0213
0214         string_type definition() const
0215         {
0216             return (0 == def_.which()) ?
0217                 boost::get<string_type>(def_) :
0218                 string_type(1, boost::get<char_type>(def_));
0219         }
0220         std::size_t state() const { return token_state_; }
0221
0222     private:
0223         variant<string_type, char_type> def_;
0224         mutable Idtype token_id_;
0225         mutable std::size_t unique_id_;
0226         mutable std::size_t token_state_;
0227     };
0228 }}}
0229
0230 namespace boost { namespace spirit { namespace traits
0231 {
0232     ///////////////////////////////////////////////////////////////////////////
0233     template<typename Attribute, typename Char, typename Idtype
0234       , typename Attr, typename Context, typename Iterator>
0235     struct handles_container<
0236             lex::token_def<Attribute, Char, Idtype>, Attr, Context, Iterator>
0237       : traits::is_container<
0238             typename attribute_of<
0239                 lex::token_def<Attribute, Char, Idtype>, Context, Iterator
0240             >::type>
0241     {};
0242 }}}
0243
0244 #if defined(BOOST_MSVC)
0245 # pragma warning(pop)
0246 #endif
0247
0248 #endif