Back to home page

EIC code displayed by LXR

 
 

    


File indexing completed on 2025-02-27 09:55:21

0001 //  Copyright (c) 2001-2011 Hartmut Kaiser
0002 // 
0003 //  Distributed under the Boost Software License, Version 1.0. (See accompanying 
0004 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
0005 
0006 #if !defined(BOOST_SPIRIT_LEX_LEXER_MAR_13_2007_0145PM)
0007 #define BOOST_SPIRIT_LEX_LEXER_MAR_13_2007_0145PM
0008 
0009 #if defined(_MSC_VER)
0010 #pragma once
0011 #endif
0012 
0013 #include <boost/spirit/home/support/info.hpp>
0014 #include <boost/spirit/home/qi/skip_over.hpp>
0015 #include <boost/spirit/home/qi/parser.hpp>
0016 #include <boost/spirit/home/qi/detail/assign_to.hpp>
0017 #include <boost/spirit/home/lex/reference.hpp>
0018 #include <boost/spirit/home/lex/meta_compiler.hpp>
0019 #include <boost/spirit/home/lex/lexer_type.hpp>
0020 #include <boost/spirit/home/lex/lexer/token_def.hpp>
0021 #include <boost/assert.hpp>
0022 #include <boost/noncopyable.hpp>
0023 #include <boost/fusion/include/vector.hpp>
0024 #include <boost/mpl/assert.hpp>
0025 #include <boost/proto/extends.hpp>
0026 #include <boost/proto/traits.hpp>
0027 #include <boost/range/iterator_range_core.hpp>
0028 #include <iterator> // for std::iterator_traits
0029 #include <string>
0030 
0031 namespace boost { namespace spirit { namespace lex
0032 {
0033     ///////////////////////////////////////////////////////////////////////////
0034     namespace detail
0035     {
0036         ///////////////////////////////////////////////////////////////////////
0037 #ifdef _MSC_VER
0038 #  pragma warning(push)
0039 #  pragma warning(disable: 4512) // assignment operator could not be generated.
0040 #endif
0041         template <typename LexerDef>
0042         struct lexer_def_
0043           : proto::extends<
0044                 typename proto::terminal<
0045                    lex::reference<lexer_def_<LexerDef> const> 
0046                 >::type
0047               , lexer_def_<LexerDef> >
0048           , qi::parser<lexer_def_<LexerDef> >
0049           , lex::lexer_type<lexer_def_<LexerDef> >
0050         {
0051         private:
0052             // avoid warnings about using 'this' in constructor
0053             lexer_def_& this_() { return *this; }
0054 
0055             typedef typename LexerDef::char_type char_type;
0056             typedef typename LexerDef::string_type string_type;
0057             typedef typename LexerDef::id_type id_type;
0058 
0059             typedef lex::reference<lexer_def_ const> reference_;
0060             typedef typename proto::terminal<reference_>::type terminal_type;
0061             typedef proto::extends<terminal_type, lexer_def_> proto_base_type;
0062 
0063             reference_ alias() const
0064             {
0065                 return reference_(*this);
0066             }
0067 
0068         public:
0069             // Qi interface: metafunction calculating parser attribute type
0070             template <typename Context, typename Iterator>
0071             struct attribute
0072             {
0073                 //  the return value of a token set contains the matched token 
0074                 //  id, and the corresponding pair of iterators
0075                 typedef typename Iterator::base_iterator_type iterator_type;
0076                 typedef 
0077                     fusion::vector2<id_type, iterator_range<iterator_type> > 
0078                 type;
0079             };
0080 
0081             // Qi interface: parse functionality
0082             template <typename Iterator, typename Context
0083               , typename Skipper, typename Attribute>
0084             bool parse(Iterator& first, Iterator const& last
0085               , Context& /*context*/, Skipper const& skipper
0086               , Attribute& attr) const
0087             {
0088                 qi::skip_over(first, last, skipper);   // always do a pre-skip
0089 
0090                 if (first != last) {
0091                     typedef typename 
0092                         std::iterator_traits<Iterator>::value_type 
0093                     token_type;
0094 
0095                     token_type const& t = *first;
0096                     if (token_is_valid(t) && t.state() == first.get_state()) {
0097                     // any of the token definitions matched
0098                         spirit::traits::assign_to(t, attr);
0099                         ++first;
0100                         return true;
0101                     }
0102                 }
0103                 return false;
0104             }
0105 
0106             // Qi interface: 'what' functionality
0107             template <typename Context>
0108             info what(Context& /*context*/) const
0109             {
0110                 return info("lexer");
0111             }
0112 
0113         private:
0114             // allow to use the lexer.self.add("regex1", id1)("regex2", id2);
0115             // syntax
0116             struct adder
0117             {
0118                 adder(lexer_def_& def_) 
0119                   : def(def_) {}
0120 
0121                 // Add a token definition based on a single character as given
0122                 // by the first parameter, the second parameter allows to 
0123                 // specify the token id to use for the new token. If no token
0124                 // id is given the character code is used.
0125                 adder const& operator()(char_type c
0126                   , id_type token_id = id_type()) const
0127                 {
0128                     if (id_type() == token_id)
0129                         token_id = static_cast<id_type>(c);
0130                     def.def.add_token (def.state.c_str(), c, token_id
0131                         , def.targetstate.empty() ? 0 : def.targetstate.c_str());
0132                     return *this;
0133                 }
0134 
0135                 // Add a token definition based on a character sequence as 
0136                 // given by the first parameter, the second parameter allows to 
0137                 // specify the token id to use for the new token. If no token
0138                 // id is given this function will generate a unique id to be 
0139                 // used as the token's id.
0140                 adder const& operator()(string_type const& s
0141                   , id_type token_id = id_type()) const
0142                 {
0143                     if (id_type() == token_id)
0144                         token_id = def.def.get_next_id();
0145                     def.def.add_token (def.state.c_str(), s, token_id
0146                         , def.targetstate.empty() ? 0 : def.targetstate.c_str());
0147                     return *this;
0148                 }
0149 
0150                 template <typename Attribute>
0151                 adder const& operator()(
0152                     token_def<Attribute, char_type, id_type>& tokdef
0153                   , id_type token_id = id_type()) const
0154                 {
0155                     // make sure we have a token id
0156                     if (id_type() == token_id) {
0157                         if (id_type() == tokdef.id()) {
0158                             token_id = def.def.get_next_id();
0159                             tokdef.id(token_id);
0160                         }
0161                         else {
0162                             token_id = tokdef.id();
0163                         }
0164                     }
0165                     else { 
0166                     // the following assertion makes sure that the token_def
0167                     // instance has not been assigned a different id earlier
0168                         BOOST_ASSERT(id_type() == tokdef.id() 
0169                                   || token_id == tokdef.id());
0170                         tokdef.id(token_id);
0171                     }
0172 
0173                     def.define(tokdef);
0174                     return *this;
0175                 }
0176 
0177 //                 template <typename F>
0178 //                 adder const& operator()(char_type c, id_type token_id, F act) const
0179 //                 {
0180 //                     if (id_type() == token_id)
0181 //                         token_id = def.def.get_next_id();
0182 //                     std::size_t unique_id = 
0183 //                         def.def.add_token (def.state.c_str(), s, token_id);
0184 //                     def.def.add_action(unique_id, def.state.c_str(), act);
0185 //                     return *this;
0186 //                 }
0187 
0188                 lexer_def_& def;
0189             };
0190             friend struct adder;
0191 
0192             // allow to use lexer.self.add_pattern("pattern1", "regex1")(...);
0193             // syntax
0194             struct pattern_adder
0195             {
0196                 pattern_adder(lexer_def_& def_) 
0197                   : def(def_) {}
0198 
0199                 pattern_adder const& operator()(string_type const& p
0200                   , string_type const& s) const
0201                 {
0202                     def.def.add_pattern (def.state.c_str(), p, s);
0203                     return *this;
0204                 }
0205 
0206                 lexer_def_& def;
0207             };
0208             friend struct pattern_adder;
0209 
0210         private:
0211             // Helper function to invoke the necessary 2 step compilation
0212             // process on token definition expressions
0213             template <typename TokenExpr>
0214             void compile2pass(TokenExpr const& expr) 
0215             {
0216                 expr.collect(def, state, targetstate);
0217                 expr.add_actions(def);
0218             }
0219 
0220         public:
0221             ///////////////////////////////////////////////////////////////////
0222             template <typename Expr>
0223             void define(Expr const& expr)
0224             {
0225                 compile2pass(compile<lex::domain>(expr));
0226             }
0227 
0228             lexer_def_(LexerDef& def_, string_type const& state_
0229                   , string_type const& targetstate_ = string_type())
0230               : proto_base_type(terminal_type::make(alias()))
0231               , add(this_()), add_pattern(this_()), def(def_)
0232               , state(state_), targetstate(targetstate_)
0233             {}
0234 
0235             // allow to switch states
0236             lexer_def_ operator()(char_type const* state_) const
0237             {
0238                 return lexer_def_(def, state_);
0239             }
0240             lexer_def_ operator()(char_type const* state_
0241               , char_type const* targetstate_) const
0242             {
0243                 return lexer_def_(def, state_, targetstate_);
0244             }
0245             lexer_def_ operator()(string_type const& state_
0246               , string_type const& targetstate_ = string_type()) const
0247             {
0248                 return lexer_def_(def, state_, targetstate_);
0249             }
0250 
0251             // allow to assign a token definition expression
0252             template <typename Expr>
0253             lexer_def_& operator= (Expr const& xpr)
0254             {
0255                 // Report invalid expression error as early as possible.
0256                 // If you got an error_invalid_expression error message here,
0257                 // then the expression (expr) is not a valid spirit lex 
0258                 // expression.
0259                 BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
0260 
0261                 def.clear(state.c_str());
0262                 define(xpr);
0263                 return *this;
0264             }
0265 
0266             // explicitly tell the lexer that the given state will be defined
0267             // (useful in conjunction with "*")
0268             std::size_t add_state(char_type const* state_ = 0)
0269             {
0270                 return def.add_state(state_ ? state_ : def.initial_state().c_str());
0271             }
0272 
0273             adder add;
0274             pattern_adder add_pattern;
0275 
0276         private:
0277             LexerDef& def;
0278             string_type state;
0279             string_type targetstate;
0280         };
0281 #ifdef _MSC_VER
0282 #  pragma warning(pop)
0283 #endif
0284 
0285 #if defined(BOOST_NO_CXX11_RVALUE_REFERENCES)
0286         // allow to assign a token definition expression
0287         template <typename LexerDef, typename Expr>
0288         inline lexer_def_<LexerDef>&
0289         operator+= (lexer_def_<LexerDef>& lexdef, Expr& xpr)
0290         {
0291             // Report invalid expression error as early as possible.
0292             // If you got an error_invalid_expression error message here,
0293             // then the expression (expr) is not a valid spirit lex 
0294             // expression.
0295             BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
0296 
0297             lexdef.define(xpr);
0298             return lexdef;
0299         }
0300 #else
0301         // allow to assign a token definition expression
0302         template <typename LexerDef, typename Expr>
0303         inline lexer_def_<LexerDef>&
0304         operator+= (lexer_def_<LexerDef>& lexdef, Expr&& xpr)
0305         {
0306             // Report invalid expression error as early as possible.
0307             // If you got an error_invalid_expression error message here,
0308             // then the expression (expr) is not a valid spirit lex 
0309             // expression.
0310             BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
0311 
0312             lexdef.define(xpr);
0313             return lexdef;
0314         }
0315 #endif
0316 
0317         template <typename LexerDef, typename Expr>
0318         inline lexer_def_<LexerDef>& 
0319         operator+= (lexer_def_<LexerDef>& lexdef, Expr const& xpr)
0320         {
0321             // Report invalid expression error as early as possible.
0322             // If you got an error_invalid_expression error message here,
0323             // then the expression (expr) is not a valid spirit lex 
0324             // expression.
0325             BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
0326 
0327             lexdef.define(xpr);
0328             return lexdef;
0329         }
0330     }
0331 
0332     ///////////////////////////////////////////////////////////////////////////
0333     //  The match_flags flags are used to influence different matching 
0334     //  modes of the lexer
0335     struct match_flags
0336     {
0337         enum enum_type 
0338         {
0339             match_default = 0,          // no flags
0340             match_not_dot_newline = 1,  // the regex '.' doesn't match newlines
0341             match_icase = 2             // all matching operations are case insensitive
0342         };
0343     };
0344 
0345     ///////////////////////////////////////////////////////////////////////////
0346     //  This represents a lexer object
0347     ///////////////////////////////////////////////////////////////////////////
0348 
0349     ///////////////////////////////////////////////////////////////////////////
0350     // This is the first token id automatically assigned by the library 
0351     // if needed
0352     enum tokenids 
0353     {
0354         min_token_id = 0x10000
0355     };
0356 
0357     template <typename Lexer>
0358     class lexer : public Lexer
0359     {
0360     private:
0361         // avoid warnings about using 'this' in constructor
0362         lexer& this_() { return *this; }
0363 
0364         std::size_t next_token_id;   // has to be an integral type
0365 
0366     public:
0367         typedef Lexer lexer_type;
0368         typedef typename Lexer::id_type id_type;
0369         typedef typename Lexer::char_type char_type;
0370         typedef typename Lexer::iterator_type iterator_type;
0371         typedef lexer base_type;
0372 
0373         typedef detail::lexer_def_<lexer> lexer_def;
0374         typedef std::basic_string<char_type> string_type;
0375 
0376         // if `id_type` was specified but `first_id` is not provided
0377         // the `min_token_id` value may be out of range for `id_type`,
0378         // but it will be a problem only if unique ids feature is in use.
0379         lexer(unsigned int flags = match_flags::match_default)
0380           : lexer_type(flags)
0381           , next_token_id(min_token_id)
0382           , self(this_(), lexer_type::initial_state())
0383         {}
0384 
0385         lexer(unsigned int flags, id_type first_id)
0386           : lexer_type(flags)
0387           , next_token_id(first_id)
0388           , self(this_(), lexer_type::initial_state()) 
0389         {}
0390 
0391         // access iterator interface
0392         template <typename Iterator>
0393         iterator_type begin(Iterator& first, Iterator const& last
0394                 , char_type const* initial_state = 0) const
0395             { return this->lexer_type::begin(first, last, initial_state); }
0396         iterator_type end() const 
0397             { return this->lexer_type::end(); }
0398 
0399         std::size_t map_state(char_type const* state)
0400             { return this->lexer_type::add_state(state); }
0401 
0402         //  create a unique token id
0403         id_type get_next_id() { return id_type(next_token_id++); }
0404 
0405         lexer_def self;  // allow for easy token definition
0406     };
0407 
0408 }}}
0409 
0410 #endif