File indexing completed on 2025-02-27 09:55:21
0001
0002
0003
0004
0005
0006 #if !defined(BOOST_SPIRIT_LEX_LEXER_MAR_13_2007_0145PM)
0007 #define BOOST_SPIRIT_LEX_LEXER_MAR_13_2007_0145PM
0008
0009 #if defined(_MSC_VER)
0010 #pragma once
0011 #endif
0012
0013 #include <boost/spirit/home/support/info.hpp>
0014 #include <boost/spirit/home/qi/skip_over.hpp>
0015 #include <boost/spirit/home/qi/parser.hpp>
0016 #include <boost/spirit/home/qi/detail/assign_to.hpp>
0017 #include <boost/spirit/home/lex/reference.hpp>
0018 #include <boost/spirit/home/lex/meta_compiler.hpp>
0019 #include <boost/spirit/home/lex/lexer_type.hpp>
0020 #include <boost/spirit/home/lex/lexer/token_def.hpp>
0021 #include <boost/assert.hpp>
0022 #include <boost/noncopyable.hpp>
0023 #include <boost/fusion/include/vector.hpp>
0024 #include <boost/mpl/assert.hpp>
0025 #include <boost/proto/extends.hpp>
0026 #include <boost/proto/traits.hpp>
0027 #include <boost/range/iterator_range_core.hpp>
0028 #include <iterator> // for std::iterator_traits
0029 #include <string>
0030
0031 namespace boost { namespace spirit { namespace lex
0032 {
0033
0034 namespace detail
0035 {
0036
0037 #ifdef _MSC_VER
0038 # pragma warning(push)
0039 # pragma warning(disable: 4512)
0040 #endif
0041 template <typename LexerDef>
0042 struct lexer_def_
0043 : proto::extends<
0044 typename proto::terminal<
0045 lex::reference<lexer_def_<LexerDef> const>
0046 >::type
0047 , lexer_def_<LexerDef> >
0048 , qi::parser<lexer_def_<LexerDef> >
0049 , lex::lexer_type<lexer_def_<LexerDef> >
0050 {
0051 private:
0052
0053 lexer_def_& this_() { return *this; }
0054
0055 typedef typename LexerDef::char_type char_type;
0056 typedef typename LexerDef::string_type string_type;
0057 typedef typename LexerDef::id_type id_type;
0058
0059 typedef lex::reference<lexer_def_ const> reference_;
0060 typedef typename proto::terminal<reference_>::type terminal_type;
0061 typedef proto::extends<terminal_type, lexer_def_> proto_base_type;
0062
0063 reference_ alias() const
0064 {
0065 return reference_(*this);
0066 }
0067
0068 public:
0069
0070 template <typename Context, typename Iterator>
0071 struct attribute
0072 {
0073
0074
0075 typedef typename Iterator::base_iterator_type iterator_type;
0076 typedef
0077 fusion::vector2<id_type, iterator_range<iterator_type> >
0078 type;
0079 };
0080
0081
0082 template <typename Iterator, typename Context
0083 , typename Skipper, typename Attribute>
0084 bool parse(Iterator& first, Iterator const& last
0085 , Context& , Skipper const& skipper
0086 , Attribute& attr) const
0087 {
0088 qi::skip_over(first, last, skipper);
0089
0090 if (first != last) {
0091 typedef typename
0092 std::iterator_traits<Iterator>::value_type
0093 token_type;
0094
0095 token_type const& t = *first;
0096 if (token_is_valid(t) && t.state() == first.get_state()) {
0097
0098 spirit::traits::assign_to(t, attr);
0099 ++first;
0100 return true;
0101 }
0102 }
0103 return false;
0104 }
0105
0106
0107 template <typename Context>
0108 info what(Context& ) const
0109 {
0110 return info("lexer");
0111 }
0112
0113 private:
0114
0115
0116 struct adder
0117 {
0118 adder(lexer_def_& def_)
0119 : def(def_) {}
0120
0121
0122
0123
0124
0125 adder const& operator()(char_type c
0126 , id_type token_id = id_type()) const
0127 {
0128 if (id_type() == token_id)
0129 token_id = static_cast<id_type>(c);
0130 def.def.add_token (def.state.c_str(), c, token_id
0131 , def.targetstate.empty() ? 0 : def.targetstate.c_str());
0132 return *this;
0133 }
0134
0135
0136
0137
0138
0139
0140 adder const& operator()(string_type const& s
0141 , id_type token_id = id_type()) const
0142 {
0143 if (id_type() == token_id)
0144 token_id = def.def.get_next_id();
0145 def.def.add_token (def.state.c_str(), s, token_id
0146 , def.targetstate.empty() ? 0 : def.targetstate.c_str());
0147 return *this;
0148 }
0149
0150 template <typename Attribute>
0151 adder const& operator()(
0152 token_def<Attribute, char_type, id_type>& tokdef
0153 , id_type token_id = id_type()) const
0154 {
0155
0156 if (id_type() == token_id) {
0157 if (id_type() == tokdef.id()) {
0158 token_id = def.def.get_next_id();
0159 tokdef.id(token_id);
0160 }
0161 else {
0162 token_id = tokdef.id();
0163 }
0164 }
0165 else {
0166
0167
0168 BOOST_ASSERT(id_type() == tokdef.id()
0169 || token_id == tokdef.id());
0170 tokdef.id(token_id);
0171 }
0172
0173 def.define(tokdef);
0174 return *this;
0175 }
0176
0177
0178
0179
0180
0181
0182
0183
0184
0185
0186
0187
0188 lexer_def_& def;
0189 };
0190 friend struct adder;
0191
0192
0193
0194 struct pattern_adder
0195 {
0196 pattern_adder(lexer_def_& def_)
0197 : def(def_) {}
0198
0199 pattern_adder const& operator()(string_type const& p
0200 , string_type const& s) const
0201 {
0202 def.def.add_pattern (def.state.c_str(), p, s);
0203 return *this;
0204 }
0205
0206 lexer_def_& def;
0207 };
0208 friend struct pattern_adder;
0209
0210 private:
0211
0212
0213 template <typename TokenExpr>
0214 void compile2pass(TokenExpr const& expr)
0215 {
0216 expr.collect(def, state, targetstate);
0217 expr.add_actions(def);
0218 }
0219
0220 public:
0221
0222 template <typename Expr>
0223 void define(Expr const& expr)
0224 {
0225 compile2pass(compile<lex::domain>(expr));
0226 }
0227
0228 lexer_def_(LexerDef& def_, string_type const& state_
0229 , string_type const& targetstate_ = string_type())
0230 : proto_base_type(terminal_type::make(alias()))
0231 , add(this_()), add_pattern(this_()), def(def_)
0232 , state(state_), targetstate(targetstate_)
0233 {}
0234
0235
0236 lexer_def_ operator()(char_type const* state_) const
0237 {
0238 return lexer_def_(def, state_);
0239 }
0240 lexer_def_ operator()(char_type const* state_
0241 , char_type const* targetstate_) const
0242 {
0243 return lexer_def_(def, state_, targetstate_);
0244 }
0245 lexer_def_ operator()(string_type const& state_
0246 , string_type const& targetstate_ = string_type()) const
0247 {
0248 return lexer_def_(def, state_, targetstate_);
0249 }
0250
0251
0252 template <typename Expr>
0253 lexer_def_& operator= (Expr const& xpr)
0254 {
0255
0256
0257
0258
0259 BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
0260
0261 def.clear(state.c_str());
0262 define(xpr);
0263 return *this;
0264 }
0265
0266
0267
0268 std::size_t add_state(char_type const* state_ = 0)
0269 {
0270 return def.add_state(state_ ? state_ : def.initial_state().c_str());
0271 }
0272
0273 adder add;
0274 pattern_adder add_pattern;
0275
0276 private:
0277 LexerDef& def;
0278 string_type state;
0279 string_type targetstate;
0280 };
0281 #ifdef _MSC_VER
0282 # pragma warning(pop)
0283 #endif
0284
0285 #if defined(BOOST_NO_CXX11_RVALUE_REFERENCES)
0286
0287 template <typename LexerDef, typename Expr>
0288 inline lexer_def_<LexerDef>&
0289 operator+= (lexer_def_<LexerDef>& lexdef, Expr& xpr)
0290 {
0291
0292
0293
0294
0295 BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
0296
0297 lexdef.define(xpr);
0298 return lexdef;
0299 }
0300 #else
0301
0302 template <typename LexerDef, typename Expr>
0303 inline lexer_def_<LexerDef>&
0304 operator+= (lexer_def_<LexerDef>& lexdef, Expr&& xpr)
0305 {
0306
0307
0308
0309
0310 BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
0311
0312 lexdef.define(xpr);
0313 return lexdef;
0314 }
0315 #endif
0316
0317 template <typename LexerDef, typename Expr>
0318 inline lexer_def_<LexerDef>&
0319 operator+= (lexer_def_<LexerDef>& lexdef, Expr const& xpr)
0320 {
0321
0322
0323
0324
0325 BOOST_SPIRIT_ASSERT_MATCH(lex::domain, Expr);
0326
0327 lexdef.define(xpr);
0328 return lexdef;
0329 }
0330 }
0331
0332
0333
0334
0335 struct match_flags
0336 {
0337 enum enum_type
0338 {
0339 match_default = 0,
0340 match_not_dot_newline = 1,
0341 match_icase = 2
0342 };
0343 };
0344
0345
0346
0347
0348
0349
0350
0351
0352 enum tokenids
0353 {
0354 min_token_id = 0x10000
0355 };
0356
0357 template <typename Lexer>
0358 class lexer : public Lexer
0359 {
0360 private:
0361
0362 lexer& this_() { return *this; }
0363
0364 std::size_t next_token_id;
0365
0366 public:
0367 typedef Lexer lexer_type;
0368 typedef typename Lexer::id_type id_type;
0369 typedef typename Lexer::char_type char_type;
0370 typedef typename Lexer::iterator_type iterator_type;
0371 typedef lexer base_type;
0372
0373 typedef detail::lexer_def_<lexer> lexer_def;
0374 typedef std::basic_string<char_type> string_type;
0375
0376
0377
0378
0379 lexer(unsigned int flags = match_flags::match_default)
0380 : lexer_type(flags)
0381 , next_token_id(min_token_id)
0382 , self(this_(), lexer_type::initial_state())
0383 {}
0384
0385 lexer(unsigned int flags, id_type first_id)
0386 : lexer_type(flags)
0387 , next_token_id(first_id)
0388 , self(this_(), lexer_type::initial_state())
0389 {}
0390
0391
0392 template <typename Iterator>
0393 iterator_type begin(Iterator& first, Iterator const& last
0394 , char_type const* initial_state = 0) const
0395 { return this->lexer_type::begin(first, last, initial_state); }
0396 iterator_type end() const
0397 { return this->lexer_type::end(); }
0398
0399 std::size_t map_state(char_type const* state)
0400 { return this->lexer_type::add_state(state); }
0401
0402
0403 id_type get_next_id() { return id_type(next_token_id++); }
0404
0405 lexer_def self;
0406 };
0407
0408 }}}
0409
0410 #endif