lex/lexer/char_token_def.hpp

0001 //  Copyright (c) 2001-2011 Hartmut Kaiser
0002 //
0003 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
0004 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
0005
0006 #if !defined(BOOST_SPIRIT_LEX_CHAR_TOKEN_DEF_MAR_28_2007_0626PM)
0007 #define BOOST_SPIRIT_LEX_CHAR_TOKEN_DEF_MAR_28_2007_0626PM
0008
0009 #if defined(_MSC_VER)
0010 #pragma once
0011 #endif
0012
0013 #include <boost/spirit/home/support/common_terminals.hpp>
0014 #include <boost/spirit/home/support/string_traits.hpp>
0015 #include <boost/spirit/home/lex/domain.hpp>
0016 #include <boost/spirit/home/lex/lexer_type.hpp>
0017 #include <boost/spirit/home/lex/meta_compiler.hpp>
0018
0019 namespace boost { namespace spirit
0020 {
0021     ///////////////////////////////////////////////////////////////////////////
0022     // Enablers
0023     ///////////////////////////////////////////////////////////////////////////
0024
0025     // enables 'x'
0026     template <>
0027     struct use_terminal<lex::domain, char>
0028       : mpl::true_ {};
0029
0030     // enables "x"
0031     template <>
0032     struct use_terminal<lex::domain, char[2]>
0033       : mpl::true_ {};
0034
0035     // enables wchar_t
0036     template <>
0037     struct use_terminal<lex::domain, wchar_t>
0038       : mpl::true_ {};
0039
0040     // enables L"x"
0041     template <>
0042     struct use_terminal<lex::domain, wchar_t[2]>
0043       : mpl::true_ {};
0044
0045     // enables char_('x'), char_("x")
0046     template <typename CharEncoding, typename A0>
0047     struct use_terminal<lex::domain
0048       , terminal_ex<
0049             tag::char_code<tag::char_, CharEncoding>
0050           , fusion::vector1<A0> > >
0051       : mpl::true_ {};
0052
0053     // enables char_('x', ID), char_("x", ID)
0054     template <typename CharEncoding, typename A0, typename A1>
0055     struct use_terminal<lex::domain
0056       , terminal_ex<
0057             tag::char_code<tag::char_, CharEncoding>
0058           , fusion::vector2<A0, A1> > >
0059       : mpl::true_ {};
0060 }}
0061
0062 namespace boost { namespace spirit { namespace lex
0063 {
0064     // use char_ from standard character set by default
0065 #ifndef BOOST_SPIRIT_NO_PREDEFINED_TERMINALS
0066     using spirit::standard::char_;
0067 #endif
0068     using spirit::standard::char_type;
0069
0070     ///////////////////////////////////////////////////////////////////////////
0071     //
0072     //  char_token_def
0073     //      represents a single character token definition
0074     //
0075     ///////////////////////////////////////////////////////////////////////////
0076     template <typename CharEncoding = char_encoding::standard
0077       , typename IdType = std::size_t>
0078     struct char_token_def
0079       : primitive_lexer<char_token_def<CharEncoding, IdType> >
0080     {
0081         typedef typename CharEncoding::char_type char_type;
0082
0083         char_token_def(char_type ch, IdType const& id)
0084           : ch(ch), id_(id), unique_id_(std::size_t(~0))
0085           , token_state_(std::size_t(~0))
0086         {}
0087
0088         template <typename LexerDef, typename String>
0089         void collect(LexerDef& lexdef, String const& state
0090           , String const& targetstate) const
0091         {
0092             std::size_t state_id = lexdef.add_state(state.c_str());
0093
0094             // If the following assertion fires you are probably trying to use
0095             // a single char_token_def instance in more than one lexer state.
0096             // This is not possible. Please create a separate token_def instance
0097             // from the same regular expression for each lexer state it needs
0098             // to be associated with.
0099             BOOST_ASSERT(
0100                 (std::size_t(~0) == token_state_ || state_id == token_state_) &&
0101                 "Can't use single char_token_def with more than one lexer state");
0102
0103             char_type const* target = targetstate.empty() ? 0 : targetstate.c_str();
0104             if (target)
0105                 lexdef.add_state(target);
0106
0107             token_state_ = state_id;
0108             unique_id_ = lexdef.add_token (state.c_str(), ch, id_, target);
0109         }
0110
0111         template <typename LexerDef>
0112         void add_actions(LexerDef&) const {}
0113
0114         IdType id() const { return id_; }
0115         std::size_t unique_id() const { return unique_id_; }
0116         std::size_t state() const { return token_state_; }
0117
0118         char_type ch;
0119         mutable IdType id_;
0120         mutable std::size_t unique_id_;
0121         mutable std::size_t token_state_;
0122     };
0123
0124     ///////////////////////////////////////////////////////////////////////////
0125     // Lexer generators: make_xxx function (objects)
0126     ///////////////////////////////////////////////////////////////////////////
0127     namespace detail
0128     {
0129         template <typename CharEncoding>
0130         struct basic_literal
0131         {
0132             typedef char_token_def<CharEncoding> result_type;
0133
0134             template <typename Char>
0135             result_type operator()(Char ch, unused_type) const
0136             {
0137                 return result_type(ch, ch);
0138             }
0139
0140             template <typename Char>
0141             result_type operator()(Char const* str, unused_type) const
0142             {
0143                 return result_type(str[0], str[0]);
0144             }
0145         };
0146     }
0147
0148     // literals: 'x', "x"
0149     template <typename Modifiers>
0150     struct make_primitive<char, Modifiers>
0151       : detail::basic_literal<char_encoding::standard> {};
0152
0153     template <typename Modifiers>
0154     struct make_primitive<char const(&)[2], Modifiers>
0155       : detail::basic_literal<char_encoding::standard> {};
0156
0157     // literals: L'x', L"x"
0158     template <typename Modifiers>
0159     struct make_primitive<wchar_t, Modifiers>
0160       : detail::basic_literal<char_encoding::standard_wide> {};
0161
0162     template <typename Modifiers>
0163     struct make_primitive<wchar_t const(&)[2], Modifiers>
0164       : detail::basic_literal<char_encoding::standard_wide> {};
0165
0166     // handle char_('x')
0167     template <typename CharEncoding, typename Modifiers, typename A0>
0168     struct make_primitive<
0169         terminal_ex<
0170             tag::char_code<tag::char_, CharEncoding>
0171           , fusion::vector1<A0>
0172         >
0173       , Modifiers>
0174     {
0175         typedef char_token_def<CharEncoding> result_type;
0176
0177         template <typename Terminal>
0178         result_type operator()(Terminal const& term, unused_type) const
0179         {
0180             return result_type(fusion::at_c<0>(term.args), fusion::at_c<0>(term.args));
0181         }
0182     };
0183
0184     // handle char_("x")
0185     template <typename CharEncoding, typename Modifiers, typename Char>
0186     struct make_primitive<
0187         terminal_ex<
0188             tag::char_code<tag::char_, CharEncoding>
0189           , fusion::vector1<Char(&)[2]>   // single char strings
0190         >
0191       , Modifiers>
0192     {
0193         typedef char_token_def<CharEncoding> result_type;
0194
0195         template <typename Terminal>
0196         result_type operator()(Terminal const& term, unused_type) const
0197         {
0198             Char ch = fusion::at_c<0>(term.args)[0];
0199             return result_type(ch, ch);
0200         }
0201     };
0202
0203     // handle char_('x', ID)
0204     template <typename CharEncoding, typename Modifiers, typename A0, typename A1>
0205     struct make_primitive<
0206         terminal_ex<
0207             tag::char_code<tag::char_, CharEncoding>
0208           , fusion::vector2<A0, A1>
0209         >
0210       , Modifiers>
0211     {
0212         typedef char_token_def<CharEncoding> result_type;
0213
0214         template <typename Terminal>
0215         result_type operator()(Terminal const& term, unused_type) const
0216         {
0217             return result_type(
0218                 fusion::at_c<0>(term.args), fusion::at_c<1>(term.args));
0219         }
0220     };
0221
0222     // handle char_("x", ID)
0223     template <typename CharEncoding, typename Modifiers, typename Char, typename A1>
0224     struct make_primitive<
0225         terminal_ex<
0226             tag::char_code<tag::char_, CharEncoding>
0227           , fusion::vector2<Char(&)[2], A1>   // single char strings
0228         >
0229       , Modifiers>
0230     {
0231         typedef char_token_def<CharEncoding> result_type;
0232
0233         template <typename Terminal>
0234         result_type operator()(Terminal const& term, unused_type) const
0235         {
0236             return result_type(
0237                 fusion::at_c<0>(term.args)[0], fusion::at_c<1>(term.args));
0238         }
0239     };
0240 }}}  // namespace boost::spirit::lex
0241
0242 #endif