lex/lexer/string_token_def.hpp

0001 //  Copyright (c) 2001-2011 Hartmut Kaiser
0002 //
0003 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
0004 //  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
0005
0006 #if !defined(BOOST_SPIRIT_LEX_STRING_TOKEN_DEF_MAR_28_2007_0722PM)
0007 #define BOOST_SPIRIT_LEX_STRING_TOKEN_DEF_MAR_28_2007_0722PM
0008
0009 #if defined(_MSC_VER)
0010 #pragma once
0011 #endif
0012
0013 #include <boost/spirit/home/support/common_terminals.hpp>
0014 #include <boost/spirit/home/support/string_traits.hpp>
0015 #include <boost/spirit/home/lex/domain.hpp>
0016 #include <boost/spirit/home/lex/lexer_type.hpp>
0017 #include <boost/spirit/home/lex/meta_compiler.hpp>
0018 #include <boost/type_traits/add_const.hpp>
0019 #include <boost/type_traits/add_reference.hpp>
0020 #include <boost/type_traits/is_integral.hpp>
0021 #include <boost/type_traits/remove_const.hpp>
0022 #include <boost/fusion/include/vector.hpp>
0023 #include <boost/fusion/include/at.hpp>
0024
0025 namespace boost { namespace spirit
0026 {
0027     ///////////////////////////////////////////////////////////////////////////
0028     // Enablers
0029     ///////////////////////////////////////////////////////////////////////////
0030
0031     // enables strings
0032     template <typename T>
0033     struct use_terminal<lex::domain, T
0034       , typename enable_if<traits::is_string<T> >::type>
0035       : mpl::true_ {};
0036
0037     // enables string(str)
0038     template <typename CharEncoding, typename A0>
0039     struct use_terminal<lex::domain
0040       , terminal_ex<
0041             tag::char_code<tag::string, CharEncoding>
0042           , fusion::vector1<A0> > >
0043       : traits::is_string<A0> {};
0044
0045     // enables string(str, ID)
0046     template <typename CharEncoding, typename A0, typename A1>
0047     struct use_terminal<lex::domain
0048       , terminal_ex<
0049             tag::char_code<tag::string, CharEncoding>
0050           , fusion::vector2<A0, A1> > >
0051       : traits::is_string<A0> {};
0052 }}
0053
0054 namespace boost { namespace spirit { namespace lex
0055 {
0056     // use string from standard character set by default
0057 #ifndef BOOST_SPIRIT_NO_PREDEFINED_TERMINALS
0058     using spirit::standard::string;
0059 #endif
0060     using spirit::standard::string_type;
0061
0062     ///////////////////////////////////////////////////////////////////////////
0063     //
0064     //  string_token_def
0065     //      represents a string based token definition
0066     //
0067     ///////////////////////////////////////////////////////////////////////////
0068     template <typename String, typename IdType = std::size_t
0069       , typename CharEncoding = char_encoding::standard>
0070     struct string_token_def
0071       : primitive_lexer<string_token_def<String, IdType, CharEncoding> >
0072     {
0073         typedef typename
0074             remove_const<typename traits::char_type_of<String>::type>::type
0075         char_type;
0076         typedef std::basic_string<char_type> string_type;
0077
0078         string_token_def(typename add_reference<String>::type str, IdType const& id)
0079           : str_(str), id_(id), unique_id_(std::size_t(~0))
0080           , token_state_(std::size_t(~0))
0081         {}
0082
0083         template <typename LexerDef, typename String_>
0084         void collect(LexerDef& lexdef, String_ const& state
0085           , String_ const& targetstate) const
0086         {
0087             std::size_t state_id = lexdef.add_state(state.c_str());
0088
0089             // If the following assertion fires you are probably trying to use
0090             // a single string_token_def instance in more than one lexer state.
0091             // This is not possible. Please create a separate token_def instance
0092             // from the same regular expression for each lexer state it needs
0093             // to be associated with.
0094             BOOST_ASSERT(
0095                 (std::size_t(~0) == token_state_ || state_id == token_state_) &&
0096                 "Can't use single string_token_def with more than one lexer state");
0097
0098             char_type const* target = targetstate.empty() ? 0 : targetstate.c_str();
0099             if (target)
0100                 lexdef.add_state(target);
0101
0102             token_state_ = state_id;
0103
0104             if (IdType(~0) == id_)
0105                 id_ = IdType(lexdef.get_next_id());
0106
0107             unique_id_ = lexdef.add_token (state.c_str(), str_, id_, target);
0108         }
0109
0110         template <typename LexerDef>
0111         void add_actions(LexerDef&) const {}
0112
0113         std::size_t id() const { return id_; }
0114         std::size_t unique_id() const { return unique_id_; }
0115         std::size_t state() const { return token_state_; }
0116
0117         string_type str_;
0118         mutable IdType id_;
0119         mutable std::size_t unique_id_;
0120         mutable std::size_t token_state_;
0121     };
0122
0123     ///////////////////////////////////////////////////////////////////////////
0124     // Lex generators: make_xxx function (objects)
0125     ///////////////////////////////////////////////////////////////////////////
0126     template <typename T, typename Modifiers>
0127     struct make_primitive<T, Modifiers
0128       , typename enable_if<traits::is_string<T> >::type>
0129     {
0130         typedef typename add_const<T>::type const_string;
0131         typedef string_token_def<const_string> result_type;
0132
0133         result_type operator()(
0134             typename add_reference<const_string>::type str, unused_type) const
0135         {
0136             return result_type(str, std::size_t(~0));
0137         }
0138     };
0139
0140     template <typename Modifiers, typename CharEncoding, typename A0>
0141     struct make_primitive<
0142         terminal_ex<
0143             tag::char_code<tag::string, CharEncoding>
0144           , fusion::vector1<A0> >
0145       , Modifiers>
0146     {
0147         typedef typename add_const<A0>::type const_string;
0148         typedef string_token_def<const_string, std::size_t, CharEncoding>
0149             result_type;
0150
0151         template <typename Terminal>
0152         result_type operator()(Terminal const& term, unused_type) const
0153         {
0154             return result_type(fusion::at_c<0>(term.args), std::size_t(~0));
0155         }
0156     };
0157
0158     template <typename Modifiers, typename CharEncoding, typename A0, typename A1>
0159     struct make_primitive<
0160         terminal_ex<
0161             tag::char_code<tag::string, CharEncoding>
0162           , fusion::vector2<A0, A1> >
0163       , Modifiers>
0164     {
0165         typedef typename add_const<A0>::type const_string;
0166         typedef string_token_def<const_string, A1, CharEncoding> result_type;
0167
0168         template <typename Terminal>
0169         result_type operator()(Terminal const& term, unused_type) const
0170         {
0171             return result_type(
0172                 fusion::at_c<0>(term.args), fusion::at_c<1>(term.args));
0173         }
0174     };
0175 }}}  // namespace boost::spirit::lex
0176
0177 #endif