wave/cpplexer/cpp_lex_token.hpp

0001 /*=============================================================================
0002     Boost.Wave: A Standard compliant C++ preprocessor library
0003
0004     A generic C++ lexer token definition
0005
0006     http://www.boost.org/
0007
0008     Copyright (c) 2001-2012 Hartmut Kaiser. Distributed under the Boost
0009     Software License, Version 1.0. (See accompanying file
0010     LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
0011 =============================================================================*/
0012
0013 #if !defined(BOOST_CPP_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED)
0014 #define BOOST_CPP_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED
0015
0016 #include <boost/wave/wave_config.hpp>
0017 #if BOOST_WAVE_SERIALIZATION != 0
0018 #include <boost/serialization/serialization.hpp>
0019 #endif
0020 #include <boost/wave/util/file_position.hpp>
0021 #include <boost/wave/token_ids.hpp>
0022 #include <boost/wave/language_support.hpp>
0023
0024 #include <boost/throw_exception.hpp>
0025 #include <boost/pool/singleton_pool.hpp>
0026 #include <boost/detail/atomic_count.hpp>
0027 #include <boost/optional.hpp>
0028
0029 // this must occur after all of the includes and before any code appears
0030 #ifdef BOOST_HAS_ABI_HEADERS
0031 #include BOOST_ABI_PREFIX
0032 #endif
0033
0034 ///////////////////////////////////////////////////////////////////////////////
0035 namespace boost {
0036 namespace wave {
0037 namespace cpplexer {
0038
0039 namespace impl {
0040
0041 template <typename StringTypeT, typename PositionT>
0042 class token_data
0043 {
0044 public:
0045     typedef StringTypeT string_type;
0046     typedef PositionT   position_type;
0047
0048     //  default constructed tokens correspond to EOI tokens
0049     token_data()
0050     :   id(T_EOI), refcnt(1)
0051     {}
0052
0053     //  construct an invalid token
0054     explicit token_data(int)
0055     :   id(T_UNKNOWN), refcnt(1)
0056     {}
0057
0058     token_data(token_id id_, string_type const &value_,
0059                position_type const &pos_,
0060                optional<position_type> const & expand_pos_ = boost::none)
0061     :   id(id_), value(value_), pos(pos_), expand_pos(expand_pos_), refcnt(1)
0062     {}
0063
0064     token_data(token_data const& rhs)
0065     :   id(rhs.id), value(rhs.value), pos(rhs.pos), expand_pos(rhs.expand_pos), refcnt(1)
0066     {}
0067
0068     ~token_data()
0069     {}
0070
0071     std::size_t addref() { return ++refcnt; }
0072     std::size_t release() { return --refcnt; }
0073     std::size_t get_refcnt() const { return refcnt; }
0074
0075     // accessors
0076     operator token_id() const { return id; }
0077     string_type const &get_value() const { return value; }
0078     position_type const &get_position() const { return pos; }
0079     position_type const &get_expand_position() const
0080     {
0081         if (expand_pos)
0082             return *expand_pos;
0083         else
0084             return pos;
0085     }
0086
0087     void set_token_id (token_id id_) { id = id_; }
0088     void set_value (string_type const &value_) { value = value_; }
0089     void set_position (position_type const &pos_) { pos = pos_; }
0090     void set_expand_position (position_type const & pos_) { expand_pos = pos_; }
0091
0092     friend bool operator== (token_data const& lhs, token_data const& rhs)
0093     {
0094         //  two tokens are considered equal even if they refer to different
0095         //  positions
0096         return (lhs.id == rhs.id && lhs.value == rhs.value) ? true : false;
0097     }
0098
0099     void init(token_id id_, string_type const &value_, position_type const &pos_)
0100     {
0101         BOOST_ASSERT(refcnt == 1);
0102         id = id_;
0103         value = value_;
0104         pos = pos_;
0105     }
0106
0107     void init(token_data const& rhs)
0108     {
0109         BOOST_ASSERT(refcnt == 1);
0110         id = rhs.id;
0111         value = rhs.value;
0112         pos = rhs.pos;
0113     }
0114
0115     static void *operator new(std::size_t size);
0116     static void operator delete(void *p, std::size_t size);
0117
0118 #if defined(BOOST_SPIRIT_DEBUG)
0119     // debug support
0120     void print (std::ostream &stream) const
0121     {
0122         stream << get_token_name(id) << "(";
0123         for (std::size_t i = 0; i < value.size(); ++i) {
0124             switch (value[i]) {
0125             case '\r':  stream << "\\r"; break;
0126             case '\n':  stream << "\\n"; break;
0127             default:
0128                 stream << value[i];
0129                 break;
0130             }
0131         }
0132         stream << ")";
0133     }
0134 #endif // defined(BOOST_SPIRIT_DEBUG)
0135
0136 #if BOOST_WAVE_SERIALIZATION != 0
0137     friend class boost::serialization::access;
0138     template<typename Archive>
0139     void serialize(Archive &ar, const unsigned int version)
0140     {
0141         using namespace boost::serialization;
0142         ar & make_nvp("id", id);
0143         ar & make_nvp("value", value);
0144         ar & make_nvp("position", pos);
0145     }
0146 #endif
0147
0148 private:
0149     token_id id;                // the token id
0150     string_type value;          // the text, which was parsed into this token
0151     position_type pos;          // the original file position
0152     boost::optional<position_type> expand_pos;    // where this token was expanded
0153     boost::detail::atomic_count refcnt;
0154 };
0155
0156 ///////////////////////////////////////////////////////////////////////////////
0157 struct token_data_tag {};
0158
0159 template <typename StringTypeT, typename PositionT>
0160 inline void *
0161 token_data<StringTypeT, PositionT>::operator new(std::size_t size)
0162 {
0163     BOOST_ASSERT(sizeof(token_data<StringTypeT, PositionT>) == size);
0164     typedef boost::singleton_pool<
0165             token_data_tag, sizeof(token_data<StringTypeT, PositionT>)
0166         > pool_type;
0167
0168     void *ret = pool_type::malloc();
0169     if (0 == ret)
0170         boost::throw_exception(std::bad_alloc());
0171     return ret;
0172 }
0173
0174 template <typename StringTypeT, typename PositionT>
0175 inline void
0176 token_data<StringTypeT, PositionT>::operator delete(void *p, std::size_t size)
0177 {
0178     BOOST_ASSERT(sizeof(token_data<StringTypeT, PositionT>) == size);
0179     typedef boost::singleton_pool<
0180             token_data_tag, sizeof(token_data<StringTypeT, PositionT>)
0181         > pool_type;
0182
0183     if (0 != p)
0184         pool_type::free(p);
0185 }
0186
0187 } // namespace impl
0188
0189 ///////////////////////////////////////////////////////////////////////////////
0190 //  forward declaration of the token type
0191 template <typename PositionT = boost::wave::util::file_position_type>
0192 class lex_token;
0193
0194 ///////////////////////////////////////////////////////////////////////////////
0195 //
0196 //  lex_token
0197 //
0198 ///////////////////////////////////////////////////////////////////////////////
0199
0200 template <typename PositionT>
0201 class lex_token
0202 {
0203 public:
0204     typedef BOOST_WAVE_STRINGTYPE   string_type;
0205     typedef PositionT               position_type;
0206
0207 private:
0208     typedef impl::token_data<string_type, position_type> data_type;
0209
0210 public:
0211     //  default constructed tokens correspond to EOI tokens
0212     lex_token()
0213     :   data(0)
0214     {}
0215
0216     //  construct an invalid token
0217     explicit lex_token(int)
0218     :   data(new data_type(0))
0219     {}
0220
0221     lex_token(lex_token const& rhs)
0222     :   data(rhs.data)
0223     {
0224         if (0 != data)
0225             data->addref();
0226     }
0227
0228     lex_token(token_id id_, string_type const &value_, PositionT const &pos_)
0229     :   data(new data_type(id_, value_, pos_))
0230     {}
0231
0232     ~lex_token()
0233     {
0234         if (0 != data && 0 == data->release())
0235             delete data;
0236         data = 0;
0237     }
0238
0239     lex_token& operator=(lex_token const& rhs)
0240     {
0241         if (&rhs != this) {
0242             if (0 != data && 0 == data->release())
0243                 delete data;
0244
0245             data = rhs.data;
0246             if (0 != data)
0247                 data->addref();
0248         }
0249         return *this;
0250     }
0251
0252     // accessors
0253     operator token_id() const { return 0 != data ? token_id(*data) : T_EOI; }
0254     string_type const &get_value() const { return data->get_value(); }
0255     position_type const &get_position() const { return data->get_position(); }
0256     position_type const &get_expand_position() const { return data->get_expand_position(); }
0257     bool is_eoi() const { return 0 == data || token_id(*data) == T_EOI; }
0258     bool is_valid() const { return 0 != data && token_id(*data) != T_UNKNOWN; }
0259
0260     void set_token_id (token_id id_) { make_unique(); data->set_token_id(id_); }
0261     void set_value (string_type const &value_) { make_unique(); data->set_value(value_); }
0262     void set_position (position_type const &pos_) { make_unique(); data->set_position(pos_); }
0263     void set_expand_position (position_type const &pos_) { make_unique(); data->set_expand_position(pos_); }
0264
0265     friend bool operator== (lex_token const& lhs, lex_token const& rhs)
0266     {
0267         if (0 == rhs.data)
0268             return 0 == lhs.data;
0269         if (0 == lhs.data)
0270             return false;
0271         return *(lhs.data) == *(rhs.data);
0272     }
0273
0274 // debug support
0275 #if BOOST_WAVE_DUMP_PARSE_TREE != 0
0276     // access functions for the tree_to_xml functionality
0277     static int get_token_id(lex_token const &t)
0278         { return token_id(t); }
0279     static string_type get_token_value(lex_token const &t)
0280         { return t.get_value(); }
0281 #endif
0282
0283 #if defined(BOOST_SPIRIT_DEBUG)
0284     // debug support
0285     void print (std::ostream &stream) const
0286     {
0287         data->print(stream);
0288     }
0289 #endif // defined(BOOST_SPIRIT_DEBUG)
0290
0291 private:
0292 #if BOOST_WAVE_SERIALIZATION != 0
0293     friend class boost::serialization::access;
0294     template<typename Archive>
0295     void serialize(Archive &ar, const unsigned int version)
0296     {
0297         data->serialize(ar, version);
0298     }
0299 #endif
0300
0301     // make a unique copy of the current object
0302     void make_unique()
0303     {
0304         if (1 == data->get_refcnt())
0305             return;
0306
0307         data_type* newdata = new data_type(*data) ;
0308
0309         data->release();          // release this reference, can't get zero
0310         data = newdata;
0311     }
0312
0313     data_type* data;
0314 };
0315
0316 ///////////////////////////////////////////////////////////////////////////////
0317 //  This overload is needed by the multi_pass/functor_input_policy to
0318 //  validate a token instance. It has to be defined in the same namespace
0319 //  as the token class itself to allow ADL to find it.
0320 ///////////////////////////////////////////////////////////////////////////////
0321 template <typename Position>
0322 inline bool
0323 token_is_valid(lex_token<Position> const& t)
0324 {
0325     return t.is_valid();
0326 }
0327
0328 ///////////////////////////////////////////////////////////////////////////////
0329 #if defined(BOOST_SPIRIT_DEBUG)
0330 template <typename PositionT>
0331 inline std::ostream &
0332 operator<< (std::ostream &stream, lex_token<PositionT> const &object)
0333 {
0334     object.print(stream);
0335     return stream;
0336 }
0337 #endif // defined(BOOST_SPIRIT_DEBUG)
0338
0339 ///////////////////////////////////////////////////////////////////////////////
0340 }   // namespace cpplexer
0341 }   // namespace wave
0342 }   // namespace boost
0343
0344 // the suffix header occurs after all of the code
0345 #ifdef BOOST_HAS_ABI_HEADERS
0346 #include BOOST_ABI_SUFFIX
0347 #endif
0348
0349 #endif // !defined(BOOST_CPP_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED)