wave/util/cpp_macromap.hpp

0001 /*=============================================================================
0002     Boost.Wave: A Standard compliant C++ preprocessor library
0003
0004     Macro expansion engine
0005
0006     http://www.boost.org/
0007
0008     Copyright (c) 2001-2012 Hartmut Kaiser. Distributed under the Boost
0009     Software License, Version 1.0. (See accompanying file
0010     LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
0011 =============================================================================*/
0012
0013 #if !defined(BOOST_CPP_MACROMAP_HPP_CB8F51B0_A3F0_411C_AEF4_6FF631B8B414_INCLUDED)
0014 #define BOOST_CPP_MACROMAP_HPP_CB8F51B0_A3F0_411C_AEF4_6FF631B8B414_INCLUDED
0015
0016 #include <cstdlib>
0017 #include <ctime>
0018
0019 #include <list>
0020 #include <map>
0021 #include <set>
0022 #include <vector>
0023 #include <iterator>
0024 #include <algorithm>
0025 #include <string>
0026
0027 #include <boost/assert.hpp>
0028 #include <boost/wave/wave_config.hpp>
0029 #if BOOST_WAVE_SERIALIZATION != 0
0030 #include <boost/serialization/serialization.hpp>
0031 #include <boost/serialization/shared_ptr.hpp>
0032 #endif
0033
0034 #include <boost/filesystem/path.hpp>
0035 #include <boost/lexical_cast.hpp>
0036 #include <boost/optional.hpp>
0037
0038 #include <boost/wave/util/time_conversion_helper.hpp>
0039 #include <boost/wave/util/unput_queue_iterator.hpp>
0040 #include <boost/wave/util/macro_helpers.hpp>
0041 #include <boost/wave/util/macro_definition.hpp>
0042 #include <boost/wave/util/symbol_table.hpp>
0043 #include <boost/wave/util/cpp_macromap_utils.hpp>
0044 #include <boost/wave/util/cpp_macromap_predef.hpp>
0045 #include <boost/wave/util/filesystem_compatibility.hpp>
0046 #include <boost/wave/grammars/cpp_defined_grammar_gen.hpp>
0047 #if BOOST_WAVE_SUPPORT_HAS_INCLUDE != 0
0048 #include <boost/wave/grammars/cpp_has_include_grammar_gen.hpp>
0049 #endif
0050
0051 #include <boost/wave/wave_version.hpp>
0052 #include <boost/wave/cpp_exceptions.hpp>
0053 #include <boost/wave/language_support.hpp>
0054
0055 // this must occur after all of the includes and before any code appears
0056 #ifdef BOOST_HAS_ABI_HEADERS
0057 #include BOOST_ABI_PREFIX
0058 #endif
0059
0060 ///////////////////////////////////////////////////////////////////////////////
0061 namespace boost { namespace wave { namespace util {
0062
0063 ///////////////////////////////////////////////////////////////////////////////
0064 //
0065 //  macromap
0066 //
0067 //      This class holds all currently defined macros and on demand expands
0068 //      those macro definitions
0069 //
0070 ///////////////////////////////////////////////////////////////////////////////
0071 template <typename ContextT>
0072 class macromap {
0073
0074     typedef macromap<ContextT>                      self_type;
0075     typedef typename ContextT::token_type           token_type;
0076     typedef typename token_type::string_type        string_type;
0077     typedef typename token_type::position_type      position_type;
0078
0079     typedef typename ContextT::token_sequence_type  definition_container_type;
0080     typedef std::vector<token_type>                 parameter_container_type;
0081
0082     typedef macro_definition<token_type, definition_container_type>
0083         macro_definition_type;
0084     typedef symbol_table<string_type, macro_definition_type>
0085         defined_macros_type;
0086     typedef typename defined_macros_type::value_type::second_type
0087         macro_ref_type;
0088
0089 public:
0090     macromap(ContextT &ctx_)
0091     :   current_macros(0), defined_macros(new defined_macros_type(1)),
0092         main_pos("", 0), ctx(ctx_), macro_uid(1)
0093     {
0094         current_macros = defined_macros.get();
0095     }
0096     ~macromap() {}
0097
0098     //  Add a new macro to the given macro scope
0099     bool add_macro(token_type const &name, bool has_parameters,
0100         parameter_container_type &parameters,
0101         definition_container_type &definition, bool is_predefined = false,
0102         defined_macros_type *scope = 0);
0103
0104     //  Tests, whether the given macro name is defined in the given macro scope
0105     bool is_defined(string_type const &name,
0106         typename defined_macros_type::iterator &it,
0107         defined_macros_type *scope = 0) const;
0108
0109     // expects a token sequence as its parameters
0110     template <typename IteratorT>
0111     bool is_defined(IteratorT const &begin, IteratorT const &end) const;
0112
0113     // expects an arbitrary string as its parameter
0114     bool is_defined(string_type const &str) const;
0115
0116 #if BOOST_WAVE_SUPPORT_HAS_INCLUDE != 0
0117     // expects a token sequence as its parameters
0118     template <typename IteratorT>
0119     bool has_include(IteratorT const &begin, IteratorT const &end,
0120                      bool is_quoted_filename, bool is_system) const;
0121 #endif
0122
0123     //  Get the macro definition for the given macro scope
0124     bool get_macro(string_type const &name, bool &has_parameters,
0125         bool &is_predefined, position_type &pos,
0126         parameter_container_type &parameters,
0127         definition_container_type &definition,
0128         defined_macros_type *scope = 0) const;
0129
0130     //  Remove a macro name from the given macro scope
0131     bool remove_macro(string_type const &name, position_type const& pos,
0132         bool even_predefined = false);
0133
0134     template <typename IteratorT, typename ContainerT>
0135     token_type const &expand_tokensequence(IteratorT &first,
0136         IteratorT const &last, ContainerT &pending, ContainerT &expanded,
0137         bool& seen_newline, bool expand_operator_defined,
0138         bool expand_operator_has_include);
0139
0140     //  Expand all macros inside the given token sequence
0141     template <typename IteratorT, typename ContainerT>
0142     void expand_whole_tokensequence(ContainerT &expanded,
0143         IteratorT &first, IteratorT const &last,
0144         bool expand_operator_defined,
0145         bool expand_operator_has_include);
0146
0147     //  Init the predefined macros (add them to the given scope)
0148     void init_predefined_macros(char const *fname = "<Unknown>",
0149         defined_macros_type *scope = 0, bool at_global_scope = true);
0150     void predefine_macro(defined_macros_type *scope, string_type const &name,
0151         token_type const &t);
0152
0153     //  Init the internal macro symbol namespace
0154     void reset_macromap();
0155
0156     position_type &get_main_pos() { return main_pos; }
0157     position_type const& get_main_pos() const { return main_pos; }
0158
0159     //  interface for macro name introspection
0160     typedef typename defined_macros_type::name_iterator name_iterator;
0161     typedef typename defined_macros_type::const_name_iterator const_name_iterator;
0162
0163     name_iterator begin()
0164         { return defined_macros_type::make_iterator(current_macros->begin()); }
0165     name_iterator end()
0166         { return defined_macros_type::make_iterator(current_macros->end()); }
0167     const_name_iterator begin() const
0168         { return defined_macros_type::make_iterator(current_macros->begin()); }
0169     const_name_iterator end() const
0170         { return defined_macros_type::make_iterator(current_macros->end()); }
0171
0172 protected:
0173     //  Helper functions for expanding all macros in token sequences
0174     template <typename IteratorT, typename ContainerT>
0175     token_type const &expand_tokensequence_worker(ContainerT &pending,
0176         unput_queue_iterator<IteratorT, token_type, ContainerT> &first,
0177         unput_queue_iterator<IteratorT, token_type, ContainerT> const &last,
0178         bool& seen_newline, bool expand_operator_defined,
0179         bool expand_operator_has_include,
0180         boost::optional<position_type> expanding_pos);
0181
0182 //  Collect all arguments supplied to a macro invocation
0183     template <typename IteratorT, typename ContainerT, typename SizeT>
0184     typename std::vector<ContainerT>::size_type collect_arguments (
0185         token_type const curr_token, std::vector<ContainerT> &arguments,
0186         IteratorT &next, IteratorT &endparen, IteratorT const &end,
0187         SizeT const &parameter_count, bool& seen_newline);
0188
0189     //  Expand a single macro name
0190     template <typename IteratorT, typename ContainerT>
0191     bool expand_macro(ContainerT &pending, token_type const &name,
0192         typename defined_macros_type::iterator it,
0193         IteratorT &first, IteratorT const &last,
0194         bool& seen_newline, bool expand_operator_defined,
0195         bool expand_operator_has_include,
0196         boost::optional<position_type> expanding_pos,
0197         defined_macros_type *scope = 0, ContainerT *queue_symbol = 0);
0198
0199     //  Expand a predefined macro (__LINE__, __FILE__ and __INCLUDE_LEVEL__)
0200     template <typename ContainerT>
0201     bool expand_predefined_macro(token_type const &curr_token,
0202         ContainerT &expanded);
0203
0204     //  Expand a single macro argument
0205     template <typename ContainerT>
0206     void expand_argument (typename std::vector<ContainerT>::size_type arg,
0207         std::vector<ContainerT> &arguments,
0208         std::vector<ContainerT> &expanded_args, bool expand_operator_defined,
0209         bool expand_operator_has_include,
0210         std::vector<bool> &has_expanded_args);
0211
0212     //  Expand the replacement list (replaces parameters with arguments)
0213     template <typename ContainerT>
0214     void expand_replacement_list(
0215         typename macro_definition_type::const_definition_iterator_t cbeg,
0216         typename macro_definition_type::const_definition_iterator_t cend,
0217         std::vector<ContainerT> &arguments,
0218         bool expand_operator_defined,
0219         bool expand_operator_has_include,
0220         ContainerT &expanded);
0221
0222     //  Rescans the replacement list for macro expansion
0223     template <typename IteratorT, typename ContainerT>
0224     void rescan_replacement_list(token_type const &curr_token,
0225         macro_definition_type &macrodef, ContainerT &replacement_list,
0226         ContainerT &expanded, bool expand_operator_defined,
0227         bool expand_operator_has_include,
0228         IteratorT &nfirst, IteratorT const &nlast);
0229
0230     //  Resolves the operator defined() and replaces the token with "0" or "1"
0231     template <typename IteratorT, typename ContainerT>
0232     token_type const &resolve_defined(IteratorT &first, IteratorT const &last,
0233         ContainerT &expanded);
0234
0235 #if BOOST_WAVE_SUPPORT_HAS_INCLUDE != 0
0236     //  Resolves the operator __has_include() and replaces the token with "0" or "1"
0237     template <typename IteratorT, typename ContainerT>
0238     token_type const &resolve_has_include(IteratorT &first, IteratorT const &last,
0239         ContainerT &expanded);
0240 #endif
0241
0242     //  Resolve operator _Pragma or the #pragma directive
0243     template <typename IteratorT, typename ContainerT>
0244     bool resolve_operator_pragma(IteratorT &first,
0245         IteratorT const &last, ContainerT &expanded, bool& seen_newline);
0246
0247     //  Handle the concatenation operator '##'
0248     template <typename ContainerT>
0249     bool concat_tokensequence(ContainerT &expanded);
0250
0251     template <typename ContainerT>
0252     bool is_valid_concat(string_type new_value,
0253         position_type const &pos, ContainerT &rescanned);
0254
0255     static bool is_space(char);
0256
0257     // batch update tokens with a single expand position
0258     template <typename ContainerT>
0259     static void set_expand_positions(ContainerT &tokens, position_type pos);
0260
0261 #if BOOST_WAVE_SERIALIZATION != 0
0262 public:
0263     BOOST_STATIC_CONSTANT(unsigned int, version = 0x10);
0264     BOOST_STATIC_CONSTANT(unsigned int, version_mask = 0x0f);
0265
0266 private:
0267     friend class boost::serialization::access;
0268     template<typename Archive>
0269     void save(Archive &ar, const unsigned int version) const
0270     {
0271         using namespace boost::serialization;
0272         ar & make_nvp("defined_macros", defined_macros);
0273     }
0274     template<typename Archive>
0275     void load(Archive &ar, const unsigned int loaded_version)
0276     {
0277         using namespace boost::serialization;
0278         if (version != (loaded_version & ~version_mask)) {
0279             BOOST_WAVE_THROW(preprocess_exception, incompatible_config,
0280                 "cpp_context state version", get_main_pos());
0281         }
0282         ar & make_nvp("defined_macros", defined_macros);
0283         current_macros = defined_macros.get();
0284     }
0285     BOOST_SERIALIZATION_SPLIT_MEMBER()
0286 #endif
0287
0288 private:
0289     defined_macros_type *current_macros;                   // current symbol table
0290     boost::shared_ptr<defined_macros_type> defined_macros; // global symbol table
0291
0292     token_type act_token;       // current token
0293     position_type main_pos;     // last token position in the pp_iterator
0294     string_type base_name;      // the name to be expanded by __BASE_FILE__
0295     ContextT &ctx;              // context object associated with the macromap
0296     long macro_uid;
0297     predefined_macros predef;   // predefined macro support
0298 };
0299 ///////////////////////////////////////////////////////////////////////////////
0300
0301 ///////////////////////////////////////////////////////////////////////////////
0302 //
0303 //  add_macro(): adds a new macro to the macromap
0304 //
0305 ///////////////////////////////////////////////////////////////////////////////
0306 template <typename ContextT>
0307 inline bool
0308 macromap<ContextT>::add_macro(token_type const &name, bool has_parameters,
0309     parameter_container_type &parameters, definition_container_type &definition,
0310     bool is_predefined, defined_macros_type *scope)
0311 {
0312     if (!is_predefined && impl::is_special_macroname (ctx, name.get_value())) {
0313         // exclude special macro names
0314         BOOST_WAVE_THROW_NAME_CTX(ctx, macro_handling_exception,
0315             illegal_redefinition, name.get_value().c_str(), main_pos,
0316             name.get_value().c_str());
0317         return false;
0318     }
0319     if (boost::wave::need_variadics(ctx.get_language()) &&
0320         "__VA_ARGS__" == name.get_value())
0321     {
0322         // can't use __VA_ARGS__ as a macro name
0323         BOOST_WAVE_THROW_NAME_CTX(ctx, macro_handling_exception,
0324             bad_define_statement_va_args, name.get_value().c_str(), main_pos,
0325             name.get_value().c_str());
0326         return false;
0327     }
0328     if (boost::wave::need_variadics(ctx.get_language()) &&
0329         "__VA_OPT__" == name.get_value())
0330     {
0331         // can't use __VA_OPT__ as a macro name
0332         BOOST_WAVE_THROW_NAME_CTX(ctx, macro_handling_exception,
0333             bad_define_statement_va_opt, name.get_value().c_str(), main_pos,
0334             name.get_value().c_str());
0335         return false;
0336     }
0337 #if BOOST_WAVE_SUPPORT_HAS_INCLUDE != 0
0338     if (boost::wave::need_has_include(ctx.get_language()) &&
0339         "__has_include" == name.get_value())
0340     {
0341         // can't use __has_include as a macro name
0342         BOOST_WAVE_THROW_NAME_CTX(ctx, macro_handling_exception,
0343             bad_define_statement_va_opt, name.get_value().c_str(), main_pos,
0344             name.get_value().c_str());
0345         return false;
0346     }
0347 #endif
0348     if (AltExtTokenType == (token_id(name) & ExtTokenOnlyMask)) {
0349         // exclude special operator names
0350         BOOST_WAVE_THROW_NAME_CTX(ctx, macro_handling_exception,
0351             illegal_operator_redefinition, name.get_value().c_str(), main_pos,
0352             name.get_value().c_str());
0353         return false;
0354     }
0355
0356     // try to define the new macro
0357     defined_macros_type* current_scope = scope ? scope : current_macros;
0358     typename defined_macros_type::iterator it = current_scope->find(name.get_value());
0359
0360     if (it != current_scope->end()) {
0361         // redefinition, should not be different
0362         macro_definition_type* macrodef = (*it).second.get();
0363         if (macrodef->is_functionlike != has_parameters ||
0364             !impl::parameters_equal(macrodef->macroparameters, parameters) ||
0365             !impl::definition_equals(macrodef->macrodefinition, definition))
0366         {
0367             BOOST_WAVE_THROW_NAME_CTX(ctx, macro_handling_exception,
0368                 macro_redefinition, name.get_value().c_str(), main_pos,
0369                 name.get_value().c_str());
0370         }
0371         return false;
0372     }
0373
0374     // test the validity of the parameter names
0375     if (has_parameters) {
0376         std::set<typename token_type::string_type> names;
0377
0378         typedef typename parameter_container_type::iterator
0379             parameter_iterator_type;
0380         typedef typename std::set<typename token_type::string_type>::iterator
0381             name_iterator_type;
0382
0383         parameter_iterator_type end = parameters.end();
0384         for (parameter_iterator_type itp = parameters.begin(); itp != end; ++itp)
0385         {
0386         name_iterator_type pit = names.find((*itp).get_value());
0387
0388             if (pit != names.end()) {
0389                 // duplicate parameter name
0390                 BOOST_WAVE_THROW_NAME_CTX(ctx, macro_handling_exception,
0391                     duplicate_parameter_name, (*pit).c_str(), main_pos,
0392                     name.get_value().c_str());
0393                 return false;
0394             }
0395             names.insert((*itp).get_value());
0396         }
0397     }
0398
0399 #if BOOST_WAVE_SUPPORT_VA_OPT != 0
0400     // check that __VA_OPT__ is used as a function macro
0401     if (boost::wave::need_va_opt(ctx.get_language())) {
0402         // __VA_OPT__, if present, must be followed by an lparen
0403         typedef typename macro_definition_type::const_definition_iterator_t iter_t;
0404         iter_t mdit = definition.begin();
0405         iter_t mdend = definition.end();
0406         for (; mdit != mdend; ++mdit) {
0407             // is this va_opt?
0408             if ((IS_EXTCATEGORY((*mdit), OptParameterTokenType)) ||  // if params replaced
0409                 ("__VA_OPT__" == (*mdit).get_value())) {             // if not
0410                 iter_t va_opt_it = mdit;
0411                 // next must be lparen
0412                 if ((++mdit == mdend) ||                             // no further tokens
0413                     (T_LEFTPAREN != token_id(*mdit))) {              // not lparen
0414                     BOOST_WAVE_THROW_NAME_CTX(ctx, macro_handling_exception,
0415                         bad_define_statement_va_opt_parens,
0416                         name.get_value().c_str(), main_pos,
0417                         name.get_value().c_str());
0418                     return false;
0419                 }
0420                 // check that no __VA_OPT__ appears inside
0421                 iter_t va_opt_end = va_opt_it;
0422                 if (!impl::find_va_opt_args(va_opt_end, mdend)) {
0423                     BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
0424                         improperly_terminated_macro, "missing ')' in __VA_OPT__",
0425                         main_pos);
0426                     return false;
0427                 }
0428                 // skip initial __VA_OPT__ and lparen
0429                 ++va_opt_it; ++va_opt_it;
0430                 for (;va_opt_it != va_opt_end; ++va_opt_it) {
0431                     if ((IS_EXTCATEGORY((*va_opt_it), OptParameterTokenType)) ||
0432                         ("__VA_OPT__" == (*va_opt_it).get_value())) {
0433                         BOOST_WAVE_THROW_NAME_CTX(ctx, macro_handling_exception,
0434                             bad_define_statement_va_opt_recurse,
0435                             name.get_value().c_str(), (*va_opt_it).get_position(),
0436                             name.get_value().c_str());
0437                     }
0438                 }
0439             }
0440         }
0441     }
0442 #endif
0443
0444     // insert a new macro node
0445     std::pair<typename defined_macros_type::iterator, bool> p =
0446         current_scope->insert(
0447             typename defined_macros_type::value_type(
0448                 name.get_value(),
0449                 macro_ref_type(new macro_definition_type(name,
0450                     has_parameters, is_predefined, ++macro_uid)
0451                 )
0452             )
0453         );
0454
0455     if (!p.second) {
0456         BOOST_WAVE_THROW_NAME_CTX(ctx, macro_handling_exception,
0457             macro_insertion_error, name.get_value().c_str(), main_pos,
0458             name.get_value().c_str());
0459         return false;
0460     }
0461
0462     // add the parameters and the definition
0463     std::swap((*p.first).second->macroparameters, parameters);
0464     std::swap((*p.first).second->macrodefinition, definition);
0465
0466 // call the context supplied preprocessing hook
0467     ctx.get_hooks().defined_macro(ctx.derived(), name, has_parameters,
0468         (*p.first).second->macroparameters,
0469         (*p.first).second->macrodefinition, is_predefined);
0470     return true;
0471 }
0472
0473 ///////////////////////////////////////////////////////////////////////////////
0474 //
0475 //  is_defined(): returns, whether a given macro is already defined
0476 //
0477 ///////////////////////////////////////////////////////////////////////////////
0478 template <typename ContextT>
0479 inline bool
0480 macromap<ContextT>::is_defined(typename token_type::string_type const &name,
0481     typename defined_macros_type::iterator &it,
0482     defined_macros_type *scope) const
0483 {
0484     if (0 == scope) scope = current_macros;
0485
0486     if ((it = scope->find(name)) != scope->end())
0487         return true;        // found in symbol table
0488
0489     // quick pre-check
0490     if (name.size() < 8 || '_' != name[0] || '_' != name[1])
0491         return false;       // quick check failed
0492
0493     if (name == "__LINE__" || name == "__FILE__" ||
0494         name == "__INCLUDE_LEVEL__")
0495         return true;
0496
0497 #if BOOST_WAVE_SUPPORT_HAS_INCLUDE != 0
0498     return (boost::wave::need_has_include(ctx.get_language()) &&
0499             (name == "__has_include"));
0500 #else
0501     return false;
0502 #endif
0503 }
0504
0505 template <typename ContextT>
0506 template <typename IteratorT>
0507 inline bool
0508 macromap<ContextT>::is_defined(IteratorT const &begin,
0509     IteratorT const &end) const
0510 {
0511     // in normal mode the name under inspection should consist of an identifier
0512     // only
0513     token_id id = token_id(*begin);
0514
0515     if (T_IDENTIFIER != id &&
0516         !IS_CATEGORY(id, KeywordTokenType) &&
0517         !IS_EXTCATEGORY(id, OperatorTokenType|AltExtTokenType) &&
0518         !IS_CATEGORY(id, BoolLiteralTokenType))
0519     {
0520         std::string msg(impl::get_full_name(begin, end));
0521         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, invalid_macroname,
0522             msg.c_str(), main_pos);
0523         return false;
0524     }
0525
0526     IteratorT it = begin;
0527     string_type name((*it).get_value());
0528     typename defined_macros_type::iterator cit;
0529
0530     if (++it != end) {
0531         // there should be only one token as the inspected name
0532         std::string msg(impl::get_full_name(begin, end));
0533         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, invalid_macroname,
0534             msg.c_str(), main_pos);
0535         return false;
0536     }
0537     return is_defined(name, cit, 0);
0538 }
0539
0540 ///////////////////////////////////////////////////////////////////////////////
0541 //  same as above, only takes an arbitrary string type as its parameter
0542 template <typename ContextT>
0543 inline bool
0544 macromap<ContextT>::is_defined(string_type const &str) const
0545 {
0546     typename defined_macros_type::iterator cit;
0547     return is_defined(str, cit, 0);
0548 }
0549
0550 #if BOOST_WAVE_SUPPORT_HAS_INCLUDE != 0
0551 ///////////////////////////////////////////////////////////////////////////////
0552 //
0553 //  has_include(): returns whether a path expression is an valid include file
0554 //
0555 ///////////////////////////////////////////////////////////////////////////////
0556 template <typename ContextT>
0557 template <typename IteratorT>
0558 inline bool
0559 macromap<ContextT>::has_include(
0560     IteratorT const &begin, IteratorT const &end,
0561     bool is_quoted_filename, bool is_system) const
0562 {
0563     typename ContextT::token_sequence_type filetoks;
0564
0565     if (is_quoted_filename) {
0566         filetoks = typename ContextT::token_sequence_type(begin, end);
0567     } else {
0568         IteratorT first = begin;
0569         IteratorT last = end;
0570         ctx.expand_whole_tokensequence(first, last, filetoks);
0571     }
0572
0573     // extract tokens into string and trim whitespace
0574     using namespace boost::wave::util::impl;
0575     std::string fn(trim_whitespace(as_string(filetoks)).c_str());
0576
0577     // verify and remove initial and final delimiters
0578     if (!((fn.size() >= 3) &&
0579           (((fn[0] == '"') && (*fn.rbegin() == '"')) ||
0580            ((fn[0] == '<') && (*fn.rbegin() == '>')))))
0581         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, bad_has_include_expression,
0582                              fn.c_str(), ctx.get_main_pos());
0583
0584     fn = fn.substr(1, fn.size() - 2);
0585
0586     // test header existence
0587     std::string dir_path;
0588     std::string native_path;
0589     return ctx.get_hooks().locate_include_file(
0590         ctx, fn, is_system, 0, dir_path, native_path);
0591
0592 }
0593 #endif
0594
0595 ///////////////////////////////////////////////////////////////////////////////
0596 //
0597 //  Get the macro definition for the given macro scope
0598 //
0599 ///////////////////////////////////////////////////////////////////////////////
0600 template <typename ContextT>
0601 inline bool
0602 macromap<ContextT>::get_macro(string_type const &name, bool &has_parameters,
0603     bool &is_predefined, position_type &pos,
0604     parameter_container_type &parameters,
0605     definition_container_type &definition,
0606     defined_macros_type *scope) const
0607 {
0608     typename defined_macros_type::iterator it;
0609     if (!is_defined(name, it, scope))
0610         return false;
0611
0612     macro_definition_type& macro_def = *(*it).second.get();
0613
0614     has_parameters = macro_def.is_functionlike;
0615     is_predefined = macro_def.is_predefined;
0616     pos = macro_def.macroname.get_position();
0617     parameters = macro_def.macroparameters;
0618     definition = macro_def.macrodefinition;
0619     return true;
0620 }
0621
0622 ///////////////////////////////////////////////////////////////////////////////
0623 //
0624 //  remove_macro(): remove a macro from the macromap
0625 //
0626 ///////////////////////////////////////////////////////////////////////////////
0627 template <typename ContextT>
0628 inline bool
0629 macromap<ContextT>::remove_macro(string_type const &name,
0630     position_type const& pos, bool even_predefined)
0631 {
0632     typename defined_macros_type::iterator it = current_macros->find(name);
0633
0634     if (it != current_macros->end()) {
0635         if ((*it).second->is_predefined) {
0636             if (!even_predefined || impl::is_special_macroname(ctx, name)) {
0637                 BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
0638                     bad_undefine_statement, name.c_str(), main_pos);
0639                 return false;
0640             }
0641         }
0642         current_macros->erase(it);
0643
0644         // call the context supplied preprocessing hook function
0645         token_type tok(T_IDENTIFIER, name, pos);
0646
0647         ctx.get_hooks().undefined_macro(ctx.derived(), tok);
0648         return true;
0649     }
0650     else if (impl::is_special_macroname(ctx, name)) {
0651         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, bad_undefine_statement,
0652             name.c_str(), pos);
0653     }
0654     return false;       // macro was not defined
0655 }
0656
0657 ///////////////////////////////////////////////////////////////////////////////
0658 //
0659 //  expand_tokensequence
0660 //
0661 //      This function is a helper function which wraps the given iterator
0662 //      range into corresponding unput_iterator's and calls the main workhorse
0663 //      of the macro expansion engine (the function expand_tokensequence_worker)
0664 //
0665 //      This is the top level macro expansion function called from the
0666 //      preprocessing iterator component only.
0667 //
0668 ///////////////////////////////////////////////////////////////////////////////
0669 template <typename ContextT>
0670 template <typename IteratorT, typename ContainerT>
0671 inline typename ContextT::token_type const &
0672 macromap<ContextT>::expand_tokensequence(IteratorT &first,
0673     IteratorT const &last, ContainerT &pending, ContainerT &expanded,
0674     bool& seen_newline, bool expand_operator_defined,
0675     bool expand_operator_has_include)
0676 {
0677     typedef impl::gen_unput_queue_iterator<IteratorT, token_type, ContainerT>
0678         gen_type;
0679     typedef typename gen_type::return_type iterator_type;
0680
0681     iterator_type first_it = gen_type::generate(expanded, first);
0682     iterator_type last_it = gen_type::generate(last);
0683
0684     on_exit::assign<IteratorT, iterator_type> on_exit(first, first_it);
0685
0686     return expand_tokensequence_worker(pending, first_it, last_it,
0687         seen_newline, expand_operator_defined, expand_operator_has_include,
0688         boost::none);
0689 }
0690
0691 ///////////////////////////////////////////////////////////////////////////////
0692 //
0693 //  expand_tokensequence_worker
0694 //
0695 //      This function is the main workhorse of the macro expansion engine. It
0696 //      expands as much tokens as needed to identify the next preprocessed
0697 //      token to return to the caller.
0698 //      It returns the next preprocessed token.
0699 //
0700 //      The iterator 'first' is adjusted accordingly.
0701 //
0702 ///////////////////////////////////////////////////////////////////////////////
0703 template <typename ContextT>
0704 template <typename IteratorT, typename ContainerT>
0705 inline typename ContextT::token_type const &
0706 macromap<ContextT>::expand_tokensequence_worker(
0707     ContainerT &pending,
0708     unput_queue_iterator<IteratorT, token_type, ContainerT> &first,
0709     unput_queue_iterator<IteratorT, token_type, ContainerT> const &last,
0710     bool& seen_newline, bool expand_operator_defined,
0711     bool expand_operator_has_include,
0712     boost::optional<position_type> expanding_pos)
0713 {
0714     // if there exist pending tokens (tokens, which are already preprocessed), then
0715     // return the next one from there
0716     if (!pending.empty()) {
0717         on_exit::pop_front<definition_container_type> pop_front_token(pending);
0718
0719         return act_token = pending.front();
0720     }
0721
0722     //  analyze the next element of the given sequence, if it is an
0723     //  T_IDENTIFIER token, try to replace this as a macro etc.
0724     using namespace boost::wave;
0725
0726     if (first != last) {
0727         token_id id = token_id(*first);
0728
0729         // ignore placeholder tokens
0730         if (T_PLACEHOLDER == id) {
0731             token_type placeholder = *first;
0732
0733             ++first;
0734             if (first == last)
0735                 return act_token = placeholder;
0736             id = token_id(*first);
0737         }
0738
0739         if (T_IDENTIFIER == id || IS_CATEGORY(id, KeywordTokenType) ||
0740             IS_EXTCATEGORY(id, OperatorTokenType|AltExtTokenType) ||
0741             IS_CATEGORY(id, BoolLiteralTokenType))
0742         {
0743         // try to replace this identifier as a macro
0744             if (expand_operator_defined && (*first).get_value() == "defined") {
0745             // resolve operator defined()
0746                 return resolve_defined(first, last, pending);
0747             }
0748 #if BOOST_WAVE_SUPPORT_HAS_INCLUDE != 0
0749             else if (boost::wave::need_has_include(ctx.get_language()) &&
0750                      expand_operator_has_include &&
0751                      (*first).get_value() == "__has_include") {
0752                 // resolve operator __has_include()
0753                 return resolve_has_include(first, last, pending);
0754             }
0755 #endif
0756             else if (boost::wave::need_variadics(ctx.get_language()) &&
0757                 (*first).get_value() == "_Pragma")
0758             {
0759                 // in C99 mode only: resolve the operator _Pragma
0760                 token_type curr_token = *first;
0761
0762                 if (!resolve_operator_pragma(first, last, pending, seen_newline) ||
0763                     pending.size() > 0)
0764                 {
0765                     // unknown to us pragma or supplied replacement, return the
0766                     // next token
0767                     on_exit::pop_front<definition_container_type> pop_token(pending);
0768
0769                     return act_token = pending.front();
0770                 }
0771
0772                 // the operator _Pragma() was eaten completely, continue
0773                 return act_token = token_type(T_PLACEHOLDER, "_",
0774                     curr_token.get_position());
0775             }
0776
0777             token_type name_token(*first);
0778             typename defined_macros_type::iterator it;
0779
0780             if (is_defined(name_token.get_value(), it)) {
0781                 // the current token contains an identifier, which is currently
0782                 // defined as a macro
0783                 if (expand_macro(pending, name_token, it, first, last,
0784                                  seen_newline, expand_operator_defined,
0785                                  expand_operator_has_include,
0786                                  expanding_pos))
0787                 {
0788                     // the tokens returned by expand_macro should be rescanned
0789                     // beginning at the last token of the returned replacement list
0790                     if (first != last) {
0791                         // splice the last token back into the input queue
0792                         typename ContainerT::reverse_iterator rit = pending.rbegin();
0793
0794                         first.get_unput_queue().splice(
0795                             first.get_unput_queue().begin(), pending,
0796                             (++rit).base(), pending.end());
0797                     }
0798
0799                     // fall through ...
0800                 }
0801                 else if (!pending.empty()) {
0802                     // return the first token from the pending queue
0803                     on_exit::pop_front<definition_container_type> pop_queue(pending);
0804
0805                     return act_token = pending.front();
0806                 }
0807                 else {
0808                     // macro expansion reached the eoi
0809                     return act_token = token_type();
0810                 }
0811
0812                 // return the next preprocessed token
0813                 if (!expanding_pos)
0814                     expanding_pos = name_token.get_expand_position();
0815
0816                 typename ContextT::token_type const & result =
0817                     expand_tokensequence_worker(
0818                         pending, first, last,
0819                         seen_newline, expand_operator_defined,
0820                         expand_operator_has_include,
0821                         expanding_pos);
0822
0823                 return result;
0824             }
0825             else {
0826                 act_token = name_token;
0827                 ++first;
0828                 return act_token;
0829             }
0830         }
0831         else if (expand_operator_defined && IS_CATEGORY(*first, BoolLiteralTokenType)) {
0832             // expanding a constant expression inside #if/#elif, special handling
0833             // of 'true' and 'false'
0834
0835             // all remaining identifiers and keywords, except for true and false,
0836             // are replaced with the pp-number 0 (C++ standard 16.1.4, [cpp.cond])
0837             return act_token = token_type(T_INTLIT, T_TRUE != id ? "0" : "1",
0838                 (*first++).get_position());
0839         }
0840         else {
0841             act_token = *first;
0842             ++first;
0843             return act_token;
0844         }
0845     }
0846     return act_token = token_type();     // eoi
0847 }
0848
0849 ///////////////////////////////////////////////////////////////////////////////
0850 //
0851 //  collect_arguments(): collect the actual arguments of a macro invocation
0852 //
0853 //      return the number of successfully detected non-empty arguments
0854 //
0855 ///////////////////////////////////////////////////////////////////////////////
0856 template <typename ContextT>
0857 template <typename IteratorT, typename ContainerT, typename SizeT>
0858 inline typename std::vector<ContainerT>::size_type
0859 macromap<ContextT>::collect_arguments (token_type const curr_token,
0860     std::vector<ContainerT> &arguments, IteratorT &next, IteratorT &endparen,
0861     IteratorT const &end, SizeT const &parameter_count, bool& seen_newline)
0862 {
0863     using namespace boost::wave;
0864
0865     arguments.push_back(ContainerT());
0866
0867     // collect the actual arguments
0868     typename std::vector<ContainerT>::size_type count_arguments = 0;
0869     int nested_parenthesis_level = 1;
0870     ContainerT* argument = &arguments[0];
0871     bool was_whitespace = false;
0872     token_type startof_argument_list = *next;
0873
0874     while (++next != end && nested_parenthesis_level) {
0875         token_id id = token_id(*next);
0876
0877         if (0 == parameter_count &&
0878             !IS_CATEGORY((*next), WhiteSpaceTokenType) && id != T_NEWLINE &&
0879             id != T_RIGHTPAREN && id != T_LEFTPAREN)
0880         {
0881         // there shouldn't be any arguments
0882             BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
0883                 too_many_macroarguments, curr_token.get_value().c_str(),
0884                 main_pos);
0885             return 0;
0886         }
0887
0888         switch (id) {
0889         case T_LEFTPAREN:
0890             ++nested_parenthesis_level;
0891             argument->push_back(*next);
0892             was_whitespace = false;
0893             break;
0894
0895         case T_RIGHTPAREN:
0896             {
0897                 if (--nested_parenthesis_level >= 1)
0898                     argument->push_back(*next);
0899                 else {
0900                 // found closing parenthesis
0901 //                    trim_sequence(argument);
0902                     endparen = next;
0903                     if (parameter_count > 0) {
0904                         if (argument->empty() ||
0905                             impl::is_whitespace_only(*argument))
0906                         {
0907 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
0908                             if (boost::wave::need_variadics(ctx.get_language())) {
0909                             // store a placemarker as the argument
0910                                 argument->push_back(token_type(T_PLACEMARKER, "\xA7",
0911                                     (*next).get_position()));
0912                                 ++count_arguments;
0913                             }
0914 #endif
0915                         }
0916                         else {
0917                             ++count_arguments;
0918                         }
0919                     }
0920                 }
0921                 was_whitespace = false;
0922             }
0923             break;
0924
0925         case T_COMMA:
0926             if (1 == nested_parenthesis_level) {
0927                 // next parameter
0928 //                trim_sequence(argument);
0929                 if (argument->empty() ||
0930                     impl::is_whitespace_only(*argument))
0931                 {
0932 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
0933                     if (boost::wave::need_variadics(ctx.get_language())) {
0934                         // store a placemarker as the argument
0935                         argument->push_back(token_type(T_PLACEMARKER, "\xA7",
0936                             (*next).get_position()));
0937                         ++count_arguments;
0938                     }
0939 #endif
0940                 }
0941                 else {
0942                     ++count_arguments;
0943                 }
0944                 arguments.push_back(ContainerT()); // add new arg
0945                 argument = &arguments[arguments.size()-1];
0946             }
0947             else {
0948                 // surrounded by parenthesises, so store to current argument
0949                 argument->push_back(*next);
0950             }
0951             was_whitespace = false;
0952             break;
0953
0954         case T_NEWLINE:
0955             seen_newline = true;
0956             /* fall through */
0957         case T_SPACE:
0958         case T_SPACE2:
0959         case T_CCOMMENT:
0960             if (!was_whitespace)
0961                 argument->push_back(token_type(T_SPACE, " ", (*next).get_position()));
0962             was_whitespace = true;
0963             break;      // skip whitespace
0964
0965         case T_PLACEHOLDER:
0966             break;      // ignore placeholder
0967
0968         default:
0969             argument->push_back(*next);
0970             was_whitespace = false;
0971             break;
0972         }
0973     }
0974
0975     if (nested_parenthesis_level >= 1) {
0976         // missing ')': improperly terminated macro invocation
0977         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
0978             improperly_terminated_macro, "missing ')'", main_pos);
0979         return 0;
0980     }
0981
0982     // if no argument was expected and we didn't find any, than remove the empty
0983     // element
0984     if (0 == parameter_count && 0 == count_arguments) {
0985         BOOST_ASSERT(1 == arguments.size());
0986         arguments.clear();
0987     }
0988     return count_arguments;
0989 }
0990
0991 ///////////////////////////////////////////////////////////////////////////////
0992 //
0993 //  expand_whole_tokensequence
0994 //
0995 //      fully expands a given token sequence
0996 //
0997 ///////////////////////////////////////////////////////////////////////////////
0998 template <typename ContextT>
0999 template <typename IteratorT, typename ContainerT>
1000 inline void
1001 macromap<ContextT>::expand_whole_tokensequence(ContainerT &expanded,
1002                                                IteratorT &first, IteratorT const &last,
1003                                                bool expand_operator_defined,
1004                                                bool expand_operator_has_include)
1005 {
1006     typedef impl::gen_unput_queue_iterator<IteratorT, token_type, ContainerT>
1007         gen_type;
1008     typedef typename gen_type::return_type iterator_type;
1009
1010     ContainerT empty;
1011     iterator_type first_it = gen_type::generate(empty, first);
1012     iterator_type last_it = gen_type::generate(last);
1013
1014     on_exit::assign<IteratorT, iterator_type> on_exit(first, first_it);
1015     ContainerT pending_queue;
1016     bool seen_newline;
1017
1018     while (!pending_queue.empty() || first_it != last_it) {
1019         expanded.push_back(
1020             expand_tokensequence_worker(
1021                 pending_queue, first_it,
1022                 last_it, seen_newline, expand_operator_defined,
1023                 expand_operator_has_include,
1024                 boost::none)
1025         );
1026     }
1027
1028     // should have returned all expanded tokens
1029     BOOST_ASSERT(pending_queue.empty()/* && unput_queue.empty()*/);
1030 }
1031
1032 ///////////////////////////////////////////////////////////////////////////////
1033 //
1034 //  expand_argument
1035 //
1036 //      fully expands the given argument of a macro call
1037 //
1038 ///////////////////////////////////////////////////////////////////////////////
1039 template <typename ContextT>
1040 template <typename ContainerT>
1041 inline void
1042 macromap<ContextT>::expand_argument (
1043     typename std::vector<ContainerT>::size_type arg,
1044     std::vector<ContainerT> &arguments, std::vector<ContainerT> &expanded_args,
1045     bool expand_operator_defined, bool expand_operator_has_include,
1046     std::vector<bool> &has_expanded_args)
1047 {
1048     if (!has_expanded_args[arg]) {
1049         // expand the argument only once
1050         typedef typename std::vector<ContainerT>::value_type::iterator
1051             argument_iterator_type;
1052
1053         argument_iterator_type begin_it = arguments[arg].begin();
1054         argument_iterator_type end_it = arguments[arg].end();
1055
1056         expand_whole_tokensequence(
1057             expanded_args[arg], begin_it, end_it,
1058             expand_operator_defined, expand_operator_has_include);
1059         impl::remove_placeholders(expanded_args[arg]);
1060         has_expanded_args[arg] = true;
1061     }
1062 }
1063
1064 ///////////////////////////////////////////////////////////////////////////////
1065 //
1066 //  expand_replacement_list
1067 //
1068 //      fully expands the replacement list of a given macro with the
1069 //      actual arguments/expanded arguments
1070 //      handles the '#' [cpp.stringize] and the '##' [cpp.concat] operator
1071 //
1072 ///////////////////////////////////////////////////////////////////////////////
1073 template <typename ContextT>
1074 template <typename ContainerT>
1075 inline void
1076 macromap<ContextT>::expand_replacement_list(
1077     typename macro_definition_type::const_definition_iterator_t cit,
1078     typename macro_definition_type::const_definition_iterator_t cend,
1079     std::vector<ContainerT> &arguments, bool expand_operator_defined,
1080     bool expand_operator_has_include,
1081     ContainerT &expanded)
1082 {
1083     using namespace boost::wave;
1084     typedef typename macro_definition_type::const_definition_iterator_t
1085         macro_definition_iter_t;
1086
1087     std::vector<ContainerT> expanded_args(arguments.size());
1088     std::vector<bool> has_expanded_args(arguments.size());
1089     bool seen_concat = false;
1090     bool adjacent_concat = false;
1091     bool adjacent_stringize = false;
1092
1093     for (;cit != cend; ++cit)
1094     {
1095         bool use_replaced_arg = true;
1096         token_id base_id = BASE_TOKEN(token_id(*cit));
1097
1098         if (T_POUND_POUND == base_id) {
1099             // concatenation operator
1100             adjacent_concat = true;
1101             seen_concat = true;
1102         }
1103         else if (T_POUND == base_id) {
1104             // stringize operator
1105             adjacent_stringize = true;
1106         }
1107         else {
1108             if (adjacent_stringize || adjacent_concat ||
1109                 T_POUND_POUND == impl::next_token<macro_definition_iter_t>
1110                     ::peek(cit, cend))
1111             {
1112                 use_replaced_arg = false;
1113             }
1114             if (adjacent_concat)    // spaces after '##' ?
1115                 adjacent_concat = IS_CATEGORY(*cit, WhiteSpaceTokenType);
1116         }
1117
1118         if (IS_CATEGORY((*cit), ParameterTokenType)) {
1119             // copy argument 'i' instead of the parameter token i
1120             typename ContainerT::size_type i;
1121 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
1122             bool is_ellipsis = false;
1123 #if BOOST_WAVE_SUPPORT_VA_OPT != 0
1124             bool is_va_opt = false;
1125 #endif
1126
1127             if (IS_EXTCATEGORY((*cit), ExtParameterTokenType)) {
1128                 BOOST_ASSERT(boost::wave::need_variadics(ctx.get_language()));
1129                 i = token_id(*cit) - T_EXTPARAMETERBASE;
1130                 is_ellipsis = true;
1131             }
1132             else
1133 #if BOOST_WAVE_SUPPORT_VA_OPT != 0
1134
1135             if (IS_EXTCATEGORY((*cit), OptParameterTokenType)) {
1136                 BOOST_ASSERT(boost::wave::need_va_opt(ctx.get_language()));
1137                 i = token_id(*cit) - T_OPTPARAMETERBASE;
1138                 is_va_opt = true;
1139             }
1140             else
1141 #endif
1142 #endif
1143             {
1144                 i = token_id(*cit) - T_PARAMETERBASE;
1145             }
1146
1147             BOOST_ASSERT(i <= arguments.size());
1148             if (use_replaced_arg) {
1149
1150 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
1151                 if (is_ellipsis) {
1152                     position_type const& pos = (*cit).get_position();
1153
1154                     BOOST_ASSERT(boost::wave::need_variadics(ctx.get_language()));
1155
1156                     // ensure all variadic arguments to be expanded
1157                     for (typename vector<ContainerT>::size_type arg = i;
1158                          arg < expanded_args.size(); ++arg)
1159                     {
1160                         expand_argument(
1161                             arg, arguments, expanded_args,
1162                             expand_operator_defined, expand_operator_has_include,
1163                             has_expanded_args);
1164                     }
1165                     impl::replace_ellipsis(expanded_args, i, expanded, pos);
1166                 }
1167                 else
1168
1169 #if BOOST_WAVE_SUPPORT_VA_OPT != 0
1170                 if (is_va_opt) {
1171                     position_type const &pos = (*cit).get_position();
1172
1173                     BOOST_ASSERT(boost::wave::need_va_opt(ctx.get_language()));
1174
1175                     // ensure all variadic arguments to be expanded
1176                     for (typename vector<ContainerT>::size_type arg = i;
1177                          arg < expanded_args.size(); ++arg)
1178                     {
1179                         expand_argument(
1180                             arg, arguments, expanded_args,
1181                             expand_operator_defined, expand_operator_has_include,
1182                             has_expanded_args);
1183                     }
1184
1185                     // locate the end of the __VA_OPT__ call
1186                     typename macro_definition_type::const_definition_iterator_t cstart = cit;
1187                     if (!impl::find_va_opt_args(cit, cend)) {
1188                         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
1189                             improperly_terminated_macro, "missing '(' or ')' in __VA_OPT__",
1190                             pos);
1191                     }
1192                     // cstart still points to __VA_OPT__; cit now points to the last rparen
1193
1194                     // locate the __VA_OPT__ arguments
1195                     typename macro_definition_type::const_definition_iterator_t arg_start = cstart;
1196                     ++arg_start;  // skip __VA_OPT__
1197                     ++arg_start;  // skip lparen
1198
1199                     // create a synthetic macro definition for use with hooks
1200                     token_type macroname(T_IDENTIFIER, "__VA_OPT__", position_type("<built-in>"));
1201                     parameter_container_type macroparameters;
1202                     macroparameters.push_back(token_type(T_ELLIPSIS, "...", position_type("<built-in>")));
1203                     definition_container_type macrodefinition;
1204
1205                     bool suppress_expand = false;
1206                     // __VA_OPT__ treats its arguments as an undifferentiated stream of tokens
1207                     // for our purposes we can consider it as a single argument
1208                     typename std::vector<ContainerT> va_opt_args(1, ContainerT(arg_start, cit));
1209                     suppress_expand = ctx.get_hooks().expanding_function_like_macro(
1210                         ctx.derived(),
1211                         macroname, macroparameters, macrodefinition,
1212                         *cstart, va_opt_args,
1213                         cstart, cit);
1214
1215                     if (suppress_expand) {
1216                         // leave the whole expression in place
1217                         std::copy(cstart, cit, std::back_inserter(expanded));
1218                         expanded.push_back(*cit);  // include the rparen
1219                     } else {
1220                         ContainerT va_expanded;
1221                         if ((i == arguments.size()) ||                 // no variadic argument
1222                             impl::is_whitespace_only(arguments[i])) {  // no visible tokens
1223                             // no args; insert placemarker
1224                             va_expanded.push_back(
1225                                 typename ContainerT::value_type(T_PLACEMARKER, "\xA7", pos));
1226                         } else if (!impl::is_blank_only(arguments[i])) {
1227                             // [cstart, cit) is now the args to va_opt
1228                             // recursively process them
1229                             expand_replacement_list(arg_start, cit, arguments,
1230                                                     expand_operator_defined,
1231                                                     expand_operator_has_include,
1232                                                     va_expanded);
1233                         }
1234                         // run final hooks
1235                         ctx.get_hooks().expanded_macro(ctx.derived(), va_expanded);
1236
1237                         // updated overall expansion with va_opt results
1238                         expanded.splice(expanded.end(), va_expanded);
1239                     }
1240                     // continue from rparen
1241                 }
1242                 else
1243
1244 #endif
1245 #endif
1246                 {
1247                     BOOST_ASSERT(i < arguments.size());
1248                     // ensure argument i to be expanded
1249                     expand_argument(
1250                         i, arguments, expanded_args,
1251                         expand_operator_defined, expand_operator_has_include,
1252                         has_expanded_args);
1253
1254                     // replace argument
1255                     BOOST_ASSERT(i < expanded_args.size());
1256                     ContainerT const& arg = expanded_args[i];
1257
1258                     std::copy(arg.begin(), arg.end(),
1259                         std::inserter(expanded, expanded.end()));
1260                 }
1261             }
1262             else if (adjacent_stringize &&
1263                     !IS_CATEGORY(*cit, WhiteSpaceTokenType))
1264             {
1265 #if BOOST_WAVE_SUPPORT_CPP2A != 0
1266                 if (i >= arguments.size()) {
1267                     // no argument supplied; do nothing (only c20 should reach here)
1268                     BOOST_ASSERT(boost::wave::need_cpp2a(ctx.get_language()));
1269                     position_type last_valid(arguments.back().back().get_position());
1270                     // insert a empty string
1271                     expanded.push_back(token_type(T_STRINGLIT, "\"\"", last_valid));
1272                 }
1273                 else
1274 #endif
1275                 {
1276                     // shouldn't be oob (w.o. cpp20)
1277                     BOOST_ASSERT(i < arguments.size() && !arguments[i].empty());
1278                     // safe a copy of the first tokens position (not a reference!)
1279                     position_type pos((*arguments[i].begin()).get_position());
1280
1281 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
1282                     if (is_ellipsis && boost::wave::need_variadics(ctx.get_language())) {
1283                         impl::trim_sequence_left(arguments[i]);
1284                         impl::trim_sequence_right(arguments.back());
1285                         expanded.push_back(token_type(T_STRINGLIT,
1286                             impl::as_stringlit(arguments, i, pos), pos));
1287                     }
1288                     else
1289 #endif
1290                     {
1291                         impl::trim_sequence(arguments[i]);
1292                         expanded.push_back(token_type(T_STRINGLIT,
1293                             impl::as_stringlit(arguments[i], pos), pos));
1294                     }
1295                 }
1296                 adjacent_stringize = false;
1297             }
1298             else {
1299                 // simply copy the original argument (adjacent '##' or '#')
1300 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
1301                 if (is_ellipsis) {
1302                     position_type const& pos = (*cit).get_position();
1303 #if BOOST_WAVE_SUPPORT_CPP2A != 0
1304                     if (i < arguments.size())
1305 #endif
1306                     {
1307
1308                         impl::trim_sequence_left(arguments[i]);
1309                         impl::trim_sequence_right(arguments.back());
1310                         BOOST_ASSERT(boost::wave::need_variadics(ctx.get_language()));
1311                         impl::replace_ellipsis(arguments, i, expanded, pos);
1312                     }
1313 #if BOOST_WAVE_SUPPORT_CPP2A != 0
1314                     else if (boost::wave::need_cpp2a(ctx.get_language())) {
1315                         BOOST_ASSERT(i == arguments.size());
1316                         // no argument supplied; insert placemarker
1317                         expanded.push_back(
1318                             typename ContainerT::value_type(T_PLACEMARKER, "\xA7", pos));
1319                     }
1320 #endif
1321                 }
1322                 else
1323 #endif
1324                 {
1325                     ContainerT& arg = arguments[i];
1326
1327                     impl::trim_sequence(arg);
1328                     std::copy(arg.begin(), arg.end(),
1329                         std::inserter(expanded, expanded.end()));
1330                 }
1331             }
1332         }
1333         else if (!adjacent_stringize || T_POUND != base_id) {
1334             // insert the actual replacement token (if it is not the '#' operator)
1335             expanded.push_back(*cit);
1336         }
1337     }
1338
1339     if (adjacent_stringize) {
1340         // error, '#' should not be the last token
1341         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, ill_formed_operator,
1342             "stringize ('#')", main_pos);
1343         return;
1344     }
1345
1346     // handle the cpp.concat operator
1347     if (seen_concat)
1348         concat_tokensequence(expanded);
1349 }
1350
1351 ///////////////////////////////////////////////////////////////////////////////
1352 //
1353 //  rescan_replacement_list
1354 //
1355 //    As the name implies, this function is used to rescan the replacement list
1356 //    after the first macro substitution phase.
1357 //
1358 ///////////////////////////////////////////////////////////////////////////////
1359 template <typename ContextT>
1360 template <typename IteratorT, typename ContainerT>
1361 inline void
1362 macromap<ContextT>::rescan_replacement_list(token_type const &curr_token,
1363     macro_definition_type &macro_def, ContainerT &replacement_list,
1364     ContainerT &expanded,
1365     bool expand_operator_defined,
1366     bool expand_operator_has_include,
1367     IteratorT &nfirst, IteratorT const &nlast)
1368 {
1369     if (!replacement_list.empty()) {
1370 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
1371         // remove the placemarkers
1372         if (boost::wave::need_variadics(ctx.get_language())) {
1373             typename ContainerT::iterator end = replacement_list.end();
1374             typename ContainerT::iterator it = replacement_list.begin();
1375
1376             while (it != end) {
1377                 using namespace boost::wave;
1378                 if (T_PLACEMARKER == token_id(*it)) {
1379                     typename ContainerT::iterator placemarker = it;
1380
1381                     ++it;
1382                     replacement_list.erase(placemarker);
1383                 }
1384                 else {
1385                     ++it;
1386                 }
1387             }
1388         }
1389 #endif
1390
1391         // rescan the replacement list, during this rescan the current macro under
1392         // expansion isn't available as an expandable macro
1393         on_exit::reset<bool> on_exit(macro_def.is_available_for_replacement, false);
1394         typename ContainerT::iterator begin_it = replacement_list.begin();
1395         typename ContainerT::iterator end_it = replacement_list.end();
1396
1397         expand_whole_tokensequence(
1398             expanded, begin_it, end_it,
1399             expand_operator_defined, expand_operator_has_include);
1400
1401         // trim replacement list, leave placeholder tokens untouched
1402         impl::trim_replacement_list(expanded);
1403     }
1404
1405     if (expanded.empty()) {
1406         // the resulting replacement list should contain at least a placeholder
1407         // token
1408         expanded.push_back(token_type(T_PLACEHOLDER, "_", curr_token.get_position()));
1409     }
1410 }
1411
1412 ///////////////////////////////////////////////////////////////////////////////
1413 //
1414 //  expand_macro(): expands a defined macro
1415 //
1416 //      This functions tries to expand the macro, to which points the 'first'
1417 //      iterator. The functions eats up more tokens, if the macro to expand is
1418 //      a function-like macro.
1419 //
1420 ///////////////////////////////////////////////////////////////////////////////
1421 template <typename ContextT>
1422 template <typename IteratorT, typename ContainerT>
1423 inline bool
1424 macromap<ContextT>::expand_macro(ContainerT &expanded,
1425     token_type const &curr_token, typename defined_macros_type::iterator it,
1426     IteratorT &first, IteratorT const &last,
1427     bool& seen_newline, bool expand_operator_defined,
1428     bool expand_operator_has_include,
1429     boost::optional<position_type> expanding_pos,
1430     defined_macros_type *scope, ContainerT *queue_symbol)
1431 {
1432     using namespace boost::wave;
1433
1434     if (0 == scope) scope = current_macros;
1435
1436     BOOST_ASSERT(T_IDENTIFIER == token_id(curr_token) ||
1437         IS_CATEGORY(token_id(curr_token), KeywordTokenType) ||
1438         IS_EXTCATEGORY(token_id(curr_token), OperatorTokenType|AltExtTokenType) ||
1439         IS_CATEGORY(token_id(curr_token), BoolLiteralTokenType));
1440
1441     if (it == scope->end()) {
1442         ++first;    // advance
1443
1444         // try to expand a predefined macro (__FILE__, __LINE__ or __INCLUDE_LEVEL__)
1445         if (expand_predefined_macro(curr_token, expanded))
1446             return false;
1447
1448         // not defined as a macro
1449         if (0 != queue_symbol) {
1450             expanded.splice(expanded.end(), *queue_symbol);
1451         }
1452         else {
1453             expanded.push_back(curr_token);
1454         }
1455         return false;
1456     }
1457
1458     // ensure the parameters to be replaced with special parameter tokens
1459     macro_definition_type& macro_def = *(*it).second.get();
1460
1461     macro_def.replace_parameters(ctx);
1462
1463     // test if this macro is currently available for replacement
1464     if (!macro_def.is_available_for_replacement) {
1465         // this macro is marked as non-replaceable
1466         // copy the macro name itself
1467         if (0 != queue_symbol) {
1468             queue_symbol->push_back(token_type(T_NONREPLACABLE_IDENTIFIER,
1469                 curr_token.get_value(), curr_token.get_position()));
1470             expanded.splice(expanded.end(), *queue_symbol);
1471         }
1472         else {
1473             expanded.push_back(token_type(T_NONREPLACABLE_IDENTIFIER,
1474                 curr_token.get_value(), curr_token.get_position()));
1475         }
1476         ++first;
1477         return false;
1478     }
1479
1480     // try to replace the current identifier as a function-like macro
1481     ContainerT replacement_list;
1482
1483     if (T_LEFTPAREN == impl::next_token<IteratorT>::peek(first, last)) {
1484         // called as a function-like macro
1485         impl::skip_to_token(ctx, first, last, T_LEFTPAREN, seen_newline);
1486
1487         IteratorT seqstart = first;
1488         IteratorT seqend = first;
1489
1490         if (macro_def.is_functionlike) {
1491             // defined as a function-like macro
1492
1493             // collect the arguments
1494             std::vector<ContainerT> arguments;
1495             typename std::vector<ContainerT>::size_type count_args =
1496                 collect_arguments(curr_token, arguments, first, seqend, last,
1497                     macro_def.macroparameters.size(), seen_newline);
1498
1499             std::size_t parm_count_required = macro_def.macroparameters.size();
1500 #if BOOST_WAVE_SUPPORT_CPP2A
1501             if (boost::wave::need_cpp2a(ctx.get_language())) {
1502                 // Starting with C++20, variable arguments may be left out
1503                 // entirely, so reduce the mandatory argument count by one
1504                 // if the last parameter is ellipsis:
1505                 if ((parm_count_required > 0) &&
1506                     (T_ELLIPSIS == token_id(macro_def.macroparameters.back()))) {
1507                     --parm_count_required;
1508                 }
1509             }
1510 #endif
1511
1512             // verify the parameter count
1513             if (count_args < parm_count_required ||
1514                 arguments.size() < parm_count_required)
1515             {
1516                 if (count_args != arguments.size()) {
1517                     // must been at least one empty argument in C++ mode
1518                     BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
1519                         empty_macroarguments, curr_token.get_value().c_str(),
1520                         main_pos);
1521                 }
1522                 else {
1523                     // too few macro arguments
1524                     BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
1525                         too_few_macroarguments, curr_token.get_value().c_str(),
1526                         main_pos);
1527                 }
1528                 return false;
1529             }
1530
1531             if (count_args > macro_def.macroparameters.size() ||
1532                 arguments.size() > macro_def.macroparameters.size())
1533             {
1534 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
1535                 if (!macro_def.has_ellipsis)
1536 #endif
1537                 {
1538                     // too many macro arguments
1539                     BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
1540                         too_many_macroarguments,
1541                         curr_token.get_value().c_str(), main_pos);
1542                     return false;
1543                 }
1544             }
1545
1546             // inject tracing support
1547             if (ctx.get_hooks().expanding_function_like_macro(ctx.derived(),
1548                     macro_def.macroname, macro_def.macroparameters,
1549                     macro_def.macrodefinition, curr_token, arguments,
1550                     seqstart, seqend))
1551             {
1552 //                 // do not expand this macro, just copy the whole sequence
1553 //                 expanded.push_back(curr_token);
1554 //                 std::copy(seqstart, first,
1555 //                     std::inserter(expanded, expanded.end()));
1556                 // do not expand macro, just copy macro name and parenthesis
1557                 expanded.push_back(curr_token);
1558                 expanded.push_back(*seqstart);
1559                 first = ++seqstart;
1560                 return false;           // no further preprocessing required
1561             }
1562
1563             // expand the replacement list of this macro
1564             expand_replacement_list(macro_def.macrodefinition.begin(),
1565                 macro_def.macrodefinition.end(),
1566                 arguments, expand_operator_defined,
1567                 expand_operator_has_include,
1568                 replacement_list);
1569
1570             if (!expanding_pos)
1571                 expanding_pos = curr_token.get_expand_position();
1572             set_expand_positions(replacement_list, *expanding_pos);
1573         }
1574         else {
1575             // defined as an object-like macro
1576             if (ctx.get_hooks().expanding_object_like_macro(ctx.derived(),
1577                   macro_def.macroname, macro_def.macrodefinition, curr_token))
1578             {
1579                 // do not expand this macro, just copy the whole sequence
1580                 expanded.push_back(curr_token);
1581                 return false;           // no further preprocessing required
1582             }
1583
1584             bool found = false;
1585             impl::find_concat_operator concat_tag(found);
1586
1587             std::remove_copy_if(macro_def.macrodefinition.begin(),
1588                 macro_def.macrodefinition.end(),
1589                 std::inserter(replacement_list, replacement_list.end()),
1590                 concat_tag);
1591
1592             // handle concatenation operators
1593             if (found && !concat_tokensequence(replacement_list))
1594                 return false;
1595         }
1596     }
1597     else {
1598         // called as an object like macro
1599         if ((*it).second->is_functionlike) {
1600             // defined as a function-like macro
1601             if (0 != queue_symbol) {
1602                 queue_symbol->push_back(curr_token);
1603                 expanded.splice(expanded.end(), *queue_symbol);
1604             }
1605             else {
1606                 expanded.push_back(curr_token);
1607             }
1608             ++first;                // skip macro name
1609             return false;           // no further preprocessing required
1610         }
1611         else {
1612             // defined as an object-like macro (expand it)
1613             if (ctx.get_hooks().expanding_object_like_macro(ctx.derived(),
1614                   macro_def.macroname, macro_def.macrodefinition, curr_token))
1615             {
1616                 // do not expand this macro, just copy the whole sequence
1617                 expanded.push_back(curr_token);
1618                 ++first;                // skip macro name
1619                 return false;           // no further preprocessing required
1620             }
1621
1622             bool found = false;
1623             impl::find_concat_operator concat_tag(found);
1624
1625             std::remove_copy_if(macro_def.macrodefinition.begin(),
1626                 macro_def.macrodefinition.end(),
1627                 std::inserter(replacement_list, replacement_list.end()),
1628                 concat_tag);
1629
1630             // handle concatenation operators
1631             if (found && !concat_tokensequence(replacement_list))
1632                 return false;
1633
1634             ++first;                // skip macro name
1635         }
1636     }
1637
1638     // rescan the replacement list
1639     ContainerT expanded_list;
1640
1641     ctx.get_hooks().expanded_macro(ctx.derived(), replacement_list);
1642
1643     rescan_replacement_list(
1644         curr_token, macro_def, replacement_list,
1645         expanded_list, expand_operator_defined,
1646         expand_operator_has_include, first, last);
1647
1648     ctx.get_hooks().rescanned_macro(ctx.derived(), expanded_list);
1649
1650     if (!expanding_pos)
1651         // set the expanding position for rescan
1652         expanding_pos = curr_token.get_expand_position();
1653
1654     // record the location where all the tokens were expanded from
1655     set_expand_positions(expanded_list, *expanding_pos);
1656
1657     expanded.splice(expanded.end(), expanded_list);
1658     return true;        // rescan is required
1659 }
1660
1661 ///////////////////////////////////////////////////////////////////////////////
1662 //
1663 //  If the token under inspection points to a certain predefined macro it will
1664 //  be expanded, otherwise false is returned.
1665 //  (only __FILE__, __LINE__ and __INCLUDE_LEVEL__ macros are expanded here)
1666 //
1667 ///////////////////////////////////////////////////////////////////////////////
1668 template <typename ContextT>
1669 template <typename ContainerT>
1670 inline bool
1671 macromap<ContextT>::expand_predefined_macro(token_type const &curr_token,
1672     ContainerT &expanded)
1673 {
1674     using namespace boost::wave;
1675
1676     string_type const& value = curr_token.get_value();
1677
1678     if ((value != "__LINE__") && (value != "__FILE__") && (value != "__INCLUDE_LEVEL__"))
1679         return false;
1680
1681     // construct a fake token for the macro's definition point
1682     token_type deftoken(T_IDENTIFIER, value, position_type("<built-in>"));
1683
1684     if (ctx.get_hooks().expanding_object_like_macro(ctx.derived(),
1685             deftoken, ContainerT(), curr_token))
1686     {
1687         // do not expand this macro, just copy the whole sequence
1688         expanded.push_back(curr_token);
1689         return false;           // no further preprocessing required
1690     }
1691
1692     token_type replacement;
1693
1694     if (value == "__LINE__") {
1695         // expand the __LINE__ macro
1696         std::string buffer = lexical_cast<std::string>(curr_token.get_expand_position().get_line());
1697
1698         replacement = token_type(T_INTLIT, buffer.c_str(), curr_token.get_position());
1699     }
1700     else if (value == "__FILE__") {
1701         // expand the __FILE__ macro
1702         namespace fs = boost::filesystem;
1703
1704         std::string file("\"");
1705         fs::path filename(
1706             wave::util::create_path(curr_token.get_expand_position().get_file().c_str()));
1707
1708         using boost::wave::util::impl::escape_lit;
1709         file += escape_lit(wave::util::native_file_string(filename)) + "\"";
1710         replacement = token_type(T_STRINGLIT, file.c_str(),
1711             curr_token.get_position());
1712     }
1713     else if (value == "__INCLUDE_LEVEL__") {
1714         // expand the __INCLUDE_LEVEL__ macro
1715         std::string buffer = std::to_string(ctx.get_iteration_depth());
1716         replacement = token_type(T_INTLIT, buffer.c_str(), curr_token.get_position());
1717     }
1718
1719     // post-expansion hooks
1720     ContainerT replacement_list;
1721     replacement_list.push_back(replacement);
1722
1723     ctx.get_hooks().expanded_macro(ctx.derived(), replacement_list);
1724
1725     expanded.push_back(replacement);
1726
1727     ctx.get_hooks().rescanned_macro(ctx.derived(), expanded);
1728
1729     return true;
1730
1731 }
1732
1733 ///////////////////////////////////////////////////////////////////////////////
1734 //
1735 //  resolve_defined(): resolve the operator defined() and replace it with the
1736 //                     correct T_INTLIT token
1737 //
1738 ///////////////////////////////////////////////////////////////////////////////
1739 template <typename ContextT>
1740 template <typename IteratorT, typename ContainerT>
1741 inline typename ContextT::token_type const &
1742 macromap<ContextT>::resolve_defined(IteratorT &first,
1743     IteratorT const &last, ContainerT &pending)
1744 {
1745     using namespace boost::wave;
1746     using namespace boost::wave::grammars;
1747
1748 ContainerT result;
1749 IteratorT start = first;
1750 boost::spirit::classic::parse_info<IteratorT> hit =
1751     defined_grammar_gen<typename ContextT::lexer_type>::
1752         parse_operator_defined(start, last, result);
1753
1754     if (!hit.hit) {
1755         string_type msg ("defined(): ");
1756         msg = msg + util::impl::as_string<string_type>(first, last);
1757         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, ill_formed_expression,
1758             msg.c_str(), main_pos);
1759
1760         // insert a dummy token
1761         pending.push_back(token_type(T_INTLIT, "0", main_pos));
1762     }
1763     else {
1764         impl::assign_iterator<IteratorT>::do_(first, hit.stop);
1765
1766         // insert a token, which reflects the outcome
1767         pending.push_back(token_type(T_INTLIT,
1768             is_defined(result.begin(), result.end()) ? "1" : "0",
1769             main_pos));
1770     }
1771
1772     on_exit::pop_front<definition_container_type> pop_front_token(pending);
1773
1774     return act_token = pending.front();
1775 }
1776
1777 #if BOOST_WAVE_SUPPORT_HAS_INCLUDE != 0
1778 ///////////////////////////////////////////////////////////////////////////////
1779 //
1780 //  resolve_has_include(): resolve the operator __has_include() and replace
1781 //                      it with the correct T_INTLIT token
1782 //
1783 ///////////////////////////////////////////////////////////////////////////////
1784 template <typename ContextT>
1785 template <typename IteratorT, typename ContainerT>
1786 inline typename ContextT::token_type const &
1787 macromap<ContextT>::resolve_has_include(IteratorT &first,
1788     IteratorT const &last, ContainerT &pending)
1789 {
1790     using namespace boost::wave;
1791     using namespace boost::wave::grammars;
1792
1793     ContainerT result;
1794     bool is_quoted_filename;
1795     bool is_system;
1796     IteratorT start = first;
1797
1798     boost::spirit::classic::parse_info<IteratorT> hit =
1799         has_include_grammar_gen<typename ContextT::lexer_type>::
1800         parse_operator_has_include(start, last, result, is_quoted_filename, is_system);
1801
1802     if (!hit.hit) {
1803         string_type msg ("__has_include(): ");
1804         msg = msg + util::impl::as_string<string_type>(first, last);
1805         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, ill_formed_expression,
1806             msg.c_str(), main_pos);
1807
1808         // insert a dummy token
1809         pending.push_back(token_type(T_INTLIT, "0", main_pos));
1810     }
1811     else {
1812         impl::assign_iterator<IteratorT>::do_(first, hit.stop);
1813
1814         // insert a token, which reflects the outcome
1815         pending.push_back(
1816             token_type(T_INTLIT,
1817                        has_include(result.begin(), result.end(),
1818                                    is_quoted_filename, is_system) ? "1" : "0",
1819                        main_pos));
1820     }
1821
1822     on_exit::pop_front<definition_container_type> pop_front_token(pending);
1823
1824     return act_token = pending.front();
1825 }
1826 #endif
1827
1828 ///////////////////////////////////////////////////////////////////////////////
1829 //
1830 //  resolve_operator_pragma(): resolve the operator _Pragma() and dispatch to
1831 //                             the associated action
1832 //
1833 //      This function returns true, if the pragma was correctly interpreted.
1834 //      The iterator 'first' is positioned behind the closing ')'.
1835 //      This function returns false, if the _Pragma was not known, the
1836 //      preprocessed token sequence is pushed back to the 'pending' sequence.
1837 //
1838 ///////////////////////////////////////////////////////////////////////////////
1839 template <typename ContextT>
1840 template <typename IteratorT, typename ContainerT>
1841 inline bool
1842 macromap<ContextT>::resolve_operator_pragma(IteratorT &first,
1843     IteratorT const &last, ContainerT &pending, bool& seen_newline)
1844 {
1845     // isolate the parameter of the operator _Pragma
1846     token_type pragma_token = *first;
1847
1848     if (!impl::skip_to_token(ctx, first, last, T_LEFTPAREN, seen_newline)) {
1849         // illformed operator _Pragma
1850         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, ill_formed_expression,
1851             "operator _Pragma()", pragma_token.get_position());
1852         return false;
1853     }
1854
1855     std::vector<ContainerT> arguments;
1856     IteratorT endparen = first;
1857     typename std::vector<ContainerT>::size_type count_args =
1858         collect_arguments (pragma_token, arguments, first, endparen, last, 1,
1859             seen_newline);
1860
1861     // verify the parameter count
1862     if (pragma_token.get_position().get_file().empty())
1863         pragma_token.set_position(act_token.get_position());
1864
1865     if (count_args < 1 || arguments.size() < 1) {
1866         // too few macro arguments
1867         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, too_few_macroarguments,
1868             pragma_token.get_value().c_str(), pragma_token.get_position());
1869         return false;
1870     }
1871     if (count_args > 1 || arguments.size() > 1) {
1872         // too many macro arguments
1873         BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, too_many_macroarguments,
1874             pragma_token.get_value().c_str(), pragma_token.get_position());
1875         return false;
1876     }
1877
1878     // preprocess the pragma token body
1879     typedef typename std::vector<ContainerT>::value_type::iterator
1880         argument_iterator_type;
1881
1882     ContainerT expanded;
1883     argument_iterator_type begin_it = arguments[0].begin();
1884     argument_iterator_type end_it = arguments[0].end();
1885     expand_whole_tokensequence(expanded, begin_it, end_it, false, false);
1886
1887     // un-escape the parameter of the operator _Pragma
1888     typedef typename token_type::string_type string_type;
1889
1890     string_type pragma_cmd;
1891     typename ContainerT::const_iterator end_exp = expanded.end();
1892     for (typename ContainerT::const_iterator it_exp = expanded.begin();
1893          it_exp != end_exp; ++it_exp)
1894     {
1895         if (T_EOF == token_id(*it_exp))
1896             break;
1897         if (IS_CATEGORY(*it_exp, WhiteSpaceTokenType))
1898             continue;
1899
1900         if (T_STRINGLIT != token_id(*it_exp)) {
1901             // ill formed operator _Pragma
1902             BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
1903                 ill_formed_pragma_option, "_Pragma",
1904                 pragma_token.get_position());
1905             return false;
1906         }
1907         if (pragma_cmd.size() > 0) {
1908             // there should be exactly one string literal (string literals are to
1909             // be concatenated at translation phase 6, but _Pragma operators are
1910             // to be executed at translation phase 4)
1911             BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
1912                 ill_formed_pragma_option, "_Pragma",
1913                 pragma_token.get_position());
1914             return false;
1915         }
1916
1917         // remove the '\"' and concat all given string literal-values
1918         string_type token_str = (*it_exp).get_value();
1919         pragma_cmd += token_str.substr(1, token_str.size() - 2);
1920     }
1921     string_type pragma_cmd_unesc = impl::unescape_lit(pragma_cmd);
1922
1923     // tokenize the pragma body
1924     typedef typename ContextT::lexer_type lexer_type;
1925
1926     ContainerT pragma;
1927     std::string pragma_cmd_str(pragma_cmd_unesc.c_str());
1928     lexer_type it = lexer_type(pragma_cmd_str.begin(), pragma_cmd_str.end(),
1929         pragma_token.get_position(), ctx.get_language());
1930     lexer_type end = lexer_type();
1931     for (/**/; it != end; ++it)
1932         pragma.push_back(*it);
1933
1934     // analyze the preprocessed token sequence and eventually dispatch to the
1935     // associated action
1936     if (interpret_pragma(ctx, pragma_token, pragma.begin(), pragma.end(),
1937         pending))
1938     {
1939         return true;    // successfully recognized a wave specific pragma
1940     }
1941
1942     // unknown pragma token sequence, push it back and return to the caller
1943     pending.push_front(token_type(T_SPACE, " ", pragma_token.get_position()));
1944     pending.push_front(token_type(T_RIGHTPAREN, ")", pragma_token.get_position()));
1945     pending.push_front(token_type(T_STRINGLIT, string_type("\"") + pragma_cmd + "\"",
1946         pragma_token.get_position()));
1947     pending.push_front(token_type(T_LEFTPAREN, "(", pragma_token.get_position()));
1948     pending.push_front(pragma_token);
1949     return false;
1950 }
1951
1952 ///////////////////////////////////////////////////////////////////////////////
1953 //
1954 //  Test, whether the result of a concat operator is well formed or not.
1955 //
1956 //  This is done by re-scanning (re-tokenizing) the resulting token sequence,
1957 //  which should give back exactly one token.
1958 //
1959 ///////////////////////////////////////////////////////////////////////////////
1960 template <typename ContextT>
1961 template <typename ContainerT>
1962 inline bool
1963 macromap<ContextT>::is_valid_concat(string_type new_value,
1964     position_type const &pos, ContainerT &rescanned)
1965 {
1966     // re-tokenize the newly generated string
1967     typedef typename ContextT::lexer_type lexer_type;
1968
1969     std::string value_to_test(new_value.c_str());
1970
1971     boost::wave::language_support lang =
1972         boost::wave::enable_prefer_pp_numbers(ctx.get_language());
1973     lang = boost::wave::enable_single_line(lang);
1974
1975     lexer_type it = lexer_type(value_to_test.begin(), value_to_test.end(), pos,
1976         lang);
1977     lexer_type end = lexer_type();
1978     for (/**/; it != end && T_EOF != token_id(*it); ++it)
1979     {
1980         // as of Wave V2.0.7 pasting of tokens is valid only if the resulting
1981         // tokens are pp_tokens (as mandated by C++11)
1982         if (!is_pp_token(*it))
1983             return false;
1984         rescanned.push_back(*it);
1985     }
1986
1987 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
1988     if (boost::wave::need_variadics(ctx.get_language()))
1989         return true;       // in variadics mode token pasting is well defined
1990 #endif
1991
1992     // test if the newly generated token sequence contains more than 1 token
1993     return 1 == rescanned.size();
1994 }
1995
1996 ///////////////////////////////////////////////////////////////////////////////
1997 //
1998 //  Bulk update expand positions
1999 //
2000 ///////////////////////////////////////////////////////////////////////////////
2001 // batch update tokens with a single expand position
2002 template <typename ContextT>
2003 template <typename ContainerT>
2004 void macromap<ContextT>::set_expand_positions(ContainerT &tokens, position_type pos)
2005 {
2006     typename ContainerT::iterator ex_end = tokens.end();
2007     for (typename ContainerT::iterator it = tokens.begin();
2008          it != ex_end; ++it) {
2009         // expand positions are only used for __LINE__, __FILE__, and macro names
2010         if (token_id(*it) == T_IDENTIFIER)
2011             it->set_expand_position(pos);
2012     }
2013 }
2014
2015 ///////////////////////////////////////////////////////////////////////////////
2016 //
2017 //  Handle all occurrences of the concatenation operator '##' inside the given
2018 //  token sequence.
2019 //
2020 ///////////////////////////////////////////////////////////////////////////////
2021 template <typename Context>
2022 inline void report_invalid_concatenation(Context& ctx,
2023     typename Context::token_type const& prev,
2024     typename Context::token_type const& next,
2025     typename Context::position_type const& main_pos)
2026 {
2027     typename Context::string_type error_string("\"");
2028
2029     error_string += prev.get_value();
2030     error_string += "\" and \"";
2031     error_string += next.get_value();
2032     error_string += "\"";
2033     BOOST_WAVE_THROW_CTX(ctx, preprocess_exception, invalid_concat,
2034         error_string.c_str(), main_pos);
2035 }
2036
2037 template <typename ContextT>
2038 template <typename ContainerT>
2039 inline bool
2040 macromap<ContextT>::concat_tokensequence(ContainerT &expanded)
2041 {
2042     using namespace boost::wave;
2043     typedef typename ContainerT::iterator iterator_type;
2044
2045     iterator_type end = expanded.end();
2046     iterator_type prev = end;
2047     for (iterator_type it = expanded.begin(); it != end; /**/)
2048     {
2049         if (T_POUND_POUND == BASE_TOKEN(token_id(*it))) {
2050             iterator_type next = it;
2051
2052             ++next;
2053             if (prev == end || next == end) {
2054                 // error, '##' should be in between two tokens
2055                 BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
2056                     ill_formed_operator, "concat ('##')", main_pos);
2057                 return false;
2058             }
2059
2060             // replace prev##next with the concatenated value, skip whitespace
2061             // before and after the '##' operator
2062             while (IS_CATEGORY(*next, WhiteSpaceTokenType)) {
2063                 ++next;
2064                 if (next == end) {
2065                     // error, '##' should be in between two tokens
2066                     BOOST_WAVE_THROW_CTX(ctx, preprocess_exception,
2067                         ill_formed_operator, "concat ('##')", main_pos);
2068                     return false;
2069                 }
2070             }
2071
2072 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
2073             if (boost::wave::need_variadics(ctx.get_language())) {
2074                 if (T_PLACEMARKER == token_id(*next)) {
2075                     // remove the '##' and the next tokens from the sequence
2076                     iterator_type first_to_delete = prev;
2077
2078                     expanded.erase(++first_to_delete, ++next);
2079                     it = next;
2080                     continue;
2081                 }
2082                 else if (T_PLACEMARKER == token_id(*prev)) {
2083                     // remove the '##' and the next tokens from the sequence
2084                     iterator_type first_to_delete = prev;
2085
2086                     *prev = *next;
2087                     expanded.erase(++first_to_delete, ++next);
2088                     it = next;
2089                     continue;
2090                 }
2091             }
2092 #endif // BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
2093
2094             // test if the concat operator has to concatenate two unrelated
2095             // tokens i.e. the result yields more then one token
2096             string_type concat_result;
2097             ContainerT rescanned;
2098
2099             concat_result = ((*prev).get_value() + (*next).get_value());
2100
2101             // analyze the validity of the concatenation result
2102             if (!is_valid_concat(concat_result, (*prev).get_position(),
2103                     rescanned) &&
2104                 !IS_CATEGORY(*prev, WhiteSpaceTokenType) &&
2105                 !IS_CATEGORY(*next, WhiteSpaceTokenType))
2106             {
2107                 report_invalid_concatenation(ctx, *prev, *next, main_pos);
2108                 return false;
2109             }
2110
2111 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
2112             if (boost::wave::need_variadics(ctx.get_language())) {
2113                 // remove the prev, '##' and the next tokens from the sequence
2114                 expanded.erase(prev, ++next); // remove not needed tokens
2115
2116                 // some stl implementations clear() the container if we erased all
2117                 // the elements, which orphans all iterators. we re-initialize these
2118                 // here
2119                 if (expanded.empty())
2120                     end = next = expanded.end();
2121
2122                 // replace the old token (pointed to by *prev) with the re-tokenized
2123                 // sequence
2124                 expanded.splice(next, rescanned);
2125
2126                 // the last token of the inserted sequence is the new previous
2127                 prev = next;
2128                 if (next != expanded.end())
2129                     --prev;
2130             }
2131             else
2132 #endif // BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
2133             {
2134                 // we leave the token_id unchanged, but unmark the token as
2135                 // disabled, if appropriate
2136                 (*prev).set_value(concat_result);
2137                 if (T_NONREPLACABLE_IDENTIFIER == token_id(*prev))
2138                     (*prev).set_token_id(T_IDENTIFIER);
2139
2140                 // remove the '##' and the next tokens from the sequence
2141                 iterator_type first_to_delete = prev;
2142
2143                 expanded.erase(++first_to_delete, ++next);
2144             }
2145             it = next;
2146             continue;
2147         }
2148
2149         // save last non-whitespace token position
2150         if (!IS_CATEGORY(*it, WhiteSpaceTokenType))
2151             prev = it;
2152
2153         ++it;           // next token, please
2154     }
2155     return true;
2156 }
2157
2158 ///////////////////////////////////////////////////////////////////////////////
2159 //
2160 //  predefine_macro(): predefine a single macro
2161 //
2162 ///////////////////////////////////////////////////////////////////////////////
2163 template <typename ContextT>
2164 inline void
2165 macromap<ContextT>::predefine_macro(defined_macros_type *scope,
2166     string_type const &name, token_type const &t)
2167 {
2168 definition_container_type macrodefinition;
2169 std::vector<token_type> param;
2170
2171     macrodefinition.push_back(t);
2172     add_macro(token_type(T_IDENTIFIER, name, t.get_position()),
2173         false, param, macrodefinition, true, scope);
2174 }
2175
2176 ///////////////////////////////////////////////////////////////////////////////
2177 //
2178 //  init_predefined_macros(): init the predefined macros
2179 //
2180 ///////////////////////////////////////////////////////////////////////////////
2181 template <typename ContextT>
2182 inline void
2183 macromap<ContextT>::init_predefined_macros(char const *fname,
2184     defined_macros_type *scope, bool at_global_scope)
2185 {
2186     // if no scope is given, use the current one
2187     defined_macros_type* current_scope = scope ? scope : current_macros;
2188
2189     // first, add the static macros
2190     position_type pos("<built-in>");
2191
2192 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
2193     if (boost::wave::need_c99(ctx.get_language())) {
2194         // define C99 specifics
2195         for (int i = 0; 0 != predef.static_data_c99(i).name; ++i) {
2196             predefined_macros::static_macros const& m = predef.static_data_c99(i);
2197             predefine_macro(current_scope, m.name,
2198                 token_type(m.token_id, m.value, pos));
2199         }
2200     }
2201     else
2202 #endif
2203     {
2204 #if BOOST_WAVE_SUPPORT_CPP0X != 0
2205         if (boost::wave::need_cpp0x(ctx.get_language())) {
2206             // define C++11 specifics
2207             for (int i = 0; 0 != predef.static_data_cpp0x(i).name; ++i) {
2208                 predefined_macros::static_macros const& m = predef.static_data_cpp0x(i);
2209                 predefine_macro(current_scope, m.name,
2210                     token_type(m.token_id, m.value, pos));
2211             }
2212         }
2213         else
2214 #endif
2215 #if BOOST_WAVE_SUPPORT_CPP2A != 0
2216             if (boost::wave::need_cpp2a(ctx.get_language())) {
2217             // define C++20 specifics
2218             for (int i = 0; 0 != predef.static_data_cpp2a(i).name; ++i) {
2219                 predefined_macros::static_macros const& m = predef.static_data_cpp2a(i);
2220                 predefine_macro(current_scope, m.name,
2221                     token_type(m.token_id, m.value, pos));
2222             }
2223         }
2224         else
2225 #endif
2226         {
2227             // define C++ specifics
2228             for (int i = 0; 0 != predef.static_data_cpp(i).name; ++i) {
2229                 predefined_macros::static_macros const& m = predef.static_data_cpp(i);
2230                 predefine_macro(current_scope, m.name,
2231                     token_type(m.token_id, m.value, pos));
2232             }
2233
2234 #if BOOST_WAVE_SUPPORT_VARIADICS_PLACEMARKERS != 0
2235             // define __WAVE_HAS_VARIADICS__, if appropriate
2236             if (boost::wave::need_variadics(ctx.get_language())) {
2237                 predefine_macro(current_scope, "__WAVE_HAS_VARIADICS__",
2238                     token_type(T_INTLIT, "1", pos));
2239             }
2240 #endif
2241         }
2242     }
2243
2244     // predefine the __BASE_FILE__ macro which contains the main file name
2245     namespace fs = boost::filesystem;
2246     if (string_type(fname) != "<Unknown>") {
2247         fs::path filename(create_path(fname));
2248
2249         using boost::wave::util::impl::escape_lit;
2250         predefine_macro(current_scope, "__BASE_FILE__",
2251             token_type(T_STRINGLIT, string_type("\"") +
2252                 escape_lit(native_file_string(filename)).c_str() + "\"", pos));
2253         base_name = fname;
2254     }
2255     else if (!base_name.empty()) {
2256         fs::path filename(create_path(base_name.c_str()));
2257
2258         using boost::wave::util::impl::escape_lit;
2259         predefine_macro(current_scope, "__BASE_FILE__",
2260             token_type(T_STRINGLIT, string_type("\"") +
2261                 escape_lit(native_file_string(filename)).c_str() + "\"", pos));
2262     }
2263
2264     // now add the dynamic macros
2265     for (int j = 0; 0 != predef.dynamic_data(j).name; ++j) {
2266         predefined_macros::dynamic_macros const& m = predef.dynamic_data(j);
2267         predefine_macro(current_scope, m.name,
2268             token_type(m.token_id, (predef.* m.generator)(), pos));
2269     }
2270 }
2271
2272 ///////////////////////////////////////////////////////////////////////////////
2273 //
2274 //  reset_macromap(): initialize the internal macro symbol namespace
2275 //
2276 ///////////////////////////////////////////////////////////////////////////////
2277 template <typename ContextT>
2278 inline void
2279 macromap<ContextT>::reset_macromap()
2280 {
2281     current_macros->clear();
2282     predef.reset();
2283     act_token = token_type();
2284 }
2285
2286 ///////////////////////////////////////////////////////////////////////////////
2287 }}}   // namespace boost::wave::util
2288
2289 #if BOOST_WAVE_SERIALIZATION != 0
2290 namespace boost { namespace serialization {
2291
2292 template<typename ContextT>
2293 struct version<boost::wave::util::macromap<ContextT> >
2294 {
2295     typedef boost::wave::util::macromap<ContextT> target_type;
2296     typedef mpl::int_<target_type::version> type;
2297     typedef mpl::integral_c_tag tag;
2298     BOOST_STATIC_CONSTANT(unsigned int, value = version::type::value);
2299 };
2300
2301 }}    // namespace boost::serialization
2302 #endif
2303
2304 // the suffix header occurs after all of the code
2305 #ifdef BOOST_HAS_ABI_HEADERS
2306 #include BOOST_ABI_SUFFIX
2307 #endif
2308
2309 #endif // !defined(BOOST_CPP_MACROMAP_HPP_CB8F51B0_A3F0_411C_AEF4_6FF631B8B414_INCLUDED)