detail/text/transcode_iterator.hpp

0001 // Copyright (C) 2020 T. Zachary Laine
0002 //
0003 // Distributed under the Boost Software License, Version 1.0. (See
0004 // accompanying file LICENSE_1_0.txt or copy at
0005 // http://www.boost.org/LICENSE_1_0.txt)
0006 #ifndef BOOST_PARSER_DETAIL_TEXT_TRANSCODE_ITERATOR_HPP
0007 #define BOOST_PARSER_DETAIL_TEXT_TRANSCODE_ITERATOR_HPP
0008
0009 #include <boost/parser/detail/debug_assert.hpp>
0010 #include <boost/parser/detail/text/transcode_iterator_fwd.hpp>
0011 #include <boost/parser/detail/text/concepts.hpp>
0012 #include <boost/parser/detail/text/utf.hpp>
0013 #include <boost/parser/detail/text/detail/algorithm.hpp>
0014
0015 #include <boost/parser/detail/stl_interfaces/iterator_interface.hpp>
0016
0017 #include <array>
0018 #include <iterator>
0019 #include <type_traits>
0020 #include <stdexcept>
0021 #include <string_view>
0022
0023 namespace boost::parser::detail { namespace text {
0024
0025     namespace {
0026         constexpr char16_t high_surrogate_base = 0xd7c0;
0027         constexpr char16_t low_surrogate_base = 0xdc00;
0028         constexpr char32_t high_surrogate_min = 0xd800;
0029         constexpr char32_t high_surrogate_max = 0xdbff;
0030         constexpr char32_t low_surrogate_min = 0xdc00;
0031         constexpr char32_t low_surrogate_max = 0xdfff;
0032         constexpr char32_t replacement_character = 0xfffd;
0033     }
0034
0035     namespace detail {
0036         constexpr bool in(unsigned char lo, unsigned char c, unsigned char hi)
0037         {
0038             return lo <= c && c <= hi;
0039         }
0040
0041         struct throw_on_encoding_error
0042         {};
0043
0044         template<typename OutIter>
0045         inline constexpr OutIter read_into_buf(char32_t cp, OutIter buf)
0046         {
0047             if (cp < 0x80) {
0048                 *buf = static_cast<char>(cp);
0049                 ++buf;
0050             } else if (cp < 0x800) {
0051                 *buf = static_cast<char>(0xC0 + (cp >> 6));
0052                 ++buf;
0053                 *buf = static_cast<char>(0x80 + (cp & 0x3f));
0054                 ++buf;
0055             } else if (cp < 0x10000) {
0056                 *buf = static_cast<char>(0xe0 + (cp >> 12));
0057                 ++buf;
0058                 *buf = static_cast<char>(0x80 + ((cp >> 6) & 0x3f));
0059                 ++buf;
0060                 *buf = static_cast<char>(0x80 + (cp & 0x3f));
0061                 ++buf;
0062             } else {
0063                 *buf = static_cast<char>(0xf0 + (cp >> 18));
0064                 ++buf;
0065                 *buf = static_cast<char>(0x80 + ((cp >> 12) & 0x3f));
0066                 ++buf;
0067                 *buf = static_cast<char>(0x80 + ((cp >> 6) & 0x3f));
0068                 ++buf;
0069                 *buf = static_cast<char>(0x80 + (cp & 0x3f));
0070                 ++buf;
0071             }
0072             return buf;
0073         }
0074
0075         template<typename OutIter>
0076         constexpr OutIter write_cp_utf8(char32_t cp, OutIter out)
0077         {
0078             return detail::read_into_buf(cp, out);
0079         }
0080
0081         template<typename OutIter>
0082         constexpr OutIter write_cp_utf16(char32_t cp, OutIter out)
0083         {
0084             if (cp < 0x10000) {
0085                 *out = static_cast<char16_t>(cp);
0086                 ++out;
0087             } else {
0088                 *out = static_cast<char16_t>(cp >> 10) + high_surrogate_base;
0089                 ++out;
0090                 *out = static_cast<char16_t>(cp & 0x3ff) + low_surrogate_base;
0091                 ++out;
0092             }
0093             return out;
0094         }
0095
0096         inline constexpr char32_t surrogates_to_cp(char16_t hi, char16_t lo)
0097         {
0098             return char32_t((hi - high_surrogate_base) << 10) +
0099                    (lo - low_surrogate_base);
0100         }
0101
0102         template<typename T, typename U>
0103         using enable_utf8_cp = std::enable_if<is_char_iter_v<T>, U>;
0104         template<typename T, typename U = T>
0105         using enable_utf8_cp_t = typename enable_utf8_cp<T, U>::type;
0106
0107         template<typename T, typename U>
0108         using enable_utf16_cp = std::enable_if<is_16_iter_v<T>, U>;
0109         template<typename T, typename U = T>
0110         using enable_utf16_cp_t = typename enable_utf16_cp<T, U>::type;
0111
0112         template<typename I>
0113         auto bidirectional_at_most()
0114         {
0115 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
0116             if constexpr (std::bidirectional_iterator<I>) {
0117                 return std::bidirectional_iterator_tag{};
0118             } else if constexpr (std::forward_iterator<I>) {
0119                 return std::forward_iterator_tag{};
0120             } else if constexpr (std::input_iterator<I>) {
0121                 return std::input_iterator_tag{};
0122             }
0123 #else
0124             using category =
0125                 typename std::iterator_traits<I>::iterator_category;
0126             if constexpr (std::is_base_of_v<
0127                               std::bidirectional_iterator_tag,
0128                               category>) {
0129                 return std::bidirectional_iterator_tag{};
0130             } else {
0131                 return category{};
0132             }
0133 #endif
0134         }
0135
0136         template<typename I>
0137         using bidirectional_at_most_t = decltype(bidirectional_at_most<I>());
0138     }
0139
0140     /** Returns true iff `c` is a Unicode surrogate. */
0141     inline constexpr bool surrogate(char32_t c)
0142     {
0143         return high_surrogate_min <= c && c <= low_surrogate_max;
0144     }
0145
0146     /** Returns true iff `c` is a Unicode high surrogate. */
0147     inline constexpr bool high_surrogate(char32_t c)
0148     {
0149         return high_surrogate_min <= c && c <= high_surrogate_max;
0150     }
0151
0152     /** Returns true iff `c` is a Unicode low surrogate. */
0153     inline constexpr bool low_surrogate(char32_t c)
0154     {
0155         return low_surrogate_min <= c && c <= low_surrogate_max;
0156     }
0157
0158     /** Returns true iff `c` is a Unicode reserved noncharacter.
0159
0160         \see Unicode 3.4/D14 */
0161     inline constexpr bool reserved_noncharacter(char32_t c)
0162     {
0163         bool const byte01_reserved = (c & 0xffff) >= 0xfffe;
0164         bool const byte2_at_most_0x10 = ((c & 0xff0000u) >> 16) <= 0x10;
0165         return (byte01_reserved && byte2_at_most_0x10) ||
0166                (0xfdd0 <= c && c <= 0xfdef);
0167     }
0168
0169     /** Returns true iff `c` is a valid Unicode scalar value.
0170
0171         \see Unicode 3.9/D90 */
0172     inline constexpr bool scalar_value(char32_t c)
0173     {
0174         return c <= 0x10ffff && !surrogate(c);
0175     }
0176
0177     /** Returns true iff `c` is a Unicode scalar value not in the reserved
0178         range.
0179
0180         \see Unicode 3.9/D90 */
0181     inline constexpr bool unreserved_scalar_value(char32_t c)
0182     {
0183         return scalar_value(c) && !reserved_noncharacter(c);
0184     }
0185
0186     /** Returns true iff `c` is a UTF-8 lead code unit (which must be followed
0187         by 1-3 following units). */
0188     constexpr bool lead_code_unit(char8_type c)
0189     {
0190         return uint8_t((unsigned char)c - 0xc2) <= 0x32;
0191     }
0192
0193     /** Returns true iff `c` is a UTF-8 continuation code unit. */
0194     constexpr bool continuation(char8_type c) { return (int8_t)c < -0x40; }
0195
0196     /** Given the first (and possibly only) code unit of a UTF-8-encoded code
0197         point, returns the number of bytes occupied by that code point (in the
0198         range `[1, 4]`).  Returns a value < 0 if `first_unit` is not a valid
0199         initial UTF-8 code unit. */
0200     inline constexpr int utf8_code_units(char8_type first_unit_)
0201     {
0202         auto first_unit = (unsigned int)first_unit_;
0203         return first_unit <= 0x7f ? 1
0204                : boost::parser::detail::text::lead_code_unit(first_unit)
0205                    ? int(0xe0 <= first_unit) + int(0xf0 <= first_unit) + 2
0206                    : -1;
0207     }
0208
0209     /** Given the first (and possibly only) code unit of a UTF-16-encoded code
0210         point, returns the number of code units occupied by that code point
0211         (in the range `[1, 2]`).  Returns a negative value if `first_unit` is
0212         not a valid initial UTF-16 code unit. */
0213     inline constexpr int utf16_code_units(char16_t first_unit)
0214     {
0215         if (boost::parser::detail::text::low_surrogate(first_unit))
0216             return -1;
0217         if (boost::parser::detail::text::high_surrogate(first_unit))
0218             return 2;
0219         return 1;
0220     }
0221
0222     namespace detail {
0223         // optional is not constexpr friendly.
0224         template<typename Iter>
0225         struct optional_iter
0226         {
0227             constexpr optional_iter() : it_(), valid_(false) {}
0228             constexpr optional_iter(Iter it) : it_(it), valid_(true) {}
0229
0230             constexpr operator bool() const { return valid_; }
0231             constexpr Iter operator*() const
0232             {
0233                 BOOST_PARSER_DEBUG_ASSERT(valid_);
0234                 return it_;
0235             }
0236             Iter & operator*()
0237             {
0238                 BOOST_PARSER_DEBUG_ASSERT(valid_);
0239                 return it_;
0240             }
0241
0242             friend BOOST_PARSER_CONSTEXPR bool
0243             operator==(optional_iter lhs, optional_iter rhs)
0244             {
0245                 return lhs.valid_ == rhs.valid_ &&
0246                        (!lhs.valid_ || lhs.it_ == rhs.it_);
0247             }
0248             friend BOOST_PARSER_CONSTEXPR bool
0249             operator!=(optional_iter lhs, optional_iter rhs)
0250             {
0251                 return !(lhs == rhs);
0252             }
0253
0254         private:
0255             Iter it_;
0256             bool valid_;
0257         };
0258
0259         // Follow Table 3-7 in Unicode 3.9/D92
0260         template<typename Iter>
0261         constexpr optional_iter<Iter> end_of_invalid_utf8(Iter it)
0262         {
0263             BOOST_PARSER_DEBUG_ASSERT(!boost::parser::detail::text::continuation(*it));
0264
0265             if (detail::in(0, *it, 0x7f))
0266                 return optional_iter<Iter>{};
0267
0268             if (detail::in(0xc2, *it, 0xdf)) {
0269                 auto next = it;
0270                 if (!boost::parser::detail::text::continuation(*++next))
0271                     return next;
0272                 return optional_iter<Iter>{};
0273             }
0274
0275             if (detail::in(0xe0, *it, 0xe0)) {
0276                 auto next = it;
0277                 if (!detail::in(0xa0, *++next, 0xbf))
0278                     return next;
0279                 if (!boost::parser::detail::text::continuation(*++next))
0280                     return next;
0281                 return optional_iter<Iter>{};
0282             }
0283             if (detail::in(0xe1, *it, 0xec)) {
0284                 auto next = it;
0285                 if (!boost::parser::detail::text::continuation(*++next))
0286                     return next;
0287                 if (!boost::parser::detail::text::continuation(*++next))
0288                     return next;
0289                 return optional_iter<Iter>{};
0290             }
0291             if (detail::in(0xed, *it, 0xed)) {
0292                 auto next = it;
0293                 if (!detail::in(0x80, *++next, 0x9f))
0294                     return next;
0295                 if (!boost::parser::detail::text::continuation(*++next))
0296                     return next;
0297                 return optional_iter<Iter>{};
0298             }
0299             if (detail::in(0xee, *it, 0xef)) {
0300                 auto next = it;
0301                 if (!boost::parser::detail::text::continuation(*++next))
0302                     return next;
0303                 if (!boost::parser::detail::text::continuation(*++next))
0304                     return next;
0305                 return optional_iter<Iter>{};
0306             }
0307
0308             if (detail::in(0xf0, *it, 0xf0)) {
0309                 auto next = it;
0310                 if (!detail::in(0x90, *++next, 0xbf))
0311                     return next;
0312                 if (!boost::parser::detail::text::continuation(*++next))
0313                     return next;
0314                 if (!boost::parser::detail::text::continuation(*++next))
0315                     return next;
0316                 return optional_iter<Iter>{};
0317             }
0318             if (detail::in(0xf1, *it, 0xf3)) {
0319                 auto next = it;
0320                 if (!boost::parser::detail::text::continuation(*++next))
0321                     return next;
0322                 if (!boost::parser::detail::text::continuation(*++next))
0323                     return next;
0324                 if (!boost::parser::detail::text::continuation(*++next))
0325                     return next;
0326                 return optional_iter<Iter>{};
0327             }
0328             if (detail::in(0xf4, *it, 0xf4)) {
0329                 auto next = it;
0330                 if (!detail::in(0x80, *++next, 0x8f))
0331                     return next;
0332                 if (!boost::parser::detail::text::continuation(*++next))
0333                     return next;
0334                 if (!boost::parser::detail::text::continuation(*++next))
0335                     return next;
0336                 return optional_iter<Iter>{};
0337             }
0338
0339             return it;
0340         }
0341
0342         template<typename Iter>
0343         constexpr Iter decrement(Iter it)
0344         {
0345             Iter retval = it;
0346
0347             int backup = 0;
0348             while (backup < 4 && boost::parser::detail::text::continuation(*--retval)) {
0349                 ++backup;
0350             }
0351             backup = it - retval;
0352
0353             if (boost::parser::detail::text::continuation(*retval))
0354                 return it - 1;
0355
0356             optional_iter<Iter> first_invalid = end_of_invalid_utf8(retval);
0357             if (first_invalid == retval)
0358                 ++*first_invalid;
0359             while (first_invalid && (*first_invalid - retval) < backup) {
0360                 backup -= *first_invalid - retval;
0361                 retval = *first_invalid;
0362                 first_invalid = end_of_invalid_utf8(retval);
0363                 if (first_invalid == retval)
0364                     ++*first_invalid;
0365             }
0366
0367             if (1 < backup) {
0368                 int const cp_bytes = boost::parser::detail::text::utf8_code_units(*retval);
0369                 if (cp_bytes < backup)
0370                     retval = it - 1;
0371             }
0372
0373             return retval;
0374         }
0375
0376         template<typename Iter>
0377         constexpr Iter decrement(Iter first, Iter it)
0378         {
0379             Iter retval = it;
0380
0381             int backup = 0;
0382             while (backup < 4 && retval != first &&
0383                    boost::parser::detail::text::continuation(*--retval)) {
0384                 ++backup;
0385             }
0386             backup = (int)std::distance(retval, it);
0387
0388             if (boost::parser::detail::text::continuation(*retval)) {
0389                 if (it != first)
0390                     --it;
0391                 return it;
0392             }
0393
0394             optional_iter<Iter> first_invalid = end_of_invalid_utf8(retval);
0395             if (first_invalid == retval)
0396                 ++*first_invalid;
0397             while (first_invalid &&
0398                    std::distance(retval, *first_invalid) < backup) {
0399                 backup -= (int)std::distance(retval, *first_invalid);
0400                 retval = *first_invalid;
0401                 first_invalid = end_of_invalid_utf8(retval);
0402                 if (first_invalid == retval)
0403                     ++*first_invalid;
0404             }
0405
0406             if (1 < backup) {
0407                 int const cp_bytes = boost::parser::detail::text::utf8_code_units(*retval);
0408                 if (cp_bytes < backup) {
0409                     if (it != first)
0410                         --it;
0411                     retval = it;
0412                 }
0413             }
0414
0415             return retval;
0416         }
0417
0418         enum char_class : uint8_t {
0419             ill = 0,
0420             asc = 1,
0421             cr1 = 2,
0422             cr2 = 3,
0423             cr3 = 4,
0424             l2a = 5,
0425             l3a = 6,
0426             l3b = 7,
0427             l3c = 8,
0428             l4a = 9,
0429             l4b = 10,
0430             l4c = 11,
0431         };
0432
0433         enum table_state : uint8_t {
0434             bgn = 0,
0435             e_d = bgn, // "end"
0436             err = 12,
0437             cs1 = 24,
0438             cs2 = 36,
0439             cs3 = 48,
0440             p3a = 60,
0441             p3b = 72,
0442             p4a = 84,
0443             p4b = 96,
0444             invalid_table_state = 200
0445         };
0446
0447         struct first_cu
0448         {
0449             unsigned char initial_octet;
0450             table_state next;
0451         };
0452
0453         namespace {
0454             constexpr first_cu first_cus[256] = {
0455                 {0x00, bgn}, {0x01, bgn}, {0x02, bgn}, {0x03, bgn}, {0x04, bgn},
0456                 {0x05, bgn}, {0x06, bgn}, {0x07, bgn}, {0x08, bgn}, {0x09, bgn},
0457                 {0x0a, bgn}, {0x0b, bgn}, {0x0c, bgn}, {0x0d, bgn}, {0x0e, bgn},
0458                 {0x0f, bgn}, {0x10, bgn}, {0x11, bgn}, {0x12, bgn}, {0x13, bgn},
0459                 {0x14, bgn}, {0x15, bgn}, {0x16, bgn}, {0x17, bgn}, {0x18, bgn},
0460                 {0x19, bgn}, {0x1a, bgn}, {0x1b, bgn}, {0x1c, bgn}, {0x1d, bgn},
0461                 {0x1e, bgn}, {0x1f, bgn}, {0x20, bgn}, {0x21, bgn}, {0x22, bgn},
0462                 {0x23, bgn}, {0x24, bgn}, {0x25, bgn}, {0x26, bgn}, {0x27, bgn},
0463                 {0x28, bgn}, {0x29, bgn}, {0x2a, bgn}, {0x2b, bgn}, {0x2c, bgn},
0464                 {0x2d, bgn}, {0x2e, bgn}, {0x2f, bgn}, {0x30, bgn}, {0x31, bgn},
0465                 {0x32, bgn}, {0x33, bgn}, {0x34, bgn}, {0x35, bgn}, {0x36, bgn},
0466                 {0x37, bgn}, {0x38, bgn}, {0x39, bgn}, {0x3a, bgn}, {0x3b, bgn},
0467                 {0x3c, bgn}, {0x3d, bgn}, {0x3e, bgn}, {0x3f, bgn}, {0x40, bgn},
0468                 {0x41, bgn}, {0x42, bgn}, {0x43, bgn}, {0x44, bgn}, {0x45, bgn},
0469                 {0x46, bgn}, {0x47, bgn}, {0x48, bgn}, {0x49, bgn}, {0x4a, bgn},
0470                 {0x4b, bgn}, {0x4c, bgn}, {0x4d, bgn}, {0x4e, bgn}, {0x4f, bgn},
0471                 {0x50, bgn}, {0x51, bgn}, {0x52, bgn}, {0x53, bgn}, {0x54, bgn},
0472                 {0x55, bgn}, {0x56, bgn}, {0x57, bgn}, {0x58, bgn}, {0x59, bgn},
0473                 {0x5a, bgn}, {0x5b, bgn}, {0x5c, bgn}, {0x5d, bgn}, {0x5e, bgn},
0474                 {0x5f, bgn}, {0x60, bgn}, {0x61, bgn}, {0x62, bgn}, {0x63, bgn},
0475                 {0x64, bgn}, {0x65, bgn}, {0x66, bgn}, {0x67, bgn}, {0x68, bgn},
0476                 {0x69, bgn}, {0x6a, bgn}, {0x6b, bgn}, {0x6c, bgn}, {0x6d, bgn},
0477                 {0x6e, bgn}, {0x6f, bgn}, {0x70, bgn}, {0x71, bgn}, {0x72, bgn},
0478                 {0x73, bgn}, {0x74, bgn}, {0x75, bgn}, {0x76, bgn}, {0x77, bgn},
0479                 {0x78, bgn}, {0x79, bgn}, {0x7a, bgn}, {0x7b, bgn}, {0x7c, bgn},
0480                 {0x7d, bgn}, {0x7e, bgn}, {0x7f, bgn}, {0x00, err}, {0x01, err},
0481                 {0x02, err}, {0x03, err}, {0x04, err}, {0x05, err}, {0x06, err},
0482                 {0x07, err}, {0x08, err}, {0x09, err}, {0x0a, err}, {0x0b, err},
0483                 {0x0c, err}, {0x0d, err}, {0x0e, err}, {0x0f, err}, {0x10, err},
0484                 {0x11, err}, {0x12, err}, {0x13, err}, {0x14, err}, {0x15, err},
0485                 {0x16, err}, {0x17, err}, {0x18, err}, {0x19, err}, {0x1a, err},
0486                 {0x1b, err}, {0x1c, err}, {0x1d, err}, {0x1e, err}, {0x1f, err},
0487                 {0x20, err}, {0x21, err}, {0x22, err}, {0x23, err}, {0x24, err},
0488                 {0x25, err}, {0x26, err}, {0x27, err}, {0x28, err}, {0x29, err},
0489                 {0x2a, err}, {0x2b, err}, {0x2c, err}, {0x2d, err}, {0x2e, err},
0490                 {0x2f, err}, {0x30, err}, {0x31, err}, {0x32, err}, {0x33, err},
0491                 {0x34, err}, {0x35, err}, {0x36, err}, {0x37, err}, {0x38, err},
0492                 {0x39, err}, {0x3a, err}, {0x3b, err}, {0x3c, err}, {0x3d, err},
0493                 {0x3e, err}, {0x3f, err}, {0xc0, err}, {0xc1, err}, {0x02, cs1},
0494                 {0x03, cs1}, {0x04, cs1}, {0x05, cs1}, {0x06, cs1}, {0x07, cs1},
0495                 {0x08, cs1}, {0x09, cs1}, {0x0a, cs1}, {0x0b, cs1}, {0x0c, cs1},
0496                 {0x0d, cs1}, {0x0e, cs1}, {0x0f, cs1}, {0x10, cs1}, {0x11, cs1},
0497                 {0x12, cs1}, {0x13, cs1}, {0x14, cs1}, {0x15, cs1}, {0x16, cs1},
0498                 {0x17, cs1}, {0x18, cs1}, {0x19, cs1}, {0x1a, cs1}, {0x1b, cs1},
0499                 {0x1c, cs1}, {0x1d, cs1}, {0x1e, cs1}, {0x1f, cs1}, {0x00, p3a},
0500                 {0x01, cs2}, {0x02, cs2}, {0x03, cs2}, {0x04, cs2}, {0x05, cs2},
0501                 {0x06, cs2}, {0x07, cs2}, {0x08, cs2}, {0x09, cs2}, {0x0a, cs2},
0502                 {0x0b, cs2}, {0x0c, cs2}, {0x0d, p3b}, {0x0e, cs2}, {0x0f, cs2},
0503                 {0x00, p4a}, {0x01, cs3}, {0x02, cs3}, {0x03, cs3}, {0x04, p4b},
0504                 {0xf5, err}, {0xf6, err}, {0xf7, err}, {0xf8, err}, {0xf9, err},
0505                 {0xfa, err}, {0xfb, err}, {0xfc, err}, {0xfd, err}, {0xfe, err},
0506                 {0xff, err},
0507             };
0508
0509             constexpr char_class octet_classes[256] = {
0510                 asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc,
0511                 asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc,
0512                 asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc,
0513                 asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc,
0514                 asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc,
0515                 asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc,
0516                 asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc,
0517                 asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc,
0518                 asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc,
0519                 asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, asc, cr1, cr1,
0520                 cr1, cr1, cr1, cr1, cr1, cr1, cr1, cr1, cr1, cr1, cr1, cr1, cr1,
0521                 cr1, cr2, cr2, cr2, cr2, cr2, cr2, cr2, cr2, cr2, cr2, cr2, cr2,
0522                 cr2, cr2, cr2, cr2, cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3,
0523                 cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3,
0524                 cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3, cr3, ill, ill, l2a,
0525                 l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a,
0526                 l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a, l2a,
0527                 l2a, l2a, l2a, l3a, l3b, l3b, l3b, l3b, l3b, l3b, l3b, l3b, l3b,
0528                 l3b, l3b, l3b, l3c, l3b, l3b, l4a, l4b, l4b, l4b, l4c, ill, ill,
0529                 ill, ill, ill, ill, ill, ill, ill, ill, ill,
0530             };
0531
0532             constexpr table_state transitions[108] = {
0533                 err, e_d, err, err, err, cs1, p3a, cs2, p3b, p4a, cs3, p4b,
0534                 err, err, err, err, err, err, err, err, err, err, err, err,
0535                 err, err, e_d, e_d, e_d, err, err, err, err, err, err, err,
0536                 err, err, cs1, cs1, cs1, err, err, err, err, err, err, err,
0537                 err, err, cs2, cs2, cs2, err, err, err, err, err, err, err,
0538                 err, err, err, err, cs1, err, err, err, err, err, err, err,
0539                 err, err, cs1, cs1, err, err, err, err, err, err, err, err,
0540                 err, err, err, cs2, cs2, err, err, err, err, err, err, err,
0541                 err, err, cs2, err, err, err, err, err, err, err, err, err,
0542             };
0543         }
0544
0545         template<typename InputIter, typename Sentinel>
0546         char32_t advance(InputIter & first, Sentinel last)
0547         {
0548             char32_t retval = 0;
0549
0550             first_cu const info = first_cus[(unsigned char)*first];
0551             ++first;
0552
0553             retval = info.initial_octet;
0554             int state = info.next;
0555
0556             while (state != bgn) {
0557                 if (first != last) {
0558                     unsigned char const cu = *first;
0559                     retval = (retval << 6) | (cu & 0x3f);
0560                     char_class const class_ = octet_classes[cu];
0561                     state = transitions[state + class_];
0562                     if (state == err)
0563                         return replacement_character;
0564                     ++first;
0565                 } else {
0566                     return replacement_character;
0567                 }
0568             }
0569
0570             return retval;
0571         }
0572
0573         template<typename Derived, typename Iter>
0574         struct trans_ins_iter
0575         {
0576             using value_type = void;
0577             using difference_type =
0578 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
0579                 std::ptrdiff_t;
0580 #else
0581                 void;
0582 #endif
0583             using pointer = void;
0584             using reference = void;
0585             using iterator_category = std::output_iterator_tag;
0586
0587             constexpr trans_ins_iter() {}
0588             constexpr trans_ins_iter(Iter it) : it_(it) {}
0589             constexpr Derived & operator*() { return derived(); }
0590             constexpr Derived & operator++() { return derived(); }
0591             constexpr Derived operator++(int) { return derived(); }
0592             constexpr Iter base() const { return it_; }
0593
0594         protected:
0595             constexpr Iter & iter() { return it_; }
0596
0597         private:
0598             constexpr Derived & derived()
0599             {
0600                 return static_cast<Derived &>(*this);
0601             }
0602             Iter it_;
0603         };
0604
0605         template<typename Derived, typename I, typename ValueType>
0606         using trans_iter = stl_interfaces::iterator_interface<
0607             Derived,
0608             bidirectional_at_most_t<I>,
0609             ValueType,
0610             ValueType>;
0611     }
0612
0613 }}
0614
0615 namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAMESPACE_V1 {
0616
0617 #if defined(BOOST_TEXT_DOXYGEN)
0618
0619     /** Returns the first code unit in `[r.begin(), r.end())` that is not
0620         properly UTF-8 encoded, or `r.begin() + std::distance(r)` if no such
0621         code unit is found. */
0622     template<utf8_range R>
0623         requires std::ranges::forward_range<R>
0624     constexpr std::ranges::borrowed_iterator_t<R> find_invalid_encoding(R && r);
0625
0626     /** Returns the first code unit in `[r.begin(), r.end())` that is not
0627         properly UTF-16 encoded, or `r.begin() + std::distance(r)` if no such
0628         code unit is found. */
0629     template<utf16_range R>
0630         requires std::ranges::forward_range<R>
0631     constexpr std::ranges::borrowed_iterator_t<R> find_invalid_encoding(R && r);
0632
0633     /** Returns true iff `r` is properly UTF-8 encoded. */
0634     template<utf8_range R>
0635         requires std::ranges::forward_range<R>
0636     constexpr bool encoded(R && r);
0637
0638     /** Returns true iff `r` is properly UTF-16 encoded */
0639     template<utf16_range R>
0640         requires std::ranges::forward_range<R>
0641     constexpr bool encoded(R && r);
0642
0643     /** Returns true iff `r` is empty or the initial UTF-8 code units in `r`
0644         form a valid Unicode code point. */
0645     template<utf8_range R>
0646         requires std::ranges::forward_range<R>
0647     constexpr bool starts_encoded(R && r);
0648
0649     /** Returns true iff `r` is empty or the initial UTF-16 code units in `r`
0650         form a valid Unicode code point. */
0651     template<utf16_range R>
0652         requires std::ranges::forward_range<R>
0653     constexpr bool starts_encoded(R && r);
0654
0655     /** Returns true iff `r` is empty or the final UTF-8 code units in `r`
0656         form a valid Unicode code point. */
0657     template<utf8_range R>
0658         requires std::ranges::bidirectional_range<R> &&
0659                  std::ranges::common_range<R>
0660     constexpr bool ends_encoded(R && r);
0661
0662     /** Returns true iff `r` is empty or the final UTF-16 code units in `r`
0663         form a valid Unicode code point. */
0664     template<utf8_range R>
0665         requires std::ranges::bidirectional_range<R> &&
0666                  std::ranges::common_range<R>
0667     constexpr bool ends_encoded(R && r);
0668
0669 #endif
0670
0671     template<typename Iter>
0672     constexpr detail::enable_utf8_cp_t<Iter>
0673     find_invalid_encoding(Iter first, Iter last)
0674     {
0675         while (first != last) {
0676             int const cp_bytes = boost::parser::detail::text::utf8_code_units(*first);
0677             if (cp_bytes == -1 || last - first < cp_bytes)
0678                 return first;
0679
0680             if (detail::end_of_invalid_utf8(first))
0681                 return first;
0682
0683             first += cp_bytes;
0684         }
0685
0686         return last;
0687     }
0688
0689     template<typename Iter>
0690     constexpr detail::enable_utf16_cp_t<Iter>
0691     find_invalid_encoding(Iter first, Iter last)
0692     {
0693         while (first != last) {
0694             int const cp_units = boost::parser::detail::text::utf16_code_units(*first);
0695             if (cp_units == -1 || last - first < cp_units)
0696                 return first;
0697
0698             if (cp_units == 2 && !boost::parser::detail::text::low_surrogate(*(first + 1)))
0699                 return first;
0700
0701             first += cp_units;
0702         }
0703
0704         return last;
0705     }
0706
0707     template<typename Iter>
0708     constexpr detail::enable_utf8_cp_t<Iter, bool> encoded(
0709         Iter first, Iter last)
0710     {
0711         return v1::find_invalid_encoding(first, last) == last;
0712     }
0713
0714     template<typename Iter>
0715     constexpr detail::enable_utf16_cp_t<Iter, bool> encoded(
0716         Iter first, Iter last)
0717     {
0718         return v1::find_invalid_encoding(first, last) == last;
0719     }
0720
0721     template<typename Iter>
0722     constexpr detail::enable_utf8_cp_t<Iter, bool>
0723     starts_encoded(Iter first, Iter last)
0724     {
0725         if (first == last)
0726             return true;
0727
0728         int const cp_bytes = boost::parser::detail::text::utf8_code_units(*first);
0729         if (cp_bytes == -1 || last - first < cp_bytes)
0730             return false;
0731
0732         return !detail::end_of_invalid_utf8(first);
0733     }
0734
0735     template<typename Iter>
0736     constexpr detail::enable_utf16_cp_t<Iter, bool>
0737     starts_encoded(Iter first, Iter last)
0738     {
0739         if (first == last)
0740             return true;
0741
0742         int const cp_units = boost::parser::detail::text::utf16_code_units(*first);
0743         if (cp_units == -1 || last - first < cp_units)
0744             return false;
0745
0746         return cp_units == 1 || boost::parser::detail::text::low_surrogate(*(first + 1));
0747     }
0748
0749     template<typename Iter>
0750     constexpr detail::enable_utf8_cp_t<Iter, bool>
0751     ends_encoded(Iter first, Iter last)
0752     {
0753         if (first == last)
0754             return true;
0755
0756         auto it = last;
0757         while (first != --it && boost::parser::detail::text::continuation(*it))
0758             ;
0759
0760         return v1::starts_encoded(it, last);
0761     }
0762
0763     template<typename Iter>
0764     constexpr detail::enable_utf16_cp_t<Iter, bool>
0765     ends_encoded(Iter first, Iter last)
0766     {
0767         if (first == last)
0768             return true;
0769
0770         auto it = last;
0771         if (boost::parser::detail::text::low_surrogate(*--it))
0772             --it;
0773
0774         return v1::starts_encoded(it, last);
0775     }
0776
0777 }}}
0778
0779 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
0780
0781 namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAMESPACE_V2 {
0782
0783     template<utf8_range R>
0784         requires std::ranges::forward_range<R>
0785     constexpr std::ranges::borrowed_iterator_t<R> find_invalid_encoding(R && r)
0786     {
0787         auto first = std::ranges::begin(r);
0788         auto last = std::ranges::end(r);
0789
0790         while (first != last) {
0791             int const cp_bytes = boost::parser::detail::text::utf8_code_units(*first);
0792             if (cp_bytes == -1 || last - first < cp_bytes)
0793                 return first;
0794
0795             if (detail::end_of_invalid_utf8(first))
0796                 return first;
0797
0798             first += cp_bytes;
0799         }
0800
0801         if constexpr (std::ranges::borrowed_range<R>) {
0802             return last;
0803         } else {
0804             return std::ranges::dangling{};
0805         }
0806     }
0807
0808     template<utf16_range R>
0809         requires std::ranges::forward_range<R>
0810     constexpr std::ranges::borrowed_iterator_t<R> find_invalid_encoding(R && r)
0811     {
0812         auto first = std::ranges::begin(r);
0813         auto last = std::ranges::end(r);
0814
0815         while (first != last) {
0816             int const cp_units = boost::parser::detail::text::utf16_code_units(*first);
0817             if (cp_units == -1 || last - first < cp_units)
0818                 return first;
0819
0820             if (cp_units == 2 && !boost::parser::detail::text::low_surrogate(*(first + 1)))
0821                 return first;
0822
0823             first += cp_units;
0824         }
0825
0826         if constexpr (std::ranges::borrowed_range<R>) {
0827             return last;
0828         } else {
0829             return std::ranges::dangling{};
0830         }
0831     }
0832
0833     template<utf8_range R>
0834         requires std::ranges::forward_range<R>
0835     constexpr bool encoded(R && r)
0836     {
0837         return boost::parser::detail::text::v1::find_invalid_encoding(r.begin(), r.end()) ==
0838                r.end();
0839     }
0840
0841     template<utf16_range R>
0842         requires std::ranges::forward_range<R>
0843     constexpr bool encoded(R && r)
0844     {
0845         return boost::parser::detail::text::v1::find_invalid_encoding(r.begin(), r.end()) ==
0846                r.end();
0847     }
0848
0849     template<utf8_range R>
0850         requires std::ranges::forward_range<R>
0851     constexpr bool starts_encoded(R && r)
0852     {
0853         auto first = std::ranges::begin(r);
0854         auto last = std::ranges::end(r);
0855
0856         if (first == last)
0857             return true;
0858
0859         int const cp_bytes = boost::parser::detail::text::utf8_code_units(*first);
0860         if (cp_bytes == -1 || last - first < cp_bytes)
0861             return false;
0862
0863         return !detail::end_of_invalid_utf8(first);
0864     }
0865
0866     template<utf16_range R>
0867         requires std::ranges::forward_range<R>
0868     constexpr bool starts_encoded(R && r)
0869     {
0870         auto first = std::ranges::begin(r);
0871         auto last = std::ranges::end(r);
0872
0873         if (first == last)
0874             return true;
0875
0876         int const cp_units = boost::parser::detail::text::utf16_code_units(*first);
0877         if (cp_units == -1 || last - first < cp_units)
0878             return false;
0879
0880         return cp_units == 1 || boost::parser::detail::text::low_surrogate(*(first + 1));
0881     }
0882
0883     template<utf8_range R>
0884         requires std::ranges::bidirectional_range<R> &&
0885                  std::ranges::common_range<R>
0886     constexpr bool ends_encoded(R && r)
0887     {
0888         auto first = std::ranges::begin(r);
0889         auto last = std::ranges::end(r);
0890
0891         if (first == last)
0892             return true;
0893
0894         auto it = last;
0895         while (first != --it && boost::parser::detail::text::continuation(*it))
0896             ;
0897
0898         return boost::parser::detail::text::starts_encoded(it, last);
0899     }
0900
0901     template<utf16_range R>
0902         requires std::ranges::bidirectional_range<R> &&
0903                  std::ranges::common_range<R>
0904     constexpr bool ends_encoded(R && r)
0905     {
0906         auto first = std::ranges::begin(r);
0907         auto last = std::ranges::end(r);
0908
0909         if (first == last)
0910             return true;
0911
0912         auto it = last;
0913         if (boost::parser::detail::text::low_surrogate(*--it))
0914             --it;
0915
0916         return boost::parser::detail::text::starts_encoded(it, last);
0917     }
0918
0919 }}}
0920
0921 #endif
0922
0923 namespace boost::parser::detail { namespace text {
0924
0925     /** An error handler type that can be used with the converting iterators;
0926         provides the Unicode replacement character on errors. */
0927     struct use_replacement_character
0928     {
0929         constexpr char32_t operator()(std::string_view) const noexcept
0930         {
0931             return replacement_character;
0932         }
0933     };
0934
0935     /** A sentinel type that compares equal to a pointer to a 1-, 2-, or
0936         4-byte integral value, iff the pointer is null. */
0937     struct null_sentinel_t
0938     {
0939 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
0940         template<std::input_iterator I>
0941             requires std::default_initializable<std::iter_value_t<I>> &&
0942                      std::equality_comparable_with<std::iter_reference_t<I>, std::iter_value_t<I>>
0943 #else
0944         template<typename I>
0945 #endif
0946         friend constexpr bool operator==(I it, null_sentinel_t)
0947         {
0948             return *it == detail::iter_value_t<I>{};
0949         }
0950 #if !defined(__cpp_impl_three_way_comparison)
0951         template<typename I>
0952         friend constexpr bool operator==(null_sentinel_t, I it)
0953         {
0954             return *it == detail::iter_value_t<I>{};
0955         }
0956         template<typename I>
0957         friend constexpr bool operator!=(I it, null_sentinel_t)
0958         {
0959             return *it != detail::iter_value_t<I>{};
0960         }
0961         template<typename I>
0962         friend constexpr bool operator!=(null_sentinel_t, I it)
0963         {
0964             return *it != detail::iter_value_t<I>{};
0965         }
0966 #endif
0967     };
0968
0969 #if defined(__cpp_inline_variables)
0970     inline constexpr null_sentinel_t null_sentinel;
0971 #else
0972     namespace {
0973         constexpr null_sentinel_t null_sentinel;
0974     }
0975 #endif
0976
0977     /** An out iterator that converts UTF-32 to UTF-8. */
0978 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
0979     template<std::output_iterator<char8_t> Iter>
0980 #else
0981     template<typename Iter>
0982 #endif
0983     struct utf_32_to_8_out_iterator
0984         : detail::trans_ins_iter<utf_32_to_8_out_iterator<Iter>, Iter>
0985     {
0986         constexpr utf_32_to_8_out_iterator() {}
0987         explicit constexpr utf_32_to_8_out_iterator(Iter it) :
0988             detail::trans_ins_iter<utf_32_to_8_out_iterator<Iter>, Iter>(it)
0989         {}
0990
0991         constexpr utf_32_to_8_out_iterator & operator=(char32_t cp)
0992         {
0993             auto & out = this->iter();
0994             out = detail::write_cp_utf8(cp, out);
0995             return *this;
0996         }
0997     };
0998
0999     /** An insert-iterator analogous to std::insert_iterator, that also
1000         converts UTF-32 to UTF-8. */
1001     template<typename Cont>
1002     struct utf_32_to_8_insert_iterator : detail::trans_ins_iter<
1003                                              utf_32_to_8_insert_iterator<Cont>,
1004                                              std::insert_iterator<Cont>>
1005     {
1006         constexpr utf_32_to_8_insert_iterator() {}
1007         constexpr utf_32_to_8_insert_iterator(
1008             Cont & c, typename Cont::iterator it) :
1009             detail::trans_ins_iter<
1010                 utf_32_to_8_insert_iterator<Cont>,
1011                 std::insert_iterator<Cont>>(std::insert_iterator<Cont>(c, it))
1012         {}
1013
1014         constexpr utf_32_to_8_insert_iterator & operator=(char32_t cp)
1015         {
1016             auto & out = this->iter();
1017             out = detail::write_cp_utf8(cp, out);
1018             return *this;
1019         }
1020     };
1021
1022     /** An insert-iterator analogous to std::front_insert_iterator, that also
1023         converts UTF-32 to UTF-8. */
1024     template<typename Cont>
1025     struct utf_32_to_8_front_insert_iterator
1026         : detail::trans_ins_iter<
1027               utf_32_to_8_front_insert_iterator<Cont>,
1028               std::front_insert_iterator<Cont>>
1029     {
1030         constexpr utf_32_to_8_front_insert_iterator() {}
1031         explicit constexpr utf_32_to_8_front_insert_iterator(Cont & c) :
1032             detail::trans_ins_iter<
1033                 utf_32_to_8_front_insert_iterator<Cont>,
1034                 std::front_insert_iterator<Cont>>(
1035                 std::front_insert_iterator<Cont>(c))
1036         {}
1037
1038         constexpr utf_32_to_8_front_insert_iterator & operator=(char32_t cp)
1039         {
1040             auto & out = this->iter();
1041             out = detail::write_cp_utf8(cp, out);
1042             return *this;
1043         }
1044     };
1045
1046     /** An insert-iterator analogous to std::back_insert_iterator, that also
1047         converts UTF-32 to UTF-8. */
1048     template<typename Cont>
1049     struct utf_32_to_8_back_insert_iterator
1050         : detail::trans_ins_iter<
1051               utf_32_to_8_back_insert_iterator<Cont>,
1052               std::back_insert_iterator<Cont>>
1053     {
1054         constexpr utf_32_to_8_back_insert_iterator() {}
1055         explicit constexpr utf_32_to_8_back_insert_iterator(Cont & c) :
1056             detail::trans_ins_iter<
1057                 utf_32_to_8_back_insert_iterator<Cont>,
1058                 std::back_insert_iterator<Cont>>(
1059                 std::back_insert_iterator<Cont>(c))
1060         {}
1061
1062         constexpr utf_32_to_8_back_insert_iterator & operator=(char32_t cp)
1063         {
1064             auto & out = this->iter();
1065             out = detail::write_cp_utf8(cp, out);
1066             return *this;
1067         }
1068     };
1069
1070
1071     namespace detail {
1072         template<typename OutIter>
1073         OutIter assign_8_to_32_insert(
1074             unsigned char cu, char32_t & cp, int & state, OutIter out)
1075         {
1076             auto write = [&] {
1077                 *out = cp;
1078                 ++out;
1079                 state = invalid_table_state;
1080             };
1081             auto start_cp = [&] {
1082                 first_cu const info = first_cus[cu];
1083                 state = info.next;
1084                 cp = info.initial_octet;
1085                 if (state == bgn)
1086                     write();
1087             };
1088             if (state == invalid_table_state) {
1089                 start_cp();
1090             } else {
1091                 cp = (cp << 6) | (cu & 0x3f);
1092                 char_class const class_ = octet_classes[cu];
1093                 state = transitions[state + class_];
1094                 if (state == bgn) {
1095                     write();
1096                 } else if (state == err) {
1097                     *out = replacement_character;
1098                     ++out;
1099                     start_cp();
1100                 }
1101             }
1102             return out;
1103         }
1104     }
1105
1106     /** An out iterator that converts UTF-8 to UTF-32. */
1107 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
1108     template<std::output_iterator<char32_t> Iter>
1109 #else
1110     template<typename Iter>
1111 #endif
1112     struct utf_8_to_32_out_iterator
1113         : detail::trans_ins_iter<utf_8_to_32_out_iterator<Iter>, Iter>
1114     {
1115         constexpr utf_8_to_32_out_iterator() {}
1116         explicit constexpr utf_8_to_32_out_iterator(Iter it) :
1117             detail::trans_ins_iter<utf_8_to_32_out_iterator<Iter>, Iter>(it),
1118             state_(detail::invalid_table_state)
1119         {}
1120
1121         constexpr utf_8_to_32_out_iterator & operator=(char8_type cu)
1122         {
1123             auto & out = this->iter();
1124             out = detail::assign_8_to_32_insert(cu, cp_, state_, out);
1125             return *this;
1126         }
1127
1128 #ifndef BOOST_TEXT_DOXYGEN
1129     private:
1130         int state_;
1131         char32_t cp_;
1132 #endif
1133     };
1134
1135     /** An insert-iterator analogous to std::insert_iterator, that also
1136         converts UTF-8 to UTF-32. */
1137     template<typename Cont>
1138     struct utf_8_to_32_insert_iterator : detail::trans_ins_iter<
1139                                              utf_8_to_32_insert_iterator<Cont>,
1140                                              std::insert_iterator<Cont>>
1141     {
1142         constexpr utf_8_to_32_insert_iterator() {}
1143         constexpr utf_8_to_32_insert_iterator(
1144             Cont & c, typename Cont::iterator it) :
1145             detail::trans_ins_iter<
1146                 utf_8_to_32_insert_iterator<Cont>,
1147                 std::insert_iterator<Cont>>(std::insert_iterator<Cont>(c, it)),
1148             state_(detail::invalid_table_state)
1149         {}
1150
1151         constexpr utf_8_to_32_insert_iterator & operator=(char16_t cu)
1152         {
1153             auto & out = this->iter();
1154             out = detail::assign_8_to_32_insert(cu, cp_, state_, out);
1155             return *this;
1156         }
1157
1158 #ifndef BOOST_TEXT_DOXYGEN
1159     private:
1160         int state_;
1161         char32_t cp_;
1162 #endif
1163     };
1164
1165     /** An insert-iterator analogous to std::front_insert_iterator, that also
1166         converts UTF-8 to UTF-32. */
1167     template<typename Cont>
1168     struct utf_8_to_32_front_insert_iterator
1169         : detail::trans_ins_iter<
1170               utf_8_to_32_front_insert_iterator<Cont>,
1171               std::front_insert_iterator<Cont>>
1172     {
1173         constexpr utf_8_to_32_front_insert_iterator() {}
1174         explicit constexpr utf_8_to_32_front_insert_iterator(Cont & c) :
1175             detail::trans_ins_iter<
1176                 utf_8_to_32_front_insert_iterator<Cont>,
1177                 std::front_insert_iterator<Cont>>(
1178                 std::front_insert_iterator<Cont>(c)),
1179             state_(detail::invalid_table_state)
1180         {}
1181
1182         constexpr utf_8_to_32_front_insert_iterator & operator=(char16_t cu)
1183         {
1184             auto & out = this->iter();
1185             out = detail::assign_8_to_32_insert(cu, cp_, state_, out);
1186             return *this;
1187         }
1188
1189 #ifndef BOOST_TEXT_DOXYGEN
1190     private:
1191         int state_;
1192         char32_t cp_;
1193 #endif
1194     };
1195
1196     /** An insert-iterator analogous to std::back_insert_iterator, that also
1197         converts UTF-8 to UTF-32. */
1198     template<typename Cont>
1199     struct utf_8_to_32_back_insert_iterator
1200         : detail::trans_ins_iter<
1201               utf_8_to_32_back_insert_iterator<Cont>,
1202               std::back_insert_iterator<Cont>>
1203     {
1204         constexpr utf_8_to_32_back_insert_iterator() {}
1205         explicit constexpr utf_8_to_32_back_insert_iterator(Cont & c) :
1206             detail::trans_ins_iter<
1207                 utf_8_to_32_back_insert_iterator<Cont>,
1208                 std::back_insert_iterator<Cont>>(
1209                 std::back_insert_iterator<Cont>(c)),
1210             state_(detail::invalid_table_state)
1211         {}
1212
1213         constexpr utf_8_to_32_back_insert_iterator & operator=(char16_t cu)
1214         {
1215             auto & out = this->iter();
1216             out = detail::assign_8_to_32_insert(cu, cp_, state_, out);
1217             return *this;
1218         }
1219
1220 #ifndef BOOST_TEXT_DOXYGEN
1221     private:
1222         int state_;
1223         char32_t cp_;
1224 #endif
1225     };
1226
1227
1228     /** An out iterator that converts UTF-8 to UTF-16. */
1229 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
1230     template<std::output_iterator<char16_t> Iter>
1231 #else
1232     template<typename Iter>
1233 #endif
1234     struct utf_32_to_16_out_iterator
1235         : detail::trans_ins_iter<utf_32_to_16_out_iterator<Iter>, Iter>
1236     {
1237         constexpr utf_32_to_16_out_iterator() {}
1238         explicit constexpr utf_32_to_16_out_iterator(Iter it) :
1239             detail::trans_ins_iter<utf_32_to_16_out_iterator<Iter>, Iter>(it)
1240         {}
1241
1242         constexpr utf_32_to_16_out_iterator & operator=(char32_t cp)
1243         {
1244             auto & out = this->iter();
1245             out = detail::write_cp_utf16(cp, out);
1246             return *this;
1247         }
1248     };
1249
1250     /** An insert-iterator analogous to std::insert_iterator, that also
1251         converts UTF-32 to UTF-16. */
1252     template<typename Cont>
1253     struct utf_32_to_16_insert_iterator
1254         : detail::trans_ins_iter<
1255               utf_32_to_16_insert_iterator<Cont>,
1256               std::insert_iterator<Cont>>
1257     {
1258         constexpr utf_32_to_16_insert_iterator() {}
1259         constexpr utf_32_to_16_insert_iterator(
1260             Cont & c, typename Cont::iterator it) :
1261             detail::trans_ins_iter<
1262                 utf_32_to_16_insert_iterator<Cont>,
1263                 std::insert_iterator<Cont>>(std::insert_iterator<Cont>(c, it))
1264         {}
1265
1266         constexpr utf_32_to_16_insert_iterator & operator=(char32_t cp)
1267         {
1268             auto & out = this->iter();
1269             out = detail::write_cp_utf16(cp, out);
1270             return *this;
1271         }
1272     };
1273
1274     /** An insert-iterator analogous to std::front_insert_iterator, that also
1275         converts UTF-32 to UTF-16. */
1276     template<typename Cont>
1277     struct utf_32_to_16_front_insert_iterator
1278         : detail::trans_ins_iter<
1279               utf_32_to_16_front_insert_iterator<Cont>,
1280               std::front_insert_iterator<Cont>>
1281     {
1282         constexpr utf_32_to_16_front_insert_iterator() {}
1283         explicit constexpr utf_32_to_16_front_insert_iterator(Cont & c) :
1284             detail::trans_ins_iter<
1285                 utf_32_to_16_front_insert_iterator<Cont>,
1286                 std::front_insert_iterator<Cont>>(
1287                 std::front_insert_iterator<Cont>(c))
1288         {}
1289
1290         constexpr utf_32_to_16_front_insert_iterator & operator=(char32_t cp)
1291         {
1292             auto & out = this->iter();
1293             out = detail::write_cp_utf16(cp, out);
1294             return *this;
1295         }
1296     };
1297
1298     /** An insert-iterator analogous to std::back_insert_iterator, that also
1299         converts UTF-32 to UTF-16. */
1300     template<typename Cont>
1301     struct utf_32_to_16_back_insert_iterator
1302         : detail::trans_ins_iter<
1303               utf_32_to_16_back_insert_iterator<Cont>,
1304               std::back_insert_iterator<Cont>>
1305     {
1306         constexpr utf_32_to_16_back_insert_iterator() {}
1307         explicit constexpr utf_32_to_16_back_insert_iterator(Cont & c) :
1308             detail::trans_ins_iter<
1309                 utf_32_to_16_back_insert_iterator<Cont>,
1310                 std::back_insert_iterator<Cont>>(
1311                 std::back_insert_iterator<Cont>(c))
1312         {}
1313
1314         constexpr utf_32_to_16_back_insert_iterator & operator=(char32_t cp)
1315         {
1316             auto & out = this->iter();
1317             out = detail::write_cp_utf16(cp, out);
1318             return *this;
1319         }
1320     };
1321
1322
1323     namespace detail {
1324         template<typename OutIter>
1325         OutIter
1326         assign_16_to_32_insert(char16_t & prev_cu, char16_t cu, OutIter out)
1327         {
1328             if (high_surrogate(cu)) {
1329                 if (prev_cu) {
1330                     *out = replacement_character;
1331                     ++out;
1332                 }
1333                 prev_cu = cu;
1334             } else if (low_surrogate(cu)) {
1335                 if (prev_cu) {
1336                     *out = detail::surrogates_to_cp(prev_cu, cu);
1337                     ++out;
1338                 } else {
1339                     *out = replacement_character;
1340                     ++out;
1341                 }
1342                 prev_cu = 0;
1343             } else {
1344                 if (prev_cu) {
1345                     *out = replacement_character;
1346                     ++out;
1347                 }
1348                 *out = cu;
1349                 ++out;
1350                 prev_cu = 0;
1351             }
1352             return out;
1353         }
1354     }
1355
1356     /** An out iterator that converts UTF-16 to UTF-32. */
1357 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
1358     template<std::output_iterator<char32_t> Iter>
1359 #else
1360     template<typename Iter>
1361 #endif
1362     struct utf_16_to_32_out_iterator
1363         : detail::trans_ins_iter<utf_16_to_32_out_iterator<Iter>, Iter>
1364     {
1365         constexpr utf_16_to_32_out_iterator() {}
1366         explicit constexpr utf_16_to_32_out_iterator(Iter it) :
1367             detail::trans_ins_iter<utf_16_to_32_out_iterator<Iter>, Iter>(it),
1368             prev_cu_(0)
1369         {}
1370
1371         constexpr utf_16_to_32_out_iterator & operator=(char16_t cu)
1372         {
1373             auto & out = this->iter();
1374             out = detail::assign_16_to_32_insert(prev_cu_, cu, out);
1375             return *this;
1376         }
1377
1378 #ifndef BOOST_TEXT_DOXYGEN
1379     private:
1380         char16_t prev_cu_;
1381 #endif
1382     };
1383
1384     /** An insert-iterator analogous to std::insert_iterator, that also
1385         converts UTF-16 to UTF-32. */
1386     template<typename Cont>
1387     struct utf_16_to_32_insert_iterator
1388         : detail::trans_ins_iter<
1389               utf_16_to_32_insert_iterator<Cont>,
1390               std::insert_iterator<Cont>>
1391     {
1392         constexpr utf_16_to_32_insert_iterator() {}
1393         constexpr utf_16_to_32_insert_iterator(
1394             Cont & c, typename Cont::iterator it) :
1395             detail::trans_ins_iter<
1396                 utf_16_to_32_insert_iterator<Cont>,
1397                 std::insert_iterator<Cont>>(std::insert_iterator<Cont>(c, it)),
1398             prev_cu_(0)
1399         {}
1400
1401         constexpr utf_16_to_32_insert_iterator & operator=(char16_t cu)
1402         {
1403             auto & out = this->iter();
1404             out = detail::assign_16_to_32_insert(prev_cu_, cu, out);
1405             return *this;
1406         }
1407
1408 #ifndef BOOST_TEXT_DOXYGEN
1409     private:
1410         char16_t prev_cu_;
1411 #endif
1412     };
1413
1414     /** An insert-iterator analogous to std::front_insert_iterator, that also
1415         converts UTF-16 to UTF-32. */
1416     template<typename Cont>
1417     struct utf_16_to_32_front_insert_iterator
1418         : detail::trans_ins_iter<
1419               utf_16_to_32_front_insert_iterator<Cont>,
1420               std::front_insert_iterator<Cont>>
1421     {
1422         constexpr utf_16_to_32_front_insert_iterator() {}
1423         explicit constexpr utf_16_to_32_front_insert_iterator(Cont & c) :
1424             detail::trans_ins_iter<
1425                 utf_16_to_32_front_insert_iterator<Cont>,
1426                 std::front_insert_iterator<Cont>>(
1427                 std::front_insert_iterator<Cont>(c)),
1428             prev_cu_(0)
1429         {}
1430
1431         constexpr utf_16_to_32_front_insert_iterator & operator=(char16_t cu)
1432         {
1433             auto & out = this->iter();
1434             out = detail::assign_16_to_32_insert(prev_cu_, cu, out);
1435             return *this;
1436         }
1437
1438 #ifndef BOOST_TEXT_DOXYGEN
1439     private:
1440         char16_t prev_cu_;
1441 #endif
1442     };
1443
1444     /** An insert-iterator analogous to std::back_insert_iterator, that also
1445         converts UTF-16 to UTF-32. */
1446     template<typename Cont>
1447     struct utf_16_to_32_back_insert_iterator
1448         : detail::trans_ins_iter<
1449               utf_16_to_32_back_insert_iterator<Cont>,
1450               std::back_insert_iterator<Cont>>
1451     {
1452         constexpr utf_16_to_32_back_insert_iterator() {}
1453         explicit constexpr utf_16_to_32_back_insert_iterator(Cont & c) :
1454             detail::trans_ins_iter<
1455                 utf_16_to_32_back_insert_iterator<Cont>,
1456                 std::back_insert_iterator<Cont>>(
1457                 std::back_insert_iterator<Cont>(c)),
1458             prev_cu_(0)
1459         {}
1460
1461         constexpr utf_16_to_32_back_insert_iterator & operator=(char16_t cu)
1462         {
1463             auto & out = this->iter();
1464             out = detail::assign_16_to_32_insert(prev_cu_, cu, out);
1465             return *this;
1466         }
1467
1468 #ifndef BOOST_TEXT_DOXYGEN
1469     private:
1470         char16_t prev_cu_;
1471 #endif
1472     };
1473
1474
1475     namespace detail {
1476         template<typename OutIter>
1477         OutIter
1478         assign_16_to_8_insert(char16_t & prev_cu, char16_t cu, OutIter out)
1479         {
1480             if (high_surrogate(cu)) {
1481                 if (prev_cu)
1482                     out = detail::write_cp_utf8(replacement_character, out);
1483                 prev_cu = cu;
1484             } else if (low_surrogate(cu)) {
1485                 if (prev_cu) {
1486                     auto const cp = detail::surrogates_to_cp(prev_cu, cu);
1487                     out = detail::write_cp_utf8(cp, out);
1488                 } else {
1489                     out = detail::write_cp_utf8(replacement_character, out);
1490                 }
1491                 prev_cu = 0;
1492             } else {
1493                 if (prev_cu)
1494                     out = detail::write_cp_utf8(replacement_character, out);
1495                 out = detail::write_cp_utf8(cu, out);
1496                 prev_cu = 0;
1497             }
1498             return out;
1499         }
1500     }
1501
1502     /** An out iterator that converts UTF-16 to UTF-8. */
1503 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
1504     template<std::output_iterator<char8_t> Iter>
1505 #else
1506     template<typename Iter>
1507 #endif
1508     struct utf_16_to_8_out_iterator
1509         : detail::trans_ins_iter<utf_16_to_8_out_iterator<Iter>, Iter>
1510     {
1511         constexpr utf_16_to_8_out_iterator() {}
1512         explicit constexpr utf_16_to_8_out_iterator(Iter it) :
1513             detail::trans_ins_iter<utf_16_to_8_out_iterator<Iter>, Iter>(it),
1514             prev_cu_(0)
1515         {}
1516
1517         constexpr utf_16_to_8_out_iterator & operator=(char16_t cu)
1518         {
1519             auto & out = this->iter();
1520             out = detail::assign_16_to_8_insert(prev_cu_, cu, out);
1521             return *this;
1522         }
1523
1524 #ifndef BOOST_TEXT_DOXYGEN
1525     private:
1526         char16_t prev_cu_;
1527 #endif
1528     };
1529
1530     /** An insert-iterator analogous to std::insert_iterator, that also
1531         converts UTF-16 to UTF-8. */
1532     template<typename Cont>
1533     struct utf_16_to_8_insert_iterator : detail::trans_ins_iter<
1534                                              utf_16_to_8_insert_iterator<Cont>,
1535                                              std::insert_iterator<Cont>>
1536     {
1537         constexpr utf_16_to_8_insert_iterator() {}
1538         constexpr utf_16_to_8_insert_iterator(
1539             Cont & c, typename Cont::iterator it) :
1540             detail::trans_ins_iter<
1541                 utf_16_to_8_insert_iterator<Cont>,
1542                 std::insert_iterator<Cont>>(std::insert_iterator<Cont>(c, it)),
1543             prev_cu_(0)
1544         {}
1545
1546         constexpr utf_16_to_8_insert_iterator & operator=(char16_t cu)
1547         {
1548             auto & out = this->iter();
1549             out = detail::assign_16_to_8_insert(prev_cu_, cu, out);
1550             return *this;
1551         }
1552
1553 #ifndef BOOST_TEXT_DOXYGEN
1554     private:
1555         char16_t prev_cu_;
1556 #endif
1557     };
1558
1559     /** An insert-iterator analogous to std::front_insert_iterator, that also
1560         converts UTF-16 to UTF-8. */
1561     template<typename Cont>
1562     struct utf_16_to_8_front_insert_iterator
1563         : detail::trans_ins_iter<
1564               utf_16_to_8_front_insert_iterator<Cont>,
1565               std::front_insert_iterator<Cont>>
1566     {
1567         constexpr utf_16_to_8_front_insert_iterator() {}
1568         explicit constexpr utf_16_to_8_front_insert_iterator(Cont & c) :
1569             detail::trans_ins_iter<
1570                 utf_16_to_8_front_insert_iterator<Cont>,
1571                 std::front_insert_iterator<Cont>>(
1572                 std::front_insert_iterator<Cont>(c)),
1573             prev_cu_(0)
1574         {}
1575
1576         constexpr utf_16_to_8_front_insert_iterator & operator=(char16_t cu)
1577         {
1578             auto & out = this->iter();
1579             out = detail::assign_16_to_8_insert(prev_cu_, cu, out);
1580             return *this;
1581         }
1582
1583 #ifndef BOOST_TEXT_DOXYGEN
1584     private:
1585         char16_t prev_cu_;
1586 #endif
1587     };
1588
1589     /** An insert-iterator analogous to std::back_insert_iterator, that also
1590         converts UTF-16 to UTF-8. */
1591     template<typename Cont>
1592     struct utf_16_to_8_back_insert_iterator
1593         : detail::trans_ins_iter<
1594               utf_16_to_8_back_insert_iterator<Cont>,
1595               std::back_insert_iterator<Cont>>
1596     {
1597         constexpr utf_16_to_8_back_insert_iterator() {}
1598         explicit constexpr utf_16_to_8_back_insert_iterator(Cont & c) :
1599             detail::trans_ins_iter<
1600                 utf_16_to_8_back_insert_iterator<Cont>,
1601                 std::back_insert_iterator<Cont>>(
1602                 std::back_insert_iterator<Cont>(c)),
1603             prev_cu_(0)
1604         {}
1605
1606         constexpr utf_16_to_8_back_insert_iterator & operator=(char16_t cu)
1607         {
1608             auto & out = this->iter();
1609             out = detail::assign_16_to_8_insert(prev_cu_, cu, out);
1610             return *this;
1611         }
1612
1613 #ifndef BOOST_TEXT_DOXYGEN
1614     private:
1615         char16_t prev_cu_;
1616 #endif
1617     };
1618
1619
1620     namespace detail {
1621         template<typename OutIter>
1622         OutIter assign_8_to_16_insert(
1623             unsigned char cu, char32_t & cp, int & state, OutIter out)
1624         {
1625             auto write = [&] {
1626                 out = detail::write_cp_utf16(cp, out);
1627                 state = invalid_table_state;
1628             };
1629             auto start_cp = [&] {
1630                 first_cu const info = first_cus[cu];
1631                 state = info.next;
1632                 cp = info.initial_octet;
1633                 if (state == bgn)
1634                     write();
1635             };
1636             if (state == invalid_table_state) {
1637                 start_cp();
1638             } else {
1639                 cp = (cp << 6) | (cu & 0x3f);
1640                 char_class const class_ = octet_classes[cu];
1641                 state = transitions[state + class_];
1642                 if (state == bgn) {
1643                     write();
1644                 } else if (state == err) {
1645                     out = detail::write_cp_utf16(replacement_character, out);
1646                     start_cp();
1647                 }
1648             }
1649             return out;
1650         }
1651     }
1652
1653     /** An out iterator that converts UTF-8 to UTF-16. */
1654 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
1655     template<std::output_iterator<char16_t> Iter>
1656 #else
1657     template<typename Iter>
1658 #endif
1659     struct utf_8_to_16_out_iterator
1660         : detail::trans_ins_iter<utf_8_to_16_out_iterator<Iter>, Iter>
1661     {
1662         constexpr utf_8_to_16_out_iterator() {}
1663         explicit constexpr utf_8_to_16_out_iterator(Iter it) :
1664             detail::trans_ins_iter<utf_8_to_16_out_iterator<Iter>, Iter>(it),
1665             state_(detail::invalid_table_state)
1666         {}
1667
1668         constexpr utf_8_to_16_out_iterator & operator=(char8_type cu)
1669         {
1670             auto & out = this->iter();
1671             out = detail::assign_8_to_16_insert(cu, cp_, state_, out);
1672             return *this;
1673         }
1674
1675 #ifndef BOOST_TEXT_DOXYGEN
1676     private:
1677         int state_;
1678         char32_t cp_;
1679 #endif
1680     };
1681
1682     /** An insert-iterator analogous to std::insert_iterator, that also
1683         converts UTF-8 to UTF-16. */
1684     template<typename Cont>
1685     struct utf_8_to_16_insert_iterator : detail::trans_ins_iter<
1686                                              utf_8_to_16_insert_iterator<Cont>,
1687                                              std::insert_iterator<Cont>>
1688     {
1689         constexpr utf_8_to_16_insert_iterator() {}
1690         constexpr utf_8_to_16_insert_iterator(
1691             Cont & c, typename Cont::iterator it) :
1692             detail::trans_ins_iter<
1693                 utf_8_to_16_insert_iterator<Cont>,
1694                 std::insert_iterator<Cont>>(std::insert_iterator<Cont>(c, it)),
1695             state_(detail::invalid_table_state)
1696         {}
1697
1698         constexpr utf_8_to_16_insert_iterator & operator=(char16_t cu)
1699         {
1700             auto & out = this->iter();
1701             out = detail::assign_8_to_16_insert(cu, cp_, state_, out);
1702             return *this;
1703         }
1704
1705 #ifndef BOOST_TEXT_DOXYGEN
1706     private:
1707         int state_;
1708         char32_t cp_;
1709 #endif
1710     };
1711
1712     /** An insert-iterator analogous to std::front_insert_iterator, that also
1713         converts UTF-8 to UTF-16. */
1714     template<typename Cont>
1715     struct utf_8_to_16_front_insert_iterator
1716         : detail::trans_ins_iter<
1717               utf_8_to_16_front_insert_iterator<Cont>,
1718               std::front_insert_iterator<Cont>>
1719     {
1720         constexpr utf_8_to_16_front_insert_iterator() {}
1721         explicit constexpr utf_8_to_16_front_insert_iterator(Cont & c) :
1722             detail::trans_ins_iter<
1723                 utf_8_to_16_front_insert_iterator<Cont>,
1724                 std::front_insert_iterator<Cont>>(
1725                 std::front_insert_iterator<Cont>(c)),
1726             state_(detail::invalid_table_state)
1727         {}
1728
1729         constexpr utf_8_to_16_front_insert_iterator & operator=(char16_t cu)
1730         {
1731             auto & out = this->iter();
1732             out = detail::assign_8_to_16_insert(cu, cp_, state_, out);
1733             return *this;
1734         }
1735
1736 #ifndef BOOST_TEXT_DOXYGEN
1737     private:
1738         int state_;
1739         char32_t cp_;
1740 #endif
1741     };
1742
1743     /** An insert-iterator analogous to std::back_insert_iterator, that also
1744         converts UTF-8 to UTF-16. */
1745     template<typename Cont>
1746     struct utf_8_to_16_back_insert_iterator
1747         : detail::trans_ins_iter<
1748               utf_8_to_16_back_insert_iterator<Cont>,
1749               std::back_insert_iterator<Cont>>
1750     {
1751         constexpr utf_8_to_16_back_insert_iterator() {}
1752         explicit constexpr utf_8_to_16_back_insert_iterator(Cont & c) :
1753             detail::trans_ins_iter<
1754                 utf_8_to_16_back_insert_iterator<Cont>,
1755                 std::back_insert_iterator<Cont>>(
1756                 std::back_insert_iterator<Cont>(c)),
1757             state_(detail::invalid_table_state)
1758         {}
1759
1760         constexpr utf_8_to_16_back_insert_iterator & operator=(char16_t cu)
1761         {
1762             auto & out = this->iter();
1763             out = detail::assign_8_to_16_insert(cu, cp_, state_, out);
1764             return *this;
1765         }
1766
1767 #ifndef BOOST_TEXT_DOXYGEN
1768     private:
1769         int state_;
1770         char32_t cp_;
1771 #endif
1772     };
1773
1774 }}
1775
1776 #include <boost/parser/detail/text/unpack.hpp>
1777
1778 namespace boost::parser::detail { namespace text { namespace detail {
1779
1780     template<format Tag>
1781     struct make_utf8_dispatch;
1782
1783     template<>
1784     struct make_utf8_dispatch<format::utf8>
1785     {
1786         template<typename Iter, typename Sentinel>
1787         static constexpr Iter call(Iter first, Iter it, Sentinel last)
1788         {
1789             return it;
1790         }
1791     };
1792
1793     template<>
1794     struct make_utf8_dispatch<format::utf16>
1795     {
1796         template<typename Iter, typename Sentinel>
1797         static constexpr utf_iterator<
1798             format::utf16,
1799             format::utf8,
1800             Iter,
1801             Sentinel>
1802         call(Iter first, Iter it, Sentinel last)
1803         {
1804             return {first, it, last};
1805         }
1806     };
1807
1808     template<>
1809     struct make_utf8_dispatch<format::utf32>
1810     {
1811         template<typename Iter, typename Sentinel>
1812         static constexpr utf_iterator<
1813             format::utf32,
1814             format::utf8,
1815             Iter,
1816             Sentinel>
1817         call(Iter first, Iter it, Sentinel last)
1818         {
1819             return {first, it, last};
1820         }
1821     };
1822
1823     template<format Tag>
1824     struct make_utf16_dispatch;
1825
1826     template<>
1827     struct make_utf16_dispatch<format::utf8>
1828     {
1829         template<typename Iter, typename Sentinel>
1830         static constexpr utf_iterator<
1831             format::utf8,
1832             format::utf16,
1833             Iter,
1834             Sentinel>
1835         call(Iter first, Iter it, Sentinel last)
1836         {
1837             return {first, it, last};
1838         }
1839     };
1840
1841     template<>
1842     struct make_utf16_dispatch<format::utf16>
1843     {
1844         template<typename Iter, typename Sentinel>
1845         static constexpr Iter call(Iter first, Iter it, Sentinel last)
1846         {
1847             return it;
1848         }
1849     };
1850
1851     template<>
1852     struct make_utf16_dispatch<format::utf32>
1853     {
1854         template<typename Iter, typename Sentinel>
1855         static constexpr utf_iterator<
1856             format::utf32,
1857             format::utf16,
1858             Iter,
1859             Sentinel>
1860         call(Iter first, Iter it, Sentinel last)
1861         {
1862             return {first, it, last};
1863         }
1864     };
1865
1866     template<format Tag>
1867     struct make_utf32_dispatch;
1868
1869     template<>
1870     struct make_utf32_dispatch<format::utf8>
1871     {
1872         template<typename Iter, typename Sentinel>
1873         static constexpr utf_iterator<
1874             format::utf8,
1875             format::utf32,
1876             Iter,
1877             Sentinel>
1878         call(Iter first, Iter it, Sentinel last)
1879         {
1880             return {first, it, last};
1881         }
1882     };
1883
1884     template<>
1885     struct make_utf32_dispatch<format::utf16>
1886     {
1887         template<typename Iter, typename Sentinel>
1888         static constexpr utf_iterator<
1889             format::utf16,
1890             format::utf32,
1891             Iter,
1892             Sentinel>
1893         call(Iter first, Iter it, Sentinel last)
1894         {
1895             return {first, it, last};
1896         }
1897     };
1898
1899     template<>
1900     struct make_utf32_dispatch<format::utf32>
1901     {
1902         template<typename Iter, typename Sentinel>
1903         static constexpr Iter call(Iter first, Iter it, Sentinel last)
1904         {
1905             return it;
1906         }
1907     };
1908
1909     template<
1910         typename Cont,
1911         typename UTF8,
1912         typename UTF16,
1913         typename UTF32,
1914         int Bytes = sizeof(typename Cont::value_type)>
1915     struct from_utf8_dispatch
1916     {
1917         using type = UTF8;
1918     };
1919
1920     template<typename Cont, typename UTF8, typename UTF16, typename UTF32>
1921     struct from_utf8_dispatch<Cont, UTF8, UTF16, UTF32, 2>
1922     {
1923         using type = UTF16;
1924     };
1925
1926     template<typename Cont, typename UTF8, typename UTF16, typename UTF32>
1927     struct from_utf8_dispatch<Cont, UTF8, UTF16, UTF32, 4>
1928     {
1929         using type = UTF32;
1930     };
1931
1932     template<typename Cont, typename UTF8, typename UTF16, typename UTF32>
1933     using from_utf8_dispatch_t =
1934         typename from_utf8_dispatch<Cont, UTF8, UTF16, UTF32>::type;
1935
1936     template<
1937         typename Cont,
1938         typename UTF8,
1939         typename UTF16,
1940         typename UTF32,
1941         int Bytes = sizeof(typename Cont::value_type)>
1942     struct from_utf16_dispatch
1943     {
1944         using type = UTF16;
1945     };
1946
1947     template<typename Cont, typename UTF8, typename UTF16, typename UTF32>
1948     struct from_utf16_dispatch<Cont, UTF8, UTF16, UTF32, 1>
1949     {
1950         using type = UTF8;
1951     };
1952
1953     template<typename Cont, typename UTF8, typename UTF16, typename UTF32>
1954     struct from_utf16_dispatch<Cont, UTF8, UTF16, UTF32, 4>
1955     {
1956         using type = UTF32;
1957     };
1958
1959     template<typename Cont, typename UTF8, typename UTF16, typename UTF32>
1960     using from_utf16_dispatch_t =
1961         typename from_utf16_dispatch<Cont, UTF8, UTF16, UTF32>::type;
1962
1963     template<
1964         typename Cont,
1965         typename UTF8,
1966         typename UTF16,
1967         typename UTF32,
1968         int Bytes = sizeof(typename Cont::value_type)>
1969     struct from_utf32_dispatch
1970     {
1971         using type = UTF32;
1972     };
1973
1974     template<typename Cont, typename UTF8, typename UTF16, typename UTF32>
1975     struct from_utf32_dispatch<Cont, UTF8, UTF16, UTF32, 1>
1976     {
1977         using type = UTF8;
1978     };
1979
1980     template<typename Cont, typename UTF8, typename UTF16, typename UTF32>
1981     struct from_utf32_dispatch<Cont, UTF8, UTF16, UTF32, 2>
1982     {
1983         using type = UTF16;
1984     };
1985
1986     template<typename Cont, typename UTF8, typename UTF16, typename UTF32>
1987     using from_utf32_dispatch_t =
1988         typename from_utf32_dispatch<Cont, UTF8, UTF16, UTF32>::type;
1989
1990 }}}
1991
1992 namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAMESPACE_V1 {
1993
1994 #if defined(BOOST_TEXT_DOXYGEN)
1995
1996     /** Returns a `utf_32_to_8_out_iterator<O>` constructed from the given
1997         iterator. */
1998     template<std::output_iterator<char8_t> O>
1999     utf_32_to_8_out_iterator<O> utf_32_to_8_out(O it);
2000
2001     /** Returns a `utf_8_to_32_out_iterator<O>` constructed from the given
2002         iterator. */
2003     template<std::output_iterator<char32_t> O>
2004     utf_8_to_32_out_iterator<O> utf_8_to_32_out(O it);
2005
2006     /** Returns a `utf_32_to_16_out_iterator<O>` constructed from the given
2007         iterator. */
2008     template<std::output_iterator<char16_t> O>
2009     utf_32_to_16_out_iterator<O> utf_32_to_16_out(O it);
2010
2011     /** Returns a `utf_16_to_32_out_iterator<O>` constructed from the given
2012         iterator. */
2013     template<std::output_iterator<char32_t> O>
2014     utf_16_to_32_out_iterator<O> utf_16_to_32_out(O it);
2015
2016     /** Returns a `utf_16_to_8_out_iterator<O>` constructed from the given
2017         iterator. */
2018     template<std::output_iterator<char8_t> O>
2019     utf_16_to_8_out_iterator<O> utf_16_to_8_out(O it);
2020
2021     /** Returns a `utf_8_to_16_out_iterator<O>` constructed from the given
2022         iterator. */
2023     template<std::output_iterator<char16_t> O>
2024     utf_8_to_16_out_iterator<O> utf_8_to_16_out(O it);
2025
2026     /** Returns an iterator equivalent to `it` that transcodes `[first, last)`
2027         to UTF-8. */
2028     template<std::input_iterator I, std::sentinel_for<I> S>
2029     auto utf8_iterator(I first, I it, S last);
2030
2031     /** Returns an iterator equivalent to `it` that transcodes `[first, last)`
2032         to UTF-16. */
2033     template<std::input_iterator I, std::sentinel_for<I> S>
2034     auto utf16_iterator(I first, I it, S last);
2035
2036     /** Returns an iterator equivalent to `it` that transcodes `[first, last)`
2037         to UTF-32. */
2038     template<std::input_iterator I, std::sentinel_for<I> S>
2039     auto utf32_iterator(I first, I it, S last);
2040
2041     /** Returns a inserting iterator that transcodes from UTF-8 to UTF-8,
2042         UTF-16, or UTF-32.  Which UTF the iterator transcodes to depends on
2043         `sizeof(Cont::value_type)`: `1` implies UTF-8; `2` implies UTF-16; and
2044         any other size implies UTF-32. */
2045     template<typename Cont>
2046     requires requires { typename Cont::value_type; } &&
2047              std::is_integral_v<typename Cont::value_type>
2048     auto from_utf8_inserter(Cont & c, typename Cont::iterator it);
2049
2050     /** Returns a inserting iterator that transcodes from UTF-16 to UTF-8,
2051         UTF-16, or UTF-32.  Which UTF the iterator transcodes to depends on
2052         `sizeof(Cont::value_type)`: `1` implies UTF-8; `2` implies UTF-16; and
2053         any other size implies UTF-32. */
2054     template<typename Cont>
2055     requires requires { typename Cont::value_type; } &&
2056              std::is_integral_v<typename Cont::value_type>
2057     auto from_utf16_inserter(Cont & c, typename Cont::iterator it);
2058
2059     /** Returns a inserting iterator that transcodes from UTF-32 to UTF-8,
2060         UTF-16, or UTF-32.  Which UTF the iterator transcodes to depends on
2061         `sizeof(Cont::value_type)`: `1` implies UTF-8; `2` implies UTF-16; and
2062         any other size implies UTF-32. */
2063     template<typename Cont>
2064     requires requires { typename Cont::value_type; } &&
2065              std::is_integral_v<typename Cont::value_type>
2066     auto from_utf32_inserter(Cont & c, typename Cont::iterator it);
2067
2068     /** Returns a back-inserting iterator that transcodes from UTF-8 to UTF-8,
2069         UTF-16, or UTF-32.  Which UTF the iterator transcodes to depends on
2070         `sizeof(Cont::value_type)`: `1` implies UTF-8; `2` implies UTF-16; and
2071         any other size implies UTF-32. */
2072     template<typename Cont>
2073     requires requires { typename Cont::value_type; } &&
2074              std::is_integral_v<typename Cont::value_type>
2075     auto from_utf8_back_inserter(Cont & c);
2076
2077     /** Returns a back-inserting iterator that transcodes from UTF-16 to UTF-8,
2078         UTF-16, or UTF-32.  Which UTF the iterator transcodes to depends on
2079         `sizeof(Cont::value_type)`: `1` implies UTF-8; `2` implies UTF-16; and
2080         any other size implies UTF-32. */
2081     template<typename Cont>
2082     requires requires { typename Cont::value_type; } &&
2083              std::is_integral_v<typename Cont::value_type>
2084     auto from_utf16_back_inserter(Cont & c);
2085
2086     /** Returns a back-inserting iterator that transcodes from UTF-32 to UTF-8,
2087         UTF-16, or UTF-32.  Which UTF the iterator transcodes to depends on
2088         `sizeof(Cont::value_type)`: `1` implies UTF-8; `2` implies UTF-16; and
2089         any other size implies UTF-32. */
2090     template<typename Cont>
2091     requires requires { typename Cont::value_type; } &&
2092              std::is_integral_v<typename Cont::value_type>
2093     auto from_utf32_back_inserter(Cont & c);
2094
2095     /** Returns a front-inserting iterator that transcodes from UTF-8 to UTF-8,
2096         UTF-16, or UTF-32.  Which UTF the iterator transcodes to depends on
2097         `sizeof(Cont::value_type)`: `1` implies UTF-8; `2` implies UTF-16; and
2098         any other size implies UTF-32. */
2099     template<typename Cont>
2100     requires requires { typename Cont::value_type; } &&
2101              std::is_integral_v<typename Cont::value_type>
2102     auto from_utf8_front_inserter(Cont & c);
2103
2104     /** Returns a front-inserting iterator that transcodes from UTF-16 to UTF-8,
2105         UTF-16, or UTF-32.  Which UTF the iterator transcodes to depends on
2106         `sizeof(Cont::value_type)`: `1` implies UTF-8; `2` implies UTF-16; and
2107         any other size implies UTF-32. */
2108     template<typename Cont>
2109     requires requires { typename Cont::value_type; } &&
2110              std::is_integral_v<typename Cont::value_type>
2111     auto from_utf16_front_inserter(Cont & c);
2112
2113     /** Returns a front-inserting iterator that transcodes from UTF-32 to UTF-8,
2114         UTF-16, or UTF-32.  Which UTF the iterator transcodes to depends on
2115         `sizeof(Cont::value_type)`: `1` implies UTF-8; `2` implies UTF-16; and
2116         any other size implies UTF-32. */
2117     template<typename Cont>
2118     requires requires { typename Cont::value_type; } &&
2119              std::is_integral_v<typename Cont::value_type>
2120     auto from_utf32_front_inserter(Cont & c);
2121
2122 #endif
2123
2124     template<typename Iter>
2125     utf_32_to_8_out_iterator<Iter> utf_32_to_8_out(Iter it)
2126     {
2127         return utf_32_to_8_out_iterator<Iter>(it);
2128     }
2129
2130     template<typename Iter>
2131     utf_8_to_32_out_iterator<Iter> utf_8_to_32_out(Iter it)
2132     {
2133         return utf_8_to_32_out_iterator<Iter>(it);
2134     }
2135
2136     template<typename Iter>
2137     utf_32_to_16_out_iterator<Iter> utf_32_to_16_out(Iter it)
2138     {
2139         return utf_32_to_16_out_iterator<Iter>(it);
2140     }
2141
2142     template<typename Iter>
2143     utf_16_to_32_out_iterator<Iter> utf_16_to_32_out(Iter it)
2144     {
2145         return utf_16_to_32_out_iterator<Iter>(it);
2146     }
2147
2148     template<typename Iter>
2149     utf_16_to_8_out_iterator<Iter> utf_16_to_8_out(Iter it)
2150     {
2151         return utf_16_to_8_out_iterator<Iter>(it);
2152     }
2153
2154     template<typename Iter>
2155     utf_8_to_16_out_iterator<Iter> utf_8_to_16_out(Iter it)
2156     {
2157         return utf_8_to_16_out_iterator<Iter>(it);
2158     }
2159
2160     template<typename Iter, typename Sentinel>
2161     auto utf8_iterator(Iter first, Iter it, Sentinel last)
2162     {
2163         auto const unpacked = text::unpack_iterator_and_sentinel(first, last);
2164         auto const unpacked_it =
2165             text::unpack_iterator_and_sentinel(it, last).first;
2166         constexpr format tag = unpacked.format_tag;
2167         return detail::make_utf8_dispatch<tag>::call(
2168             unpacked.first, unpacked_it, unpacked.last);
2169     }
2170
2171     template<typename Iter, typename Sentinel>
2172     auto utf16_iterator(Iter first, Iter it, Sentinel last)
2173     {
2174         auto const unpacked = text::unpack_iterator_and_sentinel(first, last);
2175         auto const unpacked_it =
2176             text::unpack_iterator_and_sentinel(it, last).first;
2177         constexpr format tag = unpacked.format_tag;
2178         return detail::make_utf16_dispatch<tag>::call(
2179             unpacked.first, unpacked_it, unpacked.last);
2180     }
2181
2182     template<typename Iter, typename Sentinel>
2183     auto utf32_iterator(Iter first, Iter it, Sentinel last)
2184     {
2185         auto const unpacked = text::unpack_iterator_and_sentinel(first, last);
2186         auto const unpacked_it =
2187             text::unpack_iterator_and_sentinel(it, last).first;
2188         constexpr format tag = unpacked.format_tag;
2189         return detail::make_utf32_dispatch<tag>::call(
2190             unpacked.first, unpacked_it, unpacked.last);
2191     }
2192
2193     template<typename Cont>
2194     auto from_utf8_inserter(Cont & c, typename Cont::iterator it)
2195     {
2196         using result_type = detail::from_utf8_dispatch_t<
2197             Cont,
2198             std::insert_iterator<Cont>,
2199             utf_8_to_16_insert_iterator<Cont>,
2200             utf_8_to_32_insert_iterator<Cont>>;
2201         return result_type(c, it);
2202     }
2203
2204     template<typename Cont>
2205     auto from_utf16_inserter(Cont & c, typename Cont::iterator it)
2206     {
2207         using result_type = detail::from_utf16_dispatch_t<
2208             Cont,
2209             utf_16_to_8_insert_iterator<Cont>,
2210             std::insert_iterator<Cont>,
2211             utf_16_to_32_insert_iterator<Cont>>;
2212         return result_type(c, it);
2213     }
2214
2215     template<typename Cont>
2216     auto from_utf32_inserter(Cont & c, typename Cont::iterator it)
2217     {
2218         using result_type = detail::from_utf32_dispatch_t<
2219             Cont,
2220             utf_32_to_8_insert_iterator<Cont>,
2221             utf_32_to_16_insert_iterator<Cont>,
2222             std::insert_iterator<Cont>>;
2223         return result_type(c, it);
2224     }
2225
2226     template<typename Cont>
2227     auto from_utf8_back_inserter(Cont & c)
2228     {
2229         using result_type = detail::from_utf8_dispatch_t<
2230             Cont,
2231             std::back_insert_iterator<Cont>,
2232             utf_8_to_16_back_insert_iterator<Cont>,
2233             utf_8_to_32_back_insert_iterator<Cont>>;
2234         return result_type(c);
2235     }
2236
2237     template<typename Cont>
2238     auto from_utf16_back_inserter(Cont & c)
2239     {
2240         using result_type = detail::from_utf16_dispatch_t<
2241             Cont,
2242             utf_16_to_8_back_insert_iterator<Cont>,
2243             std::back_insert_iterator<Cont>,
2244             utf_16_to_32_back_insert_iterator<Cont>>;
2245         return result_type(c);
2246     }
2247
2248     template<typename Cont>
2249     auto from_utf32_back_inserter(Cont & c)
2250     {
2251         using result_type = detail::from_utf32_dispatch_t<
2252             Cont,
2253             utf_32_to_8_back_insert_iterator<Cont>,
2254             utf_32_to_16_back_insert_iterator<Cont>,
2255             std::back_insert_iterator<Cont>>;
2256         return result_type(c);
2257     }
2258
2259     template<typename Cont>
2260     auto from_utf8_front_inserter(Cont & c)
2261     {
2262         using result_type = detail::from_utf8_dispatch_t<
2263             Cont,
2264             std::front_insert_iterator<Cont>,
2265             utf_8_to_16_front_insert_iterator<Cont>,
2266             utf_8_to_32_front_insert_iterator<Cont>>;
2267         return result_type(c);
2268     }
2269
2270     template<typename Cont>
2271     auto from_utf16_front_inserter(Cont & c)
2272     {
2273         using result_type = detail::from_utf16_dispatch_t<
2274             Cont,
2275             utf_16_to_8_front_insert_iterator<Cont>,
2276             std::front_insert_iterator<Cont>,
2277             utf_16_to_32_front_insert_iterator<Cont>>;
2278         return result_type(c);
2279     }
2280
2281     template<typename Cont>
2282     auto from_utf32_front_inserter(Cont & c)
2283     {
2284         using result_type = detail::from_utf32_dispatch_t<
2285             Cont,
2286             utf_32_to_8_front_insert_iterator<Cont>,
2287             utf_32_to_16_front_insert_iterator<Cont>,
2288             std::front_insert_iterator<Cont>>;
2289         return result_type(c);
2290     }
2291
2292 }}}
2293
2294 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2295
2296 namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAMESPACE_V2 {
2297
2298     template<std::output_iterator<char8_t> O>
2299     constexpr utf_32_to_8_out_iterator<O> utf_32_to_8_out(O it)
2300     {
2301         return utf_32_to_8_out_iterator<O>(it);
2302     }
2303
2304     template<std::output_iterator<char32_t> O>
2305     constexpr utf_8_to_32_out_iterator<O> utf_8_to_32_out(O it)
2306     {
2307         return utf_8_to_32_out_iterator<O>(it);
2308     }
2309
2310     template<std::output_iterator<char16_t> O>
2311     constexpr utf_32_to_16_out_iterator<O> utf_32_to_16_out(O it)
2312     {
2313         return utf_32_to_16_out_iterator<O>(it);
2314     }
2315
2316     template<std::output_iterator<char32_t> O>
2317     constexpr utf_16_to_32_out_iterator<O> utf_16_to_32_out(O it)
2318     {
2319         return utf_16_to_32_out_iterator<O>(it);
2320     }
2321
2322     template<std::output_iterator<char8_t> O>
2323     constexpr utf_16_to_8_out_iterator<O> utf_16_to_8_out(O it)
2324     {
2325         return utf_16_to_8_out_iterator<O>(it);
2326     }
2327
2328     template<std::output_iterator<char16_t> O>
2329     constexpr utf_8_to_16_out_iterator<O> utf_8_to_16_out(O it)
2330     {
2331         return utf_8_to_16_out_iterator<O>(it);
2332     }
2333
2334     template<std::input_iterator I, std::sentinel_for<I> S>
2335     constexpr auto utf8_iterator(I first, I it, S last)
2336     {
2337         return v1::utf8_iterator(first, it, last);
2338     }
2339
2340     template<std::input_iterator I, std::sentinel_for<I> S>
2341     constexpr auto utf16_iterator(I first, I it, S last)
2342     {
2343         return v1::utf16_iterator(first, it, last);
2344     }
2345
2346     template<std::input_iterator I, std::sentinel_for<I> S>
2347     constexpr auto utf32_iterator(I first, I it, S last)
2348     {
2349         return v1::utf32_iterator(first, it, last);
2350     }
2351
2352     template<typename Cont>
2353         requires requires { typename Cont::value_type; } &&
2354                  utf_code_unit<typename Cont::value_type>
2355     constexpr auto from_utf8_inserter(Cont & c, typename Cont::iterator it)
2356     {
2357         if constexpr (sizeof(typename Cont::value_type) == 1) {
2358             return std::insert_iterator<Cont>(c, it);
2359         } else if constexpr (sizeof(typename Cont::value_type) == 2) {
2360             return utf_8_to_16_insert_iterator<Cont>(c, it);
2361         } else {
2362             return utf_8_to_32_insert_iterator<Cont>(c, it);
2363         }
2364     }
2365
2366     template<typename Cont>
2367         requires requires { typename Cont::value_type; } &&
2368                  utf_code_unit<typename Cont::value_type>
2369     constexpr auto from_utf16_inserter(Cont & c, typename Cont::iterator it)
2370     {
2371         if constexpr (sizeof(typename Cont::value_type) == 1) {
2372             return utf_16_to_8_insert_iterator<Cont>(c, it);
2373         } else if constexpr (sizeof(typename Cont::value_type) == 2) {
2374             return std::insert_iterator<Cont>(c, it);
2375         } else {
2376             return utf_16_to_32_insert_iterator<Cont>(c, it);
2377         }
2378     }
2379
2380     template<typename Cont>
2381         requires requires { typename Cont::value_type; } &&
2382                  utf_code_unit<typename Cont::value_type>
2383     constexpr auto from_utf32_inserter(Cont & c, typename Cont::iterator it)
2384     {
2385         if constexpr (sizeof(typename Cont::value_type) == 1) {
2386             return utf_32_to_8_insert_iterator<Cont>(c, it);
2387         } else if constexpr (sizeof(typename Cont::value_type) == 2) {
2388             return utf_32_to_16_insert_iterator<Cont>(c, it);
2389         } else {
2390             return std::insert_iterator<Cont>(c, it);
2391         }
2392     }
2393
2394     template<typename Cont>
2395         requires requires { typename Cont::value_type; } &&
2396                  utf_code_unit<typename Cont::value_type>
2397     constexpr auto from_utf8_back_inserter(Cont & c)
2398     {
2399         if constexpr (sizeof(typename Cont::value_type) == 1) {
2400             return std::back_insert_iterator<Cont>(c);
2401         } else if constexpr (sizeof(typename Cont::value_type) == 2) {
2402             return utf_8_to_16_back_insert_iterator<Cont>(c);
2403         } else {
2404             return utf_8_to_32_back_insert_iterator<Cont>(c);
2405         }
2406     }
2407
2408     template<typename Cont>
2409         requires requires { typename Cont::value_type; } &&
2410                  utf_code_unit<typename Cont::value_type>
2411     constexpr auto from_utf16_back_inserter(Cont & c)
2412     {
2413         if constexpr (sizeof(typename Cont::value_type) == 1) {
2414             return utf_16_to_8_back_insert_iterator<Cont>(c);
2415         } else if constexpr (sizeof(typename Cont::value_type) == 2) {
2416             return std::back_insert_iterator<Cont>(c);
2417         } else {
2418             return utf_16_to_32_back_insert_iterator<Cont>(c);
2419         }
2420     }
2421
2422     template<typename Cont>
2423         requires requires { typename Cont::value_type; } &&
2424                  utf_code_unit<typename Cont::value_type>
2425     constexpr auto from_utf32_back_inserter(Cont & c)
2426     {
2427         if constexpr (sizeof(typename Cont::value_type) == 1) {
2428             return utf_32_to_8_back_insert_iterator<Cont>(c);
2429         } else if constexpr (sizeof(typename Cont::value_type) == 2) {
2430             return utf_32_to_16_back_insert_iterator<Cont>(c);
2431         } else {
2432             return std::back_insert_iterator<Cont>(c);
2433         }
2434     }
2435
2436     template<typename Cont>
2437         requires requires { typename Cont::value_type; } &&
2438                  utf_code_unit<typename Cont::value_type>
2439     constexpr auto from_utf8_front_inserter(Cont & c)
2440     {
2441         if constexpr (sizeof(typename Cont::value_type) == 1) {
2442             return std::front_insert_iterator<Cont>(c);
2443         } else if constexpr (sizeof(typename Cont::value_type) == 2) {
2444             return utf_8_to_16_front_insert_iterator<Cont>(c);
2445         } else {
2446             return utf_8_to_32_front_insert_iterator<Cont>(c);
2447         }
2448     }
2449
2450     template<typename Cont>
2451         requires requires { typename Cont::value_type; } &&
2452                  utf_code_unit<typename Cont::value_type>
2453     constexpr auto from_utf16_front_inserter(Cont & c)
2454     {
2455         if constexpr (sizeof(typename Cont::value_type) == 1) {
2456             return utf_16_to_8_front_insert_iterator<Cont>(c);
2457         } else if constexpr (sizeof(typename Cont::value_type) == 2) {
2458             return std::front_insert_iterator<Cont>(c);
2459         } else {
2460             return utf_16_to_32_front_insert_iterator<Cont>(c);
2461         }
2462     }
2463
2464     template<typename Cont>
2465         requires requires { typename Cont::value_type; } &&
2466                  utf_code_unit<typename Cont::value_type>
2467     constexpr auto from_utf32_front_inserter(Cont & c)
2468     {
2469         if constexpr (sizeof(typename Cont::value_type) == 1) {
2470             return utf_32_to_8_front_insert_iterator<Cont>(c);
2471         } else if constexpr (sizeof(typename Cont::value_type) == 2) {
2472             return utf_32_to_16_front_insert_iterator<Cont>(c);
2473         } else {
2474             return std::front_insert_iterator<Cont>(c);
2475         }
2476     }
2477 }}}
2478
2479 #endif
2480
2481 namespace boost::parser::detail { namespace text {
2482     namespace detail {
2483         template<format Format>
2484         constexpr auto format_to_type()
2485         {
2486             if constexpr (Format == format::utf8) {
2487                 return char8_type{};
2488             } else if constexpr (Format == format::utf16) {
2489                 return char16_t{};
2490             } else {
2491                 return char32_t{};
2492             }
2493         }
2494
2495         template<typename I>
2496         constexpr bool is_bidi =
2497 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2498             std::bidirectional_iterator<I>
2499 #else
2500             std::is_base_of_v<
2501                 std::bidirectional_iterator_tag,
2502                 typename std::iterator_traits<I>::iterator_category>
2503 #endif
2504             ;
2505
2506         template<typename I, bool SupportReverse = is_bidi<I>>
2507         struct first_and_curr
2508         {
2509             first_and_curr() = default;
2510             first_and_curr(I curr) : curr{curr} {}
2511             first_and_curr(const first_and_curr & other) = default;
2512             template<
2513                 class I2
2514 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2515                 ,
2516                 typename Enable = std::enable_if_t<std::is_convertible_v<I2, I>>
2517 #endif
2518                 >
2519 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2520             requires std::convertible_to<I2, I>
2521 #endif
2522             first_and_curr(const first_and_curr<I2> & other) : curr{other.curr}
2523             {}
2524
2525             I curr;
2526         };
2527         template<typename I>
2528         struct first_and_curr<I, true>
2529         {
2530             first_and_curr() = default;
2531             first_and_curr(I first, I curr) : first{first}, curr{curr} {}
2532             first_and_curr(const first_and_curr & other) = default;
2533             template<
2534                 class I2
2535 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2536                 ,
2537                 typename Enable = std::enable_if_t<std::is_convertible_v<I2, I>>
2538 #endif
2539                 >
2540 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2541             requires std::convertible_to<I2, I>
2542 #endif
2543             first_and_curr(const first_and_curr<I2> & other) :
2544                 first{other.first}, curr{other.curr}
2545             {}
2546
2547             I first;
2548             I curr;
2549         };
2550     }
2551
2552     namespace detail {
2553         struct iter_access
2554         {
2555             template<typename T>
2556             static auto & buf(T & it)
2557             {
2558                 return it.buf_;
2559             }
2560             template<typename T>
2561             static auto & first_and_curr(T & it)
2562             {
2563                 return it.first_and_curr_;
2564             }
2565             template<typename T>
2566             static auto & buf_index(T & it)
2567             {
2568                 return it.buf_index_;
2569             }
2570             template<typename T>
2571             static auto & buf_last(T & it)
2572             {
2573                 return it.buf_last_;
2574             }
2575             template<typename T>
2576             static auto & to_increment(T & it)
2577             {
2578                 return it.to_increment_;
2579             }
2580             template<typename T>
2581             static auto & last(T & it)
2582             {
2583                 return it.last_;
2584             }
2585         };
2586     }
2587
2588 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2589     template<
2590         format FromFormat,
2591         format ToFormat,
2592         std::input_iterator I,
2593         std::sentinel_for<I> S,
2594         transcoding_error_handler ErrorHandler>
2595         requires std::convertible_to<std::iter_value_t<I>, detail::format_to_type_t<FromFormat>>
2596 #else
2597     template<
2598         format FromFormat,
2599         format ToFormat,
2600         typename I,
2601         typename S,
2602         typename ErrorHandler>
2603 #endif
2604     class utf_iterator
2605         : public stl_interfaces::iterator_interface<
2606               utf_iterator<FromFormat, ToFormat, I, S, ErrorHandler>,
2607               detail::bidirectional_at_most_t<I>,
2608               detail::format_to_type_t<ToFormat>,
2609               detail::format_to_type_t<ToFormat>>
2610     {
2611         static_assert(
2612             FromFormat == format::utf8 || FromFormat == format::utf16 ||
2613             FromFormat == format::utf32);
2614         static_assert(
2615             ToFormat == format::utf8 || ToFormat == format::utf16 ||
2616             ToFormat == format::utf32);
2617
2618 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2619         template<typename T>
2620         constexpr static bool is_bidirectional = std::is_base_of_v<
2621             std::bidirectional_iterator_tag,
2622             detail::bidirectional_at_most_t<T>>;
2623         template<typename T>
2624         constexpr static bool is_forward = std::is_base_of_v<
2625             std::forward_iterator_tag,
2626             detail::bidirectional_at_most_t<T>>;
2627         template<typename T>
2628         constexpr static bool is_input = !is_bidirectional<T> && !is_forward<T>;
2629 #endif
2630
2631         static_assert(
2632 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2633             std::forward_iterator<I>
2634 #else
2635             is_forward<I>
2636 #endif
2637             || noexcept(ErrorHandler{}("")));
2638
2639     public:
2640         using value_type = detail::format_to_type_t<ToFormat>;
2641
2642         constexpr utf_iterator() = default;
2643
2644 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2645         template<
2646             typename J = I,
2647             typename Enable = std::enable_if_t<is_bidirectional<J>>>
2648 #endif
2649         constexpr utf_iterator(I first, I it, S last)
2650 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2651             requires std::bidirectional_iterator<I>
2652 #endif
2653             : first_and_curr_{first, it}, last_(last)
2654         {
2655             if (curr() != last_)
2656                 read();
2657         }
2658 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2659         template<
2660             typename J = I,
2661             typename Enable = std::enable_if_t<!is_bidirectional<J>>>
2662 #endif
2663         constexpr utf_iterator(I it, S last)
2664 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2665             requires(!std::bidirectional_iterator<I>)
2666 #endif
2667             :
2668             first_and_curr_{it}, last_(last)
2669         {
2670             if (curr() != last_)
2671                 read();
2672         }
2673
2674         template<
2675             class I2,
2676             class S2
2677 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2678             ,
2679             typename Enable = std::enable_if_t<
2680                 std::is_convertible_v<I2, I> && std::is_convertible_v<S2, S>>
2681 #endif
2682             >
2683 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2684         requires std::convertible_to<I2, I> && std::convertible_to<S2, S>
2685 #endif
2686         constexpr utf_iterator(
2687             utf_iterator<FromFormat, ToFormat, I2, S2, ErrorHandler> const &
2688                 other) :
2689             buf_(detail::iter_access::buf(other)),
2690             first_and_curr_(detail::iter_access::first_and_curr(other)),
2691             buf_index_(detail::iter_access::buf_index(other)),
2692             buf_last_(detail::iter_access::buf_last(other)),
2693             to_increment_(detail::iter_access::to_increment(other)),
2694             last_(detail::iter_access::last(other))
2695         {}
2696
2697 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2698         template<
2699             typename J = I,
2700             typename Enable = std::enable_if_t<is_bidirectional<J>>>
2701 #endif
2702         constexpr I begin() const
2703 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2704             requires std::bidirectional_iterator<I>
2705 #endif
2706         {
2707             return first();
2708         }
2709         constexpr S end() const { return last_; }
2710
2711 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2712         template<
2713             typename J = I,
2714             typename Enable = std::enable_if_t<is_forward<J>>>
2715 #endif
2716         constexpr I base() const
2717 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2718             requires std::forward_iterator<I>
2719 #endif
2720         {
2721             return curr();
2722         }
2723
2724         constexpr value_type operator*() const
2725         {
2726             BOOST_PARSER_DEBUG_ASSERT(buf_index_ < buf_last_);
2727             return buf_[buf_index_];
2728         }
2729
2730         constexpr utf_iterator & operator++()
2731         {
2732             BOOST_PARSER_DEBUG_ASSERT(buf_index_ != buf_last_ || curr() != last_);
2733             if (buf_index_ + 1 == buf_last_ && curr() != last_) {
2734                 if constexpr (
2735 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2736                     std::forward_iterator<I>
2737 #else
2738                     is_forward<I>
2739 #endif
2740                 ) {
2741                     std::advance(curr(), to_increment_);
2742                 }
2743                 if (curr() == last_)
2744                     buf_index_ = 0;
2745                 else
2746                     read();
2747             } else if (buf_index_ + 1 <= buf_last_) {
2748                 ++buf_index_;
2749             }
2750             return *this;
2751         }
2752
2753 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2754         template<
2755             typename J = I,
2756             typename Enable = std::enable_if_t<is_bidirectional<J>>>
2757 #endif
2758         constexpr utf_iterator & operator--()
2759 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2760             requires std::bidirectional_iterator<I>
2761 #endif
2762         {
2763             BOOST_PARSER_DEBUG_ASSERT(buf_index_ || curr() != first());
2764             if (!buf_index_ && curr() != first())
2765                 read_reverse();
2766             else if (buf_index_)
2767                 --buf_index_;
2768             return *this;
2769         }
2770
2771         friend constexpr bool operator==(
2772 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2773             utf_iterator
2774 #else
2775             std::enable_if_t<is_forward<I>, utf_iterator>
2776 #endif
2777             lhs, utf_iterator rhs)
2778 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2779             requires std::forward_iterator<I> || requires(I i) { i == i; }
2780 #endif
2781         {
2782             if constexpr (
2783 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2784                 std::forward_iterator<I>
2785 #else
2786                 is_forward<I>
2787 #endif
2788             ) {
2789                 return lhs.curr() == rhs.curr() && lhs.buf_index_ == rhs.buf_index_;
2790             } else {
2791                 if (lhs.curr() != rhs.curr())
2792                     return false;
2793
2794                 if (lhs.buf_index_ == rhs.buf_index_ &&
2795                     lhs.buf_last_ == rhs.buf_last_) {
2796                     return true;
2797                 }
2798
2799                 return lhs.buf_index_ == lhs.buf_last_ &&
2800                        rhs.buf_index_ == rhs.buf_last_;
2801             }
2802         }
2803
2804 #if !defined(__cpp_impl_three_way_comparison)
2805         friend BOOST_PARSER_CONSTEXPR bool operator!=(
2806             std::enable_if_t<is_forward<I>, utf_iterator> lhs, utf_iterator rhs)
2807         { return !(lhs == rhs); }
2808 #endif
2809
2810         friend constexpr bool operator==(utf_iterator lhs, S rhs)
2811         {
2812             if constexpr (
2813 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
2814                 std::forward_iterator<I>
2815 #else
2816                 is_forward<I>
2817 #endif
2818             ) {
2819                 return lhs.curr() == rhs;
2820             } else {
2821                 return lhs.curr() == rhs && lhs.buf_index_ == lhs.buf_last_;
2822             }
2823         }
2824
2825 #if !defined(__cpp_impl_three_way_comparison)
2826         friend BOOST_PARSER_CONSTEXPR bool operator!=(utf_iterator lhs, S rhs)
2827         { return !(lhs == rhs); }
2828 #endif
2829
2830         // exposition only
2831         using base_type = stl_interfaces::iterator_interface<
2832             utf_iterator<FromFormat, ToFormat, I, S, ErrorHandler>,
2833             detail::bidirectional_at_most_t<I>,
2834             value_type,
2835             value_type>;
2836         using base_type::operator++;
2837         using base_type::operator--;
2838
2839     private:
2840         constexpr char32_t decode_code_point()
2841         {
2842             if constexpr (FromFormat == format::utf8) {
2843                 char32_t cp = *curr();
2844                 ++curr();
2845                 to_increment_ = 1;
2846                 if (cp < 0x80)
2847                     return cp;
2848
2849                 // clang-format off
2850
2851                 // It turns out that this naive implementation is faster than
2852                 // the table implementation for the converting iterators.
2853
2854             /*
2855                 Unicode 3.9/D92
2856                 Table 3-7. Well-Formed UTF-8 Byte Sequences
2857
2858                 Code Points        First Byte Second Byte Third Byte Fourth Byte
2859                 ===========        ========== =========== ========== ===========
2860                 U+0000..U+007F     00..7F
2861                 U+0080..U+07FF     C2..DF     80..BF
2862                 U+0800..U+0FFF     E0         A0..BF      80..BF
2863                 U+1000..U+CFFF     E1..EC     80..BF      80..BF
2864                 U+D000..U+D7FF     ED         80..9F      80..BF
2865                 U+E000..U+FFFF     EE..EF     80..BF      80..BF
2866                 U+10000..U+3FFFF   F0         90..BF      80..BF     80..BF
2867                 U+40000..U+FFFFF   F1..F3     80..BF      80..BF     80..BF
2868                 U+100000..U+10FFFF F4         80..8F      80..BF     80..BF
2869             */
2870                 // clang-format on
2871
2872                 unsigned char curr_c = (unsigned char)cp;
2873
2874                 auto error = [&]() {
2875                     return ErrorHandler{}("Ill-formed UTF-8.");
2876                 };
2877                 auto next = [&]() {
2878                     ++curr();
2879                     ++to_increment_;
2880                 };
2881
2882                 // One-byte case handled above
2883
2884                 // Two-byte
2885                 if (detail::in(0xc2, curr_c, 0xdf)) {
2886                     cp = curr_c & 0b00011111;
2887                     if (curr() == last_)
2888                         return error();
2889                     curr_c = *curr();
2890                     if (!detail::in(0x80, curr_c, 0xbf))
2891                         return error();
2892                     cp = (cp << 6) + (curr_c & 0b00111111);
2893                     next();
2894                     // Three-byte
2895                 } else if (curr_c == 0xe0) {
2896                     cp = curr_c & 0b00001111;
2897                     if (curr() == last_)
2898                         return error();
2899                     curr_c = *curr();
2900                     if (!detail::in(0xa0, curr_c, 0xbf))
2901                         return error();
2902                     cp = (cp << 6) + (curr_c & 0b00111111);
2903                     next();
2904                     if (curr() == last_)
2905                         return error();
2906                     curr_c = *curr();
2907                     if (!detail::in(0x80, curr_c, 0xbf))
2908                         return error();
2909                     cp = (cp << 6) + (curr_c & 0b00111111);
2910                     next();
2911                 } else if (detail::in(0xe1, curr_c, 0xec)) {
2912                     cp = curr_c & 0b00001111;
2913                     if (curr() == last_)
2914                         return error();
2915                     curr_c = *curr();
2916                     if (!detail::in(0x80, curr_c, 0xbf))
2917                         return error();
2918                     cp = (cp << 6) + (curr_c & 0b00111111);
2919                     next();
2920                     if (curr() == last_)
2921                         return error();
2922                     curr_c = *curr();
2923                     if (!detail::in(0x80, curr_c, 0xbf))
2924                         return error();
2925                     cp = (cp << 6) + (curr_c & 0b00111111);
2926                     next();
2927                 } else if (curr_c == 0xed) {
2928                     cp = curr_c & 0b00001111;
2929                     if (curr() == last_)
2930                         return error();
2931                     curr_c = *curr();
2932                     if (!detail::in(0x80, curr_c, 0x9f))
2933                         return error();
2934                     cp = (cp << 6) + (curr_c & 0b00111111);
2935                     next();
2936                     if (curr() == last_)
2937                         return error();
2938                     curr_c = *curr();
2939                     if (!detail::in(0x80, curr_c, 0xbf))
2940                         return error();
2941                     cp = (cp << 6) + (curr_c & 0b00111111);
2942                     next();
2943                 } else if (detail::in(0xee, curr_c, 0xef)) {
2944                     cp = curr_c & 0b00001111;
2945                     if (curr() == last_)
2946                         return error();
2947                     curr_c = *curr();
2948                     if (!detail::in(0x80, curr_c, 0xbf))
2949                         return error();
2950                     cp = (cp << 6) + (curr_c & 0b00111111);
2951                     next();
2952                     if (curr() == last_)
2953                         return error();
2954                     curr_c = *curr();
2955                     if (!detail::in(0x80, curr_c, 0xbf))
2956                         return error();
2957                     cp = (cp << 6) + (curr_c & 0b00111111);
2958                     next();
2959                     // Four-byte
2960                 } else if (curr_c == 0xf0) {
2961                     cp = curr_c & 0b00000111;
2962                     if (curr() == last_)
2963                         return error();
2964                     curr_c = *curr();
2965                     if (!detail::in(0x90, curr_c, 0xbf))
2966                         return error();
2967                     cp = (cp << 6) + (curr_c & 0b00111111);
2968                     next();
2969                     if (curr() == last_)
2970                         return error();
2971                     curr_c = *curr();
2972                     if (!detail::in(0x80, curr_c, 0xbf))
2973                         return error();
2974                     cp = (cp << 6) + (curr_c & 0b00111111);
2975                     next();
2976                     if (curr() == last_)
2977                         return error();
2978                     curr_c = *curr();
2979                     if (!detail::in(0x80, curr_c, 0xbf))
2980                         return error();
2981                     cp = (cp << 6) + (curr_c & 0b00111111);
2982                     next();
2983                 } else if (detail::in(0xf1, curr_c, 0xf3)) {
2984                     cp = curr_c & 0b00000111;
2985                     if (curr() == last_)
2986                         return error();
2987                     curr_c = *curr();
2988                     if (!detail::in(0x80, curr_c, 0xbf))
2989                         return error();
2990                     cp = (cp << 6) + (curr_c & 0b00111111);
2991                     next();
2992                     if (curr() == last_)
2993                         return error();
2994                     curr_c = *curr();
2995                     if (!detail::in(0x80, curr_c, 0xbf))
2996                         return error();
2997                     cp = (cp << 6) + (curr_c & 0b00111111);
2998                     next();
2999                     if (curr() == last_)
3000                         return error();
3001                     curr_c = *curr();
3002                     if (!detail::in(0x80, curr_c, 0xbf))
3003                         return error();
3004                     cp = (cp << 6) + (curr_c & 0b00111111);
3005                     next();
3006                 } else if (curr_c == 0xf4) {
3007                     cp = curr_c & 0b00000111;
3008                     if (curr() == last_)
3009                         return error();
3010                     curr_c = *curr();
3011                     if (!detail::in(0x80, curr_c, 0x8f))
3012                         return error();
3013                     cp = (cp << 6) + (curr_c & 0b00111111);
3014                     next();
3015                     if (curr() == last_)
3016                         return error();
3017                     curr_c = *curr();
3018                     if (!detail::in(0x80, curr_c, 0xbf))
3019                         return error();
3020                     cp = (cp << 6) + (curr_c & 0b00111111);
3021                     next();
3022                     if (curr() == last_)
3023                         return error();
3024                     curr_c = *curr();
3025                     if (!detail::in(0x80, curr_c, 0xbf))
3026                         return error();
3027                     cp = (cp << 6) + (curr_c & 0b00111111);
3028                     next();
3029                 } else {
3030                     return error();
3031                 }
3032                 return cp;
3033             } else if constexpr (FromFormat == format::utf16) {
3034                 char16_t hi = *curr();
3035                 ++curr();
3036                 to_increment_ = 1;
3037                 if (!boost::parser::detail::text::surrogate(hi))
3038                     return hi;
3039
3040                 if (boost::parser::detail::text::low_surrogate(hi)) {
3041                     return ErrorHandler{}(
3042                         "Invalid UTF-16 sequence; lone trailing surrogate.");
3043                 }
3044
3045                 // high surrogate
3046                 if (curr() == last_) {
3047                     return ErrorHandler{}(
3048                         "Invalid UTF-16 sequence; lone leading surrogate.");
3049                 }
3050
3051                 char16_t lo = *curr();
3052                 ++curr();
3053                 ++to_increment_;
3054                 if (!boost::parser::detail::text::low_surrogate(lo)) {
3055                     return ErrorHandler{}(
3056                         "Invalid UTF-16 sequence; lone leading surrogate.");
3057                 }
3058
3059                 return char32_t((hi - high_surrogate_base) << 10) +
3060                        (lo - low_surrogate_base);
3061             } else {
3062                 char32_t retval = *curr();
3063                 ++curr();
3064                 to_increment_ = 1;
3065                 return retval;
3066             }
3067         }
3068
3069 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3070         template<
3071             typename J = I,
3072             typename Enable = std::enable_if_t<is_bidirectional<J>>>
3073 #endif
3074         constexpr char32_t decode_code_point_reverse()
3075 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3076             requires std::bidirectional_iterator<I>
3077 #endif
3078         {
3079             if constexpr (FromFormat == format::utf8) {
3080                 curr() = detail::decrement(first(), curr());
3081                 auto initial = curr();
3082                 char32_t cp = decode_code_point();
3083                 curr() = initial;
3084                 return cp;
3085             } else if constexpr (FromFormat == format::utf16) {
3086                 char16_t lo = *--curr();
3087                 if (!boost::parser::detail::text::surrogate(lo))
3088                     return lo;
3089
3090                 if (boost::parser::detail::text::high_surrogate(lo)) {
3091                     return ErrorHandler{}(
3092                         "Invalid UTF-16 sequence; lone leading surrogate.");
3093                 }
3094
3095                 // low surrogate
3096                 if (curr() == first()) {
3097                     return ErrorHandler{}(
3098                         "Invalid UTF-16 sequence; lone trailing surrogate.");
3099                 }
3100
3101                 char16_t hi = *detail::prev(curr());
3102                 if (!boost::parser::detail::text::high_surrogate(hi)) {
3103                     return ErrorHandler{}(
3104                         "Invalid UTF-16 sequence; lone trailing surrogate.");
3105                 }
3106                 --curr();
3107
3108                 return char32_t((hi - high_surrogate_base) << 10) +
3109                        (lo - low_surrogate_base);
3110             } else {
3111                 return *--curr();
3112             }
3113         }
3114
3115         template<class Out>
3116         static constexpr Out encode_code_point(char32_t cp, Out out)
3117         {
3118             if constexpr (ToFormat == format::utf8) {
3119                 if (cp < 0x80) {
3120                     *out++ = static_cast<char8_type>(cp);
3121                 } else if (cp < 0x800) {
3122                     *out++ = static_cast<char8_type>(0xC0 + (cp >> 6));
3123                     *out++ = static_cast<char8_type>(0x80 + (cp & 0x3f));
3124                 } else if (cp < 0x10000) {
3125                     *out++ = static_cast<char8_type>(0xe0 + (cp >> 12));
3126                     *out++ = static_cast<char8_type>(0x80 + ((cp >> 6) & 0x3f));
3127                     *out++ = static_cast<char8_type>(0x80 + (cp & 0x3f));
3128                 } else {
3129                     *out++ = static_cast<char8_type>(0xf0 + (cp >> 18));
3130                     *out++ = static_cast<char8_type>(0x80 + ((cp >> 12) & 0x3f));
3131                     *out++ = static_cast<char8_type>(0x80 + ((cp >> 6) & 0x3f));
3132                     *out++ = static_cast<char8_type>(0x80 + (cp & 0x3f));
3133                 }
3134             } else if constexpr (ToFormat == format::utf16) {
3135                 if (cp < 0x10000) {
3136                     *out++ = static_cast<char16_t>(cp);
3137                 } else {
3138                     *out++ =
3139                         static_cast<char16_t>(cp >> 10) + high_surrogate_base;
3140                     *out++ =
3141                         static_cast<char16_t>(cp & 0x3ff) + low_surrogate_base;
3142                 }
3143             } else {
3144                 *out++ = cp;
3145             }
3146             return out;
3147         }
3148
3149         constexpr void read()
3150         {
3151             I initial;
3152             if constexpr (
3153 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3154                 std::forward_iterator<I>
3155 #else
3156                 is_forward<I>
3157 #endif
3158             ) {
3159                 initial = curr();
3160             }
3161             if constexpr (noexcept(ErrorHandler{}(""))) {
3162                 char32_t cp = decode_code_point();
3163                 auto it = encode_code_point(cp, buf_.begin());
3164                 buf_index_ = 0;
3165                 buf_last_ = uint8_t(it - buf_.begin());
3166             } else {
3167 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3168                 auto buf = buf_;
3169                 try {
3170 #endif
3171                     char32_t cp = decode_code_point();
3172                     auto it = encode_code_point(cp, buf_.begin());
3173                     buf_index_ = 0;
3174                     buf_last_ = it - buf_.begin();
3175 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3176                 } catch (...) {
3177                     buf_ = buf;
3178                     curr() = initial;
3179                     throw;
3180                 }
3181 #endif
3182             }
3183             if constexpr (
3184 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3185                 std::forward_iterator<I>
3186 #else
3187             is_forward<I>
3188 #endif
3189             ) {
3190                 curr() = initial;
3191             }
3192         }
3193
3194         constexpr void read_reverse()
3195         {
3196             auto initial = curr();
3197             if constexpr (noexcept(ErrorHandler{}(""))) {
3198                 char32_t cp = decode_code_point_reverse();
3199                 auto it = encode_code_point(cp, buf_.begin());
3200                 buf_last_ = uint8_t(it - buf_.begin());
3201                 buf_index_ = buf_last_ - 1;
3202                 to_increment_ = (int)std::distance(curr(), initial);
3203             } else {
3204 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3205                 auto buf = buf_;
3206                 try {
3207 #endif
3208                     char32_t cp = decode_code_point_reverse();
3209                     auto it = encode_code_point(cp, buf_.begin());
3210                     buf_last_ = it - buf_.begin();
3211                     buf_index_ = buf_last_ - 1;
3212                     to_increment_ = std::distance(curr(), initial);
3213 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3214                 } catch (...) {
3215                     buf_ = buf;
3216                     curr() = initial;
3217                     throw;
3218                 }
3219 #endif
3220             }
3221         }
3222
3223 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3224         template<
3225             typename J = I,
3226             typename Enable = std::enable_if_t<is_bidirectional<J>>>
3227 #endif
3228         constexpr I first() const
3229 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3230             requires std::bidirectional_iterator<I>
3231 #endif
3232         {
3233             return first_and_curr_.first;
3234         }
3235         constexpr I & curr() { return first_and_curr_.curr; }
3236         constexpr I curr() const { return first_and_curr_.curr; }
3237
3238         std::array<value_type, 4 / static_cast<int>(ToFormat)> buf_ = {};
3239
3240         detail::first_and_curr<I> first_and_curr_ = {};
3241
3242         uint8_t buf_index_ = 0;
3243         uint8_t buf_last_ = 0;
3244         uint8_t to_increment_ = 0;
3245
3246         [[no_unique_address]] S last_ = {};
3247
3248         friend struct detail::iter_access;
3249     };
3250
3251 }}
3252
3253 namespace boost::parser::detail { namespace text { namespace detail {
3254
3255     template<class T>
3256     constexpr bool is_utf_iter = false;
3257     template<
3258         format FromFormat,
3259         format ToFormat,
3260         class I,
3261         class S,
3262         class ErrorHandler>
3263     constexpr bool
3264         is_utf_iter<utf_iterator<FromFormat, ToFormat, I, S, ErrorHandler>> =
3265             true;
3266
3267     // These are here because so many downstream views that use
3268     // utf_iterator use them.
3269
3270 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3271
3272     template<typename V>
3273     constexpr bool common_range_v = std::ranges::common_range<V>;
3274     template<typename V>
3275     constexpr bool forward_range_v = std::ranges::forward_range<V>;
3276     template<typename V>
3277     constexpr bool bidirectional_range_v = std::ranges::bidirectional_range<V>;
3278     template<typename T>
3279     constexpr bool default_initializable_v = std::default_initializable<T>;
3280
3281     template<typename V>
3282     constexpr bool utf32_range_v = utf32_range<V>;
3283
3284 #else
3285
3286     template<typename T>
3287     using range_expr =
3288         decltype(detail::begin(std::declval<T &>()) == detail::end(std::declval<T &>()));
3289     template<typename T>
3290     constexpr bool is_range_v = is_detected_v<range_expr, T>;
3291
3292     template<typename V>
3293     constexpr bool common_range_v =
3294         is_range_v<V> && std::is_same_v<iterator_t<V>, sentinel_t<V>>;
3295     template<typename V>
3296     constexpr bool input_range_v = is_range_v<V> && std::is_base_of_v<
3297         std::input_iterator_tag,
3298         typename std::iterator_traits<iterator_t<V>>::iterator_category>;
3299     template<typename V>
3300     constexpr bool forward_range_v = is_range_v<V> && std::is_base_of_v<
3301         std::forward_iterator_tag,
3302         typename std::iterator_traits<iterator_t<V>>::iterator_category>;
3303     template<typename V>
3304     constexpr bool bidirectional_range_v = is_range_v<V> && std::is_base_of_v<
3305         std::bidirectional_iterator_tag,
3306         typename std::iterator_traits<iterator_t<V>>::iterator_category>;
3307     template<typename T>
3308     constexpr bool default_initializable_v = std::is_default_constructible_v<T>;
3309
3310     template<typename V>
3311     constexpr bool utf_range_v = is_range_v<V> && code_unit_v<range_value_t<V>>;
3312
3313     template<typename V>
3314     constexpr bool
3315         utf32_range_v = is_range_v<V> &&
3316                         (
3317 #if !defined(_MSC_VER)
3318                             std::is_same_v<range_value_t<V>, wchar_t> ||
3319 #endif
3320                             std::is_same_v<range_value_t<V>, char32_t>);
3321
3322 #endif
3323
3324     template<typename I>
3325     constexpr bool random_access_iterator_v = std::is_base_of_v<
3326         std::random_access_iterator_tag,
3327         typename std::iterator_traits<I>::iterator_category>;
3328     template<typename I>
3329     constexpr bool bidirectional_iterator_v = std::is_base_of_v<
3330         std::bidirectional_iterator_tag,
3331         typename std::iterator_traits<I>::iterator_category>;
3332     template<typename I>
3333     constexpr bool forward_iterator_v = std::is_base_of_v<
3334         std::forward_iterator_tag,
3335         typename std::iterator_traits<I>::iterator_category>;
3336
3337     template<
3338         class V,
3339         bool StoreFirst = !is_utf_iter<iterator_t<V>> && common_range_v<V> &&
3340                           bidirectional_range_v<V>,
3341         bool StoreLast = !is_utf_iter<iterator_t<V>>>
3342     struct first_last_storage
3343     {
3344 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3345         template<
3346             typename Enable = std::enable_if_t<
3347                 default_initializable_v<iterator_t<V>> &&
3348                 default_initializable_v<sentinel_t<V>>>>
3349 #endif
3350         constexpr first_last_storage()
3351 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3352             requires default_initializable_v<iterator_t<V>> &&
3353             default_initializable_v<sentinel_t<V>>
3354 #endif
3355         {}
3356         constexpr first_last_storage(V & base) :
3357             first_{detail::begin(base)}, last_{detail::end(base)}
3358         {}
3359
3360         constexpr auto begin(iterator_t<V> & it) const { return first_; }
3361         constexpr auto end(iterator_t<V> & it) const { return last_; }
3362
3363         iterator_t<V> first_;
3364         sentinel_t<V> last_;
3365     };
3366
3367     template<typename I>
3368     using trinary_iter_ctor = decltype(I(
3369         std::declval<I>().begin(),
3370         std::declval<I>().end(),
3371         std::declval<I>().end()));
3372
3373     template<class V>
3374     struct first_last_storage<V, true, false>
3375     {
3376 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3377         template<
3378             typename Enable =
3379                 std::enable_if_t<default_initializable_v<iterator_t<V>>>>
3380 #endif
3381         constexpr first_last_storage()
3382 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3383             requires default_initializable_v<iterator_t<V>>
3384 #endif
3385         {}
3386         constexpr first_last_storage(V & base) : first_{detail::begin(base)} {}
3387
3388         constexpr auto begin(iterator_t<V> & it) const { return first_; }
3389         constexpr auto end(iterator_t<V> & it) const {
3390             if constexpr (
3391 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3392                 requires { iterator_t<V>(it.begin(), it.end(), it.end()); }
3393 #else
3394                 is_detected_v<trinary_iter_ctor, iterator_t<V>>
3395 #endif
3396             ) {
3397                 return iterator_t<V>(it.begin(), it.end(), it.end());
3398             } else {
3399                 return it.end();
3400             }
3401         }
3402
3403         iterator_t<V> first_;
3404     };
3405
3406     template<class V>
3407     struct first_last_storage<V, false, true>
3408     {
3409 #if !BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3410         template<
3411             typename Enable =
3412                 std::enable_if_t<default_initializable_v<sentinel_t<V>>>>
3413 #endif
3414         constexpr first_last_storage()
3415 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3416             requires default_initializable_v<sentinel_t<V>>
3417 #endif
3418         {}
3419         constexpr first_last_storage(V & base) : last_{detail::end(base)} {}
3420
3421         constexpr auto begin(iterator_t<V> & it) const {
3422             if constexpr (is_utf_iter<iterator_t<V>>) {
3423                 return iterator_t<V>(it.begin(), it.begin(), it.end());
3424             } else {
3425                 return;
3426             }
3427         }
3428         constexpr auto end(iterator_t<V> & it) const { return last_; }
3429
3430         sentinel_t<V> last_;
3431     };
3432
3433     template<class V>
3434     struct first_last_storage<V, false, false>
3435     {
3436         constexpr first_last_storage() = default;
3437         constexpr first_last_storage(V & base) {}
3438
3439         constexpr auto begin(iterator_t<V> & it) const {
3440             if constexpr (is_utf_iter<iterator_t<V>>) {
3441                 return iterator_t<V>(it.begin(), it.begin(), it.end());
3442             } else {
3443                 return;
3444             }
3445         }
3446         constexpr auto end(iterator_t<V> & it) const {
3447             if constexpr (
3448 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3449                 requires { iterator_t<V>(it.begin(), it.end(), it.end()); }
3450 #else
3451                 is_detected_v<trinary_iter_ctor, iterator_t<V>>
3452 #endif
3453             ) {
3454                 return iterator_t<V>(it.begin(), it.end(), it.end());
3455             } else {
3456                 return it.end();
3457             }
3458         }
3459     };
3460
3461
3462     template<class V>
3463     constexpr auto uc_view_category() {
3464         if constexpr (common_range_v<V> && bidirectional_range_v<V>) {
3465             return std::bidirectional_iterator_tag{};
3466         } else {
3467             return std::forward_iterator_tag{};
3468         }
3469     }
3470
3471     template<class V>
3472     using uc_view_category_t = decltype(uc_view_category<V>());
3473
3474     template<bool Const, class T>
3475     using maybe_const = std::conditional_t<Const, const T, T>;
3476
3477     template<class T>
3478     constexpr bool is_empty_view = false;
3479 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
3480     template<class T>
3481     constexpr bool is_empty_view<std::ranges::empty_view<T>> = true;
3482 #endif
3483
3484 }}}
3485
3486 #endif