json/internal/parser_traits.h

0001 // Protocol Buffers - Google's data interchange format
0002 // Copyright 2008 Google Inc.  All rights reserved.
0003 //
0004 // Use of this source code is governed by a BSD-style
0005 // license that can be found in the LICENSE file or at
0006 // https://developers.google.com/open-source/licenses/bsd
0007
0008 #ifndef GOOGLE_PROTOBUF_JSON_INTERNAL_PARSER_TRAITS_H__
0009 #define GOOGLE_PROTOBUF_JSON_INTERNAL_PARSER_TRAITS_H__
0010
0011 #include <cfloat>
0012 #include <cmath>
0013 #include <cstdint>
0014 #include <limits>
0015 #include <memory>
0016 #include <string>
0017 #include <utility>
0018
0019 #include "google/protobuf/type.pb.h"
0020 #include "absl/base/attributes.h"
0021 #include "absl/base/casts.h"
0022 #include "absl/container/flat_hash_map.h"
0023 #include "absl/container/flat_hash_set.h"
0024 #include "absl/status/status.h"
0025 #include "absl/strings/str_format.h"
0026 #include "absl/strings/string_view.h"
0027 #include "google/protobuf/io/coded_stream.h"
0028 #include "google/protobuf/io/zero_copy_stream.h"
0029 #include "google/protobuf/io/zero_copy_stream_impl_lite.h"
0030 #include "google/protobuf/json/internal/descriptor_traits.h"
0031 #include "google/protobuf/wire_format_lite.h"
0032 #include "google/protobuf/stubs/status_macros.h"
0033
0034 // Must be included last.
0035 #include "google/protobuf/port_def.inc"
0036
0037 namespace google {
0038 namespace protobuf {
0039 namespace json_internal {
0040 using ::google::protobuf::internal::WireFormatLite;
0041
0042 // See the comment in json_util2_parser.cc for more information.
0043 //
0044 // The type traits in this file  describe how to parse to a protobuf
0045 // representation used by the JSON API, either via proto reflection or by
0046 // emitting wire format to an output stream.
0047
0048 // Helper alias templates to avoid needing to write `typename` in function
0049 // signatures.
0050 template <typename Traits>
0051 using Msg = typename Traits::Msg;
0052
0053 struct ParseProto2Descriptor : Proto2Descriptor {
0054   // A message value that fields can be written to, but not read from.
0055   class Msg {
0056    public:
0057     explicit Msg(Message* msg) : msg_(msg) {}
0058
0059    private:
0060     friend ParseProto2Descriptor;
0061     Message* msg_;
0062     // Because `msg` might already have oneofs set, we need to track which were
0063     // set *during* the parse separately.
0064     absl::flat_hash_set<int> parsed_oneofs_indices_;
0065     absl::flat_hash_set<int> parsed_fields_;
0066   };
0067
0068   static bool HasParsed(Field f, const Msg& msg,
0069                         bool allow_repeated_non_oneof) {
0070     if (f->real_containing_oneof()) {
0071       return msg.parsed_oneofs_indices_.contains(
0072           f->real_containing_oneof()->index());
0073     }
0074     if (allow_repeated_non_oneof) {
0075       return false;
0076     }
0077     return msg.parsed_fields_.contains(f->number());
0078   }
0079
0080   /// Functions for writing fields. ///
0081
0082   // Marks a field as having been "seen". This will clear the field if it is
0083   // the first occurrence thereof.
0084   //
0085   // All setters call this function automatically, but it may also be called
0086   // eagerly to clear a pre-existing value that might not be overwritten, such
0087   // as when parsing a repeated field.
0088   static void RecordAsSeen(Field f, Msg& msg) {
0089     bool inserted = msg.parsed_fields_.insert(f->number()).second;
0090     if (inserted) {
0091       msg.msg_->GetReflection()->ClearField(msg.msg_, f);
0092     }
0093
0094     if (f->real_containing_oneof() != nullptr) {
0095       msg.parsed_oneofs_indices_.insert(f->real_containing_oneof()->index());
0096     }
0097   }
0098
0099   // Adds a new message and calls body on it.
0100   //
0101   // Body should have a signature `absl::Status(const Desc&, Msg&)`.
0102   template <typename F>
0103   static absl::Status NewMsg(Field f, Msg& msg, F body) {
0104     RecordAsSeen(f, msg);
0105
0106     Message* new_msg;
0107     if (f->is_repeated()) {
0108       new_msg = msg.msg_->GetReflection()->AddMessage(msg.msg_, f);
0109     } else {
0110       new_msg = msg.msg_->GetReflection()->MutableMessage(msg.msg_, f);
0111     }
0112     Msg wrapper(new_msg);
0113     return body(*f->message_type(), wrapper);
0114   }
0115
0116   // Adds a new dynamic message with the given type name and calls body on it.
0117   //
0118   // Body should have a signature `absl::Status(const Desc&, Msg&)`.
0119   template <typename F>
0120   static absl::Status NewDynamic(Field f, const std::string& type_url, Msg& msg,
0121                                  F body) {
0122     RecordAsSeen(f, msg);
0123     return WithDynamicType(
0124         *f->containing_type(), type_url, [&](const Desc& desc) -> absl::Status {
0125           DynamicMessageFactory factory;
0126           std::unique_ptr<Message> dynamic(factory.GetPrototype(&desc)->New());
0127           Msg wrapper(dynamic.get());
0128           RETURN_IF_ERROR(body(desc, wrapper));
0129
0130           if (f->is_repeated()) {
0131             msg.msg_->GetReflection()->AddString(
0132                 msg.msg_, f, dynamic->SerializePartialAsString());
0133           } else {
0134             msg.msg_->GetReflection()->SetString(
0135                 msg.msg_, f, dynamic->SerializePartialAsString());
0136           }
0137           return absl::OkStatus();
0138         });
0139   }
0140
0141   static void SetFloat(Field f, Msg& msg, float x) {
0142     RecordAsSeen(f, msg);
0143     if (f->is_repeated()) {
0144       msg.msg_->GetReflection()->AddFloat(msg.msg_, f, x);
0145     } else {
0146       msg.msg_->GetReflection()->SetFloat(msg.msg_, f, x);
0147     }
0148   }
0149
0150   static void SetDouble(Field f, Msg& msg, double x) {
0151     if (f->is_repeated()) {
0152       msg.msg_->GetReflection()->AddDouble(msg.msg_, f, x);
0153     } else {
0154       msg.msg_->GetReflection()->SetDouble(msg.msg_, f, x);
0155     }
0156   }
0157
0158   static void SetInt64(Field f, Msg& msg, int64_t x) {
0159     RecordAsSeen(f, msg);
0160     if (f->is_repeated()) {
0161       msg.msg_->GetReflection()->AddInt64(msg.msg_, f, x);
0162     } else {
0163       msg.msg_->GetReflection()->SetInt64(msg.msg_, f, x);
0164     }
0165   }
0166
0167   static void SetUInt64(Field f, Msg& msg, uint64_t x) {
0168     RecordAsSeen(f, msg);
0169     if (f->is_repeated()) {
0170       msg.msg_->GetReflection()->AddUInt64(msg.msg_, f, x);
0171     } else {
0172       msg.msg_->GetReflection()->SetUInt64(msg.msg_, f, x);
0173     }
0174   }
0175
0176   static void SetInt32(Field f, Msg& msg, int32_t x) {
0177     RecordAsSeen(f, msg);
0178     if (f->is_repeated()) {
0179       msg.msg_->GetReflection()->AddInt32(msg.msg_, f, x);
0180     } else {
0181       msg.msg_->GetReflection()->SetInt32(msg.msg_, f, x);
0182     }
0183   }
0184
0185   static void SetUInt32(Field f, Msg& msg, uint32_t x) {
0186     RecordAsSeen(f, msg);
0187     if (f->is_repeated()) {
0188       msg.msg_->GetReflection()->AddUInt32(msg.msg_, f, x);
0189     } else {
0190       msg.msg_->GetReflection()->SetUInt32(msg.msg_, f, x);
0191     }
0192   }
0193
0194   static void SetBool(Field f, Msg& msg, bool x) {
0195     RecordAsSeen(f, msg);
0196     if (f->is_repeated()) {
0197       msg.msg_->GetReflection()->AddBool(msg.msg_, f, x);
0198     } else {
0199       msg.msg_->GetReflection()->SetBool(msg.msg_, f, x);
0200     }
0201   }
0202
0203   static void SetString(Field f, Msg& msg, absl::string_view x) {
0204     RecordAsSeen(f, msg);
0205     if (f->is_repeated()) {
0206       msg.msg_->GetReflection()->AddString(msg.msg_, f, std::string(x));
0207     } else {
0208       msg.msg_->GetReflection()->SetString(msg.msg_, f, std::string(x));
0209     }
0210   }
0211
0212   static void SetEnum(Field f, Msg& msg, int32_t x) {
0213     RecordAsSeen(f, msg);
0214     if (f->is_repeated()) {
0215       msg.msg_->GetReflection()->AddEnumValue(msg.msg_, f, x);
0216     } else {
0217       msg.msg_->GetReflection()->SetEnumValue(msg.msg_, f, x);
0218     }
0219   }
0220 };
0221
0222 // Traits for proto3-ish deserialization.
0223 //
0224 // This includes a rudimentary proto serializer, since message fields are
0225 // written directly instead of being reflectively written to a proto field.
0226 //
0227 // See MessageTraits for API docs.
0228 struct ParseProto3Type : Proto3Type {
0229   class Msg {
0230    public:
0231     explicit Msg(io::ZeroCopyOutputStream* stream) : stream_(stream) {}
0232
0233    private:
0234     friend ParseProto3Type;
0235     io::CodedOutputStream stream_;
0236     absl::flat_hash_set<int32_t> parsed_oneofs_indices_;
0237     absl::flat_hash_set<int32_t> parsed_fields_;
0238   };
0239
0240   static bool HasParsed(Field f, const Msg& msg,
0241                         bool allow_repeated_non_oneof) {
0242     if (f->proto().oneof_index() != 0) {
0243       return msg.parsed_oneofs_indices_.contains(f->proto().oneof_index());
0244     }
0245     if (allow_repeated_non_oneof) {
0246       return false;
0247     }
0248     return msg.parsed_fields_.contains(f->proto().number());
0249   }
0250
0251   /// Functions for writing fields. ///
0252
0253   static void RecordAsSeen(Field f, Msg& msg) {
0254     msg.parsed_fields_.insert(f->proto().number());
0255     if (f->proto().oneof_index() != 0) {
0256       msg.parsed_oneofs_indices_.insert(f->proto().oneof_index());
0257     }
0258   }
0259
0260   template <typename F>
0261   static absl::Status NewMsg(Field f, Msg& msg, F body) {
0262     return NewDynamic(f, f->proto().type_url(), msg, body);
0263   }
0264
0265   template <typename F>
0266   static absl::Status NewDynamic(Field f, const std::string& type_url, Msg& msg,
0267                                  F body) {
0268     RecordAsSeen(f, msg);
0269     return WithDynamicType(
0270         f->parent(), type_url, [&](const Desc& desc) -> absl::Status {
0271           if (f->proto().kind() == google::protobuf::Field::TYPE_GROUP) {
0272             msg.stream_.WriteTag(f->proto().number() << 3 |
0273                                  WireFormatLite::WIRETYPE_START_GROUP);
0274             RETURN_IF_ERROR(body(desc, msg));
0275             msg.stream_.WriteTag(f->proto().number() << 3 |
0276                                  WireFormatLite::WIRETYPE_END_GROUP);
0277             return absl::OkStatus();
0278           }
0279
0280           std::string out;
0281           io::StringOutputStream stream(&out);
0282           Msg new_msg(&stream);
0283           RETURN_IF_ERROR(body(desc, new_msg));
0284
0285           new_msg.stream_.Trim();  // Should probably be called "Flush()".
0286           absl::string_view written(
0287               out.data(), static_cast<size_t>(new_msg.stream_.ByteCount()));
0288           SetString(f, msg, written);
0289           return absl::OkStatus();
0290         });
0291   }
0292
0293   static void SetFloat(Field f, Msg& msg, float x) {
0294     RecordAsSeen(f, msg);
0295     msg.stream_.WriteTag(f->proto().number() << 3 |
0296                          WireFormatLite::WIRETYPE_FIXED32);
0297     msg.stream_.WriteLittleEndian32(absl::bit_cast<uint32_t>(x));
0298   }
0299
0300   static void SetDouble(Field f, Msg& msg, double x) {
0301     RecordAsSeen(f, msg);
0302     msg.stream_.WriteTag(f->proto().number() << 3 |
0303                          WireFormatLite::WIRETYPE_FIXED64);
0304     msg.stream_.WriteLittleEndian64(absl::bit_cast<uint64_t>(x));
0305   }
0306
0307   static void SetInt64(Field f, Msg& msg, int64_t x) {
0308     SetInt<int64_t, google::protobuf::Field::TYPE_INT64,
0309            google::protobuf::Field::TYPE_SFIXED64,
0310            google::protobuf::Field::TYPE_SINT64>(f, msg, x);
0311   }
0312
0313   static void SetUInt64(Field f, Msg& msg, uint64_t x) {
0314     SetInt<uint64_t, google::protobuf::Field::TYPE_UINT64,
0315            google::protobuf::Field::TYPE_FIXED64,
0316            google::protobuf::Field::TYPE_UNKNOWN>(f, msg, x);
0317   }
0318
0319   static void SetInt32(Field f, Msg& msg, int32_t x) {
0320     SetInt<int32_t, google::protobuf::Field::TYPE_INT32,
0321            google::protobuf::Field::TYPE_SFIXED32,
0322            google::protobuf::Field::TYPE_SINT32>(f, msg, x);
0323   }
0324
0325   static void SetUInt32(Field f, Msg& msg, uint32_t x) {
0326     SetInt<uint32_t, google::protobuf::Field::TYPE_UINT32,
0327            google::protobuf::Field::TYPE_FIXED32,
0328            google::protobuf::Field::TYPE_UNKNOWN>(f, msg, x);
0329   }
0330
0331   static void SetBool(Field f, Msg& msg, bool x) {
0332     RecordAsSeen(f, msg);
0333     msg.stream_.WriteTag(f->proto().number() << 3);
0334     char b = x ? 0x01 : 0x00;
0335     msg.stream_.WriteRaw(&b, 1);
0336   }
0337
0338   static void SetString(Field f, Msg& msg, absl::string_view x) {
0339     RecordAsSeen(f, msg);
0340     msg.stream_.WriteTag(f->proto().number() << 3 |
0341                          WireFormatLite::WIRETYPE_LENGTH_DELIMITED);
0342     msg.stream_.WriteVarint64(static_cast<uint64_t>(x.size()));
0343     msg.stream_.WriteRaw(x.data(), x.size());
0344   }
0345
0346   static void SetEnum(Field f, Msg& msg, int32_t x) {
0347     RecordAsSeen(f, msg);
0348     msg.stream_.WriteTag(f->proto().number() << 3);
0349     // Sign extension is deliberate here.
0350     msg.stream_.WriteVarint32(x);
0351   }
0352
0353  private:
0354   using Kind = google::protobuf::Field::Kind;
0355   // Sets a field of *some* integer type, with the given kinds for the possible
0356   // encodings. This avoids quadruplicating this code in the helpers for the
0357   // four major integer types.
0358   template <typename Int, Kind varint, Kind fixed, Kind zigzag>
0359   static void SetInt(Field f, Msg& msg, Int x) {
0360     RecordAsSeen(f, msg);
0361     switch (f->proto().kind()) {
0362       case zigzag:
0363         // Regardless of the integer type, ZigZag64 will do the right thing,
0364         // because ZigZag is not dependent on the width of the integer: it is
0365         // always `2 * abs(n) + (n < 0)`.
0366         x = static_cast<Int>(
0367             internal::WireFormatLite::ZigZagEncode64(static_cast<int64_t>(x)));
0368         ABSL_FALLTHROUGH_INTENDED;
0369       case varint:
0370         msg.stream_.WriteTag(f->proto().number() << 3 |
0371                              WireFormatLite::WIRETYPE_VARINT);
0372         if (sizeof(Int) == 4) {
0373           msg.stream_.WriteVarint32(static_cast<uint32_t>(x));
0374         } else {
0375           msg.stream_.WriteVarint64(static_cast<uint64_t>(x));
0376         }
0377         break;
0378       case fixed: {
0379         if (sizeof(Int) == 4) {
0380           msg.stream_.WriteTag(f->proto().number() << 3 |
0381                                WireFormatLite::WIRETYPE_FIXED32);
0382           msg.stream_.WriteLittleEndian32(static_cast<uint32_t>(x));
0383         } else {
0384           msg.stream_.WriteTag(f->proto().number() << 3 |
0385                                WireFormatLite::WIRETYPE_FIXED64);
0386           msg.stream_.WriteLittleEndian64(static_cast<uint64_t>(x));
0387         }
0388         break;
0389       }
0390       default: {  // Unreachable.
0391       }
0392     }
0393   }
0394 };
0395 }  // namespace json_internal
0396 }  // namespace protobuf
0397 }  // namespace google
0398
0399 #include "google/protobuf/port_undef.inc"
0400 #endif  // GOOGLE_PROTOBUF_JSON_INTERNAL_PARSER_TRAITS_H__