arrow/util/utf8.h

0001 // Licensed to the Apache Software Foundation (ASF) under one
0002 // or more contributor license agreements.  See the NOTICE file
0003 // distributed with this work for additional information
0004 // regarding copyright ownership.  The ASF licenses this file
0005 // to you under the Apache License, Version 2.0 (the
0006 // "License"); you may not use this file except in compliance
0007 // with the License.  You may obtain a copy of the License at
0008 //
0009 //   http://www.apache.org/licenses/LICENSE-2.0
0010 //
0011 // Unless required by applicable law or agreed to in writing,
0012 // software distributed under the License is distributed on an
0013 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
0014 // KIND, either express or implied.  See the License for the
0015 // specific language governing permissions and limitations
0016 // under the License.
0017
0018 #pragma once
0019
0020 #include <cstdint>
0021 #include <cstring>
0022 #include <string>
0023 #include <string_view>
0024
0025 #include "arrow/type_fwd.h"
0026 #include "arrow/util/macros.h"
0027 #include "arrow/util/visibility.h"
0028
0029 namespace arrow {
0030 namespace util {
0031
0032 // Convert a UTF8 string to a wstring (either UTF16 or UTF32, depending
0033 // on the wchar_t width).
0034 ARROW_EXPORT Result<std::wstring> UTF8ToWideString(std::string_view source);
0035
0036 // Similarly, convert a wstring to a UTF8 string.
0037 ARROW_EXPORT Result<std::string> WideStringToUTF8(const std::wstring& source);
0038
0039 // Convert UTF8 string to a UTF16 string.
0040 ARROW_EXPORT Result<std::u16string> UTF8StringToUTF16(std::string_view source);
0041
0042 // Convert UTF16 string to a UTF8 string.
0043 ARROW_EXPORT Result<std::string> UTF16StringToUTF8(std::u16string_view source);
0044
0045 // This function needs to be called before doing UTF8 validation.
0046 ARROW_EXPORT void InitializeUTF8();
0047
0048 ARROW_EXPORT bool ValidateUTF8(const uint8_t* data, int64_t size);
0049
0050 ARROW_EXPORT bool ValidateUTF8(std::string_view str);
0051
0052 // Skip UTF8 byte order mark, if any.
0053 ARROW_EXPORT
0054 Result<const uint8_t*> SkipUTF8BOM(const uint8_t* data, int64_t size);
0055
0056 static constexpr uint32_t kMaxUnicodeCodepoint = 0x110000;
0057
0058 }  // namespace util
0059 }  // namespace arrow