arrow/util/binary_view_util.h

0001 // Licensed to the Apache Software Foundation (ASF) under one
0002 // or more contributor license agreements.  See the NOTICE file
0003 // distributed with this work for additional information
0004 // regarding copyright ownership.  The ASF licenses this file
0005 // to you under the Apache License, Version 2.0 (the
0006 // "License"); you may not use this file except in compliance
0007 // with the License.  You may obtain a copy of the License at
0008 //
0009 //   http://www.apache.org/licenses/LICENSE-2.0
0010 //
0011 // Unless required by applicable law or agreed to in writing,
0012 // software distributed under the License is distributed on an
0013 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
0014 // KIND, either express or implied.  See the License for the
0015 // specific language governing permissions and limitations
0016 // under the License.
0017
0018 #pragma once
0019
0020 #include <string_view>
0021 #include <utility>
0022
0023 #include "arrow/type.h"
0024 #include "arrow/util/span.h"
0025
0026 namespace arrow::util {
0027
0028 inline BinaryViewType::c_type ToInlineBinaryView(const void* data, int32_t size) {
0029   assert(size <= BinaryViewType::kInlineSize);
0030   // Small string: inlined. Bytes beyond size are zeroed
0031   BinaryViewType::c_type out;
0032   out.inlined = {size, {}};
0033   memcpy(&out.inlined.data, data, size);
0034   return out;
0035 }
0036
0037 inline BinaryViewType::c_type ToInlineBinaryView(std::string_view v) {
0038   assert(v.size() <= BinaryViewType::kInlineSize);
0039   return ToInlineBinaryView(v.data(), static_cast<int32_t>(v.size()));
0040 }
0041
0042 inline BinaryViewType::c_type ToNonInlineBinaryView(const void* data, int32_t size,
0043                                                     int32_t buffer_index,
0044                                                     int32_t offset) {
0045   // Large string: store index/offset.
0046   BinaryViewType::c_type out;
0047   out.ref = {size, {}, buffer_index, offset};
0048   memcpy(&out.ref.prefix, data, sizeof(out.ref.prefix));
0049   return out;
0050 }
0051
0052 inline BinaryViewType::c_type ToBinaryView(const void* data, int32_t size,
0053                                            int32_t buffer_index, int32_t offset) {
0054   if (size <= BinaryViewType::kInlineSize) {
0055     return ToInlineBinaryView(data, size);
0056   }
0057   return ToNonInlineBinaryView(data, size, buffer_index, offset);
0058 }
0059
0060 inline BinaryViewType::c_type ToBinaryView(std::string_view v, int32_t buffer_index,
0061                                            int32_t offset) {
0062   return ToBinaryView(v.data(), static_cast<int32_t>(v.size()), buffer_index, offset);
0063 }
0064
0065 template <typename BufferPtr>
0066 std::string_view FromBinaryView(const BinaryViewType::c_type& v,
0067                                 const BufferPtr* data_buffers) {
0068   auto* data = v.is_inline() ? v.inlined.data.data()
0069                              : data_buffers[v.ref.buffer_index]->data() + v.ref.offset;
0070   return {reinterpret_cast<const char*>(data), static_cast<size_t>(v.size())};
0071 }
0072 template <typename BufferPtr>
0073 std::string_view FromBinaryView(BinaryViewType::c_type&&, const BufferPtr*) = delete;
0074
0075 template <typename BufferPtr>
0076 bool EqualBinaryView(BinaryViewType::c_type l, BinaryViewType::c_type r,
0077                      const BufferPtr* l_buffers, const BufferPtr* r_buffers) {
0078   int64_t l_size_and_prefix, r_size_and_prefix;
0079   memcpy(&l_size_and_prefix, &l, sizeof(l_size_and_prefix));
0080   memcpy(&r_size_and_prefix, &r, sizeof(r_size_and_prefix));
0081
0082   if (l_size_and_prefix != r_size_and_prefix) return false;
0083
0084   if (l.is_inline()) {
0085     // The columnar spec mandates that the inlined part be zero-padded, so we can compare
0086     // a word at a time regardless of the exact size.
0087     int64_t l_inlined, r_inlined;
0088     memcpy(&l_inlined, l.inline_data() + BinaryViewType::kPrefixSize, sizeof(l_inlined));
0089     memcpy(&r_inlined, r.inline_data() + BinaryViewType::kPrefixSize, sizeof(r_inlined));
0090     return l_inlined == r_inlined;
0091   }
0092
0093   // Sizes are equal and this is not inline, therefore both are out
0094   // of line and have kPrefixSize first in common.
0095   const uint8_t* l_data = l_buffers[l.ref.buffer_index]->data() + l.ref.offset;
0096   const uint8_t* r_data = r_buffers[r.ref.buffer_index]->data() + r.ref.offset;
0097   return memcmp(l_data + BinaryViewType::kPrefixSize,
0098                 r_data + BinaryViewType::kPrefixSize,
0099                 l.size() - BinaryViewType::kPrefixSize) == 0;
0100 }
0101
0102 /// \brief Compute the total size of a list of binary views including null
0103 /// views.
0104 ///
0105 /// This is useful when calculating the necessary memory to store all the string
0106 /// data from the views.
0107 inline int64_t SumOfBinaryViewSizes(const BinaryViewType::c_type* views, int64_t length) {
0108   int64_t total = 0;
0109   for (int64_t i = 0; i < length; ++i) {
0110     total += views[i].size();
0111   }
0112   return total;
0113 }
0114
0115 }  // namespace arrow::util