File indexing completed on 2025-08-28 08:26:54
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020
0021 #pragma once
0022
0023 #include <cstdint>
0024 #include <memory>
0025 #include <string>
0026 #include <utility>
0027 #include <vector>
0028
0029 #include "arrow/array/array_base.h"
0030 #include "arrow/array/data.h"
0031 #include "arrow/result.h"
0032 #include "arrow/status.h"
0033 #include "arrow/type.h"
0034 #include "arrow/type_fwd.h"
0035 #include "arrow/util/checked_cast.h"
0036 #include "arrow/util/macros.h"
0037 #include "arrow/util/visibility.h"
0038
0039 namespace arrow {
0040
0041
0042
0043
0044
0045
0046
0047
0048 template <typename TYPE>
0049 class VarLengthListLikeArray;
0050
0051 namespace internal {
0052
0053
0054
0055
0056 template <typename TYPE>
0057 void SetListData(VarLengthListLikeArray<TYPE>* self,
0058 const std::shared_ptr<ArrayData>& data,
0059 Type::type expected_type_id = TYPE::type_id);
0060
0061
0062
0063
0064
0065
0066
0067
0068
0069
0070
0071
0072 ARROW_EXPORT Result<std::shared_ptr<Array>> FlattenLogicalListRecursively(
0073 const Array& in_array, MemoryPool* memory_pool);
0074
0075 }
0076
0077
0078 template <typename TYPE>
0079 class VarLengthListLikeArray : public Array {
0080 public:
0081 using TypeClass = TYPE;
0082 using offset_type = typename TypeClass::offset_type;
0083
0084 const TypeClass* var_length_list_like_type() const { return this->list_type_; }
0085
0086
0087
0088
0089 const std::shared_ptr<Array>& values() const { return values_; }
0090
0091
0092 const std::shared_ptr<Buffer>& value_offsets() const { return data_->buffers[1]; }
0093
0094 const std::shared_ptr<DataType>& value_type() const { return list_type_->value_type(); }
0095
0096
0097 const offset_type* raw_value_offsets() const { return raw_value_offsets_; }
0098
0099
0100
0101 offset_type value_offset(int64_t i) const { return raw_value_offsets_[i]; }
0102
0103
0104
0105
0106
0107
0108
0109 virtual offset_type value_length(int64_t i) const = 0;
0110
0111
0112 std::shared_ptr<Array> value_slice(int64_t i) const {
0113 return values_->Slice(value_offset(i), value_length(i));
0114 }
0115
0116
0117
0118
0119
0120 Result<std::shared_ptr<Array>> FlattenRecursively(
0121 MemoryPool* memory_pool = default_memory_pool()) const {
0122 return internal::FlattenLogicalListRecursively(*this, memory_pool);
0123 }
0124
0125 protected:
0126 friend void internal::SetListData<TYPE>(VarLengthListLikeArray<TYPE>* self,
0127 const std::shared_ptr<ArrayData>& data,
0128 Type::type expected_type_id);
0129
0130 const TypeClass* list_type_ = NULLPTR;
0131 std::shared_ptr<Array> values_;
0132 const offset_type* raw_value_offsets_ = NULLPTR;
0133 };
0134
0135
0136
0137
0138 template <typename TYPE>
0139 class BaseListArray : public VarLengthListLikeArray<TYPE> {
0140 public:
0141 using TypeClass = TYPE;
0142 using offset_type = typename TYPE::offset_type;
0143
0144 const TypeClass* list_type() const { return this->var_length_list_like_type(); }
0145
0146
0147
0148
0149
0150
0151
0152 offset_type value_length(int64_t i) const final {
0153 return this->raw_value_offsets_[i + 1] - this->raw_value_offsets_[i];
0154 }
0155 };
0156
0157
0158 class ARROW_EXPORT ListArray : public BaseListArray<ListType> {
0159 public:
0160 explicit ListArray(std::shared_ptr<ArrayData> data);
0161
0162 ListArray(std::shared_ptr<DataType> type, int64_t length,
0163 std::shared_ptr<Buffer> value_offsets, std::shared_ptr<Array> values,
0164 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0165 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0166
0167
0168
0169
0170
0171
0172
0173
0174
0175
0176
0177
0178
0179
0180
0181
0182
0183
0184
0185
0186
0187 static Result<std::shared_ptr<ListArray>> FromArrays(
0188 const Array& offsets, const Array& values, MemoryPool* pool = default_memory_pool(),
0189 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0190 int64_t null_count = kUnknownNullCount);
0191
0192 static Result<std::shared_ptr<ListArray>> FromArrays(
0193 std::shared_ptr<DataType> type, const Array& offsets, const Array& values,
0194 MemoryPool* pool = default_memory_pool(),
0195 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0196 int64_t null_count = kUnknownNullCount);
0197
0198
0199 static Result<std::shared_ptr<ListArray>> FromListView(const ListViewArray& source,
0200 MemoryPool* pool);
0201
0202
0203
0204
0205
0206
0207 Result<std::shared_ptr<Array>> Flatten(
0208 MemoryPool* memory_pool = default_memory_pool()) const;
0209
0210
0211
0212
0213
0214
0215 std::shared_ptr<Array> offsets() const;
0216
0217 protected:
0218
0219 ListArray() = default;
0220
0221 void SetData(const std::shared_ptr<ArrayData>& data);
0222 };
0223
0224
0225 class ARROW_EXPORT LargeListArray : public BaseListArray<LargeListType> {
0226 public:
0227 explicit LargeListArray(const std::shared_ptr<ArrayData>& data);
0228
0229 LargeListArray(const std::shared_ptr<DataType>& type, int64_t length,
0230 const std::shared_ptr<Buffer>& value_offsets,
0231 const std::shared_ptr<Array>& values,
0232 const std::shared_ptr<Buffer>& null_bitmap = NULLPTR,
0233 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0234
0235
0236
0237
0238
0239
0240
0241
0242
0243
0244
0245
0246
0247
0248
0249
0250
0251
0252
0253
0254
0255 static Result<std::shared_ptr<LargeListArray>> FromArrays(
0256 const Array& offsets, const Array& values, MemoryPool* pool = default_memory_pool(),
0257 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0258 int64_t null_count = kUnknownNullCount);
0259
0260 static Result<std::shared_ptr<LargeListArray>> FromArrays(
0261 std::shared_ptr<DataType> type, const Array& offsets, const Array& values,
0262 MemoryPool* pool = default_memory_pool(),
0263 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0264 int64_t null_count = kUnknownNullCount);
0265
0266
0267 static Result<std::shared_ptr<LargeListArray>> FromListView(
0268 const LargeListViewArray& source, MemoryPool* pool);
0269
0270
0271
0272
0273
0274
0275 Result<std::shared_ptr<Array>> Flatten(
0276 MemoryPool* memory_pool = default_memory_pool()) const;
0277
0278
0279 std::shared_ptr<Array> offsets() const;
0280
0281 protected:
0282 void SetData(const std::shared_ptr<ArrayData>& data);
0283 };
0284
0285
0286
0287
0288 template <typename TYPE>
0289 class BaseListViewArray : public VarLengthListLikeArray<TYPE> {
0290 public:
0291 using TypeClass = TYPE;
0292 using offset_type = typename TYPE::offset_type;
0293
0294 const TypeClass* list_view_type() const { return this->var_length_list_like_type(); }
0295
0296
0297 const std::shared_ptr<Buffer>& value_sizes() const { return this->data_->buffers[2]; }
0298
0299
0300 const offset_type* raw_value_sizes() const { return raw_value_sizes_; }
0301
0302
0303
0304
0305
0306
0307
0308
0309 offset_type value_length(int64_t i) const final { return this->raw_value_sizes_[i]; }
0310
0311 protected:
0312 const offset_type* raw_value_sizes_ = NULLPTR;
0313 };
0314
0315
0316 class ARROW_EXPORT ListViewArray : public BaseListViewArray<ListViewType> {
0317 public:
0318 explicit ListViewArray(std::shared_ptr<ArrayData> data);
0319
0320 ListViewArray(std::shared_ptr<DataType> type, int64_t length,
0321 std::shared_ptr<Buffer> value_offsets,
0322 std::shared_ptr<Buffer> value_sizes, std::shared_ptr<Array> values,
0323 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0324 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0325
0326
0327
0328
0329
0330
0331
0332
0333
0334
0335
0336
0337
0338
0339
0340
0341
0342
0343
0344
0345
0346
0347
0348
0349
0350
0351
0352 static Result<std::shared_ptr<ListViewArray>> FromArrays(
0353 const Array& offsets, const Array& sizes, const Array& values,
0354 MemoryPool* pool = default_memory_pool(),
0355 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0356 int64_t null_count = kUnknownNullCount);
0357
0358 static Result<std::shared_ptr<ListViewArray>> FromArrays(
0359 std::shared_ptr<DataType> type, const Array& offsets, const Array& sizes,
0360 const Array& values, MemoryPool* pool = default_memory_pool(),
0361 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0362 int64_t null_count = kUnknownNullCount);
0363
0364
0365 static Result<std::shared_ptr<ListViewArray>> FromList(const ListArray& list_array,
0366 MemoryPool* pool);
0367
0368
0369
0370
0371
0372
0373
0374
0375
0376
0377
0378 Result<std::shared_ptr<Array>> Flatten(
0379 MemoryPool* memory_pool = default_memory_pool()) const;
0380
0381
0382
0383
0384
0385
0386 std::shared_ptr<Array> offsets() const;
0387
0388
0389
0390
0391
0392
0393 std::shared_ptr<Array> sizes() const;
0394
0395 protected:
0396
0397 ListViewArray() = default;
0398
0399 void SetData(const std::shared_ptr<ArrayData>& data);
0400 };
0401
0402
0403
0404 class ARROW_EXPORT LargeListViewArray : public BaseListViewArray<LargeListViewType> {
0405 public:
0406 explicit LargeListViewArray(std::shared_ptr<ArrayData> data);
0407
0408 LargeListViewArray(std::shared_ptr<DataType> type, int64_t length,
0409 std::shared_ptr<Buffer> value_offsets,
0410 std::shared_ptr<Buffer> value_sizes, std::shared_ptr<Array> values,
0411 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0412 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0413
0414
0415
0416
0417
0418
0419
0420
0421
0422
0423
0424
0425
0426
0427
0428
0429
0430
0431
0432
0433
0434
0435
0436
0437
0438
0439
0440 static Result<std::shared_ptr<LargeListViewArray>> FromArrays(
0441 const Array& offsets, const Array& sizes, const Array& values,
0442 MemoryPool* pool = default_memory_pool(),
0443 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0444 int64_t null_count = kUnknownNullCount);
0445
0446 static Result<std::shared_ptr<LargeListViewArray>> FromArrays(
0447 std::shared_ptr<DataType> type, const Array& offsets, const Array& sizes,
0448 const Array& values, MemoryPool* pool = default_memory_pool(),
0449 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0450 int64_t null_count = kUnknownNullCount);
0451
0452
0453 static Result<std::shared_ptr<LargeListViewArray>> FromList(
0454 const LargeListArray& list_array, MemoryPool* pool);
0455
0456
0457
0458
0459
0460
0461
0462 Result<std::shared_ptr<Array>> Flatten(
0463 MemoryPool* memory_pool = default_memory_pool()) const;
0464
0465
0466
0467
0468
0469
0470 std::shared_ptr<Array> offsets() const;
0471
0472
0473
0474
0475
0476
0477 std::shared_ptr<Array> sizes() const;
0478
0479 protected:
0480
0481 LargeListViewArray() = default;
0482
0483 void SetData(const std::shared_ptr<ArrayData>& data);
0484 };
0485
0486
0487
0488
0489
0490
0491
0492 class ARROW_EXPORT MapArray : public ListArray {
0493 public:
0494 using TypeClass = MapType;
0495
0496 explicit MapArray(const std::shared_ptr<ArrayData>& data);
0497
0498 MapArray(const std::shared_ptr<DataType>& type, int64_t length,
0499 const std::shared_ptr<Buffer>& value_offsets,
0500 const std::shared_ptr<Array>& keys, const std::shared_ptr<Array>& items,
0501 const std::shared_ptr<Buffer>& null_bitmap = NULLPTR,
0502 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0503
0504 MapArray(const std::shared_ptr<DataType>& type, int64_t length, BufferVector buffers,
0505 const std::shared_ptr<Array>& keys, const std::shared_ptr<Array>& items,
0506 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0507
0508 MapArray(const std::shared_ptr<DataType>& type, int64_t length,
0509 const std::shared_ptr<Buffer>& value_offsets,
0510 const std::shared_ptr<Array>& values,
0511 const std::shared_ptr<Buffer>& null_bitmap = NULLPTR,
0512 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0513
0514
0515
0516
0517
0518
0519
0520
0521
0522
0523
0524
0525
0526
0527
0528 static Result<std::shared_ptr<Array>> FromArrays(
0529 const std::shared_ptr<Array>& offsets, const std::shared_ptr<Array>& keys,
0530 const std::shared_ptr<Array>& items, MemoryPool* pool = default_memory_pool(),
0531 std::shared_ptr<Buffer> null_bitmap = NULLPTR);
0532
0533 static Result<std::shared_ptr<Array>> FromArrays(
0534 std::shared_ptr<DataType> type, const std::shared_ptr<Array>& offsets,
0535 const std::shared_ptr<Array>& keys, const std::shared_ptr<Array>& items,
0536 MemoryPool* pool = default_memory_pool(),
0537 std::shared_ptr<Buffer> null_bitmap = NULLPTR);
0538
0539 const MapType* map_type() const { return map_type_; }
0540
0541
0542 const std::shared_ptr<Array>& keys() const { return keys_; }
0543
0544
0545 const std::shared_ptr<Array>& items() const { return items_; }
0546
0547
0548 static Status ValidateChildData(
0549 const std::vector<std::shared_ptr<ArrayData>>& child_data);
0550
0551 protected:
0552 void SetData(const std::shared_ptr<ArrayData>& data);
0553
0554 static Result<std::shared_ptr<Array>> FromArraysInternal(
0555 std::shared_ptr<DataType> type, const std::shared_ptr<Array>& offsets,
0556 const std::shared_ptr<Array>& keys, const std::shared_ptr<Array>& items,
0557 MemoryPool* pool, std::shared_ptr<Buffer> null_bitmap = NULLPTR);
0558
0559 private:
0560 const MapType* map_type_;
0561 std::shared_ptr<Array> keys_, items_;
0562 };
0563
0564
0565
0566
0567
0568 class ARROW_EXPORT FixedSizeListArray : public Array {
0569 public:
0570 using TypeClass = FixedSizeListType;
0571 using offset_type = TypeClass::offset_type;
0572
0573 explicit FixedSizeListArray(const std::shared_ptr<ArrayData>& data);
0574
0575 FixedSizeListArray(const std::shared_ptr<DataType>& type, int64_t length,
0576 const std::shared_ptr<Array>& values,
0577 const std::shared_ptr<Buffer>& null_bitmap = NULLPTR,
0578 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0579
0580 const FixedSizeListType* list_type() const;
0581
0582
0583 const std::shared_ptr<Array>& values() const;
0584
0585 const std::shared_ptr<DataType>& value_type() const;
0586
0587
0588 int64_t value_offset(int64_t i) const {
0589 i += data_->offset;
0590 return list_size_ * i;
0591 }
0592
0593
0594
0595
0596
0597
0598
0599 int32_t value_length(int64_t i = 0) const {
0600 ARROW_UNUSED(i);
0601 return list_size_;
0602 }
0603
0604 std::shared_ptr<Array> value_slice(int64_t i) const {
0605 return values_->Slice(value_offset(i), value_length(i));
0606 }
0607
0608
0609
0610
0611
0612 Result<std::shared_ptr<Array>> Flatten(
0613 MemoryPool* memory_pool = default_memory_pool()) const;
0614
0615
0616
0617
0618
0619 Result<std::shared_ptr<Array>> FlattenRecursively(
0620 MemoryPool* memory_pool = default_memory_pool()) const {
0621 return internal::FlattenLogicalListRecursively(*this, memory_pool);
0622 }
0623
0624
0625
0626
0627
0628
0629
0630
0631 static Result<std::shared_ptr<Array>> FromArrays(
0632 const std::shared_ptr<Array>& values, int32_t list_size,
0633 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0634 int64_t null_count = kUnknownNullCount);
0635
0636
0637
0638
0639
0640
0641
0642
0643 static Result<std::shared_ptr<Array>> FromArrays(
0644 const std::shared_ptr<Array>& values, std::shared_ptr<DataType> type,
0645 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0646 int64_t null_count = kUnknownNullCount);
0647
0648 protected:
0649 void SetData(const std::shared_ptr<ArrayData>& data);
0650 int32_t list_size_;
0651
0652 private:
0653 std::shared_ptr<Array> values_;
0654 };
0655
0656
0657
0658
0659
0660 class ARROW_EXPORT StructArray : public Array {
0661 public:
0662 using TypeClass = StructType;
0663
0664 explicit StructArray(const std::shared_ptr<ArrayData>& data);
0665
0666 StructArray(const std::shared_ptr<DataType>& type, int64_t length,
0667 const std::vector<std::shared_ptr<Array>>& children,
0668 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0669 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0670
0671
0672
0673
0674
0675 static Result<std::shared_ptr<StructArray>> Make(
0676 const ArrayVector& children, const std::vector<std::string>& field_names,
0677 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0678 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0679
0680
0681
0682
0683
0684
0685 static Result<std::shared_ptr<StructArray>> Make(
0686 const ArrayVector& children, const FieldVector& fields,
0687 std::shared_ptr<Buffer> null_bitmap = NULLPTR,
0688 int64_t null_count = kUnknownNullCount, int64_t offset = 0);
0689
0690 const StructType* struct_type() const;
0691
0692
0693
0694
0695 const std::shared_ptr<Array>& field(int pos) const;
0696
0697 const ArrayVector& fields() const;
0698
0699
0700 std::shared_ptr<Array> GetFieldByName(const std::string& name) const;
0701
0702
0703 Status CanReferenceFieldByName(const std::string& name) const;
0704
0705
0706 Status CanReferenceFieldsByNames(const std::vector<std::string>& names) const;
0707
0708
0709
0710
0711 Result<ArrayVector> Flatten(MemoryPool* pool = default_memory_pool()) const;
0712
0713
0714
0715
0716
0717
0718 Result<std::shared_ptr<Array>> GetFlattenedField(
0719 int index, MemoryPool* pool = default_memory_pool()) const;
0720
0721 private:
0722
0723
0724 mutable ArrayVector boxed_fields_;
0725 };
0726
0727
0728
0729
0730
0731 class ARROW_EXPORT UnionArray : public Array {
0732 public:
0733 using type_code_t = int8_t;
0734
0735
0736 const std::shared_ptr<Buffer>& type_codes() const { return data_->buffers[1]; }
0737
0738 const type_code_t* raw_type_codes() const { return raw_type_codes_; }
0739
0740
0741 type_code_t type_code(int64_t i) const { return raw_type_codes_[i]; }
0742
0743
0744 int child_id(int64_t i) const { return union_type_->child_ids()[raw_type_codes_[i]]; }
0745
0746 const UnionType* union_type() const { return union_type_; }
0747
0748 UnionMode::type mode() const { return union_type_->mode(); }
0749
0750
0751
0752
0753
0754 std::shared_ptr<Array> field(int pos) const;
0755
0756 protected:
0757 void SetData(std::shared_ptr<ArrayData> data);
0758
0759 const type_code_t* raw_type_codes_;
0760 const UnionType* union_type_;
0761
0762
0763 mutable std::vector<std::shared_ptr<Array>> boxed_fields_;
0764 };
0765
0766
0767 class ARROW_EXPORT SparseUnionArray : public UnionArray {
0768 public:
0769 using TypeClass = SparseUnionType;
0770
0771 explicit SparseUnionArray(std::shared_ptr<ArrayData> data);
0772
0773 SparseUnionArray(std::shared_ptr<DataType> type, int64_t length, ArrayVector children,
0774 std::shared_ptr<Buffer> type_ids, int64_t offset = 0);
0775
0776
0777
0778
0779
0780
0781
0782
0783 static Result<std::shared_ptr<Array>> Make(const Array& type_ids, ArrayVector children,
0784 std::vector<type_code_t> type_codes) {
0785 return Make(std::move(type_ids), std::move(children), std::vector<std::string>{},
0786 std::move(type_codes));
0787 }
0788
0789
0790
0791
0792
0793
0794
0795
0796
0797 static Result<std::shared_ptr<Array>> Make(const Array& type_ids, ArrayVector children,
0798 std::vector<std::string> field_names = {},
0799 std::vector<type_code_t> type_codes = {});
0800
0801 const SparseUnionType* union_type() const {
0802 return internal::checked_cast<const SparseUnionType*>(union_type_);
0803 }
0804
0805
0806
0807
0808
0809
0810 Result<std::shared_ptr<Array>> GetFlattenedField(
0811 int index, MemoryPool* pool = default_memory_pool()) const;
0812
0813 protected:
0814 void SetData(std::shared_ptr<ArrayData> data);
0815 };
0816
0817
0818
0819
0820 class ARROW_EXPORT DenseUnionArray : public UnionArray {
0821 public:
0822 using TypeClass = DenseUnionType;
0823
0824 explicit DenseUnionArray(const std::shared_ptr<ArrayData>& data);
0825
0826 DenseUnionArray(std::shared_ptr<DataType> type, int64_t length, ArrayVector children,
0827 std::shared_ptr<Buffer> type_ids,
0828 std::shared_ptr<Buffer> value_offsets = NULLPTR, int64_t offset = 0);
0829
0830
0831
0832
0833
0834
0835
0836
0837
0838
0839
0840
0841 static Result<std::shared_ptr<Array>> Make(const Array& type_ids,
0842 const Array& value_offsets,
0843 ArrayVector children,
0844 std::vector<type_code_t> type_codes) {
0845 return Make(type_ids, value_offsets, std::move(children), std::vector<std::string>{},
0846 std::move(type_codes));
0847 }
0848
0849
0850
0851
0852
0853
0854
0855
0856
0857
0858
0859
0860
0861
0862 static Result<std::shared_ptr<Array>> Make(const Array& type_ids,
0863 const Array& value_offsets,
0864 ArrayVector children,
0865 std::vector<std::string> field_names = {},
0866 std::vector<type_code_t> type_codes = {});
0867
0868 const DenseUnionType* union_type() const {
0869 return internal::checked_cast<const DenseUnionType*>(union_type_);
0870 }
0871
0872
0873 const std::shared_ptr<Buffer>& value_offsets() const { return data_->buffers[2]; }
0874
0875 int32_t value_offset(int64_t i) const { return raw_value_offsets_[i]; }
0876
0877 const int32_t* raw_value_offsets() const { return raw_value_offsets_; }
0878
0879 protected:
0880 const int32_t* raw_value_offsets_;
0881
0882 void SetData(const std::shared_ptr<ArrayData>& data);
0883 };
0884
0885
0886
0887 }