EIC code displayed by LXR master/​include/​arrow/​csv/​options.h	Source navigation Diff markup Identifier search General search
	Version: master test Revert   Change

File indexing completed on 2025-08-28 08:26:57

 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
 // regarding copyright ownership.  The ASF licenses this file
 // to you under the Apache License, Version 2.0 (the
 // "License"); you may not use this file except in compliance
 // with the License.  You may obtain a copy of the License at
 //
 //   http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing,
 // software distributed under the License is distributed on an
 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
 
 #pragma once
 
 #include <cstdint>
 #include <memory>
 #include <string>
 #include <unordered_map>
 #include <vector>
 
 #include "arrow/csv/invalid_row.h"
 #include "arrow/csv/type_fwd.h"
 #include "arrow/io/interfaces.h"
 #include "arrow/status.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
 class DataType;
 class TimestampParser;
 
 namespace csv {
 
 // Silly workaround for https://github.com/michaeljones/breathe/issues/453
 constexpr char kDefaultEscapeChar = '\\';
 
 struct ARROW_EXPORT ParseOptions {
   // Parsing options
 
   /// Field delimiter
   char delimiter = ',';
   /// Whether quoting is used
   bool quoting = true;
   /// Quoting character (if `quoting` is true)
   char quote_char = '"';
   /// Whether a quote inside a value is double-quoted
   bool double_quote = true;
   /// Whether escaping is used
   bool escaping = false;
   /// Escaping character (if `escaping` is true)
   char escape_char = kDefaultEscapeChar;
   /// Whether values are allowed to contain CR (0x0d) and LF (0x0a) characters
   bool newlines_in_values = false;
   /// Whether empty lines are ignored.  If false, an empty line represents
   /// a single empty value (assuming a one-column CSV file).
   bool ignore_empty_lines = true;
   /// A handler function for rows which do not have the correct number of columns
   InvalidRowHandler invalid_row_handler;
 
   /// Create parsing options with default values
   static ParseOptions Defaults();
 
   /// \brief Test that all set options are valid
   Status Validate() const;
 };
 
 struct ARROW_EXPORT ConvertOptions {
   // Conversion options
 
   /// Whether to check UTF8 validity of string columns
   bool check_utf8 = true;
   /// Optional per-column types (disabling type inference on those columns)
   std::unordered_map<std::string, std::shared_ptr<DataType>> column_types;
   /// Recognized spellings for null values
   std::vector<std::string> null_values;
   /// Recognized spellings for boolean true values
   std::vector<std::string> true_values;
   /// Recognized spellings for boolean false values
   std::vector<std::string> false_values;
 
   /// Whether string / binary columns can have null values.
   ///
   /// If true, then strings in "null_values" are considered null for string columns.
   /// If false, then all strings are valid string values.
   bool strings_can_be_null = false;
 
   /// Whether quoted values can be null.
   ///
   /// If true, then strings in "null_values" are also considered null when they
   /// appear quoted in the CSV file. Otherwise, quoted values are never considered null.
   bool quoted_strings_can_be_null = true;
 
   /// Whether to try to automatically dict-encode string / binary data.
   /// If true, then when type inference detects a string or binary column,
   /// it is dict-encoded up to `auto_dict_max_cardinality` distinct values
   /// (per chunk), after which it switches to regular encoding.
   ///
   /// This setting is ignored for non-inferred columns (those in `column_types`).
   bool auto_dict_encode = false;
   int32_t auto_dict_max_cardinality = 50;
 
   /// Decimal point character for floating-point and decimal data
   char decimal_point = '.';
 
   // XXX Should we have a separate FilterOptions?
 
   /// If non-empty, indicates the names of columns from the CSV file that should
   /// be actually read and converted (in the vector's order).
   /// Columns not in this vector will be ignored.
   std::vector<std::string> include_columns;
   /// If false, columns in `include_columns` but not in the CSV file will error out.
   /// If true, columns in `include_columns` but not in the CSV file will produce
   /// a column of nulls (whose type is selected using `column_types`,
   /// or null by default)
   /// This option is ignored if `include_columns` is empty.
   bool include_missing_columns = false;
 
   /// User-defined timestamp parsers, using the virtual parser interface in
   /// arrow/util/value_parsing.h. More than one parser can be specified, and
   /// the CSV conversion logic will try parsing values starting from the
   /// beginning of this vector. If no parsers are specified, we use the default
   /// built-in ISO-8601 parser.
   std::vector<std::shared_ptr<TimestampParser>> timestamp_parsers;
 
   /// Create conversion options with default values, including conventional
   /// values for `null_values`, `true_values` and `false_values`
   static ConvertOptions Defaults();
 
   /// \brief Test that all set options are valid
   Status Validate() const;
 };
 
 struct ARROW_EXPORT ReadOptions {
   // Reader options
 
   /// Whether to use the global CPU thread pool
   bool use_threads = true;
 
   /// \brief Block size we request from the IO layer.
   ///
   /// This will determine multi-threading granularity as well as
   /// the size of individual record batches.
   /// Minimum valid value for block size is 1
   int32_t block_size = 1 << 20;  // 1 MB
 
   /// Number of header rows to skip (not including the row of column names, if any)
   int32_t skip_rows = 0;
 
   /// Number of rows to skip after the column names are read, if any
   int32_t skip_rows_after_names = 0;
 
   /// Column names for the target table.
   /// If empty, fall back on autogenerate_column_names.
   std::vector<std::string> column_names;
 
   /// Whether to autogenerate column names if `column_names` is empty.
   /// If true, column names will be of the form "f0", "f1"...
   /// If false, column names will be read from the first CSV row after `skip_rows`.
   bool autogenerate_column_names = false;
 
   /// Create read options with default values
   static ReadOptions Defaults();
 
   /// \brief Test that all set options are valid
   Status Validate() const;
 };
 
 /// \brief Quoting style for CSV writing
 enum class ARROW_EXPORT QuotingStyle {
   /// Only enclose values in quotes which need them, because their CSV rendering can
   /// contain quotes itself (e.g. strings or binary values)
   Needed,
   /// Enclose all valid values in quotes. Nulls are not quoted. May cause readers to
   /// interpret all values as strings if schema is inferred.
   AllValid,
   /// Do not enclose any values in quotes. Prevents values from containing quotes ("),
   /// cell delimiters (,) or line endings (\\r, \\n), (following RFC4180). If values
   /// contain these characters, an error is caused when attempting to write.
   None
 };
 
 struct ARROW_EXPORT WriteOptions {
   /// Whether to write an initial header line with column names
   bool include_header = true;
 
   /// \brief Maximum number of rows processed at a time
   ///
   /// The CSV writer converts and writes data in batches of N rows.
   /// This number can impact performance.
   int32_t batch_size = 1024;
 
   /// Field delimiter
   char delimiter = ',';
 
   /// \brief The string to write for null values. Quotes are not allowed in this string.
   std::string null_string;
 
   /// \brief IO context for writing.
   io::IOContext io_context;
 
   /// \brief The end of line character to use for ending rows
   std::string eol = "\n";
 
   /// \brief Quoting style
   QuotingStyle quoting_style = QuotingStyle::Needed;
 
   /// Create write options with default values
   static WriteOptions Defaults();
 
   /// \brief Test that all set options are valid
   Status Validate() const;
 };
 
 }  // namespace csv
 }  // namespace arrow

This page was automatically generated by the 2.3.7 LXR engine. The LXR team