src/detail/JsonIo.cpp

0001 // This file is part of the ACTS project.
0002 //
0003 // Copyright (C) 2016 CERN for the benefit of the ACTS project
0004 //
0005 // This Source Code Form is subject to the terms of the Mozilla Public
0006 // License, v. 2.0. If a copy of the MPL was not distributed with this
0007 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
0008
0009 #include "ActsPlugins/Json/detail/JsonIo.hpp"
0010
0011 #include <algorithm>
0012 #include <array>
0013 #include <format>
0014 #include <fstream>
0015 #include <stdexcept>
0016 #include <string>
0017 #include <string_view>
0018
0019 #ifdef ACTS_JSON_ZSTD_SUPPORT
0020 #include <zstd.h>
0021 #endif
0022
0023 namespace {
0024
0025 using Acts::detail::JsonCompression;
0026 using Acts::detail::JsonEncoding;
0027 using Acts::detail::JsonFileFormat;
0028
0029 /// zstd frame magic number 0xFD2FB528, little endian as it appears on disk.
0030 constexpr std::array<std::byte, 4> kZstdMagic{std::byte{0x28}, std::byte{0xB5},
0031                                               std::byte{0x2F}, std::byte{0xFD}};
0032
0033 bool hasZstdMagic(std::span<const std::byte> data) {
0034   return data.size() >= kZstdMagic.size() &&
0035          std::equal(kZstdMagic.begin(), kZstdMagic.end(), data.begin());
0036 }
0037
0038 /// Copy any contiguous byte-like range into a byte buffer.
0039 template <typename container_t>
0040 std::vector<std::byte> toBytes(const container_t& container) {
0041   auto bytes = std::as_bytes(std::span{container});
0042   return {bytes.begin(), bytes.end()};
0043 }
0044
0045 std::string_view asStringView(std::span<const std::byte> data) {
0046   return {reinterpret_cast<const char*>(data.data()), data.size()};
0047 }
0048
0049 std::vector<std::byte> compressZstd(std::span<const std::byte> data,
0050                                     int level) {
0051 #ifdef ACTS_JSON_ZSTD_SUPPORT
0052   std::vector<std::byte> out(ZSTD_compressBound(data.size()));
0053   std::size_t written =
0054       ZSTD_compress(out.data(), out.size(), data.data(), data.size(), level);
0055   if (ZSTD_isError(written) != 0u) {
0056     throw std::runtime_error(
0057         std::format("zstd compression failed: {}", ZSTD_getErrorName(written)));
0058   }
0059   out.resize(written);
0060   return out;
0061 #else
0062   static_cast<void>(data);
0063   static_cast<void>(level);
0064   throw std::runtime_error(
0065       "zstd compressed output was requested, but ACTS was built without zstd "
0066       "support. Reconfigure with zstd available to write compressed files.");
0067 #endif
0068 }
0069
0070 std::vector<std::byte> decompressZstd(std::span<const std::byte> data,
0071                                       const std::filesystem::path& origin) {
0072 #ifdef ACTS_JSON_ZSTD_SUPPORT
0073   unsigned long long size = ZSTD_getFrameContentSize(data.data(), data.size());
0074   if (size == ZSTD_CONTENTSIZE_ERROR) {
0075     throw std::runtime_error(std::format(
0076         "'{}' starts with a zstd magic number but is not a valid zstd frame",
0077         origin.string()));
0078   }
0079   if (size == ZSTD_CONTENTSIZE_UNKNOWN) {
0080     throw std::runtime_error(std::format(
0081         "'{}' is a zstd stream without a declared content size, which is not "
0082         "supported here; recompress it as a single frame",
0083         origin.string()));
0084   }
0085   std::vector<std::byte> out(size);
0086   std::size_t written =
0087       ZSTD_decompress(out.data(), out.size(), data.data(), data.size());
0088   if (ZSTD_isError(written) != 0u) {
0089     throw std::runtime_error(
0090         std::format("'{}' could not be decompressed, the zstd frame is likely "
0091                     "truncated or corrupt: {}",
0092                     origin.string(), ZSTD_getErrorName(written)));
0093   }
0094   out.resize(written);
0095   return out;
0096 #else
0097   static_cast<void>(data);
0098   throw std::runtime_error(std::format(
0099       "'{}' is zstd compressed, but ACTS was built without zstd support. "
0100       "Reconfigure with zstd available to read this file.",
0101       origin.string()));
0102 #endif
0103 }
0104
0105 }  // namespace
0106
0107 bool Acts::detail::zstdSupported() {
0108 #ifdef ACTS_JSON_ZSTD_SUPPORT
0109   return true;
0110 #else
0111   return false;
0112 #endif
0113 }
0114
0115 Acts::detail::JsonFileFormat Acts::detail::jsonFormatFromPath(
0116     const std::filesystem::path& path) {
0117   std::string name = path.filename().string();
0118   JsonFileFormat format{};
0119
0120   if (name.ends_with(".zst")) {
0121     format.compression = JsonCompression::Zstd;
0122     name.resize(name.size() - std::string_view{".zst"}.size());
0123   }
0124
0125   if (name.ends_with(".json")) {
0126     format.encoding = JsonEncoding::Text;
0127   } else if (name.ends_with(".cbor")) {
0128     format.encoding = JsonEncoding::Cbor;
0129   } else {
0130     throw std::invalid_argument(std::format(
0131         "Cannot deduce the output format of '{}': expected one of '.json', "
0132         "'.cbor', '.json.zst' or '.cbor.zst'",
0133         path.string()));
0134   }
0135
0136   return format;
0137 }
0138
0139 std::vector<std::byte> Acts::detail::encodeJson(const nlohmann::json& payload,
0140                                                 const JsonFileFormat& format,
0141                                                 unsigned indentation,
0142                                                 int compressionLevel) {
0143   std::vector<std::byte> encoded =
0144       format.encoding == JsonEncoding::Cbor
0145           ? toBytes(nlohmann::json::to_cbor(payload))
0146           : toBytes(payload.dump(static_cast<int>(indentation)));
0147
0148   if (format.compression == JsonCompression::Zstd) {
0149     return compressZstd(encoded, compressionLevel);
0150   }
0151   return encoded;
0152 }
0153
0154 nlohmann::json Acts::detail::decodeJson(std::span<const std::byte> data,
0155                                         const std::filesystem::path& origin) {
0156   std::vector<std::byte> decompressed;
0157   bool wasCompressed = hasZstdMagic(data);
0158   if (wasCompressed) {
0159     decompressed = decompressZstd(data, origin);
0160     data = decompressed;
0161   }
0162
0163   auto isSpace = [](std::byte b) {
0164     char c = static_cast<char>(b);
0165     return c == ' ' || c == '\t' || c == '\n' || c == '\r';
0166   };
0167   auto it = std::ranges::find_if_not(data, isSpace);
0168   if (it == data.end()) {
0169     throw std::runtime_error(
0170         std::format("'{}' contains no payload to decode", origin.string()));
0171   }
0172
0173   // JSON text always starts with '{' or '['. Our CBOR payloads are always a
0174   // map or an array at the top level, i.e. major type 5 or 4, which cannot
0175   // collide with either of those two ASCII bytes.
0176   auto lead = static_cast<unsigned char>(*it);
0177   bool isText = lead == '{' || lead == '[';
0178   bool isCbor = (lead & 0xE0) == 0xA0 || (lead & 0xE0) == 0x80;
0179
0180   // A decompressed payload that does not parse points at the compressed file,
0181   // so say which stage the content came from.
0182   std::string_view stage = wasCompressed ? " (after decompression)" : "";
0183
0184   if (!isText && !isCbor) {
0185     throw std::runtime_error(std::format(
0186         "'{}'{} is neither JSON text nor CBOR, leading byte is 0x{:02X}",
0187         origin.string(), stage, lead));
0188   }
0189
0190   try {
0191     if (isText) {
0192       return nlohmann::json::parse(asStringView(data));
0193     }
0194     return nlohmann::json::from_cbor(data);
0195   } catch (const nlohmann::json::exception& e) {
0196     throw std::runtime_error(std::format(
0197         "Failed to decode {} payload from '{}'{}: {}", isText ? "JSON" : "CBOR",
0198         origin.string(), stage, e.what()));
0199   }
0200 }
0201
0202 void Acts::detail::writeJsonFile(const std::filesystem::path& path,
0203                                  const nlohmann::json& payload,
0204                                  unsigned indentation, int compressionLevel) {
0205   std::vector<std::byte> encoded = encodeJson(payload, jsonFormatFromPath(path),
0206                                               indentation, compressionLevel);
0207
0208   std::ofstream ofs{path, std::ios::binary};
0209   if (!ofs.good()) {
0210     throw std::runtime_error(
0211         std::format("Cannot open '{}' for writing", path.string()));
0212   }
0213   ofs.write(reinterpret_cast<const char*>(encoded.data()),
0214             static_cast<std::streamsize>(encoded.size()));
0215   if (!ofs.good()) {
0216     throw std::runtime_error(
0217         std::format("Failed to write '{}'", path.string()));
0218   }
0219 }
0220
0221 nlohmann::json Acts::detail::readJsonFile(const std::filesystem::path& path) {
0222   if (!std::filesystem::exists(path)) {
0223     throw std::invalid_argument(
0224         std::format("File '{}' does not exist", path.string()));
0225   }
0226
0227   std::ifstream ifs{path, std::ios::binary};
0228   if (!ifs.good()) {
0229     throw std::invalid_argument(std::format("Cannot open '{}'", path.string()));
0230   }
0231
0232   ifs.seekg(0, std::ios::end);
0233   auto size = ifs.tellg();
0234   if (size < 0) {
0235     throw std::runtime_error(
0236         std::format("Cannot determine the size of '{}'", path.string()));
0237   }
0238   ifs.seekg(0, std::ios::beg);
0239
0240   std::vector<std::byte> data(static_cast<std::size_t>(size));
0241   ifs.read(reinterpret_cast<char*>(data.data()),
0242            static_cast<std::streamsize>(data.size()));
0243   if (ifs.bad()) {
0244     throw std::runtime_error(std::format("Failed to read '{}'", path.string()));
0245   }
0246
0247   return decodeJson(data, path);
0248 }