epic/src/FileLoaderHelper.h

0001 // SPDX-License-Identifier: LGPL-3.0-or-later
0002 // Copyright (C) 2022 Wouter Deconinck
0003
0004 #pragma once
0005
0006 #include <DD4hep/DetFactoryHelper.h>
0007 #include <DD4hep/Factories.h>
0008 #include <DD4hep/Primitives.h>
0009 #include <DD4hep/Printout.h>
0010
0011 #include <fmt/core.h>
0012
0013 #include <cstdlib>
0014 #include <filesystem>
0015 #include <iostream>
0016 #include <regex>
0017 #include <string>
0018
0019 namespace fs = std::filesystem;
0020
0021 using dd4hep::ERROR, dd4hep::WARNING, dd4hep::VERBOSE, dd4hep::INFO;
0022 using dd4hep::printout;
0023
0024 namespace FileLoaderHelper {
0025 static constexpr const char* const kCommand = "curl --retry 5 --location --fail {0} --output {1}";
0026 }
0027
0028 // Function to download files
0029 inline void EnsureFileFromURLExists(std::string url, std::string file, std::string cache_str = "") {
0030   // parse cache for environment variables
0031   auto pos = std::string::npos;
0032   while ((pos = cache_str.find('$')) != std::string::npos) {
0033     auto after = cache_str.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
0034                                              "abcdefghijklmnopqrstuvwxyz"
0035                                              "0123456789"
0036                                              "_",
0037                                              pos + 1);
0038     if (after == std::string::npos)
0039       after = cache_str.size(); // cache ends on env var
0040     const std::string env_name(cache_str.substr(pos + 1, after - pos - 1));
0041     auto env_ptr = std::getenv(env_name.c_str());
0042     const std::string env_value(env_ptr != nullptr ? env_ptr : "");
0043     cache_str.erase(pos, after - pos);
0044     cache_str.insert(pos, env_value);
0045     printout(INFO, "FileLoader", "$" + env_name + " -> " + env_value);
0046   }
0047
0048   // tokenize cache on regex
0049   std::regex cache_sep(":");
0050   std::sregex_token_iterator cache_iter(cache_str.begin(), cache_str.end(), cache_sep, -1);
0051   std::sregex_token_iterator cache_end;
0052   std::vector<std::string> cache_vec(cache_iter, cache_end);
0053
0054   // create file path
0055   fs::path file_path(file);
0056
0057   // create hash from url, hex of unsigned long long
0058   std::string hash =
0059       fmt::format("{:016x}", dd4hep::detail::hash64(url)); // TODO: Use c++20 std::fmt
0060
0061   // create file parent path, if not exists
0062   fs::path parent_path = file_path.parent_path();
0063   if (!fs::exists(parent_path)) {
0064     if (fs::create_directories(parent_path) == false) {
0065       printout(ERROR, "FileLoader", "parent path " + parent_path.string() + " cannot be created");
0066       printout(ERROR, "FileLoader", "hint: try running 'mkdir -p " + parent_path.string() + "'");
0067       std::_Exit(EXIT_FAILURE);
0068     }
0069   }
0070
0071   // if file exists and is symlink to correct hash
0072   fs::path hash_path(parent_path / hash);
0073   if (fs::exists(file_path) && fs::exists(hash_path) && fs::equivalent(file_path, hash_path)) {
0074     printout(INFO, "FileLoader", "link " + file + " -> hash " + hash + " already exists");
0075     return;
0076   }
0077
0078   if (fs::exists(fs::symlink_status(hash_path))) {
0079     printout(INFO, "FileLoader", "removing symlink \"" + hash_path.string() + "\"");
0080     remove(hash_path);
0081   }
0082
0083   // if hash does not exist, we try to retrieve file from cache
0084   if (!fs::exists(hash_path)) {
0085     // recursive loop into cache directories
0086     bool success = false;
0087     for (auto cache : cache_vec) {
0088       fs::path cache_path(cache);
0089       printout(INFO, "FileLoader", "cache " + cache_path.string());
0090       if (fs::exists(cache_path)) {
0091         auto check_path = [&](const fs::path& cache_dir_path) {
0092           printout(VERBOSE, "FileLoader", "checking " + cache_dir_path.string());
0093           fs::path cache_hash_path = cache_dir_path / hash;
0094           if (fs::exists(cache_hash_path)) {
0095             // symlink hash to cache/.../hash
0096             printout(VERBOSE, "FileLoader",
0097                      "file " + file + " with hash " + hash + " found in " +
0098                          cache_hash_path.string());
0099             fs::path link_target;
0100             if (cache_hash_path.is_absolute()) {
0101               link_target = cache_hash_path;
0102             } else {
0103               link_target = fs::proximate(cache_hash_path, parent_path);
0104             }
0105             try {
0106               fs::create_symlink(link_target, hash_path);
0107               success = true;
0108             } catch (const fs::filesystem_error&) {
0109               printout(ERROR, "FileLoader",
0110                        "unable to link from " + hash_path.string() + " to " + link_target.string());
0111               std::_Exit(EXIT_FAILURE);
0112             }
0113             return true;
0114           }
0115           return false;
0116         };
0117         if (!check_path(cache_path)) {
0118           for (auto const& dir_entry : fs::recursive_directory_iterator(cache_path)) {
0119             if (!dir_entry.is_directory())
0120               continue;
0121             if (check_path(dir_entry.path())) {
0122               break;
0123             };
0124           }
0125         }
0126       }
0127       if (success)
0128         break;
0129     }
0130   }
0131
0132   // if hash does not exist, we try to retrieve file from url
0133   if (!fs::exists(hash_path)) {
0134     std::string cmd =
0135         fmt::format(FileLoaderHelper::kCommand, url, hash_path.c_str()); // TODO: Use c++20 std::fmt
0136     printout(INFO, "FileLoader", "downloading " + file + " as hash " + hash + " with " + cmd);
0137     // run cmd
0138     auto ret = std::system(cmd.c_str());
0139     if (!fs::exists(hash_path)) {
0140       printout(ERROR, "FileLoader", "unable to run the download command " + cmd);
0141       printout(ERROR, "FileLoader", "the return value was ", ret);
0142       printout(ERROR, "FileLoader", "hint: check the command and try running manually");
0143       printout(ERROR, "FileLoader",
0144                "hint: allow insecure connections on some systems with the flag -k");
0145       std::_Exit(EXIT_FAILURE);
0146     }
0147   }
0148
0149   // check if file already exists
0150   if (fs::exists(file_path)) {
0151     // file already exists
0152     if (fs::is_symlink(file_path)) {
0153       // file is symlink
0154       fs::path symlink_target = fs::read_symlink(file_path);
0155       if (fs::exists(symlink_target) && fs::equivalent(hash_path, symlink_target)) {
0156         // link points to correct path
0157         return;
0158       } else {
0159         // link points to incorrect path
0160         if (fs::remove(file_path) == false) {
0161           printout(ERROR, "FileLoader", "unable to remove symlink " + file_path.string());
0162           printout(ERROR, "FileLoader",
0163                    "we tried to create a symlink " + file_path.string() +
0164                        " to the actual resource, " +
0165                        "but a symlink already exists there and points to an incorrect location");
0166           printout(ERROR, "FileLoader",
0167                    "hint: this may be resolved by removing directory " + parent_path.string());
0168           printout(ERROR, "FileLoader",
0169                    "hint: or in that directory removing the file or link " + file_path.string());
0170           std::_Exit(EXIT_FAILURE);
0171         }
0172       }
0173     } else {
0174       // file exists but not symlink
0175       printout(ERROR, "FileLoader",
0176                "file " + file_path.string() + " already exists but is not a symlink");
0177       printout(ERROR, "FileLoader",
0178                "we tried to create a symlink " + file_path.string() + " to the actual resource, " +
0179                    "but a file already exists there and we will not remove it automatically");
0180       printout(ERROR, "FileLoader", "hint: backup the file, remove it manually, and retry");
0181       std::_Exit(EXIT_FAILURE);
0182     }
0183   }
0184   // file_path now does not exist
0185
0186   // symlink file_path to hash_path
0187   try {
0188     // use new path from hash so file link is local
0189     fs::create_symlink(fs::path(hash), file_path);
0190   } catch (const fs::filesystem_error&) {
0191     printout(ERROR, "FileLoader",
0192              "unable to link from " + file_path.string() + " to " + hash_path.string());
0193     printout(ERROR, "FileLoader", "check permissions and retry");
0194     std::_Exit(EXIT_FAILURE);
0195   }
0196
0197   // final check of the file size
0198   if (fs::file_size(file_path) == 0) {
0199     printout(ERROR, "FileLoader",
0200              "zero file size of symlink from " + file_path.string() + " to (ultimately) " +
0201                  fs::canonical(file_path).string());
0202     printout(ERROR, "FileLoader",
0203              "hint: check whether the file " + fs::canonical(file_path).string() +
0204                  " has any content");
0205     printout(ERROR, "FileLoader", "hint: check whether the URL " + url + " has any content");
0206     std::_Exit(EXIT_FAILURE);
0207   }
0208 }