epic/src/FileLoaderHelper.h

0001 // SPDX-License-Identifier: LGPL-3.0-or-later
0002 // Copyright (C) 2022 Wouter Deconinck
0003
0004 #pragma once
0005
0006 #include <DD4hep/DetFactoryHelper.h>
0007 #include <DD4hep/Factories.h>
0008 #include <DD4hep/Primitives.h>
0009 #include <DD4hep/Printout.h>
0010
0011 #include <fmt/core.h>
0012 #include <fmt/format.h>
0013
0014 #include <cstdlib>
0015 #include <filesystem>
0016 #include <iostream>
0017 #include <regex>
0018 #include <string>
0019
0020 namespace fs = std::filesystem;
0021
0022 using dd4hep::ERROR, dd4hep::WARNING, dd4hep::VERBOSE, dd4hep::INFO;
0023 using dd4hep::printout;
0024
0025 namespace FileLoaderHelper {
0026 static constexpr const char* const kCurlCommand =
0027     "curl --retry 5 --location --fail {0} --output {1}";
0028 static constexpr const char* const kXrootdCommand = "xrdcp --retry 5 {0} {1}";
0029 } // namespace FileLoaderHelper
0030
0031 // Function to download files
0032 inline void EnsureFileFromURLExists(std::string url, std::string file, std::string cache_str = "") {
0033   // parse cache for environment variables
0034   auto pos = std::string::npos;
0035   while ((pos = cache_str.find('$')) != std::string::npos) {
0036     auto after = cache_str.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
0037                                              "abcdefghijklmnopqrstuvwxyz"
0038                                              "0123456789"
0039                                              "_",
0040                                              pos + 1);
0041     if (after == std::string::npos)
0042       after = cache_str.size(); // cache ends on env var
0043     const std::string env_name(cache_str.substr(pos + 1, after - pos - 1));
0044     auto env_ptr = std::getenv(env_name.c_str());
0045     const std::string env_value(env_ptr != nullptr ? env_ptr : "");
0046     cache_str.erase(pos, after - pos);
0047     cache_str.insert(pos, env_value);
0048     printout(INFO, "FileLoader", "$" + env_name + " -> " + env_value);
0049   }
0050
0051   // tokenize cache on regex
0052   std::regex cache_sep(":");
0053   std::sregex_token_iterator cache_iter(cache_str.begin(), cache_str.end(), cache_sep, -1);
0054   std::sregex_token_iterator cache_end;
0055   std::vector<std::string> cache_vec(cache_iter, cache_end);
0056
0057   // create file path
0058   fs::path file_path(file);
0059
0060   // create hash from url, hex of unsigned long long
0061   std::string hash =
0062       fmt::format("{:016x}", dd4hep::detail::hash64(url)); // TODO: Use c++20 std::fmt
0063
0064   // create file parent path if not exists
0065   fs::path parent_path = file_path.parent_path();
0066   try {
0067     fs::create_directories(parent_path); // no-op (returns false) if already exists
0068   } catch (const fs::filesystem_error&) {
0069     printout(ERROR, "FileLoader", "parent path " + parent_path.string() + " cannot be created");
0070     printout(ERROR, "FileLoader", "hint: try running 'mkdir -p " + parent_path.string() + "'");
0071     std::exit(EXIT_FAILURE);
0072   }
0073
0074   // if file exists and is symlink to correct hash
0075   fs::path hash_path(parent_path / hash);
0076   if (fs::exists(file_path) && fs::exists(hash_path) && fs::equivalent(file_path, hash_path)) {
0077     printout(INFO, "FileLoader", "link " + file + " -> hash " + hash + " already exists");
0078     return;
0079   }
0080
0081   if (fs::exists(fs::symlink_status(hash_path))) {
0082     printout(INFO, "FileLoader", "removing symlink \"" + hash_path.string() + "\"");
0083     remove(hash_path);
0084   }
0085
0086   // if hash does not exist, we try to retrieve file from cache
0087   if (!fs::exists(hash_path)) {
0088     // recursive loop into cache directories
0089     bool success = false;
0090     for (auto cache : cache_vec) {
0091       fs::path cache_path(cache);
0092       printout(INFO, "FileLoader", "cache " + cache_path.string());
0093       if (fs::exists(cache_path)) {
0094         auto check_path = [&](const fs::path& cache_dir_path) {
0095           printout(VERBOSE, "FileLoader", "checking " + cache_dir_path.string());
0096           fs::path cache_hash_path = cache_dir_path / hash;
0097           if (fs::exists(cache_hash_path)) {
0098             // symlink hash to cache/.../hash
0099             printout(VERBOSE, "FileLoader",
0100                      "file " + file + " with hash " + hash + " found in " +
0101                          cache_hash_path.string());
0102             fs::path link_target;
0103             if (cache_hash_path.is_absolute()) {
0104               link_target = cache_hash_path;
0105             } else {
0106               link_target = fs::proximate(cache_hash_path, parent_path);
0107             }
0108             try {
0109               fs::create_symlink(link_target, hash_path);
0110               success = true;
0111             } catch (const fs::filesystem_error&) {
0112               const bool hash_path_exists = fs::exists(hash_path);
0113               if (hash_path_exists && fs::equivalent(hash_path, link_target)) {
0114                 // Another process created the correct symlink concurrently; that is fine
0115                 success = true;
0116               } else if (hash_path_exists) {
0117                 printout(ERROR, "FileLoader",
0118                          "symlink " + hash_path.string() +
0119                              " already exists but points to wrong target");
0120                 std::exit(EXIT_FAILURE);
0121               } else {
0122                 printout(ERROR, "FileLoader",
0123                          "unable to link from " + hash_path.string() + " to " +
0124                              link_target.string());
0125                 printout(ERROR, "FileLoader", "check permissions and retry");
0126                 std::exit(EXIT_FAILURE);
0127               }
0128             }
0129             return true;
0130           }
0131           return false;
0132         };
0133         if (!check_path(cache_path)) {
0134           for (auto const& dir_entry : fs::recursive_directory_iterator(cache_path)) {
0135             if (!dir_entry.is_directory())
0136               continue;
0137             if (check_path(dir_entry.path())) {
0138               break;
0139             };
0140           }
0141         }
0142       }
0143       if (success)
0144         break;
0145     }
0146   }
0147
0148   // if hash does not exist, we try to retrieve file from url
0149   if (!fs::exists(hash_path)) {
0150     std::string cmd;
0151
0152     if (url.find("root://") == 0) {
0153       cmd = fmt::format(FileLoaderHelper::kXrootdCommand, url, hash_path.c_str());
0154     } else {
0155       cmd = fmt::format(FileLoaderHelper::kCurlCommand, url, hash_path.c_str());
0156     }
0157     printout(INFO, "FileLoader", "downloading " + file + " as hash " + hash + " with " + cmd);
0158
0159     // run cmd
0160     auto ret = std::system(cmd.c_str());
0161     if (!fs::exists(hash_path)) {
0162       printout(ERROR, "FileLoader", "unable to run the download command " + cmd);
0163       printout(ERROR, "FileLoader", "the return value was ", ret);
0164       printout(ERROR, "FileLoader", "hint: check the command and try running manually");
0165       printout(ERROR, "FileLoader",
0166                "hint: allow insecure connections on some systems with the flag -k");
0167       std::exit(EXIT_FAILURE);
0168     }
0169   }
0170
0171   // check if file is symlink
0172   if (fs::is_symlink(file_path)) {
0173     // file is symlink, i.e. valid symlink
0174     if (fs::exists(file_path)) {
0175       // file already exists
0176       fs::path symlink_target = fs::read_symlink(file_path);
0177       if (fs::exists(symlink_target) && fs::equivalent(hash_path, symlink_target)) {
0178         // link points to correct path
0179         return;
0180       } else {
0181         // link points to incorrect path -> remove symlink
0182         if (fs::remove(file_path) == false) {
0183           // failure mode: cannot remove incorrect symlink
0184           printout(ERROR, "FileLoader", "unable to remove symlink " + file_path.string());
0185           printout(ERROR, "FileLoader",
0186                    "we tried to create a symlink " + file_path.string() +
0187                        " to the actual resource, " +
0188                        "but a symlink already exists there and points to an incorrect location");
0189           printout(ERROR, "FileLoader",
0190                    "hint: this may be resolved by removing directory " + parent_path.string());
0191           printout(ERROR, "FileLoader",
0192                    "hint: or in that directory removing the file or link " + file_path.string());
0193           std::exit(EXIT_FAILURE);
0194         }
0195       }
0196     } else {
0197       // file does not exists, i.e. dead symllink -> remove symlink
0198       if (fs::remove(file_path) == false) {
0199         // failure mode; cannot remove dead symlink
0200         printout(ERROR, "FileLoader", "unable to remove symlink " + file_path.string());
0201         std::exit(EXIT_FAILURE);
0202       }
0203     }
0204   } else {
0205     if (fs::exists(file_path)) {
0206       // failure mode: file exists but not symlink, and we won't remove files
0207       printout(ERROR, "FileLoader",
0208                "file " + file_path.string() + " already exists but is not a symlink");
0209       printout(ERROR, "FileLoader",
0210                "we tried to create a symlink " + file_path.string() + " to the actual resource, " +
0211                    "but a file already exists there and we will not remove it automatically");
0212       printout(ERROR, "FileLoader", "hint: backup the file, remove it manually, and retry");
0213       std::exit(EXIT_FAILURE);
0214     }
0215   }
0216   // file_path now does not exist
0217
0218   // symlink file_path to hash_path
0219   try {
0220     // use new path from hash so file link is local
0221     fs::create_symlink(fs::path(hash), file_path);
0222   } catch (const fs::filesystem_error&) {
0223     const bool file_path_exists = fs::exists(file_path);
0224     if (file_path_exists && fs::equivalent(file_path, hash_path)) {
0225       // Another process created the correct symlink concurrently; that is fine
0226     } else if (file_path_exists) {
0227       printout(ERROR, "FileLoader",
0228                "symlink " + file_path.string() + " already exists but points to wrong target");
0229       std::exit(EXIT_FAILURE);
0230     } else {
0231       printout(ERROR, "FileLoader",
0232                "unable to link from " + file_path.string() + " to " + hash_path.string());
0233       printout(ERROR, "FileLoader", "check permissions and retry");
0234       std::exit(EXIT_FAILURE);
0235     }
0236   }
0237
0238   // final check of the file size
0239   if (fs::file_size(file_path) == 0) {
0240     printout(ERROR, "FileLoader",
0241              "zero file size of symlink from " + file_path.string() + " to (ultimately) " +
0242                  fs::canonical(file_path).string());
0243     printout(ERROR, "FileLoader",
0244              "hint: check whether the file " + fs::canonical(file_path).string() +
0245                  " has any content");
0246     printout(ERROR, "FileLoader", "hint: check whether the URL " + url + " has any content");
0247     std::exit(EXIT_FAILURE);
0248   }
0249 }