athena/src/FileLoaderHelper.h

0001 #pragma once
0002
0003 #include <DD4hep/DetFactoryHelper.h>
0004 #include <DD4hep/Primitives.h>
0005 #include <DD4hep/Factories.h>
0006 #include <DD4hep/Printout.h>
0007
0008 #include <fmt/core.h>
0009
0010 #include <filesystem>
0011 #include <iostream>
0012 #include <cstdlib>
0013 #include <string>
0014
0015 namespace fs = std::filesystem;
0016
0017 using namespace dd4hep;
0018
0019 // Function to download files
0020 inline void
0021 EnsureFileFromURLExists(
0022   std::string url,
0023   std::string file,
0024   std::string cache = "",
0025   std::string cmd = "curl --retry 5 -f {0} -o {1}"
0026 ) {
0027   // parse cache for environment variables
0028   auto pos = std::string::npos;
0029   while ((pos = cache.find('$')) != std::string::npos) {
0030     auto after = cache.find_first_not_of(
0031       "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
0032       "abcdefghijklmnopqrstuvwxyz"
0033       "0123456789"
0034       "_",
0035       pos + 1);
0036     if (after == std::string::npos) after = cache.size(); // cache ends on env var
0037     const std::string env_name(cache.substr(pos + 1, after - pos - 1));
0038     auto env_ptr = std::getenv(env_name.c_str());
0039     const std::string env_value(env_ptr != nullptr ? env_ptr : "");
0040     cache.erase(pos, after - pos);
0041     cache.insert(pos, env_value);
0042     printout(INFO, "FileLoader", "$" + env_name + " -> " + env_value);
0043   }
0044
0045   // create file path
0046   fs::path file_path(file);
0047
0048   // create hash from url, hex of unsigned long long
0049   std::string hash = fmt::format("{:016x}", dd4hep::detail::hash64(url)); // TODO: Use c++20 std::fmt
0050
0051   // create file parent path, if not exists
0052   fs::path parent_path = file_path.parent_path();
0053   if (!fs::exists(parent_path)) {
0054     if (fs::create_directories(parent_path) == false) {
0055       printout(ERROR, "FileLoader", "parent path " + parent_path.string() + " cannot be created");
0056       printout(ERROR, "FileLoader", "check permissions and retry");
0057       std::_Exit(EXIT_FAILURE);
0058     }
0059   }
0060
0061   // if file exists and is symlink to correct hash
0062   fs::path hash_path(parent_path / hash);
0063   if (fs::exists(file_path)
0064    && fs::equivalent(file_path, hash_path)) {
0065     printout(INFO, "FileLoader", "Link " + file + " -> hash " + hash + " already exists");
0066     return;
0067   }
0068
0069   // if hash does not exist, we try to retrieve file from cache
0070   if (!fs::exists(hash_path)) {
0071     // recursive loop into cache directory
0072     fs::path cache_path(cache);
0073     printout(INFO, "FileLoader", "Cache " + cache_path.string());
0074     if (fs::exists(cache_path)) {
0075       for (auto const& dir_entry: fs::recursive_directory_iterator(cache_path)) {
0076         if (!dir_entry.is_directory()) continue;
0077         fs::path cache_dir_path = cache_path / dir_entry;
0078         printout(INFO, "FileLoader", "Checking " + cache_dir_path.string());
0079         fs::path cache_hash_path = cache_dir_path / hash;
0080         if (fs::exists(cache_hash_path)) {
0081           // symlink hash to cache/.../hash
0082           printout(INFO, "FileLoader", "File " + file + " with hash " + hash + " found in " + cache_hash_path.string());
0083           try {
0084             fs::create_symlink(cache_hash_path, hash_path);
0085           } catch (const fs::filesystem_error&) {
0086             printout(ERROR, "FileLoader", "unable to link from " + hash_path.string() + " to " + cache_hash_path.string());
0087             printout(ERROR, "FileLoader", "check permissions and retry");
0088             std::_Exit(EXIT_FAILURE);
0089           }
0090           break;
0091         }
0092       }
0093     }
0094   }
0095
0096   // if hash does not exist, we try to retrieve file from url
0097   if (!fs::exists(hash_path)) {
0098     cmd = fmt::format(cmd, url, hash_path.c_str()); // TODO: Use c++20 std::fmt
0099     printout(INFO, "FileLoader", "Downloading " + file + " as hash " + hash + " with " + cmd);
0100     // run cmd
0101     auto ret = std::system(cmd.c_str());
0102     if (!fs::exists(hash_path)) {
0103       printout(ERROR, "FileLoader", "unable to run cmd " + cmd);
0104       printout(ERROR, "FileLoader", "check command and retry");
0105       printout(ERROR, "FileLoader", "hint: allow insecure connections with -k");
0106       std::_Exit(EXIT_FAILURE);
0107     }
0108   }
0109
0110   // check if file already exists
0111   if (fs::exists(file_path)) {
0112     // file already exists
0113     if (fs::is_symlink(file_path)) {
0114       // file is symlink
0115       if (fs::equivalent(hash_path, fs::read_symlink(file_path))) {
0116         // link points to correct path
0117         return;
0118       } else {
0119         // link points to incorrect path
0120         if (fs::remove(file_path) == false) {
0121           printout(ERROR, "FileLoader", "unable to remove symlink " + file_path.string());
0122           printout(ERROR, "FileLoader", "check permissions or remove manually");
0123           std::_Exit(EXIT_FAILURE);
0124         }
0125       }
0126     } else {
0127       // file exists but not symlink
0128       printout(ERROR, "FileLoader", "will not remove actual file " + file_path.string());
0129       printout(ERROR, "FileLoader", "check content, remove manually, and retry");
0130       std::_Exit(EXIT_FAILURE);
0131     }
0132   }
0133   // file_path now does not exist
0134
0135   // symlink file_path to hash_path
0136   try {
0137     // use new path from hash so file link is local
0138     fs::create_symlink(fs::path(hash), file_path);
0139   } catch (const fs::filesystem_error&) {
0140     printout(ERROR, "FileLoader", "unable to link from " + file_path.string() + " to " + hash_path.string());
0141     printout(ERROR, "FileLoader", "check permissions and retry");
0142     std::_Exit(EXIT_FAILURE);
0143   }
0144 }