EIC code displayed by LXR |
|
|
|
|||
File indexing completed on 2026-01-08 10:33:39
0001 #ifndef __XRDPFC_INFO_HH__ 0002 #define __XRDPFC_INFO_HH__ 0003 //---------------------------------------------------------------------------------- 0004 // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University 0005 // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman 0006 //---------------------------------------------------------------------------------- 0007 // XRootD is free software: you can redistribute it and/or modify 0008 // it under the terms of the GNU Lesser General Public License as published by 0009 // the Free Software Foundation, either version 3 of the License, or 0010 // (at your option) any later version. 0011 // 0012 // XRootD is distributed in the hope that it will be useful, 0013 // but WITHOUT ANY WARRANTY; without even the implied warranty of 0014 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 0015 // GNU General Public License for more details. 0016 // 0017 // You should have received a copy of the GNU Lesser General Public License 0018 // along with XRootD. If not, see <http://www.gnu.org/licenses/>. 0019 //---------------------------------------------------------------------------------- 0020 0021 #include "XrdPfcTypes.hh" 0022 0023 #include <cstdio> 0024 #include <ctime> 0025 #include <cassert> 0026 #include <vector> 0027 0028 class XrdOssDF; 0029 class XrdCksCalc; 0030 class XrdSysTrace; 0031 0032 namespace XrdPfc 0033 { 0034 class Stats; 0035 0036 //---------------------------------------------------------------------------- 0037 //! Status of cached file. Can be read from and written into a binary file. 0038 //---------------------------------------------------------------------------- 0039 0040 class Info 0041 { 0042 public: 0043 struct Status { 0044 union { 0045 struct { 0046 int f_cksum_check : 3; //!< as in enum CkSumCheck_e 0047 0048 int _free_bits_ : 29; 0049 }; 0050 unsigned int _raw_; 0051 }; 0052 Status() : _raw_(0) {} 0053 }; 0054 0055 //! Access statistics 0056 struct AStat 0057 { 0058 time_t AttachTime; //!< open time 0059 time_t DetachTime; //!< close time 0060 int NumIos; //!< number of IO objects attached during this access 0061 int Duration; //!< total duration of all IOs attached 0062 int NumMerged; //!< number of times the record has been merged 0063 int Reserved; //!< reserved / alignment 0064 long long BytesHit; //!< read from cache 0065 long long BytesMissed; //!< read from remote and cached 0066 long long BytesBypassed; //!< read from remote and dropped 0067 0068 AStat() : 0069 AttachTime(0), DetachTime(0), NumIos(0), Duration(0), NumMerged(0), Reserved(0), 0070 BytesHit(0), BytesMissed(0), BytesBypassed(0) 0071 {} 0072 0073 void MergeWith(const AStat &a); 0074 }; 0075 0076 struct Store 0077 { 0078 long long m_buffer_size; //!< buffer / block size 0079 long long m_file_size; //!< size of file in bytes 0080 time_t m_creationTime; //!< time the info file was created 0081 time_t m_noCkSumTime; //!< time when first non-cksummed block was detected 0082 size_t m_accessCnt; //!< total access count for the file 0083 Status m_status; //!< status information 0084 int m_astatSize; //!< size of AStat vector 0085 0086 Store () : 0087 m_buffer_size(0), m_file_size(0), m_creationTime(0), m_noCkSumTime(0), 0088 m_accessCnt(0), m_astatSize(0) 0089 {} 0090 }; 0091 0092 0093 //------------------------------------------------------------------------ 0094 //! Constructor. 0095 //------------------------------------------------------------------------ 0096 Info(XrdSysTrace* trace, bool prefetchBuffer = false); 0097 0098 //------------------------------------------------------------------------ 0099 //! Destructor. 0100 //------------------------------------------------------------------------ 0101 ~Info(); 0102 0103 //--------------------------------------------------------------------- 0104 //! Mark block as written to disk 0105 //--------------------------------------------------------------------- 0106 void SetBitWritten(int i); 0107 0108 //--------------------------------------------------------------------- 0109 //! Test if block at the given index is written to disk 0110 //--------------------------------------------------------------------- 0111 bool TestBitWritten(int i) const; 0112 0113 //--------------------------------------------------------------------- 0114 //! Test if block at the given index has been prefetched 0115 //--------------------------------------------------------------------- 0116 bool TestBitPrefetch(int i) const; 0117 0118 //--------------------------------------------------------------------- 0119 //! Mark block as obtained through prefetch 0120 //--------------------------------------------------------------------- 0121 void SetBitPrefetch(int i); 0122 0123 //--------------------------------------------------------------------- 0124 //! Mark block as synced to disk 0125 //--------------------------------------------------------------------- 0126 void SetBitSynced(int i); 0127 0128 //--------------------------------------------------------------------- 0129 //! Mark all blocks as synced to disk 0130 //--------------------------------------------------------------------- 0131 void SetAllBitsSynced(); 0132 0133 void SetBufferSizeFileSizeAndCreationTime(long long bs, long long fs); 0134 0135 //--------------------------------------------------------------------- 0136 //! \brief Reserve bit vectors for file_size / buffer_size bytes. 0137 //--------------------------------------------------------------------- 0138 void ResizeBits(); 0139 0140 //--------------------------------------------------------------------- 0141 //! \brief Read content of cinfo file into this object 0142 //! @param fp file handle 0143 //! @param dname directory name for trace output 0144 //! @param fname optional file name for trace output (can be included in dname) 0145 //! @return true on success 0146 //--------------------------------------------------------------------- 0147 bool Read(XrdOssDF* fp, const char *dname, const char *fname = 0); 0148 0149 //--------------------------------------------------------------------- 0150 //! Write number of blocks and read buffer size 0151 //! @param fp file handle 0152 //! @param dname directory name for trace output 0153 //! @param fname optional file name for trace output (can be included in dname) 0154 //! @return true on success 0155 //--------------------------------------------------------------------- 0156 bool Write(XrdOssDF* fp, const char *dname, const char *fname = 0); 0157 0158 //--------------------------------------------------------------------- 0159 //! Compactify access records to the configured maximum. 0160 //--------------------------------------------------------------------- 0161 void CompactifyAccessRecords(); 0162 0163 //--------------------------------------------------------------------- 0164 //! Reset IO Stats 0165 //--------------------------------------------------------------------- 0166 void ResetAllAccessStats(); 0167 0168 //--------------------------------------------------------------------- 0169 //! Write open time in the last entry of access statistics 0170 //--------------------------------------------------------------------- 0171 void WriteIOStatAttach(); 0172 0173 //--------------------------------------------------------------------- 0174 //! Write bytes missed, hits, and disk 0175 //--------------------------------------------------------------------- 0176 void WriteIOStat(Stats& s); 0177 0178 //--------------------------------------------------------------------- 0179 //! Write close time together with bytes missed, hits, and disk 0180 //--------------------------------------------------------------------- 0181 void WriteIOStatDetach(Stats& s); 0182 0183 //--------------------------------------------------------------------- 0184 //! Write single open/close time for given bytes read from disk. 0185 //--------------------------------------------------------------------- 0186 void WriteIOStatSingle(long long bytes_disk); 0187 0188 //--------------------------------------------------------------------- 0189 //! Write open/close with given time and bytes read from disk. 0190 //--------------------------------------------------------------------- 0191 void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc); 0192 0193 //--------------------------------------------------------------------- 0194 //! Check download status in given block range 0195 //--------------------------------------------------------------------- 0196 int CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const; 0197 0198 //--------------------------------------------------------------------- 0199 //! Get size of download-state bit-vector in bytes. 0200 //--------------------------------------------------------------------- 0201 int GetBitvecSizeInBytes() const; 0202 0203 //--------------------------------------------------------------------- 0204 //! Get number of blocks represented in download-state bit-vector. 0205 //--------------------------------------------------------------------- 0206 int GetNBlocks() const; 0207 0208 //--------------------------------------------------------------------- 0209 //! Get file size 0210 //--------------------------------------------------------------------- 0211 long long GetFileSize() const; 0212 0213 //--------------------------------------------------------------------- 0214 //! Get latest detach time 0215 //--------------------------------------------------------------------- 0216 bool GetLatestDetachTime(time_t& t) const; 0217 0218 //--------------------------------------------------------------------- 0219 //! Get latest access stats 0220 //--------------------------------------------------------------------- 0221 const AStat* GetLastAccessStats() const; 0222 0223 //--------------------------------------------------------------------- 0224 //! Get prefetch buffer size 0225 //--------------------------------------------------------------------- 0226 long long GetBufferSize() const; 0227 0228 //--------------------------------------------------------------------- 0229 //! Get complete status 0230 //--------------------------------------------------------------------- 0231 bool IsComplete() const; 0232 0233 //--------------------------------------------------------------------- 0234 //! Get number of downloaded blocks 0235 //--------------------------------------------------------------------- 0236 int GetNDownloadedBlocks() const; 0237 0238 //--------------------------------------------------------------------- 0239 //! Get number of downloaded bytes 0240 //--------------------------------------------------------------------- 0241 long long GetNDownloadedBytes() const; 0242 0243 //--------------------------------------------------------------------- 0244 //! Get number of the last downloaded block 0245 //--------------------------------------------------------------------- 0246 int GetLastDownloadedBlock() const; 0247 0248 //--------------------------------------------------------------------- 0249 //! Get expected data file size 0250 //--------------------------------------------------------------------- 0251 long long GetExpectedDataFileSize() const; 0252 0253 //--------------------------------------------------------------------- 0254 //! Update complete status 0255 //--------------------------------------------------------------------- 0256 void UpdateDownloadCompleteStatus(); 0257 0258 //--------------------------------------------------------------------- 0259 //! Get number of accesses 0260 //--------------------------------------------------------------------- 0261 size_t GetAccessCnt() const { return m_store.m_accessCnt; } 0262 0263 //--------------------------------------------------------------------- 0264 //! Get version 0265 //--------------------------------------------------------------------- 0266 int GetVersion() { return m_version; } 0267 0268 //--------------------------------------------------------------------- 0269 //! Get stored data 0270 //--------------------------------------------------------------------- 0271 const Store& RefStoredData() const { return m_store; } 0272 const std::vector<AStat>& RefAStats() const { return m_astats; } 0273 0274 //--------------------------------------------------------------------- 0275 //! Get file size 0276 //--------------------------------------------------------------------- 0277 time_t GetCreationTime() const { return m_store.m_creationTime; } 0278 0279 //--------------------------------------------------------------------- 0280 //! Get cksum, MD5 is for backward compatibility with V2 and V3. 0281 //--------------------------------------------------------------------- 0282 uint32_t CalcCksumStore(); 0283 uint32_t CalcCksumSyncedAndAStats(); 0284 void CalcCksumMd5(unsigned char* buff, char* digest); 0285 0286 CkSumCheck_e GetCkSumState() const { return (CkSumCheck_e) m_store.m_status.f_cksum_check; } 0287 const char* GetCkSumStateAsText() const; 0288 0289 bool IsCkSumCache() const { return m_store.m_status.f_cksum_check & CSChk_Cache; } 0290 bool IsCkSumNet() const { return m_store.m_status.f_cksum_check & CSChk_Net; } 0291 bool IsCkSumAny() const { return m_store.m_status.f_cksum_check & CSChk_Both; } 0292 bool IsCkSumBoth() const { return (m_store.m_status.f_cksum_check & CSChk_Both) == CSChk_Both; } 0293 0294 void SetCkSumState(CkSumCheck_e css) { m_store.m_status.f_cksum_check = css; } 0295 void DowngradeCkSumState(CkSumCheck_e css_ref) { m_store.m_status.f_cksum_check &= css_ref; } 0296 void ResetCkSumCache(); 0297 void ResetCkSumNet(); 0298 0299 bool HasNoCkSumTime() const { return m_store.m_noCkSumTime != 0; } 0300 time_t GetNoCkSumTime() const { return m_store.m_noCkSumTime; } 0301 time_t GetNoCkSumTimeForUVKeep() const { return m_store.m_noCkSumTime ? m_store.m_noCkSumTime : m_store.m_creationTime; } 0302 void ResetNoCkSumTime() { m_store.m_noCkSumTime = 0; } 0303 0304 #ifdef XRDPFC_CKSUM_TEST 0305 static void TestCksumStuff(); 0306 #endif 0307 0308 static const char* m_traceID; // has to be m_ (convention in TRACE macros) 0309 static const char* s_infoExtension; 0310 static const size_t s_infoExtensionLen; 0311 static size_t s_maxNumAccess; // can be set from configuration 0312 static const int s_defaultVersion; 0313 0314 XrdSysTrace* GetTrace() const {return m_trace; } 0315 0316 protected: 0317 XrdSysTrace* m_trace; 0318 0319 Store m_store; 0320 unsigned char *m_buff_synced; //!< disk written state vector 0321 unsigned char *m_buff_written; //!< download state vector 0322 unsigned char *m_buff_prefetch; //!< prefetch statistics 0323 std::vector<AStat> m_astats; //!< access records 0324 0325 int m_version; 0326 int m_bitvecSizeInBits; //!< cached 0327 int m_missingBlocks; //!< cached, updated in SetBitWritten() 0328 bool m_complete; //!< cached; if false, set to true when missingBlocks hit zero 0329 bool m_hasPrefetchBuffer; //!< constains current prefetch score 0330 0331 private: 0332 inline unsigned char cfiBIT(int n) const { return 1 << n; } 0333 0334 // Reading functions for older cinfo file formats 0335 bool ReadV2(XrdOssDF* fp, off_t off, const char *dname, const char *fname); 0336 bool ReadV3(XrdOssDF* fp, off_t off, const char *dname, const char *fname); 0337 0338 XrdCksCalc* m_cksCalcMd5; 0339 }; 0340 0341 //------------------------------------------------------------------------------ 0342 0343 inline bool Info::TestBitWritten(int i) const 0344 { 0345 const int cn = i/8; 0346 assert(cn < GetBitvecSizeInBytes()); 0347 0348 const int off = i - cn*8; 0349 return (m_buff_written[cn] & cfiBIT(off)) != 0; 0350 } 0351 0352 inline void Info::SetBitWritten(int i) 0353 { 0354 const int cn = i/8; 0355 assert(cn < GetBitvecSizeInBytes()); 0356 0357 const int off = i - cn*8; 0358 0359 m_buff_written[cn] |= cfiBIT(off); 0360 0361 if (--m_missingBlocks == 0) 0362 m_complete = true; 0363 } 0364 0365 inline void Info::SetBitPrefetch(int i) 0366 { 0367 if (!m_buff_prefetch) return; 0368 0369 const int cn = i/8; 0370 assert(cn < GetBitvecSizeInBytes()); 0371 0372 const int off = i - cn*8; 0373 m_buff_prefetch[cn] |= cfiBIT(off); 0374 } 0375 0376 inline bool Info::TestBitPrefetch(int i) const 0377 { 0378 if (!m_buff_prefetch) return false; 0379 0380 const int cn = i/8; 0381 assert(cn < GetBitvecSizeInBytes()); 0382 0383 const int off = i - cn*8; 0384 return (m_buff_prefetch[cn] & cfiBIT(off)) != 0; 0385 } 0386 0387 inline void Info::SetBitSynced(int i) 0388 { 0389 const int cn = i/8; 0390 assert(cn < GetBitvecSizeInBytes()); 0391 0392 const int off = i - cn*8; 0393 m_buff_synced[cn] |= cfiBIT(off); 0394 } 0395 0396 //------------------------------------------------------------------------------ 0397 0398 inline int Info::GetNDownloadedBlocks() const 0399 { 0400 int cntd = 0; 0401 for (int i = 0; i < m_bitvecSizeInBits; ++i) 0402 if (TestBitWritten(i)) cntd++; 0403 0404 return cntd; 0405 } 0406 0407 inline long long Info::GetNDownloadedBytes() const 0408 { 0409 return m_store.m_buffer_size * GetNDownloadedBlocks(); 0410 } 0411 0412 inline int Info::GetLastDownloadedBlock() const 0413 { 0414 for (int i = m_bitvecSizeInBits - 1; i >= 0; --i) 0415 if (TestBitWritten(i)) return i; 0416 0417 return -1; 0418 } 0419 0420 inline long long Info::GetExpectedDataFileSize() const 0421 { 0422 int last_block = GetLastDownloadedBlock(); 0423 if (last_block == m_bitvecSizeInBits - 1) 0424 return m_store.m_file_size; 0425 else 0426 return (last_block + 1) * m_store.m_buffer_size; 0427 } 0428 0429 inline int Info::GetBitvecSizeInBytes() const 0430 { 0431 if (m_bitvecSizeInBits) 0432 return ((m_bitvecSizeInBits - 1)/8 + 1); 0433 else 0434 return 0; 0435 } 0436 0437 inline int Info::GetNBlocks() const 0438 { 0439 return m_bitvecSizeInBits; 0440 } 0441 0442 inline long long Info::GetFileSize() const 0443 { 0444 return m_store.m_file_size; 0445 } 0446 0447 inline bool Info::IsComplete() const 0448 { 0449 return m_complete; 0450 } 0451 0452 inline int Info::CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const 0453 { 0454 // TODO rewrite to use full byte comparisons outside of edges ? 0455 // Also, it seems to be always called with firstIdx = 0, lastIdx = m_bitvecSizeInBits. 0456 int cnt = 0; 0457 for (int i = firstIdx; i < lastIdx; ++i) 0458 if (! TestBitWritten(i)) ++cnt; 0459 0460 return cnt; 0461 } 0462 0463 inline void Info::UpdateDownloadCompleteStatus() 0464 { 0465 m_missingBlocks = CountBlocksNotWrittenInRng(0, m_bitvecSizeInBits); 0466 m_complete = (m_missingBlocks == 0); 0467 } 0468 0469 inline long long Info::GetBufferSize() const 0470 { 0471 return m_store.m_buffer_size; 0472 } 0473 0474 } 0475 #endif
| [ Source navigation ] | [ Diff markup ] | [ Identifier search ] | [ general search ] |
|
This page was automatically generated by the 2.3.7 LXR engine. The LXR team |
|