Warning, /include/Geant4/tools/wroot/branch is written in an unsupported language. File is not indexed.
0001 // Copyright (C) 2010, Guy Barrand. All rights reserved.
0002 // See the file tools.license for terms.
0003
0004 #ifndef tools_wroot_branch
0005 #define tools_wroot_branch
0006
0007 #include "leaf"
0008 #include "basket"
0009 #include "../forit"
0010
0011 #include "imutex"
0012
0013 namespace tools {
0014 namespace wroot {
0015
0016 class branch : public virtual ibo {
0017 //static uint32 kDoNotProcess() {return (1<<10);} // Active bit for branches
0018 #ifdef TOOLS_MEM
0019 static const std::string& s_class() {
0020 static const std::string s_v("tools::wroot::branch");
0021 return s_v;
0022 }
0023 #endif
0024 public: //ibo
0025 virtual const std::string& store_cls() const {
0026 static const std::string s_v("TBranch");
0027 return s_v;
0028 }
0029 virtual bool stream(buffer& a_buffer) const {
0030 unsigned int c;
0031 if(!a_buffer.write_version(8,c)) return false;
0032 if(!Named_stream(a_buffer,m_name,m_title)) return false;
0033
0034 if(!AttFill_stream(a_buffer)) return false;
0035
0036 int fEntryOffsetLen = 1000;
0037 int fOffset = 0;
0038 int fSplitLevel = 0;
0039
0040 if(!a_buffer.write(fCompress)) return false;
0041 if(!a_buffer.write(m_basket_size)) return false;
0042 if(!a_buffer.write(fEntryOffsetLen)) return false;
0043 if(!a_buffer.write(m_write_basket)) return false;
0044 int fEntryNumber = (int)m_entry_number;
0045 if(!a_buffer.write(fEntryNumber)) return false;
0046 if(!a_buffer.write(fOffset)) return false;
0047 if(!a_buffer.write(m_max_baskets)) return false;
0048 if(!a_buffer.write(fSplitLevel)) return false;
0049 double fEntries = (double)m_entries;
0050 if(!a_buffer.write(fEntries)) return false;
0051 double fTotBytes = (double)m_tot_bytes;
0052 double fZipBytes = (double)m_zip_bytes;
0053 if(!a_buffer.write(fTotBytes)) return false;
0054 if(!a_buffer.write(fZipBytes)) return false;
0055
0056 if(!m_branches.stream(a_buffer)) return false;
0057 if(!m_leaves.stream(a_buffer)) return false;
0058 if(!m_baskets.stream(a_buffer)) return false;
0059
0060 /*
0061 {uint32 remaining_baskets = 0;
0062 for(uint32 i=0;i<m_max_baskets;i++) {
0063 if(m_baskets[i]) {
0064 m_out << "debug : remaining basket : index " << i << ", bytes = " << m_baskets[i]->number_of_bytes()
0065 << std::endl;
0066 remaining_baskets++;
0067 }
0068 }
0069 m_out << "tools::wroot::branch::stream :"
0070 << " write_basket = " << m_write_basket
0071 << ", max_baskets = " << m_max_baskets
0072 << ", remaining_baskets = " << remaining_baskets << "."
0073 << std::endl;}
0074 */
0075
0076 // See TStreamerInfo::ReadBuffer::WriteBasicPointer
0077 if(!a_buffer.write((char)1)) return false;
0078 if(!a_buffer.write_fast_array(fBasketBytes,m_max_baskets)) return false;
0079 if(!a_buffer.write((char)1)) return false;
0080 if(!a_buffer.write_fast_array(fBasketEntry,m_max_baskets)) return false;
0081
0082 char isBigFile = 1;
0083 //GB : begin
0084 //if(fTree.directory().file().end()>RIO_START_BIG_FILE()) isBigFile = 2;
0085 {for(uint32 i=0;i<m_max_baskets;i++) {
0086 if(fBasketSeek[i]>START_BIG_FILE()) {
0087 isBigFile = 2;
0088 break;
0089 }
0090 }}
0091 //GB : end
0092
0093 if(!a_buffer.write(isBigFile)) return false;
0094 if(isBigFile==2) {
0095 if(!a_buffer.write_fast_array(fBasketSeek,m_max_baskets)) return false;
0096 } else {
0097 for(uint32 i=0;i<m_max_baskets;i++) {
0098 if(fBasketSeek[i]>START_BIG_FILE()) { //G.Barrand : add this test.
0099 m_out << "tools::wroot::branch::stream :"
0100 << " attempt to write big Seek "
0101 << fBasketSeek[i] << " on 32 bits."
0102 << std::endl;
0103 return false;
0104 }
0105
0106 if(!a_buffer.write((seek32)fBasketSeek[i])) return false;
0107 }
0108 }
0109
0110 // fFileName
0111 if(!a_buffer.write(std::string(""))) return false;
0112
0113 if(!a_buffer.set_byte_count(c)) return false;
0114 return true;
0115 }
0116
0117 public:
0118 branch(std::ostream& a_out,bool a_byte_swap,uint32 a_compression,
0119 seek a_seek_directory,const std::string& a_name,const std::string& a_title,bool a_verbose)
0120 :m_out(a_out)
0121 ,m_byte_swap(a_byte_swap)
0122 ,m_verbose(a_verbose)
0123 ,m_seek_directory(a_seek_directory)
0124 ,m_name(a_name)
0125 ,m_title(a_title)
0126 ,fAutoDelete(false)
0127
0128 //,m_branches(true)
0129 //,m_leaves(true)
0130 ,fCompress(a_compression)
0131 ,m_basket_size(32000)
0132 ,m_write_basket(0)
0133 ,m_entry_number(0)
0134 ,m_entries(0)
0135 ,m_tot_bytes(0)
0136 ,m_zip_bytes(0)
0137 ,m_max_baskets(10)
0138 ,fBasketBytes(0)
0139 ,fBasketEntry(0)
0140 ,fBasketSeek(0)
0141 {
0142 #ifdef TOOLS_MEM
0143 mem::increment(s_class().c_str());
0144 #endif
0145 m_baskets.resize(m_max_baskets,0);
0146 fBasketBytes = new uint32[m_max_baskets];
0147 fBasketEntry = new uint32[m_max_baskets];
0148 fBasketSeek = new seek[m_max_baskets];
0149 {for(uint32 i=0;i<m_max_baskets;i++) {
0150 m_baskets[i] = 0;
0151 fBasketBytes[i] = 0;
0152 fBasketEntry[i] = 0;
0153 fBasketSeek[i] = 0;
0154 }}
0155 m_baskets[m_write_basket] = new basket(m_out,m_byte_swap,a_seek_directory,m_name,m_title,"TBasket",m_basket_size,m_verbose);
0156 fBasketEntry[m_write_basket] = (uint32)m_entry_number;
0157 }
0158 virtual ~branch(){
0159 // {for(uint32 i=0;i<=m_write_basket;i++) {
0160 // m_out << " " << i << ", " << fBasketEntry[i] << std::endl;
0161 // }}
0162
0163 delete [] fBasketBytes;
0164 delete [] fBasketEntry;
0165 delete [] fBasketSeek;
0166 fBasketBytes = 0;
0167 fBasketEntry = 0;
0168 fBasketSeek = 0;
0169
0170 #ifdef TOOLS_MEM
0171 mem::decrement(s_class().c_str());
0172 #endif
0173 }
0174 protected:
0175 branch(const branch& a_from)
0176 :ibo(a_from)
0177 ,m_out(a_from.m_out)
0178 ,m_seek_directory(a_from.m_seek_directory)
0179 {}
0180 branch& operator=(const branch&){return *this;}
0181 public:
0182 const std::string& name() const {return m_name;}
0183 const std::string& title() const {return m_title;}
0184
0185 uint64 entries() const {return m_entries;}
0186
0187 uint64 tot_bytes() const {return m_tot_bytes;}
0188 void set_tot_bytes(uint64 a_value) {m_tot_bytes = a_value;}
0189
0190 uint64 zip_bytes() const {return m_zip_bytes;}
0191 void set_zip_bytes(uint64 a_value) {m_zip_bytes = a_value;}
0192
0193 void set_basket_size(uint32 a_size) {m_basket_size = a_size;}
0194 uint32 basket_size() const {return m_basket_size;}
0195
0196 ////////////////////////////////////////////////
0197 ////////////////////////////////////////////////
0198 ////////////////////////////////////////////////
0199 template <class T>
0200 leaf_ref<T>* create_leaf_ref(const std::string& a_name,const T& a_ref){
0201 leaf_ref<T>* lf = new leaf_ref<T>(m_out,a_name,a_ref);
0202 m_leaves.push_back(lf);
0203 return lf;
0204 }
0205
0206 leaf_string_ref* create_leaf_string_ref(const std::string& a_name,const std::string& a_ref){
0207 leaf_string_ref* lf = new leaf_string_ref(m_out,a_name,a_ref);
0208 m_leaves.push_back(lf);
0209 return lf;
0210 }
0211 ////////////////////////////////////////////////
0212 ////////////////////////////////////////////////
0213 ////////////////////////////////////////////////
0214
0215 template <class T>
0216 leaf<T>* create_leaf(const std::string& a_name){
0217 leaf<T>* lf = new leaf<T>(m_out,a_name);
0218 m_leaves.push_back(lf);
0219 return lf;
0220 }
0221
0222 leaf_string* create_leaf_string(const std::string& a_name){
0223 leaf_string* lf = new leaf_string(m_out,a_name);
0224 m_leaves.push_back(lf);
0225 return lf;
0226 }
0227
0228 leaf_element* create_leaf_element(const std::string& a_name,int a_id,int a_type){
0229 leaf_element* lf = new leaf_element(m_out,a_name,a_id,a_type);
0230 m_leaves.push_back(lf);
0231 return lf;
0232 }
0233
0234 // for row_wise ntuple :
0235 template <class T>
0236 leaf_std_vector_ref<T>* create_leaf_std_vector_ref(const std::string& a_name,
0237 base_leaf& a_leaf_count,const std::vector<T>& a_ref) {
0238 leaf_std_vector_ref<T>* lf = new leaf_std_vector_ref<T>(m_out,a_name,a_leaf_count,a_ref);
0239 m_leaves.push_back(lf);
0240 return lf;
0241 }
0242
0243 const std::vector<base_leaf*>& leaves() const {return m_leaves;}
0244
0245 void reset() {
0246 m_baskets.clear_objs();
0247 delete [] fBasketBytes;
0248 delete [] fBasketEntry;
0249 delete [] fBasketSeek;
0250 fBasketBytes = 0;
0251 fBasketEntry = 0;
0252 fBasketSeek = 0;
0253
0254 m_max_baskets = 10;
0255
0256 m_write_basket = 0;
0257 m_entry_number = 0;
0258 m_entries = 0;
0259 m_tot_bytes = 0;
0260 m_zip_bytes = 0;
0261
0262 m_baskets.resize(m_max_baskets,0);
0263 fBasketBytes = new uint32[m_max_baskets];
0264 fBasketEntry = new uint32[m_max_baskets];
0265 fBasketSeek = new seek[m_max_baskets];
0266 {for(uint32 i=0;i<m_max_baskets;i++) {
0267 m_baskets[i] = 0;
0268 fBasketBytes[i] = 0;
0269 fBasketEntry[i] = 0;
0270 fBasketSeek[i] = 0;
0271 }}
0272 fBasketEntry[m_write_basket] = (uint32)m_entry_number;
0273 m_baskets[m_write_basket] = new basket(m_out,m_byte_swap,m_seek_directory,m_name,m_title,"TBasket",m_basket_size,m_verbose);
0274
0275 {tools_vforit(branch*,m_branches,it) (*it)->reset();}
0276 }
0277
0278 bool fill(ifile& a_file,uint32& a_nbytes,uint32& a_add_bytes,uint32& a_nout) {
0279 a_nbytes = 0;
0280 a_add_bytes = 0;
0281 a_nout = 0;
0282
0283 if(m_write_basket>=m_max_baskets) {
0284 m_out << "tools::wroot::branch::fill :"
0285 << " potential overflow : m_write_basket (" << m_write_basket << ")"
0286 << " >= m_max_baskets (" << m_max_baskets << ")."
0287 << std::endl;
0288 return false;
0289 }
0290
0291 //FIXME if (TestBit(kDoNotProcess)) return 0;
0292
0293 basket* bk = m_baskets[m_write_basket];
0294 if(!bk) {
0295 m_out << "tools::wroot::branch::fill :"
0296 << " m_baskets[m_write_basket] should not be null."
0297 << std::endl;
0298 return false;
0299 }
0300
0301 buffer& buf = bk->datbuf();
0302
0303 buf.reset_objs_map();
0304
0305 uint32 lold = buf.length();
0306
0307 bk->update(bk->key_length()+lold);
0308 m_entries++;
0309 m_entry_number++;
0310
0311 if(!fill_leaves(buf)) {
0312 m_out << "tools::wroot::branch::fill :"
0313 << " fill_leaves() failed."
0314 << std::endl;
0315 return false;
0316 }
0317
0318 uint32 lnew = buf.length();
0319 uint32 nbytes = lnew - lold;
0320 uint32 nsize = 0;
0321
0322 uint32 add_bytes = 0;
0323 uint32 nout = 0;
0324
0325 // Should we create a new basket?
0326 // Compare expected next size with m_basket_size.
0327 if((lnew+2*nsize+nbytes)>=m_basket_size) {
0328 if(!bk->write_on_file(a_file,m_write_basket,nout)) {//it set bk m_seek_key, set a_file.m_END.
0329 //Does a a_file.set_END().
0330 m_out << "tools::wroot::branch::fill :"
0331 << " basket.write_on_file() failed."
0332 << std::endl;
0333 return false;
0334 }
0335 fBasketBytes[m_write_basket] = bk->number_of_bytes();
0336 //fBasketEntry[m_write_basket] = // not done here since we do not know here the first entry.
0337 fBasketSeek[m_write_basket] = bk->seek_key();
0338
0339 add_bytes = bk->object_size() + bk->key_length();
0340
0341 delete bk;
0342 m_baskets[m_write_basket] = 0;
0343
0344 m_tot_bytes += add_bytes;
0345 m_zip_bytes += nout;
0346
0347 m_write_basket++;
0348 if(!check_alloc_fBasketXxx()) return false;
0349
0350 m_baskets[m_write_basket] = new basket(m_out,m_byte_swap,
0351 m_seek_directory,m_name,m_title,"TBasket",m_basket_size,
0352 m_verbose);
0353 fBasketEntry[m_write_basket] = (uint32)m_entry_number;
0354 }
0355 a_nbytes = nbytes;
0356 a_add_bytes = add_bytes;
0357 a_nout = nout;
0358 return true;
0359 }
0360
0361 ///////////////////////////////////////////////////////////////////////////
0362 /// for parallelization : /////////////////////////////////////////////////
0363 ///////////////////////////////////////////////////////////////////////////
0364 bool add_basket(ifile& a_file,basket& a_basket,uint32& a_add_bytes,uint32& a_nout) {
0365 //done on a (main) ntuple/branch.
0366
0367 if(m_write_basket>=m_max_baskets) {
0368 m_out << "tools::wroot::branch::add_basket :"
0369 << " potential overflow : m_write_basket (" << m_write_basket << ")"
0370 << " >= m_max_baskets (" << m_max_baskets << ")."
0371 << std::endl;
0372 return false;
0373 }
0374
0375 uint32 nout;
0376 if(!a_basket.write_on_file(a_file,m_write_basket,nout)) { //a_file is the main file (a true file).
0377 //the upper set a_basket.m_seek_key, set a_file.m_END. Does a a_file.set_END().
0378 m_out << "tools::wroot::branch::add_basket :"
0379 << " basket.write_on_file() failed."
0380 << std::endl;
0381 return false;
0382 }
0383
0384 fBasketBytes[m_write_basket] = a_basket.number_of_bytes();
0385 fBasketEntry[m_write_basket] = (uint32)m_entry_number;
0386 fBasketSeek[m_write_basket] = a_basket.seek_key();
0387
0388 m_entries += a_basket.nev();
0389 m_entry_number += a_basket.nev();
0390
0391 uint32 add_bytes = a_basket.object_size() + a_basket.key_length();
0392
0393 delete m_baskets[m_write_basket];
0394 m_baskets[m_write_basket] = 0;
0395
0396 m_write_basket++;
0397 if(!check_alloc_fBasketXxx()) return false;
0398
0399 m_baskets[m_write_basket] = new basket(m_out,m_byte_swap,
0400 m_seek_directory,m_name,m_title,"TBasket",m_basket_size,
0401 m_verbose);
0402 fBasketEntry[m_write_basket] = (uint32)m_entry_number;
0403
0404 a_add_bytes = add_bytes;
0405 a_nout = nout;
0406 return true;
0407 }
0408
0409 class iadd_basket {
0410 public:
0411 virtual ~iadd_basket() {}
0412 public:
0413 virtual bool add_basket(basket*) = 0;
0414 };
0415
0416 bool pfill(iadd_basket& a_badd,uint32 a_nev) {
0417 //done on a pntuple/branch.
0418
0419 //a_nbytes = 0;
0420 //a_add_bytes = 0;
0421 //a_nout = 0;
0422
0423 // this method uses always m_baskets[0].
0424
0425 basket* bk = m_baskets[m_write_basket];
0426 if(!bk) {
0427 m_out << "tools::wroot::branch::parallel_fill :"
0428 << " get_basket failed."
0429 << std::endl;
0430 return false;
0431 }
0432
0433 buffer& buf = bk->datbuf();
0434
0435 uint32 lold = buf.length();
0436
0437 bk->update(bk->key_length()+lold);
0438
0439 //m_entries++; //not used by parallel branches.
0440 //m_entry_number++; //idem.
0441
0442 if(!fill_leaves(buf)) {
0443 m_out << "tools::wroot::branch::parallel_fill :"
0444 << " fill_leaves() failed."
0445 << std::endl;
0446 return false;
0447 }
0448
0449 uint32 lnew = buf.length();
0450 uint32 nbytes = lnew - lold;
0451
0452 // Should we create a new basket?
0453 bool store_basket = false;
0454 if(a_nev) {
0455 store_basket = (bk->nev()>=a_nev); // trigger add_basket per number of entries reached.
0456 } else {
0457 store_basket = ((lnew+nbytes)>=m_basket_size); // trigger add_basket when reaching m_basket_size.
0458 }
0459
0460 if(store_basket) {
0461 if(!a_badd.add_basket(bk)) { //we give ownership of bk.
0462 m_out << "tools::wroot::branch::parallel_fill :"
0463 << " main_branch.add_basket() failed."
0464 << std::endl;
0465 return false;
0466 }
0467 //delete bk; //no, it is deleted by the a_badd.add_basket().
0468 bool main_branch_byte_swap = m_byte_swap; //NOTE : it assumes that pntuple/branch and main/branch have same byte_swap.
0469 bool main_branch_verbose = m_verbose;
0470 m_baskets[m_write_basket] = new basket(m_out,main_branch_byte_swap,
0471 m_seek_directory,m_name,m_title,"TBasket",m_basket_size,
0472 main_branch_verbose);
0473 }
0474
0475 return true;
0476 }
0477
0478 bool end_pfill(iadd_basket& a_badd) {
0479 //done on a pntuple/branch.
0480
0481 basket* bk = m_baskets[m_write_basket];
0482 if(!bk) {
0483 m_out << "tools::wroot::branch::end_pfill :"
0484 << " m_baskets[m_write_basket] should not be null."
0485 << std::endl;
0486 return false;
0487 }
0488
0489 buffer& buf = bk->datbuf();
0490 uint32 lold = buf.length();
0491
0492 if(lold) {
0493 if(!a_badd.add_basket(bk)) { //we give ownership of bk.
0494 m_out << "tools::wroot::branch::parallel_fill :"
0495 << " main_branch.add_basket() failed."
0496 << std::endl;
0497 return false;
0498 }
0499 } else {
0500 delete bk;
0501 }
0502
0503 m_baskets[m_write_basket] = 0; // no need to recreate a basket since it is end of fill.
0504
0505 return true;
0506 }
0507
0508 ///////////////////////////////////////////////////////////////////////////
0509 ///////////////////////////////////////////////////////////////////////////
0510 ///////////////////////////////////////////////////////////////////////////
0511 protected:
0512 bool check_alloc_fBasketXxx() {
0513 if(m_write_basket>=m_max_baskets) {
0514 //Increase BasketEntry buffer of a minimum of 10 locations
0515 // and a maximum of 50 per cent of current size
0516 uint32 newsize = mx<uint32>(10,uint32(1.5*m_max_baskets));
0517 if(newsize>=START_BIG_FILE()) {
0518 //we are going to have pb with uint32[] indexing.
0519 m_out << "tools::wroot::branch::add_basket :"
0520 << " new size for fBasket[Bytes,Entry,Seek] arrays"
0521 << " is too close of 32 bits limit."
0522 << std::endl;
0523 m_out << "tools::wroot::branch::add_basket :"
0524 << " you have to work with larger basket size."
0525 << std::endl;
0526 return false;
0527 }
0528
0529 m_baskets.resize(newsize,0);
0530
0531 if(!realloc<uint32>(fBasketBytes,newsize,m_max_baskets,true)) {
0532 m_out << "tools::wroot::branch::add_basket : realloc failed." << std::endl;
0533 return false;
0534 }
0535 if(!realloc<uint32>(fBasketEntry,newsize,m_max_baskets,true)){
0536 m_out << "tools::wroot::branch::add_basket : realloc failed." << std::endl;
0537 return false;
0538 }
0539 if(!realloc<seek>(fBasketSeek,newsize,m_max_baskets,true)){
0540 m_out << "tools::wroot::branch::add_basket : realloc failed." << std::endl;
0541 return false;
0542 }
0543 m_max_baskets = newsize;
0544 }
0545
0546 m_baskets[m_write_basket] = 0;
0547 fBasketBytes[m_write_basket] = 0;
0548 fBasketEntry[m_write_basket] = 0;
0549 fBasketSeek[m_write_basket] = 0;
0550
0551 return true;
0552 }
0553
0554 virtual bool fill_leaves(buffer& a_buffer) { //virtual for branch_element.
0555 tools_vforit(base_leaf*,m_leaves,it) {
0556 if(!(*it)->fill_buffer(a_buffer)) return false;
0557 }
0558 return true;
0559 }
0560 protected:
0561 std::ostream& m_out;
0562 bool m_byte_swap;
0563 bool m_verbose;
0564 seek m_seek_directory;
0565 obj_array<basket> m_baskets;
0566 /// for parallelization :
0567 public:
0568 std::vector<basket*> m_parallel_baskets;
0569 protected:
0570 //Object
0571 //uint32 m_bits;
0572 //Named
0573 std::string m_name;
0574 std::string m_title;
0575
0576 bool fAutoDelete;
0577 private: //be sure that sub branches are not used for the moment.
0578 obj_array<branch> m_branches;
0579 protected:
0580 obj_array<base_leaf> m_leaves;
0581 uint32 fCompress; // Compression level and algorithm
0582 uint32 m_basket_size; // Initial Size of Basket Buffer
0583 uint32 m_write_basket; // Last basket number written
0584 uint64 m_entry_number; // Current entry number (last one filled in this branch)
0585 uint64 m_entries; // Number of entries
0586 uint64 m_tot_bytes;
0587 uint64 m_zip_bytes;
0588 uint32 m_max_baskets;
0589 uint32* fBasketBytes; //[m_max_baskets] Length of baskets on file
0590 uint32* fBasketEntry; //[m_max_baskets] Table of first entry in eack basket
0591 seek* fBasketSeek; //[m_max_baskets] Addresses of baskets on file
0592 };
0593
0594 }}
0595
0596 #endif
