eic-opticks/sysrap/NPX.h

0001 #pragma once
0002 /**
0003 NPX.h : NP.hh related extras such as static converters
0004 =========================================================
0005
0006
0007 **/
0008
0009 #include "NP.hh"
0010 #include <unordered_map>
0011
0012
0013 struct NPX
0014 {
0015     template<typename T>
0016     static NP* MakeValues( const std::vector<std::string>& keys, const std::vector<T>& vals );
0017     template<typename T>
0018     static NP* MakeValues( const std::vector<std::pair<std::string, T>>& values, const char* contains=nullptr );
0019     template<typename T>
0020     static std::string DescValues(const NP* a);
0021
0022     template<typename T>
0023     struct KV
0024     {
0025         std::vector<std::string> kk ;
0026         std::vector<T> vv ;
0027
0028         void add(const char* k, T v ){ kk.push_back(k); vv.push_back(v); }
0029         NP* values() const { return MakeValues<T>(kk, vv) ; }
0030     };
0031
0032
0033     static NP* MakeDemo(const char* dtype="<f4" , NP::INT ni=-1, NP::INT nj=-1, NP::INT nk=-1, NP::INT nl=-1, NP::INT nm=-1, NP::INT no=-1 );
0034
0035     template<typename T> static NP*  Make( const std::vector<T>& src );
0036     template<typename T> static NP*  Make( T d0, T v0, T d1, T v1 );
0037     template<typename T, typename... Args> static NP*  Make(const T* src, Args ... shape );  // WIP: switch to name ArrayFromData
0038     template<typename T, typename... Args> static NP*  ArrayFromData(const T* src, Args ... shape );
0039
0040     template<typename T> static NP* FromString(const char* str, char delim=' ') ;
0041
0042
0043     static NP* Holder( const std::vector<std::string>& names );
0044
0045
0046     template<typename T, int N>
0047     static NP* ArrayFromVecOfArrays(const std::vector<std::array<T,N>>& va );
0048
0049     template<typename T, int N>
0050     static void VecOfArraysFromArray( std::vector<std::array<T,N>>& va, const NP* a );
0051
0052
0053
0054     template<typename T, typename S>
0055     static NP* ArrayFromVec__(const std::vector<S>& v, const std::vector<NP::INT>& itemshape );
0056
0057     template<typename T, typename S, typename... Args>
0058     static NP* ArrayFromVec(const std::vector<S>& v, Args ... args_itemshape );   // ArrayFromVec_ellipsis
0059
0060     template<typename T, typename S>
0061     static NP* ArrayFromVec_(const std::vector<S>& v, const char* str_itemshape );
0062
0063
0064     template<typename S>
0065     static void VecFromArray(std::vector<S>& v, const NP* a );
0066
0067     template<typename S>
0068     static int VecFromMap( std::vector<S>& v,  const std::map<int, S>& m, bool contiguous_key=true );
0069
0070     template<typename S>
0071     static void VecFromMapUnordered(
0072         std::vector<S>& v,
0073         const std::unordered_map<int, S>& m,
0074         const std::vector<int>* key_order,
0075         bool* all_contiguous_key,
0076         int* _k0,
0077         int* _k1
0078         );
0079
0080
0081     template<typename S>
0082     static void MapFromVec( std::map<int, S>& m,  const std::vector<S>& v, int k0=0, bool contiguous_key=true );
0083
0084     template<typename S>
0085     static void MapUnorderedFromVec( std::unordered_map<int, S>& m,  const std::vector<S>& v, int kmin=0, bool is_contiguous_key=true );
0086
0087
0088     template<typename T, typename S>
0089     static NP* ArrayFromMap( const std::map<int, S>& m, bool contiguous_key=true );
0090
0091     template<typename T, typename S>
0092     static NP* ArrayFromMapUnordered(
0093         const std::unordered_map<int, S>& m, const std::vector<int>* key_order=nullptr );
0094
0095     template<typename S>
0096     static void KeyRangeMapUnordered( int* _k0, int* _k1, const std::unordered_map<int, S>& m );
0097
0098     template<typename S>
0099     static void MapFromArray( std::map<int, S>& m, const NP* a );
0100
0101     template<typename S>
0102     static void MapUnorderedFromArray( std::unordered_map<int, S>& m, const NP* a );
0103
0104
0105     template<typename S>
0106     static NP* ArrayFromDiscoMap( const std::map<int, S>& m );
0107     template<typename S>
0108     static void DiscoMapFromArray( std::map<int, S>& m, const NP* a );
0109     template<typename S>
0110     static std::string DescDiscoMap( const std::map<int, S>& m );
0111
0112
0113     template<typename S>
0114     static NP* ArrayFromDiscoMapUnordered( const std::unordered_map<int, S>& m );
0115     template<typename S>
0116     static void DiscoMapUnorderedFromArray( std::unordered_map<int, S>& m, const NP* a );
0117     template<typename S>
0118     static std::string DescDiscoMapUnordered( const std::unordered_map<int, S>& m );
0119
0120
0121
0122
0123
0124     template<typename T>
0125     static NP* FromNumpyString(const char* str, bool dump=false) ;
0126
0127     static NP* CategoryArrayFromString(const char* str, int catfield, const char* cats, char delim=',');
0128     static NP* LoadCategoryArrayFromTxtFile(const char* base, const char* relp, int catfield, const char* cats, char delim=',');
0129     static NP* LoadCategoryArrayFromTxtFile(const char* path, int catfield, const char* cats, char delim=',');
0130
0131     static void Import_MSD(           std::map<std::string,double>& msd, const NP* a);
0132     static NP* Serialize_MSD(   const std::map<std::string,double>& msd );
0133     static std::string Desc_MSD(const std::map<std::string,double>& msd);
0134
0135     static NP* ArrayFromEnumMap( const std::map<int, std::string>& catMap) ;
0136
0137     static NP* MakeCharArray( const std::vector<std::string>& nn );
0138
0139     template<typename F, typename T>
0140     static NP* BOA( NP* a, NP* b, NP::INT a_column=-1, NP::INT b_column=-1, std::ostream* out=nullptr  );
0141
0142 };
0143
0144
0145
0146 template<typename T>
0147 inline NP* NPX::MakeValues( const std::vector<std::string>& keys, const std::vector<T>& vals ) // static
0148 {
0149     assert( keys.size() == vals.size() );
0150     if(vals.size() == 0 ) return nullptr ;
0151
0152     NP* vv = NPX::Make<T>( vals ) ;
0153     vv->set_names( keys );
0154     return vv ;
0155 }
0156
0157 template<typename T>
0158 inline NP* NPX::MakeValues( const std::vector<std::pair<std::string, T>>& values, const char* contains ) // static
0159 {
0160     if(NP::VERBOSE) std::cout
0161         << "NPX::MakeValues values.size " << values.size()
0162         << " contains " << ( contains ? contains : "-" )
0163         << std::endl
0164         ;
0165
0166     std::vector<std::string> nams ;
0167     std::vector<T> vals ;
0168
0169     for(NP::INT i=0 ; i < NP::INT(values.size()) ; i++)
0170     {
0171         const std::pair<std::string, T>& kv = values[i] ;
0172         const char* k = kv.first.c_str() ;
0173         T v = kv.second ;
0174
0175         bool select = contains == nullptr || U::Contains( k, contains ) ;
0176
0177         if(NP::VERBOSE) std::cout
0178             << "NPX::MakeValues "
0179             << std::setw(3) << i
0180             << " v " << std::setw(10) << std::fixed << std::setprecision(4) << v
0181             << " k " << std::setw(60) << k
0182             << " select " << select
0183             <<  std::endl
0184             ;
0185
0186         if(select)
0187         {
0188             nams.push_back(k);
0189             vals.push_back(v);
0190         }
0191     }
0192     if(NP::VERBOSE) std::cout << "NPX::MakeValues vals.size " << vals.size() << std::endl ;
0193
0194     NP* vv = MakeValues<T>(nams, vals);
0195     return vv ;
0196 }
0197
0198 template<typename T>
0199 inline std::string NPX::DescValues(const NP* a) // static
0200 {
0201     std::stringstream ss ;
0202     ss << std::endl << "NPX::descValues"  << std::endl ;
0203     for(unsigned i=0 ; i < a->names.size() ; i++)
0204     {
0205         const char* k = a->names[i].c_str();
0206         T v = a->get_named_value<T>(k, 0) ;
0207         ss
0208             << std::setw(30) << k
0209             << " : "
0210             << std::setw(10) << v
0211             << std::endl
0212             ;
0213     }
0214     std::string str = ss.str();
0215     return str ;
0216 }
0217
0218
0219 inline NP* NPX::MakeDemo(const char* dtype, NP::INT ni, NP::INT nj, NP::INT nk, NP::INT nl, NP::INT nm, NP::INT no )
0220 {
0221     NP* a = new NP(dtype, ni, nj, nk, nl, nm, no);
0222     a->fillIndexFlat();
0223     return a ;
0224 }
0225
0226
0227
0228 template <typename T>
0229 inline NP* NPX::Make( const std::vector<T>& src ) // static
0230 {
0231     NP* a = NP::Make<T>(src.size());
0232     a->read(src.data());
0233     return a ;
0234 }
0235
0236 template <typename T>
0237 inline NP*  NPX::Make(T d0, T v0, T d1, T v1 ) // static
0238 {
0239     std::vector<T> src = {d0, v1, d1, v1 } ;
0240     return NPX::Make<T>(src) ;
0241 }
0242
0243 /**
0244 NPX::Make "Make_ellipsis"
0245 --------------------------
0246
0247 This "Make_ellipsis" method combines allocation of the array and populating it
0248 from the src data. This is intended to facilitate creating arrays from vectors
0249 of struct, by using simple template types  (int, float, double etc.. )
0250 together with array item shapes appropriate to the elements of the struct.
0251 For example::
0252
0253    struct demo { int x,y,z,w ; } ;
0254    std::vector<demo> dd ;
0255    dd.push_back( {1,2,3,4} );
0256
0257    NP* a = NPX::Make<int>( (int*)dd.data() , int(dd.size()) , 4 );
0258
0259 The product of the shape integers MUST correspond to the number of
0260 values provided from the src data.
0261 When the first int shape dimension is zero a nullptr is returned.
0262
0263 **/
0264
0265 template<typename T, typename... Args>
0266 inline NP* NPX::Make(const T* src, Args ... args )   // WIP switch to name ArrayFromData
0267 {
0268     std::cerr << "TODO: change NPX::Make to NPX::ArrayFromData \n" ;
0269     return ArrayFromData(src, std::forward<Args>(args)...) ;
0270 }
0271
0272 template<typename T, typename... Args>
0273 inline NP* NPX::ArrayFromData(const T* src, Args ... args )
0274 {
0275     std::string dtype = descr_<T>::dtype() ;
0276     std::vector<NP::INT> shape = {args...};
0277     if(shape.size() > 0 && shape[0] == 0) return nullptr ;
0278     NP* a = new NP(dtype.c_str(), shape );
0279     a->read2(src);
0280     return a ;
0281 }
0282
0283
0284
0285 template <typename T>
0286 inline NP* NPX::FromString(const char* str, char delim)  // static
0287 {
0288     std::vector<T> vec ;
0289     std::stringstream ss(str);
0290     std::string s ;
0291     while(getline(ss, s, delim)) vec.push_back(U::To<T>(s.c_str()));
0292     NP* a = Make<T>(vec) ;
0293     return a ;
0294 }
0295
0296
0297
0298
0299
0300
0301
0302 inline NP* NPX::Holder( const std::vector<std::string>& names )
0303 {
0304     NP* a = NP::Make<int>(0) ;
0305     a->set_names(names) ;
0306     return a ;
0307 }
0308
0309
0310
0311
0312 /**
0313 NPX::ArrayFromVecOfArrays
0314 --------------------------
0315
0316 There is potential for the impl of std::vector and std::array
0317 to add padding so although the compound vector of arrays will be contiguous
0318 it is not possible to rely on having the obvious layout ?
0319
0320 **/
0321
0322 template<typename T, int N>
0323 inline NP* NPX::ArrayFromVecOfArrays(const std::vector<std::array<T,N>>& va )
0324 {
0325     NP::INT ni = va.size();
0326     NP::INT nj = N ;
0327     NP* a = NP::Make<T>(ni, nj);
0328     T* aa = a->values<T>();
0329     for(NP::INT i=0 ; i < ni ; i++)
0330     {
0331         const std::array<T,N>& arr = va[i] ;
0332         for(NP::INT j=0 ; j < nj ; j++) aa[i*nj+j] = arr[j] ;
0333     }
0334     return a ;
0335 }
0336
0337
0338 template<typename T, int N>
0339 inline void NPX::VecOfArraysFromArray( std::vector<std::array<T,N>>& va, const NP* a )
0340 {
0341     assert( a && a->uifc == 'f' && a->ebyte == sizeof(T) );
0342     assert( a && a->shape.size() == 2 );
0343
0344     NP::INT ni = a->shape[0];
0345     NP::INT nj = a->shape[1];
0346     const T* aa = a->cvalues<T>();
0347
0348     va.resize(ni);
0349     for(NP::INT i=0 ; i < ni ; i++)
0350     {
0351         std::array<T,N>& arr = va[i] ;
0352         for(NP::INT j=0 ; j < nj ; j++) arr[j] = aa[i*nj+j] ;
0353     }
0354
0355 }
0356
0357
0358
0359
0360
0361
0362 template<typename T, typename S>
0363 inline NP* NPX::ArrayFromVec__(const std::vector<S>& v, const std::vector<NP::INT>& itemshape )
0364 {
0365     assert( sizeof(S) >= sizeof(T) );
0366     NP::INT ni = v.size() ;
0367     NP::INT nj = sizeof(S) / sizeof(T) ;
0368
0369     const T* src = (T*)v.data() ;
0370
0371     std::vector<NP::INT> shape ;
0372     shape.push_back(ni) ;
0373
0374     if(itemshape.size() == 0 )
0375     {
0376         shape.push_back(nj) ;
0377     }
0378     else
0379     {
0380         NP::INT itemcheck = 1 ;
0381         for(NP::INT i=0 ; i < NP::INT(itemshape.size()) ; i++)
0382         {
0383             shape.push_back(itemshape[i]) ;
0384             itemcheck *= itemshape[i] ;
0385         }
0386         bool consistent = itemcheck == nj ;
0387
0388         if(!consistent) std::cerr
0389             << "NPX::ArrayFromVec__"
0390             << " ERROR "
0391             << " consistent " << ( consistent ? "YES" : "NO " )
0392             << " itemcheck " << itemcheck
0393             << " nj " << nj
0394             << " itemshape.size " << itemshape.size()
0395             << std::endl
0396             ;
0397
0398         assert(consistent);
0399     }
0400
0401
0402     NP* a = NP::Make_<T>(shape) ;
0403     a->read2(src);
0404     return a ;
0405 }
0406
0407
0408
0409
0410 /**
0411 NPX::ArrayFromVec
0412 -------------------
0413
0414 The optional itemshape integers override the flat item element count
0415 obtained from the type sizeof ratio *sizeof(S)/sizeof(T)*.
0416 Note that the product of the itemshape integers must match the
0417 flat item count however.
0418
0419 Usage examples::
0420
0421     NP* _nds = NPX::ArrayFromVec<int,snode>( nds, snode::NV ) ;
0422     NP* _prim_nidx = NPX::ArrayFromVec<int,int>( prim_nidx ) ;
0423
0424 Note that with <T,S> <int,int> the array shape is (num_items, 1)
0425 without the second dimension being suppressed.
0426 Leaving like this for consistency. If that is inconvenient
0427 from python can easily reshape after np.load::
0428
0429     In [8]: f.prim_nidx.shape
0430     Out[8]: (3392, 1)
0431
0432     In [9]: f.prim_nidx[:,0].shape
0433     Out[9]: (3392,)
0434
0435
0436 **/
0437
0438 template<typename T, typename S, typename... Args>
0439 inline NP* NPX::ArrayFromVec(const std::vector<S>& v, Args ... args_itemshape )   // ArrayFromVec_ellipsis
0440 {
0441     std::vector<NP::INT> itemshape = {args_itemshape...};
0442     return ArrayFromVec__<T,S>(v, itemshape );
0443 }
0444
0445 template<typename T, typename S>
0446 inline NP* NPX::ArrayFromVec_(const std::vector<S>& v, const char* str_itemshape )
0447 {
0448     std::vector<NP::INT> itemshape ;
0449     U::MakeVec(itemshape,  str_itemshape, ',' );
0450     return ArrayFromVec__<T,S>(v, itemshape );
0451 }
0452
0453
0454
0455 /**
0456 NPX::VecFromArray
0457 -------------------
0458
0459 Direct mapping assuming type S is correct.
0460
0461 **/
0462
0463
0464 template<typename S>
0465 inline void NPX::VecFromArray(std::vector<S>& v, const NP* a )
0466 {
0467    if(a == nullptr || a->shape.size() == 0 ) return ;
0468    NP::INT ni = a->shape[0] ;
0469    unsigned ib = a->item_bytes() ;
0470    bool expected_sizeof_item = sizeof(S) == ib  ;
0471
0472    if(!expected_sizeof_item)
0473       std::cerr
0474           << "NPX::VecFromArray"
0475           << " expected_sizeof_item  " << ( expected_sizeof_item  ? "YES" : "NO" )
0476           << " sizeof(S) " << sizeof(S)
0477           << " a.item_bytes " << ib
0478           << " a.sstr " << a->sstr()
0479           << std::endl
0480           << " a.lpath " << a->get_lpath()
0481           << std::endl
0482           << " CHECK FOR COMPILATION OPTIONS THAT CHANGE STRUCT SIZES "
0483           << std::endl
0484           << " FOR EXAMPLE WITH_CHILD CHANGES sysrap/sn.h "
0485           << std::endl
0486           << " ANOTHER POSSIBILITY IS LOADING AN ARRAY WRITTEN BEFORE STRUCT SIZE CHANGES "
0487           << std::endl
0488           ;
0489
0490    assert( expected_sizeof_item );
0491
0492    v.clear() ;
0493    v.resize(ni);
0494
0495    memcpy( v.data(), a->bytes(), a->arr_bytes() );
0496 }
0497
0498
0499
0500 /**
0501 NPX::VecFromMap
0502 ---------------
0503
0504 Populate vector using map values in the default key order of the map
0505
0506 1. clear and resize the vector to match size of the map
0507 2. record k0, first key in map
0508 3. iterate through the map copying S values from the map into the vector
0509 4. return k0
0510
0511 contiguous_key:true
0512    an assert verifies that keys are contiguously incrementing from the first key
0513
0514 Used by NPX::ArrayFromMap
0515
0516 **/
0517
0518
0519 template<typename S>
0520 inline int NPX::VecFromMap( std::vector<S>& v,  const std::map<int, S>& m, bool contiguous_key ) // static
0521 {
0522     NP::INT ni = NP::INT(m.size()) ;
0523
0524     v.clear();
0525     v.resize(ni);
0526
0527     typename std::map<int,S>::const_iterator it = m.begin()  ;
0528     int k0 = it->first ;
0529
0530     for(NP::INT idx=0 ; idx < ni ; idx++)
0531     {
0532         int k = it->first ;
0533         v[idx] = it->second ;
0534
0535         if(contiguous_key) assert( k == k0 + idx );
0536         //std::cout << " k0 " << k0 << " idx " << idx << " k " << k << " v " << it->second << std::endl ;
0537
0538         std::advance(it, 1);
0539     }
0540     return k0 ;
0541 }
0542
0543 /**
0544 NPX::VecFromMapUnordered
0545 --------------------------
0546
0547 Populate vector using unordered_map values using key_order vector ordering.
0548
0549 When the key_order argument is not nullptr it must point to
0550 a vector that contains all the keys that are present in the
0551 unordered map in the desired order.
0552
0553 When the key_order argument is nullptr the keys are assumed to
0554 be contiguous integers starting from the minimum key that
0555 is present in the unordered_map.
0556
0557 1. clear vector and resize vector to the size of the map
0558 2. when key_order points to a vector iterate over keys, for each key:
0559
0560    * count if the key is a contiguous offset from the first key, k0
0561    * use the key to lookup a value from the map and place that value into the idx vector position
0562    * NB all keys in the unordered_map must be present in the key_order vector,
0563      otherwise an unordedmap::at failure will occur
0564
0565 3. when key_order is nullptr determine the range of keys and use the smallest key *_k0
0566    as the base for all keys, that are assumed to be contiguous.  If the unordered_map keys
0567    are not within a contiguous range of integers then this method
0568    will fail with an unorderedmap::at failure. In this situation it is necesssary
0569    to provide a pointer to a key_order vector containing all the keys in the unordered_map.
0570
0571 **/
0572
0573 template<typename S>
0574 inline void NPX::VecFromMapUnordered(
0575     std::vector<S>& v,
0576     const std::unordered_map<int, S>& m,
0577     const std::vector<int>* key_order,
0578     bool* all_contiguous_key,
0579     int* _k0,
0580     int* _k1
0581     )
0582 {
0583     NP::INT ni = m.size()  ;
0584     v.clear();
0585     v.resize(ni);
0586
0587     if(key_order)
0588     {
0589         assert( key_order->size() == m.size() ) ;
0590     }
0591
0592     NP::INT count(0) ;
0593     if( key_order == nullptr ) KeyRangeMapUnordered(_k0, _k1, m ) ;
0594
0595
0596     NP::INT k0_check(-999) ;
0597
0598     for(NP::INT idx=0 ; idx < ni ; idx++)
0599     {
0600         NP::INT k = key_order ? (*key_order)[idx] : *_k0 + idx  ;
0601
0602         // counting keys that are contiguous offsets from the idx 0 key
0603         if( idx == 0 )
0604         {
0605             k0_check = k ;
0606             count += 1 ;
0607         }
0608         else
0609         {
0610             if( k == k0_check + idx ) count += 1  ;
0611         }
0612
0613         const S& s = m.at(k);
0614         v[idx] = s ;
0615     }
0616
0617     *all_contiguous_key = ni > 0 && count == ni  ;
0618
0619     if(key_order == nullptr)
0620     {
0621         assert( *all_contiguous_key ) ;
0622         assert( *_k1 - *_k0 + 1 == ni ) ;
0623         assert( k0_check == *_k0 );
0624     }
0625
0626 }
0627
0628
0629
0630
0631
0632 /**
0633 NPX::MapFromVec
0634 ----------------
0635
0636 HMM: to support contiguous_key:false would need type S to follow
0637 some convention such as keeping int keys within the first 32 bit member.
0638 OR just store keys and values as separate arrays within an NPFold.
0639
0640 **/
0641
0642 template<typename S>
0643 inline void NPX::MapFromVec( std::map<int, S>& m,  const std::vector<S>& v, int k0, bool contiguous_key )
0644 {
0645     assert( contiguous_key == true );
0646
0647     NP::INT ni = NP::INT(v.size()) ;
0648     m.clear();
0649
0650     for(NP::INT i=0 ; i < ni ; i++)
0651     {
0652         const S& item = v[i] ;
0653         NP::INT key = k0 + i ;
0654         m[key] = item ;
0655     }
0656 }
0657
0658
0659
0660
0661 /**
0662 NPX::MapUnorderedFromVec
0663 ---------------------------
0664
0665
0666 **/
0667
0668 template<typename S>
0669 inline void NPX::MapUnorderedFromVec( std::unordered_map<int, S>& m,  const std::vector<S>& v, int kmin, bool is_contiguous_key )
0670 {
0671     assert( is_contiguous_key == true );
0672
0673     NP::INT ni = NP::INT(v.size()) ;
0674     m.clear();
0675
0676     for(NP::INT idx=0 ; idx < ni ; idx++)
0677     {
0678         const S& item = v[idx] ;
0679         NP::INT key = kmin + idx ;
0680         m[key] = item ;
0681     }
0682 }
0683
0684
0685
0686
0687
0688
0689
0690
0691
0692 /**
0693 NPX::ArrayFromMap
0694 -------------------
0695
0696 A vector of S structs is populated from the map in the default key order of the map.
0697 An NP array is then created from the contiguous vector data.
0698
0699 When contiguous_key:true the map keys are required to contiguously increment
0700 from the first. The first key is recorded into the metadata of the array with name "k0".
0701 For example with keys: 100,101,102 the k0 would be 100.
0702
0703 Serializing maps is most useful for contiguous_key:true as
0704 map access by key can then be mimicked by simply obtaining the
0705 array index by subtracting k0 from the map key.
0706
0707 **/
0708
0709 template<typename T, typename S>
0710 inline NP* NPX::ArrayFromMap( const std::map<int, S>& m, bool contiguous_key )
0711 {
0712     assert( sizeof(S) >= sizeof(T) );
0713
0714     std::vector<S> v ;
0715     int k0 = NPX::VecFromMap<S>( v, m, contiguous_key );
0716     NP* a = NPX::ArrayFromVec<T,S>(v) ;
0717
0718     int ContiguousKey = contiguous_key ? 1 : 0 ;
0719
0720     if(NP::VERBOSE) std::cout
0721        << "NPX::ArrayFromMap"
0722        << " k0 " << k0
0723        << " ContiguousKey " << ContiguousKey
0724        << std::endl
0725        ;
0726
0727     a->set_meta<int>("k0", k0) ;
0728     a->set_meta<int>("ContiguousKey", ContiguousKey) ;
0729
0730     return a ;
0731 }
0732
0733
0734 /**
0735 NPX::ArrayFromMapUnordered
0736 -----------------------------
0737
0738 Only useful when the keys are contiguous
0739 as key values not persisted.
0740
0741 See also NPX::MapUnorderedFromArray that
0742 does the reverse, recreating the unordered_map
0743 from the array.
0744
0745 **/
0746
0747
0748 template<typename T, typename S>
0749 inline NP* NPX::ArrayFromMapUnordered( const std::unordered_map<int, S>& m, const std::vector<int>* key_order )
0750 {
0751     assert( sizeof(S) >= sizeof(T) );
0752
0753     std::vector<S> v ;
0754     bool all_contiguous_key(false) ;
0755     int kmin(0) ;
0756     int kmax(0) ;
0757
0758     NPX::VecFromMapUnordered<S>( v, m, key_order, &all_contiguous_key, &kmin, &kmax );
0759     NP* a = NPX::ArrayFromVec<T,S>(v) ;
0760
0761     a->set_meta<int>("kmin", kmin) ;
0762     a->set_meta<int>("kmax", kmax) ;
0763     a->set_meta<int>("ContiguousKey", all_contiguous_key ) ;
0764     a->set_meta<std::string>("Creator", "NPX::ArrayFromMapUnordered" );
0765
0766     return a ;
0767 }
0768
0769
0770 /**
0771 NPX::KeyRangeMapUnordered
0772 --------------------------
0773
0774 Iterate over the int keys of the unordered_map to determine
0775 the smallest *_k0* and largest *_k1* int keys.
0776
0777 **/
0778
0779
0780 template<typename S>
0781 inline void NPX::KeyRangeMapUnordered( int* _k0, int* _k1, const std::unordered_map<int, S>& m )
0782 {
0783     assert( _k0 );
0784     assert( _k1 );
0785
0786     *_k0 = std::numeric_limits<int>::max() ;
0787     *_k1 = std::numeric_limits<int>::min() ;
0788
0789     typedef std::unordered_map<int,S> UIS ;
0790     for(typename UIS::const_iterator it=m.begin() ; it != m.end() ; it++ )
0791     {
0792         int k = it->first ;
0793         if( k < *_k0 ) *_k0 = k ;
0794         if( k > *_k1 ) *_k1 = k ;
0795     }
0796 }
0797
0798
0799
0800
0801
0802
0803
0804 template<typename S>
0805 inline void NPX::MapFromArray( std::map<int, S>& m, const NP* a )
0806 {
0807     if(a == nullptr || a->shape.size() == 0 ) return ;
0808
0809     int k0 = a->get_meta<int>("k0");
0810     int ContiguousKey = a->get_meta<int>("ContiguousKey") ;
0811     if(NP::VERBOSE) std::cout
0812         << "NPX::MapFromArray"
0813         << " k0 " << k0
0814         << " ContiguousKey " << ContiguousKey
0815         << std::endl
0816         ;
0817
0818     std::vector<S> v ;
0819     NPX::VecFromArray<S>(v, a);
0820
0821     NPX::MapFromVec(m, v, k0, ContiguousKey == 1 );
0822 }
0823
0824
0825
0826 template<typename S>
0827 inline void NPX::MapUnorderedFromArray( std::unordered_map<int, S>& m, const NP* a )
0828 {
0829     if(a == nullptr || a->shape.size() == 0 ) return ;
0830
0831     int kmin = a->get_meta<int>("kmin");
0832     int kmax = a->get_meta<int>("kmax");
0833     int ContiguousKey = a->get_meta<int>("ContiguousKey") ;
0834     bool is_contiguous_key = ContiguousKey == 1 ;
0835
0836
0837     if(NP::VERBOSE) std::cout
0838         << "NPX::MapUnorderedFromArray"
0839         << " kmin " << kmin
0840         << " kmax " << kmax
0841         << " is_contiguous_key " << ( is_contiguous_key ? "YES" : "NO " )
0842         << std::endl
0843         ;
0844
0845     std::vector<S> v ;
0846     NPX::VecFromArray<S>(v, a);
0847
0848     NPX::MapUnorderedFromVec<S>(m, v, kmin, is_contiguous_key );
0849 }
0850
0851
0852
0853
0854 /**
0855 NPX::ArrayFromDiscoMap
0856 ------------------------
0857
0858 How to handle maps with discontiguous keys ?
0859 For S:int and int keys can simply save as array of shape (10,2)
0860
0861   idx  key val
0862    0    0   0
0863    1    1   1
0864    2    2   2
0865    3    3   0
0866    4    4   1
0867    5    5   2
0868    6   10   3
0869    7   11   3
0870    8   12   3
0871    9   13   3
0872
0873 **/
0874
0875 template<typename S>
0876 inline NP* NPX::ArrayFromDiscoMap( const std::map<int, S>& m )
0877 {
0878     return nullptr ;
0879 }
0880
0881 template<typename S>
0882 inline NP* NPX::ArrayFromDiscoMapUnordered( const std::unordered_map<int, S>& m )
0883 {
0884     return nullptr ;
0885 }
0886
0887
0888
0889
0890
0891 template<>
0892 inline NP* NPX::ArrayFromDiscoMap( const std::map<int,int>& m )
0893 {
0894     NP::INT ni = m.size() ;
0895     NP::INT nj = 2 ;
0896     NP* a = NP::Make<int>(ni, nj) ;
0897     int* aa = a->values<int>();
0898
0899     typedef std::map<int,int> MII ;
0900     MII::const_iterator it = m.begin();
0901
0902     for(NP::INT i=0 ; i < ni ; i++)
0903     {
0904         aa[i*nj+0] = it->first ;
0905         aa[i*nj+1] = it->second ;
0906         it++ ;
0907     }
0908     return a ;
0909 }
0910
0911
0912 /**
0913 NPX::ArrayFromDiscoMapUnordered
0914 ---------------------------------
0915
0916 1. collect keys from the unordered_map
0917 2. sort the keys into ascending order
0918 3. check number of keys matches the size of the map, meaning keys are unique
0919 4. create empty array of shape (num_keys, 2)
0920 5. iterate through the sorted keys populating the array with keys and values from unordered_map lookups
0921
0922 The upshot of this is that the array keys follow numerically ascending order
0923
0924 **/
0925
0926 template<>
0927 inline NP* NPX::ArrayFromDiscoMapUnordered( const std::unordered_map<int,int>& m )
0928 {
0929
0930     // 1. collect keys from the unordered_map
0931     std::vector<int> keys ;
0932     typedef std::unordered_map<int,int> MII ;
0933     for(MII::const_iterator it=m.begin() ; it != m.end() ; it++) keys.push_back(it->first);
0934
0935     // 2. sort the keys into ascending order
0936     std::sort(keys.begin(), keys.end());
0937
0938     // 3. check number of keys matches the size of the map, meaning keys are unique
0939     NP::INT ni = m.size() ;
0940     assert( NP::INT(keys.size()) == ni );
0941
0942     // 4. create empty array of shape (num_keys, 2)
0943     NP::INT nj = 2 ;
0944     NP* a = NP::Make<int>(ni, nj) ;
0945     int* aa = a->values<int>();
0946
0947     // 5. iterate through the sorted keys populating the array with keys and values from unordered_map lookups
0948     for(NP::INT i=0 ; i < ni ; i++)
0949     {
0950         NP::INT key = keys[i] ;
0951         int val = m.at(key) ;
0952         aa[i*nj+0] = key ;
0953         aa[i*nj+1] = val ;
0954     }
0955     return a ;
0956 }
0957
0958
0959
0960
0961
0962
0963
0964
0965
0966
0967
0968 template<typename S>
0969 inline void NPX::DiscoMapFromArray( std::map<int, S>& m, const NP* a ){}
0970
0971 template<typename S>
0972 inline void NPX::DiscoMapUnorderedFromArray( std::unordered_map<int, S>& m, const NP* a ){}
0973
0974
0975
0976
0977 template<>
0978 inline void NPX::DiscoMapFromArray( std::map<int, int>& m, const NP* a )
0979 {
0980     assert( a && a->uifc == 'i' && a->ebyte == 4 && a->shape.size() == 2 );
0981     NP::INT ni = a->shape[0] ;
0982     NP::INT nj = a->shape[1] ;
0983     assert( nj == 2 );
0984
0985     const int* aa = a->cvalues<int>();
0986     for(NP::INT i=0 ; i < ni ; i++)
0987     {
0988         int k = aa[i*nj+0] ;
0989         int v = aa[i*nj+1] ;
0990         m[k] = v ;
0991     }
0992 }
0993
0994
0995 template<>
0996 inline void NPX::DiscoMapUnorderedFromArray( std::unordered_map<int, int>& m, const NP* a )
0997 {
0998     assert( a && a->uifc == 'i' && a->ebyte == 4 && a->shape.size() == 2 );
0999     NP::INT ni = a->shape[0] ;
1000     NP::INT nj = a->shape[1] ;
1001     assert( nj == 2 );
1002
1003     const int* aa = a->cvalues<int>();
1004     for(NP::INT i=0 ; i < ni ; i++)
1005     {
1006         int k = aa[i*nj+0] ;
1007         int v = aa[i*nj+1] ;
1008         m[k] = v ;
1009     }
1010 }
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021 template<typename S>
1022 inline std::string NPX::DescDiscoMap( const std::map<int, S>& m )
1023 {
1024     std::stringstream ss ;
1025     ss << "NPX::DescDiscoMap" << std::endl << " m.size " << m.size() ;
1026     std::string s = ss.str();
1027     return s ;
1028 }
1029
1030 template<typename S>
1031 inline std::string NPX::DescDiscoMapUnordered( const std::unordered_map<int, S>& m )
1032 {
1033     std::stringstream ss ;
1034     ss << "NPX::DescDiscoMapUnordered" << std::endl << " m.size " << m.size() ;
1035     std::string s = ss.str();
1036     return s ;
1037 }
1038
1039
1040
1041
1042 template<>
1043 inline std::string NPX::DescDiscoMap( const std::map<int,int>& m )
1044 {
1045     NP::INT ni = m.size() ;
1046     typedef std::map<int,int> MII ;
1047     MII::const_iterator it = m.begin();
1048     std::stringstream ss ;
1049     ss << "NPX::DescDiscoMap" << std::endl << " m.size " << ni << std::endl ;
1050     for(NP::INT i=0 ; i < ni ; i++)
1051     {
1052         ss << "( " << it->first << " : " << it->second << " ) " << std::endl ;
1053         it++ ;
1054     }
1055     std::string s = ss.str();
1056     return s ;
1057 }
1058
1059
1060 template<>
1061 inline std::string NPX::DescDiscoMapUnordered( const std::unordered_map<int,int>& m )
1062 {
1063     NP::INT ni = m.size() ;
1064     typedef std::unordered_map<int,int> MII ;
1065     MII::const_iterator it = m.begin();
1066     std::stringstream ss ;
1067     ss << "NPX::DescDiscoMapUnordered" << std::endl << " m.size " << ni << std::endl ;
1068     for(NP::INT i=0 ; i < ni ; i++)
1069     {
1070         ss << "( " << it->first << " : " << it->second << " ) " << std::endl ;
1071         it++ ;
1072     }
1073     std::string s = ss.str();
1074     return s ;
1075 }
1076
1077
1078
1079
1080
1081
1082 /**
1083 NPX::FromNumpyString
1084 ---------------------
1085
1086
1087 **/
1088
1089
1090 template <typename T>
1091 inline NP* NPX::FromNumpyString(const char* str, bool dump)  // static
1092 {
1093     static const char* dtype_pfx = "dtype=" ;
1094
1095     std::vector<T> vec ;
1096     std::stringstream fss(str);
1097
1098     int num_field_0 = 0 ;
1099     std::string line ;
1100     while(getline(fss, line))
1101     {
1102         if(strlen(line.c_str())==0) continue ;
1103         if(dump) std::cout << "{" << line << "}" << std::endl ;
1104         std::istringstream iss(line);
1105         std::vector<std::string> fields ;
1106         std::string field ;
1107         while( iss >> field ) fields.push_back(field) ;
1108
1109         int num_field = 0 ;
1110
1111         if(dump) std::cout << "fields.size " << fields.size() << std::endl ;
1112         if(dump) for(NP::INT j=0 ; j < NP::INT(fields.size()) ; j++ ) std::cout << "{" << fields[j].c_str() << "}\n" ;
1113
1114         for(NP::INT j=0 ; j < NP::INT(fields.size()) ; j++ )
1115         {
1116            const char* fld = fields[j].c_str() ;
1117            char* fldd = U::FirstToLastDigit(fld);
1118            if(fldd == nullptr) continue ;
1119
1120            // dtype fields like "dtype=int32" contain digits but not ones to grab
1121            bool dtype_field = 0 == strncmp(fld, dtype_pfx, strlen(dtype_pfx));
1122            if(dtype_field) continue ;
1123
1124            T val = U::To<T>(fldd) ;
1125
1126            if(dump) std::cout
1127                << "{" << fld << "}"
1128                << "{" << fldd << "}"
1129                << "{" << val << "}"
1130                << " num_field: " << num_field
1131                <<  std::endl
1132                ;
1133            num_field += 1 ;
1134            vec.push_back( val );
1135         }
1136         if( num_field_0 == 0 )
1137         {
1138             num_field_0 = num_field ;
1139         }
1140         else
1141         {
1142             assert( num_field_0 == num_field );
1143         }
1144     }
1145
1146     NP* a = Make<T>(vec) ;
1147     a->change_shape(-1, num_field_0);
1148
1149     return a ;
1150 }
1151
1152 inline NP* NPX::CategoryArrayFromString(const char* str, int catfield, const char* cats_, char delim )
1153 {
1154     std::vector<std::string> cats ;
1155     U::MakeVec(cats, cats_, delim );
1156
1157     int num_field = 0 ;
1158     std::vector<int> data ;
1159     std::string line ;
1160     std::stringstream fss(str) ;
1161     while(std::getline(fss, line))
1162     {
1163         std::istringstream iss(line);
1164         std::vector<std::string> fields ;
1165         std::string field ;
1166         while( iss >> field ) fields.push_back(field) ;
1167
1168         if(num_field == 0) num_field = fields.size() ;
1169         else  assert( int(fields.size()) == num_field ); // require consistent number of fields
1170
1171         assert( catfield < num_field );
1172         for(int i=0 ; i < num_field ; i++)
1173         {
1174             const std::string& fld = fields[i] ;
1175             int val =  i == catfield  ? U::Category(cats, fld ) : std::atoi(fld.c_str()) ;
1176             data.push_back(val);
1177         }
1178     }
1179
1180     NP* a = Make<int>( data );
1181     a->change_shape(-1, num_field);
1182     a->set_names(cats);
1183     return a ;
1184 }
1185
1186 inline NP* NPX::LoadCategoryArrayFromTxtFile(const char* base, const char* relp, int catfield, const char* cats, char delim  )  // static
1187 {
1188     std::string path = U::form_path(base, relp);
1189     return LoadCategoryArrayFromTxtFile(path.c_str(), catfield, cats, delim) ;
1190 }
1191 inline NP* NPX::LoadCategoryArrayFromTxtFile(const char* path, int catfield, const char* cats, char delim  )  // static
1192 {
1193     const char* str = U::ReadString2(path);
1194     if(str == nullptr) return nullptr ;
1195     NP* a = CategoryArrayFromString(str, catfield, cats, delim );
1196     return a ;
1197 }
1198
1199 inline void NPX::Import_MSD( std::map<std::string, double>& msd, const NP* a) // static
1200 {
1201     assert( a && a->uifc == 'f' && a->ebyte == 8 );
1202     assert( a->shape.size() == 1 );
1203     assert( NP::INT(a->names.size()) == a->shape[0] );
1204
1205     const double* vv = a->cvalues<double>() ;
1206     unsigned num_vals = a->shape[0] ;
1207
1208     for(unsigned i=0 ; i < num_vals ; i++)
1209     {
1210         const std::string& key = a->names[i] ;
1211         const double& val = vv[i] ;
1212         msd[key] = val ;
1213     }
1214 }
1215
1216 inline NP* NPX::Serialize_MSD( const std::map<std::string, double>& msd ) // static
1217 {
1218     typedef std::map<std::string, double> MSD ;
1219     MSD::const_iterator it = msd.begin();
1220
1221     std::vector<std::string> keys ;
1222     std::vector<double>      vals ;
1223
1224     for(unsigned i=0 ; i < msd.size() ; i++)
1225     {
1226         const std::string& key = it->first ;
1227         const double&      val = it->second ;
1228
1229         keys.push_back(key);
1230         vals.push_back(val);
1231
1232         std::advance(it, 1);
1233     }
1234
1235     NP* a = MakeValues( keys, vals );
1236     return a ;
1237 }
1238
1239 inline std::string NPX::Desc_MSD(const std::map<std::string, double>& msd) // static
1240 {
1241     std::stringstream ss ;
1242     ss << "NPX::Desc_MSD" << std::endl ;
1243
1244     typedef std::map<std::string, double> MSD ;
1245     MSD::const_iterator it = msd.begin();
1246     for(unsigned i=0 ; i < msd.size() ; i++)
1247     {
1248         const std::string& key = it->first ;
1249         const double& val = it->second ;
1250         ss << " key " << key << " val " << val << std::endl ;
1251         std::advance(it, 1);
1252     }
1253     std::string s = ss.str();
1254     return s ;
1255 }
1256
1257 inline NP* NPX::ArrayFromEnumMap( const std::map<int, std::string>& catMap)
1258 {
1259     unsigned num_cat = catMap.size() ;
1260     NP* a = NP::Make<int>(num_cat);
1261     int* aa = a->values<int>() ;
1262     typedef std::map<int, std::string> MIS ;
1263     MIS::const_iterator it = catMap.begin();
1264
1265     for(unsigned i=0 ; i < num_cat ; i++)
1266     {
1267         aa[i] = it->first ;
1268         a->names.push_back(it->second) ;
1269         std::advance(it, 1);
1270     }
1271     return a ;
1272 }
1273
1274 inline NP* NPX::MakeCharArray( const std::vector<std::string>& nn )
1275 {
1276     NP::INT ni = NP::INT(nn.size());
1277     NP::INT maxlen = 0 ;
1278     for(NP::INT i=0 ; i < ni ; i++) maxlen = std::max( NP::INT(strlen(nn[i].c_str())), maxlen ) ;
1279     NP::INT nj = maxlen + 1 ;
1280
1281     NP* a = NP::Make<char>(ni, nj) ;
1282     char* aa = a->values<char>() ;
1283     for(NP::INT i=0 ; i < ni ; i++) for(NP::INT j=0 ; j < nj ; j++) aa[i*nj+j] = nn[i].c_str()[j] ;
1284     return a ;
1285 }
1286
1287 /**
1288 NPX::BOA
1289 ---------
1290
1291 Forms the ratio of two columns obtained from two 2D arrays a and b
1292 with shapes (N,M_a) and (N, M_b) creating an (N, 4) array with
1293 B/A in third column and A/B in fourth.
1294
1295 T: int or int64_t of the source arrays
1296 F: float or double of the created array
1297
1298 **/
1299
1300 template<typename F, typename T>
1301 inline NP* NPX::BOA( NP* a, NP* b, NP::INT a_column, NP::INT b_column, std::ostream* out )  // static
1302 {
1303     if(out) *out
1304        << "NPX::BOA"
1305        << " A " << ( a ? a->sstr() : "-" )
1306        << " B " << ( b ? b->sstr() : "-" )
1307        << "\n"
1308        ;
1309
1310     bool abort = a == nullptr || b == nullptr ;
1311     if(abort && out) *out << "NPX::BOA ABORT A or B null \n" ;
1312     if(abort) return nullptr ;
1313
1314     assert( a->shape.size() == 2 );
1315     assert( b->shape.size() == 2 );
1316
1317     NP::INT a_ni = a->shape[0] ;
1318     NP::INT b_ni = b->shape[0] ;
1319
1320     if(a->names.size() == 0) for(NP::INT i=0 ; i < a_ni ; i++) a->names.push_back( U::FormName_("A", i, nullptr )) ;
1321     if(b->names.size() == 0) for(NP::INT i=0 ; i < b_ni ; i++) b->names.push_back( U::FormName_("B", i, nullptr )) ;
1322
1323     assert( NP::INT(a->names.size()) == a_ni );
1324     assert( NP::INT(b->names.size()) == b_ni );
1325
1326     bool samelength = a_ni == b_ni ;
1327     if(!samelength && out) *out << "NPX::BOA ABORT as not same length a_ni " << a_ni << " b_ni " << b_ni << "\n" ;
1328     if(!samelength) return nullptr ;
1329
1330     NP::INT ni = a_ni ;
1331
1332     NP::INT a_nj = a->shape[1] ;
1333     NP::INT b_nj = b->shape[1] ;
1334
1335     const T* aa = a->cvalues<T>();
1336     const T* bb = b->cvalues<T>();
1337
1338     NP::INT c_ni = ni ;
1339     NP::INT c_nj = 4 ;
1340
1341     NP* c = NP::Make<F>(c_ni, c_nj);
1342     c->set_meta<std::string>("creator", "NPX::BOA");
1343     c->set_meta<int>("a_column", a_column );
1344     c->set_meta<int>("b_column", b_column );
1345
1346     F* cc = c->values<F>();
1347
1348     c->labels = new std::vector<std::string>(c_nj)  ;
1349     (*c->labels)[0] = "A" ;
1350     (*c->labels)[1] = "B" ;
1351     (*c->labels)[2] = "B/A" ;
1352     (*c->labels)[3] = "A/B" ;
1353
1354     for(NP::INT i=0 ; i < ni ; i++)
1355     {
1356         T av = aa[i*a_nj+a_nj+a_column] ;
1357         T bv = bb[i*b_nj+b_nj+b_column] ;
1358         F boa = F(bv)/F(av);
1359         F aob = F(av)/F(bv);
1360
1361         cc[i*c_nj+0] = F(av) ;
1362         cc[i*c_nj+1] = F(bv) ;
1363         cc[i*c_nj+2] = boa ;
1364         cc[i*c_nj+3] = aob ;
1365
1366         const char* an = a->names[i].c_str() ;
1367         const char* bn = b->names[i].c_str() ;
1368         std::string name = U::FormName_(bn,":", an );
1369         c->names.push_back(name) ;
1370     }
1371     return c ;
1372 }
1373
1374
1375
1376
1377
1378
1379