eic-opticks/sysrap/SDigest.cc

0001 /*
0002  * Copyright (c) 2019 Opticks Team. All Rights Reserved.
0003  *
0004  * This file is part of Opticks
0005  * (see https://bitbucket.org/simoncblyth/opticks).
0006  *
0007  * Licensed under the Apache License, Version 2.0 (the "License");
0008  * you may not use this file except in compliance with the License.
0009  * You may obtain a copy of the License at
0010  *
0011  *   http://www.apache.org/licenses/LICENSE-2.0
0012  *
0013  * Unless required by applicable law or agreed to in writing, software
0014  * distributed under the License is distributed on an "AS IS" BASIS,
0015  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0016  * See the License for the specific language governing permissions and
0017  * limitations under the License.
0018  */
0019
0020 #include "SDigest.hh"
0021 #include "SSys.hh"
0022
0023 #include <cstdio>
0024 #include <cstdlib>
0025 #include <cstring>
0026 #include <cassert>
0027
0028 #include <iostream>
0029
0030 #include "SLOG.hh"
0031
0032 const plog::Severity SDigest::LEVEL = SLOG::EnvLevel("SDigest", "DEBUG") ;
0033
0034
0035 /**
0036
0037
0038 Byte range covering multiple bufsize::
0039
0040            ----------
0041       i0   ~~~~~~~~~~
0042            ----------
0043
0044            ----------
0045       i1   ~~~~~~~~~~
0046            ----------
0047
0048 Byte range within one bufsize::
0049
0050
0051            ----------
0052       i0   ~~~~~~~~~~
0053
0054       i1   ~~~~~~~~~~
0055            ----------
0056
0057
0058
0059            ----------
0060
0061 **/
0062
0063 std::string SDigest::DigestPathInByteRange(const char* path, int i0, int i1, unsigned bufsize)
0064 {
0065     LOG(LEVEL)
0066         << " path " << path
0067         << " i0 " << i0
0068         << " i1 " << i1
0069         << " bufsize " << bufsize
0070         ;
0071
0072     FILE* fp = fopen(path, "rb");
0073     if (fp == NULL)
0074     {
0075         LOG(error) << "failed to open path [" << path << "]"  ;
0076         return "" ;
0077     }
0078     SDigest dig ;
0079     char* data = new char[bufsize] ;
0080     int bytes ;
0081
0082     int beg = 0 ; // byte index of beginning of buffer in the file
0083     int end = 0 ;
0084     int tot = 0 ;
0085
0086     assert( i1 > i0 ) ;
0087
0088     while ((bytes = fread (data, 1, bufsize, fp)) != 0)
0089     {
0090         end = beg + bytes ;
0091
0092         bool starts = i0 >= beg && i0 < end ;  // capture starts within this bufsize full
0093         bool ends   = i1 <= end ;              // capture ends within this bufsize full
0094
0095         int idx0(-1) ;
0096         int idx1(-1) ;
0097
0098         if( starts && ends )
0099         {
0100             idx0 = i0 - beg ;
0101             idx1 = i1 - beg ;
0102         }
0103         else if( starts && !ends )
0104         {
0105             idx0 = i0 - beg ;
0106             idx1 = bytes ;
0107         }
0108         else if( !starts && ends )
0109         {
0110             idx0 = 0 ;
0111             idx1 = i1 - beg ;
0112         }
0113         else if( !starts && !ends && tot > 0 )  // entire buffer goes to update
0114         {
0115             idx0 = 0 ;
0116             idx1 = bytes ;
0117         }
0118
0119
0120         int nup =  idx1 - idx0 ;
0121         bool update = idx0 > -1 && idx1 > -1 ;
0122
0123         std::string x = update ? SSys::xxd( data+idx0, nup ) : "-" ;
0124
0125         LOG(LEVEL)
0126            << " bytes " << std::setw(8) << bytes
0127            << " beg "  << std::setw(8) << beg
0128            << " end "  << std::setw(8) << end
0129            << " idx0 " << std::setw(8) << idx0
0130            << " idx1 " << std::setw(8) << idx1
0131            << " nup "  << std::setw(8) << nup
0132            << ( starts ? " S " : "   ")
0133            << ( ends ?   " E " : "   ")
0134            << ( update ? " U " : "   ")
0135            << " x " << x
0136            ;
0137
0138         if(update)
0139         {
0140             dig.update(data+idx0, nup );
0141             tot += nup ;
0142         }
0143
0144         if( ends ) break ;
0145
0146         beg += bytes ;
0147     }
0148
0149     delete[] data ;
0150
0151     std::string sdig =  dig.finalize();
0152     LOG(LEVEL) << " sdig " << sdig ;
0153     return sdig ;
0154 }
0155
0156
0157
0158 std::string SDigest::DigestPath(const char* path, unsigned bufsize)
0159 {
0160     FILE* fp = fopen(path, "rb");
0161     if (fp == NULL)
0162     {
0163         std::cerr << "failed to open path [" << path << "]" <<  std::endl ;
0164         return "" ;
0165     }
0166     SDigest dig ;
0167     char* data = new char[bufsize] ;
0168     int bytes ;
0169     while ((bytes = fread (data, 1, bufsize, fp)) != 0) dig.update(data, bytes);
0170     // NB must update just with the bytes read, not the bufsize
0171     delete[] data ;
0172     return dig.finalize();
0173 }
0174
0175
0176
0177
0178 #if defined(__GNUC__) || defined(__clang__)
0179 #pragma GCC diagnostic push
0180 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
0181 #endif
0182
0183 // https://stackoverflow.com/questions/10324611/how-to-calculate-the-md5-hash-of-a-large-file-in-c
0184
0185
0186 std::string SDigest::DigestPath2(const char* path)
0187 {
0188     int i;
0189     FILE* fp = fopen (path, "rb");
0190     MD5_CTX mdContext;
0191     int bytes;
0192     char data[8192];
0193
0194     if (fp == NULL) {
0195         printf ("%s can't be opened.\n", path);
0196         return "";
0197     }
0198
0199     MD5_Init (&mdContext);
0200     while ((bytes = fread (data, 1, 8192, fp)) != 0)
0201         MD5_Update (&mdContext, data, bytes);
0202
0203     fclose (fp);
0204
0205     assert( MD5_DIGEST_LENGTH == 16 );
0206     unsigned char digest[MD5_DIGEST_LENGTH];
0207     MD5_Final (digest,&mdContext);
0208     char *out = (char*)malloc(MD5_DIGEST_LENGTH*2+1);
0209     for(i = 0; i < MD5_DIGEST_LENGTH; i++) snprintf(&(out[i*2]), MD5_DIGEST_LENGTH*2, "%02x", (unsigned int)digest[i]);
0210     out[MD5_DIGEST_LENGTH*2] = '\0' ;
0211
0212     return out ;
0213 }
0214
0215
0216
0217
0218 char* md5digest_str2md5_monolithic(const char *buffer, int length)
0219 {
0220     // user should free the returned string digest
0221
0222     MD5_CTX c;
0223     MD5_Init(&c);
0224
0225     const int blocksize = 512 ;
0226     while (length > 0)
0227     {
0228         if (length > blocksize) {
0229             MD5_Update(&c, buffer, blocksize);
0230         } else {
0231             MD5_Update(&c, buffer, length);
0232         }
0233         length -= blocksize ;
0234         buffer += blocksize ;
0235     }
0236
0237     unsigned char digest[16];
0238     MD5_Final(digest, &c);
0239
0240     // 16 binary bytes, into 32 char hex string
0241     char *out = (char*)malloc(32+1);
0242     for (int n = 0; n < 16; ++n) snprintf(&(out[n*2]), 16*2, "%02x", (unsigned int)digest[n]);
0243     out[32] = '\0' ;
0244     return out;
0245 }
0246
0247
0248 std::string SDigest::Buffer(const char *buffer, int length)
0249 {
0250     MD5_CTX c;
0251     MD5_Init(&c);
0252
0253     const int blocksize = 512 ;
0254     while (length > 0)
0255     {
0256         if (length > blocksize) {
0257             MD5_Update(&c, buffer, blocksize);
0258         } else {
0259             MD5_Update(&c, buffer, length);
0260         }
0261         length -= blocksize ;
0262         buffer += blocksize ;
0263     }
0264
0265     unsigned char digest[16];
0266     MD5_Final(digest, &c);
0267
0268     // 16 binary bytes, into 32 char hex string
0269
0270     char buf[32+1] ;
0271     for (int n = 0; n < 16; ++n) std::snprintf( &buf[2*n], 32+1, "%02x", (unsigned int)digest[n]) ;
0272     buf[32] = '\0' ;
0273
0274     return std::string(buf, buf + 32);
0275 }
0276
0277 void md5digest_str2md5_update(MD5_CTX& ctx, char* buffer, int length)
0278 {
0279     const int blocksize = 512 ;
0280     while (length > 0)
0281     {
0282         if (length > blocksize) {
0283             MD5_Update(&ctx, buffer, blocksize);
0284         } else {
0285             MD5_Update(&ctx, buffer, length);
0286         }
0287         length -= blocksize ;
0288         buffer += blocksize ;
0289     }
0290 }
0291
0292 char* md5digest_str2md5_finalize( MD5_CTX& ctx )
0293 {
0294     unsigned char digest[16];
0295     MD5_Final(digest, &ctx);
0296
0297     // 16 binary bytes, into 32 char hex string
0298     char *out = (char*)malloc(32+1);
0299     for (int n = 0; n < 16; ++n) snprintf(&(out[n*2]), 16*2, "%02x", (unsigned int)digest[n]);
0300     out[32] = '\0' ;
0301     return out;
0302 }
0303
0304 char* md5digest_str2md5(char* buffer, int length)
0305 {
0306     // user should free the returned string digest
0307
0308     MD5_CTX ctx;
0309     MD5_Init(&ctx);
0310
0311     md5digest_str2md5_update(ctx, buffer, length );
0312
0313     return md5digest_str2md5_finalize(ctx);
0314 }
0315
0316
0317 SDigest::SDigest()
0318 {
0319    MD5_Init(&m_ctx);
0320 }
0321
0322
0323 #if defined(__GNUC__) || defined(__clang__)
0324 #pragma GCC diagnostic pop
0325 #endif
0326
0327
0328
0329
0330
0331
0332
0333 const char* SDigest::hexchar = "0123456789abcdef" ;
0334
0335 bool SDigest::IsDigest(const char* s)
0336 {
0337     if( s == NULL ) return false ;
0338     if( strlen(s) != 32 ) return false ;
0339     for(int i=0 ; i < 32 ; i++ )
0340     {
0341         char c = *(s + i) ;
0342         if(strchr(hexchar,c) == NULL) return false  ;
0343     }
0344     return true  ;
0345 }
0346
0347
0348
0349
0350
0351
0352 SDigest::~SDigest()
0353 {
0354 }
0355
0356 void SDigest::update(const std::string& str)
0357 {
0358     update( (char*)str.c_str(), strlen(str.c_str()) );
0359 }
0360
0361 void SDigest::update(char* buffer, int length)
0362 {
0363     md5digest_str2md5_update(m_ctx, buffer, length );
0364 }
0365
0366 void SDigest::update_str(const char* str )
0367 {
0368     md5digest_str2md5_update(m_ctx, (char*)str, strlen(str) );
0369 }
0370
0371
0372 char* SDigest::finalize()
0373 {
0374     return md5digest_str2md5_finalize(m_ctx);
0375 }
0376
0377
0378
0379 std::string SDigest::md5digest( const char* buffer, int len )
0380 {
0381     char* out = md5digest_str2md5_monolithic(buffer, len);
0382     std::string digest(out);
0383     free(out);
0384     return digest;
0385 }
0386
0387 const char* SDigest::md5digest_( const char* buffer, int len )
0388 {
0389     return md5digest_str2md5_monolithic(buffer, len);
0390 }
0391
0392
0393 std::string SDigest::digest( void* buffer, int len )
0394 {
0395     SDigest dig ;
0396     dig.update( (char*)buffer, len );
0397     return dig.finalize();
0398 }
0399
0400 std::string SDigest::digest( std::vector<std::string>& ss)
0401 {
0402     SDigest dig ;
0403     for(unsigned i=0 ; i < ss.size() ; i++) dig.update( ss[i] ) ;
0404     return dig.finalize();
0405 }
0406
0407
0408 std::string SDigest::digest_skipdupe( std::vector<std::string>& ss)
0409 {
0410     SDigest dig ;
0411     for(unsigned i=0 ; i < ss.size() ; i++)
0412     {
0413         if( i > 0 && ss[i].compare(ss[i-1].c_str()) == 0 ) continue ;
0414         dig.update( ss[i] ) ;
0415     }
0416     return dig.finalize();
0417 }
0418