/* xxhsum - Command line interface for xxhash algorithms Copyright (C) Yann Collet 2012-2015 GPL v2 License This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - xxHash source repository : https://github.com/Cyan4973/xxHash */ /************************************* * Compiler Options *************************************/ /* MS Visual */ #if defined(_MSC_VER) || defined(_WIN32) # define _CRT_SECURE_NO_WARNINGS /* removes visual warnings */ # define BMK_LEGACY_TIMER 1 /* gettimeofday() not supported by MSVC */ #endif /* Under Linux at least, pull in the *64 commands */ #define _LARGEFILE64_SOURCE /************************************* * Includes *************************************/ #include /* malloc */ #include /* fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno */ #include /* strcmp */ #include /* stat64 */ #include /* stat64 */ #include "xxhash.h" /************************************* * OS-Specific Includes *************************************/ /* Use ftime() if gettimeofday() is not available on your target */ #if defined(BMK_LEGACY_TIMER) # include /* timeb, ftime */ #else # include /* gettimeofday */ #endif #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _O_BINARY */ # include /* _setmode, _isatty */ # ifdef __MINGW32__ int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into */ # endif # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else # include /* isatty, STDIN_FILENO */ # define SET_BINARY_MODE(file) # define IS_CONSOLE(stdStream) isatty(STDIN_FILENO) #endif #if !defined(S_ISREG) # define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) #endif /************************************* * Basic Types *************************************/ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; typedef int32_t S32; typedef uint64_t U64; #else typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; typedef signed int S32; typedef unsigned long long U64; #endif static unsigned BMK_isLittleEndian(void) { const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ return one.c[0]; } /************************************** * Constants **************************************/ #define PROGRAM_NAME exename #define PROGRAM_VERSION "" #define COMPILED __DATE__ static const char author[] = "Yann Collet"; #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", PROGRAM_NAME, (int)(sizeof(void*)*8), PROGRAM_VERSION, author, COMPILED #define NBLOOPS 3 /* Default number of benchmark iterations */ #define TIMELOOP 2500 /* Minimum timing per iteration */ #define KB *( 1<<10) #define MB *( 1<<20) #define GB *(1U<<30) #define MAX_MEM (2 GB - 64 MB) static const char stdinName[] = "-"; /************************************* * Display macros *************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYRESULT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) DISPLAY(__VA_ARGS__); static unsigned g_displayLevel = 1; /************************************* * Local variables *************************************/ static int g_nbIterations = NBLOOPS; static int g_fn_selection = 1; /* required within main() & usage() */ /************************************* * Benchmark Functions *************************************/ #if defined(BMK_LEGACY_TIMER) static int BMK_GetMilliStart(void) { /* Based on Legacy ftime() * Rolls over every ~ 12.1 days (0x100000/24/60/60) * Use GetMilliSpan to correct for rollover */ struct timeb tb; int nCount; ftime( &tb ); nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); return nCount; } #else static int BMK_GetMilliStart(void) { /* Based on newer gettimeofday() * Use GetMilliSpan to correct for rollover */ struct timeval tv; int nCount; gettimeofday(&tv, NULL); nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); return nCount; } #endif static int BMK_GetMilliSpan( int nTimeStart ) { int nSpan = BMK_GetMilliStart() - nTimeStart; if ( nSpan < 0 ) nSpan += 0x100000 * 1000; return nSpan; } static size_t BMK_findMaxMem(U64 requiredMem) { size_t step = 64 MB; BYTE* testmem=NULL; requiredMem = (((requiredMem >> 26) + 1) << 26); requiredMem += 2*step; if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; while (!testmem) { if (requiredMem > step) requiredMem -= step; else requiredMem >>= 1; testmem = (BYTE*) malloc ((size_t)requiredMem); } free (testmem); /* keep some space available */ if (requiredMem > step) requiredMem -= step; else requiredMem >>= 1; return (size_t)requiredMem; } static U64 BMK_GetFileSize(const char* infilename) { int r; #if defined(_MSC_VER) struct _stat64 statbuf; r = _stat64(infilename, &statbuf); #else struct stat statbuf; r = stat(infilename, &statbuf); #endif if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ return (U64)statbuf.st_size; } /* Note : buffer is supposed malloc'ed, hence aligned */ static void BMK_benchMem(const void* buffer, size_t bufferSize) { static const int nbh_perloop = 100; /* XXH32 bench */ { int iterationNb; double fastestH = 100000000.; U32 hashResult = 0; DISPLAY("\r%79s\r", ""); /* Clean display line */ if (g_nbIterations<1) g_nbIterations=1; for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) { int nbHashes = 0; int milliTime; DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "XXH32", (int)bufferSize); /* Timing loop */ milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { int i; for (i=0; i %7.1f MB/s\r", iterationNb, "XXH32", (int)bufferSize, (double)bufferSize / fastestH / 1000.); } DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)bufferSize, (double)bufferSize / fastestH / 1000., hashResult); } /* Bench XXH32 on Unaligned input */ { int iterationNb; double fastestH = 100000000.; DISPLAY("\r%79s\r", ""); /* Clean display line */ for (iterationNb = 1; (iterationNb <= g_nbIterations) && ((bufferSize>1)); iterationNb++) { int nbHashes = 0; int milliTime; const char* charPtr = (const char*)buffer; DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "(unaligned)", (int)(bufferSize-1)); /* timing loop */ milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { int i; for (i=0; i %7.1f MB/s\r", iterationNb, "XXH32 (unaligned)", (int)(bufferSize-1), (double)(bufferSize-1) / fastestH / 1000.); } DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)(bufferSize-1), (double)(bufferSize-1) / fastestH / 1000.); } /* Bench XXH64 */ { int iterationNb; double fastestH = 100000000.; unsigned long long h64 = 0; DISPLAY("\r%79s\r", ""); /* Clean display line */ for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) { int nbHashes = 0; int milliTime; DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "XXH64", (int)bufferSize); /* Timing loop */ milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { int i; for (i=0; i %7.1f MB/s\r", iterationNb, "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); } { DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x", "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); DISPLAY("%08X%08X", (U32)(h64 >> 32), (U32)h64); DISPLAY("\n"); } } } static int BMK_benchFiles(const char** fileNamesTable, int nbFiles) { int fileIdx=0; while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) { DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); } buffer = (char*)malloc((size_t )benchedSize+16); if(!buffer) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; } alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); /* align on next 16 bytes boundaries */ /* Fill input buffer */ DISPLAY("\rLoading %s... \n", inFileName); readSize = fread(alignedBuffer, 1, benchedSize, inFile); fclose(inFile); if(readSize != benchedSize) { DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); free(buffer); return 13; } /* bench */ BMK_benchMem(alignedBuffer, benchedSize); free(buffer); } return 0; } static int BMK_benchInternal(void) { static const size_t benchedSize = 100 KB; void* buffer; buffer = malloc(benchedSize); if(!buffer) { DISPLAY("\nError: not enough memory!\n"); return 12; } /* bench */ DISPLAY("\rSample of %u KB... \n", (U32)(benchedSize >> 10)); BMK_benchMem(buffer, benchedSize); free(buffer); return 0; } static void BMK_checkResult(U32 r1, U32 r2) { static int nbTests = 1; if (r1==r2) DISPLAY("\rTest%3i : %08X == %08X ok ", nbTests, r1, r2); else { DISPLAY("\rERROR : Test%3i : %08X <> %08X !!!!! \n", nbTests, r1, r2); exit(1); } nbTests++; } static void BMK_checkResult64(U64 r1, U64 r2) { static int nbTests = 1; if (r1!=r2) { DISPLAY("\rERROR : Test%3i : 64-bits values non equals !!!!! \n", nbTests); DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2>>32), (U32)r2); exit(1); } nbTests++; } static void BMK_testSequence64(void* sentence, int len, U64 seed, U64 Nresult) { U64 Dresult; XXH64_state_t state; int index; Dresult = XXH64(sentence, len, seed); BMK_checkResult64(Dresult, Nresult); XXH64_reset(&state, seed); XXH64_update(&state, sentence, len); Dresult = XXH64_digest(&state); BMK_checkResult64(Dresult, Nresult); XXH64_reset(&state, seed); for (index=0; index>24); byteGen *= byteGen; } BMK_testSequence(NULL, 0, 0, 0x02CC5D05); BMK_testSequence(NULL, 0, prime, 0x36B78AE7); BMK_testSequence(sanityBuffer, 1, 0, 0xB85CBEE5); BMK_testSequence(sanityBuffer, 1, prime, 0xD5845D64); BMK_testSequence(sanityBuffer, 14, 0, 0xE5AA0AB4); BMK_testSequence(sanityBuffer, 14, prime, 0x4481951D); BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x1F1AA412); BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0x498EC8E2); BMK_testSequence64(NULL , 0, 0, 0xEF46DB3751D8E999ULL); BMK_testSequence64(NULL , 0, prime, 0xAC75FDA2929B17EFULL); BMK_testSequence64(sanityBuffer, 1, 0, 0x4FCE394CC88952D8ULL); BMK_testSequence64(sanityBuffer, 1, prime, 0x739840CB819FA723ULL); BMK_testSequence64(sanityBuffer, 14, 0, 0xCFFA8DB881BC3A3DULL); BMK_testSequence64(sanityBuffer, 14, prime, 0x5B9611585EFCC9CBULL); BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x0EAB543384F878ADULL); BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0xCAA65939306F1E21ULL); DISPLAY("\r%79s\r", ""); /* Clean display line */ DISPLAYLEVEL(2, "Sanity check -- all tests ok\n"); } static void BMK_display_BigEndian(const void* ptr, size_t length) { const BYTE* p = (const BYTE*)ptr; size_t index = BMK_isLittleEndian() ? length-1 : 0; int incr = BMK_isLittleEndian() ? -1 : 1; while (index= XXH32_state_t */ /* Check file existence */ if (fileName == stdinName) { inFile = stdin; SET_BINARY_MODE(stdin); } else inFile = fopen( fileName, "rb" ); if (inFile==NULL) { DISPLAY( "Pb opening %s\n", fileName); return 11; } /* Memory allocation & restrictions */ buffer = malloc(blockSize); if(!buffer) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; } /* Init */ switch(hashNb) { case 0: XXH32_reset((XXH32_state_t*)&state, 0); break; case 1: XXH64_reset(&state, 0); break; default: DISPLAY("Error : bad hash algorithm ID\n"); fclose(inFile); free(buffer); return -1; } /* Load file & update hash */ DISPLAY("\rLoading %s... \r", fileName); readSize = 1; while (readSize) { readSize = fread(buffer, 1, blockSize, inFile); switch(hashNb) { case 0: XXH32_update((XXH32_state_t*)&state, buffer, readSize); break; case 1: XXH64_update(&state, buffer, readSize); break; default: break; } } fclose(inFile); free(buffer); /* display Hash */ switch(hashNb) { case 0: { U32 h32 = XXH32_digest((XXH32_state_t*)&state); BMK_display_BigEndian(&h32, 4); DISPLAYRESULT(" %s \n", fileName); break; } case 1: { U64 h64 = XXH64_digest(&state); BMK_display_BigEndian(&h64, 8); DISPLAYRESULT(" %s \n", fileName); break; } default: break; } return 0; } static int BMK_hashFiles(const char** fnList, int fnTotal, U32 hashNb) { int fnNb; int result = 0; if (fnTotal==0) { result = BMK_hash(stdinName, hashNb); } else { for (fnNb=0; fnNb 1) return badusage(exename); if (filenamesStart==0) filenamesStart = argc; return BMK_hashFiles(argv+filenamesStart, argc-filenamesStart, g_fn_selection); }