Skip to content

Commit

Permalink
Merge pull request #2847 from Svetlitski-FB/improve-verbose-output-2
Browse files Browse the repository at this point in the history 
Display command line parameters with concrete values in verbose mode
  • Loading branch information
@Cyan4973
Cyan4973 authored Nov 13, 2021
2 parents 9ba0790 + 9b28c26 commit ddae153
Show file tree
Hide file tree
Showing 4 changed files with 101 additions and 19 deletions.
44 changes: 44 additions & 0 deletions programs/fileio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1782,6 +1782,50 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
return result;
}

static const char* checked_index(const char* options[], size_t length, size_t index) {
assert(index < length);
// Necessary to avoid warnings since -O3 will omit the above `assert`
(void) length;
return options[index];
}

#define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (index))

void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) {
static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION,
LZMA_EXTENSION, LZ4_EXTENSION};
static const char* sparseOptions[3] = {" --no-sparse", "", " --sparse"};
static const char* checkSumOptions[3] = {" --no-check", "", " --check"};
static const char* rowMatchFinderOptions[3] = {"", " --no-row-match-finder", " --row-match-finder"};
static const char* compressLiteralsOptions[3] = {"", " --compress-literals", " --no-compress-literals"};

assert(g_display_prefs.displayLevel >= 4);

DISPLAY("--format=%s", formatOptions[prefs->compressionType]);
DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport));
DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID");
DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag));
DISPLAY(" --block-size=%d", prefs->blockSize);
if (prefs->adaptiveMode)
DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel);
DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder));
DISPLAY("%s", prefs->rsyncable ? " --rsyncable" : "");
if (prefs->streamSrcSize)
DISPLAY(" --stream-size=%u", (unsigned) prefs->streamSrcSize);
if (prefs->srcSizeHint)
DISPLAY(" --size-hint=%d", prefs->srcSizeHint);
if (prefs->targetCBlockSize)
DISPLAY(" --target-compressed-block-size=%u", (unsigned) prefs->targetCBlockSize);
DISPLAY("%s", INDEX(compressLiteralsOptions, prefs->literalCompressionMode));
DISPLAY(" --memory=%u", prefs->memLimit ? prefs->memLimit : 128 MB);
DISPLAY(" --threads=%d", prefs->nbWorkers);
DISPLAY("%s", prefs->excludeCompressedFiles ? " --exclude-compressed" : "");
DISPLAY(" --%scontent-size", prefs->contentSize ? "" : "no-");
DISPLAY("\n");
}

#undef INDEX

int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName,
const char* srcFileName, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams)
Expand Down
1 change: 1 addition & 0 deletions programs/fileio.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse
void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices);
void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);

/* FIO_ctx_t functions */
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);
Expand Down
68 changes: 49 additions & 19 deletions programs/zstdcli.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,48 @@ static void printVersion(void)
} }
}

#define ZSTD_NB_STRATEGIES 9
static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast",
"ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2",
"ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"};

#ifndef ZSTD_NOCOMPRESS

static void printDefaultCParams(const char* filename, const char* dictFileName, int cLevel) {
unsigned long long fileSize = UTIL_getFileSize(filename);
const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0;
const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%u bytes)\n", filename, (unsigned)fileSize);
else DISPLAY("%s (src size unknown)\n", filename);
DISPLAY(" - windowLog : %u\n", cParams.windowLog);
DISPLAY(" - chainLog : %u\n", cParams.chainLog);
DISPLAY(" - hashLog : %u\n", cParams.hashLog);
DISPLAY(" - searchLog : %u\n", cParams.searchLog);
DISPLAY(" - minMatch : %u\n", cParams.minMatch);
DISPLAY(" - targetLength : %u\n", cParams.targetLength);
assert(cParams.strategy < ZSTD_NB_STRATEGIES + 1);
DISPLAY(" - strategy : %s (%u)\n", ZSTD_strategyMap[(int)cParams.strategy], (unsigned)cParams.strategy);
}

static void printActualCParams(const char* filename, const char* dictFileName, int cLevel, const ZSTD_compressionParameters* cParams) {
unsigned long long fileSize = UTIL_getFileSize(filename);
const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0;
ZSTD_compressionParameters actualCParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
assert(g_displayLevel >= 4);
actualCParams.windowLog = cParams->windowLog == 0 ? actualCParams.windowLog : cParams->windowLog;
actualCParams.chainLog = cParams->chainLog == 0 ? actualCParams.chainLog : cParams->chainLog;
actualCParams.hashLog = cParams->hashLog == 0 ? actualCParams.hashLog : cParams->hashLog;
actualCParams.searchLog = cParams->searchLog == 0 ? actualCParams.searchLog : cParams->searchLog;
actualCParams.minMatch = cParams->minMatch == 0 ? actualCParams.minMatch : cParams->minMatch;
actualCParams.targetLength = cParams->targetLength == 0 ? actualCParams.targetLength : cParams->targetLength;
actualCParams.strategy = cParams->strategy == 0 ? actualCParams.strategy : cParams->strategy;
DISPLAY("--zstd=wlog=%d,clog=%d,hlog=%d,slog=%d,mml=%d,tlen=%d,strat=%d\n",
actualCParams.windowLog, actualCParams.chainLog, actualCParams.hashLog, actualCParams.searchLog,
actualCParams.minMatch, actualCParams.targetLength, actualCParams.strategy);
}

#endif

/* Environment variables for parameter setting */
#define ENV_CLEVEL "ZSTD_CLEVEL"
#define ENV_NBTHREADS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */
Expand Down Expand Up @@ -723,11 +765,6 @@ static unsigned init_nbThreads(void) {
val32 = readU32FromChar(&__nb); \
}

#define ZSTD_NB_STRATEGIES 9
static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast",
"ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2",
"ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"};

typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;

#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
Expand Down Expand Up @@ -1411,25 +1448,18 @@ int main(int argCount, const char* argv[])
assert(ZSTD_NB_STRATEGIES == strategyBounds.upperBound);
(void)strategyBounds; }

if (showDefaultCParams) {
if (showDefaultCParams || g_displayLevel >= 4) {
size_t fileNb;
for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) {
unsigned long long fileSize = UTIL_getFileSize(filenames->fileNames[fileNb]);
const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0;
const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize);
else DISPLAY("%s (src size unknown)\n", filenames->fileNames[fileNb]);
DISPLAY(" - windowLog : %u\n", cParams.windowLog);
DISPLAY(" - chainLog : %u\n", cParams.chainLog);
DISPLAY(" - hashLog : %u\n", cParams.hashLog);
DISPLAY(" - searchLog : %u\n", cParams.searchLog);
DISPLAY(" - minMatch : %u\n", cParams.minMatch);
DISPLAY(" - targetLength : %u\n", cParams.targetLength);
assert(cParams.strategy < ZSTD_NB_STRATEGIES + 1);
DISPLAY(" - strategy : %s (%u)\n", ZSTD_strategyMap[(int)cParams.strategy], (unsigned)cParams.strategy);
if (showDefaultCParams)
printDefaultCParams(filenames->fileNames[fileNb], dictFileName, cLevel);
if (g_displayLevel >= 4)
printActualCParams(filenames->fileNames[fileNb], dictFileName, cLevel, &compressionParams);
}
}

if (g_displayLevel >= 4)
FIO_displayCompressionParameters(prefs);
if ((filenames->tableSize==1) && outFileName)
operationResult = FIO_compressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams);
else
Expand Down
7 changes: 7 additions & 0 deletions tests/playTests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -753,6 +753,13 @@ datagen -g257000 > tmp_files/tmp3
zstd --show-default-cparams -f -r tmp_files
rm -rf tmp*

println "test : show compression parameters in verbose mode"
datagen > tmp
zstd -vv tmp 2>&1 | \
grep -q -E -- "--zstd=wlog=[[:digit:]]+,clog=[[:digit:]]+,hlog=[[:digit:]]+,\
slog=[[:digit:]]+,mml=[[:digit:]]+,tlen=[[:digit:]]+,strat=[[:digit:]]+"
rm -rf tmp*

println "\n===> Advanced compression parameters "
println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
Expand Down

0 comments on commit ddae153

Please sign in to comment.