Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Display command line parameters with concrete values in verbose mode #2847

Merged
merged 8 commits into from
Nov 13, 2021
Merged

Display command line parameters with concrete values in verbose mode #2847

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
0665d4c
Display command line parameters with concrete values in verbose mode
Svetlitski Nov 5, 2021
63fe619
Display --zstd= subparameters in command-line ready form in verbose mode
Svetlitski Nov 5, 2021
df9b775
Fix const-ness of FIO_displayCompressionParameters
Svetlitski Nov 11, 2021
f6ffd39
Add test case for detailed compression parameter verbose output
Svetlitski Nov 11, 2021
365c911
Ensure print*CParams functions are only defined when used
Svetlitski Nov 11, 2021
375e3aa
Ensure formatting directives for displaying size_t are portable
Svetlitski Nov 11, 2021
7fbd126
Suppress spurious unused parameter warning
Svetlitski Nov 11, 2021
9b28c26
Integrate verbose mode tests into playTests.sh
Svetlitski Nov 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions programs/fileio.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1782,6 +1782,50 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
return result;
}

static const char* checked_index(const char* options[], size_t length, size_t index) {
assert(index < length);
// Necessary to avoid warnings since -O3 will omit the above `assert`
(void) length;
return options[index];
}

#define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (index))

void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) {
static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION,
LZMA_EXTENSION, LZ4_EXTENSION};
static const char* sparseOptions[3] = {" --no-sparse", "", " --sparse"};
static const char* checkSumOptions[3] = {" --no-check", "", " --check"};
static const char* rowMatchFinderOptions[3] = {"", " --no-row-match-finder", " --row-match-finder"};
static const char* compressLiteralsOptions[3] = {"", " --compress-literals", " --no-compress-literals"};

assert(g_display_prefs.displayLevel >= 4);

DISPLAY("--format=%s", formatOptions[prefs->compressionType]);
DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport));
DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID");
DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag));
DISPLAY(" --block-size=%d", prefs->blockSize);
if (prefs->adaptiveMode)
DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel);
DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder));
DISPLAY("%s", prefs->rsyncable ? " --rsyncable" : "");
if (prefs->streamSrcSize)
DISPLAY(" --stream-size=%u", (unsigned) prefs->streamSrcSize);
if (prefs->srcSizeHint)
DISPLAY(" --size-hint=%d", prefs->srcSizeHint);
if (prefs->targetCBlockSize)
DISPLAY(" --target-compressed-block-size=%u", (unsigned) prefs->targetCBlockSize);
DISPLAY("%s", INDEX(compressLiteralsOptions, prefs->literalCompressionMode));
DISPLAY(" --memory=%u", prefs->memLimit ? prefs->memLimit : 128 MB);
DISPLAY(" --threads=%d", prefs->nbWorkers);
DISPLAY("%s", prefs->excludeCompressedFiles ? " --exclude-compressed" : "");
DISPLAY(" --%scontent-size", prefs->contentSize ? "" : "no-");
DISPLAY("\n");
}

#undef INDEX

int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName,
const char* srcFileName, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams)
Expand Down
1 change: 1 addition & 0 deletions programs/fileio.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse
void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices);
void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);

/* FIO_ctx_t functions */
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);
Expand Down
68 changes: 49 additions & 19 deletions programs/zstdcli.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,48 @@ static void printVersion(void)
} }
}

#define ZSTD_NB_STRATEGIES 9
static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast",
"ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2",
"ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"};

#ifndef ZSTD_NOCOMPRESS

static void printDefaultCParams(const char* filename, const char* dictFileName, int cLevel) {
unsigned long long fileSize = UTIL_getFileSize(filename);
const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0;
const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%u bytes)\n", filename, (unsigned)fileSize);
else DISPLAY("%s (src size unknown)\n", filename);
DISPLAY(" - windowLog : %u\n", cParams.windowLog);
DISPLAY(" - chainLog : %u\n", cParams.chainLog);
DISPLAY(" - hashLog : %u\n", cParams.hashLog);
DISPLAY(" - searchLog : %u\n", cParams.searchLog);
DISPLAY(" - minMatch : %u\n", cParams.minMatch);
DISPLAY(" - targetLength : %u\n", cParams.targetLength);
assert(cParams.strategy < ZSTD_NB_STRATEGIES + 1);
DISPLAY(" - strategy : %s (%u)\n", ZSTD_strategyMap[(int)cParams.strategy], (unsigned)cParams.strategy);
}

static void printActualCParams(const char* filename, const char* dictFileName, int cLevel, const ZSTD_compressionParameters* cParams) {
unsigned long long fileSize = UTIL_getFileSize(filename);
const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0;
ZSTD_compressionParameters actualCParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
assert(g_displayLevel >= 4);
actualCParams.windowLog = cParams->windowLog == 0 ? actualCParams.windowLog : cParams->windowLog;
actualCParams.chainLog = cParams->chainLog == 0 ? actualCParams.chainLog : cParams->chainLog;
actualCParams.hashLog = cParams->hashLog == 0 ? actualCParams.hashLog : cParams->hashLog;
actualCParams.searchLog = cParams->searchLog == 0 ? actualCParams.searchLog : cParams->searchLog;
actualCParams.minMatch = cParams->minMatch == 0 ? actualCParams.minMatch : cParams->minMatch;
actualCParams.targetLength = cParams->targetLength == 0 ? actualCParams.targetLength : cParams->targetLength;
actualCParams.strategy = cParams->strategy == 0 ? actualCParams.strategy : cParams->strategy;
DISPLAY("--zstd=wlog=%d,clog=%d,hlog=%d,slog=%d,mml=%d,tlen=%d,strat=%d\n",
actualCParams.windowLog, actualCParams.chainLog, actualCParams.hashLog, actualCParams.searchLog,
actualCParams.minMatch, actualCParams.targetLength, actualCParams.strategy);
}

#endif

/* Environment variables for parameter setting */
#define ENV_CLEVEL "ZSTD_CLEVEL"
#define ENV_NBTHREADS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */
Expand Down Expand Up @@ -723,11 +765,6 @@ static unsigned init_nbThreads(void) {
val32 = readU32FromChar(&__nb); \
}

#define ZSTD_NB_STRATEGIES 9
static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast",
"ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2",
"ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"};

typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;

#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
Expand Down Expand Up @@ -1411,25 +1448,18 @@ int main(int argCount, const char* argv[])
assert(ZSTD_NB_STRATEGIES == strategyBounds.upperBound);
(void)strategyBounds; }

if (showDefaultCParams) {
if (showDefaultCParams || g_displayLevel >= 4) {
size_t fileNb;
for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) {
unsigned long long fileSize = UTIL_getFileSize(filenames->fileNames[fileNb]);
const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0;
const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize);
else DISPLAY("%s (src size unknown)\n", filenames->fileNames[fileNb]);
DISPLAY(" - windowLog : %u\n", cParams.windowLog);
DISPLAY(" - chainLog : %u\n", cParams.chainLog);
DISPLAY(" - hashLog : %u\n", cParams.hashLog);
DISPLAY(" - searchLog : %u\n", cParams.searchLog);
DISPLAY(" - minMatch : %u\n", cParams.minMatch);
DISPLAY(" - targetLength : %u\n", cParams.targetLength);
assert(cParams.strategy < ZSTD_NB_STRATEGIES + 1);
DISPLAY(" - strategy : %s (%u)\n", ZSTD_strategyMap[(int)cParams.strategy], (unsigned)cParams.strategy);
if (showDefaultCParams)
printDefaultCParams(filenames->fileNames[fileNb], dictFileName, cLevel);
if (g_displayLevel >= 4)
printActualCParams(filenames->fileNames[fileNb], dictFileName, cLevel, &compressionParams);
}
}

if (g_displayLevel >= 4)
FIO_displayCompressionParameters(prefs);
if ((filenames->tableSize==1) && outFileName)
operationResult = FIO_compressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams);
else
Expand Down
7 changes: 7 additions & 0 deletions tests/playTests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -753,6 +753,13 @@ datagen -g257000 > tmp_files/tmp3
zstd --show-default-cparams -f -r tmp_files
rm -rf tmp*

println "test : show compression parameters in verbose mode"
datagen > tmp
zstd -vv tmp 2>&1 | \
grep -q -E -- "--zstd=wlog=[[:digit:]]+,clog=[[:digit:]]+,hlog=[[:digit:]]+,\
slog=[[:digit:]]+,mml=[[:digit:]]+,tlen=[[:digit:]]+,strat=[[:digit:]]+"
rm -rf tmp*

println "\n===> Advanced compression parameters "
println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
Expand Down