From 4e4ad30b4cdb32bb86ed0b591bd1954514be2021 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 20 Sep 2021 09:04:07 -0400 Subject: [PATCH] Use new paramSwitch enum for row matchfinder and block splitter --- lib/compress/zstd_compress.c | 169 ++++++++++++-------------- lib/compress/zstd_compress_internal.h | 18 +-- lib/compress/zstd_ldm.c | 2 +- lib/compress/zstd_ldm.h | 2 +- lib/compress/zstd_opt.c | 2 +- lib/zstd.h | 64 +++++----- programs/benchzstd.c | 2 +- programs/benchzstd.h | 2 +- programs/fileio.c | 6 +- programs/fileio.h | 2 +- programs/zstdcli.c | 6 +- tests/fuzz/zstd_helpers.c | 2 +- tests/fuzzer.c | 6 +- tests/regression/config.c | 8 +- tests/regression/results.csv | 72 +++++------ 15 files changed, 180 insertions(+), 183 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e68f7352ac7..75c2e3c0976 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -84,10 +84,10 @@ struct ZSTD_CDict_s { ZSTD_customMem customMem; U32 dictID; int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ - ZSTD_useRowMatchFinderMode_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use - * row-based matchfinder. Unless the cdict is reloaded, we will use - * the same greedy/lazy matchfinder at compression time. - */ + ZSTD_paramSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use + * row-based matchfinder. Unless the cdict is reloaded, we will use + * the same greedy/lazy matchfinder at compression time. + */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) @@ -226,35 +226,42 @@ static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) { /* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder * for this compression. */ -static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_useRowMatchFinderMode_e mode) { - assert(mode != ZSTD_urm_auto); - return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_urm_enableRowMatchFinder); +static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_paramSwitch_e mode) { + assert(mode != ZSTD_ps_auto); + return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable); } -/* Returns row matchfinder usage enum given an initial mode and cParams */ -static ZSTD_useRowMatchFinderMode_e ZSTD_resolveRowMatchFinderMode(ZSTD_useRowMatchFinderMode_e mode, - const ZSTD_compressionParameters* const cParams) { +/* Returns row matchfinder usage given an initial mode and cParams */ +static ZSTD_paramSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { #if defined(ZSTD_ARCH_X86_SSE2) || defined(ZSTD_ARCH_ARM_NEON) int const kHasSIMD128 = 1; #else int const kHasSIMD128 = 0; #endif - if (mode != ZSTD_urm_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */ - mode = ZSTD_urm_disableRowMatchFinder; + if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */ + mode = ZSTD_ps_disable; if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode; if (kHasSIMD128) { - if (cParams->windowLog > 14) mode = ZSTD_urm_enableRowMatchFinder; + if (cParams->windowLog > 14) mode = ZSTD_ps_enable; } else { - if (cParams->windowLog > 17) mode = ZSTD_urm_enableRowMatchFinder; + if (cParams->windowLog > 17) mode = ZSTD_ps_enable; } return mode; } +/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */ +static ZSTD_paramSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + /* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, - const ZSTD_useRowMatchFinderMode_e useRowMatchFinder, + const ZSTD_paramSwitch_e useRowMatchFinder, const U32 forDDSDict) { - assert(useRowMatchFinder != ZSTD_urm_auto); + assert(useRowMatchFinder != ZSTD_ps_auto); /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate. * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder. */ @@ -269,14 +276,6 @@ static U32 ZSTD_CParams_shouldEnableLdm(const ZSTD_compressionParameters* const return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27; } -/* Returns 1 if compression parameters are such that we should - * enable blockSplitter (wlog >= 17, strategy >= btopt). - * Returns 0 otherwise. - */ -static U32 ZSTD_CParams_useBlockSplitter(const ZSTD_compressionParameters* const cParams) { - return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17; -} - static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( ZSTD_compressionParameters cParams) { @@ -295,11 +294,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( assert(cctxParams.ldmParams.hashRateLog < 32); } - if (ZSTD_CParams_useBlockSplitter(&cParams)) { - DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including block splitting into cctx params"); - cctxParams.splitBlocks = 1; - } - + cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams); cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); assert(!ZSTD_checkCParams(cParams)); return cctxParams; @@ -360,18 +355,14 @@ static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_par */ cctxParams->compressionLevel = compressionLevel; cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams); - DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d", cctxParams->useRowMatchFinder); + cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, ¶ms->cParams); + DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d", cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter); if (ZSTD_CParams_shouldEnableLdm(¶ms->cParams)) { /* Enable LDM by default for optimal parser and window size >= 128MB */ DEBUGLOG(4, "LDM enabled by default (window size >= 128MB, strategy >= btopt)"); cctxParams->ldmParams.enableLdm = 1; } - - if (ZSTD_CParams_useBlockSplitter(¶ms->cParams)) { - DEBUGLOG(4, "Block splitter enabled by default (window size >= 128K, strategy >= btopt)"); - cctxParams->splitBlocks = 1; - } } size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) @@ -541,9 +532,9 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) return bounds; case ZSTD_c_literalCompressionMode: - ZSTD_STATIC_ASSERT(ZSTD_lcm_auto < ZSTD_lcm_huffman && ZSTD_lcm_huffman < ZSTD_lcm_uncompressed); - bounds.lowerBound = ZSTD_lcm_auto; - bounds.upperBound = ZSTD_lcm_uncompressed; + ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_disable && ZSTD_ps_disable < ZSTD_ps_enable); + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_enable; return bounds; case ZSTD_c_targetCBlockSize: @@ -572,14 +563,14 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = 1; return bounds; - case ZSTD_c_splitBlocks: - bounds.lowerBound = 0; - bounds.upperBound = 1; + case ZSTD_c_useBlockSplitter: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_enable; return bounds; case ZSTD_c_useRowMatchFinder: - bounds.lowerBound = (int)ZSTD_urm_auto; - bounds.upperBound = (int)ZSTD_urm_enableRowMatchFinder; + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_enable; return bounds; case ZSTD_c_deterministicRefPrefix: @@ -648,7 +639,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_stableOutBuffer: case ZSTD_c_blockDelimiters: case ZSTD_c_validateSequences: - case ZSTD_c_splitBlocks: + case ZSTD_c_useBlockSplitter: case ZSTD_c_useRowMatchFinder: case ZSTD_c_deterministicRefPrefix: default: @@ -703,7 +694,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_stableOutBuffer: case ZSTD_c_blockDelimiters: case ZSTD_c_validateSequences: - case ZSTD_c_splitBlocks: + case ZSTD_c_useBlockSplitter: case ZSTD_c_useRowMatchFinder: case ZSTD_c_deterministicRefPrefix: break; @@ -803,7 +794,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, } case ZSTD_c_literalCompressionMode : { - const ZSTD_literalCompressionMode_e lcm = (ZSTD_literalCompressionMode_e)value; + const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value; BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm); CCtxParams->literalCompressionMode = lcm; return CCtxParams->literalCompressionMode; @@ -917,14 +908,15 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->validateSequences = value; return CCtxParams->validateSequences; - case ZSTD_c_splitBlocks: - BOUNDCHECK(ZSTD_c_splitBlocks, value); - CCtxParams->splitBlocks = value; - return CCtxParams->splitBlocks; + case ZSTD_c_useBlockSplitter: + BOUNDCHECK(ZSTD_c_useBlockSplitter, value); + CCtxParams->useBlockSplitter = (ZSTD_paramSwitch_e)value; + return CCtxParams->useBlockSplitter; case ZSTD_c_useRowMatchFinder: BOUNDCHECK(ZSTD_c_useRowMatchFinder, value); - CCtxParams->useRowMatchFinder = (ZSTD_useRowMatchFinderMode_e)value; + + CCtxParams->useRowMatchFinder = (ZSTD_paramSwitch_e)value; return CCtxParams->useRowMatchFinder; case ZSTD_c_deterministicRefPrefix: @@ -1055,8 +1047,8 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_validateSequences : *value = (int)CCtxParams->validateSequences; break; - case ZSTD_c_splitBlocks : - *value = (int)CCtxParams->splitBlocks; + case ZSTD_c_useBlockSplitter : + *value = (int)CCtxParams->useBlockSplitter; break; case ZSTD_c_useRowMatchFinder : *value = (int)CCtxParams->useRowMatchFinder; @@ -1430,7 +1422,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( static size_t ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, - const ZSTD_useRowMatchFinderMode_e useRowMatchFinder, + const ZSTD_paramSwitch_e useRowMatchFinder, const U32 enableDedicatedDictSearch, const U32 forCCtx) { @@ -1463,7 +1455,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); - assert(useRowMatchFinder != ZSTD_urm_auto); + assert(useRowMatchFinder != ZSTD_ps_auto); DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", (U32)chainSize, (U32)hSize, (U32)h3Size); @@ -1474,7 +1466,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( const ZSTD_compressionParameters* cParams, const ldmParams_t* ldmParams, const int isStatic, - const ZSTD_useRowMatchFinderMode_e useRowMatchFinder, + const ZSTD_paramSwitch_e useRowMatchFinder, const size_t buffInSize, const size_t buffOutSize, const U64 pledgedSrcSize) @@ -1519,7 +1511,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) { ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); - ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, &cParams); RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); @@ -1537,9 +1529,9 @@ size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ size_t noRowCCtxSize; size_t rowCCtxSize; - initialParams.useRowMatchFinder = ZSTD_urm_disableRowMatchFinder; + initialParams.useRowMatchFinder = ZSTD_ps_disable; noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); - initialParams.useRowMatchFinder = ZSTD_urm_enableRowMatchFinder; + initialParams.useRowMatchFinder = ZSTD_ps_enable; rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); return MAX(noRowCCtxSize, rowCCtxSize); } else { @@ -1584,7 +1576,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) ? ZSTD_compressBound(blockSize) + 1 : 0; - ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParams); + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParams); return ZSTD_estimateCCtxSize_usingCCtxParams_internal( &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, @@ -1599,9 +1591,9 @@ size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ size_t noRowCCtxSize; size_t rowCCtxSize; - initialParams.useRowMatchFinder = ZSTD_urm_disableRowMatchFinder; + initialParams.useRowMatchFinder = ZSTD_ps_disable; noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); - initialParams.useRowMatchFinder = ZSTD_urm_enableRowMatchFinder; + initialParams.useRowMatchFinder = ZSTD_ps_enable; rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); return MAX(noRowCCtxSize, rowCCtxSize); } else { @@ -1736,7 +1728,7 @@ static size_t ZSTD_reset_matchState(ZSTD_matchState_t* ms, ZSTD_cwksp* ws, const ZSTD_compressionParameters* cParams, - const ZSTD_useRowMatchFinderMode_e useRowMatchFinder, + const ZSTD_paramSwitch_e useRowMatchFinder, const ZSTD_compResetPolicy_e crp, const ZSTD_indexResetPolicy_e forceResetIndex, const ZSTD_resetTarget_e forWho) @@ -1751,7 +1743,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); - assert(useRowMatchFinder != ZSTD_urm_auto); + assert(useRowMatchFinder != ZSTD_ps_auto); if (forceResetIndex == ZSTDirp_reset) { ZSTD_window_init(&ms->window); ZSTD_cwksp_mark_tables_dirty(ws); @@ -1847,8 +1839,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_buffered_policy_e const zbuff) { ZSTD_cwksp* const ws = &zc->workspace; - DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d", - (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder); + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d", + (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->useBlockSplitter); assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); zc->isFirstBlock = 1; @@ -1859,7 +1851,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->appliedParams = *params; params = &zc->appliedParams; - assert(params->useRowMatchFinder != ZSTD_urm_auto); + assert(params->useRowMatchFinder != ZSTD_ps_auto); + assert(params->useBlockSplitter != ZSTD_ps_auto); if (params->ldmParams.enableLdm) { /* Adjust long distance matching parameters */ ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); @@ -2138,7 +2131,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, } ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); - assert(params.useRowMatchFinder != ZSTD_urm_auto); + assert(params.useRowMatchFinder != ZSTD_ps_auto); /* copy tables */ { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */) @@ -2232,8 +2225,10 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, { ZSTD_CCtx_params params = dstCCtx->requestedParams; /* Copy only compression parameters related to tables. */ params.cParams = srcCCtx->appliedParams.cParams; - assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_urm_auto); + assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.useBlockSplitter != ZSTD_ps_auto); params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; + params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter; params.fParams = fParams; ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, /* loadedDictSize */ 0, @@ -2422,11 +2417,13 @@ static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) /* ZSTD_blockSplitterEnabled(): * Returns if block splitting param is being used * If used, compression will do best effort to split a block in order to improve compression ratio. + * At the time this function is called, the parameter must be finalized. * Returns 1 if true, 0 otherwise. */ static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams) { - DEBUGLOG(5, "ZSTD_blockSplitterEnabled(splitBlocks=%d)", cctxParams->splitBlocks); - return (cctxParams->splitBlocks != 0); + DEBUGLOG(5, "ZSTD_blockSplitterEnabled (useBlockSplitter=%d)", cctxParams->useBlockSplitter); + assert(cctxParams->useBlockSplitter != ZSTD_ps_auto); + return (cctxParams->useBlockSplitter == ZSTD_ps_enable); } /* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types @@ -2612,7 +2609,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, size_t const cSize = ZSTD_compressLiterals( &prevEntropy->huf, &nextEntropy->huf, cctxParams->cParams.strategy, - ZSTD_disableLiteralsCompression(cctxParams), + ZSTD_literalsCompressionIsDisabled(cctxParams), op, dstCapacity, literals, litSize, entropyWorkspace, entropyWkspSize, @@ -2721,7 +2718,7 @@ ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr, /* ZSTD_selectBlockCompressor() : * Not static, but internal use only (used by long distance matcher) * assumption : strat is a valid strategy */ -ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_useRowMatchFinderMode_e useRowMatchFinder, ZSTD_dictMode_e dictMode) +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode) { static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { { ZSTD_compressBlock_fast /* default for 0 */, @@ -2786,7 +2783,7 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_useRow ZSTD_compressBlock_lazy2_dedicatedDictSearch_row } }; DEBUGLOG(4, "Selecting a row-based matchfinder"); - assert(useRowMatchFinder != ZSTD_urm_auto); + assert(useRowMatchFinder != ZSTD_ps_auto); selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy]; } else { selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; @@ -3055,7 +3052,7 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi const ZSTD_hufCTables_t* prevHuf, ZSTD_hufCTables_t* nextHuf, ZSTD_hufCTablesMetadata_t* hufMetadata, - const int disableLiteralsCompression, + const int literalsCompressionIsDisabled, void* workspace, size_t wkspSize) { BYTE* const wkspStart = (BYTE*)workspace; @@ -3073,7 +3070,7 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi /* Prepare nextEntropy assuming reusing the existing table */ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - if (disableLiteralsCompression) { + if (literalsCompressionIsDisabled) { DEBUGLOG(5, "set_basic - disabled"); hufMetadata->hType = set_basic; return 0; @@ -3220,7 +3217,7 @@ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr, ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, &prevEntropy->huf, &nextEntropy->huf, &entropyMetadata->hufMetadata, - ZSTD_disableLiteralsCompression(cctxParams), + ZSTD_literalsCompressionIsDisabled(cctxParams), workspace, wkspSize); FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); entropyMetadata->fseMetadata.fseTablesSize = @@ -3723,6 +3720,7 @@ static size_t ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, U32 nbSeq; size_t cSize; DEBUGLOG(4, "ZSTD_compressBlock_splitBlock"); + assert(zc->appliedParams.useBlockSplitter == ZSTD_ps_enable); { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); @@ -3737,7 +3735,6 @@ static size_t ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); } - assert(zc->appliedParams.splitBlocks == 1); cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq); FORWARD_IF_ERROR(cSize, "Splitting blocks failed!"); return cSize; @@ -4252,8 +4249,8 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, assert(ms->chainTable != NULL); ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); } else { - assert(params->useRowMatchFinder != ZSTD_urm_auto); - if (params->useRowMatchFinder == ZSTD_urm_enableRowMatchFinder) { + assert(params->useRowMatchFinder != ZSTD_ps_auto); + if (params->useRowMatchFinder == ZSTD_ps_enable) { size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16); ZSTD_memset(ms->tagTable, 0, tagTableSize); ZSTD_row_update(ms, iend-HASH_READ_SIZE); @@ -4753,7 +4750,7 @@ size_t ZSTD_estimateCDictSize_advanced( + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small * in case we are using DDS with row-hash. */ - + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_urm_auto, &cParams), + + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams), /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); @@ -4830,7 +4827,7 @@ static size_t ZSTD_initCDict_internal( static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_compressionParameters cParams, - ZSTD_useRowMatchFinderMode_e useRowMatchFinder, + ZSTD_paramSwitch_e useRowMatchFinder, U32 enableDedicatedDictSearch, ZSTD_customMem customMem) { @@ -4985,7 +4982,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams) { - ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_urm_auto, &cParams); + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams); /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */ size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0); size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) @@ -5568,11 +5565,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, params.ldmParams.enableLdm = 1; } - if (ZSTD_CParams_useBlockSplitter(¶ms.cParams)) { - DEBUGLOG(4, "Block splitter enabled by default (window size >= 128K, strategy >= btopt)"); - params.splitBlocks = 1; - } - + params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useRowMatchFinder, ¶ms.cParams); params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); #ifdef ZSTD_MULTITHREAD diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index c032d7d18c7..c168c67e759 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -179,7 +179,7 @@ typedef struct { U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ - ZSTD_literalCompressionMode_e literalCompressionMode; + ZSTD_paramSwitch_e literalCompressionMode; } optState_t; typedef struct { @@ -297,7 +297,7 @@ struct ZSTD_CCtx_params_s { * There is no guarantee that hint is close to actual source size */ ZSTD_dictAttachPref_e attachDictPref; - ZSTD_literalCompressionMode_e literalCompressionMode; + ZSTD_paramSwitch_e literalCompressionMode; /* Multithreading: used to pass parameters to mtctx */ int nbWorkers; @@ -320,10 +320,10 @@ struct ZSTD_CCtx_params_s { int validateSequences; /* Block splitting */ - int splitBlocks; + ZSTD_paramSwitch_e useBlockSplitter; /* Param for deciding whether to use row-based matchfinder */ - ZSTD_useRowMatchFinderMode_e useRowMatchFinder; + ZSTD_paramSwitch_e useRowMatchFinder; /* Always load a dictionary in ext-dict mode (not prefix mode)? */ int deterministicRefPrefix; @@ -444,7 +444,7 @@ typedef enum { typedef size_t (*ZSTD_blockCompressor) ( ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); -ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_useRowMatchFinderMode_e rowMatchfinderMode, ZSTD_dictMode_e dictMode); +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode); MEM_STATIC U32 ZSTD_LLcode(U32 litLength) @@ -551,17 +551,17 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) return (srcSize >> minlog) + 2; } -MEM_STATIC int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams) +MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams) { switch (cctxParams->literalCompressionMode) { - case ZSTD_lcm_huffman: + case ZSTD_ps_enable: return 0; - case ZSTD_lcm_uncompressed: + case ZSTD_ps_disable: return 1; default: assert(0 /* impossible: pre-validated */); /* fall-through */ - case ZSTD_lcm_auto: + case ZSTD_ps_auto: return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); } } diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index fa4ebeabd70..c97b97a03bd 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -657,7 +657,7 @@ void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - ZSTD_useRowMatchFinderMode_e useRowMatchFinder, + ZSTD_paramSwitch_e useRowMatchFinder, void const* src, size_t srcSize) { const ZSTD_compressionParameters* const cParams = &ms->cParams; diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h index 393466fa9f5..4e68dbf52e3 100644 --- a/lib/compress/zstd_ldm.h +++ b/lib/compress/zstd_ldm.h @@ -66,7 +66,7 @@ size_t ZSTD_ldm_generateSequences( */ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - ZSTD_useRowMatchFinderMode_e useRowMatchFinder, + ZSTD_paramSwitch_e useRowMatchFinder, void const* src, size_t srcSize); /** diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 9b4b236c0bc..40c25daa1f0 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -65,7 +65,7 @@ MEM_STATIC double ZSTD_fCost(U32 price) static int ZSTD_compressedLiterals(optState_t const* const optPtr) { - return optPtr->literalCompressionMode != ZSTD_lcm_uncompressed; + return optPtr->literalCompressionMode != ZSTD_ps_disable; } static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) diff --git a/lib/zstd.h b/lib/zstd.h index ebe2e04b5b7..96c4c52e3b4 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -417,7 +417,7 @@ typedef enum { * ZSTD_c_stableOutBuffer * ZSTD_c_blockDelimiters * ZSTD_c_validateSequences - * ZSTD_c_splitBlocks + * ZSTD_c_useBlockSplitter * ZSTD_c_useRowMatchFinder * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; @@ -1290,19 +1290,14 @@ typedef enum { } ZSTD_dictAttachPref_e; typedef enum { - ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. - * Negative compression levels will be uncompressed, and positive compression - * levels will be compressed. */ - ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be - * emitted if Huffman compression is not profitable. */ - ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ -} ZSTD_literalCompressionMode_e; - -typedef enum { - ZSTD_urm_auto = 0, /* Automatically determine whether or not we use row matchfinder */ - ZSTD_urm_disableRowMatchFinder = 1, /* Never use row matchfinder */ - ZSTD_urm_enableRowMatchFinder = 2 /* Always use row matchfinder when applicable */ -} ZSTD_useRowMatchFinderMode_e; + /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final + * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable + * or ZSTD_ps_disable allow for a force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_disable = 1, /* Do not use the feature */ + ZSTD_ps_enable = 2 /* Force-enable the feature */ +} ZSTD_paramSwitch_e; /*************************************** * Frame size functions @@ -1729,9 +1724,15 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * See the comments on that enum for an explanation of the feature. */ #define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 -/* Controls how the literals are compressed (default is auto). - * The value must be of type ZSTD_literalCompressionMode_e. - * See ZSTD_literalCompressionMode_e enum definition for details. +/* Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. */ #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 @@ -1883,23 +1884,26 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre */ #define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 -/* ZSTD_c_splitBlocks - * Default is 0 == disabled. Set to 1 to enable block splitting. +/* ZSTD_c_useBlockSplitter + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. * - * Will attempt to split blocks in order to improve compression ratio at the cost of speed. + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. */ -#define ZSTD_c_splitBlocks ZSTD_c_experimentalParam13 +#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13 /* ZSTD_c_useRowMatchFinder - * Default is ZSTD_urm_auto. - * Controlled with ZSTD_useRowMatchFinderMode_e enum. - * - * By default, in ZSTD_urm_auto, when finalizing the compression parameters, the library - * will decide at runtime whether to use the row-based matchfinder based on support for SIMD - * instructions as well as the windowLog. - * - * Set to ZSTD_urm_disableRowMatchFinder to never use row-based matchfinder. - * Set to ZSTD_urm_enableRowMatchFinder to force usage of row-based matchfinder. + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 */ #define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 5cfe624b072..09035ea458e 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -131,7 +131,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) { 0, /* ldmHashLog */ 0, /* ldmBuckSizeLog */ 0, /* ldmHashRateLog */ - ZSTD_lcm_auto, /* literalCompressionMode */ + ZSTD_ps_auto, /* literalCompressionMode */ 0 /* useRowMatchFinder */ }; return res; diff --git a/programs/benchzstd.h b/programs/benchzstd.h index 9b40dcc29bc..11ac85da7f9 100644 --- a/programs/benchzstd.h +++ b/programs/benchzstd.h @@ -116,7 +116,7 @@ typedef struct { int ldmHashLog; int ldmBucketSizeLog; int ldmHashRateLog; - ZSTD_literalCompressionMode_e literalCompressionMode; + ZSTD_paramSwitch_e literalCompressionMode; int useRowMatchFinder; /* use row-based matchfinder if possible */ } BMK_advancedParams_t; diff --git a/programs/fileio.c b/programs/fileio.c index 317ec87e05e..db64ecb645f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -320,7 +320,7 @@ struct FIO_prefs_s { size_t targetCBlockSize; int srcSizeHint; int testMode; - ZSTD_literalCompressionMode_e literalCompressionMode; + ZSTD_paramSwitch_e literalCompressionMode; /* IO preferences */ U32 removeSrcFile; @@ -392,7 +392,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->targetCBlockSize = 0; ret->srcSizeHint = 0; ret->testMode = 0; - ret->literalCompressionMode = ZSTD_lcm_auto; + ret->literalCompressionMode = ZSTD_ps_auto; ret->excludeCompressedFiles = 0; ret->allowBlockDevices = 0; return ret; @@ -510,7 +510,7 @@ void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) { void FIO_setLiteralCompressionMode( FIO_prefs_t* const prefs, - ZSTD_literalCompressionMode_e mode) { + ZSTD_paramSwitch_e mode) { prefs->literalCompressionMode = mode; } diff --git a/programs/fileio.h b/programs/fileio.h index 9d97ec8bfb9..82d5bc84929 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -100,7 +100,7 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint); void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode); void FIO_setLiteralCompressionMode( FIO_prefs_t* const prefs, - ZSTD_literalCompressionMode_e mode); + ZSTD_paramSwitch_e mode); void FIO_setProgressSetting(FIO_progressSetting_e progressSetting); void FIO_setNotificationLevel(int level); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9178a4bef4f..7e4f530f870 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -804,7 +804,7 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOBENCH BMK_advancedParams_t benchParams = BMK_initAdvancedParams(); #endif - ZSTD_literalCompressionMode_e literalCompressionMode = ZSTD_lcm_auto; + ZSTD_paramSwitch_e literalCompressionMode = ZSTD_ps_auto; /* init */ @@ -900,8 +900,8 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); continue; } #endif if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; } - if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; } - if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; } + if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_ps_enable; continue; } + if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_ps_disable; continue; } if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; } if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; } if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; } diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index 4d889deeb45..0fbf3bed7ab 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -96,7 +96,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer); setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer); setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer); - setRand(cctx, ZSTD_c_splitBlocks, 0, 1, producer); + setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer); setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer); if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index c4de54247bf..9be7a38ab03 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1718,7 +1718,7 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++) { ZSTD_CCtx* cctx = ZSTD_createCCtx(); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitBlocks, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) ); cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); CHECK(cSize); ZSTD_freeCCtx(cctx); @@ -1732,7 +1732,7 @@ static int basicUnitTests(U32 const seed, double compressibility) CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) ); cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); CHECK(cSize1); - CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_lcm_uncompressed) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) ); cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); CHECK(cSize2); CHECK_LT(cSize1, cSize2); @@ -2019,7 +2019,7 @@ static int basicUnitTests(U32 const seed, double compressibility) ZSTD_CCtx* const cctx = ZSTD_createCCtx(); size_t nodict_cSize; ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_urm_enableRowMatchFinder); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable); nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, contentStart, contentSize); if (nodict_cSize > target_nodict_cSize[l]) { diff --git a/tests/regression/config.c b/tests/regression/config.c index 4c66dd15064..ac34b1ddc16 100644 --- a/tests/regression/config.c +++ b/tests/regression/config.c @@ -215,7 +215,7 @@ static config_t mt_ldm = { static param_value_t mt_advanced_param_values[] = { {.param = ZSTD_c_nbWorkers, .value = 2}, - {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable}, }; static config_t mt_advanced = { @@ -258,7 +258,7 @@ static config_t small_clog = { static param_value_t const uncompressed_literals_param_values[] = { {.param = ZSTD_c_compressionLevel, .value = 3}, - {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable}, }; static config_t uncompressed_literals = { @@ -269,7 +269,7 @@ static config_t uncompressed_literals = { static param_value_t const uncompressed_literals_opt_param_values[] = { {.param = ZSTD_c_compressionLevel, .value = 19}, - {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable}, }; static config_t uncompressed_literals_opt = { @@ -280,7 +280,7 @@ static config_t uncompressed_literals_opt = { static param_value_t const huffman_literals_param_values[] = { {.param = ZSTD_c_compressionLevel, .value = -1}, - {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_huffman}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable}, }; static config_t huffman_literals = { diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 9909a15a014..879923b1cf1 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -236,17 +236,17 @@ silesia, level 1, advanced silesia, level 3, advanced one pass, 4849553 silesia, level 4, advanced one pass, 4786968 silesia, level 5 row 1, advanced one pass, 4640752 -silesia, level 5 row 2, advanced one pass, 4638961 +silesia, level 5 row 2, advanced one pass, 4620764 silesia, level 5, advanced one pass, 4638961 silesia, level 6, advanced one pass, 4605369 silesia, level 7 row 1, advanced one pass, 4564868 -silesia, level 7 row 2, advanced one pass, 4567204 +silesia, level 7 row 2, advanced one pass, 4548490 silesia, level 7, advanced one pass, 4567204 silesia, level 9, advanced one pass, 4543310 silesia, level 11 row 1, advanced one pass, 4519288 -silesia, level 11 row 2, advanced one pass, 4521399 +silesia, level 11 row 2, advanced one pass, 4501980 silesia, level 12 row 1, advanced one pass, 4503116 -silesia, level 12 row 2, advanced one pass, 4505153 +silesia, level 12 row 2, advanced one pass, 4485616 silesia, level 13, advanced one pass, 4493990 silesia, level 16, advanced one pass, 4359864 silesia, level 19, advanced one pass, 4296880 @@ -260,7 +260,7 @@ silesia, small chain log, advanced silesia, explicit params, advanced one pass, 4795856 silesia, uncompressed literals, advanced one pass, 5127982 silesia, uncompressed literals optimal, advanced one pass, 4319518 -silesia, huffman literals, advanced one pass, 5326346 +silesia, huffman literals, advanced one pass, 6177565 silesia, multithreaded with advanced params, advanced one pass, 5127982 silesia.tar, level -5, advanced one pass, 7359401 silesia.tar, level -3, advanced one pass, 6901672 @@ -270,17 +270,17 @@ silesia.tar, level 1, advanced silesia.tar, level 3, advanced one pass, 4861424 silesia.tar, level 4, advanced one pass, 4799632 silesia.tar, level 5 row 1, advanced one pass, 4652862 -silesia.tar, level 5 row 2, advanced one pass, 4650202 +silesia.tar, level 5 row 2, advanced one pass, 4625000 silesia.tar, level 5, advanced one pass, 4650202 silesia.tar, level 6, advanced one pass, 4616811 silesia.tar, level 7 row 1, advanced one pass, 4575393 -silesia.tar, level 7 row 2, advanced one pass, 4576829 +silesia.tar, level 7 row 2, advanced one pass, 4550307 silesia.tar, level 7, advanced one pass, 4576829 silesia.tar, level 9, advanced one pass, 4552584 silesia.tar, level 11 row 1, advanced one pass, 4529461 -silesia.tar, level 11 row 2, advanced one pass, 4530256 +silesia.tar, level 11 row 2, advanced one pass, 4501989 silesia.tar, level 12 row 1, advanced one pass, 4513604 -silesia.tar, level 12 row 2, advanced one pass, 4514568 +silesia.tar, level 12 row 2, advanced one pass, 4486023 silesia.tar, level 13, advanced one pass, 4502956 silesia.tar, level 16, advanced one pass, 4360527 silesia.tar, level 19, advanced one pass, 4267266 @@ -294,7 +294,7 @@ silesia.tar, small chain log, advanced silesia.tar, explicit params, advanced one pass, 4807381 silesia.tar, uncompressed literals, advanced one pass, 5129458 silesia.tar, uncompressed literals optimal, advanced one pass, 4310141 -silesia.tar, huffman literals, advanced one pass, 5344545 +silesia.tar, huffman literals, advanced one pass, 6182241 silesia.tar, multithreaded with advanced params, advanced one pass, 5129555 github, level -5, advanced one pass, 232315 github, level -5 with dict, advanced one pass, 46718 @@ -419,7 +419,7 @@ github, small chain log, advanced github, explicit params, advanced one pass, 137727 github, uncompressed literals, advanced one pass, 165915 github, uncompressed literals optimal, advanced one pass, 157227 -github, huffman literals, advanced one pass, 142365 +github, huffman literals, advanced one pass, 175468 github, multithreaded with advanced params, advanced one pass, 165915 github.tar, level -5, advanced one pass, 66914 github.tar, level -5 with dict, advanced one pass, 51525 @@ -544,7 +544,7 @@ github.tar, small chain log, advanced github.tar, explicit params, advanced one pass, 41227 github.tar, uncompressed literals, advanced one pass, 41122 github.tar, uncompressed literals optimal, advanced one pass, 35397 -github.tar, huffman literals, advanced one pass, 38853 +github.tar, huffman literals, advanced one pass, 42560 github.tar, multithreaded with advanced params, advanced one pass, 41122 silesia, level -5, advanced one pass small out, 7354675 silesia, level -3, advanced one pass small out, 6902374 @@ -554,17 +554,17 @@ silesia, level 1, advanced silesia, level 3, advanced one pass small out, 4849553 silesia, level 4, advanced one pass small out, 4786968 silesia, level 5 row 1, advanced one pass small out, 4640752 -silesia, level 5 row 2, advanced one pass small out, 4638961 +silesia, level 5 row 2, advanced one pass small out, 4620764 silesia, level 5, advanced one pass small out, 4638961 silesia, level 6, advanced one pass small out, 4605369 silesia, level 7 row 1, advanced one pass small out, 4564868 -silesia, level 7 row 2, advanced one pass small out, 4567204 +silesia, level 7 row 2, advanced one pass small out, 4548490 silesia, level 7, advanced one pass small out, 4567204 silesia, level 9, advanced one pass small out, 4543310 silesia, level 11 row 1, advanced one pass small out, 4519288 -silesia, level 11 row 2, advanced one pass small out, 4521399 +silesia, level 11 row 2, advanced one pass small out, 4501980 silesia, level 12 row 1, advanced one pass small out, 4503116 -silesia, level 12 row 2, advanced one pass small out, 4505153 +silesia, level 12 row 2, advanced one pass small out, 4485616 silesia, level 13, advanced one pass small out, 4493990 silesia, level 16, advanced one pass small out, 4359864 silesia, level 19, advanced one pass small out, 4296880 @@ -578,7 +578,7 @@ silesia, small chain log, advanced silesia, explicit params, advanced one pass small out, 4795856 silesia, uncompressed literals, advanced one pass small out, 5127982 silesia, uncompressed literals optimal, advanced one pass small out, 4319518 -silesia, huffman literals, advanced one pass small out, 5326346 +silesia, huffman literals, advanced one pass small out, 6177565 silesia, multithreaded with advanced params, advanced one pass small out, 5127982 silesia.tar, level -5, advanced one pass small out, 7359401 silesia.tar, level -3, advanced one pass small out, 6901672 @@ -588,17 +588,17 @@ silesia.tar, level 1, advanced silesia.tar, level 3, advanced one pass small out, 4861424 silesia.tar, level 4, advanced one pass small out, 4799632 silesia.tar, level 5 row 1, advanced one pass small out, 4652862 -silesia.tar, level 5 row 2, advanced one pass small out, 4650202 +silesia.tar, level 5 row 2, advanced one pass small out, 4625000 silesia.tar, level 5, advanced one pass small out, 4650202 silesia.tar, level 6, advanced one pass small out, 4616811 silesia.tar, level 7 row 1, advanced one pass small out, 4575393 -silesia.tar, level 7 row 2, advanced one pass small out, 4576829 +silesia.tar, level 7 row 2, advanced one pass small out, 4550307 silesia.tar, level 7, advanced one pass small out, 4576829 silesia.tar, level 9, advanced one pass small out, 4552584 silesia.tar, level 11 row 1, advanced one pass small out, 4529461 -silesia.tar, level 11 row 2, advanced one pass small out, 4530256 +silesia.tar, level 11 row 2, advanced one pass small out, 4501989 silesia.tar, level 12 row 1, advanced one pass small out, 4513604 -silesia.tar, level 12 row 2, advanced one pass small out, 4514568 +silesia.tar, level 12 row 2, advanced one pass small out, 4486023 silesia.tar, level 13, advanced one pass small out, 4502956 silesia.tar, level 16, advanced one pass small out, 4360527 silesia.tar, level 19, advanced one pass small out, 4267266 @@ -612,7 +612,7 @@ silesia.tar, small chain log, advanced silesia.tar, explicit params, advanced one pass small out, 4807381 silesia.tar, uncompressed literals, advanced one pass small out, 5129458 silesia.tar, uncompressed literals optimal, advanced one pass small out, 4310141 -silesia.tar, huffman literals, advanced one pass small out, 5344545 +silesia.tar, huffman literals, advanced one pass small out, 6182241 silesia.tar, multithreaded with advanced params, advanced one pass small out, 5129555 github, level -5, advanced one pass small out, 232315 github, level -5 with dict, advanced one pass small out, 46718 @@ -737,7 +737,7 @@ github, small chain log, advanced github, explicit params, advanced one pass small out, 137727 github, uncompressed literals, advanced one pass small out, 165915 github, uncompressed literals optimal, advanced one pass small out, 157227 -github, huffman literals, advanced one pass small out, 142365 +github, huffman literals, advanced one pass small out, 175468 github, multithreaded with advanced params, advanced one pass small out, 165915 github.tar, level -5, advanced one pass small out, 66914 github.tar, level -5 with dict, advanced one pass small out, 51525 @@ -862,7 +862,7 @@ github.tar, small chain log, advanced github.tar, explicit params, advanced one pass small out, 41227 github.tar, uncompressed literals, advanced one pass small out, 41122 github.tar, uncompressed literals optimal, advanced one pass small out, 35397 -github.tar, huffman literals, advanced one pass small out, 38853 +github.tar, huffman literals, advanced one pass small out, 42560 github.tar, multithreaded with advanced params, advanced one pass small out, 41122 silesia, level -5, advanced streaming, 7292053 silesia, level -3, advanced streaming, 6867875 @@ -872,17 +872,17 @@ silesia, level 1, advanced silesia, level 3, advanced streaming, 4849553 silesia, level 4, advanced streaming, 4786968 silesia, level 5 row 1, advanced streaming, 4640752 -silesia, level 5 row 2, advanced streaming, 4638961 +silesia, level 5 row 2, advanced streaming, 4620764 silesia, level 5, advanced streaming, 4638961 silesia, level 6, advanced streaming, 4605369 silesia, level 7 row 1, advanced streaming, 4564868 -silesia, level 7 row 2, advanced streaming, 4567204 +silesia, level 7 row 2, advanced streaming, 4548490 silesia, level 7, advanced streaming, 4567204 silesia, level 9, advanced streaming, 4543310 silesia, level 11 row 1, advanced streaming, 4519288 -silesia, level 11 row 2, advanced streaming, 4521399 +silesia, level 11 row 2, advanced streaming, 4501980 silesia, level 12 row 1, advanced streaming, 4503116 -silesia, level 12 row 2, advanced streaming, 4505153 +silesia, level 12 row 2, advanced streaming, 4485616 silesia, level 13, advanced streaming, 4493990 silesia, level 16, advanced streaming, 4359864 silesia, level 19, advanced streaming, 4296880 @@ -896,7 +896,7 @@ silesia, small chain log, advanced silesia, explicit params, advanced streaming, 4795883 silesia, uncompressed literals, advanced streaming, 5127982 silesia, uncompressed literals optimal, advanced streaming, 4319518 -silesia, huffman literals, advanced streaming, 5332234 +silesia, huffman literals, advanced streaming, 6183923 silesia, multithreaded with advanced params, advanced streaming, 5127982 silesia.tar, level -5, advanced streaming, 7260007 silesia.tar, level -3, advanced streaming, 6845151 @@ -906,17 +906,17 @@ silesia.tar, level 1, advanced silesia.tar, level 3, advanced streaming, 4861426 silesia.tar, level 4, advanced streaming, 4799632 silesia.tar, level 5 row 1, advanced streaming, 4652866 -silesia.tar, level 5 row 2, advanced streaming, 4650207 +silesia.tar, level 5 row 2, advanced streaming, 4625005 silesia.tar, level 5, advanced streaming, 4650207 silesia.tar, level 6, advanced streaming, 4616816 silesia.tar, level 7 row 1, advanced streaming, 4575394 -silesia.tar, level 7 row 2, advanced streaming, 4576831 +silesia.tar, level 7 row 2, advanced streaming, 4550309 silesia.tar, level 7, advanced streaming, 4576831 silesia.tar, level 9, advanced streaming, 4552590 silesia.tar, level 11 row 1, advanced streaming, 4529461 -silesia.tar, level 11 row 2, advanced streaming, 4530258 +silesia.tar, level 11 row 2, advanced streaming, 4501991 silesia.tar, level 12 row 1, advanced streaming, 4513604 -silesia.tar, level 12 row 2, advanced streaming, 4514569 +silesia.tar, level 12 row 2, advanced streaming, 4486023 silesia.tar, level 13, advanced streaming, 4502956 silesia.tar, level 16, advanced streaming, 4360527 silesia.tar, level 19, advanced streaming, 4267266 @@ -930,7 +930,7 @@ silesia.tar, small chain log, advanced silesia.tar, explicit params, advanced streaming, 4807401 silesia.tar, uncompressed literals, advanced streaming, 5129461 silesia.tar, uncompressed literals optimal, advanced streaming, 4310141 -silesia.tar, huffman literals, advanced streaming, 5350519 +silesia.tar, huffman literals, advanced streaming, 6187938 silesia.tar, multithreaded with advanced params, advanced streaming, 5129555 github, level -5, advanced streaming, 232315 github, level -5 with dict, advanced streaming, 46718 @@ -1055,7 +1055,7 @@ github, small chain log, advanced github, explicit params, advanced streaming, 137727 github, uncompressed literals, advanced streaming, 165915 github, uncompressed literals optimal, advanced streaming, 157227 -github, huffman literals, advanced streaming, 142365 +github, huffman literals, advanced streaming, 175468 github, multithreaded with advanced params, advanced streaming, 165915 github.tar, level -5, advanced streaming, 64132 github.tar, level -5 with dict, advanced streaming, 48642 @@ -1180,7 +1180,7 @@ github.tar, small chain log, advanced github.tar, explicit params, advanced streaming, 41227 github.tar, uncompressed literals, advanced streaming, 41122 github.tar, uncompressed literals optimal, advanced streaming, 35397 -github.tar, huffman literals, advanced streaming, 38874 +github.tar, huffman literals, advanced streaming, 42536 github.tar, multithreaded with advanced params, advanced streaming, 41122 silesia, level -5, old streaming, 7292053 silesia, level -3, old streaming, 6867875