Skip to content

Commit

Permalink
Enable multithreading in lib build by default
Browse files Browse the repository at this point in the history
  • Loading branch information
senhuang42 committed May 6, 2021
1 parent 12c045f commit 31897fd
Show file tree
Hide file tree
Showing 7 changed files with 59 additions and 11 deletions.
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,6 @@ clean:
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku))

HOST_OS = POSIX
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release

HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
EGREP_OPTIONS ?=
Expand Down Expand Up @@ -357,6 +356,8 @@ lz4install:
endif


CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DZSTD_MULTITHREAD_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release

ifneq (,$(filter MSYS%,$(shell uname)))
HOST_OS = MSYS
CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
Expand Down
4 changes: 4 additions & 0 deletions contrib/largeNbDicts/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wredundant-decls
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)

ifneq ($(ZSTD_NO_MULTITHREAD), 1)
CPPFLAGS += -DZSTD_MULTITHREAD
LDFLAGS += -pthread
endif

default: largeNbDicts

Expand Down
5 changes: 5 additions & 0 deletions contrib/seekable_format/examples/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ CPPFLAGS += -I../ -I../../../lib -I../../../lib/common
CFLAGS ?= -O3
CFLAGS += -g

ifneq ($(ZSTD_NO_MULTITHREAD), 1)
CPPFLAGS += -DZSTD_MULTITHREAD
LDFLAGS += -pthread
endif

SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)

.PHONY: default all clean test
Expand Down
5 changes: 5 additions & 0 deletions contrib/seekable_format/tests/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ CFLAGS ?= -O3
CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \
-Wformat=2 -Wstrict-aliasing=1

ifneq ($(ZSTD_NO_MULTITHREAD), 1)
CPPFLAGS += -DZSTD_MULTITHREAD
LDFLAGS += -pthread
endif

SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)

.PHONY: default clean test
Expand Down
40 changes: 32 additions & 8 deletions lib/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
VERSION?= $(LIBVER)
CCVER := $(shell $(CC) --version)

# Multithreading can be forcibly disabled
ZSTD_MULTITHREAD_MODE = 0

# ZSTD_LIB_MINIFY is a helper variable that
# configures a bunch of other variables to space-optimized defaults.
ZSTD_LIB_MINIFY ?= 0
Expand Down Expand Up @@ -216,6 +219,22 @@ SET_CACHE_DIRECTORY = \
CFLAGS="$(CFLAGS)" \
LDFLAGS="$(LDFLAGS)"

%-mt : ZSTD_MULTITHREAD_MODE=1 %
@echo multi-threading build completed

%-nomt : ZSTD_MULTITHREAD_MODE=2
%-nomt : %
@echo single-threaded build completed

ZSTD_MULTITHREAD_MSG = single-threaded static lib, multi-threaded dynamic lib
ifeq ($(ZSTD_MULTITHREAD_MODE), 1)
CPPFLAGS += -DZSTD_MULTITHREAD
LDFLAGS += -pthread
ZSTD_MULTITHREAD_MSG = multi-threaded static and dynamic lib
endif
ifeq ($(ZSTD_MULTITHREAD_MODE), 2)
ZSTD_MULTITHREAD_MSG = single-threaded static and dynamic lib
endif

.PHONY: all
all: lib
Expand All @@ -239,6 +258,7 @@ $(ZSTD_STATLIB): ARFLAGS = rcs
$(ZSTD_STATLIB): | $(ZSTD_STATLIB_DIR)
$(ZSTD_STATLIB): $(ZSTD_STATLIB_OBJ)
@echo compiling static library
@echo ZSTD_MULTITHREAD_MODE is $(ZSTD_MULTITHREAD_MODE)
$(AR) $(ARFLAGS) $@ $^

libzstd.a: $(ZSTD_STATLIB)
Expand Down Expand Up @@ -269,6 +289,11 @@ $(LIBZSTD):
else
# BUILD_DIR is defined

ifeq ($(ZSTD_MULTITHREAD_MODE), 0)
FLAGS += -DZSTD_MULTITHREAD
LDFLAGS += -pthread
endif

ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynamic
ZSTD_DYNLIB := $(ZSTD_DYNLIB_DIR)/$(LIBZSTD)
ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
Expand All @@ -278,6 +303,7 @@ $(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ)
@echo compiling dynamic library $(LIBVER)
$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
@echo creating versioned links
@echo ZSTD_MULTITHREAD_MODE is $(ZSTD_MULTITHREAD_MODE)
ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
ln -sf $@ libzstd.$(SHARED_EXT)

Expand All @@ -287,21 +313,17 @@ $(LIBZSTD): $(ZSTD_DYNLIB)
endif # ifndef BUILD_DIR
endif # if windows


.PHONY: libzstd
libzstd : $(LIBZSTD)

.PHONY: lib
lib : libzstd.a libzstd
@echo $(ZSTD_MULTITHREAD_MSG)


# note : do not define lib-mt or lib-release as .PHONY
# note : do not define lib-release as .PHONY
# make does not consider implicit pattern rule for .PHONY target

%-mt : CPPFLAGS += -DZSTD_MULTITHREAD
%-mt : LDFLAGS += -pthread
%-mt : %
@echo multi-threading build completed

%-release : DEBUGFLAGS :=
%-release : %
@echo release build completed
Expand Down Expand Up @@ -332,11 +354,13 @@ include $(wildcard $(DEPFILES))
# Special case : building library in single-thread mode _and_ without zstdmt_compress.c
ZSTDMT_FILES = compress/zstdmt_compress.c
ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES))
ZSTD_NOMT_FLAGS = $(filter-out -DZSTD_MULTITHREAD,$(FLAGS))
ZSTD_NOMT_LDFLAGS = $(filter-out -pthread,$(LDFLAGS))
libzstd-nomt: LDFLAGS += -shared -fPIC -fvisibility=hidden
libzstd-nomt: $(ZSTD_NOMT_FILES)
@echo compiling single-thread dynamic library $(LIBVER)
@echo files : $(ZSTD_NOMT_FILES)
$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
$(CC) $(ZSTD_NOMT_FLAGS) $^ $(ZSTD_NOMT_LDFLAGS) $(SONAME_FLAGS) -o $@

.PHONY: clean
clean:
Expand Down
8 changes: 6 additions & 2 deletions lib/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,16 @@ The scope can be reduced on demand (see paragraph _modular build_).

#### Multithreading support

Multithreading is disabled by default when building with `make`.
When building with `make`, by default the dynamic library is multithreaded and static library is single-threaded (for compatibility reasons).

Enabling multithreading requires 2 conditions :
- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`)
- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)

Both conditions are automatically applied when invoking `make lib-mt` target.
For convenience, we provide a build target to generate multi and single threaded libraries:
- Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`.
- Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`.
- And as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`.

When linking a POSIX program with a multithreaded version of `libzstd`,
note that it's necessary to invoke the `-pthread` flag during link stage.
Expand Down
5 changes: 5 additions & 0 deletions zlibWrapper/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ TEST_FILE = ../doc/zstd_compression_format.md

vpath %.c $(PROGRAMS_PATH) $(EXAMPLE_PATH) $(ZLIBWRAPPER_PATH)

ZSTD_NO_MULTITHREAD ?= 0
ifneq ($(ZSTD_NO_MULTITHREAD), 1)
CPPFLAGS += -DZSTD_MULTITHREAD
LDFLAGS += -pthread
endif

CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \
-I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
Expand Down

0 comments on commit 31897fd

Please sign in to comment.