Skip to content

Commit

Permalink
build,tools: make addons tests work with GN
Browse files Browse the repository at this point in the history
PR-URL: #50737
Reviewed-By: Joyee Cheung <[email protected]>
Reviewed-By: Benjamin Gruenbaum <[email protected]>
Reviewed-By: Yagiz Nizipli <[email protected]>
  • Loading branch information
zcbenz authored and marco-ippolito committed Feb 27, 2024
1 parent 9be6b7c commit 5911731
Show file tree
Hide file tree
Showing 9 changed files with 469 additions and 331 deletions.
48 changes: 32 additions & 16 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -187,11 +187,11 @@ config.gypi: configure configure.py src/node_version.h

.PHONY: install
install: all ## Installs node into $PREFIX (default=/usr/local).
$(PYTHON) tools/install.py $@ '$(DESTDIR)' '$(PREFIX)'
$(PYTHON) tools/install.py $@ --dest-dir '$(DESTDIR)' --prefix '$(PREFIX)'

.PHONY: uninstall
uninstall: ## Uninstalls node from $PREFIX (default=/usr/local).
$(PYTHON) tools/install.py $@ '$(DESTDIR)' '$(PREFIX)'
$(PYTHON) tools/install.py $@ --dest-dir '$(DESTDIR)' --prefix '$(PREFIX)'

.PHONY: clean
.NOTPARALLEL: clean
Expand Down Expand Up @@ -379,6 +379,28 @@ test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) tools/doc/node_modules
[ $$? -eq 0 ] && touch $@; \
fi

# All files that will be included in headers tarball should be listed as deps
# for generating headers. The list is manually synchronized with install.py.
ADDONS_HEADERS_PREREQS := tools/install.py \
config.gypi common.gypi \
$(wildcard deps/openssl/config/*.h) \
$(wildcard deps/openssl/openssl/include/openssl/*.h) \
$(wildcard deps/uv/include/*.h) \
$(wildcard deps/uv/include/*/*.h) \
$(wildcard deps/v8/include/*.h) \
$(wildcard deps/v8/include/*/*.h) \
deps/zlib/zconf.h deps/zlib/zlib.h \
src/node.h src/node_api.h src/js_native_api.h src/js_native_api_types.h \
src/node_api_types.h src/node_buffer.h src/node_object_wrap.h \
src/node_version.h

ADDONS_HEADERS_DIR = out/$(BUILDTYPE)/addons_headers

# Generate node headers which will be used for building addons.
test/addons/.headersbuildstamp: $(ADDONS_HEADERS_PREREQS)
$(PYTHON) tools/install.py install --headers-only --dest-dir '$(ADDONS_HEADERS_DIR)' --prefix '/'
@touch $@

ADDONS_BINDING_GYPS := \
$(filter-out test/addons/??_*/binding.gyp, \
$(wildcard test/addons/*/binding.gyp))
Expand All @@ -387,16 +409,11 @@ ADDONS_BINDING_SOURCES := \
$(filter-out test/addons/??_*/*.cc, $(wildcard test/addons/*/*.cc)) \
$(filter-out test/addons/??_*/*.h, $(wildcard test/addons/*/*.h))

ADDONS_PREREQS := config.gypi \
deps/npm/node_modules/node-gyp/package.json tools/build-addons.mjs \
deps/uv/include/*.h deps/v8/include/*.h \
src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h
ADDONS_PREREQS := test/addons/.headersbuildstamp \
deps/npm/node_modules/node-gyp/package.json tools/build_addons.py

define run_build_addons
env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \
npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons.mjs" \
"$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" \
$1
env $(PYTHON) "$$PWD/tools/build_addons.py" --loglevel=$(LOGLEVEL) --headers-dir "$(ADDONS_HEADERS_DIR)" $1
touch $2
endef

Expand Down Expand Up @@ -429,8 +446,7 @@ JS_NATIVE_API_BINDING_SOURCES := \
# Implicitly depends on $(NODE_EXE), see the build-js-native-api-tests rule for rationale.
test/js-native-api/.buildstamp: $(ADDONS_PREREQS) \
$(JS_NATIVE_API_BINDING_GYPS) $(JS_NATIVE_API_BINDING_SOURCES) \
src/node_api.h src/node_api_types.h src/js_native_api.h \
src/js_native_api_types.h src/js_native_api_v8.h src/js_native_api_v8_internals.h
src/js_native_api_v8.h src/js_native_api_v8_internals.h
@$(call run_build_addons,"$$PWD/test/js-native-api",$@)

.PHONY: build-js-native-api-tests
Expand All @@ -454,8 +470,7 @@ NODE_API_BINDING_SOURCES := \
# Implicitly depends on $(NODE_EXE), see the build-node-api-tests rule for rationale.
test/node-api/.buildstamp: $(ADDONS_PREREQS) \
$(NODE_API_BINDING_GYPS) $(NODE_API_BINDING_SOURCES) \
src/node_api.h src/node_api_types.h src/js_native_api.h \
src/js_native_api_types.h src/js_native_api_v8.h src/js_native_api_v8_internals.h
src/js_native_api_v8.h src/js_native_api_v8_internals.h
@$(call run_build_addons,"$$PWD/test/node-api",$@)

.PHONY: build-node-api-tests
Expand Down Expand Up @@ -660,9 +675,10 @@ test-addons: test-build test-js-native-api test-node-api
.PHONY: test-addons-clean
.NOTPARALLEL: test-addons-clean
test-addons-clean:
$(RM) -r "$(ADDONS_HEADERS_DIR)"
$(RM) -r test/addons/??_*/
$(RM) -r test/addons/*/build
$(RM) test/addons/.buildstamp test/addons/.docbuildstamp
$(RM) test/addons/.buildstamp test/addons/.docbuildstamp test/addons/.headersbuildstamp
$(MAKE) test-js-native-api-clean
$(MAKE) test-node-api-clean

Expand Down Expand Up @@ -1216,7 +1232,7 @@ $(TARBALL)-headers: release-only
--tag=$(TAG) \
--release-urlbase=$(RELEASE_URLBASE) \
$(CONFIG_FLAGS) $(BUILD_RELEASE_FLAGS)
HEADERS_ONLY=1 $(PYTHON) tools/install.py install '$(TARNAME)' '/'
$(PYTHON) tools/install.py install --headers-only --dest-dir '$(TARNAME)' --prefix '/'
find $(TARNAME)/ -type l | xargs $(RM)
tar -cf $(TARNAME)-headers.tar $(TARNAME)
$(RM) -r $(TARNAME)
Expand Down
5 changes: 5 additions & 0 deletions deps/openssl/unofficial.gni
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,11 @@ template("openssl_gn_build") {
configs += [ ":openssl_internal_config" ]
public_configs = [ ":openssl_external_config" ]

if (is_posix) {
configs -= [ "//build/config/gcc:symbol_visibility_hidden" ]
configs += [ "//build/config/gcc:symbol_visibility_default" ]
}

config_path_name = ""
if (is_win) {
if (target_cpu == "x86") {
Expand Down
5 changes: 5 additions & 0 deletions deps/uv/unofficial.gni
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,11 @@ template("uv_gn_build") {
configs += [ ":uv_internal_config" ]
public_configs = [ ":uv_external_config" ]

if (is_posix) {
configs -= [ "//build/config/gcc:symbol_visibility_hidden" ]
configs += [ "//build/config/gcc:symbol_visibility_default" ]
}

if (is_win) {
libs = [
"advapi32.lib",
Expand Down
1 change: 0 additions & 1 deletion doc/contributing/collaborator-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,6 @@ There are some other files that touch the build chain. Changes in the following
files also qualify as affecting the `node` binary:

* `tools/*.py`
* `tools/build-addons.mjs`
* `*.gyp`
* `*.gypi`
* `configure`
Expand Down
65 changes: 0 additions & 65 deletions tools/build-addons.mjs

This file was deleted.

145 changes: 145 additions & 0 deletions tools/build_addons.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
#!/usr/bin/env python3

import argparse
import os
import shutil
import subprocess
import sys
import tempfile
from concurrent.futures import ThreadPoolExecutor

ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..'))

# Run install.py to install headers.
def generate_headers(headers_dir, install_args):
print('Generating headers')
subprocess.check_call([
sys.executable,
os.path.join(ROOT_DIR, 'tools/install.py'),
'install',
'--silent',
'--headers-only',
'--prefix', '/',
'--dest-dir', headers_dir,
] + install_args)

# Rebuild addons in parallel.
def rebuild_addons(args):
headers_dir = os.path.abspath(args.headers_dir)
out_dir = os.path.abspath(args.out_dir)
node_bin = os.path.join(out_dir, 'node')
if args.is_win:
node_bin += '.exe'

if os.path.isabs(args.node_gyp):
node_gyp = args.node_gyp
else:
node_gyp = os.path.join(ROOT_DIR, args.node_gyp)

# Copy node.lib.
if args.is_win:
node_lib_dir = os.path.join(headers_dir, 'Release')
os.makedirs(node_lib_dir)
shutil.copy2(os.path.join(args.out_dir, 'node.lib'),
os.path.join(node_lib_dir, 'node.lib'))

def node_gyp_rebuild(test_dir):
print('Building addon in', test_dir)
try:
process = subprocess.Popen([
node_bin,
node_gyp,
'rebuild',
'--directory', test_dir,
'--nodedir', headers_dir,
'--python', sys.executable,
'--loglevel', args.loglevel,
], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

# We buffer the output and print it out once the process is done in order
# to avoid interleaved output from multiple builds running at once.
return_code = process.wait()
stdout, stderr = process.communicate()
if return_code != 0:
print(f'Failed to build addon in {test_dir}:')
if stdout:
print(stdout.decode())
if stderr:
print(stderr.decode())

except Exception as e:
print(f'Unexpected error when building addon in {test_dir}. Error: {e}')

test_dirs = []
skip_tests = args.skip_tests.split(',')
only_tests = args.only_tests.split(',') if args.only_tests else None
for child_dir in os.listdir(args.target):
full_path = os.path.join(args.target, child_dir)
if not os.path.isdir(full_path):
continue
if 'binding.gyp' not in os.listdir(full_path):
continue
if child_dir in skip_tests:
continue
if only_tests and child_dir not in only_tests:
continue
test_dirs.append(full_path)

with ThreadPoolExecutor() as executor:
executor.map(node_gyp_rebuild, test_dirs)

def get_default_out_dir(args):
default_out_dir = os.path.join('out', 'Release')
if not args.is_win:
# POSIX platforms only have one out dir.
return default_out_dir
# On Windows depending on the args of GYP and configure script, the out dir
# could be 'out/Release' or just 'Release'.
if os.path.exists(default_out_dir):
return default_out_dir
if os.path.exists('Release'):
return 'Release'
raise RuntimeError('Can not find out dir, did you run configure?')

def main():
if sys.platform == 'cygwin':
raise RuntimeError('This script does not support running with cygwin python.')

parser = argparse.ArgumentParser(
description='Install headers and rebuild child directories')
parser.add_argument('target', help='target directory to build addons')
parser.add_argument('--headers-dir',
help='path to node headers directory, if not specified '
'new headers will be generated for building',
default=None)
parser.add_argument('--out-dir', help='path to the output directory',
default=None)
parser.add_argument('--loglevel', help='loglevel of node-gyp',
default='silent')
parser.add_argument('--skip-tests', help='skip building tests',
default='')
parser.add_argument('--only-tests', help='only build tests in the list',
default='')
parser.add_argument('--node-gyp', help='path to node-gyp script',
default='deps/npm/node_modules/node-gyp/bin/node-gyp.js')
parser.add_argument('--is-win', help='build for Windows target',
action='store_true', default=(sys.platform == 'win32'))
args, unknown_args = parser.parse_known_args()

if not args.out_dir:
args.out_dir = get_default_out_dir(args)

if args.headers_dir:
rebuild_addons(args)
else:
# When --headers-dir is not specified, generate headers into a temp dir and
# build with the new headers.
try:
args.headers_dir = tempfile.mkdtemp()
generate_headers(args.headers_dir, unknown_args)
rebuild_addons(args)
finally:
shutil.rmtree(args.headers_dir)

if __name__ == '__main__':
main()
Loading

0 comments on commit 5911731

Please sign in to comment.