build,tools: make addons tests work with GN

Makefile

@@ -187,11 +187,11 @@ config.gypi: configure configure.py src/node_version.h
 
 .PHONY: install
 install: all ## Installs node into $PREFIX (default=/usr/local).
-	$(PYTHON) tools/install.py $@ '$(DESTDIR)' '$(PREFIX)'
+	$(PYTHON) tools/install.py $@ --dest-dir '$(DESTDIR)' --prefix '$(PREFIX)'
 
 .PHONY: uninstall
 uninstall: ## Uninstalls node from $PREFIX (default=/usr/local).
-	$(PYTHON) tools/install.py $@ '$(DESTDIR)' '$(PREFIX)'
+	$(PYTHON) tools/install.py $@ --dest-dir '$(DESTDIR)' --prefix '$(PREFIX)'
 
 .PHONY: clean
 .NOTPARALLEL: clean
@@ -379,6 +379,28 @@ test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) tools/doc/node_modules
 		[ $$? -eq 0 ] && touch $@; \
 	fi
 
+# All files that will be included in headers tarball should be listed as deps
+# for generating headers. The list is manually synchronized with install.py.
+ADDONS_HEADERS_PREREQS := tools/install.py \
+	config.gypi common.gypi \
+	$(wildcard deps/openssl/config/*.h) \
+	$(wildcard deps/openssl/openssl/include/openssl/*.h) \
+	$(wildcard deps/uv/include/*.h) \
+	$(wildcard deps/uv/include/*/*.h) \
+	$(wildcard deps/v8/include/*.h) \
+	$(wildcard deps/v8/include/*/*.h) \
+	deps/zlib/zconf.h deps/zlib/zlib.h \
+	src/node.h src/node_api.h src/js_native_api.h src/js_native_api_types.h \
+	src/node_api_types.h src/node_buffer.h src/node_object_wrap.h \
+	src/node_version.h
+
+ADDONS_HEADERS_DIR = out/$(BUILDTYPE)/addons_headers
+
+# Generate node headers which will be used for building addons.
+test/addons/.headersbuildstamp: $(ADDONS_HEADERS_PREREQS)
+	$(PYTHON) tools/install.py install --headers-only --dest-dir '$(ADDONS_HEADERS_DIR)' --prefix '/'
+	@touch $@
+
 ADDONS_BINDING_GYPS := \
 	$(filter-out test/addons/??_*/binding.gyp, \
 		$(wildcard test/addons/*/binding.gyp))
@@ -387,16 +409,11 @@ ADDONS_BINDING_SOURCES := \
 	$(filter-out test/addons/??_*/*.cc, $(wildcard test/addons/*/*.cc)) \
 	$(filter-out test/addons/??_*/*.h, $(wildcard test/addons/*/*.h))
 
-ADDONS_PREREQS := config.gypi \
-	deps/npm/node_modules/node-gyp/package.json tools/build-addons.mjs \
-	deps/uv/include/*.h deps/v8/include/*.h \
-	src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h
+ADDONS_PREREQS := test/addons/.headersbuildstamp \
+	deps/npm/node_modules/node-gyp/package.json tools/build_addons.py
 
 define run_build_addons
-env npm_config_loglevel=$(LOGLEVEL) npm_config_nodedir="$$PWD" \
-	npm_config_python="$(PYTHON)" $(NODE) "$$PWD/tools/build-addons.mjs" \
-	"$$PWD/deps/npm/node_modules/node-gyp/bin/node-gyp.js" \
-	$1
+env $(PYTHON) "$$PWD/tools/build_addons.py" --loglevel=$(LOGLEVEL) --headers-dir "$(ADDONS_HEADERS_DIR)" $1
 touch $2
 endef
 
@@ -429,8 +446,7 @@ JS_NATIVE_API_BINDING_SOURCES := \
 # Implicitly depends on $(NODE_EXE), see the build-js-native-api-tests rule for rationale.
 test/js-native-api/.buildstamp: $(ADDONS_PREREQS) \
 	$(JS_NATIVE_API_BINDING_GYPS) $(JS_NATIVE_API_BINDING_SOURCES) \
-	src/node_api.h src/node_api_types.h src/js_native_api.h \
-	src/js_native_api_types.h src/js_native_api_v8.h src/js_native_api_v8_internals.h
+	src/js_native_api_v8.h src/js_native_api_v8_internals.h
 	@$(call run_build_addons,"$$PWD/test/js-native-api",$@)
 
 .PHONY: build-js-native-api-tests
@@ -454,8 +470,7 @@ NODE_API_BINDING_SOURCES := \
 # Implicitly depends on $(NODE_EXE), see the build-node-api-tests rule for rationale.
 test/node-api/.buildstamp: $(ADDONS_PREREQS) \
 	$(NODE_API_BINDING_GYPS) $(NODE_API_BINDING_SOURCES) \
-	src/node_api.h src/node_api_types.h src/js_native_api.h \
-	src/js_native_api_types.h src/js_native_api_v8.h src/js_native_api_v8_internals.h
+	src/js_native_api_v8.h src/js_native_api_v8_internals.h
 	@$(call run_build_addons,"$$PWD/test/node-api",$@)
 
 .PHONY: build-node-api-tests
@@ -660,9 +675,10 @@ test-addons: test-build test-js-native-api test-node-api
 .PHONY: test-addons-clean
 .NOTPARALLEL: test-addons-clean
 test-addons-clean:
+	$(RM) -r "$(ADDONS_HEADERS_DIR)"
 	$(RM) -r test/addons/??_*/
 	$(RM) -r test/addons/*/build
-	$(RM) test/addons/.buildstamp test/addons/.docbuildstamp
+	$(RM) test/addons/.buildstamp test/addons/.docbuildstamp test/addons/.headersbuildstamp
 	$(MAKE) test-js-native-api-clean
 	$(MAKE) test-node-api-clean
 
@@ -1216,7 +1232,7 @@ $(TARBALL)-headers: release-only
 		--tag=$(TAG) \
 		--release-urlbase=$(RELEASE_URLBASE) \
 		$(CONFIG_FLAGS) $(BUILD_RELEASE_FLAGS)
-	HEADERS_ONLY=1 $(PYTHON) tools/install.py install '$(TARNAME)' '/'
+	$(PYTHON) tools/install.py install --headers-only --dest-dir '$(TARNAME)' --prefix '/'
 	find $(TARNAME)/ -type l | xargs $(RM)
 	tar -cf $(TARNAME)-headers.tar $(TARNAME)
 	$(RM) -r $(TARNAME)

deps/openssl/unofficial.gni

@@ -88,6 +88,11 @@ template("openssl_gn_build") {
     configs += [ ":openssl_internal_config" ]
     public_configs = [ ":openssl_external_config" ]
 
+    if (is_posix) {
+      configs -= [ "//build/config/gcc:symbol_visibility_hidden" ]
+      configs += [ "//build/config/gcc:symbol_visibility_default" ]
+    }
+
     config_path_name = ""
     if (is_win) {
       if (target_cpu == "x86") {

deps/uv/unofficial.gni

@@ -64,6 +64,11 @@ template("uv_gn_build") {
     configs += [ ":uv_internal_config" ]
     public_configs = [ ":uv_external_config" ]
 
+    if (is_posix) {
+      configs -= [ "//build/config/gcc:symbol_visibility_hidden" ]
+      configs += [ "//build/config/gcc:symbol_visibility_default" ]
+    }
+
     if (is_win) {
       libs = [
         "advapi32.lib",

doc/contributing/collaborator-guide.md

@@ -232,7 +232,6 @@ There are some other files that touch the build chain. Changes in the following
 files also qualify as affecting the `node` binary:
 
 * `tools/*.py`
-* `tools/build-addons.mjs`
 * `*.gyp`
 * `*.gypi`
 * `configure`

tools/build_addons.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+
+ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..'))
+
+# Run install.py to install headers.
+def generate_headers(headers_dir, install_args):
+  print('Generating headers')
+  subprocess.check_call([
+      sys.executable,
+      os.path.join(ROOT_DIR, 'tools/install.py'),
+      'install',
+      '--silent',
+      '--headers-only',
+      '--prefix', '/',
+      '--dest-dir', headers_dir,
+  ] + install_args)
+
+# Rebuild addons in parallel.
+def rebuild_addons(args):
+  headers_dir = os.path.abspath(args.headers_dir)
+  out_dir = os.path.abspath(args.out_dir)
+  node_bin = os.path.join(out_dir, 'node')
+  if args.is_win:
+    node_bin += '.exe'
+
+  if os.path.isabs(args.node_gyp):
+    node_gyp = args.node_gyp
+  else:
+    node_gyp = os.path.join(ROOT_DIR, args.node_gyp)
+
+  # Copy node.lib.
+  if args.is_win:
+    node_lib_dir = os.path.join(headers_dir, 'Release')
+    os.makedirs(node_lib_dir)
+    shutil.copy2(os.path.join(args.out_dir, 'node.lib'),
+                 os.path.join(node_lib_dir, 'node.lib'))
+
+  def node_gyp_rebuild(test_dir):
+    print('Building addon in', test_dir)
+    try:
+      process = subprocess.Popen([
+          node_bin,
+          node_gyp,
+          'rebuild',
+          '--directory', test_dir,
+          '--nodedir', headers_dir,
+          '--python', sys.executable,
+          '--loglevel', args.loglevel,
+      ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+      # We buffer the output and print it out once the process is done in order
+      # to avoid interleaved output from multiple builds running at once.
+      return_code = process.wait()
+      stdout, stderr = process.communicate()
+      if return_code != 0:
+        print(f'Failed to build addon in {test_dir}:')
+        if stdout:
+          print(stdout.decode())
+        if stderr:
+          print(stderr.decode())
+
+    except Exception as e:
+      print(f'Unexpected error when building addon in {test_dir}. Error: {e}')
+
+  test_dirs = []
+  skip_tests = args.skip_tests.split(',')
+  only_tests = args.only_tests.split(',') if args.only_tests else None
+  for child_dir in os.listdir(args.target):
+    full_path = os.path.join(args.target, child_dir)
+    if not os.path.isdir(full_path):
+      continue
+    if 'binding.gyp' not in os.listdir(full_path):
+      continue
+    if child_dir in skip_tests:
+      continue
+    if only_tests and child_dir not in only_tests:
+      continue
+    test_dirs.append(full_path)
+
+  with ThreadPoolExecutor() as executor:
+    executor.map(node_gyp_rebuild, test_dirs)
+
+def get_default_out_dir(args):
+  default_out_dir = os.path.join('out', 'Release')
+  if not args.is_win:
+    # POSIX platforms only have one out dir.
+    return default_out_dir
+  # On Windows depending on the args of GYP and configure script, the out dir
+  # could be 'out/Release' or just 'Release'.
+  if os.path.exists(default_out_dir):
+    return default_out_dir
+  if os.path.exists('Release'):
+    return 'Release'
+  raise RuntimeError('Can not find out dir, did you run configure?')
+
+def main():
+  if sys.platform == 'cygwin':
+    raise RuntimeError('This script does not support running with cygwin python.')
+
+  parser = argparse.ArgumentParser(
+      description='Install headers and rebuild child directories')
+  parser.add_argument('target', help='target directory to build addons')
+  parser.add_argument('--headers-dir',
+                      help='path to node headers directory, if not specified '
+                           'new headers will be generated for building',
+                      default=None)
+  parser.add_argument('--out-dir', help='path to the output directory',
+                      default=None)
+  parser.add_argument('--loglevel', help='loglevel of node-gyp',
+                      default='silent')
+  parser.add_argument('--skip-tests', help='skip building tests',
+                      default='')
+  parser.add_argument('--only-tests', help='only build tests in the list',
+                      default='')
+  parser.add_argument('--node-gyp', help='path to node-gyp script',
+                      default='deps/npm/node_modules/node-gyp/bin/node-gyp.js')
+  parser.add_argument('--is-win', help='build for Windows target',
+                      action='store_true', default=(sys.platform == 'win32'))
+  args, unknown_args = parser.parse_known_args()
+
+  if not args.out_dir:
+    args.out_dir = get_default_out_dir(args)
+
+  if args.headers_dir:
+    rebuild_addons(args)
+  else:
+    # When --headers-dir is not specified, generate headers into a temp dir and
+    # build with the new headers.
+    try:
+      args.headers_dir = tempfile.mkdtemp()
+      generate_headers(args.headers_dir, unknown_args)
+      rebuild_addons(args)
+    finally:
+      shutil.rmtree(args.headers_dir)
+
+if __name__ == '__main__':
+  main()