From f146fd70e3e413db5901afd3e07597ca54ac37c5 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 24 Feb 2022 10:51:50 -0800 Subject: [PATCH 1/7] scripts/meson_exe: Use Protocol for better type checking --- mesonbuild/scripts/meson_exe.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mesonbuild/scripts/meson_exe.py b/mesonbuild/scripts/meson_exe.py index e928e9c1ee85..b8a96d24fa71 100644 --- a/mesonbuild/scripts/meson_exe.py +++ b/mesonbuild/scripts/meson_exe.py @@ -3,6 +3,7 @@ from __future__ import annotations +from __future__ import annotations import os import sys import argparse @@ -13,6 +14,16 @@ from ..utils.core import ExecutableSerialisation +if T.TYPE_CHECKING: + from typing_extensions import Protocol + + class Args(Protocol): + + unpickle: bool + capture: bool + feed: bool + + def buildparser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description='Custom executable wrapper for Meson. Do not run on your own, mmm\'kay?') parser.add_argument('--unpickle') @@ -93,6 +104,7 @@ def run_exe(exe: ExecutableSerialisation, extra_env: T.Optional[T.Dict[str, str] def run(args: T.List[str]) -> int: parser = buildparser() + options: Args options, cmd_args = parser.parse_known_args(args) # argparse supports double dash to separate options and positional arguments, # but the user has to remove it manually. From 9be7c7c960216357e1b5b19f0bd9a7f7cb6c1ad4 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 24 Feb 2022 10:57:17 -0800 Subject: [PATCH 2/7] scripts/meson_exe: allow passing environment variables to the meson internal wrapper This will be used by rustc when we're setting environment variables, but don't have support for --set-env --- mesonbuild/scripts/meson_exe.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mesonbuild/scripts/meson_exe.py b/mesonbuild/scripts/meson_exe.py index b8a96d24fa71..18f35af40d41 100644 --- a/mesonbuild/scripts/meson_exe.py +++ b/mesonbuild/scripts/meson_exe.py @@ -3,7 +3,6 @@ from __future__ import annotations -from __future__ import annotations import os import sys import argparse @@ -12,7 +11,7 @@ import typing as T import locale -from ..utils.core import ExecutableSerialisation +from ..utils.core import ExecutableSerialisation, EnvironmentVariables if T.TYPE_CHECKING: from typing_extensions import Protocol @@ -22,6 +21,7 @@ class Args(Protocol): unpickle: bool capture: bool feed: bool + env: T.List[str] def buildparser() -> argparse.ArgumentParser: @@ -29,6 +29,7 @@ def buildparser() -> argparse.ArgumentParser: parser.add_argument('--unpickle') parser.add_argument('--capture') parser.add_argument('--feed') + parser.add_argument('--env', action='append', default=[]) return parser def run_exe(exe: ExecutableSerialisation, extra_env: T.Optional[T.Dict[str, str]] = None) -> int: @@ -119,7 +120,12 @@ def run(args: T.List[str]) -> int: exe = pickle.load(f) exe.pickled = True else: - exe = ExecutableSerialisation(cmd_args, capture=options.capture, feed=options.feed) + env = EnvironmentVariables() + for e in options.env: + name, value = e.split('=') + # This is internal, we can pick whatever separator we want! + env.append(name, value.split(':')) + exe = ExecutableSerialisation(cmd_args, capture=options.capture, feed=options.feed, env=env) return run_exe(exe) From 2eb9c4e0eefb9679ffce3105eed5db23f84ccaa6 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 16 Jan 2024 13:22:05 -0800 Subject: [PATCH 3/7] compilers/rust: track whether we have a nightly or non-nightly compiler There is currently no reason to track whether we have a beta compiler, and this is not user visible, so we're not differentiating stable and beta, just nightly vs !nightly --- mesonbuild/compilers/detect.py | 3 ++- mesonbuild/compilers/rust.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mesonbuild/compilers/detect.py b/mesonbuild/compilers/detect.py index 9d4852bf1524..0f81ad6bff03 100644 --- a/mesonbuild/compilers/detect.py +++ b/mesonbuild/compilers/detect.py @@ -987,6 +987,7 @@ def detect_rust_compiler(env: 'Environment', for_machine: MachineChoice) -> Rust continue version = search_version(out) + is_nightly = 'nightly' in out cls: T.Type[RustCompiler] = rust.RustCompiler # Clippy is a wrapper around rustc, but it doesn't have rustc in it's @@ -1078,7 +1079,7 @@ def detect_rust_compiler(env: 'Environment', for_machine: MachineChoice) -> Rust env.coredata.add_lang_args(cls.language, cls, for_machine, env) return cls( compiler, version, for_machine, is_cross, info, exe_wrap, - linker=linker) + linker=linker, is_nightly=is_nightly) _handle_exceptions(popen_exceptions, compilers) raise EnvironmentException('Unreachable code (exception to make mypy happy)') diff --git a/mesonbuild/compilers/rust.py b/mesonbuild/compilers/rust.py index e10cb4cf3d5f..1a4f6e6892e7 100644 --- a/mesonbuild/compilers/rust.py +++ b/mesonbuild/compilers/rust.py @@ -60,7 +60,8 @@ def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoic is_cross: bool, info: 'MachineInfo', exe_wrapper: T.Optional['ExternalProgram'] = None, full_version: T.Optional[str] = None, - linker: T.Optional['DynamicLinker'] = None): + linker: T.Optional['DynamicLinker'] = None, + is_nightly: bool = False): super().__init__([], exelist, version, for_machine, info, is_cross=is_cross, full_version=full_version, linker=linker) @@ -69,6 +70,7 @@ def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoic if 'link' in self.linker.id: self.base_options.add(OptionKey('b_vscrt')) self.native_static_libs: T.List[str] = [] + self.is_nightly = is_nightly def needs_static_linker(self) -> bool: return False From ce5bdcbe3bb393287b9ef77c58bfbf8fb030025b Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 24 Feb 2022 10:35:25 -0800 Subject: [PATCH 4/7] tests/rust: Add a test for using environment variables --- mesonbuild/compilers/rust.py | 3 ++- test cases/rust/23 env/generator.py | 21 +++++++++++++++++++++ test cases/rust/23 env/main.rs | 5 +++++ test cases/rust/23 env/meson.build | 16 ++++++++++++++++ test cases/rust/23 env/static.rs | 3 +++ 5 files changed, 47 insertions(+), 1 deletion(-) create mode 100755 test cases/rust/23 env/generator.py create mode 100644 test cases/rust/23 env/main.rs create mode 100644 test cases/rust/23 env/meson.build create mode 100644 test cases/rust/23 env/static.rs diff --git a/mesonbuild/compilers/rust.py b/mesonbuild/compilers/rust.py index 1a4f6e6892e7..ca106ba18863 100644 --- a/mesonbuild/compilers/rust.py +++ b/mesonbuild/compilers/rust.py @@ -9,7 +9,7 @@ import typing as T from .. import coredata -from ..mesonlib import EnvironmentException, MesonException, Popen_safe_logged, OptionKey +from ..mesonlib import EnvironmentException, MesonException, Popen_safe_logged, OptionKey, version_compare from .compilers import Compiler, clike_debug_args if T.TYPE_CHECKING: @@ -71,6 +71,7 @@ def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoic self.base_options.add(OptionKey('b_vscrt')) self.native_static_libs: T.List[str] = [] self.is_nightly = is_nightly + self.needs_env_wrapper = not (version_compare(self.version, '>= 1.76') and self.is_nightly) def needs_static_linker(self) -> bool: return False diff --git a/test cases/rust/23 env/generator.py b/test cases/rust/23 env/generator.py new file mode 100755 index 000000000000..35cf2a1c3d75 --- /dev/null +++ b/test cases/rust/23 env/generator.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +import argparse +import os +import textwrap + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('outdir') + args = parser.parse_args() + + with open(os.path.join(args.outdir, 'generated.rs'), 'w') as f: + f.write(textwrap.dedent('''\ + fn generated() { + std::process::exit(0); + }''')) + + +if __name__ == '__main__': + main() diff --git a/test cases/rust/23 env/main.rs b/test cases/rust/23 env/main.rs new file mode 100644 index 000000000000..2ea3864ebc22 --- /dev/null +++ b/test cases/rust/23 env/main.rs @@ -0,0 +1,5 @@ +include!(concat!(env!("OUT_DIR"), "/generated.rs")); + +fn main() { + generated(); +} diff --git a/test cases/rust/23 env/meson.build b/test cases/rust/23 env/meson.build new file mode 100644 index 000000000000..3f9961290a96 --- /dev/null +++ b/test cases/rust/23 env/meson.build @@ -0,0 +1,16 @@ +project('rust environment variables', 'rust') + +gen = find_program('generator.py') + +ct = custom_target( + 'generated.rs', + output : 'generated.rs', + command : [gen, '@OUTDIR@'], +) + +exe = executable('main', 'main.rs', ct, env: {'OUT_DIR': meson.current_build_dir()}) + +test('main test', exe) + +# test that generated and non-generated work together +executable('static', 'static.rs') diff --git a/test cases/rust/23 env/static.rs b/test cases/rust/23 env/static.rs new file mode 100644 index 000000000000..44490fa24647 --- /dev/null +++ b/test cases/rust/23 env/static.rs @@ -0,0 +1,3 @@ +fn main() { + std::process::exit(0); +} From e9290a4cb5ac02b9d5fac79624876c1559fe8b0e Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 16 Jan 2024 13:18:01 -0800 Subject: [PATCH 5/7] backend/ninja: set env variables when the `--env-set` option is available TODO: fallback to using a wrapper --- mesonbuild/backend/ninjabackend.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 9acc39c2a82e..85370f8240ea 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -2063,6 +2063,9 @@ def _link_library(libname: str, static: bool, bundle: bool = False): for rpath_arg in rpath_args: args += ['-C', 'link-arg=' + rpath_arg + ':' + os.path.join(rustc.get_sysroot(), 'lib')] + if target.env: + args.extend(rustc.get_env_args(target.env)) + proc_macro_dylib_path = None if getattr(target, 'rust_crate_type', '') == 'proc-macro': proc_macro_dylib_path = os.path.abspath(os.path.join(target.subdir, target.get_filename())) From c6d474786d66ea9c1c3050da6b900a0ae4e64335 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 16 Jan 2024 14:23:53 -0800 Subject: [PATCH 6/7] backend/ninja: add fallback path for wrapping rustc with env vars At least for a while, we will need to support wrapping rustc in a script to set environment variables. This is far from ideal, since it's slow, clunky, and complicated. --- mesonbuild/backend/ninjabackend.py | 32 +++++++++++++++++++++++++----- test cases/rust/23 env/meson.build | 2 +- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 85370f8240ea..702020d3856b 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -44,6 +44,7 @@ from ..linkers.linkers import DynamicLinker, StaticLinker from ..compilers.cs import CsCompiler from ..compilers.fortran import FortranCompiler + from ..compilers.rust import RustCompiler CommandArgOrStr = T.List[T.Union['NinjaCommandArg', str]] RUST_EDITIONS = Literal['2015', '2018', '2021'] @@ -1846,7 +1847,7 @@ def _get_rust_dependency_name(self, target: build.BuildTarget, dependency: LibTy return target.rust_dependency_map.get(dependency.name, dependency.name).replace('-', '_') def generate_rust_target(self, target: build.BuildTarget) -> None: - rustc = target.compilers['rust'] + rustc = T.cast('RustCompiler', target.compilers['rust']) # Rust compiler takes only the main file as input and # figures out what other files are needed via import # statements and magic. @@ -2063,8 +2064,9 @@ def _link_library(libname: str, static: bool, bundle: bool = False): for rpath_arg in rpath_args: args += ['-C', 'link-arg=' + rpath_arg + ':' + os.path.join(rustc.get_sysroot(), 'lib')] - if target.env: - args.extend(rustc.get_env_args(target.env)) + compiler_name = self.compiler_to_rule_name(rustc) + if rustc.needs_env_wrapper: + compiler_name = self.get_compiler_rule_name(rustc.get_language(), rustc.for_machine, 'ENV') proc_macro_dylib_path = None if getattr(target, 'rust_crate_type', '') == 'proc-macro': @@ -2077,12 +2079,26 @@ def _link_library(libname: str, static: bool, bundle: bool = False): proc_macro_dylib_path, project_deps) - compiler_name = self.compiler_to_rule_name(rustc) element = NinjaBuildElement(self.all_outputs, target_name, compiler_name, main_rust_file) if orderdeps: element.add_orderdep(orderdeps) if deps: element.add_dep(deps) + if rustc.needs_env_wrapper: + env: T.List[str] = [] + nargs: T.List[str] = [] + itr = iter(args) + for a in itr: + if a == '--env-set': + a = next(itr) + elif a.startswith('--env-set='): + a = a.split('=', 1)[1] + else: + nargs.append(a) + continue + env.extend(['--env', a]) + element.add_item('ENV', env) + args = nargs element.add_item('ARGS', args) element.add_item('targetdep', depfile) element.add_item('cratetype', cratetype) @@ -2369,7 +2385,7 @@ def generate_cython_compile_rules(self, compiler: 'Compiler') -> None: depfile=depfile, extra='restat = 1')) - def generate_rust_compile_rules(self, compiler): + def generate_rust_compile_rules(self, compiler: RustCompiler) -> None: rule = self.compiler_to_rule_name(compiler) command = compiler.get_exelist() + ['$ARGS', '$in'] description = 'Compiling Rust source $in' @@ -2377,6 +2393,12 @@ def generate_rust_compile_rules(self, compiler): depstyle = 'gcc' self.add_rule(NinjaRule(rule, command, [], description, deps=depstyle, depfile=depfile)) + if compiler.needs_env_wrapper: + rule = self.get_compiler_rule_name(compiler.get_language(), compiler.for_machine, 'ENV') + command = self.environment.get_build_command() + ['--internal', 'exe', '$ENV', '--'] + command + description = f'{description} (wrapped to set environment variables)' + self.add_rule(NinjaRule(rule, command, [], description, deps=depstyle, + depfile=depfile)) def generate_swift_compile_rules(self, compiler): rule = self.compiler_to_rule_name(compiler) diff --git a/test cases/rust/23 env/meson.build b/test cases/rust/23 env/meson.build index 3f9961290a96..42ddf47cd940 100644 --- a/test cases/rust/23 env/meson.build +++ b/test cases/rust/23 env/meson.build @@ -8,7 +8,7 @@ ct = custom_target( command : [gen, '@OUTDIR@'], ) -exe = executable('main', 'main.rs', ct, env: {'OUT_DIR': meson.current_build_dir()}) +exe = executable('main', 'main.rs', ct, rust_args: ['--env-set', 'OUT_DIR=' + meson.current_build_dir()]) test('main test', exe) From 145353a00648b478462056b3fddcda52d5c03119 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 16 Jan 2024 15:57:04 -0800 Subject: [PATCH 7/7] hack: add unstable-options --- mesonbuild/backend/ninjabackend.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 702020d3856b..00bc65c3ddaf 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -2064,7 +2064,7 @@ def _link_library(libname: str, static: bool, bundle: bool = False): for rpath_arg in rpath_args: args += ['-C', 'link-arg=' + rpath_arg + ':' + os.path.join(rustc.get_sysroot(), 'lib')] - compiler_name = self.compiler_to_rule_name(rustc) + compiler_name: str = self.compiler_to_rule_name(rustc) if rustc.needs_env_wrapper: compiler_name = self.get_compiler_rule_name(rustc.get_language(), rustc.for_machine, 'ENV') @@ -2099,6 +2099,9 @@ def _link_library(libname: str, static: bool, bundle: bool = False): env.extend(['--env', a]) element.add_item('ENV', env) args = nargs + else: + # XXX: this needs a heiristc + args.extend(['-Z', 'unstable-options']) element.add_item('ARGS', args) element.add_item('targetdep', depfile) element.add_item('cratetype', cratetype)