llb

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Builder for LLVM project.

import argparse
import os
import sys
import re
import subprocess
import shlex
import shutil
import tempfile
import logging
import platform
from copy import copy


def GetRuntimesLdFlags():
    # libc++ uses libc++abi and libunwind being built.
    return ['-Wl,-rpath,$ORIGIN']


def GetRuntimesCmakeFlags():
    return [
        '-DLIBCXX_USE_COMPILER_RT=TRUE',
        '-DLIBCXXABI_USE_COMPILER_RT=TRUE',
        '-DLIBUNWIND_USE_COMPILER_RT=TRUE',
        '-DLIBCXXABI_USE_LLVM_UNWINDER=TRUE',
    ]


def GetNativeCPUFlags():
    m = platform.machine()
    if m.startswith('x86'):
        return ['-march=native']
    return ['-mcpu=native']


def GetDefaultFlags():
    return ['-pipe'] + GetNativeCPUFlags()


def GetLdPathFlags(cc, ld):
    if not cc.endswith('clang'):
        return []
    return [f'--ld-path={ld}']


def FeasibleToEnableThinLTO(cc, ld):
    return cc.endswith('clang') and ld.endswith('lld')


def GetMinimalToolchainTargets():
    return [
        'install-llvm-ar',
        'install-llvm-ranlib',
        'install-llvm-profdata',
        'install-clang',
        'install-lld',
        'install-clang-resource-headers',
        'install-runtimes',
        'install-builtins',
    ]


def GetRuntimes():
    return ['libcxx', 'libcxxabi', 'libunwind']


def GetRuntimeTargets():
    return ['install-cxx', 'install-cxxabi', 'install-unwind']


def GetClangToolsExtraTargets():
    return [
        'install-clangd',
        'install-clang-format',
        'install-clang-tidy',
    ]


def InferCXX(cc):
    if cc.endswith('gcc'):
        return cc[:-len('gcc')] + 'g++'
    if cc.endswith('clang'):
        return cc[:-len('clang')] + 'clang++'
    if cc.endswith('icx'):
        return cc[:-len('icx')] + 'icpx'
    raise Exception(f'Unrecognized compiler: {cc}')


def BuildForStaging(config):
    BS = BuildSpace(config.build_dir)
    B1 = Builder(
        jobs=config.jobs,
        monorepo=config.monorepo,
        build_dir=BS.GetBuild('stage1'),
        install_prefix=BS.GetInstall('stage1'),
        ccache=True,
        cc=config.cc,
        cxx=InferCXX(config.cc),
        cflags=GetDefaultFlags(),
        cxxflags=GetDefaultFlags(),
        ld=config.ld,
        ldflags=GetLdPathFlags(config.cc, config.ld),
        use_malloc='mimalloc',
        projects=['clang', 'lld'],
        runtimes=['compiler-rt'],
        ninja_targets=GetMinimalToolchainTargets(),
    )
    rc = B1.ConfigAndBuild()
    if rc != 0:
        return rc
    cc = os.path.join(BS.GetInstall('stage1'), 'bin/clang')
    ld = os.path.join(BS.GetInstall('stage1'), 'bin/ld.lld')
    BS.RemoveBuild('stage2')
    B2 = Builder(
        jobs=config.jobs,
        monorepo=config.monorepo,
        build_dir=BS.GetBuild('stage2'),
        install_prefix=BS.GetInstall('stage2'),
        cc=cc,
        cxx=InferCXX(cc),
        cflags=GetDefaultFlags(),
        cxxflags=GetDefaultFlags(),
        ld=ld,
        ldflags=GetLdPathFlags(cc, ld),
        projects=['clang', 'lld'],
        runtimes=['compiler-rt'],
        ninja_targets=[
            'check-all',
            'check-runtimes',
        ],
    )
    return B2.ConfigAndBuild()


def BuildForDev(config):
    Builder(
        monorepo=config.monorepo,
        build_dir=config.build_dir,
        cmake=config.cmake_binary,
        ninja=config.ninja_binary,
        ccache=True,
        cc=config.cc,
        cxx=InferCXX(config.cc),
        cflags=GetDefaultFlags(),
        cxxflags=GetDefaultFlags(),
        ld=config.ld,
        ldflags=GetLdPathFlags(config.cc, config.ld),
        use_malloc='mimalloc',
        projects=['clang', 'lld'],
        runtimes=['compiler-rt'],
        config_only=True,
    ).ConfigAndBuild()


def BuildForDebug(config):
    Builder(monorepo=config.monorepo,
            build_dir=config.build_dir,
            cmake=config.cmake_binary,
            ninja=config.ninja_binary,
            build_type='Debug',
            cc=config.cc,
            cxx=InferCXX(config.cc),
            cflags=['-pipe'],
            cxxflags=['-pipe'],
            ld=config.ld,
            ldflags=GetLdPathFlags(config.cc, config.ld),
            projects=['clang;lld'],
            runtimes=['compiler-rt'],
            config_only=True).ConfigAndBuild()


def BuildWithGCC(config):
    BS = BuildSpace(config.build_dir)
    # Force to use GNU toolchain.
    cc = shutil.which('gcc')
    ld = shutil.which('ld')
    B = Builder(
        jobs=config.jobs,
        monorepo=config.monorepo,
        build_dir=BS.GetBuild('stage1'),
        install_prefix=BS.GetInstall('stage1'),
        cc=cc,
        cxx=InferCXX(cc),
        cflags=['-pipe'],
        cxxflags=['-pipe'],
        ld=ld,
        projects=['clang', 'lld'],
        runtimes=['compiler-rt'] + GetRuntimes(),
        ninja_targets=GetMinimalToolchainTargets() + GetRuntimeTargets(),
    )
    return B.ConfigAndBuild()


class Builder(object):

    def __init__(
        self,
        monorepo=None,
        build_dir=None,
        install_prefix=None,
        cmake=shutil.which('cmake'),
        ninja=shutil.which('ninja'),
        build_type='Release',
        werror=False,
        ccache=False,
        cc=None,
        cxx=None,
        ld=None,
        cflags=[],
        cxxflags=[],
        ldflags=[],
        projects=[],
        runtimes=[],
        extra_cmake_flags=[],
        config_only=False,
        jobs=min(8, os.cpu_count()),
        ninja_targets=[],
        use_malloc=None,
        enable_assert=True,
        env={},
    ):
        assert monorepo
        assert build_dir
        self.werror = werror
        self.ccache = ccache
        self.cmake = cmake
        self.ninja = ninja
        self.monorepo = os.path.abspath(monorepo)
        self.build_dir = os.path.abspath(build_dir)
        self.install_prefix = os.path.abspath(
            install_prefix) if install_prefix else None
        self.cc = os.path.abspath(cc) if cc else None
        self.cxx = os.path.abspath(cxx) if cxx else None
        self.ld = os.path.abspath(ld) if ld else None
        self.cflags = cflags
        self.cxxflags = cxxflags
        self.ldflags = ldflags
        self.projects = projects
        self.runtimes = runtimes
        self.extra_cmake_flags = extra_cmake_flags
        self.ninja_targets = ninja_targets
        self.build_type = build_type
        self.config_only = config_only
        self.jobs = jobs
        self.use_malloc = use_malloc
        self.enable_assert = enable_assert
        self.env = env

    def BuildCMakeCommand(self):
        cmake_command = [
            self.cmake,
            '-GNinja',
            f'-DCMAKE_MAKE_PROGRAM={self.ninja}',
            '-DLLVM_ENABLE_BINDINGS=OFF',
            '-DLLVM_ENABLE_ASSERTIONS={}'.format(
                'TRUE' if self.enable_assert else 'FALSE'),
            f'-DCMAKE_BUILD_TYPE={self.build_type}',
        ]
        if self.werror:
            cmake_command.append('-DLLVM_ENABLE_WERROR=ON')
        if self.ccache:
            cmake_command.append('-DLLVM_CCACHE_BUILD=ON')
        if self.install_prefix:
            cmake_command.append(
                f'-DCMAKE_INSTALL_PREFIX={self.install_prefix}')
        if self.use_malloc:
            self.ldflags.append(f'-l{self.use_malloc}')
        if self.cc:
            cmake_command.append(f'-DCMAKE_C_COMPILER={self.cc}')
        if self.cxx:
            cmake_command.append(f'-DCMAKE_CXX_COMPILER={self.cxx}')
        # GCC doesn't support -fuse-ld yet.
        if self.ld and not self.cc.endswith('gcc'):
            cmake_command.append(f'-DLLVM_USE_LINKER={self.ld}')
        if self.cflags:
            cmake_command.append('-DCMAKE_C_FLAGS={}'.format(
                shlex.join(self.cflags)))
        if self.cxxflags:
            cmake_command.append('-DCMAKE_CXX_FLAGS={}'.format(
                shlex.join(self.cxxflags)))
        if self.projects:
            cmake_command.append('-DLLVM_ENABLE_PROJECTS={}'.format(';'.join(
                self.projects)))
        if self.runtimes:
            cmake_command.append('-DLLVM_ENABLE_RUNTIMES={}'.format(';'.join(
                self.runtimes)))
        if self.ldflags:
            cmake_command.append('-DCMAKE_EXE_LINKER_FLAGS={}'.format(
                shlex.join(self.ldflags)))
            cmake_command.append('-DCMAKE_SHARED_LINKER_FLAGS={}'.format(
                shlex.join(self.ldflags)))
            cmake_command.append('-DCMAKE_MODULE_LINKER_FLAGS={}'.format(
                shlex.join(self.ldflags)))
        cmake_command.extend(self.extra_cmake_flags)
        return cmake_command

    def _Config(self, src_dir, build_dir):
        cmake_command = self.BuildCMakeCommand()
        cmake_command.extend(['-S', src_dir])
        cmake_command.extend(['-B', build_dir])
        env = os.environ.copy()
        env.update(self.env)
        err = subprocess.call(cmake_command, env=env)
        if err != 0:
            logging.error(f'cmake failed: {cmake_command}')
        return err

    def ConfigAndBuild(self):
        err = self.Config()
        if self.config_only or err != 0:
            return err
        return self.Build()

    def Config(self):
        return self._Config(os.path.join(self.monorepo, 'llvm'),
                            self.build_dir)

    def Build(self):
        ninja_command = [
            self.ninja,
            '-j%d' % self.jobs,
        ]
        ninja_command.extend(self.ninja_targets)
        env = os.environ.copy()
        env.update(self.env)
        err = subprocess.call(ninja_command, cwd=self.build_dir, env=env)
        if err != 0:
            logging.error(f'ninja failed: {ninja_command}')
        return err


class RuntimeBuilder(Builder):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def Config(self):
        return self._Config(os.path.join(self.monorepo, 'runtimes'),
                            self.build_dir)


class BuildSpace(object):

    def __init__(self, root):
        self.root = os.path.abspath(root)

    def Clear(self):
        shutil.rmtree(self.root)

    def _GetDir(self, path):
        return os.path.join(self.root, path)

    def _AddDir(self, path):
        os.makedirs(self._GetDir(path), exist_ok=True)

    def _RemoveDir(self, path):
        if os.path.exists(self._GetDir(path)):
            shutil.rmtree(self._GetDir(path))

    def GetBuild(self, name):
        self._AddDir(f'build/{name}')
        return self._GetDir(f'build/{name}')

    def GetInstall(self, name):
        self._AddDir(f'install/{name}')
        return self._GetDir(f'install/{name}')

    def RemoveBuild(self, name):
        self._RemoveDir(f'build/{name}')

    def RemoveInstall(self, name):
        self._RemoveDir(f'install/{name}')


def Bootstrap(config, stage):
    if stage < 1:
        return 1
    if stage > 3:
        stage = 3
    cc = shutil.which(config.cc)
    cxx = InferCXX(cc)
    ld = shutil.which(config.ld)
    BS = BuildSpace(config.build_dir)
    for i in range(1, stage + 1):
        ninja_targets = GetMinimalToolchainTargets()
        ldflags = []
        cflags = GetDefaultFlags()
        cxxflags = GetDefaultFlags()
        # CMake does not set the -fpic option when building static libraries.
        # See https://github.com/gabime/spdlog/issues/1405.
        extra_cmake_flags = [
            '-DCMAKE_POSITION_INDEPENDENT_CODE=On',
        ]
        projects = ['clang', 'lld']
        runtimes = ['compiler-rt']
        if i > 1:
            install_prefix = BS.GetInstall(f'stage{i-1}')
            cc = os.path.join(install_prefix, 'bin/clang')
            cxx = os.path.join(install_prefix, 'bin/clang++')
            ld = os.path.join(install_prefix, 'bin/ld.lld')
        if FeasibleToEnableThinLTO(cc, ld):
            extra_cmake_flags.append('-DLLVM_ENABLE_LTO=Thin')
        # Build additional tools at the last stage.
        if i == stage:
            projects.extend(['clang-tools-extra', 'polly', 'bolt', 'lldb'])
            ninja_targets.extend(GetClangToolsExtraTargets())
            ninja_targets.extend(['install-bolt', 'install-lldb'])
        # Build runtimes by stage{i} driver.
        RB = RuntimeBuilder(
            # To compare stage2 and stage3 binaries, assertion must be disabled.
            enable_assert=False,
            jobs=config.jobs,
            monorepo=config.monorepo,
            build_dir=BS.GetBuild(f'stage{i}-runtimes'),
            install_prefix=BS.GetInstall(f'stage{i}'),
            cc=cc,
            cxx=cxx,
            cflags=GetDefaultFlags(),
            cxxflags=GetDefaultFlags(),
            ld=ld,
            ldflags=ldflags + GetLdPathFlags(cc, ld) + GetRuntimesLdFlags(),
            runtimes=GetRuntimes(),
            use_malloc='mimalloc',
            ninja_targets=GetRuntimeTargets(),
            extra_cmake_flags=extra_cmake_flags + GetRuntimesCmakeFlags(),
        )
        rc = RB.ConfigAndBuild()
        if rc != 0:
            return rc
        libdir = os.path.join(BS.GetInstall(f'stage{i}'), 'lib')
        B = Builder(
            enable_assert=False,
            jobs=config.jobs,
            monorepo=config.monorepo,
            build_dir=BS.GetBuild(f'stage{i}'),
            install_prefix=BS.GetInstall(f'stage{i}'),
            cc=cc,
            cxx=cxx,
            cflags=cflags,
            cxxflags=cxxflags + [
                '-nostdinc++',
                '-nostdlib++',
                '-isystem',
                os.path.join(BS.GetInstall(f'stage{i}'), 'include/c++/v1'),
            ],
            ld=ld,
            ldflags=ldflags + GetLdPathFlags(cc, ld) + [
                # Link against the runtimes we just built.
                '-L{}'.format(libdir),
                '--rtlib=compiler-rt',
                '--unwindlib=libunwind',
                '--stdlib=libc++',
                '-lc++',
            ],
            runtimes=runtimes,
            projects=projects,
            use_malloc='mimalloc',
            # Currently, binaries are in build tree, not install tree, so RUNPATH may be
            # not pointing the libdir in need.
            env={'LD_LIBRARY_PATH': libdir},
            ninja_targets=ninja_targets,
            extra_cmake_flags=extra_cmake_flags,
        )
        rc = B.ConfigAndBuild()
        if rc != 0:
            return rc
    return 0


def Bootstrap1(config):
    return Bootstrap(config, 1)


def Bootstrap2(config):
    return Bootstrap(config, 2)


def Bootstrap3(config):
    return Bootstrap(config, 3)


def BuildRuntimes(config):
    extra_cmake_flags = []
    if FeasibleToEnableThinLTO(config.cc, config.ld):
        extra_cmake_flags.append('-DLLVM_ENABLE_LTO=Thin')
    return RuntimeBuilder(
        enable_assert=False,
        jobs=config.jobs,
        monorepo=config.monorepo,
        build_dir=config.build_dir,
        install_prefix=config.install_prefix,
        cc=config.cc,
        cxx=InferCXX(config.cc),
        cflags=GetDefaultFlags(),
        cxxflags=GetDefaultFlags(),
        ld=config.ld,
        ldflags=GetLdPathFlags(config.cc, config.ld) + GetRuntimesLdFlags(),
        runtimes=GetRuntimes(),
        use_malloc='mimalloc',
        ninja_targets=GetRuntimeTargets(),
        extra_cmake_flags=extra_cmake_flags + GetRuntimesCmakeFlags(),
    ).ConfigAndBuild()


def BuildLibatomic(config):
    os.makedirs(config.build_dir, exist_ok=True)
    os.makedirs(config.install_prefix, exist_ok=True)
    cflags = GetDefaultFlags()
    if FeasibleToEnableThinLTO(config.cc, config.ld):
        cflags.append(f'--ld-path={config.ld}')
        cflags.append('-flto=thin')
    srcs = [
        os.path.join(config.monorepo, 'compiler-rt/lib/builtins/atomic.c'),
    ]
    out = os.path.join(config.build_dir, 'libatomic.so.1.0')
    cmd = [
        config.cc,
        '-O3',
        '-fPIC',
        '-shared',
        '-Wl,-rpath,$ORIGIN',
        '-o',
        out,
    ] + srcs
    cmd.extend(cflags)
    rc = subprocess.call(cmd)
    if rc != 0:
        return rc
    cmd = [
        'install',
        out,
        config.install_prefix,
    ]
    rc = subprocess.call(cmd)
    if rc != 0:
        return rc
    cmd = [
        'ln',
        '-sfn',
        'libatomic.so.1.0',
        'libatomic.so.1',
    ]
    rc = subprocess.call(cmd, cwd=config.install_prefix)
    if rc != 0:
        return rc
    cmd = [
        'ln',
        '-sfn',
        'libatomic.so.1',
        'libatomic.so',
    ]
    return subprocess.call(cmd, cwd=config.install_prefix)


def PrintCmakeCommand(config):
    cmd = Builder(
        monorepo=config.monorepo,
        build_dir=config.build_dir,
        cmake=config.cmake_binary,
        ninja=config.ninja_binary,
        ccache=True,
        cc=config.cc,
        cxx=InferCXX(config.cc),
        cflags=GetDefaultFlags(),
        cxxflags=GetDefaultFlags(),
        ld=config.ld,
        ldflags=GetLdPathFlags(config.cc, config.ld),
        use_malloc='mimalloc',
        projects=['clang', 'lld'],
        runtimes=['compiler-rt'],
        config_only=True,
    ).BuildCMakeCommand()
    print(shlex.join(cmd))
    return 0


Profiles = {
    'dev': BuildForDev,
    'debug': BuildForDebug,
    'staging': BuildForStaging,
    'gcc': BuildWithGCC,
    'clang1': Bootstrap1,
    'clang2': Bootstrap2,
    'clang3': Bootstrap3,
    'runtimes': BuildRuntimes,
    'libatomic': BuildLibatomic,
    'print_cmake': PrintCmakeCommand,
}


def main():
    parser = argparse.ArgumentParser(
        description='Build Clang/LLVM using different profile.')
    parser.add_argument('--cmake_binary', default=shutil.which('cmake'))
    parser.add_argument('--ninja_binary', default=shutil.which('ninja'))
    parser.add_argument('--cc', default=shutil.which('clang'))
    parser.add_argument('--ld', default=shutil.which('ld.lld'))
    parser.add_argument('--ccache', default=False, action='store_true')
    parser.add_argument('-j', dest='jobs', type=int, default=os.cpu_count())
    parser.add_argument('--monorepo', required=True)
    parser.add_argument('--build_dir', required=True)
    parser.add_argument('--install_prefix')
    parser.add_argument('--werror', default=False, action='store_true')
    parser.add_argument('-p', dest='profile', type=str, default='dev')
    config = parser.parse_args()
    if config.profile in Profiles:
        return Profiles[config.profile](config)
    logging.error(f"No builder for {config.profile}")
    return 1


if __name__ == '__main__':
    sys.exit(main())