Skip to content

Commit

Permalink
Merge pull request #860 from altheaden/pip-install-flags
Browse files Browse the repository at this point in the history 
Add additional flags to `pip install` command
  • Loading branch information
@xylar
xylar authored Sep 23, 2024
2 parents 1c39854 + d32d67e commit 2e06f2e
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 65 deletions.
117 changes: 67 additions & 50 deletions conda/bootstrap.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,8 +312,9 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
# conda packages don't like dashes
version_conda = version.replace('-', '')
packages = f'{packages} "compass={version_conda}={mpi_prefix}_*"'
commands = f'{activate_base} && ' \
f'conda create -y -n {env_name} {channels} {packages}'
commands = \
f'{activate_base} && ' \
f'conda create -y -n {env_name} {channels} {packages}'
check_call(commands, logger=logger)
else:
if env_type == 'dev':
Expand All @@ -336,7 +337,7 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
f'git submodule update --init alphaBetaLab &&' \
f'cd {source_path}/alphaBetaLab&& ' \
f'conda install -y --file dev-spec.txt && ' \
f'python -m pip install --no-deps -e .'
f'python -m pip install --no-deps --no-build-isolation -e .'
check_call(commands, logger=logger)

if recreate or update_jigsaw:
Expand All @@ -348,7 +349,7 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
f'{activate_env} && ' \
f'cd {source_path} && ' \
f'rm -rf compass.egg-info && ' \
f'python -m pip install --no-deps -e .'
f'python -m pip install --no-deps --no-build-isolation -e .'
check_call(commands, logger=logger)

print('Installing pre-commit\n')
Expand Down Expand Up @@ -402,13 +403,13 @@ def build_jigsaw(activate_env, source_path, env_path, logger):
commands = \
f'{activate_env} && ' \
f'cd {source_path}/jigsaw-python && ' \
f'python -m pip install --no-deps -e . && ' \
f'python -m pip install --no-deps --no-build-isolation -e . && ' \
f'cp jigsawpy/_bin/* ${{CONDA_PREFIX}}/bin'
check_call(commands, logger=logger)

t1 = time.time()
total = int(t1 - t0 + 0.5)
message = f'JIGSAW install took {total:.1f} s.'
message = f'JIGSAW install took {total:.1f} s.' # noqa: 231
if logger is None:
print(message)
else:
Expand All @@ -424,11 +425,12 @@ def get_env_vars(machine, compiler, mpilib):
env_vars = 'export MPAS_EXTERNAL_LIBS=""\n'

if 'intel' in compiler and machine == 'anvil':
env_vars = f'{env_vars}' \
f'export I_MPI_CC=icc\n' \
f'export I_MPI_CXX=icpc\n' \
f'export I_MPI_F77=ifort\n' \
f'export I_MPI_F90=ifort\n'
env_vars = \
f'{env_vars}' \
f'export I_MPI_CC=icc\n' \
f'export I_MPI_CXX=icpc\n' \
f'export I_MPI_F77=ifort\n' \
f'export I_MPI_F90=ifort\n'

if machine.startswith('conda'):
# we're using parallelio so we don't have ADIOS support
Expand All @@ -442,9 +444,10 @@ def get_env_vars(machine, compiler, mpilib):
f'export MPAS_EXTERNAL_LIBS="${{MPAS_EXTERNAL_LIBS}} -lgomp"\n'

if mpilib == 'mvapich':
env_vars = f'{env_vars}' \
f'export MV2_ENABLE_AFFINITY=0\n' \
f'export MV2_SHOW_CPU_BINDING=1\n'
env_vars = \
f'{env_vars}' \
f'export MV2_ENABLE_AFFINITY=0\n' \
f'export MV2_SHOW_CPU_BINDING=1\n'

if machine.startswith('chicoma') or machine.startswith('pm'):
env_vars = \
Expand Down Expand Up @@ -525,8 +528,9 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901
f'"scorpio'
f'@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"')
# make sure scorpio, not esmf, libraries are linked
lib_path = f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view/lib'
lib_path = \
f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view/lib'
scorpio_lib_path = '$(spack find --format "{prefix}" scorpio)'
custom_spack = \
f'{custom_spack}' \
Expand Down Expand Up @@ -572,8 +576,9 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901
custom_spack=custom_spack, spack_mirror=spack_mirror)

# remove ESMC/ESMF include files that interfere with MPAS time keeping
include_path = f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view/include'
include_path = \
f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view/include'
for prefix in ['ESMC', 'esmf']:
files = glob.glob(os.path.join(include_path, f'{prefix}*'))
for filename in files:
Expand All @@ -587,10 +592,12 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901
include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf,
yaml_template=yaml_template)

spack_view = f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view'
env_vars = f'{env_vars}' \
f'export PIO={spack_view}\n'
spack_view = \
f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view'
env_vars = \
f'{env_vars}' \
f'export PIO={spack_view}\n'
if parallelio != 'None':
env_vars = \
f'{env_vars}' \
Expand Down Expand Up @@ -619,14 +626,16 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901
f'${{ALBANY_LINK_LIBS}} {stdcxx} {mpicxx}"\n'

if lapack != 'None':
env_vars = f'{env_vars}' \
f'export LAPACK={spack_view}\n' \
f'export USE_LAPACK=true\n'
env_vars = \
f'{env_vars}' \
f'export LAPACK={spack_view}\n' \
f'export USE_LAPACK=true\n'

if petsc != 'None':
env_vars = f'{env_vars}' \
f'export PETSC={spack_view}\n' \
f'export USE_PETSC=true\n'
env_vars = \
f'{env_vars}' \
f'export PETSC={spack_view}\n' \
f'export USE_PETSC=true\n'

return spack_branch_base, spack_script, env_vars

Expand All @@ -636,7 +645,7 @@ def set_ld_library_path(spack_branch_base, spack_env, logger):
f'source {spack_branch_base}/share/spack/setup-env.sh && ' \
f'spack env activate {spack_env} && ' \
f'spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH] && ' \
f'spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' # noqa: E501
f'spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' # noqa: E501,E231
check_call(commands, logger=logger)


Expand All @@ -658,21 +667,26 @@ def write_load_compass(template_path, activ_path, conda_base, env_type,
script_filename = '{}/{}{}.sh'.format(activ_path, prefix, activ_suffix)

if not env_only:
env_vars = f'{env_vars}\n' \
f'export USE_PIO2=true'
env_vars = \
f'{env_vars}\n' \
f'export USE_PIO2=true'
if without_openmp:
env_vars = f'{env_vars}\n' \
f'export OPENMP=false'
env_vars = \
f'{env_vars}\n' \
f'export OPENMP=false'
else:
env_vars = f'{env_vars}\n' \
f'export OPENMP=true'
env_vars = \
f'{env_vars}\n' \
f'export OPENMP=true'

env_vars = f'{env_vars}\n' \
f'export HDF5_USE_FILE_LOCKING=FALSE\n' \
f'export LOAD_COMPASS_ENV={script_filename}'
env_vars = \
f'{env_vars}\n' \
f'export HDF5_USE_FILE_LOCKING=FALSE\n' \
f'export LOAD_COMPASS_ENV={script_filename}'
if machine is not None and not machine.startswith('conda'):
env_vars = f'{env_vars}\n' \
f'export COMPASS_MACHINE={machine}'
env_vars = \
f'{env_vars}\n' \
f'export COMPASS_MACHINE={machine}'

filename = f'{template_path}/load_compass.template'
with open(filename, 'r') as f:
Expand All @@ -688,7 +702,8 @@ def write_load_compass(template_path, activ_path, conda_base, env_type,
mkdir -p conda/logs
echo Reinstalling compass package in edit mode...
rm -rf compass.egg-info
python -m pip install --no-deps -e . &> conda/logs/install_compass.log
python -m pip install --no-deps --no-build-isolation -e . \\
&> conda/logs/install_compass.log
echo Done.
echo
fi
Expand Down Expand Up @@ -1089,11 +1104,12 @@ def main(): # noqa: C901

if local_mache:
print('Install local mache\n')
commands = f'source {conda_base}/etc/profile.d/conda.sh && ' \
f'conda activate {conda_env_name} && ' \
f'cd ../build_mache/mache && ' \
f'conda install -y --file spec-file.txt && ' \
f'python -m pip install --no-deps .'
commands = \
f'source {conda_base}/etc/profile.d/conda.sh && ' \
f'conda activate {conda_env_name} && ' \
f'cd ../build_mache/mache && ' \
f'conda install -y --file spec-file.txt && ' \
f'python -m pip install --no-deps --no-build-isolation .'
check_call(commands, logger=logger)

previous_conda_env = conda_env_name
Expand All @@ -1109,10 +1125,11 @@ def main(): # noqa: C901
config, args.update_spack, machine, compiler, mpi,
spack_env, spack_base, spack_template_path, env_vars,
args.tmpdir, logger)
spack_script = f'echo Loading Spack environment...\n' \
f'{spack_script}\n' \
f'echo Done.\n' \
f'echo\n'
spack_script = \
f'echo Loading Spack environment...\n' \
f'{spack_script}\n' \
f'echo Done.\n' \
f'echo\n'
else:
env_vars = \
f'{env_vars}' \
Expand Down
29 changes: 16 additions & 13 deletions conda/configure_compass_env.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ def bootstrap(activate_install_env, source_path, local_conda_build):

print('Creating the compass conda environment\n')
bootstrap_command = f'{source_path}/conda/bootstrap.py'
command = f'{activate_install_env} && ' \
f'{bootstrap_command} {" ".join(sys.argv[1:])}'
command = \
f'{activate_install_env} && ' \
f'{bootstrap_command} {" ".join(sys.argv[1:])}'
if local_conda_build is not None:
command = f'{command} --local_conda_build {local_conda_build}'
check_call(command)
Expand All @@ -51,8 +52,9 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate,
f'conda create -y -n {env_name} {channels} {packages}'
else:
print('Updating conda environment for installing compass\n')
commands = f'{activate_base} && ' \
f'conda install -y -n {env_name} {channels} {packages}'
commands = \
f'{activate_base} && ' \
f'conda install -y -n {env_name} {channels} {packages}'

check_call(commands, logger=logger)

Expand Down Expand Up @@ -107,15 +109,16 @@ def main():

if local_mache:
print('Clone and install local mache\n')
commands = f'{activate_install_env} && ' \
f'rm -rf conda/build_mache && ' \
f'mkdir -p conda/build_mache && ' \
f'cd conda/build_mache && ' \
f'git clone -b {args.mache_branch} ' \
f'[email protected]:{args.mache_fork}.git mache && ' \
f'cd mache && ' \
f'conda install -y --file spec-file.txt && ' \
f'python -m pip install --no-deps .'
commands = \
f'{activate_install_env} && ' \
f'rm -rf conda/build_mache && ' \
f'mkdir -p conda/build_mache && ' \
f'cd conda/build_mache && ' \
f'git clone -b {args.mache_branch} ' \
f'[email protected]:{args.mache_fork}.git mache && ' \
f'cd mache && ' \
f'conda install -y --file spec-file.txt && ' \
f'python -m pip install --no-deps --no-build-isolation .' # noqa: E231,E501

check_call(commands, logger=logger)

Expand Down
4 changes: 2 additions & 2 deletions docs/developers_guide/quick_start.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@ switch branches.

.. code-block:: bash
python -m pip install -e .
python -m pip install --no-deps --no-build-isolation -e .
The activation script will do this automatically when you source it in
the root directory of your compass branch. The activation script will also
Expand Down Expand Up @@ -446,7 +446,7 @@ to the current directory.

.. code-block:: bash
python -m pip install -e .
python -m pip install --no-deps --no-build-isolation -e .
This will be substantially faster than rerunning
``./conda/configure_compass_env.py ...`` but at the risk that dependencies are
Expand Down

0 comments on commit 2e06f2e

Please sign in to comment.