⚡️ 🚑 Update environment locking script (#46)

microsoft · Jun 29, 2022 · 5de3842 · 5de3842
1 parent f782e12
commit 5de3842
Show file tree

Hide file tree

Showing 4 changed files with 146 additions and 137 deletions.
diff --git a/create_and_lock_environment.sh b/create_and_lock_environment.sh
@@ -15,25 +15,35 @@ env_name="${name_arr[1]}"
 
 # clear old conda envs, create new one
 export CONDA_ALWAYS_YES="true"
-conda env remove --name ${env_name::-1}
+conda env remove --name ${env_name}
 conda env create --file primary_deps.yml
 
 # export new environment to environment.yml
-conda env export -n ${env_name::-1} | grep -v "prefix:" > environment.yml
+conda env export -n ${env_name} | grep -v "prefix:" > environment.yml
 unset CONDA_ALWAYS_YES
 
 # remove python version hash (technically not locked, so still potential for problems here if python secondary deps change)
 while IFS='' read -r line; do
-    if [[ $line == *"- python="* ]]; then
+    if [[ $line == "    "* ]]; then
+        echo "${line:6}" >> requirements_tmp.txt
+
+    elif [[ $line == *"- python="* ]]; then
 
         IFS='=' read -ra python_arr <<< "$line"
         unset python_arr[-1]
-        echo "${python_arr[0]}"="${python_arr[1]}"
+        echo "${python_arr[0]}"="${python_arr[1]}" >> environment.yml.tmp
     elif [[ ! $line == "#"* ]]; then
-        echo "${line}"
+        echo "${line}" >> environment.yml.tmp
     fi
-done < environment.yml > environment.yml.tmp
+done < environment.yml
+echo "    - -r file:requirements.txt" >> environment.yml.tmp
+
+# create environment.yml and requirements.txt
 echo "# WARNING - DO NOT EDIT THIS FILE MANUALLY" > environment.yml
 echo "# Please refer to the environment documentation for instructions on how to create a new version of this file: https://github.com/microsoft/InnerEye-DeepLearning/blob/main/docs/environment.md" >> environment.yml
+echo "# WARNING - DO NOT EDIT THIS FILE MANUALLY" > requirements.txt
+echo "# Please refer to the environment documentation for instructions on how to create a new version of this file: https://github.com/microsoft/InnerEye-DeepLearning/blob/main/docs/environment.md" >> requirements.txt
+
 cat environment.yml.tmp >> environment.yml
 rm environment.yml.tmp
+mv requirements_tmp.txt requirements.txt
diff --git a/environment.yml b/environment.yml
@@ -7,14 +7,14 @@ dependencies:
   - _libgcc_mutex=0.1=main
   - _openmp_mutex=5.1=1_gnu
   - ca-certificates=2022.4.26=h06a4308_0
-  - certifi=2022.5.18.1=py37h06a4308_0
+  - certifi=2022.6.15=py37h06a4308_0
   - libedit=3.1.20210910=h7f8727e_0
   - libffi=3.2.1=hf484d3e_1007
   - libgcc-ng=11.2.0=h1234567_1
   - libgomp=11.2.0=h1234567_1
   - libstdcxx-ng=11.2.0=h1234567_1
   - ncurses=6.3=h7f8727e_2
-  - openssl=1.1.1o=h7f8727e_0
+  - openssl=1.1.1p=h5eee18b_0
   - pip=20.1.1=py37_1
   - python=3.7.3
   - readline=7.0=h7b6447c_5
@@ -25,129 +25,4 @@ dependencies:
   - xz=5.2.5=h7f8727e_1
   - zlib=1.2.12=h7f8727e_2
   - pip:
-    - absl-py==1.1.0
-    - adal==1.2.7
-    - applicationinsights==0.11.10
-    - argcomplete==2.0.0
-    - attrs==21.4.0
-    - azure-common==1.1.28
-    - azure-core==1.22.1
-    - azure-graphrbac==0.61.1
-    - azure-identity==1.7.0
-    - azure-mgmt-authorization==2.0.0
-    - azure-mgmt-containerregistry==9.1.0
-    - azure-mgmt-core==1.3.0
-    - azure-mgmt-keyvault==9.3.0
-    - azure-mgmt-resource==20.1.0
-    - azure-mgmt-storage==19.1.0
-    - azureml-automl-core==1.41.1.post1
-    - azureml-core==1.41.0.post3
-    - azureml-dataprep==3.1.4
-    - azureml-dataprep-native==38.0.0
-    - azureml-dataprep-rslex==2.5.5
-    - azureml-dataset-runtime==1.41.0
-    - azureml-pipeline==1.41.0
-    - azureml-pipeline-core==1.41.0
-    - azureml-pipeline-steps==1.41.0
-    - azureml-sdk==1.41.0
-    - azureml-telemetry==1.41.0
-    - azureml-tensorboard==1.42.0
-    - azureml-train-automl-client==1.41.0
-    - azureml-train-core==1.41.0
-    - azureml-train-restclients-hyperdrive==1.41.0
-    - backports-tempfile==1.0
-    - backports-weakref==1.0.post1
-    - bcrypt==3.2.2
-    - cachetools==5.2.0
-    - cffi==1.15.0
-    - charset-normalizer==2.0.12
-    - click==8.1.3
-    - cloudpickle==2.1.0
-    - conda-merge==0.2.0
-    - contextlib2==21.6.0
-    - coverage==6.4.1
-    - cryptography==36.0.2
-    - distro==1.7.0
-    - docker==5.0.3
-    - dotnetcore2==2.1.23
-    - flake8==3.8.4
-    - flask==2.1.1
-    - flask-injector==0.13.0
-    - fusepy==3.0.1
-    - google-auth==2.7.0
-    - google-auth-oauthlib==0.4.6
-    - grpcio==1.46.3
-    - hi-ml-azure==0.2.1
-    - humanfriendly==10.0
-    - idna==3.3
-    - importlib-metadata==4.11.4
-    - iniconfig==1.1.1
-    - injector==0.19.0
-    - isodate==0.6.1
-    - itsdangerous==2.1.2
-    - jeepney==0.8.0
-    - jinja2==3.1.2
-    - jmespath==0.10.0
-    - jsonpickle==2.2.0
-    - knack==0.9.0
-    - markdown==3.3.7
-    - markupsafe==2.1.1
-    - mccabe==0.6.1
-    - memory-tempfile==2.2.3
-    - more-itertools==8.13.0
-    - msal==1.18.0
-    - msal-extensions==0.3.1
-    - msrest==0.6.21
-    - msrestazure==0.6.4
-    - mypy==0.790
-    - mypy-extensions==0.4.3
-    - ndg-httpsclient==0.5.1
-    - numpy==1.21.6
-    - oauthlib==3.2.0
-    - packaging==21.3
-    - pandas==1.3.5
-    - param==1.12.1
-    - paramiko==2.11.0
-    - pathspec==0.9.0
-    - pkginfo==1.8.2
-    - pluggy==0.13.1
-    - portalocker==2.4.0
-    - protobuf==4.21.1
-    - py==1.11.0
-    - pyarrow==3.0.0
-    - pyasn1==0.4.8
-    - pyasn1-modules==0.2.8
-    - pycodestyle==2.6.0
-    - pycparser==2.21
-    - pydicom==2.1.2
-    - pyflakes==2.2.0
-    - pygments==2.12.0
-    - pyjwt==2.4.0
-    - pynacl==1.5.0
-    - pyopenssl==22.0.0
-    - pyparsing==3.0.9
-    - pysocks==1.7.1
-    - pytest==6.0.1
-    - pytest-cov==2.10.1
-    - python-dateutil==2.8.2
-    - pytz==2022.1
-    - pyyaml==6.0
-    - requests==2.27.1
-    - requests-oauthlib==1.3.1
-    - rsa==4.8
-    - ruamel-yaml==0.17.21
-    - ruamel-yaml-clib==0.2.6
-    - secretstorage==3.3.2
-    - simpleitk==2.0.2
-    - six==1.16.0
-    - tabulate==0.8.9
-    - tensorboard==2.9.0
-    - tensorboard-data-server==0.6.1
-    - tensorboard-plugin-wit==1.8.1
-    - toml==0.10.2
-    - typed-ast==1.4.3
-    - typing-extensions==4.2.0
-    - urllib3==1.26.7
-    - websocket-client==1.3.2
-    - werkzeug==2.1.1
-    - zipp==3.8.0
+    - -r file:requirements.txt
diff --git a/primary_deps.yml b/primary_deps.yml
@@ -14,6 +14,7 @@ dependencies:
       - Flask==2.1.1
       - hi-ml-azure==0.2.1
       - memory-tempfile==2.2.3
+      - msrest==0.6.21
       - mypy==0.790
       - pydicom==2.1.2
       - pytest-cov==2.10.1

diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,126 @@
-# DO NOT REMOVE THIS FILE
-# Package dependencies are stored in environment.yml
-# This file is only here to allow "az webapp up" to run without errors.
+absl-py==1.1.0
+adal==1.2.7
+applicationinsights==0.11.10
+argcomplete==2.0.0
+attrs==21.4.0
+azure-common==1.1.28
+azure-core==1.22.1
+azure-graphrbac==0.61.1
+azure-identity==1.7.0
+azure-mgmt-authorization==2.0.0
+azure-mgmt-containerregistry==9.1.0
+azure-mgmt-core==1.3.1
+azure-mgmt-keyvault==9.3.0
+azure-mgmt-resource==20.1.0
+azure-mgmt-storage==19.1.0
+azureml-automl-core==1.41.1.post1
+azureml-core==1.41.0.post3
+azureml-dataprep==3.1.4
+azureml-dataprep-native==38.0.0
+azureml-dataprep-rslex==2.5.5
+azureml-dataset-runtime==1.41.0
+azureml-pipeline==1.41.0
+azureml-pipeline-core==1.41.0
+azureml-pipeline-steps==1.41.0
+azureml-sdk==1.41.0
+azureml-telemetry==1.41.0
+azureml-tensorboard==1.43.0
+azureml-train-automl-client==1.41.0
+azureml-train-core==1.41.0
+azureml-train-restclients-hyperdrive==1.41.0
+backports-tempfile==1.0
+backports-weakref==1.0.post1
+bcrypt==3.2.2
+cachetools==5.2.0
+cffi==1.15.0
+charset-normalizer==2.0.12
+click==8.1.3
+cloudpickle==2.1.0
+conda-merge==0.2.0
+contextlib2==21.6.0
+coverage==6.4.1
+cryptography==36.0.2
+distro==1.7.0
+docker==5.0.3
+dotnetcore2==2.1.23
+flake8==3.8.4
+flask==2.1.1
+flask-injector==0.13.0
+fusepy==3.0.1
+google-auth==2.9.0
+google-auth-oauthlib==0.4.6
+grpcio==1.47.0
+hi-ml-azure==0.2.1
+humanfriendly==10.0
+idna==3.3
+importlib-metadata==4.12.0
+iniconfig==1.1.1
+injector==0.20.0
+isodate==0.6.1
+itsdangerous==2.1.2
+jeepney==0.8.0
+jinja2==3.1.2
+jmespath==0.10.0
+jsonpickle==2.2.0
+knack==0.9.0
+markdown==3.3.7
+markupsafe==2.1.1
+mccabe==0.6.1
+memory-tempfile==2.2.3
+more-itertools==8.13.0
+msal==1.18.0
+msal-extensions==0.3.1
+msrest==0.6.21
+msrestazure==0.6.4
+mypy==0.790
+mypy-extensions==0.4.3
+ndg-httpsclient==0.5.1
+numpy==1.21.6
+oauthlib==3.2.0
+packaging==21.3
+pandas==1.3.5
+param==1.12.2
+paramiko==2.11.0
+pathspec==0.9.0
+pkginfo==1.8.3
+pluggy==0.13.1
+portalocker==2.4.0
+protobuf==3.19.4
+py==1.11.0
+pyarrow==3.0.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycodestyle==2.6.0
+pycparser==2.21
+pydicom==2.1.2
+pyflakes==2.2.0
+pygments==2.12.0
+pyjwt==2.4.0
+pynacl==1.5.0
+pyopenssl==22.0.0
+pyparsing==3.0.9
+pysocks==1.7.1
+pytest==6.0.1
+pytest-cov==2.10.1
+python-dateutil==2.8.2
+pytz==2022.1
+pyyaml==6.0
+requests==2.28.0
+requests-oauthlib==1.3.1
+rsa==4.8
+ruamel-yaml==0.17.21
+ruamel-yaml-clib==0.2.6
+secretstorage==3.3.2
+simpleitk==2.0.2
+six==1.16.0
+tabulate==0.8.10
+tensorboard==2.9.1
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+toml==0.10.2
+typed-ast==1.4.3
+typing-extensions==4.2.0
+urllib3==1.26.7
+websocket-client==1.3.3
+werkzeug==2.1.1
+zipp==3.8.0