azure-pipelines.yml

trigger:
  tags:
    include:
    - v*
  branches:
    include:
    - master

pr:
- master

variables:
  package_name: cygrid
  is.tag: ${{ startsWith(variables['Build.SourceBranch'], 'refs/tags/') }}
  publish.wheels: ${{ startsWith(variables['Build.SourceBranch'], 'refs/tags/') }}
  is.master: ${{ startsWith(variables['Build.SourceBranch'], 'refs/heads/master') }}
  is.pullrequest: ${{ startsWith(variables['Build.Reason'], 'PullRequest') }}
  # is.tag: ${{ true }}
  # why is the following not working? --> because Agent.OS is not available
  # at compile time (would be necessary for template expressions to work)
  is.linux: ${{ startsWith(variables['Agent.OS'], 'Linux') }}
  is.macos: ${{ startsWith(variables['Agent.OS'], 'Darwin') }}
  is.win: ${{ startsWith(variables['Agent.OS'], 'Windows') }}

jobs:

- job:
  displayName: Job

  variables:
    arch: x86_64
    plat: manylinux_2_28_x86_64
    image: quay.io/pypa/manylinux_2_28_x86_64
    # for very old systems, the following would also be an alternative:
    # plat: manylinux1_x86_64
    # image: skhep/manylinuxgcc-x86_64
    astropy.version: ''
    python.architecture: x64
    sphinx.version: '>=3.0'
    # sphinx.version: '<3.0'
    pytest.version: ''
    doctestplus.version: ''
    extra.packages: ''

  strategy:
    matrix:
      linux38:
        imageName: 'ubuntu-latest'
        python.version: '3.8'
        py_whl: cp38-
        numpy.version: '1.18.1'
        astropy.version: '==4'
        extra.packages: 'gcc_linux-64 gxx_linux-64 sysroot_linux-64 "jinja2<3.1"'
      linux39:
        imageName: 'ubuntu-latest'
        python.version: '3.9'
        py_whl: cp39-
        numpy.version: '1.20.3'
        extra.packages: 'gcc_linux-64 gxx_linux-64'
      linux310:
        imageName: 'ubuntu-latest'
        python.version: '3.10'
        py_whl: cp310-
        numpy.version: '1.21.6'
        extra.packages: 'gcc_linux-64 gxx_linux-64'
      linux311:
        imageName: 'ubuntu-latest'
        python.version: '3.11'
        py_whl: cp311-
        numpy.version: '1.23.4'
        publish.docs: ${{ or(eq(variables['is.master'], true), eq(variables['is.tag'], true)) }}
        # publish.docs: ${{ true }}
        ${{ if eq(variables['is.tag'], true) }}:
          docs.target: '.'
        ${{ if eq(variables['is.tag'], false) }}:
          docs.target: 'latest'
        # tarball should only be published by one thread, as it is version-
        # independent
        build.tarball: ${{ true }}
        publish.tarball: ${{ eq(variables['is.tag'], true) }}
        extra.packages: 'gcc_linux-64 gxx_linux-64'
      linux312:
        imageName: 'ubuntu-latest'
        python.version: '3.12'
        py_whl: cp312-
        numpy.version: '1.26.4'
        extra.packages: 'gcc_linux-64 gxx_linux-64'
      mac38:
        imageName: 'macos-latest'
        python.version: '3.8'
        numpy.version: '1.18.1'
        astropy.version: '==4'
        extra.packages: 'compilers llvm-openmp clang_osx-64 clangxx_osx-64'
      mac39:
        imageName: 'macos-latest'
        python.version: '3.9'
        numpy.version: '1.20.3'
        extra.packages: 'compilers llvm-openmp clang_osx-64 clangxx_osx-64'
      mac310:
        imageName: 'macos-latest'
        python.version: '3.10'
        numpy.version: '1.21.6'
        extra.packages: 'compilers llvm-openmp clang_osx-64 clangxx_osx-64'
      mac311:
        imageName: 'macos-latest'
        python.version: '3.11'
        numpy.version: '1.23.4'
        extra.packages: 'compilers llvm-openmp clang_osx-64 clangxx_osx-64'
      mac312:
        imageName: 'macos-latest'
        python.version: '3.12'
        numpy.version: '1.26.4'
        extra.packages: 'compilers llvm-openmp clang_osx-64 clangxx_osx-64'
      win38:
        imageName: 'windows-latest'
        python.version: '3.8'
        # numpy.version: '1.17.4'
        numpy.version: '1.18.1'
        astropy.version: '==4'
      win39:
        imageName: 'windows-latest'
        python.version: '3.9'
        numpy.version: '1.20.3'
      win310:
        imageName: 'windows-latest'
        python.version: '3.10'
        numpy.version: '1.21.6'
      win311:
        imageName: 'windows-latest'
        python.version: '3.11'
        numpy.version: '1.23.4'
      win312:
        imageName: 'windows-latest'
        python.version: '3.12'
        numpy.version: '1.26.4'

  pool:
    vmImage: $(imageName)

  steps:

  - bash: |
      sudo chown -R $USER $CONDA
    displayName: Fix a conda bug on MacOS
    condition: startsWith(variables['Agent.OS'], 'Darwin')

  # see https://github.com/mamba-org/mamba/issues/488
  # and https://github.com/MPAS-Dev/compass/pull/320
  - bash: |
      rm /usr/share/miniconda/pkgs/cache/*.json
    displayName: Fix a conda bug on Linux
    condition: startsWith(variables['Agent.OS'], 'Linux')

  - bash: |
      echo "##vso[task.prependpath]$CONDA/bin"
      echo "##vso[task.setvariable variable=conda.activate]source activate"
    displayName: Add conda to PATH, Linux/MacOS
    condition: not(startsWith(variables['Agent.OS'], 'Windows'))

  - bash: |
      echo "##vso[task.prependpath]/usr/bin"
      echo "##vso[task.setvariable variable=SETUPTOOLS_ENABLE_FEATURES]legacy-editable"
    displayName: Bugfix MacOS command line interface
    condition: startsWith(variables['Agent.OS'], 'Darwin')

  # echo "##vso[task.setvariable variable=conda.activate]call $env:CONDA\Scripts\activate.bat"
  - powershell: |
      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
      echo "##vso[task.setvariable variable=conda.activate]call activate.bat"
    displayName: Add conda to PATH, Windows
    condition: startsWith(variables['Agent.OS'], 'Windows')

  - script: |
      set -x
    displayName: Verbosity on Linux/MacOS
    condition: not(startsWith(variables['Agent.OS'], 'Windows'))

  - script: |
      echo on
    displayName: Verbosity on Win
    condition: startsWith(variables['Agent.OS'], 'Windows')

  - script: |
      echo "Build.SourceBranch" $(Build.SourceBranch)
      echo "Agent.BuildDirectory" $(Agent.BuildDirectory)
      echo "Agent.OS" $(Agent.OS) "/" ${{ variables['Agent.OS'] }}
      echo "is.linux" $(is.linux) "/" ${{ variables['is.linux'] }}
      echo "is.macos" $(is.macos) "/" ${{ variables['is.macos'] }}
      echo "is.win" $(is.win) "/" ${{ variables['is.win'] }}
      echo "conda.activate" $(conda.activate)
      echo "is.tag" $(is.tag)
      echo "publish.docs" $(publish.docs)
      echo "is.pullrequest" $(is.pullrequest)
      echo "docs.target" $(docs.target)
      echo "publish.wheels" $(publish.wheels)
      echo "python.version" $(python.version)
      echo "py_whl" $(py_whl)
      echo "numpy.version" $(numpy.version)
      echo "astropy.version" $(astropy.version)
      echo "sphinx.version" $(sphinx.version)
      echo "imageName" $(imageName)
      echo "conda version"; conda -V
    displayName: Show Vars

  - bash: |
      gcc -v
    displayName: Show gcc version
    # condition: not(eq(variables['is.win'], true))
    condition: not(startsWith(variables['Agent.OS'], 'Windows'))

  - script: |
      conda create --yes --quiet --name cygrid-env -c conda-forge
    displayName: Create Anaconda environment

  # note: with project.toml it should not be necessary to have cython
  # installed already, but on PRs building wheels and the tarball still fails
  # for some reason!? therefore, we add it here (should also be faster than
  # pip installation)
  - script: |
      $(conda.activate) cygrid-env
      conda install --yes --quiet -c conda-forge python=$(python.version) astropy$(astropy.version) astroquery build cython extension-helpers matplotlib-base numpy==$(numpy.version) pip "pytest$(pytest.version)" pytest-astropy pytest-doctestplus reproject scipy setuptools-scm twine $(extra.packages)
    displayName: Install Anaconda packages

  - script: |
      $(conda.activate) cygrid-env
      # pip install -e .
      python -m pip install --no-build-isolation --no-deps -v -v -v -e .
    displayName: Building

  # pytest -rsx --doctest-rst docs cygrid
  - script: |
      $(conda.activate) cygrid-env
      pytest -rsx --ignore-glob="*/setup_package.py" cygrid
    displayName: Testing
    condition: succeeded()

  - script: |
      $(conda.activate) cygrid-env
      # --doctest-rst not recommended, use --doctest-glob instead
      # the following only works because of pip install -e
      pytest -rsx --doctest-plus --doctest-glob="*.rst" --doctest-ignore-import-errors docs
    displayName: Testing docs
    condition: succeeded()
    # condition: and(succeeded(), startsWith(variables['Agent.OS'], 'Linux'), eq(variables['python.version'], 3.9))

  - script: |
      $(conda.activate) cygrid-env
      conda install --yes --quiet --name cygrid-env -c conda-forge "sphinx$(sphinx.version)" sphinx-astropy sphinx-astropy[confv2] sphinx-copybutton pydata-sphinx-theme sphinx-design
      cd docs
      sphinx-build . _build/html -b html
    displayName: Make docs
    # condition: and(succeeded(), startsWith(variables['Agent.OS'], 'Linux'), eq(variables['python.version'], 3.8))
    condition: and(succeeded(), startsWith(variables['Agent.OS'], 'Linux'), eq(variables['python.version'], '3.11'))

  - task: DownloadSecureFile@1
    inputs:
      secureFile: ghpages_deploy_cygrid
    displayName: 'Get the deploy key'
    condition: and(succeeded(), eq(variables['publish.docs'], true))

  - bash: |
      mkdir ~/.ssh && mv $DOWNLOADSECUREFILE_SECUREFILEPATH ~/.ssh/id_rsa
      chmod 700 ~/.ssh && chmod 600 ~/.ssh/id_rsa
      ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
      $(conda.activate) cygrid-env
      mkdir gh_pages && cd gh_pages
      git init
      git config --local user.name "Benjamin Winkel"
      git config --local user.email "bwinkel@mpifr.de"
      git remote add github git@github.com:bwinkel/cygrid.git
      git fetch github gh-pages
      git checkout --track github/gh-pages
      rsync -av ../docs/_build/html/ ${DOCS_TARGET}/
      docs_version=$(python -c "import cygrid; print(cygrid.__version__)")
      git add -A && git commit -m "Update docs for $docs_version ***NO_CI***"
      bash ../retry.sh git push github
    displayName: Publish docs
    condition: and(succeeded(), eq(variables['publish.docs'], true))

  - task: TwineAuthenticate@1
    inputs:
      pythonUploadServiceConnection: pypi_cygrid
    displayName: 'Twine Authenticate'

  - bash: |
      rm -rf build
      docker run -e NPY_NUM_BUILD_JOBS=4 -e PLAT=$(plat) -e package_name=$(package_name) -e py_whl=$(py_whl) --rm -v `pwd`:/io $(image) /io/.ci/build-wheels.sh
      ls -lh wheelhouse/
      mkdir -p dist
      cp wheelhouse/$(package_name)*.whl dist/.
      ls dist
    displayName: Build wheels, Linux
    condition: and(succeeded(), startsWith(variables['Agent.OS'], 'Linux'))

  - script: |
      $(conda.activate) cygrid-env
      # python setup.py bdist_wheel
      python -m build --wheel
    displayName: Build wheels, MacOS/Windows
    condition: and(succeeded(), not(startsWith(variables['Agent.OS'], 'Linux')))

  # # twine upload with python 3.5 doesn't work?
  # # (pkg_resources.DistributionNotFound: The 'readme-renderer>=21.0' distribution was not found)
  # # will use system python for upload
  # - task: UsePythonVersion@0
  #   inputs:
  #     versionSpec: '3.9'

  - script: |
      $(conda.activate) cygrid-env
      ls dist/*.whl
      python -m twine upload -r "pypi_cygrid" --skip-existing --config-file $(PYPIRC_PATH) dist/*.whl
    displayName: Upload wheels, Linux/MacOS
    condition: and(succeeded(), eq(variables['publish.wheels'], true), not(startsWith(variables['Agent.OS'], 'Windows')))

  - script: |
      $(conda.activate) cygrid-env
      ls dist/*.whl
      python -m twine upload -r "pypi_cygrid" --skip-existing --config-file %PYPIRC_PATH% dist/*.whl
    displayName: Upload wheels, Windows
    condition: and(succeeded(), eq(variables['publish.wheels'], true), startsWith(variables['Agent.OS'], 'Windows'))

  - script: |
      $(conda.activate) cygrid-env
      # python setup.py sdist
      sudo rm -rfv *.egg-info
      python -m build --sdist
    displayName: Build tarball
    condition: and(succeeded(), eq(variables['build.tarball'], true))

  - script: |
      $(conda.activate) cygrid-env
      ls dist/*.tar.gz
      python -m twine upload -r "pypi_cygrid" --config-file $(PYPIRC_PATH) dist/*.tar.gz
    displayName: Upload tarball
    condition: and(succeeded(), eq(variables['publish.tarball'], true))

  - script: rm ~/.ssh/id_rsa || true
    displayName: 'Cleanup docs deploy key'
    condition: and(always(), eq(variables['publish.docs'], true))