Collect and publish catalogs #63

Workflow file for this run

.github/workflows/catalogs.yml at e1effee

	name: Collect and publish catalogs
	on:
	workflow_dispatch:
	inputs:
	channel:
	description: 'Channel to publish catalogs to'
	required: true
	default: stgn
	schedule:
	- cron: '0 1 * * *' # 01:00 UTC every day

	env:
	PIP_DISABLE_PIP_VERSION_CHECK: on
	PIP_DEFAULT_TIMEOUT: 10
	PIP_PROGRESS_BAR: off

	jobs:
	catalog-aws:
	name: Collect AWS catalog
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: 3.11
	- name: Install dependencies
	run: \|
	pip install pip -U
	pip install -e '.[aws]'
	- name: Collect catalog
	working-directory: src
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	run: python -m gpuhunt aws --output ../aws.csv
	- uses: actions/upload-artifact@v4
	with:
	name: catalogs-aws
	path: aws.csv
	retention-days: 1

	catalog-azure:
	name: Collect Azure catalog
	runs-on: ubuntu-latest
	permissions:
	id-token: write
	contents: read
	steps:
	- uses: actions/checkout@v4
	- uses: azure/login@v1
	with:
	creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}'
	- uses: actions/setup-python@v5
	with:
	python-version: 3.11
	- name: Install dependencies
	run: \|
	pip install pip -U
	pip install -e '.[azure]'
	- name: Collect catalog
	working-directory: src
	env:
	AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
	run: python -m gpuhunt azure --output ../azure.csv
	- uses: actions/upload-artifact@v4
	with:
	name: catalogs-azure
	path: azure.csv
	retention-days: 1

	catalog-cudo:
	name: Collect Cudo catalog
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: 3.11
	- name: Install dependencies
	run: \|
	pip install pip -U
	pip install -e '.'
	- name: Collect catalog
	working-directory: src
	run: python -m gpuhunt cudo --output ../cudo.csv
	- uses: actions/upload-artifact@v4
	with:
	name: catalogs-cudo
	path: cudo.csv
	retention-days: 1

	catalog-datacrunch:
	name: Collect DataCrunch catalog
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: 3.11
	- name: Install dependencies
	run: \|
	pip install pip -U
	pip install -e '.[datacrunch]'
	- name: Collect catalog
	working-directory: src
	env:
	DATACRUNCH_CLIENT_ID: ${{ secrets.DATACRUNCH_CLIENT_ID }}
	DATACRUNCH_CLIENT_SECRET: ${{ secrets.DATACRUNCH_CLIENT_SECRET }}
	run: python -m gpuhunt datacrunch --output ../datacrunch.csv
	- uses: actions/upload-artifact@v4
	with:
	name: catalogs-datacrunch
	path: datacrunch.csv
	retention-days: 1

	catalog-gcp:
	name: Collect GCP catalog
	runs-on: ubuntu-latest
	permissions:
	contents: read
	id-token: write
	steps:
	- uses: actions/checkout@v4
	- uses: google-github-actions/auth@v2
	with:
	workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider'
	service_account: '[email protected]'
	create_credentials_file: true
	- uses: actions/setup-python@v5
	with:
	python-version: 3.11
	- name: Install dependencies
	run: \|
	pip install pip -U
	pip install -e '.[gcp]'
	- name: Collect catalog
	working-directory: src
	env:
	GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
	run: python -m gpuhunt gcp --output ../gcp.csv
	- uses: actions/upload-artifact@v4
	with:
	name: catalogs-gcp
	path: gcp.csv
	retention-days: 1

	catalog-lambdalabs:
	name: Collect LambdaLabs catalog
	runs-on: ubuntu-latest
	permissions:
	contents: read
	id-token: write
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: 3.11
	- name: Install dependencies
	run: \|
	pip install pip -U
	pip install -e '.'
	- name: Collect catalog
	working-directory: src
	env:
	LAMBDALABS_TOKEN: ${{ secrets.LAMBDALABS_TOKEN }}
	run: python -m gpuhunt lambdalabs --output ../lambdalabs.csv
	- uses: actions/upload-artifact@v4
	with:
	name: catalogs-lambdalabs
	path: lambdalabs.csv
	retention-days: 1

	catalog-oci:
	name: Collect OCI catalog
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: 3.11
	- name: Install dependencies
	run: \|
	pip install pip -U
	pip install -e '.[oci]'
	- name: Collect catalog
	working-directory: src
	run: python -m gpuhunt oci --output ../oci.csv
	env:
	OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }}
	OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }}
	OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }}
	OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }}
	OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }}
	- uses: actions/upload-artifact@v4
	with:
	name: catalogs-oci
	path: oci.csv
	retention-days: 1

	catalog-runpod:
	name: Collect Runpod catalog
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: 3.11
	- name: Install dependencies
	run: \|
	pip install pip -U
	pip install -e '.'
	- name: Collect catalog
	working-directory: src
	run: python -m gpuhunt runpod --output ../runpod.csv
	- uses: actions/upload-artifact@v4
	with:
	name: catalogs-runpod
	path: runpod.csv
	retention-days: 1

	test-catalog:
	name: Test catalogs integrity
	needs:
	- catalog-aws
	- catalog-azure
	- catalog-cudo
	- catalog-datacrunch
	- catalog-gcp
	- catalog-lambdalabs
	- catalog-oci
	- catalog-runpod
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: 3.11
	- name: Install dependencies
	run: \|
	pip install pip -U
	pip install '.[all]'
	pip install pytest
	- uses: actions/download-artifact@v4
	with:
	pattern: catalogs-*
	merge-multiple: true
	- name: Run integrity tests
	env:
	CATALOG_DIR: .
	run: pytest src/integrity_tests

	publish-catalog:
	name: Publish catalogs
	needs: [ test-catalog ]
	runs-on: ubuntu-latest
	steps:
	- name: Install AWS CLI
	run: pip install awscli
	- uses: actions/download-artifact@v4
	with:
	pattern: catalogs-*
	merge-multiple: true
	- name: Write version
	run: date +%Y%m%d > version
	- name: Package catalogs
	run: zip catalog.zip *.csv version
	- name: Set channel
	run: \|
	if [[ ${{ github.event_name == 'workflow_dispatch' }} == true ]]; then
	CHANNEL=${{ inputs.channel }}
	else
	CHANNEL=${{ vars.CHANNEL }}
	fi
	echo "CHANNEL=$CHANNEL" >> $GITHUB_ENV
	- name: Upload to S3
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	run: \|
	VERSION=$(cat version)
	aws s3 cp catalog.zip "s3://dstack-gpu-pricing/$CHANNEL/$VERSION/catalog.zip" --acl public-read
	cat version \| aws s3 cp - "s3://dstack-gpu-pricing/$CHANNEL/version" --acl public-read
	aws s3 cp "s3://dstack-gpu-pricing/$CHANNEL/$VERSION/catalog.zip" "s3://dstack-gpu-pricing/$CHANNEL/latest/catalog.zip" --acl public-read

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Collect and publish catalogs #63

Workflow file

Collect and publish catalogs #63

Jobs

Run details

Workflow file for this run