diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 1897f33801597b..4e556e636158de 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -10,7 +10,6 @@ on: branches: - master paths-ignore: - - "docker/**" - "docs/**" - "**.md" release: diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml index 2fe82c7062015b..3da2400cddc924 100644 --- a/.github/workflows/check-datahub-jars.yml +++ b/.github/workflows/check-datahub-jars.yml @@ -5,6 +5,7 @@ on: branches: - master paths-ignore: + - "docker/**" - "docs/**" - "**.md" pull_request: diff --git a/.github/workflows/docker-feast-source.yml b/.github/workflows/docker-feast-source.yml index 865d2f5cf92259..a6526c4e447887 100644 --- a/.github/workflows/docker-feast-source.yml +++ b/.github/workflows/docker-feast-source.yml @@ -3,21 +3,22 @@ on: push: branches: - master - paths-ignore: - - 'docs/**' - - '**.md' + paths: + - 'metadata-ingestion/src/datahub/ingestion/source/feast_image/**' + - '.github/workflows/docker-feast-source.yml' pull_request: branches: - master paths: - 'metadata-ingestion/src/datahub/ingestion/source/feast_image/**' - '.github/workflows/docker-feast-source.yml' - paths_ignore: - - '**.md' - - '**.env' release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: setup: runs-on: ubuntu-latest diff --git a/.github/workflows/docker-ingestion.yml b/.github/workflows/docker-ingestion.yml index d78ba2f9b11f05..752bcc606a3232 100644 --- a/.github/workflows/docker-ingestion.yml +++ b/.github/workflows/docker-ingestion.yml @@ -10,14 +10,17 @@ on: branches: - master paths: - - "docker/**" + - "metadata-ingestion/**" + - "metadata-models/**" + - "docker/datahub-ingestion/**" - ".github/workflows/docker-ingestion.yml" - paths_ignore: - - "**.md" - - "**.env" release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: setup: runs-on: ubuntu-latest diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml index 41bb76adb2ea37..e1c775634bcaae 100644 --- a/.github/workflows/docker-postgres-setup.yml +++ b/.github/workflows/docker-postgres-setup.yml @@ -3,9 +3,9 @@ on: push: branches: - master - paths-ignore: - - 'docs/**' - - '**.md' + paths: + - './docker/postgres-setup/**' + - '.github/workflows/docker-postgres-setup.yml' pull_request: branches: - master @@ -15,6 +15,10 @@ on: release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: setup: runs-on: ubuntu-latest diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index facf3c61aa8328..5d479b0d627217 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -663,6 +663,23 @@ jobs: run: | echo "$DATAHUB_VERSION" ./smoke-test/smoke.sh + - name: store logs + if: failure() + run: | + docker ps -a + docker logs datahub-gms >& gms.log + - name: Upload logs + uses: actions/upload-artifact@v2 + if: failure() + with: + name: docker logs + path: "*.log" + - name: Upload screenshots + uses: actions/upload-artifact@v2 + if: failure() + with: + name: cypress-snapshots + path: smoke-test/tests/cypress/cypress/screenshots/ - uses: actions/upload-artifact@v2 if: always() with: diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index 3e5607e80d66eb..bf3b7d4695580c 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -70,6 +70,9 @@ jobs: run: ./metadata-ingestion/scripts/install_deps.sh - name: Run metadata-ingestion tests run: ./gradlew :metadata-ingestion:build :metadata-ingestion:${{ matrix.command }} -x:metadata-ingestion:testQuick -x:metadata-ingestion:check + - name: pip freeze show list installed + if: always() + run: source metadata-ingestion/venv/bin/activate && pip freeze - uses: actions/upload-artifact@v2 if: always() with: diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index 6f936ae7cdc2d6..807e14319d0a88 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -19,6 +19,10 @@ on: release: types: [published] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml index 06f9145645477c..ad3a774462ac40 100644 --- a/.github/workflows/metadata-model.yml +++ b/.github/workflows/metadata-model.yml @@ -9,6 +9,10 @@ on: release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: setup: runs-on: ubuntu-latest diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml index 0fb2f99f9dcde9..a2253d56c43681 100644 --- a/.github/workflows/publish-datahub-jars.yml +++ b/.github/workflows/publish-datahub-jars.yml @@ -12,6 +12,11 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check-secret: diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml index cdafb132c82459..327667f7461d93 100644 --- a/.github/workflows/spark-smoke-test.yml +++ b/.github/workflows/spark-smoke-test.yml @@ -19,6 +19,10 @@ on: release: types: [published, edited] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: spark-smoke-test: runs-on: ubuntu-latest diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml index 8c5fd123ee257b..c67b3afc9b1d90 100644 --- a/docker/docker-compose-with-cassandra.yml +++ b/docker/docker-compose-with-cassandra.yml @@ -33,7 +33,7 @@ services: kafka-setup: build: context: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} env_file: kafka-setup/env/docker.env hostname: kafka-setup container_name: kafka-setup @@ -85,7 +85,7 @@ services: build: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} env_file: elasticsearch-setup/env/docker.env hostname: elasticsearch-setup container_name: elasticsearch-setup @@ -120,7 +120,7 @@ services: build: context: ../ dockerfile: docker/datahub-gms/Dockerfile - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} env_file: ./datahub-gms/env/docker.cassandra.env hostname: datahub-gms container_name: datahub-gms @@ -136,7 +136,7 @@ services: build: context: ../ dockerfile: docker/datahub-frontend/Dockerfile - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} env_file: datahub-frontend/env/docker.env hostname: datahub-frontend-react container_name: datahub-frontend-react diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml index 481dee2be8ec79..d349d493639d05 100644 --- a/docker/docker-compose-without-neo4j.yml +++ b/docker/docker-compose-without-neo4j.yml @@ -31,7 +31,7 @@ services: kafka-setup: build: context: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} env_file: kafka-setup/env/docker.env hostname: kafka-setup container_name: kafka-setup @@ -72,7 +72,7 @@ services: build: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} env_file: elasticsearch-setup/env/docker.env hostname: elasticsearch-setup container_name: elasticsearch-setup @@ -83,7 +83,7 @@ services: build: context: ../ dockerfile: docker/datahub-gms/Dockerfile - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} env_file: datahub-gms/env/docker-without-neo4j.env hostname: datahub-gms container_name: datahub-gms @@ -98,7 +98,7 @@ services: build: context: ../ dockerfile: docker/datahub-frontend/Dockerfile - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} env_file: datahub-frontend/env/docker.env hostname: datahub-frontend-react container_name: datahub-frontend-react diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 402916fd717ccd..455b2a62cc7c9e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -32,7 +32,7 @@ services: kafka-setup: build: context: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} env_file: kafka-setup/env/docker.env hostname: kafka-setup container_name: kafka-setup @@ -84,7 +84,7 @@ services: build: context: ../ dockerfile: docker/elasticsearch-setup/Dockerfile - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} env_file: elasticsearch-setup/env/docker.env hostname: elasticsearch-setup container_name: elasticsearch-setup @@ -95,7 +95,7 @@ services: build: context: ../ dockerfile: docker/datahub-gms/Dockerfile - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} hostname: datahub-gms container_name: datahub-gms ports: @@ -110,7 +110,7 @@ services: build: context: ../ dockerfile: docker/datahub-frontend/Dockerfile - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} env_file: datahub-frontend/env/docker.env hostname: datahub-frontend-react container_name: datahub-frontend-react diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index 80928f2f1118b7..40dad574fff373 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -52,7 +52,7 @@ services: - ELASTIC_CLIENT_HOST=elasticsearch - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: @@ -81,7 +81,7 @@ services: - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - PE_CONSUMER_ENABLED=true hostname: datahub-gms - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 volumes: @@ -115,7 +115,7 @@ services: - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} kafka-setup: container_name: kafka-setup depends_on: @@ -125,7 +125,7 @@ services: - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_BOOTSTRAP_SERVER=broker:29092 hostname: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} mysql: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 8b7679f8da4a4b..cb3ded5b3ff630 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -53,7 +53,7 @@ services: - ELASTIC_CLIENT_HOST=elasticsearch - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - 9002:9002 volumes: @@ -84,7 +84,7 @@ services: - UI_INGESTION_ENABLED=true - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.41 hostname: datahub-gms - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - 8080:8080 volumes: @@ -118,7 +118,7 @@ services: - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} kafka-setup: container_name: kafka-setup depends_on: @@ -128,7 +128,7 @@ services: - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_BOOTSTRAP_SERVER=broker:29092 hostname: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} mysql: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin container_name: mysql diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 291dacb7c45043..78aad9227ac956 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -54,7 +54,7 @@ services: - ELASTIC_CLIENT_HOST=elasticsearch - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react - image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head} + image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: @@ -89,7 +89,7 @@ services: - UI_INGESTION_ENABLED=true - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.41 hostname: datahub-gms - image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} + image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 volumes: @@ -123,7 +123,7 @@ services: - ELASTICSEARCH_PORT=9200 - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup - image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} kafka-setup: container_name: kafka-setup depends_on: @@ -133,7 +133,7 @@ services: - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_BOOTSTRAP_SERVER=broker:29092 hostname: kafka-setup - image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head} + image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} mysql: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin container_name: mysql