From ff50596c06c3bec91ac21a014ffe823cc53e4de4 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 28 Jan 2020 16:16:25 +0100 Subject: [PATCH] [AIRFLOW-6662] Switch to --init docker flag for signal propagation We are now using native --init flag of docker run and init: parameter of docker compose to pass signals and reap child processes --- Dockerfile | 2 +- scripts/ci/_utils.sh | 55 ++++++++++++------- scripts/ci/docker-compose/backend-mysql.yml | 1 + .../ci/docker-compose/backend-postgres.yml | 1 + .../docker-compose/integration-cassandra.yml | 1 + .../docker-compose/integration-kerberos.yml | 1 + .../ci/docker-compose/integration-mongo.yml | 1 + .../docker-compose/integration-openldap.yml | 1 + .../docker-compose/integration-rabbitmq.yml | 1 + .../ci/docker-compose/integration-redis.yml | 1 + setup.py | 1 - 11 files changed, 44 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 480f913877093..7195d378263c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -405,6 +405,6 @@ ENV PATH="${HOME}:${PATH}" EXPOSE 8080 -ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/entrypoint.sh"] +ENTRYPOINT ["/entrypoint.sh"] CMD ["--help"] diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index 592297b11dfe7..3441aef046cf2 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -640,27 +640,30 @@ function run_flake8() { if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint "/usr/local/bin/dumb-init" \ + --init \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --entrypoint \ + "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint "/usr/local/bin/dumb-init" \ + --init \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ + --entrypoint \ + "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \ + "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } @@ -679,29 +682,31 @@ function run_bats_tests() { function run_docs() { docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ - --entrypoint "/usr/local/bin/dumb-init" \ + --init \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --entrypoint \ + "/opt/airflow/docs/build.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/docs/build.sh" \ | tee -a "${OUTPUT_LOG}" } function run_check_license() { docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ - --entrypoint "/usr/local/bin/dumb-init" \ + --init \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --entrypoint \ + "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \ | tee -a "${OUTPUT_LOG}" } @@ -709,27 +714,31 @@ function run_mypy() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint "/usr/local/bin/dumb-init" \ + --init \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ + --entrypoint \ + "/opt/airflow/scripts/ci/in_container/run_mypy.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "airflow" "tests" "docs" \ + "airflow" "tests" "docs" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint "/usr/local/bin/dumb-init" \ + --init \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --entrypoint \ + "/opt/airflow/scripts/ci/in_container/run_mypy.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "${FILES[@]}" \ + "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } @@ -738,27 +747,30 @@ function run_pylint_main() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint "/usr/local/bin/dumb-init" \ + --init \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ + --entrypoint \ + "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint "/usr/local/bin/dumb-init" \ + --init \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ + --entrypoint \ + "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" "${FILES[@]}" \ + "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } @@ -768,27 +780,30 @@ function run_pylint_tests() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint "/usr/local/bin/dumb-init" \ + --init \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ + --entrypoint \ + "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint "/usr/local/bin/dumb-init" \ + --init \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ + --entrypoint \ + "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" "${FILES[@]}" \ + "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } diff --git a/scripts/ci/docker-compose/backend-mysql.yml b/scripts/ci/docker-compose/backend-mysql.yml index d9773aac1a97b..d2bfd8dad9e49 100644 --- a/scripts/ci/docker-compose/backend-mysql.yml +++ b/scripts/ci/docker-compose/backend-mysql.yml @@ -25,6 +25,7 @@ services: depends_on: - mysql mysql: + init: true image: mysql:5.6 environment: - MYSQL_ALLOW_EMPTY_PASSWORD=true diff --git a/scripts/ci/docker-compose/backend-postgres.yml b/scripts/ci/docker-compose/backend-postgres.yml index 10a6fd5227bbb..9cfdc7b54b44f 100644 --- a/scripts/ci/docker-compose/backend-postgres.yml +++ b/scripts/ci/docker-compose/backend-postgres.yml @@ -25,6 +25,7 @@ services: depends_on: - postgres postgres: + init: true image: postgres:9.6 environment: - POSTGRES_USER=postgres diff --git a/scripts/ci/docker-compose/integration-cassandra.yml b/scripts/ci/docker-compose/integration-cassandra.yml index 12a5537156b30..304c36c9d51f8 100644 --- a/scripts/ci/docker-compose/integration-cassandra.yml +++ b/scripts/ci/docker-compose/integration-cassandra.yml @@ -18,6 +18,7 @@ version: "2.2" services: cassandra: + init: true image: cassandra:3.0 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/integration-kerberos.yml b/scripts/ci/docker-compose/integration-kerberos.yml index 6714a104f5819..79d841f21a2fa 100644 --- a/scripts/ci/docker-compose/integration-kerberos.yml +++ b/scripts/ci/docker-compose/integration-kerberos.yml @@ -18,6 +18,7 @@ version: "2.2" services: kerberos: + init: true image: godatadriven/krb5-kdc-server hostname: kerberos domainname: example.com diff --git a/scripts/ci/docker-compose/integration-mongo.yml b/scripts/ci/docker-compose/integration-mongo.yml index 5d48c5022761e..06dfea113b18a 100644 --- a/scripts/ci/docker-compose/integration-mongo.yml +++ b/scripts/ci/docker-compose/integration-mongo.yml @@ -18,6 +18,7 @@ version: "2.2" services: mongo: + init: true image: mongo:3 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/integration-openldap.yml b/scripts/ci/docker-compose/integration-openldap.yml index cf8cfd3019c4b..44e2c2f2de50e 100644 --- a/scripts/ci/docker-compose/integration-openldap.yml +++ b/scripts/ci/docker-compose/integration-openldap.yml @@ -18,6 +18,7 @@ version: "2.2" services: openldap: + init: true image: osixia/openldap:1.2.0 command: --copy-service environment: diff --git a/scripts/ci/docker-compose/integration-rabbitmq.yml b/scripts/ci/docker-compose/integration-rabbitmq.yml index d14db8aaef984..1864122f8d34b 100644 --- a/scripts/ci/docker-compose/integration-rabbitmq.yml +++ b/scripts/ci/docker-compose/integration-rabbitmq.yml @@ -18,6 +18,7 @@ version: "2.2" services: rabbitmq: + init: true image: rabbitmq:3.7 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/integration-redis.yml b/scripts/ci/docker-compose/integration-redis.yml index 6893af7074860..819a35416f99c 100644 --- a/scripts/ci/docker-compose/integration-redis.yml +++ b/scripts/ci/docker-compose/integration-redis.yml @@ -18,6 +18,7 @@ version: "2.2" services: redis: + init: true image: redis:5.0.1 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/setup.py b/setup.py index b9616327376a1..0bd35a73d6f84 100644 --- a/setup.py +++ b/setup.py @@ -376,7 +376,6 @@ def write_version(filename: str = os.path.join(*["airflow", "git_version"])): 'click==6.7', 'contextdecorator;python_version<"3.4"', 'coverage', - 'dumb-init>=1.2.2', 'flake8>=3.6.0', 'flake8-colors', 'flaky',