From 603247f735e1bbc831a18cfef27255d744cb8e84 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 3 Apr 2024 10:33:31 +0200 Subject: [PATCH] GH-40954: [CI] Fix use of obsolete docker-compose command on Github Actions (#40949) ### Rationale for this change The `docker-compose` utility is progressively being removed from GHA-provided runners: https://github.com/actions/runner-images/issues/9557 ### What changes are included in this PR? Use `docker` client CLI directly instead of `docker-compose` where possible. ### Are these changes tested? Yes, this should fix the sporadic CI failures because of the above migration. ### Are there any user-facing changes? No, except additional optional env var `ARCHERY_DEBUG`. * GitHub Issue: #40954 Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- .github/workflows/archery.yml | 2 + .github/workflows/cpp.yml | 5 ++ .github/workflows/dev.yml | 4 + .github/workflows/docs.yml | 2 + .github/workflows/docs_light.yml | 2 + .github/workflows/go.yml | 9 +++ .github/workflows/integration.yml | 2 + .github/workflows/java.yml | 2 + .github/workflows/java_jni.yml | 2 + .github/workflows/js.yml | 4 + .github/workflows/python.yml | 2 + .github/workflows/r.yml | 2 + .github/workflows/ruby.yml | 2 + .github/workflows/swift.yml | 2 + dev/archery/archery/cli.py | 1 + dev/archery/archery/docker/cli.py | 79 ++++++------------- dev/archery/archery/docker/core.py | 50 +++++++----- .../archery/docker/tests/test_docker_cli.py | 19 +---- dev/tasks/java-jars/github.yml | 2 + dev/tasks/linux-packages/github.linux.yml | 1 + dev/tasks/macros.jinja | 4 + dev/tasks/python-wheels/github.linux.yml | 2 + dev/tasks/tasks.yml | 4 + 23 files changed, 117 insertions(+), 87 deletions(-) diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml index cb783dd66c3fb..2aa6b0dcbb91d 100644 --- a/.github/workflows/archery.yml +++ b/.github/workflows/archery.yml @@ -32,7 +32,9 @@ on: - 'docker-compose.yml' env: + ARCHERY_DEBUG: 1 ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + ARCHERY_USE_DOCKER_CLI: 1 concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index e8e41f1bcb90c..1d10be3b5bc82 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -53,6 +53,7 @@ permissions: contents: read env: + ARCHERY_DEBUG: 1 ARROW_ENABLE_TIMING_TESTS: OFF DOCKER_VOLUME_PREFIX: ".docker/" @@ -94,6 +95,7 @@ jobs: cat <> "$GITHUB_OUTPUT" { "arch": "arm64v8", + "archery-use-docker-cli": "0", "clang-tools": "10", "image": "ubuntu-cpp", "llvm": "10", @@ -118,6 +120,9 @@ jobs: include: ${{ fromJson(needs.docker-targets.outputs.targets) }} env: ARCH: ${{ matrix.arch }} + # By default, use Docker CLI because docker-compose v1 is obsolete, + # except where the Docker client version is too old. + ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }} ARROW_SIMD_LEVEL: ${{ matrix.simd-level }} CLANG_TOOLS: ${{ matrix.clang-tools }} LLVM: ${{ matrix.llvm }} diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 37fda2e313ae2..8af5832f15948 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -29,6 +29,10 @@ concurrency: permissions: contents: read +env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 + jobs: lint: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9c7701f25f756..fe49e275d908d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,6 +24,8 @@ permissions: contents: read env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 ARROW_ENABLE_TIMING_TESTS: OFF DOCKER_VOLUME_PREFIX: ".docker/" diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml index 6ec4c3d53d0e3..376c87651d2d0 100644 --- a/.github/workflows/docs_light.yml +++ b/.github/workflows/docs_light.yml @@ -33,6 +33,8 @@ permissions: contents: read env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 ARROW_ENABLE_TIMING_TESTS: OFF DOCKER_VOLUME_PREFIX: ".docker/" diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 7fca38528260f..11dc29dcae54e 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -41,6 +41,10 @@ concurrency: permissions: contents: read +env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 + jobs: docker-targets: @@ -75,12 +79,14 @@ jobs: { "arch-label": "ARM64", "arch": "arm64v8", + "archery-use-docker-cli": "0", "go": "1.21", "runs-on": ["self-hosted", "arm", "linux"] }, { "arch-label": "ARM64", "arch": "arm64v8", + "archery-use-docker-cli": "0", "go": "1.22", "runs-on": ["self-hosted", "arm", "linux"] } @@ -101,6 +107,9 @@ jobs: include: ${{ fromJson(needs.docker-targets.outputs.targets) }} env: ARCH: ${{ matrix.arch }} + # By default, use Docker CLI because docker-compose v1 is obsolete, + # except where the Docker client version is too old. + ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }} GO: ${{ matrix.go }} steps: - name: Checkout Arrow diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0f186ff6a4527..2c3499c160f9c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -51,6 +51,8 @@ permissions: contents: read env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 423f54cd93547..611e202ca0624 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -45,6 +45,8 @@ permissions: contents: read env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index 790ffd5c650e0..958216ac7669d 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -45,6 +45,8 @@ permissions: contents: read env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index dab89da44c861..c9b7d7b742d88 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -38,6 +38,10 @@ concurrency: permissions: contents: read +env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 + jobs: docker: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 1147ac13e6f93..2db9b17e895b0 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -41,6 +41,8 @@ permissions: contents: read env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 78677499f3e45..05c85fa6dc2c2 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -51,6 +51,8 @@ permissions: contents: read env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 311c1c822baf6..ea3e61d55787d 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -53,6 +53,8 @@ permissions: contents: read env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index f55e9e77503c0..3f039315b505a 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -41,6 +41,8 @@ permissions: contents: read env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index 32921afb2e61b..5fa41e28a3208 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -44,6 +44,7 @@ @click.group(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("--debug", type=BOOL, is_flag=True, default=False, + envvar='ARCHERY_DEBUG', help="Increase logging with debugging output.") @click.option("--pdb", type=BOOL, is_flag=True, default=False, help="Invoke pdb on uncaught exception.") diff --git a/dev/archery/archery/docker/cli.py b/dev/archery/archery/docker/cli.py index 20d9a16138bac..e6baf0ca1f002 100644 --- a/dev/archery/archery/docker/cli.py +++ b/dev/archery/archery/docker/cli.py @@ -46,10 +46,19 @@ def _execute(self, *args, **kwargs): callback=validate_arrow_sources, help="Specify Arrow source directory.") @click.option('--dry-run/--execute', default=False, - help="Display the docker-compose commands instead of executing " - "them.") + help="Display the docker commands instead of executing them.") +@click.option('--using-docker-cli', default=False, is_flag=True, + envvar='ARCHERY_USE_DOCKER_CLI', + help="Use docker CLI directly for building instead of calling " + "docker-compose. This may help to reuse cached layers.") +@click.option('--using-docker-buildx', default=False, is_flag=True, + envvar='ARCHERY_USE_DOCKER_BUILDX', + help="Use buildx with docker CLI directly for building instead " + "of calling docker-compose or the plain docker build " + "command. This option makes the build cache reusable " + "across hosts.") @click.pass_context -def docker(ctx, src, dry_run): +def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx): """ Interact with docker-compose based builds. """ @@ -64,7 +73,10 @@ def docker(ctx, src, dry_run): # take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the # environment variables to keep the usage similar to docker-compose + using_docker_cli |= using_docker_buildx compose = DockerCompose(config_path, params=os.environ, + using_docker=using_docker_cli, + using_buildx=using_docker_buildx, debug=ctx.obj.get('debug', False)) if dry_run: _mock_compose_calls(compose) @@ -83,24 +95,19 @@ def check_config(obj): @docker.command('pull') @click.argument('image') -@click.option('--using-docker-cli', default=False, is_flag=True, - envvar='ARCHERY_USE_DOCKER_CLI', - help="Use docker CLI directly for pulling instead of calling " - "docker-compose. This may help to reuse cached layers.") @click.option('--pull-leaf/--no-leaf', default=True, help="Whether to pull leaf images too.") @click.option('--ignore-pull-failures/--no-ignore-pull-failures', default=True, help="Whether to ignore pull failures.") @click.pass_obj -def docker_pull(obj, image, *, using_docker_cli, pull_leaf, - ignore_pull_failures): +def docker_pull(obj, image, *, pull_leaf, ignore_pull_failures): """ Execute docker-compose pull. """ compose = obj['compose'] try: - compose.pull(image, pull_leaf=pull_leaf, using_docker=using_docker_cli, + compose.pull(image, pull_leaf=pull_leaf, ignore_pull_failures=ignore_pull_failures) except UndefinedImage as e: raise click.ClickException( @@ -115,16 +122,6 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf, @click.argument('image') @click.option('--force-pull/--no-pull', default=True, help="Whether to force pull the image and its ancestor images") -@click.option('--using-docker-cli', default=False, is_flag=True, - envvar='ARCHERY_USE_DOCKER_CLI', - help="Use docker CLI directly for building instead of calling " - "docker-compose. This may help to reuse cached layers.") -@click.option('--using-docker-buildx', default=False, is_flag=True, - envvar='ARCHERY_USE_DOCKER_BUILDX', - help="Use buildx with docker CLI directly for building instead " - "of calling docker-compose or the plain docker build " - "command. This option makes the build cache reusable " - "across hosts.") @click.option('--use-cache/--no-cache', default=True, help="Whether to use cache when building the image and its " "ancestor images") @@ -133,22 +130,17 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf, "passed as the argument. To disable caching for both the " "image and its ancestors use --no-cache option.") @click.pass_obj -def docker_build(obj, image, *, force_pull, using_docker_cli, - using_docker_buildx, use_cache, use_leaf_cache): +def docker_build(obj, image, *, force_pull, use_cache, use_leaf_cache): """ Execute docker-compose builds. """ compose = obj['compose'] - using_docker_cli |= using_docker_buildx try: if force_pull: - compose.pull(image, pull_leaf=use_leaf_cache, - using_docker=using_docker_cli) + compose.pull(image, pull_leaf=use_leaf_cache) compose.build(image, use_cache=use_cache, use_leaf_cache=use_leaf_cache, - using_docker=using_docker_cli, - using_buildx=using_docker_buildx, pull_parents=force_pull) except UndefinedImage as e: raise click.ClickException( @@ -172,16 +164,6 @@ def docker_build(obj, image, *, force_pull, using_docker_cli, help="Whether to force build the image and its ancestor images") @click.option('--build-only', default=False, is_flag=True, help="Pull and/or build the image, but do not run it") -@click.option('--using-docker-cli', default=False, is_flag=True, - envvar='ARCHERY_USE_DOCKER_CLI', - help="Use docker CLI directly for building instead of calling " - "docker-compose. This may help to reuse cached layers.") -@click.option('--using-docker-buildx', default=False, is_flag=True, - envvar='ARCHERY_USE_DOCKER_BUILDX', - help="Use buildx with docker CLI directly for building instead " - "of calling docker-compose or the plain docker build " - "command. This option makes the build cache reusable " - "across hosts.") @click.option('--use-cache/--no-cache', default=True, help="Whether to use cache when building the image and its " "ancestor images") @@ -191,7 +173,7 @@ def docker_build(obj, image, *, force_pull, using_docker_cli, "image and its ancestors use --no-cache option.") @click.option('--resource-limit', default=None, help="A CPU/memory limit preset to mimic CI environments like " - "GitHub Actions. Implies --using-docker-cli. Note that " + "GitHub Actions. Mandates --using-docker-cli. Note that " "exporting ARCHERY_DOCKER_BIN=\"sudo docker\" is likely " "required, unless Docker is configured with cgroups v2 " "(else Docker will silently ignore the limits).") @@ -199,8 +181,8 @@ def docker_build(obj, image, *, force_pull, using_docker_cli, help="Set volume within the container") @click.pass_obj def docker_run(obj, image, command, *, env, user, force_pull, force_build, - build_only, using_docker_cli, using_docker_buildx, use_cache, - use_leaf_cache, resource_limit, volume): + build_only, use_cache, use_leaf_cache, resource_limit, + volume): """ Execute docker-compose builds. @@ -234,18 +216,14 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build, archery docker run ubuntu-cpp bash """ compose = obj['compose'] - using_docker_cli |= using_docker_buildx env = dict(kv.split('=', 1) for kv in env) try: if force_pull: - compose.pull(image, pull_leaf=use_leaf_cache, - using_docker=using_docker_cli) + compose.pull(image, pull_leaf=use_leaf_cache) if force_build: compose.build(image, use_cache=use_cache, - use_leaf_cache=use_leaf_cache, - using_docker=using_docker_cli, - using_buildx=using_docker_buildx) + use_leaf_cache=use_leaf_cache) if build_only: return compose.run( @@ -253,7 +231,6 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build, command=command, env=env, user=user, - using_docker=using_docker_cli, resource_limit=resource_limit, volumes=volume ) @@ -273,15 +250,11 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build, @click.option('--password', '-p', required=False, envvar='ARCHERY_DOCKER_PASSWORD', help='Docker repository password') -@click.option('--using-docker-cli', default=False, is_flag=True, - help="Use docker CLI directly for building instead of calling " - "docker-compose. This may help to reuse cached layers.") @click.pass_obj -def docker_compose_push(obj, image, user, password, using_docker_cli): +def docker_compose_push(obj, image, user, password): """Push the generated docker-compose image.""" compose = obj['compose'] - compose.push(image, user=user, password=password, - using_docker=using_docker_cli) + compose.push(image, user=user, password=password) @docker.command('images') diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py index 184d9808759b8..38720e5856a14 100644 --- a/dev/archery/archery/docker/core.py +++ b/dev/archery/archery/docker/core.py @@ -58,12 +58,21 @@ class UndefinedImage(Exception): class ComposeConfig: - def __init__(self, config_path, dotenv_path, compose_bin, params=None): + def __init__(self, config_path, dotenv_path, compose_bin, + using_docker=False, using_buildx=False, + params=None, debug=False): + self.using_docker = using_docker + self.using_buildx = using_buildx + self.debug = debug config_path = _ensure_path(config_path) if dotenv_path: dotenv_path = _ensure_path(dotenv_path) else: dotenv_path = config_path.parent / '.env' + if self.debug: + # Log docker version + Docker().run('version') + self._read_env(dotenv_path, params) self._read_config(config_path, compose_bin) @@ -122,8 +131,13 @@ def _read_config(self, config_path, compose_bin): ) # trigger docker-compose's own validation - compose = Command('docker-compose') - args = ['--file', str(config_path), 'config'] + if self.using_docker: + compose = Docker() + args = ['compose'] + else: + compose = Command('docker-compose') + args = [] + args += ['--file', str(config_path), 'config'] result = compose.run(*args, env=self.env, check=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) @@ -164,12 +178,13 @@ def __init__(self, docker_bin=None): class DockerCompose(Command): def __init__(self, config_path, dotenv_path=None, compose_bin=None, - params=None, debug=False): + using_docker=False, using_buildx=False, params=None, + debug=False): compose_bin = default_bin(compose_bin, 'docker-compose') self.config = ComposeConfig(config_path, dotenv_path, compose_bin, - params) + params=params, using_docker=using_docker, + using_buildx=using_buildx, debug=debug) self.bin = compose_bin - self.debug = debug self.pull_memory = set() def clear_pull_memory(self): @@ -215,14 +230,13 @@ def _execute_docker(self, *args, **kwargs): ) ) - def pull(self, service_name, pull_leaf=True, using_docker=False, - ignore_pull_failures=True): + def pull(self, service_name, pull_leaf=True, ignore_pull_failures=True): def _pull(service): args = ['pull'] if service['image'] in self.pull_memory: return - if using_docker: + if self.config.using_docker: try: self._execute_docker(*args, service['image']) except Exception as e: @@ -245,7 +259,7 @@ def _pull(service): _pull(service) def build(self, service_name, use_cache=True, use_leaf_cache=True, - using_docker=False, using_buildx=False, pull_parents=True): + pull_parents=True): def _build(service, use_cache): if 'build' not in service: # nothing to do @@ -273,7 +287,7 @@ def _build(service, use_cache): if self.config.env.get('BUILDKIT_INLINE_CACHE') == '1': args.extend(['--build-arg', 'BUILDKIT_INLINE_CACHE=1']) - if using_buildx: + if self.config.using_buildx: for k, v in service['build'].get('args', {}).items(): args.extend(['--build-arg', '{}={}'.format(k, v)]) @@ -295,9 +309,9 @@ def _build(service, use_cache): service['build'].get('context', '.') ]) self._execute_docker("buildx", "build", *args) - elif using_docker: + elif self.config.using_docker: # better for caching - if self.debug: + if self.config.debug: args.append("--progress=plain") for k, v in service['build'].get('args', {}).items(): args.extend(['--build-arg', '{}={}'.format(k, v)]) @@ -310,7 +324,7 @@ def _build(service, use_cache): ]) self._execute_docker("build", *args) else: - if self.debug: + if self.config.debug: args.append("--progress=plain") self._execute_compose("build", *args, service['name']) @@ -322,7 +336,7 @@ def _build(service, use_cache): _build(service, use_cache=use_cache and use_leaf_cache) def run(self, service_name, command=None, *, env=None, volumes=None, - user=None, using_docker=False, resource_limit=None): + user=None, resource_limit=None): service = self.config.get(service_name) args = [] @@ -337,7 +351,7 @@ def run(self, service_name, command=None, *, env=None, volumes=None, for volume in volumes: args.extend(['--volume', volume]) - if using_docker or service['need_gpu'] or resource_limit: + if self.config.using_docker or service['need_gpu'] or resource_limit: # use gpus, requires docker>=19.03 if service['need_gpu']: args.extend(['--gpus', 'all']) @@ -399,9 +413,9 @@ def run(self, service_name, command=None, *, env=None, volumes=None, args.append(command) self._execute_compose('run', '--rm', *args) - def push(self, service_name, user=None, password=None, using_docker=False): + def push(self, service_name, user=None, password=None): def _push(service): - if using_docker: + if self.config.using_docker: return self._execute_docker('push', service['image']) else: return self._execute_compose('push', service['name']) diff --git a/dev/archery/archery/docker/tests/test_docker_cli.py b/dev/archery/archery/docker/tests/test_docker_cli.py index ab39c7b9dbb4a..c117a3edfff65 100644 --- a/dev/archery/archery/docker/tests/test_docker_cli.py +++ b/dev/archery/archery/docker/tests/test_docker_cli.py @@ -33,14 +33,12 @@ def test_docker_run_with_custom_command(run, build, pull): assert result.exit_code == 0 pull.assert_called_once_with( - "ubuntu-cpp", pull_leaf=True, using_docker=False + "ubuntu-cpp", pull_leaf=True, ) build.assert_called_once_with( "ubuntu-cpp", use_cache=True, use_leaf_cache=True, - using_docker=False, - using_buildx=False ) run.assert_called_once_with( "ubuntu-cpp", @@ -48,7 +46,6 @@ def test_docker_run_with_custom_command(run, build, pull): env={}, resource_limit=None, user=None, - using_docker=False, volumes=(), ) @@ -75,14 +72,12 @@ def test_docker_run_options(run, build, pull): result = CliRunner().invoke(docker, args) assert result.exit_code == 0 pull.assert_called_once_with( - "ubuntu-cpp", pull_leaf=True, using_docker=False + "ubuntu-cpp", pull_leaf=True, ) build.assert_called_once_with( "ubuntu-cpp", use_cache=True, use_leaf_cache=True, - using_docker=False, - using_buildx=False ) run.assert_called_once_with( "ubuntu-cpp", @@ -90,7 +85,6 @@ def test_docker_run_options(run, build, pull): env={"ARROW_GANDIVA": "OFF", "ARROW_FLIGHT": "ON"}, resource_limit=None, user="root", - using_docker=False, volumes=( "./build:/build", "./ccache:/ccache:delegated", @@ -126,7 +120,6 @@ def test_docker_limit_options(run): env={"ARROW_GANDIVA": "OFF", "ARROW_FLIGHT": "ON"}, resource_limit="github", user="root", - using_docker=False, volumes=( "./build:/build", "./ccache:/ccache:delegated", @@ -145,7 +138,6 @@ def test_docker_run_without_pulling_or_building(run): env={}, resource_limit=None, user=None, - using_docker=False, volumes=(), ) @@ -157,14 +149,12 @@ def test_docker_run_only_pulling_and_building(build, pull): result = CliRunner().invoke(docker, args) assert result.exit_code == 0 pull.assert_called_once_with( - "ubuntu-cpp", pull_leaf=True, using_docker=False + "ubuntu-cpp", pull_leaf=True, ) build.assert_called_once_with( "ubuntu-cpp", use_cache=True, use_leaf_cache=True, - using_docker=False, - using_buildx=False ) @@ -187,8 +177,6 @@ def test_docker_run_without_build_cache(run, build): "ubuntu-cpp", use_cache=False, use_leaf_cache=False, - using_docker=False, - using_buildx=False ) run.assert_called_once_with( "ubuntu-cpp", @@ -196,6 +184,5 @@ def test_docker_run_without_build_cache(run, build): env={}, resource_limit=None, user="me", - using_docker=False, volumes=(), ) diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 03cbcc7c98fcc..0437ee7864979 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -30,6 +30,7 @@ jobs: ARCH: {{ '${{ matrix.platform.archery_arch }}' }} ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }} ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }} + ARCHERY_USE_DOCKER_CLI: {{ "${{matrix.platform.archery_use_docker_cli || '1'}}" }} strategy: fail-fast: false matrix: @@ -44,6 +45,7 @@ jobs: archery_arch: "arm64v8" archery_arch_alias: "aarch64" archery_arch_short: "arm64" + archery_use_docker_cli: "0" steps: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_free_space()|indent }} diff --git a/dev/tasks/linux-packages/github.linux.yml b/dev/tasks/linux-packages/github.linux.yml index 6de3edfce07e1..9e24835b8b627 100644 --- a/dev/tasks/linux-packages/github.linux.yml +++ b/dev/tasks/linux-packages/github.linux.yml @@ -29,6 +29,7 @@ jobs: {% endif %} env: ARCHITECTURE: {{ architecture }} + ARCHERY_USE_DOCKER_CLI: {{ '0' if architecture == 'arm64' else '1' }} steps: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_login_dockerhub()|indent }} diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index bcafe53066ef8..f55a7f9481e56 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -23,6 +23,10 @@ on: push: branches: - "*-github-*" + +env: + ARCHERY_DEBUG: 1 + ARCHERY_USE_DOCKER_CLI: 1 {% endmacro %} {%- macro github_checkout_arrow(fetch_depth=1, submodules="recursive", action_v="4") -%} diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index 41b18684cee10..0ff3c56b695eb 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -31,8 +31,10 @@ jobs: # archery uses these environment variables {% if arch == "amd64" %} ARCH: amd64 + ARCHERY_USE_DOCKER_CLI: 1 {% else %} ARCH: arm64v8 + ARCHERY_USE_DOCKER_CLI: 0 {% endif %} PYTHON: "{{ python_version }}" diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 5e1ef8d13b988..cf46cb8c6ad70 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1456,12 +1456,16 @@ tasks: ci: github template: docker-tests/github.cuda.yml params: + env: + ARCHERY_USE_DOCKER_CLI: 0 image: ubuntu-cuda-cpp test-cuda-python: ci: github template: docker-tests/github.cuda.yml params: + env: + ARCHERY_USE_DOCKER_CLI: 0 image: ubuntu-cuda-python ############################## Fuzz tests #################################