From de0d818f756d5d9c905ab1b0f2e4f57aa37540a0 Mon Sep 17 00:00:00 2001 From: Irina Truong Date: Tue, 14 Jun 2022 12:50:40 -0700 Subject: [PATCH] Git doesn't respect HTTP_PROXY or http_proxy. Provide -c 'http.proxy' instead. --- esrally/utils/git.py | 51 +++++++++++++++++++++++++------------------- it/proxy_test.py | 7 ++++-- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/esrally/utils/git.py b/esrally/utils/git.py index 0b1389418..16999b038 100644 --- a/esrally/utils/git.py +++ b/esrally/utils/git.py @@ -22,13 +22,20 @@ from esrally.utils import io, process +def git_cmd(): + """Command line to call git with http_proxy if needed""" + if "http_proxy" in os.environ: + return f"git -c 'http.proxy={os.environ['http_proxy']}'" + return "git" + + def probed(f): def probe(src, *args, **kwargs): # Probe for -C if not process.exit_status_as_bool( - lambda: process.run_subprocess_with_logging("git -C {} --version".format(io.escape_path(src)), level=logging.DEBUG), quiet=True + lambda: process.run_subprocess_with_logging(f"{git_cmd()} -C {io.escape_path(src)} --version", level=logging.DEBUG), quiet=True ): - version = process.run_subprocess_with_output("git --version") + version = process.run_subprocess_with_output(f"{git_cmd()} --version") if version: version = str(version).strip() else: @@ -51,27 +58,27 @@ def is_working_copy(src): def clone(src, remote): io.ensure_dir(src) # Don't swallow subprocess output, user might need to enter credentials... - if process.run_subprocess_with_logging("git clone %s %s" % (remote, io.escape_path(src))): - raise exceptions.SupplyError("Could not clone from [%s] to [%s]" % (remote, src)) + if process.run_subprocess_with_logging(f"{git_cmd()} clone {remote} {io.escape_path(src)}"): + raise exceptions.SupplyError(f"Could not clone from [{remote}] to [{src}]") @probed def fetch(src, remote="origin"): - if process.run_subprocess_with_logging("git -C {0} fetch --prune --tags {1}".format(io.escape_path(src), remote)): - raise exceptions.SupplyError("Could not fetch source tree from [%s]" % remote) + if process.run_subprocess_with_logging(f"{git_cmd()} -C {io.escape_path(src)} fetch --prune --tags {remote}"): + raise exceptions.SupplyError(f"Could not fetch source tree from [{remote}]") @probed def checkout(src_dir, branch="master"): - if process.run_subprocess_with_logging("git -C {0} checkout {1}".format(io.escape_path(src_dir), branch)): - raise exceptions.SupplyError("Could not checkout [%s]. Do you have uncommitted changes?" % branch) + if process.run_subprocess_with_logging(f"{git_cmd()} -C {io.escape_path(src_dir)} checkout {branch}"): + raise exceptions.SupplyError(f"Could not checkout [{branch}]. Do you have uncommitted changes?") @probed def rebase(src_dir, remote="origin", branch="master"): checkout(src_dir, branch) - if process.run_subprocess_with_logging("git -C {0} rebase {1}/{2}".format(io.escape_path(src_dir), remote, branch)): - raise exceptions.SupplyError("Could not rebase on branch [%s]" % branch) + if process.run_subprocess_with_logging(f"{git_cmd()} -C {io.escape_path(src_dir)} rebase {remote}/{branch}"): + raise exceptions.SupplyError(f"Could not rebase on branch [{branch}]") @probed @@ -84,28 +91,28 @@ def pull(src_dir, remote="origin", branch="master"): def pull_ts(src_dir, ts): fetch(src_dir) clean_src = io.escape_path(src_dir) - revision = process.run_subprocess_with_output( - 'git -C {0} rev-list -n 1 --before="{1}" --date=iso8601 origin/master'.format(clean_src, ts) - )[0].strip() - if process.run_subprocess_with_logging("git -C {0} checkout {1}".format(clean_src, revision)): - raise exceptions.SupplyError("Could not checkout source tree for timestamped revision [%s]" % ts) + revision = process.run_subprocess_with_output(f'{git_cmd()} -C {clean_src} rev-list -n 1 --before="{ts}" --date=iso8601 origin/master')[ + 0 + ].strip() + if process.run_subprocess_with_logging(f"{git_cmd()} -C {clean_src} checkout {revision}"): + raise exceptions.SupplyError(f"Could not checkout source tree for timestamped revision [{ts}]") @probed def pull_revision(src_dir, revision): fetch(src_dir) - if process.run_subprocess_with_logging("git -C {0} checkout {1}".format(io.escape_path(src_dir), revision)): - raise exceptions.SupplyError("Could not checkout source tree for revision [%s]" % revision) + if process.run_subprocess_with_logging(f"{git_cmd()} -C {io.escape_path(src_dir)} checkout {revision}"): + raise exceptions.SupplyError(f"Could not checkout source tree for revision [{revision}]") @probed def head_revision(src_dir): - return process.run_subprocess_with_output("git -C {0} rev-parse --short HEAD".format(io.escape_path(src_dir)))[0].strip() + return process.run_subprocess_with_output(f"{git_cmd()} -C {io.escape_path(src_dir)} rev-parse --short HEAD")[0].strip() @probed def current_branch(src_dir): - return process.run_subprocess_with_output("git -C {0} rev-parse --abbrev-ref HEAD".format(io.escape_path(src_dir)))[0].strip() + return process.run_subprocess_with_output(f"{git_cmd()} -C {io.escape_path(src_dir)} rev-parse --abbrev-ref HEAD")[0].strip() @probed @@ -114,17 +121,17 @@ def branches(src_dir, remote=True): if remote: # alternatively: git for-each-ref refs/remotes/ --format='%(refname:short)' return _cleanup_remote_branch_names( - process.run_subprocess_with_output("git -C {src} for-each-ref refs/remotes/ --format='%(refname:short)'".format(src=clean_src)) + process.run_subprocess_with_output(f"{git_cmd()} -C {clean_src} for-each-ref refs/remotes/ --format='%(refname:short)'") ) else: return _cleanup_local_branch_names( - process.run_subprocess_with_output("git -C {src} for-each-ref refs/heads/ --format='%(refname:short)'".format(src=clean_src)) + process.run_subprocess_with_output(f"{git_cmd()} -C {clean_src} for-each-ref refs/heads/ --format='%(refname:short)'") ) @probed def tags(src_dir): - return _cleanup_tag_names(process.run_subprocess_with_output("git -C {0} tag".format(io.escape_path(src_dir)))) + return _cleanup_tag_names(process.run_subprocess_with_output(f"{git_cmd()} -C {io.escape_path(src_dir)} tag")) def _cleanup_remote_branch_names(branch_names): diff --git a/it/proxy_test.py b/it/proxy_test.py index b92dae507..7f9510bd1 100644 --- a/it/proxy_test.py +++ b/it/proxy_test.py @@ -54,6 +54,7 @@ def fresh_log_file(): bak = os.path.join(tempfile.mkdtemp(), "rally.log") shutil.move(log_file, bak) yield log_file + shutil.copy(log_file, "tmp.log") # append log lines to the original file and move it back to its original with open(log_file, "r") as src: with open(bak, "a") as dst: @@ -81,7 +82,7 @@ def test_anonymous_proxy_no_connection(cfg, http_proxy, fresh_log_file): assert process.run_subprocess_with_logging(it.esrally_command_line_for(cfg, "list tracks"), env=env) == 0 assert_log_line_present(fresh_log_file, f"Connecting via proxy URL [{http_proxy.anonymous_url}] to the Internet") # unauthenticated proxy access is prevented - assert_log_line_present(fresh_log_file, "[ERROR] Cannot list") + assert_log_line_present(fresh_log_file, "Received HTTP code 407 from proxy") @it.rally_in_mem @@ -89,4 +90,6 @@ def test_authenticated_proxy_user_can_connect(cfg, http_proxy, fresh_log_file): env = dict(os.environ) env["http_proxy"] = http_proxy.authenticated_url assert process.run_subprocess_with_logging(it.esrally_command_line_for(cfg, "list tracks"), env=env) == 0 - assert_log_line_present(fresh_log_file, "[INFO] SUCCESS") + assert_log_line_present(fresh_log_file, f"Connecting via proxy URL [{http_proxy.authenticated_url}] to the Internet") + # authenticated proxy access is allowed + assert_log_line_present(fresh_log_file, "Rebasing on [master]")