Fix retrieving GitHub token from gh

RELEASE_NOTES.md

@@ -2,6 +2,8 @@
 
 ## New in git-machete 3.17.6
 
+- fixed: `git machete github` not being able to retrieve token used by `gh` for `gh` version >= 2.31.0 (reported by @domesticsimian)
+
 ## New in git-machete 3.17.5
 
 - fixed: `machete-post-slide-out`, `machete-pre-rebase` and `machete-status-branch` hooks can now be executed on Windows

ci/checks/enforce-mocking-only-whitelisted-methods.sh

@@ -14,11 +14,11 @@ git_machete.git_operations.GitContext.fetch_remote
 git_machete.github.GitHubClient.MAX_PULLS_PER_PAGE_COUNT
 git_machete.github.GitHubToken.for_domain
 git_machete.github.RemoteAndOrganizationAndRepository.from_url
+git_machete.utils._popen_cmd
 git_machete.utils._run_cmd
 git_machete.utils.find_executable
 os.path.isfile
 shutil.which
-subprocess.run
 sys.argv
 urllib.request.urlopen
 '

git_machete/cli.py

@@ -67,7 +67,7 @@ def get_help_description(display_help_topics: bool, command: Optional[str] = Non
             usage_str += underline(hdr) + '\n\n'
             for cm in cmds:
                 alias = f", {alias_by_command[cm]}" if cm in alias_by_command else ""
-                usage_str += f'    {bold(cm + alias) : <{18 if utils.ascii_only else 26}}{short_docs[cm]}'
+                usage_str += f'    {bold(cm + alias) : <{18 if utils.ascii_only else 27}}{short_docs[cm]}'
                 usage_str += '\n'
             usage_str += '\n'
         usage_str += fmt(textwrap.dedent("""

git_machete/client.py

@@ -21,7 +21,7 @@
                              RemoteBranchShortName)
 from .github import (GitHubClient, GitHubPullRequest, GitHubToken,
                      RemoteAndOrganizationAndRepository, is_github_remote_url)
-from .utils import (GITHUB_NEW_ISSUE_MESSAGE, AnsiEscapeCodes,
+from .utils import (GITHUB_NEW_ISSUE_MESSAGE, AnsiEscapeCodes, PopenResult,
                     SyncToParentStatus, bold, colored, debug, dim, excluding,
                     flat_map, fmt, get_pretty_choices, get_second,
                     sync_to_parent_status_to_edge_color_map,
@@ -1146,7 +1146,7 @@ def print_line_prefix(branch_: LocalBranchShortName, suffix: str) -> None:
             warn(f"{first_part}.\n\n{second_part}.")
 
     @staticmethod
-    def __popen_hook(*args: str, **kwargs: Any) -> Tuple[int, str, str]:
+    def __popen_hook(*args: str, **kwargs: Any) -> PopenResult:
         if sys.platform == "win32":
             # This is a poor-man's solution to the problem of Windows **not** recognizing Unix-style shebangs :/
             return utils.popen_cmd("sh", *args, **kwargs)  # pragma: no cover

git_machete/github.py

@@ -3,18 +3,17 @@
 import os
 import re
 import shutil
-import subprocess
 import urllib.error
 # Deliberately NOT using much more convenient `requests` to avoid external dependencies in production code
 import urllib.request
 from pathlib import Path
-from typing import Any, Dict, List, NamedTuple, Optional
+from typing import Any, Dict, List, NamedTuple, Optional, Tuple
 
 from git_machete import git_config_keys
 
 from .exceptions import MacheteException, UnprocessableEntityHTTPError
 from .git_operations import GitContext, LocalBranchShortName
-from .utils import bold, debug, fmt, warn
+from .utils import bold, debug, fmt, popen_cmd, warn
 
 
 class GitHubPullRequest(NamedTuple):
@@ -126,29 +125,46 @@ def __get_token_from_gh(cls, domain: str) -> Optional["GitHubToken"]:
         if not gh:
             return None
 
-        # There is also `gh auth token`, but it's only been added in gh v2.17.0, in Oct 2022.
-        # Let's stick to the older `gh auth status --show-token` for compatibility.
-        proc = subprocess.run(
-            [gh, "auth", "status", "--hostname", domain, "--show-token"],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE
-        )
-        if proc.returncode != 0:
+        gh_version_returncode, gh_version_stdout, _ = popen_cmd(gh, "--version")
+        if gh_version_returncode != 0:
             return None
 
-        # `gh auth status --show-token` outputs to stderr in the form:
+        # The stdout of `gh --version` looks like:
         #
-        # {domain}:
-        #   ✓ Logged in to {domain} as {username} ({config_path})
-        #   ✓ Git operations for {domain} configured to use {protocol} protocol.
-        #   ✓ Token: <token>
-        #
-        # with non-zero exit code on failure.
-        stderr = proc.stderr.decode()
-        match = re.search(r"Token: (\w+)", stderr)
-        if match:
-            return cls(value=match.group(1),
-                       provider=f'auth token for {domain} from `gh` GitHub CLI')
+        # gh version 2.18.0 (2022-10-18)
+        # https://github.com/cli/cli/releases/tag/v2.18.0
+
+        gh_version_match = re.search(r"gh version (\d+).(\d+).(\d+) ", gh_version_stdout)
+        gh_version: Optional[Tuple[int, int, int]] = None
+        if gh_version_match:  # pragma: no branch
+            gh_version = int(gh_version_match.group(1)), int(gh_version_match.group(2)), int(gh_version_match.group(3))
+
+        if gh_version and gh_version >= (2, 17, 0):
+            gh_token_returncode, gh_token_stdout, _ = \
+                popen_cmd(gh, "auth", "token", "--hostname", domain, hide_debug_output=True)
+            if gh_token_returncode != 0:
+                return None
+            if gh_token_stdout:
+                return cls(value=gh_token_stdout.strip(), provider=f'auth token for {domain} from `gh` GitHub CLI')
+        else:
+            gh_token_returncode, _, gh_token_stderr = \
+                popen_cmd(gh, "auth", "status", "--hostname", domain, "--show-token", hide_debug_output=True)
+            if gh_token_returncode != 0:
+                return None
+
+            # The stderr of `gh auth status --show-token` looks like:
+            #
+            # {domain}:
+            #   ✓ Logged in to {domain} as {username} ({config_path})
+            #   ✓ Git operations for {domain} configured to use {protocol} protocol.
+            #   ✓ Token: <token>
+            #
+            # with non-zero exit code on failure.
+            # Note that since v2.31.0 (https://github.com/cli/cli/pull/7540), this output goes to stdout instead.
+            # Still, we're only handling here the versions < 2.17.0 that don't provide `gh auth token` yet.
+            match = re.search(r"Token: (\w+)", gh_token_stderr)
+            if match:
+                return cls(value=match.group(1), provider=f'auth token for {domain} from `gh` GitHub CLI')
         return None
 
     @classmethod

git_machete/utils.py

@@ -161,30 +161,50 @@ def chdir_upwards_until_current_directory_exists() -> None:
         current_directory_confirmed_to_exist = True
 
 
-def popen_cmd(cmd: str, *args: str, **kwargs: Any) -> Tuple[int, str, str]:
+class PopenResult(NamedTuple):
+    exit_code: int
+    stdout: str
+    stderr: str
+
+
+def _popen_cmd(cmd: str, *args: str, **kwargs: Any) -> PopenResult:
+    # capture_output argument is only supported since Python 3.7
+    process = subprocess.Popen([cmd] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
+    stdout_bytes, stderr_bytes = process.communicate()
+    exit_code: int = process.returncode  # must be retrieved after process.communicate()
+    stdout: str = stdout_bytes.decode('utf-8')
+    stderr: str = stderr_bytes.decode('utf-8')
+    return PopenResult(exit_code, stdout, stderr)
+
+
+def popen_cmd(cmd: str, *args: str, **kwargs: Any) -> PopenResult:
     chdir_upwards_until_current_directory_exists()
 
-    flat_cmd = get_cmd_shell_repr(cmd, *args, env=kwargs.get('env'))
+    kwargs_ = kwargs.copy()
+    hide_debug_output = kwargs_.pop("hide_debug_output", False)
+    flat_cmd = get_cmd_shell_repr(cmd, *args, env=kwargs_.get('env'))
     if debug_mode:
         print(bold(f">>> {flat_cmd}"), file=sys.stderr)
     elif verbose_mode:
         print(flat_cmd, file=sys.stderr)
 
-    process = subprocess.Popen([cmd] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
-    stdout_bytes, stderr_bytes = process.communicate()
-    stdout: str = stdout_bytes.decode('utf-8')
-    stderr: str = stderr_bytes.decode('utf-8')
-    exit_code: int = process.returncode
+    exit_code, stdout, stderr = result = _popen_cmd(cmd, *args, **kwargs_)
 
     if debug_mode:
         if exit_code != 0:
             print(colored(f"<exit code: {exit_code}>\n", AnsiEscapeCodes.RED), file=sys.stderr)
         if stdout:
-            print(f"{dim('<stdout>:')}\n{dim(stdout)}", file=sys.stderr)
+            if hide_debug_output:
+                print(f"{dim('<stdout>:')}\n{dim('<REDACTED>')}", file=sys.stderr)
+            else:
+                print(f"{dim('<stdout>:')}\n{dim(stdout)}", file=sys.stderr)
         if stderr:
-            print(f"{dim('<stderr>:')}\n{colored(stderr, AnsiEscapeCodes.RED)}", file=sys.stderr)
+            if hide_debug_output:
+                print(f"{dim('<stderr>:')}\n{dim('<REDACTED>')}", file=sys.stderr)
+            else:
+                print(f"{dim('<stderr>:')}\n{colored(stderr, AnsiEscapeCodes.RED)}", file=sys.stderr)
 
-    return exit_code, stdout, stderr
+    return result
 
 
 def get_cmd_shell_repr(cmd: str, *args: str, env: Optional[Dict[str, str]]) -> str:

tests/mockers.py

@@ -6,12 +6,13 @@
 import textwrap
 from contextlib import (AbstractContextManager, contextmanager,
                         redirect_stderr, redirect_stdout)
-from typing import Any, Callable, Iterable, Iterator, Type
+from typing import Any, Callable, Iterable, Iterator, Tuple, Type
 
 import pytest
 
 from git_machete import cli, utils
 from git_machete.exceptions import MacheteException
+from git_machete.utils import PopenResult
 
 
 @contextmanager
@@ -80,6 +81,14 @@ def rewrite_definition_file(new_body: str) -> None:
         def_file.writelines(new_body)
 
 
+def mock__popen_cmd_with_fixed_results(*results: Tuple[int, str, str]) -> Callable[..., PopenResult]:
+    gen = (i for i in results)
+
+    def inner(*args: Any, **kwargs: Any) -> PopenResult:  # noqa: U100
+        return PopenResult(*next(gen))
+    return inner
+
+
 def mock__run_cmd_and_forward_stdout(cmd: str, *args: str, **kwargs: Any) -> int:
     """Execute command in the new subprocess but capture together process's stdout and print it into sys.stdout.
     This sys.stdout is later being redirected via the `redirect_stdout` in `launch_command()` and gets returned by this function.

tests/mockers_github.py

@@ -2,7 +2,6 @@
 from collections import defaultdict
 from contextlib import AbstractContextManager, contextmanager
 from http import HTTPStatus
-from subprocess import CompletedProcess
 from typing import Any, Callable, Dict, Iterator, List, Optional, Union
 from urllib.error import HTTPError
 from urllib.parse import ParseResult, parse_qs, urlencode, urlparse
@@ -30,10 +29,6 @@ def mock_shutil_which(path: Optional[str]) -> Callable[[Any], Optional[str]]:
     return lambda _cmd: path
 
 
-def mock_subprocess_run(returncode: int, stdout: str = '', stderr: str = '') -> Callable[..., CompletedProcess]:  # type: ignore[type-arg]
-    return lambda *args, **_kwargs: CompletedProcess(args, returncode, bytes(stdout, 'utf-8'), bytes(stderr, 'utf-8'))
-
-
 class MockGitHubAPIResponse:
     def __init__(self,
                  status_code: int,

tests/test_github.py

@@ -7,13 +7,14 @@
                                 RemoteAndOrganizationAndRepository)
 from tests.base_test import BaseTest
 from tests.mockers import (assert_failure, assert_success, launch_command,
+                           mock__popen_cmd_with_fixed_results,
                            mock_input_returning_y, overridden_environment,
                            rewrite_definition_file)
 from tests.mockers_github import (MockGitHubAPIState, mock_from_url,
                                   mock_github_token_for_domain_fake,
                                   mock_github_token_for_domain_none,
                                   mock_repository_info, mock_shutil_which,
-                                  mock_subprocess_run, mock_urlopen)
+                                  mock_urlopen)
 
 
 class TestGitHub(BaseTest):
@@ -209,22 +210,38 @@ def test_github_get_token_from_gh(self, mocker: MockerFixture) -> None:
 
         domain = 'git.example.com'
 
-        # Let's first cover the case where `gh` is present, but not authenticated.
-        self.patch_symbol(mocker, 'subprocess.run', mock_subprocess_run(returncode=0, stdout='stdout', stderr='''
-        You are not logged into any GitHub hosts. Run gh auth login to authenticate.
-        '''))
+        fixed_popen_cmd_results = [(1, "unknown error", "")]
+        self.patch_symbol(mocker, 'git_machete.utils._popen_cmd', mock__popen_cmd_with_fixed_results(*fixed_popen_cmd_results))
+        github_token = GitHubToken.for_domain(domain=domain)
+        assert github_token is None
 
+        fixed_popen_cmd_results = [(0, "gh version 2.0.0 (2099-12-31)\nhttps://github.com/cli/cli/releases/tag/v2.0.0\n", ""),
+                                   (0, "", "You are not logged into any GitHub hosts. Run gh auth login to authenticate.")]
+        self.patch_symbol(mocker, 'git_machete.utils._popen_cmd', mock__popen_cmd_with_fixed_results(*fixed_popen_cmd_results))
         github_token = GitHubToken.for_domain(domain=domain)
         assert github_token is None
 
-        self.patch_symbol(mocker, 'subprocess.run', mock_subprocess_run(returncode=0, stdout='stdout', stderr='''
-        github.com
-            ✓ Logged in to github.com as Foo Bar (/Users/foo_bar/.config/gh/hosts.yml)
-            ✓ Git operations for github.com configured to use ssh protocol.
-            ✓ Token: ghp_mytoken_for_github_com_from_gh_cli
-            ✓ Token scopes: gist, read:discussion, read:org, repo, workflow
-        '''))
+        fixed_popen_cmd_results = [(0, "gh version 2.16.0 (2099-12-31)\nhttps://github.com/cli/cli/releases/tag/v2.16.0\n", ""),
+                                   (0, "", """github.com
+                                                ✓ Logged in to git.example.com as Foo Bar (/Users/foo_bar/.config/gh/hosts.yml)
+                                                ✓ Git operations for git.example.com configured to use ssh protocol.
+                                                ✓ Token: ghp_mytoken_for_github_com_from_gh_cli
+                                                ✓ Token scopes: gist, read:discussion, read:org, repo, workflow""")]
+        self.patch_symbol(mocker, 'git_machete.utils._popen_cmd', mock__popen_cmd_with_fixed_results(*fixed_popen_cmd_results))
+        github_token = GitHubToken.for_domain(domain=domain)
+        assert github_token is not None
+        assert github_token.provider == f'auth token for {domain} from `gh` GitHub CLI'
+        assert github_token.value == 'ghp_mytoken_for_github_com_from_gh_cli'
+
+        fixed_popen_cmd_results = [(0, "gh version 2.17.0 (2099-12-31)\nhttps://github.com/cli/cli/releases/tag/v2.17.0\n", ""),
+                                   (0, "", "You are not logged into any GitHub hosts. Run gh auth login to authenticate.")]
+        self.patch_symbol(mocker, 'git_machete.utils._popen_cmd', mock__popen_cmd_with_fixed_results(*fixed_popen_cmd_results))
+        github_token = GitHubToken.for_domain(domain=domain)
+        assert github_token is None
 
+        fixed_popen_cmd_results = [(0, "gh version 2.17.0 (2099-12-31)\nhttps://github.com/cli/cli/releases/tag/v2.17.0\n", ""),
+                                   (0, "ghp_mytoken_for_github_com_from_gh_cli", "")]
+        self.patch_symbol(mocker, 'git_machete.utils._popen_cmd', mock__popen_cmd_with_fixed_results(*fixed_popen_cmd_results))
         github_token = GitHubToken.for_domain(domain=domain)
         assert github_token is not None
         assert github_token.provider == f'auth token for {domain} from `gh` GitHub CLI'
@@ -247,18 +264,24 @@ def test_github_get_token_from_hub(self, mocker: MockerFixture) -> None:
 
         # Let's pretend that `gh` is available, but fails for whatever reason.
         self.patch_symbol(mocker, 'shutil.which', mock_shutil_which('/path/to/gh'))
-        self.patch_symbol(mocker, 'subprocess.run', mock_subprocess_run(returncode=1))
         self.patch_symbol(mocker, 'os.path.isfile', lambda file: '.github-token' not in file)
         self.patch_symbol(mocker, 'builtins.open', mock_open(read_data=dedent(config_hub_contents)))
 
+        fixed_popen_cmd_results = [(0, "gh version 2.31.0 (2099-12-31)\nhttps://github.com/cli/cli/releases/tag/v2.31.0\n", ""),
+                                   (1, "", "unknown error")]
+        self.patch_symbol(mocker, 'git_machete.utils._popen_cmd', mock__popen_cmd_with_fixed_results(*fixed_popen_cmd_results))
         github_token = GitHubToken.for_domain(domain=domain0)
         assert github_token is None
 
+        self.patch_symbol(mocker, 'git_machete.utils._popen_cmd', mock__popen_cmd_with_fixed_results(*fixed_popen_cmd_results))
         github_token = GitHubToken.for_domain(domain=domain1)
         assert github_token is not None
         assert github_token.provider == f'auth token for {domain1} from `hub` GitHub CLI'
         assert github_token.value == 'ghp_mytoken_for_github_com'
 
+        fixed_popen_cmd_results = [(0, "gh version 2.16.0 (2099-12-31)\nhttps://github.com/cli/cli/releases/tag/v2.16.0\n", ""),
+                                   (1, "", "unknown error")]
+        self.patch_symbol(mocker, 'git_machete.utils._popen_cmd', mock__popen_cmd_with_fixed_results(*fixed_popen_cmd_results))
         github_token = GitHubToken.for_domain(domain=domain2)
         assert github_token is not None
         assert github_token.provider == f'auth token for {domain2} from `hub` GitHub CLI'