From 747a0fff805b1b86722d3f8a8b5d8df1f1fb491e Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 26 Sep 2016 01:36:57 +0200 Subject: [PATCH] apveyor, #519: FIX incomplete Popen pump + The code in `_read_lines_from_fno()` was reading the stream only once per invocation, so when input was larger than `mmap.PAGESIZE`, bytes were forgotten in the stream. + Replaced buffer-banging code with iterate-on-file-descriptors. + Also set deamon-threads. --- git/cmd.py | 16 +++++++++++++--- git/test/test_git.py | 4 +++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 1cc656bf5..a613a0595 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -193,14 +193,24 @@ def _deplete_buffer(fno, handler, buf_list, wg=None): else: # Oh ... probably we are on windows. select.select() can only handle sockets, we have files # The only reliable way to do this now is to use threads and wait for both to finish + def _handle_lines(fd, handler, wg): + for line in fd: + line = line.decode(defenc) + if line and handler: + handler(line) + if wg: + wg.done() + # Since the finalizer is expected to wait, we don't have to introduce our own wait primitive # NO: It's not enough unfortunately, and we will have to sync the threads wg = WaitGroup() - for fno, (handler, buf_list) in fdmap.items(): + for fd, handler in zip((process.stdout, process.stderr), + (stdout_handler, stderr_handler)): wg.add(1) - t = threading.Thread(target=lambda: _deplete_buffer(fno, handler, buf_list, wg)) + t = threading.Thread(target=_handle_lines, args=(fd, handler, wg)) + t.setDaemon(True) t.start() - # end + # NOTE: Just joining threads can possibly fail as there is a gap between .start() and when it's # actually started, which could make the wait() call to just return because the thread is not yet # active diff --git a/git/test/test_git.py b/git/test/test_git.py index 534539d78..1695993ac 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -238,7 +238,9 @@ def counter_stderr(line): stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=False) + shell=False, + creationflags=Git.CREATE_NO_WINDOW if sys.platform == 'win32' else 0, + ) handle_process_output(proc, counter_stdout, counter_stderr, lambda proc: proc.wait())