Skip to content

Commit

Permalink
apveyor, gitpython-developers#519: FIX incomplete Popen pump
Browse files Browse the repository at this point in the history
+ The code in `_read_lines_from_fno()` was reading the stream only once
per invocation, so when input was larger than `mmap.PAGESIZE`, bytes
were forgotten in the stream.
+ Replaced buffer-banging code with iterate-on-file-descriptors.
+ Also set deamon-threads.
  • Loading branch information
ankostis committed Sep 26, 2016
1 parent d12fdca commit 747a0ff
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 4 deletions.
16 changes: 13 additions & 3 deletions git/cmd.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,14 +193,24 @@ def _deplete_buffer(fno, handler, buf_list, wg=None):
else:
# Oh ... probably we are on windows. select.select() can only handle sockets, we have files
# The only reliable way to do this now is to use threads and wait for both to finish
def _handle_lines(fd, handler, wg):
for line in fd:
line = line.decode(defenc)
if line and handler:
handler(line)
if wg:
wg.done()

# Since the finalizer is expected to wait, we don't have to introduce our own wait primitive
# NO: It's not enough unfortunately, and we will have to sync the threads
wg = WaitGroup()
for fno, (handler, buf_list) in fdmap.items():
for fd, handler in zip((process.stdout, process.stderr),
(stdout_handler, stderr_handler)):
wg.add(1)
t = threading.Thread(target=lambda: _deplete_buffer(fno, handler, buf_list, wg))
t = threading.Thread(target=_handle_lines, args=(fd, handler, wg))
t.setDaemon(True)
t.start()
# end

# NOTE: Just joining threads can possibly fail as there is a gap between .start() and when it's
# actually started, which could make the wait() call to just return because the thread is not yet
# active
Expand Down
4 changes: 3 additions & 1 deletion git/test/test_git.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,9 @@ def counter_stderr(line):
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False)
shell=False,
creationflags=Git.CREATE_NO_WINDOW if sys.platform == 'win32' else 0,
)

handle_process_output(proc, counter_stdout, counter_stderr, lambda proc: proc.wait())

Expand Down

0 comments on commit 747a0ff

Please sign in to comment.