From 4ec6e6f104c82fbf96665e9654226ad91fa1613e Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Thu, 11 Jul 2024 16:32:47 +0100 Subject: [PATCH 01/13] split long lines of js input for node.js to avoid tty issues --- pytest_pyodide/__init__.py | 3 ++- pytest_pyodide/node_test_driver.js | 6 +++++- pytest_pyodide/runner.py | 12 +++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/pytest_pyodide/__init__.py b/pytest_pyodide/__init__.py index fe1d4fd8..8436edb3 100644 --- a/pytest_pyodide/__init__.py +++ b/pytest_pyodide/__init__.py @@ -1,6 +1,6 @@ from importlib.metadata import PackageNotFoundError, version -from .decorator import run_in_pyodide +from .decorator import run_in_pyodide,copy_files_to_pyodide from .runner import ( NodeRunner, PlaywrightChromeRunner, @@ -28,5 +28,6 @@ "set_webdriver_script_timeout", "parse_driver_timeout", "run_in_pyodide", + "copy_files_to_pyodide", "spawn_web_server", ] diff --git a/pytest_pyodide/node_test_driver.js b/pytest_pyodide/node_test_driver.js index b95cb0fc..ddf0e3cf 100644 --- a/pytest_pyodide/node_test_driver.js +++ b/pytest_pyodide/node_test_driver.js @@ -55,7 +55,11 @@ rl.on("line", async function (line) { return; } if (line !== cur_uuid) { - cur_code += line + "\n"; + if( line.trim() === ""){ + cur_code += "\n"; + }else{ + cur_code += line; + } } else { evalCode(cur_uuid, cur_code, context); cur_code = ""; diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py index f60b046f..86f7ac5c 100644 --- a/pytest_pyodide/runner.py +++ b/pytest_pyodide/runner.py @@ -610,7 +610,17 @@ def run_js_inner(self, code, check_code): cmd_id = str(uuid4()) self.p.sendline(cmd_id) - self.p.sendline(wrapped) + # split lines into shorter buffers + # because some ttys don't like long + # single lines + all_lines = wrapped.split("\n") + for l in all_lines: + count = 0 + while count < len(l): + to_read = min(128, len(l)-count) + self.p.sendline(l[count:count+to_read]) + count += to_read + self.p.sendline("") self.p.sendline(cmd_id) self.p.expect_exact(f"{cmd_id}:UUID\r\n", timeout=self.script_timeout) self.p.expect_exact(f"{cmd_id}:UUID\r\n") From cd3c1f7511baf433df4cd7ff590af34257309bfd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:40:27 +0000 Subject: [PATCH 02/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytest_pyodide/__init__.py | 2 +- pytest_pyodide/node_test_driver.js | 2 +- pytest_pyodide/runner.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pytest_pyodide/__init__.py b/pytest_pyodide/__init__.py index 8436edb3..bf147e39 100644 --- a/pytest_pyodide/__init__.py +++ b/pytest_pyodide/__init__.py @@ -1,6 +1,6 @@ from importlib.metadata import PackageNotFoundError, version -from .decorator import run_in_pyodide,copy_files_to_pyodide +from .decorator import copy_files_to_pyodide, run_in_pyodide from .runner import ( NodeRunner, PlaywrightChromeRunner, diff --git a/pytest_pyodide/node_test_driver.js b/pytest_pyodide/node_test_driver.js index ddf0e3cf..bfd85e39 100644 --- a/pytest_pyodide/node_test_driver.js +++ b/pytest_pyodide/node_test_driver.js @@ -58,7 +58,7 @@ rl.on("line", async function (line) { if( line.trim() === ""){ cur_code += "\n"; }else{ - cur_code += line; + cur_code += line; } } else { evalCode(cur_uuid, cur_code, context); diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py index 86f7ac5c..4d75ea1d 100644 --- a/pytest_pyodide/runner.py +++ b/pytest_pyodide/runner.py @@ -617,8 +617,8 @@ def run_js_inner(self, code, check_code): for l in all_lines: count = 0 while count < len(l): - to_read = min(128, len(l)-count) - self.p.sendline(l[count:count+to_read]) + to_read = min(128, len(l) - count) + self.p.sendline(l[count : count + to_read]) count += to_read self.p.sendline("") self.p.sendline(cmd_id) From b528c1edfe5106567203fcfcd4cbe699ef566225 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Thu, 11 Jul 2024 16:46:41 +0100 Subject: [PATCH 03/13] strip whitespace from received lines --- pytest_pyodide/node_test_driver.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pytest_pyodide/node_test_driver.js b/pytest_pyodide/node_test_driver.js index ddf0e3cf..aa194080 100644 --- a/pytest_pyodide/node_test_driver.js +++ b/pytest_pyodide/node_test_driver.js @@ -55,6 +55,12 @@ rl.on("line", async function (line) { return; } if (line !== cur_uuid) { + if(line.endsWith("\n")){ + line=line.substring(0,line.length()-1) + } + if(line.endsWith("\r")){ + line=line.substring(0,line.length()-1) + } if( line.trim() === ""){ cur_code += "\n"; }else{ From b5f95af983c47b63bbfe100c41dca86baddc8693 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Thu, 11 Jul 2024 16:52:58 +0100 Subject: [PATCH 04/13] revert whitespace changes --- pytest_pyodide/node_test_driver.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pytest_pyodide/node_test_driver.js b/pytest_pyodide/node_test_driver.js index 73f85765..bfd85e39 100644 --- a/pytest_pyodide/node_test_driver.js +++ b/pytest_pyodide/node_test_driver.js @@ -55,12 +55,6 @@ rl.on("line", async function (line) { return; } if (line !== cur_uuid) { - if(line.endsWith("\n")){ - line=line.substring(0,line.length()-1) - } - if(line.endsWith("\r")){ - line=line.substring(0,line.length()-1) - } if( line.trim() === ""){ cur_code += "\n"; }else{ From 5658af5f204c7b08d671a0dfa79a670deeabae47 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 12 Jul 2024 11:50:55 +0100 Subject: [PATCH 05/13] use delimiter to end line --- pytest_pyodide/node_test_driver.js | 3 ++- pytest_pyodide/runner.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pytest_pyodide/node_test_driver.js b/pytest_pyodide/node_test_driver.js index bfd85e39..9edadcc5 100644 --- a/pytest_pyodide/node_test_driver.js +++ b/pytest_pyodide/node_test_driver.js @@ -55,7 +55,8 @@ rl.on("line", async function (line) { return; } if (line !== cur_uuid) { - if( line.trim() === ""){ + line = line.substring(0,line.lastIndexOf('$')) + if( line === ""){ cur_code += "\n"; }else{ cur_code += line; diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py index 4d75ea1d..663003c2 100644 --- a/pytest_pyodide/runner.py +++ b/pytest_pyodide/runner.py @@ -614,13 +614,13 @@ def run_js_inner(self, code, check_code): # because some ttys don't like long # single lines all_lines = wrapped.split("\n") - for l in all_lines: + for line in all_lines: count = 0 - while count < len(l): - to_read = min(128, len(l) - count) - self.p.sendline(l[count : count + to_read]) + while count < len(line): + to_read = min(128, len(line) - count) + self.p.sendline(line[count : count + to_read]+"$") count += to_read - self.p.sendline("") + self.p.sendline("$") self.p.sendline(cmd_id) self.p.expect_exact(f"{cmd_id}:UUID\r\n", timeout=self.script_timeout) self.p.expect_exact(f"{cmd_id}:UUID\r\n") From ad857ca3ef4560580d8ff2904ca4391986d50234 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Jul 2024 10:51:13 +0000 Subject: [PATCH 06/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytest_pyodide/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py index 663003c2..e9d5dce7 100644 --- a/pytest_pyodide/runner.py +++ b/pytest_pyodide/runner.py @@ -618,7 +618,7 @@ def run_js_inner(self, code, check_code): count = 0 while count < len(line): to_read = min(128, len(line) - count) - self.p.sendline(line[count : count + to_read]+"$") + self.p.sendline(line[count : count + to_read] + "$") count += to_read self.p.sendline("$") self.p.sendline(cmd_id) From 25cfa70807e2ec05b82987ac612d4f84447ebbbf Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Fri, 12 Jul 2024 12:01:43 +0100 Subject: [PATCH 07/13] make sure no extra carriage returns are sent --- pytest_pyodide/runner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py index 663003c2..0b0db8b1 100644 --- a/pytest_pyodide/runner.py +++ b/pytest_pyodide/runner.py @@ -614,13 +614,14 @@ def run_js_inner(self, code, check_code): # because some ttys don't like long # single lines all_lines = wrapped.split("\n") - for line in all_lines: + for c,line in enumerate(all_lines): count = 0 while count < len(line): to_read = min(128, len(line) - count) self.p.sendline(line[count : count + to_read]+"$") count += to_read - self.p.sendline("$") + if c Date: Fri, 12 Jul 2024 11:02:24 +0000 Subject: [PATCH 08/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytest_pyodide/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py index a007f45a..db3eabf5 100644 --- a/pytest_pyodide/runner.py +++ b/pytest_pyodide/runner.py @@ -614,13 +614,13 @@ def run_js_inner(self, code, check_code): # because some ttys don't like long # single lines all_lines = wrapped.split("\n") - for c,line in enumerate(all_lines): + for c, line in enumerate(all_lines): count = 0 while count < len(line): to_read = min(128, len(line) - count) self.p.sendline(line[count : count + to_read] + "$") count += to_read - if c Date: Fri, 12 Jul 2024 14:13:36 +0100 Subject: [PATCH 09/13] made runner wait for line response to avoid buffer overruns --- pytest_pyodide/node_test_driver.js | 1 + pytest_pyodide/runner.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pytest_pyodide/node_test_driver.js b/pytest_pyodide/node_test_driver.js index 9edadcc5..36a040dd 100644 --- a/pytest_pyodide/node_test_driver.js +++ b/pytest_pyodide/node_test_driver.js @@ -61,6 +61,7 @@ rl.on("line", async function (line) { }else{ cur_code += line; } + console.log("OK") } else { evalCode(cur_uuid, cur_code, context); cur_code = ""; diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py index a007f45a..1f7d1b26 100644 --- a/pytest_pyodide/runner.py +++ b/pytest_pyodide/runner.py @@ -619,9 +619,11 @@ def run_js_inner(self, code, check_code): while count < len(line): to_read = min(128, len(line) - count) self.p.sendline(line[count : count + to_read] + "$") + self.p.expect_exact("OK\r\n") count += to_read if c Date: Fri, 12 Jul 2024 14:17:32 +0100 Subject: [PATCH 10/13] remove long line handling stty setting --- pytest_pyodide/runner.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py index eff20c57..28f855ca 100644 --- a/pytest_pyodide/runner.py +++ b/pytest_pyodide/runner.py @@ -536,9 +536,6 @@ def init_node(self, jspi=False): self.p = pexpect.spawn("/bin/bash", timeout=60) self.p.setecho(False) self.p.delaybeforesend = None - # disable canonical input processing mode to allow sending longer lines - # See: https://pexpect.readthedocs.io/en/stable/api/pexpect.html#pexpect.spawn.send - self.p.sendline("stty -icanon") node_version = pexpect.spawn("node --version").read().decode("utf-8") node_major = int(node_version.split(".")[0][1:]) # vAA.BB.CC -> AA From beb4465d185c88cf94042c8655cf244c487e9764 Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Mon, 15 Jul 2024 13:23:47 +0100 Subject: [PATCH 11/13] review tidying Co-authored-by: Gyeongjae Choi --- pytest_pyodide/node_test_driver.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytest_pyodide/node_test_driver.js b/pytest_pyodide/node_test_driver.js index 36a040dd..95a88187 100644 --- a/pytest_pyodide/node_test_driver.js +++ b/pytest_pyodide/node_test_driver.js @@ -55,10 +55,10 @@ rl.on("line", async function (line) { return; } if (line !== cur_uuid) { - line = line.substring(0,line.lastIndexOf('$')) - if( line === ""){ + line = line.substring(0, line.lastIndexOf('$')) + if(line === ""){ cur_code += "\n"; - }else{ + } else { cur_code += line; } console.log("OK") From 0f27843d0acad056b97dd616cc4aa5266ce7df2c Mon Sep 17 00:00:00 2001 From: Joe Marshall Date: Mon, 15 Jul 2024 13:42:32 +0100 Subject: [PATCH 12/13] comments and clarifications --- pytest_pyodide/node_test_driver.js | 7 ++++++- pytest_pyodide/runner.py | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/pytest_pyodide/node_test_driver.js b/pytest_pyodide/node_test_driver.js index 95a88187..97dbeebc 100644 --- a/pytest_pyodide/node_test_driver.js +++ b/pytest_pyodide/node_test_driver.js @@ -55,13 +55,18 @@ rl.on("line", async function (line) { return; } if (line !== cur_uuid) { + // each line ends with an extra $, to avoid problems with end-of-line + // translation etc. line = line.substring(0, line.lastIndexOf('$')) if(line === ""){ cur_code += "\n"; } else { cur_code += line; } - console.log("OK") + // tell runner.py that the line has been read + // so it can send the next line without worrying about + // filling buffers + console.log("{LINE_OK}") } else { evalCode(cur_uuid, cur_code, context); cur_code = ""; diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py index 28f855ca..dcbc2cb6 100644 --- a/pytest_pyodide/runner.py +++ b/pytest_pyodide/runner.py @@ -607,7 +607,7 @@ def run_js_inner(self, code, check_code): cmd_id = str(uuid4()) self.p.sendline(cmd_id) - # split lines into shorter buffers + # split long lines into shorter buffers # because some ttys don't like long # single lines all_lines = wrapped.split("\n") @@ -615,12 +615,21 @@ def run_js_inner(self, code, check_code): count = 0 while count < len(line): to_read = min(128, len(line) - count) + # each sent line ends with an extra $, to avoid problems with end-of-line + # translation etc. it doesn't matter if there are $ in the string elsewhere + # because the last $ is always the one we added self.p.sendline(line[count : count + to_read] + "$") - self.p.expect_exact("OK\r\n") + # after we send a line, we wait for a response + # before sending the next line + # this means we don't overflow input buffers + self.p.expect_exact("{LINE_OK}\r\n") count += to_read if c < len(all_lines) - 1: + # to insert a line break into the received code string + # we send a blank line with just the $ end-of-line character + # and await response self.p.sendline("$") - self.p.expect_exact("OK\r\n") + self.p.expect_exact("{LINE_OK}\r\n") self.p.sendline(cmd_id) self.p.expect_exact(f"{cmd_id}:UUID\r\n", timeout=self.script_timeout) self.p.expect_exact(f"{cmd_id}:UUID\r\n") From 4b69cbe22cd3c1a24f589909a59c2bca58bb68de Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 12:43:02 +0000 Subject: [PATCH 13/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytest_pyodide/node_test_driver.js | 4 ++-- pytest_pyodide/runner.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pytest_pyodide/node_test_driver.js b/pytest_pyodide/node_test_driver.js index 97dbeebc..cd8e66d5 100644 --- a/pytest_pyodide/node_test_driver.js +++ b/pytest_pyodide/node_test_driver.js @@ -55,7 +55,7 @@ rl.on("line", async function (line) { return; } if (line !== cur_uuid) { - // each line ends with an extra $, to avoid problems with end-of-line + // each line ends with an extra $, to avoid problems with end-of-line // translation etc. line = line.substring(0, line.lastIndexOf('$')) if(line === ""){ @@ -64,7 +64,7 @@ rl.on("line", async function (line) { cur_code += line; } // tell runner.py that the line has been read - // so it can send the next line without worrying about + // so it can send the next line without worrying about // filling buffers console.log("{LINE_OK}") } else { diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py index dcbc2cb6..e2781343 100644 --- a/pytest_pyodide/runner.py +++ b/pytest_pyodide/runner.py @@ -615,7 +615,7 @@ def run_js_inner(self, code, check_code): count = 0 while count < len(line): to_read = min(128, len(line) - count) - # each sent line ends with an extra $, to avoid problems with end-of-line + # each sent line ends with an extra $, to avoid problems with end-of-line # translation etc. it doesn't matter if there are $ in the string elsewhere # because the last $ is always the one we added self.p.sendline(line[count : count + to_read] + "$")