diff --git a/python/tvm/rpc/server.py b/python/tvm/rpc/server.py
index ea9c2da31e996..ea677350d171f 100644
--- a/python/tvm/rpc/server.py
+++ b/python/tvm/rpc/server.py
@@ -403,7 +403,10 @@ def terminate(self):
         """Terminate the server process"""
         if self.use_popen:
             if self.proc:
-                os.killpg(self.proc.pid, signal.SIGTERM)
+                if platform.system() == "Windows":
+                    os.kill(self.proc.pid, signal.CTRL_C_EVENT)
+                else:
+                    os.killpg(self.proc.pid, signal.SIGTERM)
                 self.proc = None
         else:
             if self.proc: