diff --git a/src/node_quic_session.cc b/src/node_quic_session.cc
index 87cdd8b9ae..7d877ead78 100644
--- a/src/node_quic_session.cc
+++ b/src/node_quic_session.cc
@@ -1527,7 +1527,9 @@ void QuicSession::StreamReset(
 
 void QuicSession::UpdateIdleTimer() {
   CHECK_NOT_NULL(idle_);
-  uint64_t timeout = ngtcp2_conn_get_idle_expiry(Connection()) / 1000000UL;
+  uint64_t timeout = (ngtcp2_conn_get_idle_expiry(Connection()) - uv_hrtime()) / 1000000UL;
+  if (timeout < 0)
+    timeout = 0;
   Debug(this, "Updating idle timeout to %" PRIu64, timeout);
   idle_->Update(timeout);
 }
diff --git a/test/parallel/test-quic-idle-timeout.js b/test/parallel/test-quic-idle-timeout.js
index 646fbe0cfb..0dab601eed 100644
--- a/test/parallel/test-quic-idle-timeout.js
+++ b/test/parallel/test-quic-idle-timeout.js
@@ -71,10 +71,12 @@ const idleTimeout = common.platformTimeout(1000);
     idleTimeout,
   });
 
-  server.on('session', common.mustCall(() => {
-    client.close();
-    server.close();
-    assert(Date.now() - start < idleTimeout + error);
+  server.on('session', common.mustCall((serverSession) => {
+    serverSession.on('close', common.mustCall(() => {
+      client.close();
+      server.close();
+      assert(Date.now() - start < idleTimeout + error);
+    }));
   }));
 
   server.on('ready', common.mustCall(() => {