diff --git a/dubbo-plugin/dubbo-qos/src/main/java/com/alibaba/dubbo/qos/protocol/QosProtocolWrapper.java b/dubbo-plugin/dubbo-qos/src/main/java/com/alibaba/dubbo/qos/protocol/QosProtocolWrapper.java
index 73d1f56a797..e7a8ff6bee6 100644
--- a/dubbo-plugin/dubbo-qos/src/main/java/com/alibaba/dubbo/qos/protocol/QosProtocolWrapper.java
+++ b/dubbo-plugin/dubbo-qos/src/main/java/com/alibaba/dubbo/qos/protocol/QosProtocolWrapper.java
@@ -74,13 +74,10 @@ public <T> Invoker<T> refer(Class<T> type, URL url) throws RpcException {
     @Override
     public void destroy() {
         protocol.destroy();
+        stopServer();
     }
 
     private void startQosServer(URL url) {
-        if (!hasStarted.compareAndSet(false, true)) {
-            return;
-        }
-
         try {
             boolean qosEnable = url.getParameter(QOS_ENABLE,true);
             if (!qosEnable) {
@@ -90,6 +87,10 @@ private void startQosServer(URL url) {
                 return;
             }
 
+            if (!hasStarted.compareAndSet(false, true)) {
+                return;
+            }
+
             int port = url.getParameter(QOS_PORT, DEFAULT_PORT);
             boolean acceptForeignIp = Boolean.parseBoolean(url.getParameter(ACCEPT_FOREIGN_IP,"false"));
             Server server = com.alibaba.dubbo.qos.server.Server.getInstance();
@@ -101,4 +102,11 @@ private void startQosServer(URL url) {
             logger.warn("Fail to start qos server: ", throwable);
         }
     }
+
+    /*package*/ void stopServer() {
+        if (hasStarted.compareAndSet(true, false)) {
+            Server server = Server.getInstance();
+            server.stop();
+        }
+    }
 }