diff --git a/src/brpc/socket_map.cpp b/src/brpc/socket_map.cpp
index 3984f6b866..856cdccb60 100644
--- a/src/brpc/socket_map.cpp
+++ b/src/brpc/socket_map.cpp
@@ -295,8 +295,10 @@ void SocketMap::RemoveInternal(const SocketMapKey& key,
         const int defer_close_second = _options.defer_close_second_dynamic ?
             *_options.defer_close_second_dynamic
             : _options.defer_close_second;
-        if (!remove_orphan && defer_close_second > 0) {
-            // Start count down on this Socket 
+        if (!remove_orphan && defer_close_second > 0
+            && sc->no_ref_us <= (sc->socket->last_active_time_us() + defer_close_second * 1000000L)) {
+            // Start count down on this Socket. If the socket has been idle for
+            // longer than defer_close_second already, close it immediately.
             sc->no_ref_us = butil::cpuwide_time_us();
         } else {
             Socket* const s = sc->socket;