diff --git a/net/core/sock_map.c b/net/core/sock_map.c
index 83e3ef655684c1b84ba0ab1024a6dd371e9bf4a6..96b8b8d1dac9af241ef0a210d0dfade753addaec 100644
--- a/net/core/sock_map.c
+++ b/net/core/sock_map.c
@@ -1604,18 +1604,22 @@ void sock_map_close(struct sock *sk, long timeout)
 
 	lock_sock(sk);
 	rcu_read_lock();
-	psock = sk_psock_get(sk);
-	if (unlikely(!psock)) {
-		rcu_read_unlock();
-		release_sock(sk);
-		saved_close = READ_ONCE(sk->sk_prot)->close;
-	} else {
+	psock = sk_psock(sk);
+	if (likely(psock)) {
 		saved_close = psock->saved_close;
 		sock_map_remove_links(sk, psock);
+		psock = sk_psock_get(sk);
+		if (unlikely(!psock))
+			goto no_psock;
 		rcu_read_unlock();
 		sk_psock_stop(psock, true);
 		sk_psock_put(sk, psock);
 		release_sock(sk);
+	} else {
+		saved_close = READ_ONCE(sk->sk_prot)->close;
+no_psock:
+		rcu_read_unlock();
+		release_sock(sk);
 	}
 	/* Make sure we do not recurse. This is a bug.
 	 * Leak the socket instead of crashing on a stack overflow.