XDNS finalmask: Fix resolver close deadlock and socket leaks (#7089)

https://github.com/XTLS/Xray-core/pull/7089#issuecomment-5987521972

transport/internet/finalmask/xdns/client.go

@@ -80,6 +80,9 @@ func NewClient(c *Config, dialer *finalmask.Dialer) (net.PacketConn, error) {
 	for i := range c.Resolvers {
 		resolver, err := NewResolver(c.Resolvers[i], dialer)
 		if err != nil {
+			for _, resolver := range resolvers {
+				resolver.Close()
+			}
 			return nil, err
 		}
 		resolvers = append(resolvers, resolver)

transport/internet/finalmask/xdns/resolver_tcp.go

@@ -130,13 +130,15 @@ func (r *TCPResolver) Send(p []byte) {
 
 func (r *TCPResolver) Close() {
 	r.mu.Lock()
-	defer r.mu.Unlock()
 	if r.closed() {
+		r.mu.Unlock()
 		return
 	}
 	close(r.closeCh)
-	if r.conn != nil {
-		_ = r.conn.Close()
+	conn := r.conn
+	r.mu.Unlock()
+	if conn != nil {
+		_ = conn.Close()
 	}
 	r.wg.Wait()
 	close(r.readCh)

transport/internet/finalmask/xdns/resolver_udp.go

@@ -117,13 +117,15 @@ func (r *UDPResolver) Send(p []byte) {
 
 func (r *UDPResolver) Close() {
 	r.mu.Lock()
-	defer r.mu.Unlock()
 	if r.closed() {
+		r.mu.Unlock()
 		return
 	}
 	close(r.closeCh)
-	if r.conn != nil {
-		_ = r.conn.Close()
+	conn := r.conn
+	r.mu.Unlock()
+	if conn != nil {
+		_ = conn.Close()
 	}
 	r.wg.Wait()
 	close(r.readCh)