sftp/conn.go

134 lines
2.7 KiB
Go
Raw Normal View History

package sftp
import (
"encoding"
"io"
"sync"
"github.com/pkg/errors"
)
// conn implements a bidirectional channel on which client and server
// connections are multiplexed.
type conn struct {
io.Reader
io.WriteCloser
sync.Mutex // used to serialise writes to sendPacket
// sendPacketTest is needed to replicate packet issues in testing
sendPacketTest func(w io.Writer, m encoding.BinaryMarshaler) error
}
func (c *conn) recvPacket() (uint8, []byte, error) {
return recvPacket(c)
}
func (c *conn) sendPacket(m encoding.BinaryMarshaler) error {
c.Lock()
defer c.Unlock()
if c.sendPacketTest != nil {
return c.sendPacketTest(c, m)
}
return sendPacket(c, m)
}
type clientConn struct {
conn
2016-06-15 16:23:51 +08:00
wg sync.WaitGroup
sync.Mutex // protects inflight
inflight map[uint32]chan<- result // outstanding requests
}
2016-06-15 16:23:51 +08:00
// Close closes the SFTP session.
func (c *clientConn) Close() error {
defer c.wg.Wait()
return c.conn.Close()
2016-06-15 16:23:51 +08:00
}
func (c *clientConn) loop() {
defer c.wg.Done()
err := c.recv()
if err != nil {
c.broadcastErr(err)
}
}
// recv continuously reads from the server and forwards responses to the
// appropriate channel.
func (c *clientConn) recv() error {
defer func() {
c.conn.Lock()
c.conn.Close()
c.conn.Unlock()
}()
for {
typ, data, err := c.recvPacket()
if err != nil {
return err
}
sid, _ := unmarshalUint32(data)
c.Lock()
ch, ok := c.inflight[sid]
delete(c.inflight, sid)
c.Unlock()
if !ok {
// This is an unexpected occurrence. Send the error
// back to all listeners so that they terminate
// gracefully.
return errors.Errorf("sid: %v not fond", sid)
}
ch <- result{typ: typ, data: data}
}
}
2016-06-15 16:30:05 +08:00
// result captures the result of receiving the a packet from the server
type result struct {
typ byte
data []byte
err error
}
type idmarshaler interface {
id() uint32
encoding.BinaryMarshaler
}
func (c *clientConn) sendPacket(p idmarshaler) (byte, []byte, error) {
2016-06-15 16:30:05 +08:00
ch := make(chan result, 1)
c.dispatchRequest(ch, p)
s := <-ch
return s.typ, s.data, s.err
}
func (c *clientConn) dispatchRequest(ch chan<- result, p idmarshaler) {
c.Lock()
c.inflight[p.id()] = ch
Fix deadlock when using Client + Service over io.Pipe Client + Server might deadlock while being connected via io.Pipe with high request concurrency. The scenario is as follows: - Server is trying to send reply to a previous client's packet and is waiting on Pipe's condvar for the client to read it: 3 0x0000000000460e39 in sync.(*Cond).Wait at /usr/lib/go/src/sync/cond.go:57 4 0x0000000000464519 in io.(*pipe).write at /usr/lib/go/src/io/pipe.go:90 5 0x0000000000464a4c in io.(*PipeWriter).Write at /usr/lib/go/src/io/pipe.go:157 6 0x00000000007a0b66 in go.(*struct { io.Reader; io.WriteCloser }).Write at <autogenerated>:246 7 0x0000000000790cf6 in github.com/pkg/sftp.(*conn).Write at <autogenerated>:3 8 0x000000000073a68d in github.com/pkg/sftp.sendPacket at ./packet.go:130 9 0x000000000073338c in github.com/pkg/sftp.(*conn).sendPacket at ./conn.go:31 10 0x00000000007374b4 in github.com/pkg/sftp.(*packetManager).maybeSendPackets at ./packet-manager.go:87 - Client is sending a new packet and is waiting on Pipe's condvar for the server to read it, while holding clientConn mutex: 5 0x0000000000464a4c in io.(*PipeWriter).Write at /usr/lib/go/src/io/pipe.go:157 6 0x0000000000790cf6 in github.com/pkg/sftp.(*conn).Write at <autogenerated>:3 7 0x000000000073a68d in github.com/pkg/sftp.sendPacket at ./packet.go:130 8 0x000000000073338c in github.com/pkg/sftp.(*conn).sendPacket at ./conn.go:31 9 0x0000000000733bfb in github.com/pkg/sftp.(*clientConn).dispatchRequest at ./conn.go:105 10 0x0000000000733a0b in github.com/pkg/sftp.(*clientConn).sendPacket at ./conn.go:97 - Client's receive loop is processing a reply from the server and is trying to acquire clientConn mutex: 4 0x000000000046108d in sync.(*Mutex).Lock at /usr/lib/go/src/sync/mutex.go:87 5 0x00000000007336cb in github.com/pkg/sftp.(*clientConn).recv at ./conn.go:69 6 0x000000000073350d in github.com/pkg/sftp.(*clientConn).loop at ./conn.go:49 7 0x000000000045ab51 in runtime.goexit at /usr/lib/go/src/runtime/asm_amd64.s:2197 In this scenario neither server nor client can progress. Fix this by releasing clienConn mutex *before* trying to send the packet. Signed-off-by: Pavel Borzenkov <pavel.borzenkov@gmail.com>
2017-04-05 17:53:22 +08:00
c.Unlock()
if err := c.conn.sendPacket(p); err != nil {
Fix deadlock when using Client + Service over io.Pipe Client + Server might deadlock while being connected via io.Pipe with high request concurrency. The scenario is as follows: - Server is trying to send reply to a previous client's packet and is waiting on Pipe's condvar for the client to read it: 3 0x0000000000460e39 in sync.(*Cond).Wait at /usr/lib/go/src/sync/cond.go:57 4 0x0000000000464519 in io.(*pipe).write at /usr/lib/go/src/io/pipe.go:90 5 0x0000000000464a4c in io.(*PipeWriter).Write at /usr/lib/go/src/io/pipe.go:157 6 0x00000000007a0b66 in go.(*struct { io.Reader; io.WriteCloser }).Write at <autogenerated>:246 7 0x0000000000790cf6 in github.com/pkg/sftp.(*conn).Write at <autogenerated>:3 8 0x000000000073a68d in github.com/pkg/sftp.sendPacket at ./packet.go:130 9 0x000000000073338c in github.com/pkg/sftp.(*conn).sendPacket at ./conn.go:31 10 0x00000000007374b4 in github.com/pkg/sftp.(*packetManager).maybeSendPackets at ./packet-manager.go:87 - Client is sending a new packet and is waiting on Pipe's condvar for the server to read it, while holding clientConn mutex: 5 0x0000000000464a4c in io.(*PipeWriter).Write at /usr/lib/go/src/io/pipe.go:157 6 0x0000000000790cf6 in github.com/pkg/sftp.(*conn).Write at <autogenerated>:3 7 0x000000000073a68d in github.com/pkg/sftp.sendPacket at ./packet.go:130 8 0x000000000073338c in github.com/pkg/sftp.(*conn).sendPacket at ./conn.go:31 9 0x0000000000733bfb in github.com/pkg/sftp.(*clientConn).dispatchRequest at ./conn.go:105 10 0x0000000000733a0b in github.com/pkg/sftp.(*clientConn).sendPacket at ./conn.go:97 - Client's receive loop is processing a reply from the server and is trying to acquire clientConn mutex: 4 0x000000000046108d in sync.(*Mutex).Lock at /usr/lib/go/src/sync/mutex.go:87 5 0x00000000007336cb in github.com/pkg/sftp.(*clientConn).recv at ./conn.go:69 6 0x000000000073350d in github.com/pkg/sftp.(*clientConn).loop at ./conn.go:49 7 0x000000000045ab51 in runtime.goexit at /usr/lib/go/src/runtime/asm_amd64.s:2197 In this scenario neither server nor client can progress. Fix this by releasing clienConn mutex *before* trying to send the packet. Signed-off-by: Pavel Borzenkov <pavel.borzenkov@gmail.com>
2017-04-05 17:53:22 +08:00
c.Lock()
delete(c.inflight, p.id())
Fix deadlock when using Client + Service over io.Pipe Client + Server might deadlock while being connected via io.Pipe with high request concurrency. The scenario is as follows: - Server is trying to send reply to a previous client's packet and is waiting on Pipe's condvar for the client to read it: 3 0x0000000000460e39 in sync.(*Cond).Wait at /usr/lib/go/src/sync/cond.go:57 4 0x0000000000464519 in io.(*pipe).write at /usr/lib/go/src/io/pipe.go:90 5 0x0000000000464a4c in io.(*PipeWriter).Write at /usr/lib/go/src/io/pipe.go:157 6 0x00000000007a0b66 in go.(*struct { io.Reader; io.WriteCloser }).Write at <autogenerated>:246 7 0x0000000000790cf6 in github.com/pkg/sftp.(*conn).Write at <autogenerated>:3 8 0x000000000073a68d in github.com/pkg/sftp.sendPacket at ./packet.go:130 9 0x000000000073338c in github.com/pkg/sftp.(*conn).sendPacket at ./conn.go:31 10 0x00000000007374b4 in github.com/pkg/sftp.(*packetManager).maybeSendPackets at ./packet-manager.go:87 - Client is sending a new packet and is waiting on Pipe's condvar for the server to read it, while holding clientConn mutex: 5 0x0000000000464a4c in io.(*PipeWriter).Write at /usr/lib/go/src/io/pipe.go:157 6 0x0000000000790cf6 in github.com/pkg/sftp.(*conn).Write at <autogenerated>:3 7 0x000000000073a68d in github.com/pkg/sftp.sendPacket at ./packet.go:130 8 0x000000000073338c in github.com/pkg/sftp.(*conn).sendPacket at ./conn.go:31 9 0x0000000000733bfb in github.com/pkg/sftp.(*clientConn).dispatchRequest at ./conn.go:105 10 0x0000000000733a0b in github.com/pkg/sftp.(*clientConn).sendPacket at ./conn.go:97 - Client's receive loop is processing a reply from the server and is trying to acquire clientConn mutex: 4 0x000000000046108d in sync.(*Mutex).Lock at /usr/lib/go/src/sync/mutex.go:87 5 0x00000000007336cb in github.com/pkg/sftp.(*clientConn).recv at ./conn.go:69 6 0x000000000073350d in github.com/pkg/sftp.(*clientConn).loop at ./conn.go:49 7 0x000000000045ab51 in runtime.goexit at /usr/lib/go/src/runtime/asm_amd64.s:2197 In this scenario neither server nor client can progress. Fix this by releasing clienConn mutex *before* trying to send the packet. Signed-off-by: Pavel Borzenkov <pavel.borzenkov@gmail.com>
2017-04-05 17:53:22 +08:00
c.Unlock()
ch <- result{err: err}
}
}
// broadcastErr sends an error to all goroutines waiting for a response.
func (c *clientConn) broadcastErr(err error) {
c.Lock()
listeners := make([]chan<- result, 0, len(c.inflight))
for _, ch := range c.inflight {
listeners = append(listeners, ch)
}
c.Unlock()
for _, ch := range listeners {
ch <- result{err: err}
}
}
2016-06-15 19:08:29 +08:00
type serverConn struct {
conn
}
func (s *serverConn) sendError(p ider, err error) error {
2016-06-15 19:08:29 +08:00
return s.sendPacket(statusFromError(p, err))
}