From e83480a785265806b558ceba1400aa9a2815e97d Mon Sep 17 00:00:00 2001 From: sukun Date: Fri, 1 Dec 2023 21:37:20 +0530 Subject: [PATCH 01/30] webrtc: wait for fin_ack for closing datachannel --- core/network/mux.go | 7 + p2p/net/swarm/swarm_stream.go | 6 + p2p/net/swarm/swarm_stream_test.go | 45 +++++ p2p/test/swarm/swarm_test.go | 83 +++++++++ p2p/test/transport/transport_test.go | 15 +- p2p/transport/webrtc/connection.go | 110 ++++++------ p2p/transport/webrtc/pb/message.pb.go | 29 ++-- p2p/transport/webrtc/pb/message.proto | 4 + p2p/transport/webrtc/stream.go | 227 +++++++++++++++++-------- p2p/transport/webrtc/stream_read.go | 38 ++--- p2p/transport/webrtc/stream_test.go | 178 +++++++++++++++++-- p2p/transport/webrtc/stream_write.go | 69 ++------ p2p/transport/webrtc/transport_test.go | 49 ++++++ 13 files changed, 628 insertions(+), 232 deletions(-) create mode 100644 p2p/net/swarm/swarm_stream_test.go diff --git a/core/network/mux.go b/core/network/mux.go index d12e2ea34b..fdda55365a 100644 --- a/core/network/mux.go +++ b/core/network/mux.go @@ -61,6 +61,13 @@ type MuxedStream interface { SetWriteDeadline(time.Time) error } +// AsyncCloser is implemented by streams that need to do expensive operations on close before +// releasing the resources. Closing the stream async avoids blocking the calling goroutine. +type AsyncCloser interface { + // AsyncClose closes the stream and executes onDone after the stream is closed + AsyncClose(onDone func()) error +} + // MuxedConn represents a connection to a remote peer that has been // extended to support stream multiplexing. // diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index b7846adec2..1339709db2 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -78,6 +78,12 @@ func (s *Stream) Write(p []byte) (int, error) { // Close closes the stream, closing both ends and freeing all associated // resources. func (s *Stream) Close() error { + if as, ok := s.stream.(network.AsyncCloser); ok { + err := as.AsyncClose(func() { + s.closeAndRemoveStream() + }) + return err + } err := s.stream.Close() s.closeAndRemoveStream() return err diff --git a/p2p/net/swarm/swarm_stream_test.go b/p2p/net/swarm/swarm_stream_test.go new file mode 100644 index 0000000000..653489fe8f --- /dev/null +++ b/p2p/net/swarm/swarm_stream_test.go @@ -0,0 +1,45 @@ +package swarm + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peerstore" + "github.com/stretchr/testify/require" +) + +type asyncStreamWrapper struct { + network.MuxedStream + beforeClose func() +} + +func (s *asyncStreamWrapper) AsyncClose(onDone func()) error { + s.beforeClose() + err := s.Close() + onDone() + return err +} + +func TestStreamAsyncCloser(t *testing.T) { + s1 := makeSwarm(t) + s2 := makeSwarm(t) + + s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.TempAddrTTL) + s, err := s1.NewStream(context.Background(), s2.LocalPeer()) + require.NoError(t, err) + ss, ok := s.(*Stream) + require.True(t, ok) + + var called atomic.Bool + as := &asyncStreamWrapper{ + MuxedStream: ss.stream, + beforeClose: func() { + called.Store(true) + }, + } + ss.stream = as + ss.Close() + require.True(t, called.Load()) +} diff --git a/p2p/test/swarm/swarm_test.go b/p2p/test/swarm/swarm_test.go index 9874431441..7f2e731f25 100644 --- a/p2p/test/swarm/swarm_test.go +++ b/p2p/test/swarm/swarm_test.go @@ -2,6 +2,7 @@ package swarm_test import ( "context" + "fmt" "io" "sync" "testing" @@ -14,6 +15,7 @@ import ( rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/client" "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay" + libp2pwebrtc "github.com/libp2p/go-libp2p/p2p/transport/webrtc" ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -243,3 +245,84 @@ func TestLimitStreamsWhenHangingHandlers(t *testing.T) { return false }, 5*time.Second, 100*time.Millisecond) } + +func TestLimitStreamsWhenHangingHandlersWebRTC(t *testing.T) { + var partial rcmgr.PartialLimitConfig + const streamLimit = 10 + partial.System.Streams = streamLimit + mgr, err := rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(partial.Build(rcmgr.InfiniteLimits))) + require.NoError(t, err) + + maddr, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/0/webrtc-direct") + require.NoError(t, err) + + receiver, err := libp2p.New( + libp2p.ResourceManager(mgr), + libp2p.ListenAddrs(maddr), + libp2p.Transport(libp2pwebrtc.New), + ) + require.NoError(t, err) + t.Cleanup(func() { receiver.Close() }) + + var wg sync.WaitGroup + wg.Add(1) + + const pid = "/test" + receiver.SetStreamHandler(pid, func(s network.Stream) { + defer s.Close() + s.Write([]byte{42}) + wg.Wait() + }) + + // Open streamLimit streams + success := 0 + // we make a lot of tries because identify and identify push take up a few streams + for i := 0; i < 1000 && success < streamLimit; i++ { + mgr, err = rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(rcmgr.InfiniteLimits)) + require.NoError(t, err) + + sender, err := libp2p.New(libp2p.ResourceManager(mgr), libp2p.Transport(libp2pwebrtc.New)) + require.NoError(t, err) + t.Cleanup(func() { sender.Close() }) + + sender.Peerstore().AddAddrs(receiver.ID(), receiver.Addrs(), peerstore.PermanentAddrTTL) + + s, err := sender.NewStream(context.Background(), receiver.ID(), pid) + if err != nil { + continue + } + + var b [1]byte + _, err = io.ReadFull(s, b[:]) + if err == nil { + success++ + } + sender.Close() + } + require.Equal(t, streamLimit, success) + // We have the maximum number of streams open. Next call should fail. + mgr, err = rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(rcmgr.InfiniteLimits)) + require.NoError(t, err) + + sender, err := libp2p.New(libp2p.ResourceManager(mgr), libp2p.Transport(libp2pwebrtc.New)) + require.NoError(t, err) + t.Cleanup(func() { sender.Close() }) + + sender.Peerstore().AddAddrs(receiver.ID(), receiver.Addrs(), peerstore.PermanentAddrTTL) + + _, err = sender.NewStream(context.Background(), receiver.ID(), pid) + require.Error(t, err) + // Close the open streams + wg.Done() + + // Next call should succeed + require.Eventually(t, func() bool { + s, err := sender.NewStream(context.Background(), receiver.ID(), pid) + if err == nil { + s.Close() + return true + } + fmt.Println(err) + return false + }, 5*time.Second, 1*time.Second) +} diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index a7e98a0d85..57af510c2a 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -382,9 +382,6 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { const streamCount = 1024 for _, tc := range transportsToTest { t.Run(tc.Name, func(t *testing.T) { - if strings.Contains(tc.Name, "WebRTC") { - t.Skip("This test potentially exhausts the uint16 WebRTC stream ID space.") - } listenerLimits := rcmgr.PartialLimitConfig{ PeerDefault: rcmgr.ResourceLimits{ Streams: 32, @@ -428,7 +425,9 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { workerCount := 4 var startWorker func(workerIdx int) + var wCount atomic.Int32 startWorker = func(workerIdx int) { + fmt.Println("worker count", wCount.Add(1)) wg.Add(1) defer wg.Done() for { @@ -440,7 +439,10 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { // Inline function so we can use defer func() { var didErr bool - defer completedStreams.Add(1) + defer func() { + x := completedStreams.Add(1) + fmt.Println("completed streams", x) + }() defer func() { // Only the first worker adds more workers if workerIdx == 0 && !didErr && !sawFirstErr.Load() { @@ -483,7 +485,6 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { return } err = func(s network.Stream) error { - defer s.Close() err = s.SetDeadline(time.Now().Add(100 * time.Millisecond)) if err != nil { return err @@ -511,8 +512,12 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { return nil }(s) if err != nil && shouldRetry(err) { + fmt.Println("failed to write stream!", err) + s.Reset() time.Sleep(50 * time.Millisecond) continue + } else { + s.Close() } return diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index fd31f8351a..3241ce46cd 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -4,10 +4,8 @@ import ( "context" "errors" "fmt" - "math" "net" "sync" - "sync/atomic" ic "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/network" @@ -25,9 +23,7 @@ import ( var _ tpt.CapableConn = &connection{} -const maxAcceptQueueLen = 10 - -const maxDataChannelID = 1 << 10 +const maxAcceptQueueLen = 256 type errConnectionTimeout struct{} @@ -47,7 +43,8 @@ type connection struct { transport *WebRTCTransport scope network.ConnManagementScope - closeErr error + closeOnce sync.Once + closeErr error localPeer peer.ID localMultiaddr ma.Multiaddr @@ -56,9 +53,8 @@ type connection struct { remoteKey ic.PubKey remoteMultiaddr ma.Multiaddr - m sync.Mutex - streams map[uint16]*stream - nextStreamID atomic.Int32 + m sync.Mutex + streams map[uint16]*stream acceptQueue chan dataChannel @@ -97,25 +93,12 @@ func newConnection( acceptQueue: make(chan dataChannel, maxAcceptQueueLen), } - switch direction { - case network.DirInbound: - c.nextStreamID.Store(1) - case network.DirOutbound: - // stream ID 0 is used for the Noise handshake stream - c.nextStreamID.Store(2) - } pc.OnConnectionStateChange(c.onConnectionStateChange) pc.OnDataChannel(func(dc *webrtc.DataChannel) { if c.IsClosed() { return } - // Limit the number of streams, since we're not able to actually properly close them. - // See https://github.com/libp2p/specs/issues/575 for details. - if *dc.ID() > maxDataChannelID { - c.Close() - return - } dc.OnOpen(func() { rwc, err := dc.Detach() if err != nil { @@ -133,7 +116,6 @@ func newConnection( } }) }) - return c, nil } @@ -144,16 +126,41 @@ func (c *connection) ConnState() network.ConnectionState { // Close closes the underlying peerconnection. func (c *connection) Close() error { - if c.IsClosed() { - return nil - } + c.closeOnce.Do(func() { + c.closeErr = errors.New("connection closed") + // cancel must be called after closeErr is set. This ensures interested goroutines waiting on + // ctx.Done can read closeErr without holding the conn lock. + c.cancel() + c.m.Lock() + streams := c.streams + c.streams = nil + c.m.Unlock() + for _, str := range streams { + str.Reset() + } + c.pc.Close() + c.scope.Done() + }) + return nil +} - c.m.Lock() - defer c.m.Unlock() - c.scope.Done() - c.closeErr = errors.New("connection closed") - c.cancel() - return c.pc.Close() +func (c *connection) closeTimedOut() error { + c.closeOnce.Do(func() { + c.closeErr = errConnectionTimeout{} + // cancel must be called after closeErr is set. This ensures interested goroutines waiting on + // ctx.Done can read closeErr without holding the conn lock. + c.cancel() + c.m.Lock() + streams := c.streams + c.streams = nil + c.m.Unlock() + for _, str := range streams { + str.closeWithError(errConnectionTimeout{}) + } + c.pc.Close() + c.scope.Done() + }) + return nil } func (c *connection) IsClosed() bool { @@ -170,19 +177,7 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error return nil, c.closeErr } - id := c.nextStreamID.Add(2) - 2 - if id > math.MaxUint16 { - return nil, errors.New("exhausted stream ID space") - } - // Limit the number of streams, since we're not able to actually properly close them. - // See https://github.com/libp2p/specs/issues/575 for details. - if id > maxDataChannelID { - c.Close() - return c.OpenStream(ctx) - } - - streamID := uint16(id) - dc, err := c.pc.CreateDataChannel("", &webrtc.DataChannelInit{ID: &streamID}) + dc, err := c.pc.CreateDataChannel("", nil) if err != nil { return nil, err } @@ -190,9 +185,10 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error if err != nil { return nil, fmt.Errorf("open stream: %w", err) } - str := newStream(dc, rwc, func() { c.removeStream(streamID) }) + fmt.Println("opened dc with ID: ", *dc.ID()) + str := newStream(dc, rwc, func() { c.removeStream(*dc.ID()) }) if err := c.addStream(str); err != nil { - str.Close() + str.Reset() return nil, err } return str, nil @@ -205,7 +201,7 @@ func (c *connection) AcceptStream() (network.MuxedStream, error) { case dc := <-c.acceptQueue: str := newStream(dc.channel, dc.stream, func() { c.removeStream(*dc.channel.ID()) }) if err := c.addStream(str); err != nil { - str.Close() + str.Reset() return nil, err } return str, nil @@ -223,6 +219,9 @@ func (c *connection) Transport() tpt.Transport { return c.transport } func (c *connection) addStream(str *stream) error { c.m.Lock() defer c.m.Unlock() + if c.IsClosed() { + return fmt.Errorf("connection closed: %w", c.closeErr) + } if _, ok := c.streams[str.id]; ok { return errors.New("stream ID already exists") } @@ -238,20 +237,7 @@ func (c *connection) removeStream(id uint16) { func (c *connection) onConnectionStateChange(state webrtc.PeerConnectionState) { if state == webrtc.PeerConnectionStateFailed || state == webrtc.PeerConnectionStateClosed { - // reset any streams - if c.IsClosed() { - return - } - c.m.Lock() - defer c.m.Unlock() - c.closeErr = errConnectionTimeout{} - for k, str := range c.streams { - str.setCloseError(c.closeErr) - delete(c.streams, k) - } - c.cancel() - c.scope.Done() - c.pc.Close() + c.closeTimedOut() } } diff --git a/p2p/transport/webrtc/pb/message.pb.go b/p2p/transport/webrtc/pb/message.pb.go index fffc025f7f..384bddd289 100644 --- a/p2p/transport/webrtc/pb/message.pb.go +++ b/p2p/transport/webrtc/pb/message.pb.go @@ -31,6 +31,10 @@ const ( // The sender abruptly terminates the sending part of the stream. The // receiver can discard any data that it already received on that stream. Message_RESET Message_Flag = 2 + // Sending the FIN_ACK flag acknowledges the previous receipt of a message + // with the FIN flag set. Receiving a FIN_ACK flag gives the recipient + // confidence that the remote has received all sent messages. + Message_FIN_ACK Message_Flag = 3 ) // Enum value maps for Message_Flag. @@ -39,11 +43,13 @@ var ( 0: "FIN", 1: "STOP_SENDING", 2: "RESET", + 3: "FIN_ACK", } Message_Flag_value = map[string]int32{ "FIN": 0, "STOP_SENDING": 1, "RESET": 2, + "FIN_ACK": 3, } ) @@ -143,17 +149,18 @@ var File_message_proto protoreflect.FileDescriptor var file_message_proto_rawDesc = []byte{ 0x0a, 0x0d, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, - 0x74, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x04, 0x66, 0x6c, - 0x61, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x2e, 0x46, 0x6c, 0x61, 0x67, 0x52, 0x04, 0x66, 0x6c, 0x61, 0x67, 0x12, 0x18, 0x0a, - 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x2c, 0x0a, 0x04, 0x46, 0x6c, 0x61, 0x67, 0x12, - 0x07, 0x0a, 0x03, 0x46, 0x49, 0x4e, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x4f, 0x50, - 0x5f, 0x53, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, - 0x53, 0x45, 0x54, 0x10, 0x02, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x69, 0x62, 0x70, 0x32, 0x70, 0x2f, 0x67, 0x6f, 0x2d, 0x6c, 0x69, - 0x62, 0x70, 0x32, 0x70, 0x2f, 0x70, 0x32, 0x70, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, - 0x72, 0x74, 0x2f, 0x77, 0x65, 0x62, 0x72, 0x74, 0x63, 0x2f, 0x70, 0x62, + 0x81, 0x01, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x04, 0x66, + 0x6c, 0x61, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x2e, 0x46, 0x6c, 0x61, 0x67, 0x52, 0x04, 0x66, 0x6c, 0x61, 0x67, 0x12, 0x18, + 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x39, 0x0a, 0x04, 0x46, 0x6c, 0x61, 0x67, + 0x12, 0x07, 0x0a, 0x03, 0x46, 0x49, 0x4e, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x4f, + 0x50, 0x5f, 0x53, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, + 0x45, 0x53, 0x45, 0x54, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x49, 0x4e, 0x5f, 0x41, 0x43, + 0x4b, 0x10, 0x03, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x6c, 0x69, 0x62, 0x70, 0x32, 0x70, 0x2f, 0x67, 0x6f, 0x2d, 0x6c, 0x69, 0x62, 0x70, + 0x32, 0x70, 0x2f, 0x70, 0x32, 0x70, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, + 0x2f, 0x77, 0x65, 0x62, 0x72, 0x74, 0x63, 0x2f, 0x70, 0x62, } var ( diff --git a/p2p/transport/webrtc/pb/message.proto b/p2p/transport/webrtc/pb/message.proto index d6b1957beb..aab885b0da 100644 --- a/p2p/transport/webrtc/pb/message.proto +++ b/p2p/transport/webrtc/pb/message.proto @@ -12,6 +12,10 @@ message Message { // The sender abruptly terminates the sending part of the stream. The // receiver can discard any data that it already received on that stream. RESET = 2; + // Sending the FIN_ACK flag acknowledges the previous receipt of a message + // with the FIN flag set. Receiving a FIN_ACK flag gives the recipient + // confidence that the remote has received all sent messages. + FIN_ACK = 3; } optional Flag flag=1; diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 0358dce56c..db7109e4bf 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -1,6 +1,8 @@ package libp2pwebrtc import ( + "errors" + "os" "sync" "time" @@ -52,6 +54,7 @@ type sendState uint8 const ( sendStateSending sendState = iota sendStateDataSent + sendStateDataReceived sendStateReset ) @@ -59,27 +62,38 @@ const ( // and then a network.MuxedStream type stream struct { mx sync.Mutex - // pbio.Reader is not thread safe, - // and while our Read is not promised to be thread safe, - // we ourselves internally read from multiple routines... - reader pbio.Reader + + // readerOnce ensures that only a single goroutine reads from the reader. Read is not threadsafe + // But we may need to read from reader for control messages from a different goroutine. + readerOnce chan struct{} + reader pbio.Reader + // this buffer is limited up to a single message. Reason we need it // is because a reader might read a message midway, and so we need a // wait to buffer that for as long as the remaining part is not (yet) read nextMessage *pb.Message receiveState receiveState - // The public Write API is not promised to be thread safe, - // but we need to be able to write control messages. + // writerMx ensures that only a single goroutine is calling WriteMsg on writer. writer is a + // pbio.uvarintWriter which is not thread safe. The public Write API is not promised to be + // thread safe, but we need to be able to write control messages concurrently + writerMx sync.Mutex writer pbio.Writer sendStateChanged chan struct{} sendState sendState - controlMsgQueue []*pb.Message writeDeadline time.Time writeDeadlineUpdated chan struct{} writeAvailable chan struct{} - readLoopOnce sync.Once + controlMessageReaderOnce sync.Once + // controlMessageReaderEndTime is the end time for reading FIN_ACK from the control + // message reader. We cannot rely on SetReadDeadline to do this since that is prone to + // race condition where a previous deadline timer fires after the latest call to + // SetReadDeadline + // See: https://github.com/pion/sctp/pull/290 + controlMessageReaderEndTime time.Time + controlMessageReaderStarted chan struct{} + controlMessageReaderDone chan struct{} onDone func() id uint16 // for logging purposes @@ -95,13 +109,17 @@ func newStream( onDone func(), ) *stream { s := &stream{ - reader: pbio.NewDelimitedReader(rwc, maxMessageSize), - writer: pbio.NewDelimitedWriter(rwc), + readerOnce: make(chan struct{}, 1), + reader: pbio.NewDelimitedReader(rwc, maxMessageSize), + writer: pbio.NewDelimitedWriter(rwc), sendStateChanged: make(chan struct{}, 1), writeDeadlineUpdated: make(chan struct{}, 1), writeAvailable: make(chan struct{}, 1), + controlMessageReaderStarted: make(chan struct{}), + controlMessageReaderDone: make(chan struct{}), + id: *channel.ID(), dataChannel: rwc.(*datachannel.DataChannel), onDone: onDone, @@ -111,35 +129,6 @@ func newStream( channel.OnBufferedAmountLow(func() { s.mx.Lock() defer s.mx.Unlock() - // first send out queued control messages - for len(s.controlMsgQueue) > 0 { - msg := s.controlMsgQueue[0] - available := s.availableSendSpace() - if controlMsgSize < available { - s.writer.WriteMsg(msg) // TODO: handle error - s.controlMsgQueue = s.controlMsgQueue[1:] - } else { - return - } - } - - if s.isDone() { - // onDone removes the stream from the connection and requires the connection lock. - // This callback(onBufferedAmountLow) is executing in the sctp readLoop goroutine. - // If Connection.Close is called concurrently, the closing goroutine will acquire - // the connection lock and wait for sctp readLoop to exit, the sctp readLoop will - // wait for the connection lock before exiting, causing a deadlock. - // Run this in a different goroutine to avoid the deadlock. - go func() { - s.mx.Lock() - defer s.mx.Unlock() - // TODO: we should be closing the underlying datachannel, but this resets the stream - // See https://github.com/libp2p/specs/issues/575 for details. - // _ = s.dataChannel.Close() - // TODO: write for the spawned reader to return - s.onDone() - }() - } select { case s.writeAvailable <- struct{}{}: @@ -150,15 +139,50 @@ func newStream( } func (s *stream) Close() error { + defer s.cleanup() + closeWriteErr := s.CloseWrite() closeReadErr := s.CloseRead() - if closeWriteErr != nil { - return closeWriteErr + if closeWriteErr != nil || closeReadErr != nil { + s.Reset() + return errors.Join(closeWriteErr, closeReadErr) } - return closeReadErr + + s.mx.Lock() + s.controlMessageReaderEndTime = time.Now().Add(10 * time.Second) + s.mx.Unlock() + s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) + <-s.controlMessageReaderDone + return nil +} + +func (s *stream) AsyncClose(onDone func()) error { + closeWriteErr := s.CloseWrite() + closeReadErr := s.CloseRead() + if closeWriteErr != nil || closeReadErr != nil { + s.Reset() + if onDone != nil { + onDone() + } + return errors.Join(closeWriteErr, closeReadErr) + } + s.mx.Lock() + s.controlMessageReaderEndTime = time.Now().Add(10 * time.Second) + s.mx.Unlock() + s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) + go func() { + <-s.controlMessageReaderDone + s.cleanup() + if onDone != nil { + onDone() + } + }() + return nil } func (s *stream) Reset() error { + defer s.cleanup() + cancelWriteErr := s.cancelWrite() closeReadErr := s.CloseRead() if cancelWriteErr != nil { @@ -167,14 +191,20 @@ func (s *stream) Reset() error { return closeReadErr } +func (s *stream) closeWithError(e error) { + defer s.cleanup() + + s.mx.Lock() + defer s.mx.Unlock() + s.closeErr = e +} + func (s *stream) SetDeadline(t time.Time) error { _ = s.SetReadDeadline(t) return s.SetWriteDeadline(t) } // processIncomingFlag process the flag on an incoming message -// It needs to be called with msg.Flag, not msg.GetFlag(), -// otherwise we'd misinterpret the default value. // It needs to be called while the mutex is locked. func (s *stream) processIncomingFlag(flag *pb.Message_Flag) { if flag == nil { @@ -182,50 +212,101 @@ func (s *stream) processIncomingFlag(flag *pb.Message_Flag) { } switch *flag { - case pb.Message_FIN: - if s.receiveState == receiveStateReceiving { - s.receiveState = receiveStateDataRead - } case pb.Message_STOP_SENDING: - if s.sendState == sendStateSending { + // We must process STOP_SENDING after sending a FIN(sendStateDataSent). Remote peer + // may not send a FIN_ACK once it has sent a STOP_SENDING + if s.sendState == sendStateSending || s.sendState == sendStateDataSent { s.sendState = sendStateReset } select { case s.sendStateChanged <- struct{}{}: default: } + case pb.Message_FIN_ACK: + s.sendState = sendStateDataReceived + select { + case s.sendStateChanged <- struct{}{}: + default: + } + case pb.Message_FIN: + if s.receiveState == receiveStateReceiving { + s.receiveState = receiveStateDataRead + } + if err := s.writeMsgOnWriter(&pb.Message{Flag: pb.Message_FIN_ACK.Enum()}); err != nil { + log.Debugf("failed to send FIN_ACK: %s", err) + // Remote has finished writing all the data It'll stop waiting for the + // FIN_ACK eventually or will be notified when we close the datachannel + } + s.controlMessageReaderOnce.Do(s.spawnControlMessageReader) case pb.Message_RESET: if s.receiveState == receiveStateReceiving { s.receiveState = receiveStateReset } + s.controlMessageReaderOnce.Do(s.spawnControlMessageReader) } - s.maybeDeclareStreamDone() } -// maybeDeclareStreamDone is used to force reset a stream. It should be called with -// the stream lock acquired. It calls stream.onDone which requires the connection lock. -func (s *stream) maybeDeclareStreamDone() { - if s.isDone() { - _ = s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) // pion ignores zero times - // TODO: we should be closing the underlying datachannel, but this resets the stream - // See https://github.com/libp2p/specs/issues/575 for details. - // _ = s.dataChannel.Close() - // TODO: write for the spawned reader to return - s.onDone() - } -} +// spawnControlMessageReader is used for processing control messages after the reader is closed. +func (s *stream) spawnControlMessageReader() { -// isDone indicates whether the stream is completed and all the control messages have also been -// flushed. It must be called with the stream lock acquired. -func (s *stream) isDone() bool { - return (s.sendState == sendStateReset || s.sendState == sendStateDataSent) && - (s.receiveState == receiveStateReset || s.receiveState == receiveStateDataRead) && - len(s.controlMsgQueue) == 0 -} + // Spawn a goroutine to ensure that we're not holding any locks + go func() { + defer close(s.controlMessageReaderDone) + // cleanup the sctp deadline timer goroutine + defer s.SetReadDeadline(time.Time{}) -func (s *stream) setCloseError(e error) { - s.mx.Lock() - defer s.mx.Unlock() + isSendCompleted := func() bool { + s.mx.Lock() + defer s.mx.Unlock() + return s.sendState == sendStateDataReceived || s.sendState == sendStateReset + } - s.closeErr = e + setDeadline := func() bool { + s.mx.Lock() + if s.controlMessageReaderEndTime.IsZero() || time.Now().Before(s.controlMessageReaderEndTime) { + s.SetReadDeadline(s.controlMessageReaderEndTime) + s.mx.Unlock() + return true + } + s.mx.Unlock() + return false + } + + // Unblock any Read call waiting on reader.ReadMsg + s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) + // We have the lock, any waiting reader has exited. + s.readerOnce <- struct{}{} + <-s.readerOnce + // From this point onwards only this goroutine can do reader.ReadMsg + + s.mx.Lock() + if s.nextMessage != nil { + s.processIncomingFlag(s.nextMessage.Flag) + s.nextMessage = nil + } + s.mx.Unlock() + + for !isSendCompleted() { + var msg pb.Message + if !setDeadline() { + return + } + if err := s.reader.ReadMsg(&msg); err != nil { + if errors.Is(err, os.ErrDeadlineExceeded) { + continue + } + return + } + s.mx.Lock() + s.processIncomingFlag(msg.Flag) + s.mx.Unlock() + } + }() +} + +func (s *stream) cleanup() { + s.dataChannel.Close() + if s.onDone != nil { + s.onDone() + } } diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index e064c8558b..209af0a634 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -1,7 +1,6 @@ package libp2pwebrtc import ( - "errors" "io" "time" @@ -10,9 +9,8 @@ import ( ) func (s *stream) Read(b []byte) (int, error) { - if len(b) == 0 { - return 0, nil - } + s.readerOnce <- struct{}{} + defer func() { <-s.readerOnce }() s.mx.Lock() defer s.mx.Unlock() @@ -27,6 +25,10 @@ func (s *stream) Read(b []byte) (int, error) { return 0, network.ErrReset } + if len(b) == 0 { + return 0, nil + } + var read int for { if s.nextMessage == nil { @@ -40,13 +42,19 @@ func (s *stream) Read(b []byte) (int, error) { if s.receiveState == receiveStateDataRead { return 0, io.EOF } - // This case occurs when the remote node closes the stream without writing a FIN message - // There's little we can do here - return 0, errors.New("didn't receive final state for stream") + // This case occurs when remote closes the datachannel without writing a FIN + // message. Some implementations discard the buffered data on closing the + // datachannel. For these implementations a stream reset will be observed as an + // abrupt closing of the datachannel. + s.receiveState = receiveStateReset + return 0, network.ErrReset } if s.receiveState == receiveStateReset { return 0, network.ErrReset } + if s.receiveState == receiveStateDataRead { + return 0, io.EOF + } return 0, err } s.mx.Lock() @@ -70,7 +78,6 @@ func (s *stream) Read(b []byte) (int, error) { case receiveStateDataRead: return read, io.EOF case receiveStateReset: - s.dataChannel.SetReadDeadline(time.Time{}) return read, network.ErrReset } } @@ -81,20 +88,11 @@ func (s *stream) SetReadDeadline(t time.Time) error { return s.dataChannel.SetRe func (s *stream) CloseRead() error { s.mx.Lock() defer s.mx.Unlock() - - if s.nextMessage != nil { - s.processIncomingFlag(s.nextMessage.Flag) - s.nextMessage = nil - } var err error if s.receiveState == receiveStateReceiving && s.closeErr == nil { - err = s.sendControlMessage(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) + err = s.writeMsgOnWriter(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) + s.receiveState = receiveStateReset } - s.receiveState = receiveStateReset - s.maybeDeclareStreamDone() - - // make any calls to Read blocking on ReadMsg return immediately - s.dataChannel.SetReadDeadline(time.Now()) - + s.controlMessageReaderOnce.Do(s.spawnControlMessageReader) return err } diff --git a/p2p/transport/webrtc/stream_test.go b/p2p/transport/webrtc/stream_test.go index f1442b9bfd..851171c2cc 100644 --- a/p2p/transport/webrtc/stream_test.go +++ b/p2p/transport/webrtc/stream_test.go @@ -5,6 +5,7 @@ import ( "errors" "io" "os" + "sync/atomic" "testing" "time" @@ -14,6 +15,7 @@ import ( "github.com/pion/datachannel" "github.com/pion/webrtc/v3" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -24,6 +26,7 @@ type detachedChan struct { func getDetachedDataChannels(t *testing.T) (detachedChan, detachedChan) { s := webrtc.SettingEngine{} + s.SetIncludeLoopbackCandidate(true) s.DetachDataChannels() api := webrtc.NewAPI(webrtc.WithSettingEngine(s)) @@ -97,9 +100,9 @@ func getDetachedDataChannels(t *testing.T) (detachedChan, detachedChan) { func TestStreamSimpleReadWriteClose(t *testing.T) { client, server := getDetachedDataChannels(t) - var clientDone, serverDone bool - clientStr := newStream(client.dc, client.rwc, func() { clientDone = true }) - serverStr := newStream(server.dc, server.rwc, func() { serverDone = true }) + var clientDone, serverDone atomic.Bool + clientStr := newStream(client.dc, client.rwc, func() { clientDone.Store(true) }) + serverStr := newStream(server.dc, server.rwc, func() { serverDone.Store(true) }) // send a foobar from the client n, err := clientStr.Write([]byte("foobar")) @@ -109,7 +112,7 @@ func TestStreamSimpleReadWriteClose(t *testing.T) { // writing after closing should error _, err = clientStr.Write([]byte("foobar")) require.Error(t, err) - require.False(t, clientDone) + require.False(t, clientDone.Load()) // now read all the data on the server side b, err := io.ReadAll(serverStr) @@ -119,19 +122,26 @@ func TestStreamSimpleReadWriteClose(t *testing.T) { n, err = serverStr.Read(make([]byte, 10)) require.Zero(t, n) require.ErrorIs(t, err, io.EOF) - require.False(t, serverDone) + require.False(t, serverDone.Load()) // send something back _, err = serverStr.Write([]byte("lorem ipsum")) require.NoError(t, err) require.NoError(t, serverStr.CloseWrite()) - require.True(t, serverDone) + // and read it at the client - require.False(t, clientDone) + require.False(t, clientDone.Load()) b, err = io.ReadAll(clientStr) require.NoError(t, err) require.Equal(t, []byte("lorem ipsum"), b) - require.True(t, clientDone) + + // stream is only cleaned up on calling Close or AsyncClose or Reset + clientStr.AsyncClose(nil) + serverStr.AsyncClose(nil) + require.Eventually(t, func() bool { return clientDone.Load() }, 10*time.Second, 100*time.Millisecond) + // Need to call Close for cleanup. Otherwise the FIN_ACK is never read + require.NoError(t, serverStr.Close()) + require.Eventually(t, func() bool { return serverDone.Load() }, 10*time.Second, 100*time.Millisecond) } func TestStreamPartialReads(t *testing.T) { @@ -201,14 +211,17 @@ func TestStreamReadReturnsOnClose(t *testing.T) { _, err := clientStr.Read([]byte{0}) errChan <- err }() - time.Sleep(50 * time.Millisecond) // give the Read call some time to hit the loop - require.NoError(t, clientStr.Close()) + time.Sleep(100 * time.Millisecond) // give the Read call some time to hit the loop + require.NoError(t, clientStr.AsyncClose(nil)) select { case err := <-errChan: require.ErrorIs(t, err, network.ErrReset) case <-time.After(500 * time.Millisecond): t.Fatal("timeout") } + + _, err := clientStr.Read([]byte{0}) + require.ErrorIs(t, err, network.ErrReset) } func TestStreamResets(t *testing.T) { @@ -242,6 +255,7 @@ func TestStreamResets(t *testing.T) { _, err := serverStr.Write([]byte("foobar")) return errors.Is(err, network.ErrReset) }, time.Second, 50*time.Millisecond) + serverStr.Close() require.True(t, serverDone) } @@ -305,3 +319,147 @@ func TestStreamWriteDeadlineAsync(t *testing.T) { require.GreaterOrEqual(t, took, timeout) require.LessOrEqual(t, took, timeout*3/2) } + +func TestStreamReadAfterClose(t *testing.T) { + client, server := getDetachedDataChannels(t) + + clientStr := newStream(client.dc, client.rwc, func() {}) + serverStr := newStream(server.dc, server.rwc, func() {}) + + serverStr.AsyncClose(nil) + b := make([]byte, 1) + _, err := clientStr.Read(b) + require.Equal(t, io.EOF, err) + _, err = clientStr.Read(nil) + require.Equal(t, io.EOF, err) + + client, server = getDetachedDataChannels(t) + + clientStr = newStream(client.dc, client.rwc, func() {}) + serverStr = newStream(server.dc, server.rwc, func() {}) + + serverStr.Reset() + b = make([]byte, 1) + _, err = clientStr.Read(b) + require.ErrorIs(t, err, network.ErrReset) + _, err = clientStr.Read(nil) + require.ErrorIs(t, err, network.ErrReset) +} + +func TestStreamCloseAfterFINACK(t *testing.T) { + client, server := getDetachedDataChannels(t) + + done := make(chan bool, 1) + clientStr := newStream(client.dc, client.rwc, func() { done <- true }) + serverStr := newStream(server.dc, server.rwc, func() {}) + + go func() { + done <- true + err := clientStr.Close() + assert.NoError(t, err) + }() + <-done + + select { + case <-done: + t.Fatalf("Close should not have completed without processing FIN_ACK") + case <-time.After(2 * time.Second): + } + + b := make([]byte, 1) + _, err := serverStr.Read(b) + require.Error(t, err) + require.ErrorIs(t, err, io.EOF) + select { + case <-done: + case <-time.After(3 * time.Second): + t.Errorf("Close should have completed") + } +} + +func TestStreamFinAckAfterStopSending(t *testing.T) { + client, server := getDetachedDataChannels(t) + + done := make(chan bool, 1) + clientStr := newStream(client.dc, client.rwc, func() { done <- true }) + serverStr := newStream(server.dc, server.rwc, func() {}) + + go func() { + clientStr.CloseRead() + clientStr.Write([]byte("hello world")) + done <- true + err := clientStr.Close() + assert.NoError(t, err) + }() + <-done + + select { + case <-done: + t.Errorf("Close should not have completed without processing FIN_ACK") + case <-time.After(500 * time.Millisecond): + } + + // serverStr has write half of the stream closed but the read half should + // respond correctly + b := make([]byte, 24) + _, err := serverStr.Read(b) + require.NoError(t, err) + serverStr.Close() // Sends stop_sending, fin + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatalf("Close should have completed") + } +} + +func TestStreamConcurrentClose(t *testing.T) { + client, server := getDetachedDataChannels(t) + + start := make(chan bool, 1) + done := make(chan bool, 2) + clientStr := newStream(client.dc, client.rwc, func() { done <- true }) + serverStr := newStream(server.dc, server.rwc, func() { done <- true }) + + go func() { + start <- true + clientStr.Close() + }() + go func() { + start <- true + serverStr.Close() + }() + <-start + <-start + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("concurrent close should succeed quickly") + } + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("concurrent close should succeed quickly") + } +} + +func TestStreamResetAfterAsyncClose(t *testing.T) { + client, _ := getDetachedDataChannels(t) + + done := make(chan bool, 1) + clientStr := newStream(client.dc, client.rwc, func() { done <- true }) + clientStr.AsyncClose(nil) + + select { + case <-done: + t.Fatalf("AsyncClose shouldn't run cleanup immediately") + case <-time.After(500 * time.Millisecond): + } + + clientStr.Reset() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("Reset should run callback immediately") + } +} diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index 698af9c4d6..7a99957288 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -7,6 +7,7 @@ import ( "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "google.golang.org/protobuf/proto" ) var errWriteAfterClose = errors.New("write after close") @@ -25,16 +26,10 @@ func (s *stream) Write(b []byte) (int, error) { switch s.sendState { case sendStateReset: return 0, network.ErrReset - case sendStateDataSent: + case sendStateDataSent, sendStateDataReceived: return 0, errWriteAfterClose } - // Check if there is any message on the wire. This is used for control - // messages only when the read side of the stream is closed - if s.receiveState != receiveStateReceiving { - s.readLoopOnce.Do(s.spawnControlMessageReader) - } - if !s.writeDeadline.IsZero() && time.Now().After(s.writeDeadline) { return 0, os.ErrDeadlineExceeded } @@ -54,7 +49,7 @@ func (s *stream) Write(b []byte) (int, error) { switch s.sendState { case sendStateReset: return n, network.ErrReset - case sendStateDataSent: + case sendStateDataSent, sendStateDataReceived: return n, errWriteAfterClose } @@ -100,7 +95,7 @@ func (s *stream) Write(b []byte) (int, error) { end = len(b) } msg := &pb.Message{Message: b[:end]} - if err := s.writer.WriteMsg(msg); err != nil { + if err := s.writeMsgOnWriter(msg); err != nil { return n, err } n += end @@ -109,30 +104,6 @@ func (s *stream) Write(b []byte) (int, error) { return n, nil } -// used for reading control messages while writing, in case the reader is closed, -// as to ensure we do still get control messages. This is important as according to the spec -// our data and control channels are intermixed on the same conn. -func (s *stream) spawnControlMessageReader() { - if s.nextMessage != nil { - s.processIncomingFlag(s.nextMessage.Flag) - s.nextMessage = nil - } - - go func() { - // no deadline needed, Read will return once there's a new message, or an error occurred - _ = s.dataChannel.SetReadDeadline(time.Time{}) - for { - var msg pb.Message - if err := s.reader.ReadMsg(&msg); err != nil { - return - } - s.mx.Lock() - s.processIncomingFlag(msg.Flag) - s.mx.Unlock() - } - }() -} - func (s *stream) SetWriteDeadline(t time.Time) error { s.mx.Lock() defer s.mx.Unlock() @@ -153,24 +124,12 @@ func (s *stream) availableSendSpace() int { return availableSpace } -// There's no way to determine the size of a Protobuf message in the pbio package. -// Setting the size to 100 works as long as the control messages (incl. the varint prefix) are smaller than that value. -const controlMsgSize = 100 - -func (s *stream) sendControlMessage(msg *pb.Message) error { - available := s.availableSendSpace() - if controlMsgSize < available { - return s.writer.WriteMsg(msg) - } - s.controlMsgQueue = append(s.controlMsgQueue, msg) - return nil -} - func (s *stream) cancelWrite() error { s.mx.Lock() defer s.mx.Unlock() - if s.sendState != sendStateSending { + // Don't wait for FIN_ACK on reset + if s.sendState != sendStateSending && s.sendState != sendStateDataSent { return nil } s.sendState = sendStateReset @@ -178,10 +137,9 @@ func (s *stream) cancelWrite() error { case s.sendStateChanged <- struct{}{}: default: } - if err := s.sendControlMessage(&pb.Message{Flag: pb.Message_RESET.Enum()}); err != nil { + if err := s.writeMsgOnWriter(&pb.Message{Flag: pb.Message_RESET.Enum()}); err != nil { return err } - s.maybeDeclareStreamDone() return nil } @@ -193,9 +151,18 @@ func (s *stream) CloseWrite() error { return nil } s.sendState = sendStateDataSent - if err := s.sendControlMessage(&pb.Message{Flag: pb.Message_FIN.Enum()}); err != nil { + select { + case s.sendStateChanged <- struct{}{}: + default: + } + if err := s.writeMsgOnWriter(&pb.Message{Flag: pb.Message_FIN.Enum()}); err != nil { return err } - s.maybeDeclareStreamDone() return nil } + +func (s *stream) writeMsgOnWriter(msg proto.Message) error { + s.writerMx.Lock() + defer s.writerMx.Unlock() + return s.writer.WriteMsg(msg) +} diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 7f4df94fc1..c596851007 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -481,6 +481,7 @@ func TestTransportWebRTC_RemoteReadsAfterClose(t *testing.T) { return } err = stream.Close() + fmt.Println("closed!") if err != nil { done <- err return @@ -496,7 +497,55 @@ func TestTransportWebRTC_RemoteReadsAfterClose(t *testing.T) { require.NoError(t, err) // require write and close to complete require.NoError(t, <-done) + stream.SetReadDeadline(time.Now().Add(5 * time.Second)) + + buf := make([]byte, 10) + n, err := stream.Read(buf) + require.NoError(t, err) + require.Equal(t, n, 4) +} + +func TestTransportWebRTC_RemoteReadsAfterAsyncClose(t *testing.T) { + tr, listeningPeer := getTransport(t) + listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") + listener, err := tr.Listen(listenMultiaddr) + require.NoError(t, err) + + tr1, _ := getTransport(t) + + done := make(chan error) + go func() { + lconn, err := listener.Accept() + if err != nil { + done <- err + return + } + s, err := lconn.AcceptStream() + if err != nil { + done <- err + return + } + _, err = s.Write([]byte{1, 2, 3, 4}) + if err != nil { + done <- err + return + } + err = s.(*stream).AsyncClose(nil) + if err != nil { + done <- err + return + } + close(done) + }() + conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) + require.NoError(t, err) + // create a stream + stream, err := conn.OpenStream(context.Background()) + + require.NoError(t, err) + // require write and close to complete + require.NoError(t, <-done) stream.SetReadDeadline(time.Now().Add(5 * time.Second)) buf := make([]byte, 10) From cba757767c5f09342fb0a3d2b987d4d124c4360a Mon Sep 17 00:00:00 2001 From: Sukun Date: Wed, 6 Dec 2023 03:30:48 +0530 Subject: [PATCH 02/30] fix conn closing on peerID mismatch --- p2p/test/transport/transport_test.go | 1 - p2p/transport/webrtc/listener.go | 3 ++- p2p/transport/webrtc/transport.go | 35 +++++++++++++++++----------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 57af510c2a..bc51328c6c 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -684,7 +684,6 @@ func TestDiscoverPeerIDFromSecurityNegotiation(t *testing.T) { // Try connecting with the bogus peer ID err = h2.Connect(ctx, *ai) require.Error(t, err, "somehow we successfully connected to a bogus peerID!") - // Extract the actual peer ID from the error newPeerId, err := extractPeerIDFromError(err) require.NoError(t, err) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index 0b29bf655d..c494da7072 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -291,8 +291,9 @@ func (l *listener) setupConnection( } // we do not yet know A's peer ID so accept any inbound - remotePubKey, err := l.transport.noiseHandshake(ctx, pc, handshakeChannel, "", crypto.SHA256, true) + remotePubKey, err := l.transport.noiseHandshake(ctx, conn, handshakeChannel, "", crypto.SHA256, true) if err != nil { + conn.Close() return nil, err } remotePeer, err := peer.IDFromPublicKey(remotePubKey) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index c13ad19b1e..aad23e82e4 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -331,7 +331,7 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement // We need to set negotiated = true for this channel on both // the client and server to avoid DCEP errors. negotiated, id := handshakeChannelNegotiated, handshakeChannelID - rawHandshakeChannel, err := pc.CreateDataChannel("", &webrtc.DataChannelInit{ + handshakeChannel, err := pc.CreateDataChannel("", &webrtc.DataChannelInit{ Negotiated: &negotiated, ID: &id, }) @@ -371,12 +371,12 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement return nil, errors.New("peerconnection opening timed out") } - detached, err := getDetachedChannel(ctx, rawHandshakeChannel) + detached, err := getDetachedChannel(ctx, handshakeChannel) if err != nil { return nil, err } // set the local address from the candidate pair - cp, err := rawHandshakeChannel.Transport().Transport().ICETransport().GetSelectedCandidatePair() + cp, err := handshakeChannel.Transport().Transport().ICETransport().GetSelectedCandidatePair() if cp == nil { return nil, errors.New("ice connection did not have selected candidate pair: nil result") } @@ -384,7 +384,7 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement return nil, fmt.Errorf("ice connection did not have selected candidate pair: error: %w", err) } - channel := newStream(rawHandshakeChannel, detached, func() {}) + s := newStream(handshakeChannel, detached, func() {}) // the local address of the selected candidate pair should be the // local address for the connection, since different datachannels // are multiplexed over the same SCTP connection @@ -412,9 +412,10 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement return nil, err } - remotePubKey, err := t.noiseHandshake(ctx, pc, channel, p, remoteHashFunction, false) + remotePubKey, err := t.noiseHandshake(ctx, conn, s, p, remoteHashFunction, false) if err != nil { - return conn, err + conn.Close() + return nil, err } if t.gater != nil && !t.gater.InterceptSecured(network.DirOutbound, p, conn) { conn.Close() @@ -497,8 +498,8 @@ func (t *WebRTCTransport) generateNoisePrologue(pc *webrtc.PeerConnection, hash return result, nil } -func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerConnection, datachannel *stream, peer peer.ID, hash crypto.Hash, inbound bool) (ic.PubKey, error) { - prologue, err := t.generateNoisePrologue(pc, hash, inbound) +func (t *WebRTCTransport) noiseHandshake(ctx context.Context, c *connection, s *stream, peer peer.ID, hash crypto.Hash, inbound bool) (ic.PubKey, error) { + prologue, err := t.generateNoisePrologue(c.pc, hash, inbound) if err != nil { return nil, fmt.Errorf("generate prologue: %w", err) } @@ -513,12 +514,12 @@ func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerCon } var secureConn sec.SecureConn if inbound { - secureConn, err = sessionTransport.SecureOutbound(ctx, fakeStreamConn{datachannel}, peer) + secureConn, err = sessionTransport.SecureOutbound(ctx, netConnWrapper{s, c}, peer) if err != nil { return nil, fmt.Errorf("failed to secure inbound connection: %w", err) } } else { - secureConn, err = sessionTransport.SecureInbound(ctx, fakeStreamConn{datachannel}, peer) + secureConn, err = sessionTransport.SecureInbound(ctx, netConnWrapper{s, c}, peer) if err != nil { return nil, fmt.Errorf("failed to secure outbound connection: %w", err) } @@ -526,7 +527,15 @@ func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerCon return secureConn.RemotePublicKey(), nil } -type fakeStreamConn struct{ *stream } +type netConnWrapper struct { + *stream + *connection +} -func (fakeStreamConn) LocalAddr() net.Addr { return nil } -func (fakeStreamConn) RemoteAddr() net.Addr { return nil } +func (netConnWrapper) LocalAddr() net.Addr { return nil } +func (netConnWrapper) RemoteAddr() net.Addr { return nil } + +func (f netConnWrapper) Close() error { + f.connection.Close() + return nil +} From 846c9ab2ecf3714b7e04be327183ae81aa73e2f5 Mon Sep 17 00:00:00 2001 From: Sukun Date: Wed, 6 Dec 2023 23:05:46 +0530 Subject: [PATCH 03/30] webrtc: don't test for ManyStreams in transport integration test --- p2p/test/transport/transport_test.go | 19 ++++++------------- p2p/transport/webrtc/connection.go | 1 - p2p/transport/webrtc/stream.go | 8 ++------ p2p/transport/webrtc/stream_read.go | 2 +- p2p/transport/webrtc/stream_write.go | 13 +++---------- p2p/transport/webrtc/transport_test.go | 1 - 6 files changed, 12 insertions(+), 32 deletions(-) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index bc51328c6c..bfbd2c0bce 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -314,9 +314,6 @@ func TestManyStreams(t *testing.T) { const streamCount = 128 for _, tc := range transportsToTest { t.Run(tc.Name, func(t *testing.T) { - if strings.Contains(tc.Name, "WebRTC") { - t.Skip("Pion doesn't correctly handle large queues of streams.") - } h1 := tc.HostGenerator(t, TransportTestCaseOpts{NoRcmgr: true}) h2 := tc.HostGenerator(t, TransportTestCaseOpts{NoListen: true, NoRcmgr: true}) defer h1.Close() @@ -382,6 +379,9 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { const streamCount = 1024 for _, tc := range transportsToTest { t.Run(tc.Name, func(t *testing.T) { + if strings.Contains(tc.Name, "WebRTC") { + t.Skip("This test potentially exhausts the uint16 WebRTC stream ID space.") + } listenerLimits := rcmgr.PartialLimitConfig{ PeerDefault: rcmgr.ResourceLimits{ Streams: 32, @@ -425,9 +425,7 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { workerCount := 4 var startWorker func(workerIdx int) - var wCount atomic.Int32 startWorker = func(workerIdx int) { - fmt.Println("worker count", wCount.Add(1)) wg.Add(1) defer wg.Done() for { @@ -439,10 +437,7 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { // Inline function so we can use defer func() { var didErr bool - defer func() { - x := completedStreams.Add(1) - fmt.Println("completed streams", x) - }() + defer completedStreams.Add(1) defer func() { // Only the first worker adds more workers if workerIdx == 0 && !didErr && !sawFirstErr.Load() { @@ -485,6 +480,7 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { return } err = func(s network.Stream) error { + defer s.Close() err = s.SetDeadline(time.Now().Add(100 * time.Millisecond)) if err != nil { return err @@ -512,12 +508,8 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { return nil }(s) if err != nil && shouldRetry(err) { - fmt.Println("failed to write stream!", err) - s.Reset() time.Sleep(50 * time.Millisecond) continue - } else { - s.Close() } return @@ -684,6 +676,7 @@ func TestDiscoverPeerIDFromSecurityNegotiation(t *testing.T) { // Try connecting with the bogus peer ID err = h2.Connect(ctx, *ai) require.Error(t, err, "somehow we successfully connected to a bogus peerID!") + // Extract the actual peer ID from the error newPeerId, err := extractPeerIDFromError(err) require.NoError(t, err) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index 3241ce46cd..5ac6cb8d44 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -185,7 +185,6 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error if err != nil { return nil, fmt.Errorf("open stream: %w", err) } - fmt.Println("opened dc with ID: ", *dc.ID()) str := newStream(dc, rwc, func() { c.removeStream(*dc.ID()) }) if err := c.addStream(str); err != nil { str.Reset() diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index db7109e4bf..abc3deef6c 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -74,11 +74,7 @@ type stream struct { nextMessage *pb.Message receiveState receiveState - // writerMx ensures that only a single goroutine is calling WriteMsg on writer. writer is a - // pbio.uvarintWriter which is not thread safe. The public Write API is not promised to be - // thread safe, but we need to be able to write control messages concurrently - writerMx sync.Mutex - writer pbio.Writer + writer pbio.Writer // concurrent writes prevented by mx sendStateChanged chan struct{} sendState sendState writeDeadline time.Time @@ -232,7 +228,7 @@ func (s *stream) processIncomingFlag(flag *pb.Message_Flag) { if s.receiveState == receiveStateReceiving { s.receiveState = receiveStateDataRead } - if err := s.writeMsgOnWriter(&pb.Message{Flag: pb.Message_FIN_ACK.Enum()}); err != nil { + if err := s.writer.WriteMsg(&pb.Message{Flag: pb.Message_FIN_ACK.Enum()}); err != nil { log.Debugf("failed to send FIN_ACK: %s", err) // Remote has finished writing all the data It'll stop waiting for the // FIN_ACK eventually or will be notified when we close the datachannel diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index 209af0a634..1ad171564c 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -90,7 +90,7 @@ func (s *stream) CloseRead() error { defer s.mx.Unlock() var err error if s.receiveState == receiveStateReceiving && s.closeErr == nil { - err = s.writeMsgOnWriter(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) + err = s.writer.WriteMsg(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) s.receiveState = receiveStateReset } s.controlMessageReaderOnce.Do(s.spawnControlMessageReader) diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index 7a99957288..ec0feadb58 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -7,7 +7,6 @@ import ( "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" - "google.golang.org/protobuf/proto" ) var errWriteAfterClose = errors.New("write after close") @@ -95,7 +94,7 @@ func (s *stream) Write(b []byte) (int, error) { end = len(b) } msg := &pb.Message{Message: b[:end]} - if err := s.writeMsgOnWriter(msg); err != nil { + if err := s.writer.WriteMsg(msg); err != nil { return n, err } n += end @@ -137,7 +136,7 @@ func (s *stream) cancelWrite() error { case s.sendStateChanged <- struct{}{}: default: } - if err := s.writeMsgOnWriter(&pb.Message{Flag: pb.Message_RESET.Enum()}); err != nil { + if err := s.writer.WriteMsg(&pb.Message{Flag: pb.Message_RESET.Enum()}); err != nil { return err } return nil @@ -155,14 +154,8 @@ func (s *stream) CloseWrite() error { case s.sendStateChanged <- struct{}{}: default: } - if err := s.writeMsgOnWriter(&pb.Message{Flag: pb.Message_FIN.Enum()}); err != nil { + if err := s.writer.WriteMsg(&pb.Message{Flag: pb.Message_FIN.Enum()}); err != nil { return err } return nil } - -func (s *stream) writeMsgOnWriter(msg proto.Message) error { - s.writerMx.Lock() - defer s.writerMx.Unlock() - return s.writer.WriteMsg(msg) -} diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index c596851007..983e03c00f 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -481,7 +481,6 @@ func TestTransportWebRTC_RemoteReadsAfterClose(t *testing.T) { return } err = stream.Close() - fmt.Println("closed!") if err != nil { done <- err return From f5b59826a38c173d18dbd7e9d392f06550bfdbbc Mon Sep 17 00:00:00 2001 From: Sukun Date: Thu, 7 Dec 2023 11:16:01 +0530 Subject: [PATCH 04/30] remove lock for channel push --- p2p/transport/webrtc/stream.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index abc3deef6c..672ba71b13 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -123,9 +123,6 @@ func newStream( channel.SetBufferedAmountLowThreshold(bufferedAmountLowThreshold) channel.OnBufferedAmountLow(func() { - s.mx.Lock() - defer s.mx.Unlock() - select { case s.writeAvailable <- struct{}{}: default: From 4865037f80e8e8cb012708cff1b981cc1e27bdc5 Mon Sep 17 00:00:00 2001 From: Sukun Date: Fri, 8 Dec 2023 11:12:26 +0530 Subject: [PATCH 05/30] don't track streams on connection --- p2p/transport/webrtc/connection.go | 93 ++++++++------------------ p2p/transport/webrtc/listener.go | 7 +- p2p/transport/webrtc/stream.go | 8 --- p2p/transport/webrtc/transport.go | 10 +-- p2p/transport/webrtc/transport_test.go | 4 +- 5 files changed, 37 insertions(+), 85 deletions(-) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index 5ac6cb8d44..3a10d959ca 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -53,9 +53,6 @@ type connection struct { remoteKey ic.PubKey remoteMultiaddr ma.Multiaddr - m sync.Mutex - streams map[uint16]*stream - acceptQueue chan dataChannel ctx context.Context @@ -89,7 +86,6 @@ func newConnection( remoteMultiaddr: remoteMultiaddr, ctx: ctx, cancel: cancel, - streams: make(map[uint16]*stream), acceptQueue: make(chan dataChannel, maxAcceptQueueLen), } @@ -126,41 +122,26 @@ func (c *connection) ConnState() network.ConnectionState { // Close closes the underlying peerconnection. func (c *connection) Close() error { - c.closeOnce.Do(func() { - c.closeErr = errors.New("connection closed") - // cancel must be called after closeErr is set. This ensures interested goroutines waiting on - // ctx.Done can read closeErr without holding the conn lock. - c.cancel() - c.m.Lock() - streams := c.streams - c.streams = nil - c.m.Unlock() - for _, str := range streams { - str.Reset() - } - c.pc.Close() - c.scope.Done() - }) + c.closeOnce.Do(func() { c.closeWithError(errors.New("connection closed")) }) return nil } -func (c *connection) closeTimedOut() error { - c.closeOnce.Do(func() { - c.closeErr = errConnectionTimeout{} - // cancel must be called after closeErr is set. This ensures interested goroutines waiting on - // ctx.Done can read closeErr without holding the conn lock. - c.cancel() - c.m.Lock() - streams := c.streams - c.streams = nil - c.m.Unlock() - for _, str := range streams { - str.closeWithError(errConnectionTimeout{}) +// closeWithError is used to Close the connection when the underlying DTLS connection fails +func (c *connection) closeWithError(err error) { + c.closeErr = err + // cancel must be called after closeErr is set. This ensures interested goroutines waiting on + // ctx.Done can read closeErr without holding the conn lock. + c.cancel() + c.pc.Close() +loop: + for { + select { + case <-c.acceptQueue: + default: + break loop } - c.pc.Close() - c.scope.Done() - }) - return nil + } + c.scope.Done() } func (c *connection) IsClosed() bool { @@ -176,7 +157,6 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error if c.IsClosed() { return nil, c.closeErr } - dc, err := c.pc.CreateDataChannel("", nil) if err != nil { return nil, err @@ -185,12 +165,11 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error if err != nil { return nil, fmt.Errorf("open stream: %w", err) } - str := newStream(dc, rwc, func() { c.removeStream(*dc.ID()) }) - if err := c.addStream(str); err != nil { - str.Reset() - return nil, err + if c.IsClosed() { + dc.Close() + return nil, c.closeErr } - return str, nil + return newStream(dc, rwc, nil), nil } func (c *connection) AcceptStream() (network.MuxedStream, error) { @@ -198,12 +177,11 @@ func (c *connection) AcceptStream() (network.MuxedStream, error) { case <-c.ctx.Done(): return nil, c.closeErr case dc := <-c.acceptQueue: - str := newStream(dc.channel, dc.stream, func() { c.removeStream(*dc.channel.ID()) }) - if err := c.addStream(str); err != nil { - str.Reset() - return nil, err + if c.IsClosed() { + dc.channel.Close() + return nil, c.closeErr } - return str, nil + return newStream(dc.channel, dc.stream, nil), nil } } @@ -215,28 +193,11 @@ func (c *connection) RemoteMultiaddr() ma.Multiaddr { return c.remoteMultiaddr } func (c *connection) Scope() network.ConnScope { return c.scope } func (c *connection) Transport() tpt.Transport { return c.transport } -func (c *connection) addStream(str *stream) error { - c.m.Lock() - defer c.m.Unlock() - if c.IsClosed() { - return fmt.Errorf("connection closed: %w", c.closeErr) - } - if _, ok := c.streams[str.id]; ok { - return errors.New("stream ID already exists") - } - c.streams[str.id] = str - return nil -} - -func (c *connection) removeStream(id uint16) { - c.m.Lock() - defer c.m.Unlock() - delete(c.streams, id) -} - func (c *connection) onConnectionStateChange(state webrtc.PeerConnectionState) { if state == webrtc.PeerConnectionStateFailed || state == webrtc.PeerConnectionStateClosed { - c.closeTimedOut() + c.closeOnce.Do(func() { + c.closeWithError(errConnectionTimeout{}) + }) } } diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index c494da7072..6e61406618 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -271,7 +271,8 @@ func (l *listener) setupConnection( localMultiaddrWithoutCerthash, _ := ma.SplitFunc(l.localMultiaddr, func(c ma.Component) bool { return c.Protocol().Code == ma.P_CERTHASH }) - handshakeChannel := newStream(rawDatachannel, rwc, func() {}) + s := newStream(rawDatachannel, rwc, func() {}) + // The connection is instantiated before performing the Noise handshake. This is // to handle the case where the remote is faster and attempts to initiate a stream // before the ondatachannel callback can be set. @@ -291,18 +292,20 @@ func (l *listener) setupConnection( } // we do not yet know A's peer ID so accept any inbound - remotePubKey, err := l.transport.noiseHandshake(ctx, conn, handshakeChannel, "", crypto.SHA256, true) + remotePubKey, err := l.transport.noiseHandshake(ctx, conn, s, "", crypto.SHA256, true) if err != nil { conn.Close() return nil, err } remotePeer, err := peer.IDFromPublicKey(remotePubKey) if err != nil { + conn.Close() return nil, err } // earliest point where we know the remote's peerID if err := scope.SetPeer(remotePeer); err != nil { + conn.Close() return nil, err } diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 672ba71b13..e519bc9f83 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -184,14 +184,6 @@ func (s *stream) Reset() error { return closeReadErr } -func (s *stream) closeWithError(e error) { - defer s.cleanup() - - s.mx.Lock() - defer s.mx.Unlock() - s.closeErr = e -} - func (s *stream) SetDeadline(t time.Time) error { _ = s.SetReadDeadline(t) return s.SetWriteDeadline(t) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index aad23e82e4..8f4ed07bb1 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -514,12 +514,12 @@ func (t *WebRTCTransport) noiseHandshake(ctx context.Context, c *connection, s * } var secureConn sec.SecureConn if inbound { - secureConn, err = sessionTransport.SecureOutbound(ctx, netConnWrapper{s, c}, peer) + secureConn, err = sessionTransport.SecureOutbound(ctx, netConnWrapper{s}, peer) if err != nil { return nil, fmt.Errorf("failed to secure inbound connection: %w", err) } } else { - secureConn, err = sessionTransport.SecureInbound(ctx, netConnWrapper{s, c}, peer) + secureConn, err = sessionTransport.SecureInbound(ctx, netConnWrapper{s}, peer) if err != nil { return nil, fmt.Errorf("failed to secure outbound connection: %w", err) } @@ -529,13 +529,7 @@ func (t *WebRTCTransport) noiseHandshake(ctx context.Context, c *connection, s * type netConnWrapper struct { *stream - *connection } func (netConnWrapper) LocalAddr() net.Addr { return nil } func (netConnWrapper) RemoteAddr() net.Addr { return nil } - -func (f netConnWrapper) Close() error { - f.connection.Close() - return nil -} diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 983e03c00f..e927ee4fa7 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -713,9 +713,11 @@ func TestConnectionTimeoutOnListener(t *testing.T) { // TODO: return timeout errors here for { if _, err := str.Write([]byte("test")); err != nil { - require.True(t, os.IsTimeout(err)) + // TODO (sukunrt): Decide whether we want to keep this behaviour + //require.True(t, os.IsTimeout(err)) break } + if time.Since(start) > 5*time.Second { t.Fatal("timeout") } From c2a30c7432c6daa2dc5f6bb58b3a95961fa31290 Mon Sep 17 00:00:00 2001 From: Sukun Date: Fri, 8 Dec 2023 11:56:57 +0530 Subject: [PATCH 06/30] set receive MTU --- p2p/transport/webrtc/listener.go | 1 + p2p/transport/webrtc/transport.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index 6e61406618..1b365af202 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -222,6 +222,7 @@ func (l *listener) setupConnection( l.transport.peerConnectionTimeouts.Failed, l.transport.peerConnectionTimeouts.Keepalive, ) + settingEngine.SetReceiveMTU(receiveMTUBytes) settingEngine.DetachDataChannels() api := webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine)) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 8f4ed07bb1..ac507100f6 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -76,6 +76,7 @@ const ( // timeout values for the peerconnection // https://github.com/pion/webrtc/blob/v3.1.50/settingengine.go#L102-L109 const ( + receiveMTUBytes = 1501 DefaultDisconnectedTimeout = 20 * time.Second DefaultFailedTimeout = 30 * time.Second DefaultKeepaliveTimeout = 15 * time.Second @@ -315,10 +316,9 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement // This is disallowed in the ICE specification. However, implementations // do not strictly follow this, for eg. Chrome gathers TCP loopback candidates. // If you run pion on a system with only the loopback interface UP, - // it will not connect to anything. However, if it has any other interface - // (even a private one, eg. 192.168.0.0/16), it will gather candidates on it and - // will be able to connect to other pion instances on the same interface. + // it will not connect to anything. settingEngine.SetIncludeLoopbackCandidate(true) + settingEngine.SetReceiveMTU(receiveMTUBytes) api := webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine)) From d0ad152db60cbf3be9081ca9da52efbc7aeb96df Mon Sep 17 00:00:00 2001 From: Sukun Date: Fri, 8 Dec 2023 12:19:08 +0530 Subject: [PATCH 07/30] debug log --- p2p/test/transport/transport_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index bfbd2c0bce..8132c8d326 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -280,6 +280,7 @@ func TestLotsOfDataManyStreams(t *testing.T) { sem := make(chan struct{}, parallel) var wg sync.WaitGroup + var done atomic.Int32 for i := 0; i < totalStreams; i++ { wg.Add(1) sem <- struct{}{} @@ -302,6 +303,7 @@ func TestLotsOfDataManyStreams(t *testing.T) { _, err = s.Read([]byte{0}) require.ErrorIs(t, err, io.EOF) + fmt.Println("done", done.Add(1)) }() } From d21921bd6047d7703aa39e333f0a3ebf624989a7 Mon Sep 17 00:00:00 2001 From: Sukun Date: Sat, 9 Dec 2023 19:32:53 +0530 Subject: [PATCH 08/30] webrtc: update pion --- go.mod | 24 ++++---- go.sum | 63 +++++++++++++++---- p2p/test/transport/transport_test.go | 8 +-- p2p/transport/webrtc/datachannel.go | 6 +- p2p/transport/webrtc/stream.go | 4 +- p2p/transport/webrtc/stream_write.go | 2 +- p2p/transport/webrtc/transport.go | 6 +- p2p/transport/webrtc/transport_test.go | 83 ++++++++++++++++++-------- test-plans/go.mod | 24 ++++---- test-plans/go.sum | 63 +++++++++++++++---- 10 files changed, 199 insertions(+), 84 deletions(-) diff --git a/go.mod b/go.mod index 624aeefd16..5e26fb6ce1 100644 --- a/go.mod +++ b/go.mod @@ -45,9 +45,9 @@ require ( github.com/multiformats/go-varint v0.0.7 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pion/datachannel v1.5.5 - github.com/pion/ice/v2 v2.3.6 + github.com/pion/ice/v2 v2.3.11 github.com/pion/logging v0.2.2 - github.com/pion/stun v0.6.0 + github.com/pion/stun v0.6.1 github.com/pion/webrtc/v3 v3.2.9 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.4.0 @@ -88,7 +88,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.4.0 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -101,17 +101,17 @@ require ( github.com/multiformats/go-base36 v0.2.0 // indirect github.com/onsi/ginkgo/v2 v2.13.0 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect - github.com/pion/dtls/v2 v2.2.7 // indirect - github.com/pion/interceptor v0.1.17 // indirect - github.com/pion/mdns v0.0.7 // indirect + github.com/pion/dtls/v2 v2.2.8 // indirect + github.com/pion/interceptor v0.1.25 // indirect + github.com/pion/mdns v0.0.9 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.10 // indirect - github.com/pion/rtp v1.7.13 // indirect - github.com/pion/sctp v1.8.7 // indirect + github.com/pion/rtcp v1.2.12 // indirect + github.com/pion/rtp v1.8.3 // indirect + github.com/pion/sctp v1.8.9 // indirect github.com/pion/sdp/v3 v3.0.6 // indirect - github.com/pion/srtp/v2 v2.0.15 // indirect - github.com/pion/transport/v2 v2.2.1 // indirect - github.com/pion/turn/v2 v2.1.0 // indirect + github.com/pion/srtp/v2 v2.0.18 // indirect + github.com/pion/transport/v2 v2.2.4 // indirect + github.com/pion/turn/v2 v2.1.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.37.0 // indirect diff --git a/go.sum b/go.sum index 53a93d9444..01bef9515e 100644 --- a/go.sum +++ b/go.sum @@ -212,8 +212,10 @@ github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVe github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -395,41 +397,58 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= -github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v2 v2.3.6 h1:Jgqw36cAud47iD+N6rNX225uHvrgWtAlHfVyOQc3Heg= +github.com/pion/dtls/v2 v2.2.8 h1:BUroldfiIbV9jSnC6cKOMnyiORRWrWWpV11JUyEu5OA= +github.com/pion/dtls/v2 v2.2.8/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/ice/v2 v2.3.6/go.mod h1:9/TzKDRwBVAPsC+YOrKH/e3xDrubeTRACU9/sHQarsU= -github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w= +github.com/pion/ice/v2 v2.3.11 h1:rZjVmUwyT55cmN8ySMpL7rsS8KYsJERsrxJLLxpKhdw= +github.com/pion/ice/v2 v2.3.11/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E= github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= +github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= +github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/mdns v0.0.7 h1:P0UB4Sr6xDWEox0kTVxF0LmQihtCbSAdW0H2nEgkA3U= github.com/pion/mdns v0.0.7/go.mod h1:4iP2UbeFhLI/vWju/bw6ZfwjJzk0z8DNValjGxR/dD8= +github.com/pion/mdns v0.0.8/go.mod h1:hYE72WX8WDveIhg7fmXgMKivD3Puklk0Ymzog0lSyaI= +github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4= +github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.10 h1:nkr3uj+8Sp97zyItdN60tE/S6vk4al5CPRR6Gejsdjc= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= -github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA= +github.com/pion/rtcp v1.2.12 h1:bKWiX93XKgDZENEXCijvHRU/wRifm6JV5DGcH6twtSM= +github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= +github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= +github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.7 h1:JnABvFakZueGAn4KU/4PSKg+GWbF6QWbKTWZOSGJjXw= github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= +github.com/pion/sctp v1.8.9 h1:TP5ZVxV5J7rz7uZmbyvnUvsn7EJ2x/5q9uhsTtXbI3g= +github.com/pion/sctp v1.8.9/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= -github.com/pion/srtp/v2 v2.0.15 h1:+tqRtXGsGwHC0G0IUIAzRmdkHvriF79IHVfZGfHrQoA= github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= +github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= +github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= -github.com/pion/stun v0.6.0 h1:JHT/2iyGDPrFWE8NNC15wnddBN8KifsEDw8swQmrEmU= github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= +github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= +github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= -github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= -github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= +github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++NjXLOUorc= +github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QAlo= +github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= +github.com/pion/turn/v2 v2.1.3 h1:pYxTVWG2gpC97opdRc5IGsQ1lJ9O/IlNhkzj7MMrGAA= +github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/webrtc/v3 v3.2.9 h1:U8NSjQDlZZ+Iy/hg42Q/u6mhEVSXYvKrOIZiZwYTfLc= github.com/pion/webrtc/v3 v3.2.9/go.mod h1:gjQLMZeyN3jXBGdxGmUYCyKjOuYX/c99BDjGqmadq0A= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -596,6 +615,10 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -686,6 +709,10 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -780,6 +807,10 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -790,6 +821,10 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -802,6 +837,10 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 8132c8d326..e29b1562a0 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -9,7 +9,6 @@ import ( "io" "net" "runtime" - "strings" "sync" "sync/atomic" "testing" @@ -381,9 +380,6 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { const streamCount = 1024 for _, tc := range transportsToTest { t.Run(tc.Name, func(t *testing.T) { - if strings.Contains(tc.Name, "WebRTC") { - t.Skip("This test potentially exhausts the uint16 WebRTC stream ID space.") - } listenerLimits := rcmgr.PartialLimitConfig{ PeerDefault: rcmgr.ResourceLimits{ Streams: 32, @@ -439,7 +435,9 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { // Inline function so we can use defer func() { var didErr bool - defer completedStreams.Add(1) + defer func() { + fmt.Println("completed", completedStreams.Add(1)) + }() defer func() { // Only the first worker adds more workers if workerIdx == 0 && !didErr && !sawFirstErr.Load() { diff --git a/p2p/transport/webrtc/datachannel.go b/p2p/transport/webrtc/datachannel.go index aedca6d5b0..b23c33ef27 100644 --- a/p2p/transport/webrtc/datachannel.go +++ b/p2p/transport/webrtc/datachannel.go @@ -11,8 +11,10 @@ import ( // will be called immediately. Only use after the peerconnection is open. // The context should close if the peerconnection underlying the datachannel // is closed. -func getDetachedChannel(ctx context.Context, dc *webrtc.DataChannel) (rwc datachannel.ReadWriteCloser, err error) { +func getDetachedChannel(ctx context.Context, dc *webrtc.DataChannel) (datachannel.ReadWriteCloser, error) { done := make(chan struct{}) + var rwc datachannel.ReadWriteCloser + var err error dc.OnOpen(func() { defer close(done) rwc, err = dc.Detach() @@ -21,8 +23,8 @@ func getDetachedChannel(ctx context.Context, dc *webrtc.DataChannel) (rwc datach // callback immediately if the SCTP transport is also connected. select { case <-done: + return rwc, err case <-ctx.Done(): return nil, ctx.Err() } - return } diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index e519bc9f83..04f8f66f15 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -21,6 +21,9 @@ const ( // We can change this value in the SettingEngine before creating the peerconnection. // https://github.com/pion/webrtc/blob/v3.1.49/sctptransport.go#L341 maxBufferedAmount = 2 * maxMessageSize + // maxTotalControlMessageSize is the maximum of the total size of all control messages we will + // write on this stream + maxTotalControlMessageSize = 500 // bufferedAmountLowThreshold and maxBufferedAmount are bound // to a stream but congestion control is done on the whole // SCTP association. This means that a single stream can monopolize @@ -133,7 +136,6 @@ func newStream( func (s *stream) Close() error { defer s.cleanup() - closeWriteErr := s.CloseWrite() closeReadErr := s.CloseRead() if closeWriteErr != nil || closeReadErr != nil { diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index ec0feadb58..6b0e0b8d7b 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -117,7 +117,7 @@ func (s *stream) SetWriteDeadline(t time.Time) error { func (s *stream) availableSendSpace() int { buffered := int(s.dataChannel.BufferedAmount()) availableSpace := maxBufferedAmount - buffered - if availableSpace < 0 { // this should never happen, but better check + if availableSpace+maxTotalControlMessageSize < 0 { // this should never happen, but better check log.Errorw("data channel buffered more data than the maximum amount", "max", maxBufferedAmount, "buffered", buffered) } return availableSpace diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index ac507100f6..2f8b0d5765 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -76,7 +76,7 @@ const ( // timeout values for the peerconnection // https://github.com/pion/webrtc/blob/v3.1.50/settingengine.go#L102-L109 const ( - receiveMTUBytes = 1501 + receiveMTUBytes = 1500 DefaultDisconnectedTimeout = 20 * time.Second DefaultFailedTimeout = 30 * time.Second DefaultKeepaliveTimeout = 15 * time.Second @@ -533,3 +533,7 @@ type netConnWrapper struct { func (netConnWrapper) LocalAddr() net.Addr { return nil } func (netConnWrapper) RemoteAddr() net.Addr { return nil } +func (w netConnWrapper) Close() error { + w.stream.Reset() + return nil +} diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index e927ee4fa7..b691dd733f 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -17,6 +17,7 @@ import ( quicproxy "github.com/quic-go/quic-go/integrationtests/tools/proxy" "golang.org/x/crypto/sha3" + "golang.org/x/exp/rand" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/network" @@ -294,7 +295,6 @@ func TestTransportWebRTC_DialerCanCreateStreams(t *testing.T) { } func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { - count := 5 tr, listeningPeer := getTransport(t) listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") listener, err := tr.Listen(listenMultiaddr) @@ -303,27 +303,40 @@ func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { tr1, connectingPeer := getTransport(t) done := make(chan struct{}) + const ( + numListeners = 10 + numStreams = 100 + numWriters = 10 + size = 20 << 10 + ) + go func() { lconn, err := listener.Accept() require.NoError(t, err) require.Equal(t, connectingPeer, lconn.RemotePeer()) var wg sync.WaitGroup - - for i := 0; i < count; i++ { - stream, err := lconn.AcceptStream() - require.NoError(t, err) + var doneStreams atomic.Int32 + var concurrency atomic.Int32 + for i := 0; i < numListeners; i++ { wg.Add(1) go func() { defer wg.Done() - buf := make([]byte, 100) - n, err := stream.Read(buf) - require.NoError(t, err) - require.Equal(t, "test", string(buf[:n])) - _, err = stream.Write([]byte("test")) - require.NoError(t, err) + for { + var nn int32 + if nn = doneStreams.Add(1); nn > int32(numStreams) { + return + } + s, err := lconn.AcceptStream() + fmt.Println("concurrency", concurrency.Add(1)) + require.NoError(t, err) + n, err := io.Copy(s, s) + require.Equal(t, n, int64(size)) + require.NoError(t, err) + s.Close() + concurrency.Add(-1) + } }() } - wg.Wait() done <- struct{}{} }() @@ -332,23 +345,41 @@ func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { require.NoError(t, err) t.Logf("dialer opened connection") - for i := 0; i < count; i++ { - idx := i + var wwg sync.WaitGroup + var cnt atomic.Int32 + var streamsStarted atomic.Int32 + for i := 0; i < numWriters; i++ { + wwg.Add(1) go func() { - stream, err := conn.OpenStream(context.Background()) - require.NoError(t, err) - t.Logf("dialer opened stream: %d", idx) - buf := make([]byte, 100) - _, err = stream.Write([]byte("test")) - require.NoError(t, err) - n, err := stream.Read(buf) - require.NoError(t, err) - require.Equal(t, "test", string(buf[:n])) + defer wwg.Done() + for { + var nn int32 + if nn = streamsStarted.Add(1); nn > int32(numStreams) { + return + } + s, err := conn.OpenStream(context.Background()) + require.NoError(t, err) + // t.Logf("dialer opened stream: %d %d", idx, s.(*stream).id) + buf := make([]byte, size) + rand.Read(buf) + n, err := s.Write(buf) + require.Equal(t, n, size) + require.NoError(t, err) + + s.CloseWrite() + resp := make([]byte, size+10) + n, err = io.ReadFull(s, resp) + require.ErrorIs(t, err, io.ErrUnexpectedEOF) + require.Equal(t, n, size) + if string(buf) != string(resp[:size]) { + t.Errorf("bytes not equal: %d %d", len(buf), len(resp)) + } + s.Close() + fmt.Println("completed stream: ", cnt.Add(1), s.(*stream).id) + } }() - if i%10 == 0 && i > 0 { - time.Sleep(100 * time.Millisecond) - } } + wwg.Wait() select { case <-done: case <-time.After(100 * time.Second): diff --git a/test-plans/go.mod b/test-plans/go.mod index 4b47834a22..d2d188951b 100644 --- a/test-plans/go.mod +++ b/test-plans/go.mod @@ -39,7 +39,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.4.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect @@ -78,20 +78,20 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pion/datachannel v1.5.5 // indirect - github.com/pion/dtls/v2 v2.2.7 // indirect - github.com/pion/ice/v2 v2.3.6 // indirect - github.com/pion/interceptor v0.1.17 // indirect + github.com/pion/dtls/v2 v2.2.8 // indirect + github.com/pion/ice/v2 v2.3.11 // indirect + github.com/pion/interceptor v0.1.25 // indirect github.com/pion/logging v0.2.2 // indirect - github.com/pion/mdns v0.0.7 // indirect + github.com/pion/mdns v0.0.9 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.10 // indirect - github.com/pion/rtp v1.7.13 // indirect - github.com/pion/sctp v1.8.7 // indirect + github.com/pion/rtcp v1.2.12 // indirect + github.com/pion/rtp v1.8.3 // indirect + github.com/pion/sctp v1.8.9 // indirect github.com/pion/sdp/v3 v3.0.6 // indirect - github.com/pion/srtp/v2 v2.0.15 // indirect - github.com/pion/stun v0.6.0 // indirect - github.com/pion/transport/v2 v2.2.1 // indirect - github.com/pion/turn/v2 v2.1.0 // indirect + github.com/pion/srtp/v2 v2.0.18 // indirect + github.com/pion/stun v0.6.1 // indirect + github.com/pion/transport/v2 v2.2.4 // indirect + github.com/pion/turn/v2 v2.1.3 // indirect github.com/pion/webrtc/v3 v3.2.9 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/test-plans/go.sum b/test-plans/go.sum index 32186695a6..83567334d4 100644 --- a/test-plans/go.sum +++ b/test-plans/go.sum @@ -194,8 +194,10 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -346,41 +348,58 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= -github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v2 v2.3.6 h1:Jgqw36cAud47iD+N6rNX225uHvrgWtAlHfVyOQc3Heg= +github.com/pion/dtls/v2 v2.2.8 h1:BUroldfiIbV9jSnC6cKOMnyiORRWrWWpV11JUyEu5OA= +github.com/pion/dtls/v2 v2.2.8/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/ice/v2 v2.3.6/go.mod h1:9/TzKDRwBVAPsC+YOrKH/e3xDrubeTRACU9/sHQarsU= -github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w= +github.com/pion/ice/v2 v2.3.11 h1:rZjVmUwyT55cmN8ySMpL7rsS8KYsJERsrxJLLxpKhdw= +github.com/pion/ice/v2 v2.3.11/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E= github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= +github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= +github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/mdns v0.0.7 h1:P0UB4Sr6xDWEox0kTVxF0LmQihtCbSAdW0H2nEgkA3U= github.com/pion/mdns v0.0.7/go.mod h1:4iP2UbeFhLI/vWju/bw6ZfwjJzk0z8DNValjGxR/dD8= +github.com/pion/mdns v0.0.8/go.mod h1:hYE72WX8WDveIhg7fmXgMKivD3Puklk0Ymzog0lSyaI= +github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4= +github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.10 h1:nkr3uj+8Sp97zyItdN60tE/S6vk4al5CPRR6Gejsdjc= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= -github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA= +github.com/pion/rtcp v1.2.12 h1:bKWiX93XKgDZENEXCijvHRU/wRifm6JV5DGcH6twtSM= +github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= +github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= +github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.7 h1:JnABvFakZueGAn4KU/4PSKg+GWbF6QWbKTWZOSGJjXw= github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= +github.com/pion/sctp v1.8.9 h1:TP5ZVxV5J7rz7uZmbyvnUvsn7EJ2x/5q9uhsTtXbI3g= +github.com/pion/sctp v1.8.9/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= -github.com/pion/srtp/v2 v2.0.15 h1:+tqRtXGsGwHC0G0IUIAzRmdkHvriF79IHVfZGfHrQoA= github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= +github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= +github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= -github.com/pion/stun v0.6.0 h1:JHT/2iyGDPrFWE8NNC15wnddBN8KifsEDw8swQmrEmU= github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= +github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= +github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= -github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= -github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= +github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++NjXLOUorc= +github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QAlo= +github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= +github.com/pion/turn/v2 v2.1.3 h1:pYxTVWG2gpC97opdRc5IGsQ1lJ9O/IlNhkzj7MMrGAA= +github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/webrtc/v3 v3.2.9 h1:U8NSjQDlZZ+Iy/hg42Q/u6mhEVSXYvKrOIZiZwYTfLc= github.com/pion/webrtc/v3 v3.2.9/go.mod h1:gjQLMZeyN3jXBGdxGmUYCyKjOuYX/c99BDjGqmadq0A= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -529,6 +548,10 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -617,6 +640,10 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -707,6 +734,10 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -717,6 +748,10 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -729,6 +764,10 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 046d450054388699ba4c6273b273e507649cf993 Mon Sep 17 00:00:00 2001 From: Sukun Date: Sat, 9 Dec 2023 20:38:45 +0530 Subject: [PATCH 09/30] better debug log --- p2p/test/transport/transport_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index e29b1562a0..c52690ad93 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -436,7 +436,7 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { func() { var didErr bool defer func() { - fmt.Println("completed", completedStreams.Add(1)) + t.Log("completed", completedStreams.Add(1)) }() defer func() { // Only the first worker adds more workers From 29ed5408c90411e2d6e3d971d53e241220670b17 Mon Sep 17 00:00:00 2001 From: Sukun Date: Tue, 12 Dec 2023 13:57:59 +0530 Subject: [PATCH 10/30] don't rely on pion for channel numbering --- p2p/test/transport/transport_test.go | 8 +-- p2p/transport/webrtc/connection.go | 85 ++++++++++++++++++---------- p2p/transport/webrtc/stream.go | 4 +- p2p/transport/webrtc/stream_write.go | 2 +- p2p/transport/webrtc/transport.go | 2 + 5 files changed, 65 insertions(+), 36 deletions(-) diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index c52690ad93..3b018d5747 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -9,6 +9,7 @@ import ( "io" "net" "runtime" + "strings" "sync" "sync/atomic" "testing" @@ -380,6 +381,9 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { const streamCount = 1024 for _, tc := range transportsToTest { t.Run(tc.Name, func(t *testing.T) { + if strings.Contains(tc.Name, "WebRTC") { + t.Skip("Pion doesn't correctly handle large queues of streams.") + } listenerLimits := rcmgr.PartialLimitConfig{ PeerDefault: rcmgr.ResourceLimits{ Streams: 32, @@ -417,7 +421,6 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { wg := sync.WaitGroup{} errCh := make(chan error, 1) - var completedStreams atomic.Int32 const maxWorkerCount = streamCount workerCount := 4 @@ -435,9 +438,6 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { // Inline function so we can use defer func() { var didErr bool - defer func() { - t.Log("completed", completedStreams.Add(1)) - }() defer func() { // Only the first worker adds more workers if workerIdx == 0 && !didErr && !sawFirstErr.Load() { diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index 3a10d959ca..06c52f47ec 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" + "math" "net" "sync" + "sync/atomic" ic "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/network" @@ -53,6 +55,10 @@ type connection struct { remoteKey ic.PubKey remoteMultiaddr ma.Multiaddr + m sync.Mutex + streams map[uint16]*stream + nextStreamID atomic.Int32 + acceptQueue chan dataChannel ctx context.Context @@ -88,13 +94,18 @@ func newConnection( cancel: cancel, acceptQueue: make(chan dataChannel, maxAcceptQueueLen), + streams: make(map[uint16]*stream), + } + switch direction { + case network.DirInbound: + c.nextStreamID.Store(1) + case network.DirOutbound: + // stream ID 0 is used for the Noise handshake stream + c.nextStreamID.Store(2) } pc.OnConnectionStateChange(c.onConnectionStateChange) pc.OnDataChannel(func(dc *webrtc.DataChannel) { - if c.IsClosed() { - return - } dc.OnOpen(func() { rwc, err := dc.Detach() if err != nil { @@ -132,44 +143,56 @@ func (c *connection) closeWithError(err error) { // cancel must be called after closeErr is set. This ensures interested goroutines waiting on // ctx.Done can read closeErr without holding the conn lock. c.cancel() + // closing peerconnection will close the datachannels associated with the streams c.pc.Close() -loop: - for { - select { - case <-c.acceptQueue: - default: - break loop - } - } c.scope.Done() } func (c *connection) IsClosed() bool { - select { - case <-c.ctx.Done(): - return true - default: - return false - } + return c.ctx.Err() != nil } func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error) { if c.IsClosed() { return nil, c.closeErr } - dc, err := c.pc.CreateDataChannel("", nil) + + id := c.nextStreamID.Add(2) - 2 + if id > math.MaxUint16 { + return nil, errors.New("exhausted stream ID space") + } + streamID := uint16(id) + dc, err := c.pc.CreateDataChannel("", &webrtc.DataChannelInit{ID: &streamID}) if err != nil { return nil, err } rwc, err := c.detachChannel(ctx, dc) if err != nil { - return nil, fmt.Errorf("open stream: %w", err) - } - if c.IsClosed() { dc.Close() - return nil, c.closeErr + return nil, fmt.Errorf("detach channel failed for stream(%d): %w", streamID, err) + } + str := newStream(dc, rwc, func() { c.removeStream(streamID) }) + if err := c.addStream(str); err != nil { + str.Reset() + return nil, fmt.Errorf("failed to add stream(%d) to connection: %w", streamID, err) + } + return str, nil +} + +func (c *connection) addStream(str *stream) error { + c.m.Lock() + defer c.m.Unlock() + if _, ok := c.streams[str.id]; ok { + return errors.New("stream ID already exists") } - return newStream(dc, rwc, nil), nil + c.streams[str.id] = str + return nil +} + +func (c *connection) removeStream(id uint16) { + c.m.Lock() + defer c.m.Unlock() + delete(c.streams, id) } func (c *connection) AcceptStream() (network.MuxedStream, error) { @@ -177,11 +200,12 @@ func (c *connection) AcceptStream() (network.MuxedStream, error) { case <-c.ctx.Done(): return nil, c.closeErr case dc := <-c.acceptQueue: - if c.IsClosed() { - dc.channel.Close() - return nil, c.closeErr + str := newStream(dc.channel, dc.stream, func() { c.removeStream(*dc.channel.ID()) }) + if err := c.addStream(str); err != nil { + str.Reset() + return nil, err } - return newStream(dc.channel, dc.stream, nil), nil + return str, nil } } @@ -218,8 +242,11 @@ func (c *connection) onConnectionStateChange(state webrtc.PeerConnectionState) { // This was desired because it was not feasible to introduce backpressure // with the OnMessage callbacks. The tradeoff is a change in the semantics of // the OnOpen callback, and having to force close Read locally. -func (c *connection) detachChannel(ctx context.Context, dc *webrtc.DataChannel) (rwc datachannel.ReadWriteCloser, err error) { +func (c *connection) detachChannel(ctx context.Context, dc *webrtc.DataChannel) (datachannel.ReadWriteCloser, error) { done := make(chan struct{}) + + var rwc datachannel.ReadWriteCloser + var err error // OnOpen will return immediately for detached datachannels // refer: https://github.com/pion/webrtc/blob/7ab3174640b3ce15abebc2516a2ca3939b5f105f/datachannel.go#L278-L282 dc.OnOpen(func() { @@ -233,8 +260,8 @@ func (c *connection) detachChannel(ctx context.Context, dc *webrtc.DataChannel) case <-ctx.Done(): return nil, ctx.Err() case <-done: + return rwc, err } - return } // A note on these setters and why they are needed: diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 04f8f66f15..4b8c9d9bb2 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -21,9 +21,9 @@ const ( // We can change this value in the SettingEngine before creating the peerconnection. // https://github.com/pion/webrtc/blob/v3.1.49/sctptransport.go#L341 maxBufferedAmount = 2 * maxMessageSize - // maxTotalControlMessageSize is the maximum of the total size of all control messages we will + // maxTotalControlMessagesSize is the maximum total size of all control messages we will // write on this stream - maxTotalControlMessageSize = 500 + maxTotalControlMessagesSize = 500 // bufferedAmountLowThreshold and maxBufferedAmount are bound // to a stream but congestion control is done on the whole // SCTP association. This means that a single stream can monopolize diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index 6b0e0b8d7b..52f3ce214f 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -117,7 +117,7 @@ func (s *stream) SetWriteDeadline(t time.Time) error { func (s *stream) availableSendSpace() int { buffered := int(s.dataChannel.BufferedAmount()) availableSpace := maxBufferedAmount - buffered - if availableSpace+maxTotalControlMessageSize < 0 { // this should never happen, but better check + if availableSpace+maxTotalControlMessagesSize < 0 { // this should never happen, but better check log.Errorw("data channel buffered more data than the maximum amount", "max", maxBufferedAmount, "buffered", buffered) } return availableSpace diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 2f8b0d5765..6f56a6b43d 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -534,6 +534,8 @@ type netConnWrapper struct { func (netConnWrapper) LocalAddr() net.Addr { return nil } func (netConnWrapper) RemoteAddr() net.Addr { return nil } func (w netConnWrapper) Close() error { + // Close called while running the security handshake is an error and we should Reset the + // stream in that case rather than gracefully closing w.stream.Reset() return nil } From 340d322b82885e318b0890d7a74425efbe8e2e25 Mon Sep 17 00:00:00 2001 From: Sukun Date: Tue, 12 Dec 2023 19:35:41 +0530 Subject: [PATCH 11/30] remove dependency on personal sctp fork --- core/network/mux.go | 7 ------- p2p/net/swarm/swarm_stream.go | 6 +++++- p2p/transport/webrtc/stream.go | 16 ++++++++-------- p2p/transport/webrtc/stream_read.go | 4 ++-- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/core/network/mux.go b/core/network/mux.go index fdda55365a..d12e2ea34b 100644 --- a/core/network/mux.go +++ b/core/network/mux.go @@ -61,13 +61,6 @@ type MuxedStream interface { SetWriteDeadline(time.Time) error } -// AsyncCloser is implemented by streams that need to do expensive operations on close before -// releasing the resources. Closing the stream async avoids blocking the calling goroutine. -type AsyncCloser interface { - // AsyncClose closes the stream and executes onDone after the stream is closed - AsyncClose(onDone func()) error -} - // MuxedConn represents a connection to a remote peer that has been // extended to support stream multiplexing. // diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 1339709db2..a56c2c91b4 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -75,10 +75,14 @@ func (s *Stream) Write(p []byte) (int, error) { return n, err } +type asyncCloser interface { + AsyncClose(onDone func()) error +} + // Close closes the stream, closing both ends and freeing all associated // resources. func (s *Stream) Close() error { - if as, ok := s.stream.(network.AsyncCloser); ok { + if as, ok := s.stream.(asyncCloser); ok { err := as.AsyncClose(func() { s.closeAndRemoveStream() }) diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 4b8c9d9bb2..b32e88de59 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -66,10 +66,10 @@ const ( type stream struct { mx sync.Mutex - // readerOnce ensures that only a single goroutine reads from the reader. Read is not threadsafe + // readerSem ensures that only a single goroutine reads from the reader. Read is not threadsafe // But we may need to read from reader for control messages from a different goroutine. - readerOnce chan struct{} - reader pbio.Reader + readerSem chan struct{} + reader pbio.Reader // this buffer is limited up to a single message. Reason we need it // is because a reader might read a message midway, and so we need a @@ -108,9 +108,9 @@ func newStream( onDone func(), ) *stream { s := &stream{ - readerOnce: make(chan struct{}, 1), - reader: pbio.NewDelimitedReader(rwc, maxMessageSize), - writer: pbio.NewDelimitedWriter(rwc), + readerSem: make(chan struct{}, 1), + reader: pbio.NewDelimitedReader(rwc, maxMessageSize), + writer: pbio.NewDelimitedWriter(rwc), sendStateChanged: make(chan struct{}, 1), writeDeadlineUpdated: make(chan struct{}, 1), @@ -262,8 +262,8 @@ func (s *stream) spawnControlMessageReader() { // Unblock any Read call waiting on reader.ReadMsg s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) // We have the lock, any waiting reader has exited. - s.readerOnce <- struct{}{} - <-s.readerOnce + s.readerSem <- struct{}{} + <-s.readerSem // From this point onwards only this goroutine can do reader.ReadMsg s.mx.Lock() diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index 1ad171564c..d4fcc8bf5e 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -9,8 +9,8 @@ import ( ) func (s *stream) Read(b []byte) (int, error) { - s.readerOnce <- struct{}{} - defer func() { <-s.readerOnce }() + s.readerSem <- struct{}{} + defer func() { <-s.readerSem }() s.mx.Lock() defer s.mx.Unlock() From 1e39ac5840644ef8fed701b7e1658ebab3dea5a9 Mon Sep 17 00:00:00 2001 From: Sukun Date: Tue, 12 Dec 2023 20:14:25 +0530 Subject: [PATCH 12/30] remove unimportant changes --- p2p/test/transport/transport_test.go | 6 ++--- p2p/transport/webrtc/connection.go | 34 ++++++++++++------------ p2p/transport/webrtc/datachannel.go | 30 --------------------- p2p/transport/webrtc/listener.go | 10 +++---- p2p/transport/webrtc/stream.go | 7 +++-- p2p/transport/webrtc/transport.go | 36 +++++++++++++++++++------- p2p/transport/webrtc/transport_test.go | 5 +--- 7 files changed, 56 insertions(+), 72 deletions(-) delete mode 100644 p2p/transport/webrtc/datachannel.go diff --git a/p2p/test/transport/transport_test.go b/p2p/test/transport/transport_test.go index 3b018d5747..bfbd2c0bce 100644 --- a/p2p/test/transport/transport_test.go +++ b/p2p/test/transport/transport_test.go @@ -280,7 +280,6 @@ func TestLotsOfDataManyStreams(t *testing.T) { sem := make(chan struct{}, parallel) var wg sync.WaitGroup - var done atomic.Int32 for i := 0; i < totalStreams; i++ { wg.Add(1) sem <- struct{}{} @@ -303,7 +302,6 @@ func TestLotsOfDataManyStreams(t *testing.T) { _, err = s.Read([]byte{0}) require.ErrorIs(t, err, io.EOF) - fmt.Println("done", done.Add(1)) }() } @@ -382,7 +380,7 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { for _, tc := range transportsToTest { t.Run(tc.Name, func(t *testing.T) { if strings.Contains(tc.Name, "WebRTC") { - t.Skip("Pion doesn't correctly handle large queues of streams.") + t.Skip("This test potentially exhausts the uint16 WebRTC stream ID space.") } listenerLimits := rcmgr.PartialLimitConfig{ PeerDefault: rcmgr.ResourceLimits{ @@ -421,6 +419,7 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { wg := sync.WaitGroup{} errCh := make(chan error, 1) + var completedStreams atomic.Int32 const maxWorkerCount = streamCount workerCount := 4 @@ -438,6 +437,7 @@ func TestMoreStreamsThanOurLimits(t *testing.T) { // Inline function so we can use defer func() { var didErr bool + defer completedStreams.Add(1) defer func() { // Only the first worker adds more workers if workerIdx == 0 && !didErr && !sawFirstErr.Load() { diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index 06c52f47ec..e23975f4da 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -92,9 +92,9 @@ func newConnection( remoteMultiaddr: remoteMultiaddr, ctx: ctx, cancel: cancel, + streams: make(map[uint16]*stream), acceptQueue: make(chan dataChannel, maxAcceptQueueLen), - streams: make(map[uint16]*stream), } switch direction { case network.DirInbound: @@ -179,22 +179,6 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error return str, nil } -func (c *connection) addStream(str *stream) error { - c.m.Lock() - defer c.m.Unlock() - if _, ok := c.streams[str.id]; ok { - return errors.New("stream ID already exists") - } - c.streams[str.id] = str - return nil -} - -func (c *connection) removeStream(id uint16) { - c.m.Lock() - defer c.m.Unlock() - delete(c.streams, id) -} - func (c *connection) AcceptStream() (network.MuxedStream, error) { select { case <-c.ctx.Done(): @@ -217,6 +201,22 @@ func (c *connection) RemoteMultiaddr() ma.Multiaddr { return c.remoteMultiaddr } func (c *connection) Scope() network.ConnScope { return c.scope } func (c *connection) Transport() tpt.Transport { return c.transport } +func (c *connection) addStream(str *stream) error { + c.m.Lock() + defer c.m.Unlock() + if _, ok := c.streams[str.id]; ok { + return errors.New("stream ID already exists") + } + c.streams[str.id] = str + return nil +} + +func (c *connection) removeStream(id uint16) { + c.m.Lock() + defer c.m.Unlock() + delete(c.streams, id) +} + func (c *connection) onConnectionStateChange(state webrtc.PeerConnectionState) { if state == webrtc.PeerConnectionStateFailed || state == webrtc.PeerConnectionStateClosed { c.closeOnce.Do(func() { diff --git a/p2p/transport/webrtc/datachannel.go b/p2p/transport/webrtc/datachannel.go deleted file mode 100644 index b23c33ef27..0000000000 --- a/p2p/transport/webrtc/datachannel.go +++ /dev/null @@ -1,30 +0,0 @@ -package libp2pwebrtc - -import ( - "context" - - "github.com/pion/datachannel" - "github.com/pion/webrtc/v3" -) - -// only use this if the datachannels are detached, since the OnOpen callback -// will be called immediately. Only use after the peerconnection is open. -// The context should close if the peerconnection underlying the datachannel -// is closed. -func getDetachedChannel(ctx context.Context, dc *webrtc.DataChannel) (datachannel.ReadWriteCloser, error) { - done := make(chan struct{}) - var rwc datachannel.ReadWriteCloser - var err error - dc.OnOpen(func() { - defer close(done) - rwc, err = dc.Detach() - }) - // this is safe since for detached datachannels, the peerconnection runs the onOpen - // callback immediately if the SCTP transport is also connected. - select { - case <-done: - return rwc, err - case <-ctx.Done(): - return nil, ctx.Err() - } -} diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index 1b365af202..c7c3ba401d 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -265,15 +265,14 @@ func (l *listener) setupConnection( } } - rwc, err := getDetachedChannel(ctx, rawDatachannel) + rwc, err := detachHandshakeDataChannel(ctx, rawDatachannel) if err != nil { return nil, err } localMultiaddrWithoutCerthash, _ := ma.SplitFunc(l.localMultiaddr, func(c ma.Component) bool { return c.Protocol().Code == ma.P_CERTHASH }) - s := newStream(rawDatachannel, rwc, func() {}) - + handshakeChannel := newStream(rawDatachannel, rwc, func() {}) // The connection is instantiated before performing the Noise handshake. This is // to handle the case where the remote is faster and attempts to initiate a stream // before the ondatachannel callback can be set. @@ -293,20 +292,17 @@ func (l *listener) setupConnection( } // we do not yet know A's peer ID so accept any inbound - remotePubKey, err := l.transport.noiseHandshake(ctx, conn, s, "", crypto.SHA256, true) + remotePubKey, err := l.transport.noiseHandshake(ctx, pc, handshakeChannel, "", crypto.SHA256, true) if err != nil { - conn.Close() return nil, err } remotePeer, err := peer.IDFromPublicKey(remotePubKey) if err != nil { - conn.Close() return nil, err } // earliest point where we know the remote's peerID if err := scope.SetPeer(remotePeer); err != nil { - conn.Close() return nil, err } diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index b32e88de59..747668764a 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -42,6 +42,9 @@ const ( // is less than or equal to 2 ^ 14, the varint will not be more than // 2 bytes in length. varintOverhead = 2 + // maxFINACKWait is the maximum amount of time a stream will wait to read + // FIN_ACK before closing the data channel + maxFINACKWait = 10 * time.Second ) type receiveState uint8 @@ -144,7 +147,7 @@ func (s *stream) Close() error { } s.mx.Lock() - s.controlMessageReaderEndTime = time.Now().Add(10 * time.Second) + s.controlMessageReaderEndTime = time.Now().Add(maxFINACKWait) s.mx.Unlock() s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) <-s.controlMessageReaderDone @@ -162,7 +165,7 @@ func (s *stream) AsyncClose(onDone func()) error { return errors.Join(closeWriteErr, closeReadErr) } s.mx.Lock() - s.controlMessageReaderEndTime = time.Now().Add(10 * time.Second) + s.controlMessageReaderEndTime = time.Now().Add(maxFINACKWait) s.mx.Unlock() s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) go func() { diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index 6f56a6b43d..dab5d3efb0 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -42,6 +42,7 @@ import ( manet "github.com/multiformats/go-multiaddr/net" "github.com/multiformats/go-multihash" + "github.com/pion/datachannel" pionlogger "github.com/pion/logging" "github.com/pion/webrtc/v3" ) @@ -331,7 +332,7 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement // We need to set negotiated = true for this channel on both // the client and server to avoid DCEP errors. negotiated, id := handshakeChannelNegotiated, handshakeChannelID - handshakeChannel, err := pc.CreateDataChannel("", &webrtc.DataChannelInit{ + rawHandshakeChannel, err := pc.CreateDataChannel("", &webrtc.DataChannelInit{ Negotiated: &negotiated, ID: &id, }) @@ -371,12 +372,12 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement return nil, errors.New("peerconnection opening timed out") } - detached, err := getDetachedChannel(ctx, handshakeChannel) + detached, err := detachHandshakeDataChannel(ctx, rawHandshakeChannel) if err != nil { return nil, err } // set the local address from the candidate pair - cp, err := handshakeChannel.Transport().Transport().ICETransport().GetSelectedCandidatePair() + cp, err := rawHandshakeChannel.Transport().Transport().ICETransport().GetSelectedCandidatePair() if cp == nil { return nil, errors.New("ice connection did not have selected candidate pair: nil result") } @@ -384,7 +385,7 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement return nil, fmt.Errorf("ice connection did not have selected candidate pair: error: %w", err) } - s := newStream(handshakeChannel, detached, func() {}) + channel := newStream(rawHandshakeChannel, detached, func() {}) // the local address of the selected candidate pair should be the // local address for the connection, since different datachannels // are multiplexed over the same SCTP connection @@ -412,13 +413,11 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement return nil, err } - remotePubKey, err := t.noiseHandshake(ctx, conn, s, p, remoteHashFunction, false) + remotePubKey, err := t.noiseHandshake(ctx, pc, channel, p, remoteHashFunction, false) if err != nil { - conn.Close() return nil, err } if t.gater != nil && !t.gater.InterceptSecured(network.DirOutbound, p, conn) { - conn.Close() return nil, fmt.Errorf("secured connection gated") } conn.setRemotePublicKey(remotePubKey) @@ -498,8 +497,8 @@ func (t *WebRTCTransport) generateNoisePrologue(pc *webrtc.PeerConnection, hash return result, nil } -func (t *WebRTCTransport) noiseHandshake(ctx context.Context, c *connection, s *stream, peer peer.ID, hash crypto.Hash, inbound bool) (ic.PubKey, error) { - prologue, err := t.generateNoisePrologue(c.pc, hash, inbound) +func (t *WebRTCTransport) noiseHandshake(ctx context.Context, pc *webrtc.PeerConnection, s *stream, peer peer.ID, hash crypto.Hash, inbound bool) (ic.PubKey, error) { + prologue, err := t.generateNoisePrologue(pc, hash, inbound) if err != nil { return nil, fmt.Errorf("generate prologue: %w", err) } @@ -539,3 +538,22 @@ func (w netConnWrapper) Close() error { w.stream.Reset() return nil } + +// detachHandshakeDataChannel detaches the handshake data channel +func detachHandshakeDataChannel(ctx context.Context, dc *webrtc.DataChannel) (datachannel.ReadWriteCloser, error) { + done := make(chan struct{}) + var rwc datachannel.ReadWriteCloser + var err error + dc.OnOpen(func() { + defer close(done) + rwc, err = dc.Detach() + }) + // this is safe since for detached datachannels, the peerconnection runs the onOpen + // callback immediately if the SCTP transport is also connected. + select { + case <-done: + return rwc, err + case <-ctx.Done(): + return nil, ctx.Err() + } +} diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index b691dd733f..b831d52f08 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -316,7 +316,6 @@ func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { require.Equal(t, connectingPeer, lconn.RemotePeer()) var wg sync.WaitGroup var doneStreams atomic.Int32 - var concurrency atomic.Int32 for i := 0; i < numListeners; i++ { wg.Add(1) go func() { @@ -327,13 +326,11 @@ func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { return } s, err := lconn.AcceptStream() - fmt.Println("concurrency", concurrency.Add(1)) require.NoError(t, err) n, err := io.Copy(s, s) require.Equal(t, n, int64(size)) require.NoError(t, err) s.Close() - concurrency.Add(-1) } }() } @@ -375,7 +372,7 @@ func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { t.Errorf("bytes not equal: %d %d", len(buf), len(resp)) } s.Close() - fmt.Println("completed stream: ", cnt.Add(1), s.(*stream).id) + t.Log("completed stream: ", cnt.Add(1), s.(*stream).id) } }() } From b8578f22a43af07c6656ea3b34db088fbaee6a7f Mon Sep 17 00:00:00 2001 From: Sukun Date: Tue, 12 Dec 2023 20:16:57 +0530 Subject: [PATCH 13/30] update pion --- go.mod | 6 +++--- go.sum | 33 +++++++-------------------------- test-plans/go.mod | 6 +++--- test-plans/go.sum | 33 +++++++-------------------------- 4 files changed, 20 insertions(+), 58 deletions(-) diff --git a/go.mod b/go.mod index 5e26fb6ce1..9f5cf52823 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/pion/ice/v2 v2.3.11 github.com/pion/logging v0.2.2 github.com/pion/stun v0.6.1 - github.com/pion/webrtc/v3 v3.2.9 + github.com/pion/webrtc/v3 v3.2.23 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.4.0 github.com/quic-go/quic-go v0.39.3 @@ -105,13 +105,13 @@ require ( github.com/pion/interceptor v0.1.25 // indirect github.com/pion/mdns v0.0.9 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.12 // indirect + github.com/pion/rtcp v1.2.13 // indirect github.com/pion/rtp v1.8.3 // indirect github.com/pion/sctp v1.8.9 // indirect github.com/pion/sdp/v3 v3.0.6 // indirect github.com/pion/srtp/v2 v2.0.18 // indirect github.com/pion/transport/v2 v2.2.4 // indirect - github.com/pion/turn/v2 v2.1.3 // indirect + github.com/pion/turn/v2 v2.1.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.37.0 // indirect diff --git a/go.sum b/go.sum index 01bef9515e..b791175a1f 100644 --- a/go.sum +++ b/go.sum @@ -212,7 +212,6 @@ github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVe github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -400,45 +399,36 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.8 h1:BUroldfiIbV9jSnC6cKOMnyiORRWrWWpV11JUyEu5OA= github.com/pion/dtls/v2 v2.2.8/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v2 v2.3.6/go.mod h1:9/TzKDRwBVAPsC+YOrKH/e3xDrubeTRACU9/sHQarsU= github.com/pion/ice/v2 v2.3.11 h1:rZjVmUwyT55cmN8ySMpL7rsS8KYsJERsrxJLLxpKhdw= github.com/pion/ice/v2 v2.3.11/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E= -github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/mdns v0.0.7/go.mod h1:4iP2UbeFhLI/vWju/bw6ZfwjJzk0z8DNValjGxR/dD8= github.com/pion/mdns v0.0.8/go.mod h1:hYE72WX8WDveIhg7fmXgMKivD3Puklk0Ymzog0lSyaI= github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4= github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= -github.com/pion/rtcp v1.2.12 h1:bKWiX93XKgDZENEXCijvHRU/wRifm6JV5DGcH6twtSM= github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= -github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= +github.com/pion/rtcp v1.2.13 h1:+EQijuisKwm/8VBs8nWllr0bIndR7Lf7cZG200mpbNo= +github.com/pion/rtcp v1.2.13/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= +github.com/pion/sctp v1.8.8/go.mod h1:igF9nZBrjh5AtmKc7U30jXltsFHicFCXSmWA2GWRaWs= github.com/pion/sctp v1.8.9 h1:TP5ZVxV5J7rz7uZmbyvnUvsn7EJ2x/5q9uhsTtXbI3g= github.com/pion/sctp v1.8.9/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= -github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= -github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= -github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= -github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= -github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= -github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++NjXLOUorc= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= @@ -446,11 +436,11 @@ github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QA github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= -github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= -github.com/pion/turn/v2 v2.1.3 h1:pYxTVWG2gpC97opdRc5IGsQ1lJ9O/IlNhkzj7MMrGAA= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.9 h1:U8NSjQDlZZ+Iy/hg42Q/u6mhEVSXYvKrOIZiZwYTfLc= -github.com/pion/webrtc/v3 v3.2.9/go.mod h1:gjQLMZeyN3jXBGdxGmUYCyKjOuYX/c99BDjGqmadq0A= +github.com/pion/turn/v2 v2.1.4 h1:2xn8rduI5W6sCZQkEnIUDAkrBQNl2eYIBCHMZ3QMmP8= +github.com/pion/turn/v2 v2.1.4/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/webrtc/v3 v3.2.23 h1:GbqEuxBbVLFhXk0GwxKAoaIJYiEa9TyoZPEZC+2HZxM= +github.com/pion/webrtc/v3 v3.2.23/go.mod h1:1CaT2fcZzZ6VZA+O1i9yK2DU4EOcXVvSbWG9pr5jefs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -552,7 +542,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -614,7 +603,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= @@ -704,9 +692,7 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= @@ -802,7 +788,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -816,9 +801,7 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= @@ -833,9 +816,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= diff --git a/test-plans/go.mod b/test-plans/go.mod index d2d188951b..1baf299f67 100644 --- a/test-plans/go.mod +++ b/test-plans/go.mod @@ -84,15 +84,15 @@ require ( github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.9 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.12 // indirect + github.com/pion/rtcp v1.2.13 // indirect github.com/pion/rtp v1.8.3 // indirect github.com/pion/sctp v1.8.9 // indirect github.com/pion/sdp/v3 v3.0.6 // indirect github.com/pion/srtp/v2 v2.0.18 // indirect github.com/pion/stun v0.6.1 // indirect github.com/pion/transport/v2 v2.2.4 // indirect - github.com/pion/turn/v2 v2.1.3 // indirect - github.com/pion/webrtc/v3 v3.2.9 // indirect + github.com/pion/turn/v2 v2.1.4 // indirect + github.com/pion/webrtc/v3 v3.2.23 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect diff --git a/test-plans/go.sum b/test-plans/go.sum index 83567334d4..7237c4ac64 100644 --- a/test-plans/go.sum +++ b/test-plans/go.sum @@ -194,7 +194,6 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -351,45 +350,36 @@ github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.8 h1:BUroldfiIbV9jSnC6cKOMnyiORRWrWWpV11JUyEu5OA= github.com/pion/dtls/v2 v2.2.8/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v2 v2.3.6/go.mod h1:9/TzKDRwBVAPsC+YOrKH/e3xDrubeTRACU9/sHQarsU= github.com/pion/ice/v2 v2.3.11 h1:rZjVmUwyT55cmN8ySMpL7rsS8KYsJERsrxJLLxpKhdw= github.com/pion/ice/v2 v2.3.11/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E= -github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/mdns v0.0.7/go.mod h1:4iP2UbeFhLI/vWju/bw6ZfwjJzk0z8DNValjGxR/dD8= github.com/pion/mdns v0.0.8/go.mod h1:hYE72WX8WDveIhg7fmXgMKivD3Puklk0Ymzog0lSyaI= github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4= github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= -github.com/pion/rtcp v1.2.12 h1:bKWiX93XKgDZENEXCijvHRU/wRifm6JV5DGcH6twtSM= github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= -github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= +github.com/pion/rtcp v1.2.13 h1:+EQijuisKwm/8VBs8nWllr0bIndR7Lf7cZG200mpbNo= +github.com/pion/rtcp v1.2.13/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= +github.com/pion/sctp v1.8.8/go.mod h1:igF9nZBrjh5AtmKc7U30jXltsFHicFCXSmWA2GWRaWs= github.com/pion/sctp v1.8.9 h1:TP5ZVxV5J7rz7uZmbyvnUvsn7EJ2x/5q9uhsTtXbI3g= github.com/pion/sctp v1.8.9/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= -github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= -github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= -github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= -github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= -github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= -github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++NjXLOUorc= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= @@ -397,11 +387,11 @@ github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QA github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= -github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= -github.com/pion/turn/v2 v2.1.3 h1:pYxTVWG2gpC97opdRc5IGsQ1lJ9O/IlNhkzj7MMrGAA= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.9 h1:U8NSjQDlZZ+Iy/hg42Q/u6mhEVSXYvKrOIZiZwYTfLc= -github.com/pion/webrtc/v3 v3.2.9/go.mod h1:gjQLMZeyN3jXBGdxGmUYCyKjOuYX/c99BDjGqmadq0A= +github.com/pion/turn/v2 v2.1.4 h1:2xn8rduI5W6sCZQkEnIUDAkrBQNl2eYIBCHMZ3QMmP8= +github.com/pion/turn/v2 v2.1.4/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/webrtc/v3 v3.2.23 h1:GbqEuxBbVLFhXk0GwxKAoaIJYiEa9TyoZPEZC+2HZxM= +github.com/pion/webrtc/v3 v3.2.23/go.mod h1:1CaT2fcZzZ6VZA+O1i9yK2DU4EOcXVvSbWG9pr5jefs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -496,7 +486,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -547,7 +536,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= @@ -635,9 +623,7 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= @@ -729,7 +715,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -743,9 +728,7 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= @@ -760,9 +743,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= From acf1687cb85595753b2a4f7f6bdd6487044421e3 Mon Sep 17 00:00:00 2001 From: sukun Date: Sat, 6 Jan 2024 20:15:44 +0530 Subject: [PATCH 14/30] add tests for chunking --- p2p/transport/webrtc/stream.go | 18 ++++--- p2p/transport/webrtc/stream_test.go | 77 +++++++++++++++++++++++++--- p2p/transport/webrtc/stream_write.go | 5 +- 3 files changed, 83 insertions(+), 17 deletions(-) diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 747668764a..8fc34c2531 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -94,7 +94,6 @@ type stream struct { // SetReadDeadline // See: https://github.com/pion/sctp/pull/290 controlMessageReaderEndTime time.Time - controlMessageReaderStarted chan struct{} controlMessageReaderDone chan struct{} onDone func() @@ -119,8 +118,7 @@ func newStream( writeDeadlineUpdated: make(chan struct{}, 1), writeAvailable: make(chan struct{}, 1), - controlMessageReaderStarted: make(chan struct{}), - controlMessageReaderDone: make(chan struct{}), + controlMessageReaderDone: make(chan struct{}), id: *channel.ID(), dataChannel: rwc.(*datachannel.DataChannel), @@ -168,6 +166,7 @@ func (s *stream) AsyncClose(onDone func()) error { s.controlMessageReaderEndTime = time.Now().Add(maxFINACKWait) s.mx.Unlock() s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) + go func() { <-s.controlMessageReaderDone s.cleanup() @@ -253,21 +252,23 @@ func (s *stream) spawnControlMessageReader() { setDeadline := func() bool { s.mx.Lock() + defer s.mx.Unlock() if s.controlMessageReaderEndTime.IsZero() || time.Now().Before(s.controlMessageReaderEndTime) { s.SetReadDeadline(s.controlMessageReaderEndTime) - s.mx.Unlock() return true } - s.mx.Unlock() return false } // Unblock any Read call waiting on reader.ReadMsg s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) - // We have the lock, any waiting reader has exited. s.readerSem <- struct{}{} + // We have the lock any readers blocked on reader.ReadMsg have exited. + // From this point onwards only this goroutine will do reader.ReadMsg. + + // Release the semaphore because calls to stream.Read will return immediately + // since the read half of the stream is closed. <-s.readerSem - // From this point onwards only this goroutine can do reader.ReadMsg s.mx.Lock() if s.nextMessage != nil { @@ -282,6 +283,9 @@ func (s *stream) spawnControlMessageReader() { return } if err := s.reader.ReadMsg(&msg); err != nil { + // We have to manually manage deadline exceeded errors since pion/sctp can + // return deadline exceeded error for cancelled deadlines + // see: https://github.com/pion/sctp/pull/290/files if errors.Is(err, os.ErrDeadlineExceeded) { continue } diff --git a/p2p/transport/webrtc/stream_test.go b/p2p/transport/webrtc/stream_test.go index 851171c2cc..34655e4358 100644 --- a/p2p/transport/webrtc/stream_test.go +++ b/p2p/transport/webrtc/stream_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/libp2p/go-libp2p/p2p/transport/webrtc/pb" + "github.com/libp2p/go-msgio/pbio" "github.com/libp2p/go-libp2p/core/network" @@ -33,13 +34,13 @@ func getDetachedDataChannels(t *testing.T) (detachedChan, detachedChan) { offerPC, err := api.NewPeerConnection(webrtc.Configuration{}) require.NoError(t, err) t.Cleanup(func() { offerPC.Close() }) - offerRWCChan := make(chan datachannel.ReadWriteCloser, 1) + offerRWCChan := make(chan detachedChan, 1) offerDC, err := offerPC.CreateDataChannel("data", nil) require.NoError(t, err) offerDC.OnOpen(func() { rwc, err := offerDC.Detach() require.NoError(t, err) - offerRWCChan <- rwc + offerRWCChan <- detachedChan{rwc: rwc, dc: offerDC} }) answerPC, err := api.NewPeerConnection(webrtc.Configuration{}) @@ -94,7 +95,7 @@ func getDetachedDataChannels(t *testing.T) (detachedChan, detachedChan) { require.NoError(t, offerPC.SetRemoteDescription(answer)) require.NoError(t, answerPC.SetLocalDescription(answer)) - return <-answerChan, detachedChan{rwc: <-offerRWCChan, dc: offerDC} + return <-answerChan, <-offerRWCChan } func TestStreamSimpleReadWriteClose(t *testing.T) { @@ -138,10 +139,10 @@ func TestStreamSimpleReadWriteClose(t *testing.T) { // stream is only cleaned up on calling Close or AsyncClose or Reset clientStr.AsyncClose(nil) serverStr.AsyncClose(nil) - require.Eventually(t, func() bool { return clientDone.Load() }, 10*time.Second, 100*time.Millisecond) + require.Eventually(t, func() bool { return clientDone.Load() }, 5*time.Second, 100*time.Millisecond) // Need to call Close for cleanup. Otherwise the FIN_ACK is never read require.NoError(t, serverStr.Close()) - require.Eventually(t, func() bool { return serverDone.Load() }, 10*time.Second, 100*time.Millisecond) + require.Eventually(t, func() bool { return serverDone.Load() }, 5*time.Second, 100*time.Millisecond) } func TestStreamPartialReads(t *testing.T) { @@ -377,6 +378,8 @@ func TestStreamCloseAfterFINACK(t *testing.T) { } } +// TestStreamFinAckAfterStopSending tests that FIN_ACK is sent even after the write half +// of the stream is closed. func TestStreamFinAckAfterStopSending(t *testing.T) { client, server := getDetachedDataChannels(t) @@ -399,8 +402,8 @@ func TestStreamFinAckAfterStopSending(t *testing.T) { case <-time.After(500 * time.Millisecond): } - // serverStr has write half of the stream closed but the read half should - // respond correctly + // serverStr has write half closed and read half open + // serverStr should still send FIN_ACK b := make([]byte, 24) _, err := serverStr.Read(b) require.NoError(t, err) @@ -415,7 +418,7 @@ func TestStreamFinAckAfterStopSending(t *testing.T) { func TestStreamConcurrentClose(t *testing.T) { client, server := getDetachedDataChannels(t) - start := make(chan bool, 1) + start := make(chan bool, 2) done := make(chan bool, 2) clientStr := newStream(client.dc, client.rwc, func() { done <- true }) serverStr := newStream(server.dc, server.rwc, func() { done <- true }) @@ -463,3 +466,61 @@ func TestStreamResetAfterAsyncClose(t *testing.T) { t.Fatalf("Reset should run callback immediately") } } + +func TestStreamDataChannelCloseOnFINACK(t *testing.T) { + client, server := getDetachedDataChannels(t) + + done := make(chan bool, 1) + clientStr := newStream(client.dc, client.rwc, func() { done <- true }) + + clientStr.AsyncClose(nil) + + select { + case <-done: + t.Fatalf("AsyncClose shouldn't run cleanup immediately") + case <-time.After(500 * time.Millisecond): + } + + serverWriter := pbio.NewDelimitedWriter(server.rwc) + err := serverWriter.WriteMsg(&pb.Message{Flag: pb.Message_FIN_ACK.Enum()}) + require.NoError(t, err) + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("Callback should be run on reading FIN_ACK") + } + b := make([]byte, 100) + N := 0 + for { + n, err := server.rwc.Read(b) + N += n + if err != nil { + require.ErrorIs(t, err, io.EOF) + break + } + } + require.Less(t, N, 10) +} + +func TestStreamChunking(t *testing.T) { + client, server := getDetachedDataChannels(t) + + clientStr := newStream(client.dc, client.rwc, func() {}) + serverStr := newStream(server.dc, server.rwc, func() {}) + + const N = (16 << 10) + 1000 + go func() { + data := make([]byte, N) + _, err := clientStr.Write(data) + require.NoError(t, err) + }() + + data := make([]byte, N) + n, err := serverStr.Read(data) + require.NoError(t, err) + require.LessOrEqual(t, n, 16<<10) + + nn, err := serverStr.Read(data) + require.NoError(t, err) + require.Equal(t, nn+n, N) +} diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index 52f3ce214f..6515aba8ff 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -127,8 +127,9 @@ func (s *stream) cancelWrite() error { s.mx.Lock() defer s.mx.Unlock() - // Don't wait for FIN_ACK on reset - if s.sendState != sendStateSending && s.sendState != sendStateDataSent { + // There's no need to reset the write half if the write half has been closed + // successfully or has been reset previously + if s.sendState == sendStateDataReceived || s.sendState == sendStateReset { return nil } s.sendState = sendStateReset From cffc6a80c6f4a312f9f68602773678697f2128ab Mon Sep 17 00:00:00 2001 From: sukun Date: Tue, 9 Jan 2024 18:32:22 +0530 Subject: [PATCH 15/30] use default MTU --- p2p/transport/webrtc/listener.go | 1 - p2p/transport/webrtc/transport.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/p2p/transport/webrtc/listener.go b/p2p/transport/webrtc/listener.go index c7c3ba401d..7848bd5c6f 100644 --- a/p2p/transport/webrtc/listener.go +++ b/p2p/transport/webrtc/listener.go @@ -222,7 +222,6 @@ func (l *listener) setupConnection( l.transport.peerConnectionTimeouts.Failed, l.transport.peerConnectionTimeouts.Keepalive, ) - settingEngine.SetReceiveMTU(receiveMTUBytes) settingEngine.DetachDataChannels() api := webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine)) diff --git a/p2p/transport/webrtc/transport.go b/p2p/transport/webrtc/transport.go index dab5d3efb0..6102669dd9 100644 --- a/p2p/transport/webrtc/transport.go +++ b/p2p/transport/webrtc/transport.go @@ -77,7 +77,6 @@ const ( // timeout values for the peerconnection // https://github.com/pion/webrtc/blob/v3.1.50/settingengine.go#L102-L109 const ( - receiveMTUBytes = 1500 DefaultDisconnectedTimeout = 20 * time.Second DefaultFailedTimeout = 30 * time.Second DefaultKeepaliveTimeout = 15 * time.Second @@ -319,7 +318,6 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement // If you run pion on a system with only the loopback interface UP, // it will not connect to anything. settingEngine.SetIncludeLoopbackCandidate(true) - settingEngine.SetReceiveMTU(receiveMTUBytes) api := webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine)) From f6b86de31cc46332c4663045ae9f3761626bc2ec Mon Sep 17 00:00:00 2001 From: sukun Date: Tue, 9 Jan 2024 18:56:36 +0530 Subject: [PATCH 16/30] improve test for swarm stream --- p2p/net/swarm/swarm_stream.go | 6 +++ p2p/net/swarm/swarm_stream_test.go | 69 +++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index a56c2c91b4..2b6f07c92a 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -75,6 +75,12 @@ func (s *Stream) Write(p []byte) (int, error) { return n, err } +// asyncCloser interface is implemented by transports that need to do expensive cleanup work +// on stream close. Currently only WebRTC transport needs this. This interface is kept private +// to swarm to discourage other transport implementations from using this mechanism. It is only +// required because WebRTC datachannels have no Close functionality that would send the flush +// the enqueued data. +// see: https://github.com/libp2p/specs/issues/575 type asyncCloser interface { AsyncClose(onDone func()) error } diff --git a/p2p/net/swarm/swarm_stream_test.go b/p2p/net/swarm/swarm_stream_test.go index 653489fe8f..3fe47f4a07 100644 --- a/p2p/net/swarm/swarm_stream_test.go +++ b/p2p/net/swarm/swarm_stream_test.go @@ -2,8 +2,9 @@ package swarm import ( "context" - "sync/atomic" + "sync" "testing" + "time" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peerstore" @@ -13,12 +14,16 @@ import ( type asyncStreamWrapper struct { network.MuxedStream beforeClose func() + done chan bool } func (s *asyncStreamWrapper) AsyncClose(onDone func()) error { s.beforeClose() err := s.Close() - onDone() + go func() { + <-s.done + onDone() + }() return err } @@ -27,19 +32,51 @@ func TestStreamAsyncCloser(t *testing.T) { s2 := makeSwarm(t) s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.TempAddrTTL) - s, err := s1.NewStream(context.Background(), s2.LocalPeer()) - require.NoError(t, err) - ss, ok := s.(*Stream) - require.True(t, ok) - - var called atomic.Bool - as := &asyncStreamWrapper{ - MuxedStream: ss.stream, - beforeClose: func() { - called.Store(true) - }, + + var mx sync.Mutex + var wg1, wg2 sync.WaitGroup + var closed int + done := make(chan bool) + const N = 100 + wg1.Add(N) + // wg2 blocks all goroutines in the beforeClose method. This allows us to check GetStreams + // works concurrently with Close + wg2.Add(N) + for i := 0; i < N; i++ { + go func() { + s, err := s1.NewStream(context.Background(), s2.LocalPeer()) + require.NoError(t, err) + ss, ok := s.(*Stream) + require.True(t, ok) + as := &asyncStreamWrapper{ + MuxedStream: ss.stream, + beforeClose: func() { + wg2.Done() + mx.Lock() + defer mx.Unlock() + closed++ + wg1.Done() + }, + done: done, + } + ss.stream = as + ss.Close() + }() + } + wg2.Wait() + require.Eventually(t, func() bool { return s1.Connectedness(s2.LocalPeer()) == network.Connected }, + 5*time.Second, 100*time.Millisecond) + require.Equal(t, len(s1.ConnsToPeer(s2.LocalPeer())[0].GetStreams()), N) + + wg1.Wait() + require.Equal(t, closed, N) + // Streams should only be removed from the connection after the onDone call back is executed + require.Equal(t, len(s1.ConnsToPeer(s2.LocalPeer())[0].GetStreams()), N) + + for i := 0; i < N; i++ { + done <- true } - ss.stream = as - ss.Close() - require.True(t, called.Load()) + require.Eventually(t, func() bool { + return len(s1.ConnsToPeer(s2.LocalPeer())[0].GetStreams()) == 0 + }, 5*time.Second, 100*time.Millisecond) } From bed82fee8afd9f956610064dc699effd6f7201d1 Mon Sep 17 00:00:00 2001 From: sukun Date: Fri, 26 Jan 2024 18:22:59 +0530 Subject: [PATCH 17/30] fix sync stuff from review comments --- p2p/transport/webrtc/stream.go | 122 ++++++++++++++-------------- p2p/transport/webrtc/stream_read.go | 6 +- 2 files changed, 62 insertions(+), 66 deletions(-) diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 8fc34c2531..b711a82534 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -69,10 +69,10 @@ const ( type stream struct { mx sync.Mutex - // readerSem ensures that only a single goroutine reads from the reader. Read is not threadsafe + // readerMx ensures that only a single goroutine reads from the reader. Read is not threadsafe // But we may need to read from reader for control messages from a different goroutine. - readerSem chan struct{} - reader pbio.Reader + readerMx sync.Mutex + reader pbio.Reader // this buffer is limited up to a single message. Reason we need it // is because a reader might read a message midway, and so we need a @@ -94,7 +94,7 @@ type stream struct { // SetReadDeadline // See: https://github.com/pion/sctp/pull/290 controlMessageReaderEndTime time.Time - controlMessageReaderDone chan struct{} + controlMessageReaderDone sync.WaitGroup onDone func() id uint16 // for logging purposes @@ -110,21 +110,21 @@ func newStream( onDone func(), ) *stream { s := &stream{ - readerSem: make(chan struct{}, 1), - reader: pbio.NewDelimitedReader(rwc, maxMessageSize), - writer: pbio.NewDelimitedWriter(rwc), + reader: pbio.NewDelimitedReader(rwc, maxMessageSize), + writer: pbio.NewDelimitedWriter(rwc), sendStateChanged: make(chan struct{}, 1), writeDeadlineUpdated: make(chan struct{}, 1), writeAvailable: make(chan struct{}, 1), - controlMessageReaderDone: make(chan struct{}), + controlMessageReaderDone: sync.WaitGroup{}, id: *channel.ID(), dataChannel: rwc.(*datachannel.DataChannel), onDone: onDone, } - + // released when the controlMessageReader goroutine exits + s.controlMessageReaderDone.Add(1) channel.SetBufferedAmountLowThreshold(bufferedAmountLowThreshold) channel.OnBufferedAmountLow(func() { select { @@ -148,7 +148,7 @@ func (s *stream) Close() error { s.controlMessageReaderEndTime = time.Now().Add(maxFINACKWait) s.mx.Unlock() s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) - <-s.controlMessageReaderDone + s.controlMessageReaderDone.Wait() return nil } @@ -168,7 +168,7 @@ func (s *stream) AsyncClose(onDone func()) error { s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) go func() { - <-s.controlMessageReaderDone + s.controlMessageReaderDone.Wait() s.cleanup() if onDone != nil { onDone() @@ -226,76 +226,72 @@ func (s *stream) processIncomingFlag(flag *pb.Message_Flag) { // Remote has finished writing all the data It'll stop waiting for the // FIN_ACK eventually or will be notified when we close the datachannel } - s.controlMessageReaderOnce.Do(s.spawnControlMessageReader) + s.spawnControlMessageReader() case pb.Message_RESET: if s.receiveState == receiveStateReceiving { s.receiveState = receiveStateReset } - s.controlMessageReaderOnce.Do(s.spawnControlMessageReader) + s.spawnControlMessageReader() } } // spawnControlMessageReader is used for processing control messages after the reader is closed. func (s *stream) spawnControlMessageReader() { + s.controlMessageReaderOnce.Do(func() { + // Spawn a goroutine to ensure that we're not holding any locks + go func() { + defer s.controlMessageReaderDone.Done() + // cleanup the sctp deadline timer goroutine + defer s.SetReadDeadline(time.Time{}) + + setDeadline := func() bool { + if s.controlMessageReaderEndTime.IsZero() || time.Now().Before(s.controlMessageReaderEndTime) { + s.SetReadDeadline(s.controlMessageReaderEndTime) + return true + } + return false + } - // Spawn a goroutine to ensure that we're not holding any locks - go func() { - defer close(s.controlMessageReaderDone) - // cleanup the sctp deadline timer goroutine - defer s.SetReadDeadline(time.Time{}) + // Unblock any Read call waiting on reader.ReadMsg + s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) - isSendCompleted := func() bool { - s.mx.Lock() - defer s.mx.Unlock() - return s.sendState == sendStateDataReceived || s.sendState == sendStateReset - } + s.readerMx.Lock() + // We have the lock any readers blocked on reader.ReadMsg have exited. + // From this point onwards only this goroutine will do reader.ReadMsg. + + //lint:ignore SA2001 we just want to ensure any exising readers have exited. + // Read calls from this point onwards will exit immediately on checking + // s.readState + s.readerMx.Unlock() - setDeadline := func() bool { s.mx.Lock() defer s.mx.Unlock() - if s.controlMessageReaderEndTime.IsZero() || time.Now().Before(s.controlMessageReaderEndTime) { - s.SetReadDeadline(s.controlMessageReaderEndTime) - return true - } - return false - } - // Unblock any Read call waiting on reader.ReadMsg - s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) - s.readerSem <- struct{}{} - // We have the lock any readers blocked on reader.ReadMsg have exited. - // From this point onwards only this goroutine will do reader.ReadMsg. - - // Release the semaphore because calls to stream.Read will return immediately - // since the read half of the stream is closed. - <-s.readerSem - - s.mx.Lock() - if s.nextMessage != nil { - s.processIncomingFlag(s.nextMessage.Flag) - s.nextMessage = nil - } - s.mx.Unlock() - - for !isSendCompleted() { - var msg pb.Message - if !setDeadline() { - return + if s.nextMessage != nil { + s.processIncomingFlag(s.nextMessage.Flag) + s.nextMessage = nil } - if err := s.reader.ReadMsg(&msg); err != nil { - // We have to manually manage deadline exceeded errors since pion/sctp can - // return deadline exceeded error for cancelled deadlines - // see: https://github.com/pion/sctp/pull/290/files - if errors.Is(err, os.ErrDeadlineExceeded) { - continue + for s.sendState != sendStateDataReceived && s.sendState != sendStateReset { + var msg pb.Message + if !setDeadline() { + return + } + s.mx.Unlock() + if err := s.reader.ReadMsg(&msg); err != nil { + s.mx.Lock() + // We have to manually manage deadline exceeded errors since pion/sctp can + // return deadline exceeded error for cancelled deadlines + // see: https://github.com/pion/sctp/pull/290/files + if errors.Is(err, os.ErrDeadlineExceeded) { + continue + } + return } - return + s.mx.Lock() + s.processIncomingFlag(msg.Flag) } - s.mx.Lock() - s.processIncomingFlag(msg.Flag) - s.mx.Unlock() - } - }() + }() + }) } func (s *stream) cleanup() { diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index d4fcc8bf5e..29c8c0cfe7 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -9,8 +9,8 @@ import ( ) func (s *stream) Read(b []byte) (int, error) { - s.readerSem <- struct{}{} - defer func() { <-s.readerSem }() + s.readerMx.Lock() + defer s.readerMx.Unlock() s.mx.Lock() defer s.mx.Unlock() @@ -93,6 +93,6 @@ func (s *stream) CloseRead() error { err = s.writer.WriteMsg(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) s.receiveState = receiveStateReset } - s.controlMessageReaderOnce.Do(s.spawnControlMessageReader) + s.spawnControlMessageReader() return err } From bc7390f746160f15d5dd70eb7664dacfba00718e Mon Sep 17 00:00:00 2001 From: Sukun Date: Tue, 30 Jan 2024 14:59:06 +0530 Subject: [PATCH 18/30] remove the onBufferedAmount callback to clear all references to stream --- p2p/transport/webrtc/stream.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index b711a82534..2604115b41 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -125,8 +125,8 @@ func newStream( } // released when the controlMessageReader goroutine exits s.controlMessageReaderDone.Add(1) - channel.SetBufferedAmountLowThreshold(bufferedAmountLowThreshold) - channel.OnBufferedAmountLow(func() { + s.dataChannel.SetBufferedAmountLowThreshold(bufferedAmountLowThreshold) + s.dataChannel.OnBufferedAmountLow(func() { select { case s.writeAvailable <- struct{}{}: default: @@ -295,6 +295,10 @@ func (s *stream) spawnControlMessageReader() { } func (s *stream) cleanup() { + // Even if we close the datachannel pion keeps a reference to the datachannel around. + // Remove the onBufferedAmountLow callback to ensure that we at least garbage collect + // memory we allocated for this stream. + s.dataChannel.OnBufferedAmountLow(nil) s.dataChannel.Close() if s.onDone != nil { s.onDone() From bb4aea6e4bb7934ce647bb3f2030ac3226da09d1 Mon Sep 17 00:00:00 2001 From: Sukun Date: Thu, 1 Feb 2024 14:31:54 +0530 Subject: [PATCH 19/30] rename udpmux.ReceiveMTU to udpmux.ReceiveBufSize --- p2p/transport/webrtc/stream_write.go | 2 +- p2p/transport/webrtc/transport_test.go | 22 ++++++++++------------ p2p/transport/webrtc/udpmux/mux.go | 7 +++++-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index 6515aba8ff..53a6260577 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -89,7 +89,7 @@ func (s *stream) Write(b []byte) (int, error) { if end > availableSpace { end = availableSpace } - end -= protoOverhead + varintOverhead + end -= (protoOverhead + varintOverhead) if end > len(b) { end = len(b) } diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index b831d52f08..072f6a1f9a 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -301,7 +301,7 @@ func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { require.NoError(t, err) tr1, connectingPeer := getTransport(t) - done := make(chan struct{}) + readerDone := make(chan struct{}) const ( numListeners = 10 @@ -335,34 +335,32 @@ func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { }() } wg.Wait() - done <- struct{}{} + readerDone <- struct{}{} }() conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) require.NoError(t, err) - t.Logf("dialer opened connection") - var wwg sync.WaitGroup + var writerWG sync.WaitGroup var cnt atomic.Int32 var streamsStarted atomic.Int32 for i := 0; i < numWriters; i++ { - wwg.Add(1) + writerWG.Add(1) go func() { - defer wwg.Done() + defer writerWG.Done() + buf := make([]byte, size) for { var nn int32 if nn = streamsStarted.Add(1); nn > int32(numStreams) { return } + rand.Read(buf) + s, err := conn.OpenStream(context.Background()) require.NoError(t, err) - // t.Logf("dialer opened stream: %d %d", idx, s.(*stream).id) - buf := make([]byte, size) - rand.Read(buf) n, err := s.Write(buf) require.Equal(t, n, size) require.NoError(t, err) - s.CloseWrite() resp := make([]byte, size+10) n, err = io.ReadFull(s, resp) @@ -376,9 +374,9 @@ func TestTransportWebRTC_DialerCanCreateStreamsMultiple(t *testing.T) { } }() } - wwg.Wait() + writerWG.Wait() select { - case <-done: + case <-readerDone: case <-time.After(100 * time.Second): t.Fatal("timed out") } diff --git a/p2p/transport/webrtc/udpmux/mux.go b/p2p/transport/webrtc/udpmux/mux.go index ca54c18593..30cc61edab 100644 --- a/p2p/transport/webrtc/udpmux/mux.go +++ b/p2p/transport/webrtc/udpmux/mux.go @@ -17,7 +17,10 @@ import ( var log = logging.Logger("webrtc-udpmux") -const ReceiveMTU = 1500 +// ReceiveBufSize is the size of the buffer used to receive packets from the PacketConn. +// It is fine for this number to be higher than the actual path MTU as this value is not +// used to decide the packet size on the write path. +const ReceiveBufSize = 1500 type Candidate struct { Ufrag string @@ -134,7 +137,7 @@ func (mux *UDPMux) readLoop() { default: } - buf := pool.Get(ReceiveMTU) + buf := pool.Get(ReceiveBufSize) n, addr, err := mux.socket.ReadFrom(buf) if err != nil { From fbcb8f91126f1e4685c47866f615a5b241030b66 Mon Sep 17 00:00:00 2001 From: Sukun Date: Mon, 5 Feb 2024 20:57:16 +0530 Subject: [PATCH 20/30] reuse msg for writes --- p2p/transport/webrtc/stream_write.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index 53a6260577..dee76851cc 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -41,6 +41,7 @@ func (s *stream) Write(b []byte) (int, error) { }() var n int + var msg pb.Message for len(b) > 0 { if s.closeErr != nil { return n, s.closeErr @@ -89,12 +90,12 @@ func (s *stream) Write(b []byte) (int, error) { if end > availableSpace { end = availableSpace } - end -= (protoOverhead + varintOverhead) + end -= protoOverhead + varintOverhead if end > len(b) { end = len(b) } - msg := &pb.Message{Message: b[:end]} - if err := s.writer.WriteMsg(msg); err != nil { + msg = pb.Message{Message: b[:end]} + if err := s.writer.WriteMsg(&msg); err != nil { return n, err } n += end From 5f0faa3d43e666ac9c979b2485e78cea497fad96 Mon Sep 17 00:00:00 2001 From: Sukun Date: Tue, 13 Feb 2024 19:44:13 +0530 Subject: [PATCH 21/30] remove AsyncClose --- p2p/net/swarm/swarm_stream.go | 16 ----- p2p/net/swarm/swarm_stream_test.go | 82 -------------------------- p2p/transport/webrtc/stream.go | 23 -------- p2p/transport/webrtc/stream_test.go | 34 ++++++----- p2p/transport/webrtc/transport_test.go | 49 --------------- 5 files changed, 18 insertions(+), 186 deletions(-) delete mode 100644 p2p/net/swarm/swarm_stream_test.go diff --git a/p2p/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go index 2b6f07c92a..b7846adec2 100644 --- a/p2p/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -75,25 +75,9 @@ func (s *Stream) Write(p []byte) (int, error) { return n, err } -// asyncCloser interface is implemented by transports that need to do expensive cleanup work -// on stream close. Currently only WebRTC transport needs this. This interface is kept private -// to swarm to discourage other transport implementations from using this mechanism. It is only -// required because WebRTC datachannels have no Close functionality that would send the flush -// the enqueued data. -// see: https://github.com/libp2p/specs/issues/575 -type asyncCloser interface { - AsyncClose(onDone func()) error -} - // Close closes the stream, closing both ends and freeing all associated // resources. func (s *Stream) Close() error { - if as, ok := s.stream.(asyncCloser); ok { - err := as.AsyncClose(func() { - s.closeAndRemoveStream() - }) - return err - } err := s.stream.Close() s.closeAndRemoveStream() return err diff --git a/p2p/net/swarm/swarm_stream_test.go b/p2p/net/swarm/swarm_stream_test.go deleted file mode 100644 index 3fe47f4a07..0000000000 --- a/p2p/net/swarm/swarm_stream_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package swarm - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/peerstore" - "github.com/stretchr/testify/require" -) - -type asyncStreamWrapper struct { - network.MuxedStream - beforeClose func() - done chan bool -} - -func (s *asyncStreamWrapper) AsyncClose(onDone func()) error { - s.beforeClose() - err := s.Close() - go func() { - <-s.done - onDone() - }() - return err -} - -func TestStreamAsyncCloser(t *testing.T) { - s1 := makeSwarm(t) - s2 := makeSwarm(t) - - s1.Peerstore().AddAddrs(s2.LocalPeer(), s2.ListenAddresses(), peerstore.TempAddrTTL) - - var mx sync.Mutex - var wg1, wg2 sync.WaitGroup - var closed int - done := make(chan bool) - const N = 100 - wg1.Add(N) - // wg2 blocks all goroutines in the beforeClose method. This allows us to check GetStreams - // works concurrently with Close - wg2.Add(N) - for i := 0; i < N; i++ { - go func() { - s, err := s1.NewStream(context.Background(), s2.LocalPeer()) - require.NoError(t, err) - ss, ok := s.(*Stream) - require.True(t, ok) - as := &asyncStreamWrapper{ - MuxedStream: ss.stream, - beforeClose: func() { - wg2.Done() - mx.Lock() - defer mx.Unlock() - closed++ - wg1.Done() - }, - done: done, - } - ss.stream = as - ss.Close() - }() - } - wg2.Wait() - require.Eventually(t, func() bool { return s1.Connectedness(s2.LocalPeer()) == network.Connected }, - 5*time.Second, 100*time.Millisecond) - require.Equal(t, len(s1.ConnsToPeer(s2.LocalPeer())[0].GetStreams()), N) - - wg1.Wait() - require.Equal(t, closed, N) - // Streams should only be removed from the connection after the onDone call back is executed - require.Equal(t, len(s1.ConnsToPeer(s2.LocalPeer())[0].GetStreams()), N) - - for i := 0; i < N; i++ { - done <- true - } - require.Eventually(t, func() bool { - return len(s1.ConnsToPeer(s2.LocalPeer())[0].GetStreams()) == 0 - }, 5*time.Second, 100*time.Millisecond) -} diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 2604115b41..dbbdd5cb63 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -136,32 +136,12 @@ func newStream( } func (s *stream) Close() error { - defer s.cleanup() closeWriteErr := s.CloseWrite() closeReadErr := s.CloseRead() if closeWriteErr != nil || closeReadErr != nil { s.Reset() return errors.Join(closeWriteErr, closeReadErr) } - - s.mx.Lock() - s.controlMessageReaderEndTime = time.Now().Add(maxFINACKWait) - s.mx.Unlock() - s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) - s.controlMessageReaderDone.Wait() - return nil -} - -func (s *stream) AsyncClose(onDone func()) error { - closeWriteErr := s.CloseWrite() - closeReadErr := s.CloseRead() - if closeWriteErr != nil || closeReadErr != nil { - s.Reset() - if onDone != nil { - onDone() - } - return errors.Join(closeWriteErr, closeReadErr) - } s.mx.Lock() s.controlMessageReaderEndTime = time.Now().Add(maxFINACKWait) s.mx.Unlock() @@ -170,9 +150,6 @@ func (s *stream) AsyncClose(onDone func()) error { go func() { s.controlMessageReaderDone.Wait() s.cleanup() - if onDone != nil { - onDone() - } }() return nil } diff --git a/p2p/transport/webrtc/stream_test.go b/p2p/transport/webrtc/stream_test.go index 34655e4358..884e688bad 100644 --- a/p2p/transport/webrtc/stream_test.go +++ b/p2p/transport/webrtc/stream_test.go @@ -136,9 +136,9 @@ func TestStreamSimpleReadWriteClose(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte("lorem ipsum"), b) - // stream is only cleaned up on calling Close or AsyncClose or Reset - clientStr.AsyncClose(nil) - serverStr.AsyncClose(nil) + // stream is only cleaned up on calling Close or Reset + clientStr.Close() + serverStr.Close() require.Eventually(t, func() bool { return clientDone.Load() }, 5*time.Second, 100*time.Millisecond) // Need to call Close for cleanup. Otherwise the FIN_ACK is never read require.NoError(t, serverStr.Close()) @@ -213,7 +213,7 @@ func TestStreamReadReturnsOnClose(t *testing.T) { errChan <- err }() time.Sleep(100 * time.Millisecond) // give the Read call some time to hit the loop - require.NoError(t, clientStr.AsyncClose(nil)) + require.NoError(t, clientStr.Close()) select { case err := <-errChan: require.ErrorIs(t, err, network.ErrReset) @@ -228,9 +228,9 @@ func TestStreamReadReturnsOnClose(t *testing.T) { func TestStreamResets(t *testing.T) { client, server := getDetachedDataChannels(t) - var clientDone, serverDone bool - clientStr := newStream(client.dc, client.rwc, func() { clientDone = true }) - serverStr := newStream(server.dc, server.rwc, func() { serverDone = true }) + var clientDone, serverDone atomic.Bool + clientStr := newStream(client.dc, client.rwc, func() { clientDone.Store(true) }) + serverStr := newStream(server.dc, server.rwc, func() { serverDone.Store(true) }) // send a foobar from the client _, err := clientStr.Write([]byte("foobar")) @@ -238,7 +238,7 @@ func TestStreamResets(t *testing.T) { _, err = serverStr.Write([]byte("lorem ipsum")) require.NoError(t, err) require.NoError(t, clientStr.Reset()) // resetting resets both directions - require.True(t, clientDone) + require.True(t, clientDone.Load()) // attempting to write more data should result in a reset error _, err = clientStr.Write([]byte("foobar")) require.ErrorIs(t, err, network.ErrReset) @@ -248,7 +248,7 @@ func TestStreamResets(t *testing.T) { require.ErrorIs(t, err, network.ErrReset) // read the data on the server side - require.False(t, serverDone) + require.False(t, serverDone.Load()) b, err = io.ReadAll(serverStr) require.Equal(t, []byte("foobar"), b) require.ErrorIs(t, err, network.ErrReset) @@ -257,7 +257,9 @@ func TestStreamResets(t *testing.T) { return errors.Is(err, network.ErrReset) }, time.Second, 50*time.Millisecond) serverStr.Close() - require.True(t, serverDone) + require.Eventually(t, func() bool { + return serverDone.Load() + }, time.Second, 50*time.Millisecond) } func TestStreamReadDeadlineAsync(t *testing.T) { @@ -327,7 +329,7 @@ func TestStreamReadAfterClose(t *testing.T) { clientStr := newStream(client.dc, client.rwc, func() {}) serverStr := newStream(server.dc, server.rwc, func() {}) - serverStr.AsyncClose(nil) + serverStr.Close() b := make([]byte, 1) _, err := clientStr.Read(b) require.Equal(t, io.EOF, err) @@ -446,16 +448,16 @@ func TestStreamConcurrentClose(t *testing.T) { } } -func TestStreamResetAfterAsyncClose(t *testing.T) { +func TestStreamResetAfterClose(t *testing.T) { client, _ := getDetachedDataChannels(t) done := make(chan bool, 1) clientStr := newStream(client.dc, client.rwc, func() { done <- true }) - clientStr.AsyncClose(nil) + clientStr.Close() select { case <-done: - t.Fatalf("AsyncClose shouldn't run cleanup immediately") + t.Fatalf("Close shouldn't run cleanup immediately") case <-time.After(500 * time.Millisecond): } @@ -473,11 +475,11 @@ func TestStreamDataChannelCloseOnFINACK(t *testing.T) { done := make(chan bool, 1) clientStr := newStream(client.dc, client.rwc, func() { done <- true }) - clientStr.AsyncClose(nil) + clientStr.Close() select { case <-done: - t.Fatalf("AsyncClose shouldn't run cleanup immediately") + t.Fatalf("Close shouldn't run cleanup immediately") case <-time.After(500 * time.Millisecond): } diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index c69e732cf5..e851eedc27 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -530,55 +530,6 @@ func TestTransportWebRTC_RemoteReadsAfterClose(t *testing.T) { require.Equal(t, n, 4) } -func TestTransportWebRTC_RemoteReadsAfterAsyncClose(t *testing.T) { - tr, listeningPeer := getTransport(t) - listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") - listener, err := tr.Listen(listenMultiaddr) - require.NoError(t, err) - - tr1, _ := getTransport(t) - - done := make(chan error) - go func() { - lconn, err := listener.Accept() - if err != nil { - done <- err - return - } - s, err := lconn.AcceptStream() - if err != nil { - done <- err - return - } - _, err = s.Write([]byte{1, 2, 3, 4}) - if err != nil { - done <- err - return - } - err = s.(*stream).AsyncClose(nil) - if err != nil { - done <- err - return - } - close(done) - }() - - conn, err := tr1.Dial(context.Background(), listener.Multiaddr(), listeningPeer) - require.NoError(t, err) - // create a stream - stream, err := conn.OpenStream(context.Background()) - - require.NoError(t, err) - // require write and close to complete - require.NoError(t, <-done) - stream.SetReadDeadline(time.Now().Add(5 * time.Second)) - - buf := make([]byte, 10) - n, err := stream.Read(buf) - require.NoError(t, err) - require.Equal(t, n, 4) -} - func TestTransportWebRTC_RemoteReadsAfterClose2(t *testing.T) { tr, listeningPeer := getTransport(t) listenMultiaddr := ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct") From 17c89bdd5dbc43751a39956bcebf2f2b9176e3cc Mon Sep 17 00:00:00 2001 From: Sukun Date: Tue, 13 Feb 2024 19:45:55 +0530 Subject: [PATCH 22/30] go mod tidy --- go.sum | 5 ----- test-plans/go.sum | 5 ----- 2 files changed, 10 deletions(-) diff --git a/go.sum b/go.sum index ed18f429ed..32854b62af 100644 --- a/go.sum +++ b/go.sum @@ -607,9 +607,6 @@ golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -799,8 +796,6 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/test-plans/go.sum b/test-plans/go.sum index ea16f8267a..c1f7a91201 100644 --- a/test-plans/go.sum +++ b/test-plans/go.sum @@ -540,9 +540,6 @@ golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -726,8 +723,6 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From 37ff2f6b75d4f66fb010bd708d7b667cb0710efb Mon Sep 17 00:00:00 2001 From: sukun Date: Fri, 9 Feb 2024 22:17:49 +0530 Subject: [PATCH 23/30] return connection timed out errors on connection close for timeout --- p2p/transport/webrtc/connection.go | 3 +++ p2p/transport/webrtc/stream.go | 35 ++++++++++++++++++++++---- p2p/transport/webrtc/stream_read.go | 18 +++++++++---- p2p/transport/webrtc/stream_write.go | 8 +++--- p2p/transport/webrtc/transport_test.go | 4 +-- 5 files changed, 51 insertions(+), 17 deletions(-) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index e23975f4da..a9463a199b 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -145,6 +145,9 @@ func (c *connection) closeWithError(err error) { c.cancel() // closing peerconnection will close the datachannels associated with the streams c.pc.Close() + for _, s := range c.streams { + s.closeForShutdown(err) + } c.scope.Done() } diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index dbbdd5cb63..b7c91af079 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -96,10 +96,10 @@ type stream struct { controlMessageReaderEndTime time.Time controlMessageReaderDone sync.WaitGroup - onDone func() - id uint16 // for logging purposes - dataChannel *datachannel.DataChannel - closeErr error + onDone func() + id uint16 // for logging purposes + dataChannel *datachannel.DataChannel + closeForShutdownErr error } var _ network.MuxedStream = &stream{} @@ -136,6 +136,13 @@ func newStream( } func (s *stream) Close() error { + s.mx.Lock() + if s.closeForShutdownErr != nil { + return s.closeForShutdownErr + } + s.mx.Unlock() + + defer s.cleanup() closeWriteErr := s.CloseWrite() closeReadErr := s.CloseRead() if closeWriteErr != nil || closeReadErr != nil { @@ -155,8 +162,13 @@ func (s *stream) Close() error { } func (s *stream) Reset() error { - defer s.cleanup() + s.mx.Lock() + if s.closeForShutdownErr != nil { + return nil + } + s.mx.Unlock() + defer s.cleanup() cancelWriteErr := s.cancelWrite() closeReadErr := s.CloseRead() if cancelWriteErr != nil { @@ -165,6 +177,19 @@ func (s *stream) Reset() error { return closeReadErr } +func (s *stream) closeForShutdown(closeErr error) { + s.mx.Lock() + defer s.mx.Unlock() + + s.closeForShutdownErr = closeErr + s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) + select { + case s.sendStateChanged <- struct{}{}: + default: + } + s.cleanup() +} + func (s *stream) SetDeadline(t time.Time) error { _ = s.SetReadDeadline(t) return s.SetWriteDeadline(t) diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index 29c8c0cfe7..9d54ec51be 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -15,8 +15,8 @@ func (s *stream) Read(b []byte) (int, error) { s.mx.Lock() defer s.mx.Unlock() - if s.closeErr != nil { - return 0, s.closeErr + if s.closeForShutdownErr != nil { + return 0, s.closeForShutdownErr } switch s.receiveState { case receiveStateDataRead: @@ -38,6 +38,10 @@ func (s *stream) Read(b []byte) (int, error) { if err := s.reader.ReadMsg(&msg); err != nil { s.mx.Lock() if err == io.EOF { + // connection was closed + if s.closeForShutdownErr != nil { + return 0, s.closeForShutdownErr + } // if the channel was properly closed, return EOF if s.receiveState == receiveStateDataRead { return 0, io.EOF @@ -55,6 +59,10 @@ func (s *stream) Read(b []byte) (int, error) { if s.receiveState == receiveStateDataRead { return 0, io.EOF } + // connection was closed + if s.closeForShutdownErr != nil { + return 0, s.closeForShutdownErr + } return 0, err } s.mx.Lock() @@ -71,8 +79,8 @@ func (s *stream) Read(b []byte) (int, error) { // process flags on the message after reading all the data s.processIncomingFlag(s.nextMessage.Flag) s.nextMessage = nil - if s.closeErr != nil { - return read, s.closeErr + if s.closeForShutdownErr != nil { + return read, s.closeForShutdownErr } switch s.receiveState { case receiveStateDataRead: @@ -89,7 +97,7 @@ func (s *stream) CloseRead() error { s.mx.Lock() defer s.mx.Unlock() var err error - if s.receiveState == receiveStateReceiving && s.closeErr == nil { + if s.receiveState == receiveStateReceiving && s.closeForShutdownErr == nil { err = s.writer.WriteMsg(&pb.Message{Flag: pb.Message_STOP_SENDING.Enum()}) s.receiveState = receiveStateReset } diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index dee76851cc..b0b2837d91 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -19,8 +19,8 @@ func (s *stream) Write(b []byte) (int, error) { s.mx.Lock() defer s.mx.Unlock() - if s.closeErr != nil { - return 0, s.closeErr + if s.closeForShutdownErr != nil { + return 0, s.closeForShutdownErr } switch s.sendState { case sendStateReset: @@ -43,8 +43,8 @@ func (s *stream) Write(b []byte) (int, error) { var n int var msg pb.Message for len(b) > 0 { - if s.closeErr != nil { - return n, s.closeErr + if s.closeForShutdownErr != nil { + return n, s.closeForShutdownErr } switch s.sendState { case sendStateReset: diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index e851eedc27..18e677f7c7 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -687,11 +687,9 @@ func TestConnectionTimeoutOnListener(t *testing.T) { // start dropping all packets drop.Store(true) start := time.Now() - // TODO: return timeout errors here for { if _, err := str.Write([]byte("test")); err != nil { - // TODO (sukunrt): Decide whether we want to keep this behaviour - //require.True(t, os.IsTimeout(err)) + require.True(t, os.IsTimeout(err)) break } From bd78a36bd3300d6f93a4b94b7dd3f279914daeaa Mon Sep 17 00:00:00 2001 From: sukun Date: Fri, 9 Feb 2024 22:33:44 +0530 Subject: [PATCH 24/30] add rationale for maxTotalControlMessagesSize --- p2p/transport/webrtc/stream.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index b7c91af079..42e90d5fad 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -22,8 +22,11 @@ const ( // https://github.com/pion/webrtc/blob/v3.1.49/sctptransport.go#L341 maxBufferedAmount = 2 * maxMessageSize // maxTotalControlMessagesSize is the maximum total size of all control messages we will - // write on this stream - maxTotalControlMessagesSize = 500 + // write on this stream. + // 4 control messages of size 10 bytes + 10 bytes buffer. This number doesn't need to be + // exact. In the worst case, we enqueue these many bytes more in the webrtc peer connection + // send queue. + maxTotalControlMessagesSize = 50 // bufferedAmountLowThreshold and maxBufferedAmount are bound // to a stream but congestion control is done on the whole // SCTP association. This means that a single stream can monopolize @@ -142,7 +145,6 @@ func (s *stream) Close() error { } s.mx.Unlock() - defer s.cleanup() closeWriteErr := s.CloseWrite() closeReadErr := s.CloseRead() if closeWriteErr != nil || closeReadErr != nil { From 8abd3fefdfc6aca7c487d863cd38adbfd161ef30 Mon Sep 17 00:00:00 2001 From: sukun Date: Tue, 13 Feb 2024 23:51:17 +0530 Subject: [PATCH 25/30] fix race in connection timeout --- p2p/transport/webrtc/connection.go | 10 +++++++++- p2p/transport/webrtc/stream.go | 10 +++++++--- p2p/transport/webrtc/stream_read.go | 12 ++++-------- p2p/transport/webrtc/transport_test.go | 4 ++-- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/p2p/transport/webrtc/connection.go b/p2p/transport/webrtc/connection.go index a9463a199b..f220360ed4 100644 --- a/p2p/transport/webrtc/connection.go +++ b/p2p/transport/webrtc/connection.go @@ -145,7 +145,12 @@ func (c *connection) closeWithError(err error) { c.cancel() // closing peerconnection will close the datachannels associated with the streams c.pc.Close() - for _, s := range c.streams { + + c.m.Lock() + streams := c.streams + c.streams = nil + c.m.Unlock() + for _, s := range streams { s.closeForShutdown(err) } c.scope.Done() @@ -207,6 +212,9 @@ func (c *connection) Transport() tpt.Transport { return c.transport } func (c *connection) addStream(str *stream) error { c.m.Lock() defer c.m.Unlock() + if c.streams == nil { + return c.closeErr + } if _, ok := c.streams[str.id]; ok { return errors.New("stream ID already exists") } diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 42e90d5fad..23fef07bbc 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -141,7 +141,8 @@ func newStream( func (s *stream) Close() error { s.mx.Lock() if s.closeForShutdownErr != nil { - return s.closeForShutdownErr + s.mx.Unlock() + return nil } s.mx.Unlock() @@ -166,6 +167,7 @@ func (s *stream) Close() error { func (s *stream) Reset() error { s.mx.Lock() if s.closeForShutdownErr != nil { + s.mx.Unlock() return nil } s.mx.Unlock() @@ -180,6 +182,8 @@ func (s *stream) Reset() error { } func (s *stream) closeForShutdown(closeErr error) { + defer s.cleanup() + s.mx.Lock() defer s.mx.Unlock() @@ -189,7 +193,6 @@ func (s *stream) closeForShutdown(closeErr error) { case s.sendStateChanged <- struct{}{}: default: } - s.cleanup() } func (s *stream) SetDeadline(t time.Time) error { @@ -275,7 +278,8 @@ func (s *stream) spawnControlMessageReader() { s.processIncomingFlag(s.nextMessage.Flag) s.nextMessage = nil } - for s.sendState != sendStateDataReceived && s.sendState != sendStateReset { + for s.closeForShutdownErr == nil && + s.sendState != sendStateDataReceived && s.sendState != sendStateReset { var msg pb.Message if !setDeadline() { return diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index 9d54ec51be..002ebac0ec 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -37,11 +37,11 @@ func (s *stream) Read(b []byte) (int, error) { var msg pb.Message if err := s.reader.ReadMsg(&msg); err != nil { s.mx.Lock() + // connection was closed + if s.closeForShutdownErr != nil { + return 0, s.closeForShutdownErr + } if err == io.EOF { - // connection was closed - if s.closeForShutdownErr != nil { - return 0, s.closeForShutdownErr - } // if the channel was properly closed, return EOF if s.receiveState == receiveStateDataRead { return 0, io.EOF @@ -59,10 +59,6 @@ func (s *stream) Read(b []byte) (int, error) { if s.receiveState == receiveStateDataRead { return 0, io.EOF } - // connection was closed - if s.closeForShutdownErr != nil { - return 0, s.closeForShutdownErr - } return 0, err } s.mx.Lock() diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 18e677f7c7..11a0cd1600 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -689,7 +689,7 @@ func TestConnectionTimeoutOnListener(t *testing.T) { start := time.Now() for { if _, err := str.Write([]byte("test")); err != nil { - require.True(t, os.IsTimeout(err)) + require.True(t, os.IsTimeout(err), "invalid error type: %v", err) break } @@ -697,7 +697,7 @@ func TestConnectionTimeoutOnListener(t *testing.T) { t.Fatal("timeout") } // make sure to not write too often, we don't want to fill the flow control window - time.Sleep(5 * time.Millisecond) + time.Sleep(20 * time.Millisecond) } // make sure that accepting a stream also returns an error... _, err = conn.AcceptStream() From 033ec8782953937b7b1c63703cb17b88025c35e3 Mon Sep 17 00:00:00 2001 From: sukun Date: Sun, 18 Feb 2024 17:59:01 +0530 Subject: [PATCH 26/30] fix check for timeout error --- p2p/transport/webrtc/transport_test.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index a86998aa64..807a498491 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -7,6 +7,7 @@ import ( "io" "net" "os" + "strings" "sync" "sync/atomic" "testing" @@ -683,7 +684,20 @@ func TestConnectionTimeoutOnListener(t *testing.T) { start := time.Now() for { if _, err := str.Write([]byte("test")); err != nil { - require.True(t, os.IsTimeout(err), "invalid error type: %v", err) + if os.IsTimeout(err) { + break + } + // If we write when a connection timeout happens, sctp provides + // a "stream closed" error. This occurs concurrently with the + // callback we receive for connection timeout. + // Test once more after sleep that we provide the correct error. + if strings.Contains(err.Error(), "stream closed") { + time.Sleep(50 * time.Millisecond) + _, err = str.Write([]byte("test")) + require.True(t, os.IsTimeout(err), "invalid error type: %v", err) + } else { + t.Fatal("invalid error type", err) + } break } From ec4a9739ba3bd8eea107d95a87a3bc1fa10b30ac Mon Sep 17 00:00:00 2001 From: Sukun Date: Mon, 19 Feb 2024 18:04:40 +0530 Subject: [PATCH 27/30] remove redundant hanging handlers test --- p2p/test/swarm/swarm_test.go | 83 ------------------------------------ 1 file changed, 83 deletions(-) diff --git a/p2p/test/swarm/swarm_test.go b/p2p/test/swarm/swarm_test.go index 3761c452ee..8027cebe53 100644 --- a/p2p/test/swarm/swarm_test.go +++ b/p2p/test/swarm/swarm_test.go @@ -2,7 +2,6 @@ package swarm_test import ( "context" - "fmt" "io" "sync" "testing" @@ -15,7 +14,6 @@ import ( rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/client" "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay" - libp2pwebrtc "github.com/libp2p/go-libp2p/p2p/transport/webrtc" ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -245,84 +243,3 @@ func TestLimitStreamsWhenHangingHandlers(t *testing.T) { return false }, 5*time.Second, 100*time.Millisecond) } - -func TestLimitStreamsWhenHangingHandlersWebRTC(t *testing.T) { - var partial rcmgr.PartialLimitConfig - const streamLimit = 10 - partial.System.Streams = streamLimit - mgr, err := rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(partial.Build(rcmgr.InfiniteLimits))) - require.NoError(t, err) - - maddr, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/0/webrtc-direct") - require.NoError(t, err) - - receiver, err := libp2p.New( - libp2p.ResourceManager(mgr), - libp2p.ListenAddrs(maddr), - libp2p.Transport(libp2pwebrtc.New), - ) - require.NoError(t, err) - t.Cleanup(func() { receiver.Close() }) - - var wg sync.WaitGroup - wg.Add(1) - - const pid = "/test" - receiver.SetStreamHandler(pid, func(s network.Stream) { - defer s.Close() - s.Write([]byte{42}) - wg.Wait() - }) - - // Open streamLimit streams - success := 0 - // we make a lot of tries because identify and identify push take up a few streams - for i := 0; i < 1000 && success < streamLimit; i++ { - mgr, err = rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(rcmgr.InfiniteLimits)) - require.NoError(t, err) - - sender, err := libp2p.New(libp2p.ResourceManager(mgr), libp2p.Transport(libp2pwebrtc.New)) - require.NoError(t, err) - t.Cleanup(func() { sender.Close() }) - - sender.Peerstore().AddAddrs(receiver.ID(), receiver.Addrs(), peerstore.PermanentAddrTTL) - - s, err := sender.NewStream(context.Background(), receiver.ID(), pid) - if err != nil { - continue - } - - var b [1]byte - _, err = io.ReadFull(s, b[:]) - if err == nil { - success++ - } - sender.Close() - } - require.Equal(t, streamLimit, success) - // We have the maximum number of streams open. Next call should fail. - mgr, err = rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(rcmgr.InfiniteLimits)) - require.NoError(t, err) - - sender, err := libp2p.New(libp2p.ResourceManager(mgr), libp2p.Transport(libp2pwebrtc.New)) - require.NoError(t, err) - t.Cleanup(func() { sender.Close() }) - - sender.Peerstore().AddAddrs(receiver.ID(), receiver.Addrs(), peerstore.PermanentAddrTTL) - - _, err = sender.NewStream(context.Background(), receiver.ID(), pid) - require.Error(t, err) - // Close the open streams - wg.Done() - - // Next call should succeed - require.Eventually(t, func() bool { - s, err := sender.NewStream(context.Background(), receiver.ID(), pid) - if err == nil { - s.Close() - return true - } - fmt.Println(err) - return false - }, 5*time.Second, 1*time.Second) -} From 0094df6fac47660d8a10641a2c3a096d240e9a6d Mon Sep 17 00:00:00 2001 From: Sukun Date: Tue, 20 Feb 2024 08:49:28 +0530 Subject: [PATCH 28/30] cleanup write path channels --- p2p/transport/webrtc/stream.go | 67 ++++++++++------------------ p2p/transport/webrtc/stream_read.go | 6 +-- p2p/transport/webrtc/stream_write.go | 26 +++++------ 3 files changed, 38 insertions(+), 61 deletions(-) diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index 23fef07bbc..a78604b99e 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -83,12 +83,10 @@ type stream struct { nextMessage *pb.Message receiveState receiveState - writer pbio.Writer // concurrent writes prevented by mx - sendStateChanged chan struct{} - sendState sendState - writeDeadline time.Time - writeDeadlineUpdated chan struct{} - writeAvailable chan struct{} + writer pbio.Writer // concurrent writes prevented by mx + writeStateChanged chan struct{} + sendState sendState + writeDeadline time.Time controlMessageReaderOnce sync.Once // controlMessageReaderEndTime is the end time for reading FIN_ACK from the control @@ -113,38 +111,30 @@ func newStream( onDone func(), ) *stream { s := &stream{ - reader: pbio.NewDelimitedReader(rwc, maxMessageSize), - writer: pbio.NewDelimitedWriter(rwc), - - sendStateChanged: make(chan struct{}, 1), - writeDeadlineUpdated: make(chan struct{}, 1), - writeAvailable: make(chan struct{}, 1), - - controlMessageReaderDone: sync.WaitGroup{}, - - id: *channel.ID(), - dataChannel: rwc.(*datachannel.DataChannel), - onDone: onDone, + reader: pbio.NewDelimitedReader(rwc, maxMessageSize), + writer: pbio.NewDelimitedWriter(rwc), + writeStateChanged: make(chan struct{}, 1), + id: *channel.ID(), + dataChannel: rwc.(*datachannel.DataChannel), + onDone: onDone, } // released when the controlMessageReader goroutine exits s.controlMessageReaderDone.Add(1) s.dataChannel.SetBufferedAmountLowThreshold(bufferedAmountLowThreshold) s.dataChannel.OnBufferedAmountLow(func() { - select { - case s.writeAvailable <- struct{}{}: - default: - } + s.notifyWriteStateChanged() + }) return s } func (s *stream) Close() error { s.mx.Lock() - if s.closeForShutdownErr != nil { - s.mx.Unlock() + isClosed := s.closeForShutdownErr != nil + s.mx.Unlock() + if isClosed { return nil } - s.mx.Unlock() closeWriteErr := s.CloseWrite() closeReadErr := s.CloseRead() @@ -166,11 +156,11 @@ func (s *stream) Close() error { func (s *stream) Reset() error { s.mx.Lock() - if s.closeForShutdownErr != nil { - s.mx.Unlock() + isClosed := s.closeForShutdownErr != nil + s.mx.Unlock() + if isClosed { return nil } - s.mx.Unlock() defer s.cleanup() cancelWriteErr := s.cancelWrite() @@ -189,10 +179,7 @@ func (s *stream) closeForShutdown(closeErr error) { s.closeForShutdownErr = closeErr s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) - select { - case s.sendStateChanged <- struct{}{}: - default: - } + s.notifyWriteStateChanged() } func (s *stream) SetDeadline(t time.Time) error { @@ -214,16 +201,10 @@ func (s *stream) processIncomingFlag(flag *pb.Message_Flag) { if s.sendState == sendStateSending || s.sendState == sendStateDataSent { s.sendState = sendStateReset } - select { - case s.sendStateChanged <- struct{}{}: - default: - } + s.notifyWriteStateChanged() case pb.Message_FIN_ACK: s.sendState = sendStateDataReceived - select { - case s.sendStateChanged <- struct{}{}: - default: - } + s.notifyWriteStateChanged() case pb.Message_FIN: if s.receiveState == receiveStateReceiving { s.receiveState = receiveStateDataRead @@ -285,8 +266,9 @@ func (s *stream) spawnControlMessageReader() { return } s.mx.Unlock() - if err := s.reader.ReadMsg(&msg); err != nil { - s.mx.Lock() + err := s.reader.ReadMsg(&msg) + s.mx.Lock() + if err != nil { // We have to manually manage deadline exceeded errors since pion/sctp can // return deadline exceeded error for cancelled deadlines // see: https://github.com/pion/sctp/pull/290/files @@ -295,7 +277,6 @@ func (s *stream) spawnControlMessageReader() { } return } - s.mx.Lock() s.processIncomingFlag(msg.Flag) } }() diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index 002ebac0ec..f9df06eca2 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -35,8 +35,9 @@ func (s *stream) Read(b []byte) (int, error) { // load the next message s.mx.Unlock() var msg pb.Message - if err := s.reader.ReadMsg(&msg); err != nil { - s.mx.Lock() + err := s.reader.ReadMsg(&msg) + s.mx.Lock() + if err != nil { // connection was closed if s.closeForShutdownErr != nil { return 0, s.closeForShutdownErr @@ -61,7 +62,6 @@ func (s *stream) Read(b []byte) (int, error) { } return 0, err } - s.mx.Lock() s.nextMessage = &msg } diff --git a/p2p/transport/webrtc/stream_write.go b/p2p/transport/webrtc/stream_write.go index b0b2837d91..82d4ac287d 100644 --- a/p2p/transport/webrtc/stream_write.go +++ b/p2p/transport/webrtc/stream_write.go @@ -76,12 +76,10 @@ func (s *stream) Write(b []byte) (int, error) { if availableSpace < minMessageSize { s.mx.Unlock() select { - case <-s.writeAvailable: case <-writeDeadlineChan: s.mx.Lock() return n, os.ErrDeadlineExceeded - case <-s.sendStateChanged: - case <-s.writeDeadlineUpdated: + case <-s.writeStateChanged: } s.mx.Lock() continue @@ -108,10 +106,7 @@ func (s *stream) SetWriteDeadline(t time.Time) error { s.mx.Lock() defer s.mx.Unlock() s.writeDeadline = t - select { - case s.writeDeadlineUpdated <- struct{}{}: - default: - } + s.notifyWriteStateChanged() return nil } @@ -134,10 +129,7 @@ func (s *stream) cancelWrite() error { return nil } s.sendState = sendStateReset - select { - case s.sendStateChanged <- struct{}{}: - default: - } + s.notifyWriteStateChanged() if err := s.writer.WriteMsg(&pb.Message{Flag: pb.Message_RESET.Enum()}); err != nil { return err } @@ -152,12 +144,16 @@ func (s *stream) CloseWrite() error { return nil } s.sendState = sendStateDataSent - select { - case s.sendStateChanged <- struct{}{}: - default: - } + s.notifyWriteStateChanged() if err := s.writer.WriteMsg(&pb.Message{Flag: pb.Message_FIN.Enum()}); err != nil { return err } return nil } + +func (s *stream) notifyWriteStateChanged() { + select { + case s.writeStateChanged <- struct{}{}: + default: + } +} From 2924291c5a66fca4d99a1a99964ab20f3e22880b Mon Sep 17 00:00:00 2001 From: Sukun Date: Tue, 20 Feb 2024 09:14:42 +0530 Subject: [PATCH 29/30] use crypto/rand --- p2p/transport/webrtc/transport_test.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/p2p/transport/webrtc/transport_test.go b/p2p/transport/webrtc/transport_test.go index 807a498491..3bfcaaf5a7 100644 --- a/p2p/transport/webrtc/transport_test.go +++ b/p2p/transport/webrtc/transport_test.go @@ -2,6 +2,7 @@ package libp2pwebrtc import ( "context" + "crypto/rand" "encoding/hex" "fmt" "io" @@ -13,22 +14,17 @@ import ( "testing" "time" - manet "github.com/multiformats/go-multiaddr/net" - - quicproxy "github.com/quic-go/quic-go/integrationtests/tools/proxy" - - "golang.org/x/crypto/sha3" - "golang.org/x/exp/rand" - "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" - ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" "github.com/multiformats/go-multibase" "github.com/multiformats/go-multihash" + quicproxy "github.com/quic-go/quic-go/integrationtests/tools/proxy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/crypto/sha3" ) func getTransport(t *testing.T, opts ...Option) (*WebRTCTransport, peer.ID) { From 8992b8c91e46e2698e1f72d234284375ddbad1dd Mon Sep 17 00:00:00 2001 From: sukun Date: Tue, 20 Feb 2024 18:08:56 +0530 Subject: [PATCH 30/30] disallow SetReadDeadline after read half is closed --- p2p/transport/webrtc/stream.go | 29 ++++++++++++++--------------- p2p/transport/webrtc/stream_read.go | 13 ++++++++++++- p2p/transport/webrtc/stream_test.go | 2 +- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/p2p/transport/webrtc/stream.go b/p2p/transport/webrtc/stream.go index a78604b99e..45641321dc 100644 --- a/p2p/transport/webrtc/stream.go +++ b/p2p/transport/webrtc/stream.go @@ -142,15 +142,17 @@ func (s *stream) Close() error { s.Reset() return errors.Join(closeWriteErr, closeReadErr) } + s.mx.Lock() - s.controlMessageReaderEndTime = time.Now().Add(maxFINACKWait) + if s.controlMessageReaderEndTime.IsZero() { + s.controlMessageReaderEndTime = time.Now().Add(maxFINACKWait) + s.setDataChannelReadDeadline(time.Now().Add(-1 * time.Hour)) + go func() { + s.controlMessageReaderDone.Wait() + s.cleanup() + }() + } s.mx.Unlock() - s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) - - go func() { - s.controlMessageReaderDone.Wait() - s.cleanup() - }() return nil } @@ -165,10 +167,8 @@ func (s *stream) Reset() error { defer s.cleanup() cancelWriteErr := s.cancelWrite() closeReadErr := s.CloseRead() - if cancelWriteErr != nil { - return cancelWriteErr - } - return closeReadErr + s.setDataChannelReadDeadline(time.Now().Add(-1 * time.Hour)) + return errors.Join(closeReadErr, cancelWriteErr) } func (s *stream) closeForShutdown(closeErr error) { @@ -178,7 +178,6 @@ func (s *stream) closeForShutdown(closeErr error) { defer s.mx.Unlock() s.closeForShutdownErr = closeErr - s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) s.notifyWriteStateChanged() } @@ -230,18 +229,18 @@ func (s *stream) spawnControlMessageReader() { go func() { defer s.controlMessageReaderDone.Done() // cleanup the sctp deadline timer goroutine - defer s.SetReadDeadline(time.Time{}) + defer s.setDataChannelReadDeadline(time.Time{}) setDeadline := func() bool { if s.controlMessageReaderEndTime.IsZero() || time.Now().Before(s.controlMessageReaderEndTime) { - s.SetReadDeadline(s.controlMessageReaderEndTime) + s.setDataChannelReadDeadline(s.controlMessageReaderEndTime) return true } return false } // Unblock any Read call waiting on reader.ReadMsg - s.SetReadDeadline(time.Now().Add(-1 * time.Hour)) + s.setDataChannelReadDeadline(time.Now().Add(-1 * time.Hour)) s.readerMx.Lock() // We have the lock any readers blocked on reader.ReadMsg have exited. diff --git a/p2p/transport/webrtc/stream_read.go b/p2p/transport/webrtc/stream_read.go index f9df06eca2..80d99ea91c 100644 --- a/p2p/transport/webrtc/stream_read.go +++ b/p2p/transport/webrtc/stream_read.go @@ -87,7 +87,18 @@ func (s *stream) Read(b []byte) (int, error) { } } -func (s *stream) SetReadDeadline(t time.Time) error { return s.dataChannel.SetReadDeadline(t) } +func (s *stream) SetReadDeadline(t time.Time) error { + s.mx.Lock() + defer s.mx.Unlock() + if s.receiveState == receiveStateReceiving { + s.setDataChannelReadDeadline(t) + } + return nil +} + +func (s *stream) setDataChannelReadDeadline(t time.Time) error { + return s.dataChannel.SetReadDeadline(t) +} func (s *stream) CloseRead() error { s.mx.Lock() diff --git a/p2p/transport/webrtc/stream_test.go b/p2p/transport/webrtc/stream_test.go index 884e688bad..8f1ec165cf 100644 --- a/p2p/transport/webrtc/stream_test.go +++ b/p2p/transport/webrtc/stream_test.go @@ -451,7 +451,7 @@ func TestStreamConcurrentClose(t *testing.T) { func TestStreamResetAfterClose(t *testing.T) { client, _ := getDetachedDataChannels(t) - done := make(chan bool, 1) + done := make(chan bool, 2) clientStr := newStream(client.dc, client.rwc, func() { done <- true }) clientStr.Close()