http2: prevent transport deadlock due to WINDOW_UPDATE that exceeds limit

Transport currently deadlocks when receiving a WINDOW_UPDATE for a
non-zero stream that increments its window beyond the 2^31-1 bytes
limit.

This is because endStreamError is called to end the non-zero stream,
which tries to lock an already-locked mutex. Therefore, create and use
endStreamErrorLocked instead, which assumes the mutex is already locked.

Fixes golang/go#77331

Change-Id: Iea212f49a1f305d1bddefb8831dbaca00840870c
Reviewed-on: https://go-review.googlesource.com/c/net/+/739700
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Nicholas Husin <husin@google.com>
Reviewed-by: Damien Neil <dneil@google.com>

Author: nicholashusin

http2/transport.go

@@ -2779,6 +2779,11 @@ func (rl *clientConnReadLoop) endStreamError(cs *clientStream, err error) {
 	cs.abortStream(err)
 }
 
+func (rl *clientConnReadLoop) endStreamErrorLocked(cs *clientStream, err error) {
+	cs.readAborted = true
+	cs.abortStreamLocked(err)
+}
+
 // Constants passed to streamByID for documentation purposes.
 const (
 	headerOrDataFrame    = true
@@ -2946,7 +2951,7 @@ func (rl *clientConnReadLoop) processWindowUpdate(f *WindowUpdateFrame) error {
 	if !fl.add(int32(f.Increment)) {
 		// For stream, the sender sends RST_STREAM with an error code of FLOW_CONTROL_ERROR
 		if cs != nil {
-			rl.endStreamError(cs, StreamError{
+			rl.endStreamErrorLocked(cs, StreamError{
 				StreamID: f.StreamID,
 				Code:     ErrCodeFlowControl,
 			})

http2/transport_test.go

@@ -1881,6 +1881,29 @@ func testTransportResponseHeaderTimeout(t testing.TB, body bool) {
 	}
 }
 
+// https://go.dev/issue/77331
+func TestTransportWindowUpdateBeyondLimit(t *testing.T) {
+	synctestTest(t, testTransportWindowUpdateBeyondLimit)
+}
+func testTransportWindowUpdateBeyondLimit(t testing.TB) {
+	const windowIncrease uint32 = (1 << 31) - 1 // Will cause window to exceed limit of 2^31-1.
+	tc := newTestClientConn(t)
+	tc.greet()
+
+	req, _ := http.NewRequest("GET", "https://dummy.tld/", nil)
+	rt := tc.roundTrip(req)
+	tc.wantHeaders(wantHeader{
+		streamID:  rt.streamID(),
+		endStream: true,
+	})
+
+	tc.writeWindowUpdate(rt.streamID(), windowIncrease)
+	tc.wantRSTStream(rt.streamID(), ErrCodeFlowControl)
+
+	tc.writeWindowUpdate(0, windowIncrease)
+	tc.wantClosed()
+}
+
 func TestTransportDisableCompression(t *testing.T) {
 	const body = "sup"
 	ts := newTestServer(t, func(w http.ResponseWriter, r *http.Request) {