ttrpc client send hang, even ctx is timeout.

[ningmingxiao]

ttrpc send hang, even ctx is timeout

goroutine 6132118 [IO wait, 1818 minutes]:

internal/poll.runtime_pollWait(0x7f9e8013bbd0, 0x77)

        /usr/local/go/src/runtime/netpoll.go:306 +0x89

internal/poll.(*pollDesc).wait(0xc000f2b900?, 0xc0021e6000?, 0x0)

        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32

internal/poll.(*pollDesc).waitWrite(...)

        /usr/local/go/src/internal/poll/fd_poll_runtime.go:93

internal/poll.(*FD).Write(0xc000f2b900, {0xc0021e6000, 0x74, 0x1000})

        /usr/local/go/src/internal/poll/fd_unix.go:391 +0x2f6

net.(*netFD).Write(0xc000f2b900, {0xc0021e6000?, 0xc000abf260?, 0x41c4f1?})

        /usr/local/go/src/net/fd_posix.go:96 +0x29

net.(*conn).Write(0xc00128a238, {0xc0021e6000?, 0x5222f5?, 0x7f9e8008bbd0?})

        /usr/local/go/src/net/net.go:195 +0x45

bufio.(*Writer).Flush(0xc0021de800)

        /usr/local/go/src/bufio/bufio.go:628 +0x62

github.com/containerd/ttrpc.(*channel).send(0xc0021de840, 0x21a7680?, 0xc0?, 0x0?, {0xc002fa2070, 0x6a, 0x6a})

        github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/channel.go:161 +0x98

github.com/containerd/ttrpc.(*Client).createStream(0xc001b59290, 0x0?, {0xc002fa2070, 0x6a, 0x6a})

        github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:370 +0x245

github.com/containerd/ttrpc.(*Client).dispatch(0xc001b59290, {0x1cba358, 0xc005081d70}, 0xc000462800?, 0xc0019feda0?)

        github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:480 +0x95

github.com/containerd/ttrpc.defaultClientInterceptor({0x1cba358?, 0xc005081d70?}, 0x2b5d3c0?, 0xc0056ecc40?, 0xc000abf5e0?, 0x45a7b1?)

        github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26

github.com/containerd/ttrpc.(*Client).Call(0xc001b59290, {0x1cba358, 0xc005081d70}, {0x1a4b86d, 0x17}, {0x1a2ad25, 0x5}, {0x18ddd60?, 0xc003a836d0?}, {0x1993ba0, ...})

        github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353

github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).State(0xc00128b950, {0x1cba358, 0xc005081d70}, 0x2b5d3c0?)

        github.com/containerd/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:169 +0x98

github.com/containerd/containerd/runtime/v2.(*shimTask).State(0xc000f2f960, {0x1cba358, 0xc005081d70})

        github.com/containerd/containerd/runtime/v2/shim.go:698 +0xd4

github.com/containerd/containerd/services/tasks.getProcessState({0x1cba390?, 0xc005081aa0?}, {0x7f9e800dcf10, 0xc000f2f960})

        github.com/containerd/containerd/services/tasks/local.go:340 +0xef

github.com/containerd/containerd/services/tasks.(*local).Get(0xc003a83540?, {0x1cba390, 0xc005081aa0}, 0xc003a83540, {0x194a040?, 0x19be760?, 0x40f000?})

        github.com/containerd/containerd/services/tasks/local.go:386 +0xe5

github.com/containerd/containerd/services/tasks.(*service).Get(0x18e0160?, {0x1cba390?, 0xc005081aa0?}, 0x0?)

        github.com/containerd/containerd/services/tasks/service.go:86 +0x33

github.com/containerd/containerd/api/services/tasks/v1._Tasks_Get_Handler({0x19be760?, 0xc000014fc0}, {0x1cba390, 0xc005081aa0}, 0xc0036e90a0, 0x0)

        github.com/containerd/containerd/api/services/tasks/v1/tasks_grpc.pb.go:384 +0x170

google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003e4000, {0x1cc2a40, 0xc003e6e000}, 0xc002ae5e60, 0xc00047de90, 0x2abe980, 0x0)

        github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:1336 +0xd33

google.golang.org/grpc.(*Server).handleStream(0xc0003e4000, {0x1cc2a40, 0xc003e6e000}, 0xc002ae5e60, 0x0)

        github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:1704 +0xa36

google.golang.org/grpc.(*Server).serveStreams.func1.2()

        github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:965 +0x98

created by google.golang.org/grpc.(*Server).serveStreams.func1

        github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:963 +0x28a

[kevpar]

What version of ttrpc are you using on the client? From your line numbers, it appears you may be using an old version.

We fixed a deadlock in the client in v1.2.4, it could be what you are hitting: 1b4f6f8

[ningmingxiao]

some stack

goroutine 4796 [IO wait]:
internal/poll.runtime_pollWait(0x7f9e8013bea0, 0x72)
        /usr/local/go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000138e80?, 0xc00075f000?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000138e80, {0xc00075f000, 0x1000, 0x1000})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000138e80, {0xc00075f000?, 0xc00336dcf8?, 0x2?})
        /usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126858, {0xc00075f000?, 0x418313?, 0xc000905918?})
        /usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000d60060, {0xc000731220, 0xa, 0xc000bb14a0?})
        /usr/local/go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x1ca1b00, 0xc000d60060}, {0xc000731220, 0xa, 0xa}, 0xa)
        /usr/local/go/src/io/io.go:332 +0x9a
io.ReadFull(...)
        /usr/local/go/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000731220?, 0xa?, 0x30?}, {0x1ca1b00?, 0xc000d60060?})
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000731200)
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0008047e0)
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0008047e0)
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6


goroutine 6132118 [IO wait, 1818 minutes]:
internal/poll.runtime_pollWait(0x7f9e8013bbd0, 0x77)
        /usr/local/go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000f2b900?, 0xc0021e6000?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitWrite(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:93
internal/poll.(*FD).Write(0xc000f2b900, {0xc0021e6000, 0x74, 0x1000})
        /usr/local/go/src/internal/poll/fd_unix.go:391 +0x2f6
net.(*netFD).Write(0xc000f2b900, {0xc0021e6000?, 0xc000abf260?, 0x41c4f1?})
        /usr/local/go/src/net/fd_posix.go:96 +0x29
net.(*conn).Write(0xc00128a238, {0xc0021e6000?, 0x5222f5?, 0x7f9e8008bbd0?})
        /usr/local/go/src/net/net.go:195 +0x45
bufio.(*Writer).Flush(0xc0021de800)
        /usr/local/go/src/bufio/bufio.go:628 +0x62
github.com/containerd/ttrpc.(*channel).send(0xc0021de840, 0x21a7680?, 0xc0?, 0x0?, {0xc002fa2070, 0x6a, 0x6a})
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/channel.go:161 +0x98
github.com/containerd/ttrpc.(*Client).createStream(0xc001b59290, 0x0?, {0xc002fa2070, 0x6a, 0x6a})
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:370 +0x245
github.com/containerd/ttrpc.(*Client).dispatch(0xc001b59290, {0x1cba358, 0xc005081d70}, 0xc000462800?, 0xc0019feda0?)
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:480 +0x95
github.com/containerd/ttrpc.defaultClientInterceptor({0x1cba358?, 0xc005081d70?}, 0x2b5d3c0?, 0xc0056ecc40?, 0xc000abf5e0?, 0x45a7b1?)
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001b59290, {0x1cba358, 0xc005081d70}, {0x1a4b86d, 0x17}, {0x1a2ad25, 0x5}, {0x18ddd60?, 0xc003a836d0?}, {0x1993ba0, ...})
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).State(0xc00128b950, {0x1cba358, 0xc005081d70}, 0x2b5d3c0?)
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:169 +0x98
		

sync.runtime_SemacquireMutex(0xc00124b800?, 0x4b?, 0x0?)
        /usr/local/go/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc001b592cc)
        /usr/local/go/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
        /usr/local/go/src/sync/mutex.go:90
github.com/containerd/ttrpc.(*Client).createStream(0xc001b59290, 0x0?, {0xc000f390a0, 0x6a, 0x6a})
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:366 +0x1ce
github.com/containerd/ttrpc.(*Client).dispatch(0xc001b59290, {0x1cba358, 0xc0035bfdd0}, 0xc00124b800?, 0xc003836d30?)
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:480 +0x95
github.com/containerd/ttrpc.defaultClientInterceptor({0x1cba358?, 0xc0035bfdd0?}, 0x2b5d3c0?, 0xc00190ff80?, 0xc0053675e0?, 0x45a7b1?)
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001b59290, {0x1cba358, 0xc0035bfdd0}, {0x1a4b86d, 0x17}, {0x1a2ad25, 0x5}, {0x18ddd60?, 0xc003e172c0?}, {0x1993ba0, ...})
        /root/rpmbuild/BUILD/containerd.io-1.7.6/_build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353

It will hit 1b4f6f8
I don't see it was blocked at "s := c.getStream(sid)" （It will block at c.getStream(sid)）

func (c *Client) receiveLoop() error {
	for {
		select {
		case <-c.ctx.Done():
			return ErrClosed
		default:
			var (
				msg = &streamMessage{}
				err error
			)

			msg.header, msg.payload, err = c.channel.recv()
			if err != nil {
				_, ok := status.FromError(err)
				if !ok {
					// treat all errors that are not an rpc status as terminal.
					// all others poison the connection.
					return filterCloseErr(err)
				}
			}
			sid := streamID(msg.header.StreamID)
                        s := c.getStream(sid)

@kevpar i use https://github.com/containerd/ttrpc-rust as ttrpc server.

[ningmingxiao]

I review this pr 1b4f6f8

func (c *Client) createStream(flags uint8, b []byte) (*stream, error) {
	c.streamLock.Lock()

	// Check if closed since lock acquired to prevent adding
	// anything after cleanup completes
	select {
	case <-c.ctx.Done():
		c.streamLock.Unlock()
		return nil, ErrClosed
	default:
	}

	// Stream ID should be allocated at same time
	s := newStream(c.nextStreamID, c)
	c.streams[s.id] = s
	c.nextStreamID = c.nextStreamID + 2

	c.sendLock.Lock()
	defer c.sendLock.Unlock()
	c.streamLock.Unlock()

	if err := c.channel.send(uint32(s.id), messageTypeRequest, flags, b); err != nil {
		return s, filterCloseErr(err)
	}

	return s, nil
}

If c.channel.send return there will not be dead lock.

If reader don't have chance to read data(hang at c.getStream ) ,will c.channel.send hang ?

func (c *Client) receiveLoop() error {
	for {
		select {
		case <-c.ctx.Done():
			return ErrClosed
		default:
			var (
				msg = &streamMessage{}
				err error
			)

			msg.header, msg.payload, err = c.channel.recv()
			if err != nil {
				_, ok := status.FromError(err)
				if !ok {
					// treat all errors that are not an rpc status as terminal.
					// all others poison the connection.
					return filterCloseErr(err)
				}
			}
			sid := streamID(msg.header.StreamID)
			s := c.getStream(sid)
			if s == nil {
				logrus.WithField("stream", sid).Errorf("ttrpc: received message on inactive stream")
				continue
			}

			if err != nil {
				s.closeWithError(err)
			} else {
				if err := s.receive(c.ctx, msg); err != nil {
					logrus.WithError(err).WithField("stream", sid).Errorf("ttrpc: failed to handle message")
				}
			}
		}
	}
}

can you show me your testcase ?
@kevpar thank you.

[novahe]

We also encountered this issue in v1.2.7(containerd 1.7.29 ), where the crictl ps command was stuck.

goroutine 45534323 [sync.Mutex.Lock, 799 minutes]:
internal/sync.runtime_SemacquireMutex(0x2?, 0x9?, 0xc014b1ba80?)
	/usr/local/go/src/runtime/sema.go:95 +0x25
internal/sync.(*Mutex).lockSlow(0xc011179bcc)
	/usr/local/go/src/internal/sync/mutex.go:149 +0x15d
internal/sync.(*Mutex).Lock(...)
	/usr/local/go/src/internal/sync/mutex.go:70
sync.(*Mutex).Lock(...)
	/usr/local/go/src/sync/mutex.go:46
github.com/containerd/ttrpc.(*Client).createStream(0xc011179b90, 0x0, {0xc015df7c20, 0x90, 0x90})
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:394 +0x74
github.com/containerd/ttrpc.(*Client).dispatch(0xc011179b90, {0x55c6b4460258, 0xc0188a68c0}, 0x55c6b4213b60?, 0xc0176086e0)
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:538 +0x86
github.com/containerd/ttrpc.defaultClientInterceptor({0x55c6b4460258?, 0xc0188a68c0?}, 0x55c6b534d4c0?, 0xc01308d260?, 0x55c6b42c84e0?, 0xc01a28c008?)
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x22
github.com/containerd/ttrpc.(*Client).Call(0xc011179b90, {0x55c6b4460258, 0xc0188a68c0}, {0x55c6b3cc5798, 0x17}, {0x55c6b3ca1c62, 0x5}, {0x55c6b42cb060?, 0xc017608690?}, {0x55c6b4392cc0, ...})
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/github.com/containerd/ttrpc/client.go:173 +0x323
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).State(0xc003298688, {0x55c6b4460258, 0xc0188a68c0}, 0xc017608690)
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/github.com/containerd/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:169 +0x92
github.com/containerd/containerd/runtime/v2.(*shimTask).State(0xc01101b360, {0x55c6b4460258, 0xc0188a68c0})
	/tmp/containerd-build/src/github.com/containerd/containerd/runtime/v2/shim.go:717 +0xd1
github.com/containerd/containerd/services/tasks.getProcessState({0x55c6b44601b0?, 0xc01a905770?}, {0x7f437c48f0d8, 0xc01101b360})
	/tmp/containerd-build/src/github.com/containerd/containerd/services/tasks/local.go:367 +0xcf
github.com/containerd/containerd/services/tasks.(*local).Get(0xa8c9776?, {0x55c6b44601b0, 0xc01a905770}, 0xc0176085a0, {0xc0026b1fc0?, 0x40?, 0xc015192540?})
	/tmp/containerd-build/src/github.com/containerd/containerd/services/tasks/local.go:413 +0xc5
github.com/containerd/containerd.(*container).loadTask(0xc0009d96c0, {0x55c6b44601b0, 0xc01a905770}, 0x0)
	/tmp/containerd-build/src/github.com/containerd/containerd/container.go:423 +0xaf
github.com/containerd/containerd.(*container).Task(0x55c6b44601b0?, {0x55c6b44601b0?, 0xc01a905770?}, 0xc0009d96c0?)
	/tmp/containerd-build/src/github.com/containerd/containerd/container.go:192 +0x1d
github.com/containerd/containerd/pkg/cri/sbserver.(*criService).updateContainerResources(_, {_, _}, {{{0xc0198b7840, 0x40}, {0xc003f47800, 0x74}, {0xc00725cd40, 0x40}, 0xc00037ba40, ...}, ...}, ...)
	/tmp/containerd-build/src/github.com/containerd/containerd/pkg/cri/sbserver/container_update_resources.go:126 +0x449
github.com/containerd/containerd/pkg/cri/sbserver.(*criService).UpdateContainerResources.func1({0xa5, 0x189d4d50a5c69e57, 0x189d4d50af43d47a, 0x0, 0x0, {0x0, 0x0}, {0x0, 0x0}, 0x0, ...})
	/tmp/containerd-build/src/github.com/containerd/containerd/pkg/cri/sbserver/container_update_resources.go:65 +0xfb
github.com/containerd/containerd/pkg/cri/store/container.(*statusStorage).UpdateSync(0xc016f89300, 0xc01a9057a0)
	/tmp/containerd-build/src/github.com/containerd/containerd/pkg/cri/store/container/status.go:265 +0x135
github.com/containerd/containerd/pkg/cri/sbserver.(*criService).UpdateContainerResources(0xc000892f08, {0x55c6b44601b0, 0xc01a905770}, 0xc01a905530)
	/tmp/containerd-build/src/github.com/containerd/containerd/pkg/cri/sbserver/container_update_resources.go:64 +0x5c2
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).UpdateContainerResources(0xc000597500, {0x55c6b44601b0, 0xc01a905500}, 0xc01a905530)
	/tmp/containerd-build/src/github.com/containerd/containerd/pkg/cri/instrument/instrumented_service.go:936 +0x243
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_UpdateContainerResources_Handler.func1({0x55c6b44601b0?, 0xc01a905500?}, {0x55c6b43b9dc0?, 0xc01a905530?})
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10953 +0xce
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55c6b44601b0, 0xc01a905500}, {0x55c6b43b9dc0, 0xc01a905530}, 0x400000004181e00?, 0xc010725d58)
	/tmp/containerd-build/src/github.com/containerd/containerd/services/server/namespace.go:31 +0x65
github.com/containerd/containerd/services/server.New.ChainUnaryServer.func6.1.1({0x55c6b44601b0?, 0xc01a905500?}, {0x55c6b43b9dc0?, 0xc01a905530?})
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x34
github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).UnaryServerInterceptor.func3({0x55c6b44601b0, 0xc01a905500}, {0x55c6b43b9dc0, 0xc01a905530}, 0xc01a28c008?, 0xc01101ac60)
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x77
github.com/containerd/containerd/services/server.New.ChainUnaryServer.func6.1.1({0x55c6b44601b0?, 0xc01a905500?}, {0x55c6b43b9dc0?, 0xc01a905530?})
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x34
github.com/containerd/containerd/services/server.New.ChainUnaryServer.func6({0x55c6b44601b0, 0xc01a905500}, {0x55c6b43b9dc0, 0xc01a905530}, 0x55c6b290dca5?, 0x78?)
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xb4
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_UpdateContainerResources_Handler({0x55c6b43fd7c0, 0xc000597500}, {0x55c6b44601b0, 0xc01a905500}, 0xc014b1aa80, 0xc000632510)
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10955 +0x143
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002ce200, {0x55c6b44601b0, 0xc01a905410}, {0x55c6b446af40, 0xc001ab8000}, 0xc0164dd560, 0xc0003e7dd0, 0x55c6b52ac2e0, 0x0)
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:1394 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc0002ce200, {0x55c6b446af40, 0xc001ab8000}, 0xc0164dd560)
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:1805 +0xe8a
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:1029 +0x7f
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 8999
	/tmp/containerd-build/src/github.com/containerd/containerd/vendor/google.golang.org/grpc/server.go:1040 +0x11d

[ningmingxiao]

@novahe can you show me containerd full stack ?

[novahe]

@ningmingxiao We are on an internal network; these cannot be copied to the external network. If you want to see something else from other goroutines, I can check it for you.

[ningmingxiao]

can you show who hold the lock and shim stack ? kill -10 pidof shimpid will print stack in containerd log @novahe