goroutine profile: total 43848

21740 @ 0x43e0e5 0x44fc65 0x44fc4e 0x471a27 0xc1d97f 0xc1d707 0x475921

# 0x471a26 sync.runtime_SemacquireMutex+0x46
/opt/go/src/runtime/sema.go:71
# 0xc1d97e sync.(*RWMutex).RLock+0x5e
/opt/go/src/sync/rwmutex.go:63
# 0xc1d706 github.com/sasha-s/go-deadlock.lock.func1+0x26
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:153

21581 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe24b3f 0xe24ad7 0xe23385 0xe1e2a5 0xe1dc94

0xdf133d 0xdf1dc4 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe24b3e github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xbe
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe24ad6 ygame/pkg/net.(*pipeMgr).findRandPipe+0x56
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:738
# 0xe23384 ygame/pkg/net.(*pipeMgr).forwardPkg+0x164
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:490
# 0xe1e2a4 ygame/pkg/net.doForwardSSNetPkg+0x204
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:250
# 0xe1dc93 ygame/pkg/net.ForwardToClusterRandom+0xf3
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:150
# 0xdf133c ygame/pkg/metrics.(*pipeConnector).Call+0xdc
/home/hk5/perforce/goworks/src/ygame/pkg/metrics/metrics_connector.go:84
# 0xdf1dc3 ygame/pkg/metrics.(*MetricClient).send.func1+0x63
/home/hk5/perforce/goworks/src/ygame/pkg/metrics/metrics_client.go:226

182 @ 0x43e0e5 0x44fc65 0x44fc4e 0x471a27 0x48c845 0x48dfb0 0x48df42 0xc1d707

0x475921
# 0x471a26 sync.runtime_SemacquireMutex+0x46
/opt/go/src/runtime/sema.go:71
# 0x48c844 sync.(*Mutex).lockSlow+0x104
/opt/go/src/sync/mutex.go:138
# 0x48dfaf sync.(*Mutex).Lock+0x8f
/opt/go/src/sync/mutex.go:81
# 0x48df41 sync.(*RWMutex).Lock+0x21
/opt/go/src/sync/rwmutex.go:111
# 0xc1d706 github.com/sasha-s/go-deadlock.lock.func1+0x26
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:153

68 @ 0x43e0e5 0x44f197 0xc1ad3f 0xc62a92 0xc62a33 0xe940ec 0xc1b430 0xe22711

0xe226a3 0xe285c5 0xe28596 0xe11a63 0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xc62a91 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xf51
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xc62a32 ygame/pkg/alerter.ReportCrash+0xef2
/home/hk5/perforce/goworks/src/ygame/pkg/alerter/alerter.go:125
# 0xe940eb ygame/pkg/storm/app.(*App).prepare.func1+0x8b
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:129
# 0xc1b42f github.com/sasha-s/go-deadlock.lock+0x86f
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:194
# 0xe22710 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xb0
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xe226a2 ygame/pkg/net.(*pipeMgr).pingPipe+0x42
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:388
# 0xe285c4 ygame/pkg/net.onRecvPing+0x884
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:616
# 0xe28595 ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x855
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:178
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

61 @ 0x43e0e5 0x40848f 0x4080cb 0xc1b44a 0xe22711 0xe226a3 0xe285c5 0xe28596

0xe11a63 0xc63586 0x475921
# 0xc1b449 github.com/sasha-s/go-deadlock.lock+0x889
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:195
# 0xe22710 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xb0
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xe226a2 ygame/pkg/net.(*pipeMgr).pingPipe+0x42
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:388
# 0xe285c4 ygame/pkg/net.onRecvPing+0x884
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:616
# 0xe28595 ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x855
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:178
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

48 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe20f99 0xe20f2f 0xe27fa9 0xe27f7f 0xe11a63

0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe20f98 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xb8
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe20f2e ygame/pkg/net.(*pipeMgr).getSvrType+0x4e
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:191
# 0xe27fa8 ygame/pkg/net.getConnSvrType+0x268
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:628
# 0xe27f7e ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x23e
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:155
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

25 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe24b3f 0xe24ad7 0xe23645 0xe2e1bb 0xe2e1c6

0xe2cbee 0xe28394 0xe11a63 0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe24b3e github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xbe
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe24ad6 ygame/pkg/net.(*pipeMgr).findRandPipe+0x56
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:738
# 0xe23644 ygame/pkg/net.(*pipeMgr).sendDataByRand+0x44
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:520
# 0xe2e1ba ygame/pkg/net.SendDataByRandom+0x153a
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:267
# 0xe2e1c5 ygame/internal/route/logic.RouteTcpBusiness.routePkg+0x1545

/home/hk5/perforce/goworks/src/ygame/internal/route/logic/logic_handler.go:319
# 0xe2cbed ygame/internal/route/logic.RouteTcpBusiness.DealConnMsg+0x5cd

/home/hk5/perforce/goworks/src/ygame/internal/route/logic/logic_handler.go:184
# 0xe28393 ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x653
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:176
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

19 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe22711 0xe226a3 0xe285c5 0xe28596 0xe11a63

0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe22710 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xb0
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xe226a2 ygame/pkg/net.(*pipeMgr).pingPipe+0x42
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:388
# 0xe285c4 ygame/pkg/net.onRecvPing+0x884
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:616
# 0xe28595 ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x855
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:178
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

16 @ 0x43e0e5 0x44f197 0xe110b5 0xc63586 0x475921

# 0xe110b4 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg+0x174
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:255
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

16 @ 0x43e0e5 0x44f197 0xe11435 0xc63586 0x475921

# 0xe11434 ygame/pkg/storm/stnet/server.(*TcpServer).handleWritePkg+0x174
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:282
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

15 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe22918 0xe228a5 0xe20705 0xe2882f 0xe118f3

0xe108df 0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe22917 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xf7
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xe228a4 ygame/pkg/net.(*pipeMgr).detachPipeConnection+0x84
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:410
# 0xe20704 ygame/pkg/net.onRecvConnClose+0x44
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:602
# 0xe2882e ygame/pkg/net.(*ygameTcpBusiness).OnClose+0x12e
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:191
# 0xe118f2 ygame/pkg/storm/stnet/server.(*TcpServer).handleConn.func1+0x172
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:187
# 0xe108de ygame/pkg/storm/stnet/server.(*TcpServer).handleConn+0x6fe
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:239
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

15 @ 0x43e0e5 0x44f197 0xe2a35c 0xc63586 0x475921

# 0xe2a35b ygame/pkg/net.(*pipeSession).onEstablish.func2+0xfb
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:86
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
 /home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

7 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe20d55 0xe20ceb 0xe27f73 0xe27f79 0xe11a63

0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe20d54 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xb4
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe20cea ygame/pkg/net.(*pipeMgr).checkPipeAuth+0x4a
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:174
# 0xe27f72 ygame/pkg/net.checkConnAuth+0x232
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:624
# 0xe27f78 ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x238
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:151
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

6 @ 0x43e0e5 0x44f197 0xc1ad3f 0xc61bf2 0xc61b91 0xe940ec 0xc1b430 0xe22711

0xe226a3 0xe285c5 0xe28596 0xe11a63 0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xc61bf1 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xb1
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xc61b90 ygame/pkg/alerter.ReportCrash+0x50
/home/hk5/perforce/goworks/src/ygame/pkg/alerter/alerter.go:59
# 0xe940eb ygame/pkg/storm/app.(*App).prepare.func1+0x8b
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:129
# 0xc1b42f github.com/sasha-s/go-deadlock.lock+0x86f
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:194
# 0xe22710 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xb0
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xe226a2 ygame/pkg/net.(*pipeMgr).pingPipe+0x42
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:388
# 0xe285c4 ygame/pkg/net.onRecvPing+0x884
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:616
# 0xe28595 ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x855
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:178
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

6 @ 0x43e0e5 0x44f197 0xc1ad3f 0xc62a92 0xc62a33 0xe940ec 0xc1b430 0xe20d55

0xe20ceb 0xe27f73 0xe27f79 0xe11a63 0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xc62a91 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xf51
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xc62a32 ygame/pkg/alerter.ReportCrash+0xef2
/home/hk5/perforce/goworks/src/ygame/pkg/alerter/alerter.go:125
# 0xe940eb ygame/pkg/storm/app.(*App).prepare.func1+0x8b
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:129
# 0xc1b42f github.com/sasha-s/go-deadlock.lock+0x86f
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:194
# 0xe20d54 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xb4
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe20cea ygame/pkg/net.(*pipeMgr).checkPipeAuth+0x4a
 /home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:174
# 0xe27f72 ygame/pkg/net.checkConnAuth+0x232
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:624
# 0xe27f78 ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x238
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:151
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

4 @ 0x43e0e5 0x40848f 0x40810b 0xdc685a 0x475921

# 0xdc6859 ygame/bytedance/metrics.init.0.func1+0x59
/home/hk5/perforce/goworks/src/ygame/bytedance/metrics/cache.go:19

2 @ 0x43e0e5 0x40848f 0x4080cb 0xbe5bb2 0x475921

# 0xbe5bb1 github.com/cihub/seelog.(*bufferedWriter).flushPeriodically+0x51
/home/hk5/go/pkg/mod/github.com/cihub/[email protected]
d2c6e5aa9fbf/writers_bufferedwriter.go:153

2 @ 0x43e0e5 0x44f197 0xc1ad3f 0xc61bf2 0xc61b91 0xe940ec 0xc1b430 0xe20d55

0xe20ceb 0xe27f73 0xe27f79 0xe11a63 0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xc61bf1 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xb1
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xc61b90 ygame/pkg/alerter.ReportCrash+0x50
/home/hk5/perforce/goworks/src/ygame/pkg/alerter/alerter.go:59
# 0xe940eb ygame/pkg/storm/app.(*App).prepare.func1+0x8b
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:129
# 0xc1b42f github.com/sasha-s/go-deadlock.lock+0x86f
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:194
# 0xe20d54 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xb4
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe20cea ygame/pkg/net.(*pipeMgr).checkPipeAuth+0x4a
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:174
# 0xe27f72 ygame/pkg/net.checkConnAuth+0x232
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:624
# 0xe27f78 ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x238
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:151
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

2 @ 0x43e0e5 0x471bb8 0x471b8e 0x48a779 0xbce6ea 0xbce805 0x475921

# 0x471b8d sync.runtime_notifyListWait+0xcd
/opt/go/src/runtime/sema.go:513
# 0x48a778 sync.(*Cond).Wait+0x98
/opt/go/src/sync/cond.go:56
# 0xbce6e9 github.com/cihub/seelog.(*asyncLoopLogger).processItem+0x89
/home/hk5/go/pkg/mod/github.com/cihub/[email protected]
d2c6e5aa9fbf/behavior_asynclooplogger.go:50
# 0xbce804 github.com/cihub/seelog.(*asyncLoopLogger).processQueue+0x44
/home/hk5/go/pkg/mod/github.com/cihub/[email protected]
d2c6e5aa9fbf/behavior_asynclooplogger.go:63

1 @ 0x40e4d4 0x472125 0xe35a45 0x475921

# 0x472124 os/signal.signal_recv+0xa4 /opt/go/src/runtime/sigqueue.go:168
# 0xe35a44 os/signal.loop+0x24
 /opt/go/src/os/signal/signal_unix.go:23

1 @ 0x43e0e5 0x40848f 0x4080cb 0xdf3834 0x475921

# 0xdf3833 ygame/pkg/storm/stdb.(*RedisProxy).stat+0x93
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stdb/redis_proxy.go:242

1 @ 0x43e0e5 0x40848f 0x4080cb 0xe1918f 0xc63586 0x475921

# 0xe1918e ygame/pkg/storm/sthttp.(*SessionMgr).GC+0x4e
/home/hk5/perforce/goworks/src/ygame/pkg/storm/sthttp/session.go:172
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x40848f 0x4080cb 0xe2a039 0xc63586 0x475921

# 0xe2a038 ygame/pkg/net.InitPipeByServerID.func1+0x38
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:95
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x40848f 0x4080cb 0xe315cf 0xc63586 0x475921

# 0xe315ce ygame/internal/route/logic.InitCache.func1+0x4e

/home/hk5/perforce/goworks/src/ygame/internal/route/logic/route_obj_cache.go:65
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x40848f 0x4080cb 0xe9354c 0x475921

# 0xe9354b ygame/pkg/storm/app.(*App).stat+0x8b
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:304

1 @ 0x43e0e5 0x40848f 0x40810b 0xdc200c 0x475921

# 0xdc200b ygame/bytedance/metrics.flushClients+0x30b
/home/hk5/perforce/goworks/src/ygame/bytedance/metrics/metrics.go:167

1 @ 0x43e0e5 0x43667b 0x46fed5 0x4daf85 0x4dc075 0x4dc057 0x5ac8cf 0x5bc9f1

0x4b8047 0xe39f5d 0xe39e5d 0xe42dc5 0x475921
# 0x46fed4 internal/poll.runtime_pollWait+0x54
/opt/go/src/runtime/netpoll.go:222
# 0x4daf84 internal/poll.(*pollDesc).wait+0x44
/opt/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4dc074 internal/poll.(*pollDesc).waitRead+0x1d4
/opt/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4dc056 internal/poll.(*FD).Read+0x1b6
/opt/go/src/internal/poll/fd_unix.go:166
# 0x5ac8ce net.(*netFD).Read+0x4e
/opt/go/src/net/fd_posix.go:55
# 0x5bc9f0 net.(*conn).Read+0x90
/opt/go/src/net/net.go:183
# 0x4b8046 io.ReadAtLeast+0x86
/opt/go/src/io/io.go:328
# 0xe39f5c io.ReadFull+0x17c
/opt/go/src/io/io.go:347
# 0xe39e5c github.com/samuel/go-zookeeper/zk.(*Conn).recvLoop+0x7c
/home/hk5/go/pkg/mod/github.com/samuel/[email protected]
c4fab1ac1bec/zk/conn.go:827
# 0xe42dc4 github.com/samuel/go-zookeeper/zk.(*Conn).loop.func2+0x1e4
/home/hk5/go/pkg/mod/github.com/samuel/[email protected]
c4fab1ac1bec/zk/conn.go:523

1 @ 0x43e0e5 0x43667b 0x46fed5 0x4daf85 0x4dc075 0x4dc057 0x5ac8cf 0x5bc9f1

0x6cc9b8 0x475921
# 0x46fed4 internal/poll.runtime_pollWait+0x54
/opt/go/src/runtime/netpoll.go:222
# 0x4daf84 internal/poll.(*pollDesc).wait+0x44
/opt/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4dc074 internal/poll.(*pollDesc).waitRead+0x1d4
/opt/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4dc056 internal/poll.(*FD).Read+0x1b6
/opt/go/src/internal/poll/fd_unix.go:166
# 0x5ac8ce net.(*netFD).Read+0x4e
/opt/go/src/net/fd_posix.go:55
# 0x5bc9f0 net.(*conn).Read+0x90
/opt/go/src/net/net.go:183
# 0x6cc9b7 net/http.(*connReader).backgroundRead+0x57
/opt/go/src/net/http/server.go:672

1 @ 0x43e0e5 0x43667b 0x46fed5 0x4daf85 0x4dc075 0x4dc057 0x5ac8cf 0x5bc9f1

0x6ccf39 0x574f28 0x575c9d 0x575ef4 0x657476 0x6c716a 0x6c716b 0x6ce3dd 0x6d3065
0x475921
# 0x46fed4 internal/poll.runtime_pollWait+0x54
/opt/go/src/runtime/netpoll.go:222
# 0x4daf84 internal/poll.(*pollDesc).wait+0x44
/opt/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4dc074 internal/poll.(*pollDesc).waitRead+0x1d4
/opt/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4dc056 internal/poll.(*FD).Read+0x1b6
/opt/go/src/internal/poll/fd_unix.go:166
# 0x5ac8ce net.(*netFD).Read+0x4e
/opt/go/src/net/fd_posix.go:55
# 0x5bc9f0 net.(*conn).Read+0x90
/opt/go/src/net/net.go:183
# 0x6ccf38 net/http.(*connReader).Read+0x1b8
/opt/go/src/net/http/server.go:780
# 0x574f27 bufio.(*Reader).fill+0x107
/opt/go/src/bufio/bufio.go:101
# 0x575c9c bufio.(*Reader).ReadSlice+0x3c
/opt/go/src/bufio/bufio.go:360
# 0x575ef3 bufio.(*Reader).ReadLine+0x33
/opt/go/src/bufio/bufio.go:389
# 0x657475 net/textproto.(*Reader).readLineSlice+0xd5
/opt/go/src/net/textproto/reader.go:57
# 0x6c7169 net/textproto.(*Reader).ReadLine+0xa9
/opt/go/src/net/textproto/reader.go:38
# 0x6c716a net/http.readRequest+0xaa
/opt/go/src/net/http/request.go:1027
# 0x6ce3dc net/http.(*conn).readRequest+0x19c
/opt/go/src/net/http/server.go:966
# 0x6d3064 net/http.(*conn).serve+0x704
/opt/go/src/net/http/server.go:1858

1 @ 0x43e0e5 0x43667b 0x46fed5 0x4daf85 0x4dddf2 0x4dddd4 0x5adf05 0x5c6f12

0x5c58e5 0x6d80e5 0x6d7dfa 0xe9459c 0xe944e6 0x475921
# 0x46fed4 internal/poll.runtime_pollWait+0x54
/opt/go/src/runtime/netpoll.go:222
# 0x4daf84 internal/poll.(*pollDesc).wait+0x44
/opt/go/src/internal/poll/fd_poll_runtime.go:87
# 0x4dddf1 internal/poll.(*pollDesc).waitRead+0x211
/opt/go/src/internal/poll/fd_poll_runtime.go:92
# 0x4dddd3 internal/poll.(*FD).Accept+0x1f3
 /opt/go/src/internal/poll/fd_unix.go:401
# 0x5adf04 net.(*netFD).accept+0x44
/opt/go/src/net/fd_unix.go:172
# 0x5c6f11 net.(*TCPListener).accept+0x31
/opt/go/src/net/tcpsock_posix.go:139
# 0x5c58e4 net.(*TCPListener).Accept+0x64
/opt/go/src/net/tcpsock.go:261
# 0x6d80e4 net/http.(*Server).Serve+0x284
/opt/go/src/net/http/server.go:2961
# 0x6d7df9 net/http.(*Server).ListenAndServe+0xb9
/opt/go/src/net/http/server.go:2890
# 0xe9459b net/http.ListenAndServe+0x13b
/opt/go/src/net/http/server.go:3144
# 0xe944e5 ygame/pkg/storm/app.WithPprof.func2+0x85
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app_option.go:65

1 @ 0x43e0e5 0x44f197 0xc1ad3f 0xc61bf2 0xc61b91 0xe940ec 0xc1b430 0xc61bf2

0xc61b91 0xe940ec 0xc1b430 0xe22711 0xe226a3 0xe285c5 0xe28596 0xe11a63 0xc63586
0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xc61bf1 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xb1
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xc61b90 ygame/pkg/alerter.ReportCrash+0x50
/home/hk5/perforce/goworks/src/ygame/pkg/alerter/alerter.go:59
# 0xe940eb ygame/pkg/storm/app.(*App).prepare.func1+0x8b
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:129
# 0xc1b42f github.com/sasha-s/go-deadlock.lock+0x86f
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:194
# 0xc61bf1 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xb1
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xc61b90 ygame/pkg/alerter.ReportCrash+0x50
/home/hk5/perforce/goworks/src/ygame/pkg/alerter/alerter.go:59
# 0xe940eb ygame/pkg/storm/app.(*App).prepare.func1+0x8b
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:129
# 0xc1b42f github.com/sasha-s/go-deadlock.lock+0x86f
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:194
# 0xe22710 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xb0
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xe226a2 ygame/pkg/net.(*pipeMgr).pingPipe+0x42
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:388
# 0xe285c4 ygame/pkg/net.onRecvPing+0x884
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:616
# 0xe28595 ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x855
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:178
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x44f197 0xc1ad3f 0xc61bf2 0xc61b91 0xe940ec 0xc1b430 0xe24b3f

0xe24ad7 0xe23385 0xe1e2a5 0xe1dc94 0xdf133d 0xdf1dc4 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xc61bf1 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xb1
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xc61b90 ygame/pkg/alerter.ReportCrash+0x50
/home/hk5/perforce/goworks/src/ygame/pkg/alerter/alerter.go:59
# 0xe940eb ygame/pkg/storm/app.(*App).prepare.func1+0x8b
 /home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:129
# 0xc1b42f github.com/sasha-s/go-deadlock.lock+0x86f
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:194
# 0xe24b3e github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xbe
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe24ad6 ygame/pkg/net.(*pipeMgr).findRandPipe+0x56
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:738
# 0xe23384 ygame/pkg/net.(*pipeMgr).forwardPkg+0x164
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:490
# 0xe1e2a4 ygame/pkg/net.doForwardSSNetPkg+0x204
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:250
# 0xe1dc93 ygame/pkg/net.ForwardToClusterRandom+0xf3
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:150
# 0xdf133c ygame/pkg/metrics.(*pipeConnector).Call+0xdc
/home/hk5/perforce/goworks/src/ygame/pkg/metrics/metrics_connector.go:84
# 0xdf1dc3 ygame/pkg/metrics.(*MetricClient).send.func1+0x63
/home/hk5/perforce/goworks/src/ygame/pkg/metrics/metrics_client.go:226

1 @ 0x43e0e5 0x44f197 0xc1ad3f 0xc62a92 0xc62a33 0xe940ec 0xc1b430 0xe20f99

0xe20f2f 0xe27fa9 0xe27f7f 0xe11a63 0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xc62a91 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xf51
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xc62a32 ygame/pkg/alerter.ReportCrash+0xef2
/home/hk5/perforce/goworks/src/ygame/pkg/alerter/alerter.go:125
# 0xe940eb ygame/pkg/storm/app.(*App).prepare.func1+0x8b
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:129
# 0xc1b42f github.com/sasha-s/go-deadlock.lock+0x86f
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:194
# 0xe20f98 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xb8
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe20f2e ygame/pkg/net.(*pipeMgr).getSvrType+0x4e
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:191
# 0xe27fa8 ygame/pkg/net.getConnSvrType+0x268
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:628
# 0xe27f7e ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x23e
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:155
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x44f197 0xc1ad3f 0xc62a92 0xc62a33 0xe940ec 0xc1b430 0xe5586d

0xe55800 0xe56b25 0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xc62a91 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xf51
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xc62a32 ygame/pkg/alerter.ReportCrash+0xef2
/home/hk5/perforce/goworks/src/ygame/pkg/alerter/alerter.go:125
# 0xe940eb ygame/pkg/storm/app.(*App).prepare.func1+0x8b
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:129
# 0xc1b42f github.com/sasha-s/go-deadlock.lock+0x86f
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:194
# 0xe5586c github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xac
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xe557ff ygame/pkg/registry.(*registryZookeeper).reconnectFailConn+0x3f
/home/hk5/perforce/goworks/src/ygame/pkg/registry/registry_zookeeper.go:1260
# 0xe56b24 ygame/pkg/registry.(*registryZookeeper).RegularSync.func2+0x44
/home/hk5/perforce/goworks/src/ygame/pkg/registry/registry_zookeeper.go:292
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe212d0 0xe21263 0xe20192 0xe286b4 0xe11a63

0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe212cf github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0x1ef
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xe21262 ygame/pkg/net.(*pipeMgr).attachPipeConnection+0x182
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:247
# 0xe20191 ygame/pkg/net.onRecvConnAuth+0x151
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:568
# 0xe286b3 ygame/pkg/net.(*ygameTcpBusiness).OnProcPkg+0x973
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:143
# 0xe11a62 ygame/pkg/storm/stnet/server.(*TcpServer).handlePkg.func1+0xa2
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:267
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe22918 0xe228a5 0xe20705 0xe2882f 0xe118f3

0xe10a51 0xc63586 0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe22917 github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xf7
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xe228a4 ygame/pkg/net.(*pipeMgr).detachPipeConnection+0x84
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_mgr.go:410
# 0xe20704 ygame/pkg/net.onRecvConnClose+0x44
/home/hk5/perforce/goworks/src/ygame/pkg/net/pipe_business.go:602
# 0xe2882e ygame/pkg/net.(*ygameTcpBusiness).OnClose+0x12e
/home/hk5/perforce/goworks/src/ygame/pkg/net/ygame_tcp_business.go:191
# 0xe118f2 ygame/pkg/storm/stnet/server.(*TcpServer).handleConn.func1+0x172
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:187
# 0xe10a50 ygame/pkg/storm/stnet/server.(*TcpServer).handleConn+0x870
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:236
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe4e1ad 0xe4e140 0xe47bca 0xc63586 0x475921

# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe4e1ac github.com/sasha-s/go-deadlock.(*RWMutex).Lock+0xac
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:89
# 0xe4e13f ygame/pkg/registry.(*registryZookeeper).onZKReconnected+0x3f
/home/hk5/perforce/goworks/src/ygame/pkg/registry/registry_zookeeper.go:566
# 0xe47bc9 ygame/pkg/storm/zkcli.handleConnEvent+0x269
/home/hk5/perforce/goworks/src/ygame/pkg/storm/zkcli/zkcli.go:427
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe53825 0xe537c3 0xe50385 0xe4f90b 0xe48b96

0x475921
# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe53824 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xa4
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe537c2 ygame/pkg/registry.(*registryZookeeper).queryCacheCallbacks+0x42
/home/hk5/perforce/goworks/src/ygame/pkg/registry/registry_zookeeper.go:1066
# 0xe50384 ygame/pkg/registry.(*registryZookeeper).handleClusterChange+0x504
/home/hk5/perforce/goworks/src/ygame/pkg/registry/registry_zookeeper.go:764
# 0xe4f90a ygame/pkg/registry.(*registryZookeeper).onWatchDir+0x6aa
/home/hk5/perforce/goworks/src/ygame/pkg/registry/registry_zookeeper.go:729
# 0xe48b95 ygame/pkg/storm/zkcli.(*Conn).WatchDir.func1+0x255
/home/hk5/perforce/goworks/src/ygame/pkg/storm/zkcli/zkcli.go:370

1 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe54925 0xe548ae 0xe569b6 0xc63586 0x475921

# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe54924 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xe4
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe548ad ygame/pkg/registry.(*registryZookeeper).checkCacheDiff+0x6d
/home/hk5/perforce/goworks/src/ygame/pkg/registry/registry_zookeeper.go:1178
# 0xe569b5 ygame/pkg/registry.(*registryZookeeper).RegularSync.func1+0x1f5
/home/hk5/perforce/goworks/src/ygame/pkg/registry/registry_zookeeper.go:264
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x44f197 0xc1ad3f 0xe553b4 0xe55348 0xe4f3d3 0xe48b96 0x475921

# 0xc1ad3e github.com/sasha-s/go-deadlock.lock+0x17e
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:159
# 0xe553b3 github.com/sasha-s/go-deadlock.(*RWMutex).RLock+0xb3
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:110
# 0xe55347 ygame/pkg/registry.(*registryZookeeper).checkSelfPath+0x47
/home/hk5/perforce/goworks/src/ygame/pkg/registry/registry_zookeeper.go:1238
# 0xe4f3d2 ygame/pkg/registry.(*registryZookeeper).onWatchDir+0x172
/home/hk5/perforce/goworks/src/ygame/pkg/registry/registry_zookeeper.go:688
# 0xe48b95 ygame/pkg/storm/zkcli.(*Conn).WatchDir.func1+0x255
/home/hk5/perforce/goworks/src/ygame/pkg/storm/zkcli/zkcli.go:370

1 @ 0x43e0e5 0x44f197 0xc661bc 0x475921

# 0xc661bb ygame/pkg/common.(*ActionStat).Start.func1+0x15b
/home/hk5/perforce/goworks/src/ygame/pkg/common/action_stat.go:142

1 @ 0x43e0e5 0x44f197 0xdf0bdb 0x475921

# 0xdf0bda ygame/pkg/metrics.(*MetricClient).mainLoop+0x11a
/home/hk5/perforce/goworks/src/ygame/pkg/metrics/metrics_client.go:175

1 @ 0x43e0e5 0x44f197 0xe2741b 0xc63586 0x475921

# 0xe2741a ygame/pkg/net.run+0xda
/home/hk5/perforce/goworks/src/ygame/pkg/net/ws_conn_mgr.go:60
# 0xc63585 ygame/pkg/goroutine.Go.func1+0x65
/home/hk5/perforce/goworks/src/ygame/pkg/goroutine/go.go:18

1 @ 0x43e0e5 0x44f197 0xe39a88 0xe42a93 0x475921

# 0xe39a87 github.com/samuel/go-zookeeper/zk.(*Conn).sendLoop+0x127
/home/hk5/go/pkg/mod/github.com/samuel/[email protected]
c4fab1ac1bec/zk/conn.go:792
# 0xe42a92 github.com/samuel/go-zookeeper/zk.(*Conn).loop.func1+0x52
/home/hk5/go/pkg/mod/github.com/samuel/[email protected]
c4fab1ac1bec/zk/conn.go:509

1 @ 0x43e0e5 0x44f197 0xe48093 0x475921

# 0xe48092 ygame/pkg/storm/zkcli.(*Conn).Watch.func1+0x272
/home/hk5/perforce/goworks/src/ygame/pkg/storm/zkcli/zkcli.go:234
1 @ 0x43e0e5 0x44fc65 0x44fc4e 0x471905 0x48e3e5 0xe0fdbf 0xe955f3 0xe93462
0x475921
# 0x471904 sync.runtime_Semacquire+0x44
/opt/go/src/runtime/sema.go:56
# 0x48e3e4 sync.(*WaitGroup).Wait+0x64
/opt/go/src/sync/waitgroup.go:130
# 0xe0fdbe ygame/pkg/storm/stnet/server.(*TcpServer).Stop+0x5e
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stnet/server/server_tcp.go:95
# 0xe955f2 ygame/internal/route.(*serviceTcp).Close+0x32
/home/hk5/perforce/goworks/src/ygame/internal/route/tcp_service.go:104
# 0xe93461 ygame/pkg/storm/app.(*App).listenInterupt+0x141
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:292

1 @ 0x43e0e5 0x44fc65 0x44fc4e 0x471905 0x48e3e5 0xe379f6 0xe428cf 0x475921

# 0x471904 sync.runtime_Semacquire+0x44
/opt/go/src/runtime/sema.go:56
# 0x48e3e4 sync.(*WaitGroup).Wait+0x64
/opt/go/src/sync/waitgroup.go:130
# 0xe379f5 github.com/samuel/go-zookeeper/zk.(*Conn).loop+0x3d5
/home/hk5/go/pkg/mod/github.com/samuel/[email protected]
c4fab1ac1bec/zk/conn.go:538
# 0xe428ce github.com/samuel/go-zookeeper/zk.Connect.func1+0x2e
/home/hk5/go/pkg/mod/github.com/samuel/[email protected]
c4fab1ac1bec/zk/conn.go:223

1 @ 0x43e0e5 0x44fc65 0x44fc4e 0x471905 0x48e3e5 0xe928b8 0xe95905 0xe95890

0x43dcb6 0x475921
# 0x471904 sync.runtime_Semacquire+0x44
/opt/go/src/runtime/sema.go:56
# 0x48e3e4 sync.(*WaitGroup).Wait+0x64
/opt/go/src/sync/waitgroup.go:130
# 0xe928b7 ygame/pkg/storm/app.(*App).Run+0x5d7
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:216
# 0xe95904 ygame/pkg/storm/app.Run+0x1c4
/home/hk5/perforce/goworks/src/ygame/pkg/storm/app/app.go:378
# 0xe9588f main.main+0x14f
/home/hk5/perforce/goworks/src/ygame/cmd/routeserver/main.go:27
# 0x43dcb5 runtime.main+0x255
/opt/go/src/runtime/proc.go:225

1 @ 0x43e0e5 0x44fc65 0x44fc4e 0x471a27 0x48dfa5 0xc1d707 0x475921

# 0x471a26 sync.runtime_SemacquireMutex+0x46
/opt/go/src/runtime/sema.go:71
# 0x48dfa4 sync.(*RWMutex).Lock+0x84
/opt/go/src/sync/rwmutex.go:116
# 0xc1d706 github.com/sasha-s/go-deadlock.lock.func1+0x26
/home/hk5/go/pkg/mod/github.com/sasha-s/[email protected]/deadlock.go:153

1 @ 0x43e0e5 0x471bb8 0x471b8e 0x48a779 0xbcd5b3 0xbcd7e5 0x475921

# 0x471b8d sync.runtime_notifyListWait+0xcd
/opt/go/src/runtime/sema.go:513
# 0x48a778 sync.(*Cond).Wait+0x98
/opt/go/src/sync/cond.go:56
# 0xbcd5b2 github.com/cihub/seelog.(*asyncAdaptiveLogger).processItem+0x92
/home/hk5/go/pkg/mod/github.com/cihub/[email protected]
d2c6e5aa9fbf/behavior_adaptivelogger.go:94
# 0xbcd7e4 github.com/cihub/seelog.(*asyncAdaptiveLogger).processQueue+0x84
/home/hk5/go/pkg/mod/github.com/cihub/[email protected]
d2c6e5aa9fbf/behavior_adaptivelogger.go:119
1 @ 0x43e0e5 0x4729d2 0xdf3691 0x475921
# 0x4729d1 time.Sleep+0xd1
/opt/go/src/runtime/time.go:193
# 0xdf3690 ygame/pkg/storm/stdb.(*RedisProxy).connect+0x270
/home/hk5/perforce/goworks/src/ygame/pkg/storm/stdb/redis_proxy.go:230

1 @ 0x46fa7d 0xe8104e 0xe80e25 0xe7d9b2 0xe8f285 0xe90ad7 0x6d4744 0x6d65cd

0x6d7d03 0x6d322d 0x475921
# 0x46fa7c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c
/opt/go/src/runtime/mprof.go:716
# 0xe8104d runtime/pprof.writeRuntimeProfile+0xcd
/opt/go/src/runtime/pprof/pprof.go:724
# 0xe80e24 runtime/pprof.writeGoroutine+0xa4
/opt/go/src/runtime/pprof/pprof.go:684
# 0xe7d9b1 runtime/pprof.(*Profile).WriteTo+0x3f1
/opt/go/src/runtime/pprof/pprof.go:331
# 0xe8f284 net/http/pprof.handler.ServeHTTP+0x384
/opt/go/src/net/http/pprof/pprof.go:253
# 0xe90ad6 net/http/pprof.Index+0x8d6
/opt/go/src/net/http/pprof/pprof.go:371
# 0x6d4743 net/http.HandlerFunc.ServeHTTP+0x43
/opt/go/src/net/http/server.go:2049
# 0x6d65cc net/http.(*ServeMux).ServeHTTP+0x1ac
/opt/go/src/net/http/server.go:2428
# 0x6d7d02 net/http.serverHandler.ServeHTTP+0xa2
/opt/go/src/net/http/server.go:2867
# 0x6d322c net/http.(*conn).serve+0x8cc
/opt/go/src/net/http/server.go:1932