Hello. we have problem. We have a set of microservices that communicate with each other using the GRPC protocol. And when we turn on the linker on any of them, we begin to lose some of the connections. Logs:
{"timestamp":"[ 214.010685s]","level":"WARN","fields":{"message":"Unexpected error","error":"logical service 10.2.97.49:50002: route default.endpoint: backend default.unknown: endpoint 10.2.97.49:50002: operation was canceled: connection closed"},"target":"linkerd_app_outbound::http::server","spans":[{"name":"outbound"},{"addr":"10.2.97.49:50002","name":"proxy"},{"client.addr":"10.2.120.153:55906","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 214.162698s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.97.49:50002","name":"proxy"},{"addr":"10.2.97.49:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 214.775475s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.125.109:50002","name":"proxy"},{"addr":"10.2.125.109:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 214.886837s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.120.144:50002","name":"proxy"},{"addr":"10.2.120.144:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 216.564612s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.118.192:50002","name":"proxy"},{"addr":"10.2.118.192:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 217.671428s]","level":"INFO","fields":{"message":"gRPC request failed","error":"endpoint 10.2.122.227:50002: operation was canceled: connection closed"},"target":"linkerd_app_core::errors::respond","spans":[{"name":"outbound"},{"addr":"10.2.122.227:50002","name":"proxy"},{"addr":"10.2.122.227:50002","name":"forward"},{"client.addr":"10.2.120.153:43420","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 217.671466s]","level":"INFO","fields":{"message":"gRPC request failed","error":"logical service 10.2.122.227:50002: route default.endpoint: backend default.unknown: endpoint 10.2.122.227:50002: operation was canceled: connection closed"},"target":"linkerd_app_core::errors::respond","spans":[{"name":"outbound"},{"addr":"10.2.122.227:50002","name":"proxy"},{"client.addr":"10.2.120.153:43420","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 217.671476s]","level":"WARN","fields":{"message":"Unexpected error","error":"logical service 10.2.122.227:50002: route default.endpoint: backend default.unknown: endpoint 10.2.122.227:50002: operation was canceled: connection closed"},"target":"linkerd_app_outbound::http::server","spans":[{"name":"outbound"},{"addr":"10.2.122.227:50002","name":"proxy"},{"client.addr":"10.2.120.153:43420","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 217.724199s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.122.227:50002","name":"proxy"},{"addr":"10.2.122.227:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 218.822834s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.93.55:50002","name":"proxy"},{"addr":"10.2.93.55:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 221.459179s]","level":"INFO","fields":{"message":"gRPC request failed","error":"endpoint 10.2.123.155:50002: operation was canceled: connection closed"},"target":"linkerd_app_core::errors::respond","spans":[{"name":"outbound"},{"addr":"10.2.123.155:50002","name":"proxy"},{"addr":"10.2.123.155:50002","name":"forward"},{"client.addr":"10.2.120.153:33040","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 221.459217s]","level":"INFO","fields":{"message":"gRPC request failed","error":"logical service 10.2.123.155:50002: route default.endpoint: backend default.unknown: endpoint 10.2.123.155:50002: operation was canceled: connection closed"},"target":"linkerd_app_core::errors::respond","spans":[{"name":"outbound"},{"addr":"10.2.123.155:50002","name":"proxy"},{"client.addr":"10.2.120.153:33040","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 221.459231s]","level":"WARN","fields":{"message":"Unexpected error","error":"logical service 10.2.123.155:50002: route default.endpoint: backend default.unknown: endpoint 10.2.123.155:50002: operation was canceled: connection closed"},"target":"linkerd_app_outbound::http::server","spans":[{"name":"outbound"},{"addr":"10.2.123.155:50002","name":"proxy"},{"client.addr":"10.2.120.153:33040","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 221.517824s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.123.155:50002","name":"proxy"},{"addr":"10.2.123.155:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 222.011855s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.98.67:50002","name":"proxy"},{"addr":"10.2.98.67:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 224.446579s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.120.156:50002","name":"proxy"},{"addr":"10.2.120.156:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 225.074819s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.98.73:50002","name":"proxy"},{"addr":"10.2.98.73:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 225.186729s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.124.177:50002","name":"proxy"},{"addr":"10.2.124.177:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 226.650727s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.122.177:50002","name":"proxy"},{"addr":"10.2.122.177:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 226.755992s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.115.186:50002","name":"proxy"},{"addr":"10.2.115.186:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 228.142275s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.122.239:50002","name":"proxy"},{"addr":"10.2.122.239:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 231.674511s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.120.152:50002","name":"proxy"},{"addr":"10.2.120.152:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 231.899970s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.95.53:50002","name":"proxy"},{"addr":"10.2.95.53:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 238.350626s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.118.200:50002","name":"proxy"},{"addr":"10.2.118.200:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 242.127652s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.97.49:50002","name":"proxy"},{"addr":"10.2.97.49:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 244.606469s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.120.144:50002","name":"proxy"},{"addr":"10.2.120.144:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 246.410222s]","level":"INFO","fields":{"message":"gRPC request failed","error":"endpoint 10.2.93.55:50002: operation was canceled: connection closed"},"target":"linkerd_app_core::errors::respond","spans":[{"name":"outbound"},{"addr":"10.2.93.55:50002","name":"proxy"},{"addr":"10.2.93.55:50002","name":"forward"},{"client.addr":"10.2.120.153:57458","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 246.410260s]","level":"INFO","fields":{"message":"gRPC request failed","error":"logical service 10.2.93.55:50002: route default.endpoint: backend default.unknown: endpoint 10.2.93.55:50002: operation was canceled: connection closed"},"target":"linkerd_app_core::errors::respond","spans":[{"name":"outbound"},{"addr":"10.2.93.55:50002","name":"proxy"},{"client.addr":"10.2.120.153:57458","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 246.410270s]","level":"WARN","fields":{"message":"Unexpected error","error":"logical service 10.2.93.55:50002: route default.endpoint: backend default.unknown: endpoint 10.2.93.55:50002: operation was canceled: connection closed"},"target":"linkerd_app_outbound::http::server","spans":[{"name":"outbound"},{"addr":"10.2.93.55:50002","name":"proxy"},{"client.addr":"10.2.120.153:57458","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 246.852404s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.93.55:50002","name":"proxy"},{"addr":"10.2.93.55:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 247.347250s]","level":"INFO","fields":{"message":"gRPC request failed","error":"endpoint 10.2.125.109:50002: operation was canceled: connection closed"},"target":"linkerd_app_core::errors::respond","spans":[{"name":"outbound"},{"addr":"10.2.125.109:50002","name":"proxy"},{"addr":"10.2.125.109:50002","name":"forward"},{"client.addr":"10.2.120.153:57760","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 247.347290s]","level":"INFO","fields":{"message":"gRPC request failed","error":"logical service 10.2.125.109:50002: route default.endpoint: backend default.unknown: endpoint 10.2.125.109:50002: operation was canceled: connection closed"},"target":"linkerd_app_core::errors::respond","spans":[{"name":"outbound"},{"addr":"10.2.125.109:50002","name":"proxy"},{"client.addr":"10.2.120.153:57760","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 247.347300s]","level":"WARN","fields":{"message":"Unexpected error","error":"logical service 10.2.125.109:50002: route default.endpoint: backend default.unknown: endpoint 10.2.125.109:50002: operation was canceled: connection closed"},"target":"linkerd_app_outbound::http::server","spans":[{"name":"outbound"},{"addr":"10.2.125.109:50002","name":"proxy"},{"client.addr":"10.2.120.153:57760","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 247.428982s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.125.109:50002","name":"proxy"},{"addr":"10.2.125.109:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 247.627939s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.118.192:50002","name":"proxy"},{"addr":"10.2.118.192:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 247.770892s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.122.227:50002","name":"proxy"},{"addr":"10.2.122.227:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 251.352863s]","level":"INFO","fields":{"message":"gRPC request failed","error":"endpoint 10.2.123.155:50002: operation was canceled: connection closed"},"target":"linkerd_app_core::errors::respond","spans":[{"name":"outbound"},{"addr":"10.2.123.155:50002","name":"proxy"},{"addr":"10.2.123.155:50002","name":"forward"},{"client.addr":"10.2.120.153:48952","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 251.352912s]","level":"INFO","fields":{"message":"gRPC request failed","error":"logical service 10.2.123.155:50002: route default.endpoint: backend default.unknown: endpoint 10.2.123.155:50002: operation was canceled: connection closed"},"target":"linkerd_app_core::errors::respond","spans":[{"name":"outbound"},{"addr":"10.2.123.155:50002","name":"proxy"},{"client.addr":"10.2.120.153:48952","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 251.352924s]","level":"WARN","fields":{"message":"Unexpected error","error":"logical service 10.2.123.155:50002: route default.endpoint: backend default.unknown: endpoint 10.2.123.155:50002: operation was canceled: connection closed"},"target":"linkerd_app_outbound::http::server","spans":[{"name":"outbound"},{"addr":"10.2.123.155:50002","name":"proxy"},{"client.addr":"10.2.120.153:48952","name":"rescue"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 251.360674s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.123.155:50002","name":"proxy"},{"addr":"10.2.123.155:50002","name":"forward"}],"threadId":"ThreadId(1)"}
{"timestamp":"[ 253.704899s]","level":"WARN","fields":{"message":"Service failed","error":"channel closed"},"target":"linkerd_reconnect","spans":[{"name":"outbound"},{"addr":"10.2.98.73:50002","name":"proxy"},{"addr":"10.2.98.73:50002","name":"forward"}],"threadId":"ThreadId(1)"}