receive data timeout when using cluster, view, with and hedged connection

when a query statement has a cluster, view, with, and by using hedged connection, a receive data timeout is displayed and a new replica is created, but the previous replica is running, and I can get the right result in the end, is this a bug?

for me, this caused a performance problem, as half the nodes were invalid calculations, resulting in a high load on my cluster.

of course I can fix this temporarily, by disabling use_hedged_requests or raising receive_data_timeout_ms(default = 2000ms), is it a bad idea to raise receive_data_timeout_ms?

also, I found that removing with did not trigger the problem, but the actual production business needed with.

```
## version

SELECT version()

┌─version()──┐
│ 23.9.1.400 │
└────────────┘

## cluster info

┌─cluster────────────────────────────────┬─shard_num─┬─replica_num─┬─host_name─┬─port─┐
│ test_multiple_shards_multiple_replicas │ 1 │ 1 │ 127.0.0.1 │ 9000 │
│ test_multiple_shards_multiple_replicas │ 1 │ 2 │ 127.0.0.1 │ 9001 │
│ test_multiple_shards_multiple_replicas │ 2 │ 1 │ 127.0.0.1 │ 9002 │
│ test_multiple_shards_multiple_replicas │ 2 │ 2 │ 127.0.0.1 │ 9003 │
└────────────────────────────────────────┴───────────┴─────────────┴───────────┴──────┘

## query
SELECT
sum(sn),
groupArray(sn)
FROM cluster(test_multiple_shards_multiple_replicas, view(
WITH (
SELECT sum(rand())
FROM
(
SELECT number
FROM system.numbers
LIMIT 500000000
)
) AS sn
SELECT sn
))
SETTINGS prefer_localhost_replica = 0, enable_global_with_statement = 0

Query id: 989f9ec3-69b0-4dab-a5f7-7e2f88574386

┌─────────────sum(sn)─┬─groupArray(sn)────────────────────────────┐
│ 2147537985959717488 │ [1073758051612807304,1073779934346910184] │
└─────────────────────┴───────────────────────────────────────────┘

1 rows in set. Elapsed: 5.877 sec. Processed 1.00 billion rows, 8.00 GB (170.17 million rows/s., 1.36 GB/s.)

## query log
SELECT *
FROM system.query_log_dst
WHERE (event_date = today()) AND (type != 1) AND (initial_query_id = '989f9ec3-69b0-4dab-a5f7-7e2f88574386')
ORDER BY is_initial_query DESC

Query id: 3ea742cd-5116-426e-b7a1-830d7278702a

Row 1:
──────
type: QueryFinish
event_date: 2023-09-18
event_time: 2023-09-18 23:29:53
event_time_microseconds: 2023-09-18 23:29:53.547553
query_start_time: 2023-09-18 23:29:47
query_start_time_microseconds: 2023-09-18 23:29:47.677002
query_duration_ms: 5870
read_rows: 1000103612
read_bytes: 8000828882
written_rows: 0
written_bytes: 0
result_rows: 1
result_bytes: 896
memory_usage: 419480
current_database: default
sum(sn), SELECT
groupArray(sn)
FROM cluster(test_multiple_shards_multiple_replicas, view(
WITH (
SELECT sum(rand())
FROM
(
SELECT number
FROM system.numbers
LIMIT 500000000
)
) AS sn
SELECT sn
))
SETTINGS prefer_localhost_replica = 0, enable_global_with_statement = 0;
formatted_query:
normalized_query_hash: 17703569033388686795
query_kind: Select
databases: ['_table_function']
tables: ['_table_function.cluster']
columns: ['_table_function.cluster.sn']
projections: []
views: []
exception_code: 0
exception:
stack_trace:
is_initial_query: 1
user: default
query_id: 989f9ec3-69b0-4dab-a5f7-7e2f88574386
address: ::ffff:127.0.0.1
port: 53342
initial_user: default
initial_query_id: 989f9ec3-69b0-4dab-a5f7-7e2f88574386
initial_address: ::ffff:127.0.0.1
initial_port: 53342
initial_query_start_time: 2023-09-18 23:29:47
initial_query_start_time_microseconds: 2023-09-18 23:29:47.677002
interface: 1
os_user:
client_hostname: DESKTOP-M5IG93K.localdomain
client_name: ClickHouse client
client_revision: 54455
client_version_major: 22
client_version_minor: 3
client_version_patch: 15
http_method: 0
http_user_agent:
http_referer:
forwarded_for:
quota_key:
distributed_depth: 0
revision: 54478
log_comment:
thread_ids: [5076,5094,5106,5090,5097,5069,5085,5108,5100,5078,5088,5109,5132,6214,5087,5082,5135,4828]
ProfileEvents: {'Query':1,'SelectQuery':1,'QueriesWithSubqueries':1,'SelectQueriesWithSubqueries':1,'ReadCompressedBytes':279,'CompressedReadBufferBlocks':3,'CompressedReadBufferBytes':276,'IOBufferAllocs':8,'IOBufferAllocBytes':7340742,'ArenaAllocChunks':6,'ArenaAllocBytes':24576,'TableFunctionExecute':1,'NetworkReceiveElapsedMicroseconds':6858,'NetworkSendElapsedMicroseconds':5518,'NetworkReceiveBytes':5859,'NetworkSendBytes':46488,'DistributedConnectionTries':4,'DistributedConnectionUsable':4,'HedgedRequestsChangeReplica':4,'SuspendSendingQueryToShard':2,'SelectedRows':1000103612,'SelectedBytes':8000828882,'ContextLock':104,'RWLockAcquiredReadLocks':6,'RealTimeMicroseconds':104896131,'UserTimeMicroseconds':124903,'SystemTimeMicroseconds':11872,'SoftPageFaults':53,'OSCPUWaitMicroseconds':162,'OSCPUVirtualTimeMicroseconds':136763,'OSWriteBytes':16384,'OSReadChars':10490,'OSWriteChars':77204,'QueryProfilerRuns':103,'ScalarSubqueriesCacheMiss':2,'LogTrace':11,'LogDebug':103}
Settings: {'use_uncompressed_cache':'0','load_balancing':'random','http_receive_timeout':'240','max_memory_usage':'10000000000','prefer_localhost_replica':'0','enable_global_with_statement':'0','use_structure_from_insertion_table_in_table_functions':'0','allow_experimental_map_type':'1'}
used_aggregate_functions: ['groupArray','sum']
used_aggregate_function_combinators: []
used_database_engines: []
used_data_type_families: ['Int64','Enum8','DateTime','AggregateFunction','String','Nullable','UInt64']
used_dictionaries: []
used_formats: []
used_functions: ['rand','identity','_CAST']
used_storages: []
used_table_functions: ['view','cluster']

Row 2:
──────
type: QueryFinish
event_date: 2023-09-18
event_time: 2023-09-18 23:29:52
event_time_microseconds: 2023-09-18 23:29:52.908910
query_start_time: 2023-09-18 23:29:47
query_start_time_microseconds: 2023-09-18 23:29:47.677002
query_duration_ms: 5183
read_rows: 500051806
read_bytes: 4000414441
written_rows: 0
written_bytes: 0
result_rows: 1
result_bytes: 512
memory_usage: 93264
current_database: default
query: SELECT sum(`sn`), groupArray(`sn`) FROM view(WITH (SELECT sum(rand()) FROM (SELECT `number` FROM `system`.`numbers` LIMIT 500000000)) AS `sn` SELECT `sn`)
formatted_query:
normalized_query_hash: 12510725402136903933
query_kind: Select
databases: ['_table_function','system']
tables: ['_table_function.view','system.numbers','system.one']
columns: ['_table_function.view.sn','system.numbers.number','system.one.dummy']
projections: []
views: []
exception_code: 0
exception:
stack_trace:
is_initial_query: 0
user: default
query_id: e6860ab7-a33a-46b3-a498-dd712b4dd6da
address: ::ffff:127.0.0.1
port: 53352
initial_user: default
initial_query_id: 989f9ec3-69b0-4dab-a5f7-7e2f88574386
initial_address: ::ffff:127.0.0.1
initial_port: 53342
initial_query_start_time: 2023-09-18 23:29:47
initial_query_start_time_microseconds: 2023-09-18 23:29:47.677002
interface: 1
os_user:
client_hostname: DESKTOP-M5IG93K.localdomain
client_name: ClickHouse client
client_revision: 54455
client_version_major: 22
client_version_minor: 3
client_version_patch: 15
http_method: 0
http_user_agent:
http_referer:
forwarded_for:
quota_key:
distributed_depth: 1
revision: 54478
log_comment:
thread_ids: [5102,5130,6217,5136,5073,5133,5131,5077,5099,5093,5070,6215,5134,5091,5080,6216,4829,5074]
ProfileEvents: {'Query':1,'SelectQuery':1,'QueriesWithSubqueries':4,'SelectQueriesWithSubqueries':4,'ArenaAllocChunks':2,'ArenaAllocBytes':8192,'FunctionExecute':7646,'TableFunctionExecute':1,'NetworkReceiveElapsedMicroseconds':4,'NetworkSendElapsedMicroseconds':212,'NetworkReceiveBytes':57,'NetworkSendBytes':2572,'SelectedRows':500051806,'SelectedBytes':4000414441,'ContextLock':118,'RWLockAcquiredReadLocks':8,'RealTimeMicroseconds':92481834,'UserTimeMicroseconds':4137384,'SystemTimeMicroseconds':1056244,'SoftPageFaults':1467817,'OSCPUWaitMicroseconds':2135,'OSCPUVirtualTimeMicroseconds':5193508,'OSWriteBytes':4096,'OSReadChars':121523,'OSWriteChars':43626,'QueryProfilerRuns':98,'ScalarSubqueriesGlobalCacheHit':1,'ScalarSubqueriesCacheMiss':3,'LogTrace':21,'LogDebug':1}
Settings: {'use_uncompressed_cache':'0','load_balancing':'random','http_receive_timeout':'240','max_memory_usage':'10000000000','prefer_localhost_replica':'0','enable_global_with_statement':'0','use_structure_from_insertion_table_in_table_functions':'0','allow_experimental_map_type':'1'}
used_aggregate_functions: ['groupArray','sum']
used_aggregate_function_combinators: []
used_database_engines: []
used_data_type_families: ['UInt32','UInt64','Nullable']
used_dictionaries: []
used_formats: []
used_functions: ['rand','identity','_CAST']
used_storages: []
used_table_functions: ['view']

Row 3:
──────
type: ExceptionWhileProcessing
event_date: 2023-09-18
event_time: 2023-09-18 23:29:54
event_time_microseconds: 2023-09-18 23:29:54.933705
query_start_time: 2023-09-18 23:29:47
query_start_time_microseconds: 2023-09-18 23:29:47.677002
query_duration_ms: 5206
read_rows: 500051806
read_bytes: 4000414441
written_rows: 0
written_bytes: 0
result_rows: 0
result_bytes: 0
memory_usage: 97112
current_database: default
query: SELECT sum(`sn`), groupArray(`sn`) FROM view(WITH (SELECT sum(rand()) FROM (SELECT `number` FROM `system`.`numbers` LIMIT 500000000)) AS `sn` SELECT `sn`)
formatted_query:
normalized_query_hash: 12510725402136903933
query_kind: Select
databases: ['_table_function','system']
tables: ['_table_function.view','system.numbers','system.one']
columns: ['_table_function.view.sn','system.numbers.number','system.one.dummy']
projections: []
views: []
exception_code: 210
exception: Code: 210. DB::NetException: I/O error: Broken pipe, while writing to socket (127.0.0.1:9001 -> 127.0.0.1:60702). (NETWORK_ERROR) (version 23.9.1.1)
stack_trace: 0. /home/wzb/ck_code/ck_master/ClickHouse/contrib/llvm-project/libcxx/include/exception:134: std::exception::capture() @ 0x000000000ad93d42 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
1. /home/wzb/ck_code/ck_master/ClickHouse/contrib/llvm-project/libcxx/include/exception:112: std::exception::exception[abi:v15000]() @ 0x000000000ad93d0d in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
2. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Foundation/src/Exception.cpp:27: Poco::Exception::Exception(String const&, int) @ 0x000000002326cd80 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
3. /home/wzb/ck_code/ck_master/ClickHouse/src/Common/Exception.cpp:96: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x00000000134ccbee in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
4. /home/wzb/ck_code/ck_master/ClickHouse/src/Common/Exception.h:73: DB::Exception::Exception(String&&, int, bool) @ 0x000000000ad85d8a in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
5. /home/wzb/ck_code/ck_master/ClickHouse/src/Common/NetException.h:26: DB::NetException::NetException(int, FormatStringHelperImpl::type, std::type_identity::type, std::type_identity::type>, String&&, String&&, String&&) @ 0x00000000138dcfa8 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
6. /home/wzb/ck_code/ck_master/ClickHouse/src/IO/WriteBufferFromPocoSocket.cpp:107: DB::WriteBufferFromPocoSocket::nextImpl() @ 0x00000000138dc5c4 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
7. /home/wzb/ck_code/ck_master/ClickHouse/src/IO/WriteBuffer.h:48: DB::WriteBuffer::next() @ 0x000000000adf50ed in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
8. /home/wzb/ck_code/ck_master/ClickHouse/src/Server/TCPHandler.cpp:2002: DB::TCPHandler::sendData(DB::Block const&) @ 0x000000001e09b60f in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
9. /home/wzb/ck_code/ck_master/ClickHouse/src/Server/TCPHandler.cpp:924: DB::TCPHandler::processOrdinaryQueryWithProcessors() @ 0x000000001e098c19 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
10. /home/wzb/ck_code/ck_master/ClickHouse/src/Server/TCPHandler.cpp:518: DB::TCPHandler::runImpl() @ 0x000000001e090f71 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
11. /home/wzb/ck_code/ck_master/ClickHouse/src/Server/TCPHandler.cpp:2124: DB::TCPHandler::run() @ 0x000000001e0a30c5 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
12. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Net/src/TCPServerConnection.cpp:43: Poco::Net::TCPServerConnection::start() @ 0x0000000023094ff9 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
13. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Net/src/TCPServerDispatcher.cpp:115: Poco::Net::TCPServerDispatcher::run() @ 0x000000002309583c in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
14. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Foundation/src/ThreadPool.cpp:188: Poco::PooledThread::run() @ 0x00000000232ea294 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
15. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Foundation/src/Thread.cpp:46: Poco::(anonymous namespace)::RunnableHolder::run() @ 0x00000000232e703a in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
16. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Foundation/src/Thread_POSIX.cpp:335: Poco::ThreadImpl::runnableEntry(void*) @ 0x00000000232e5d3e in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
17. ? @ 0x00007fb34dac9b43 in ?
18. ? @ 0x00007fb34db5ba00 in ?

is_initial_query: 0
user: default
query_id: 164dc572-89f8-470b-8ee7-ad4d83610ed7
address: ::ffff:127.0.0.1
port: 60702
initial_user: default
initial_query_id: 989f9ec3-69b0-4dab-a5f7-7e2f88574386
initial_address: ::ffff:127.0.0.1
initial_port: 53342
initial_query_start_time: 2023-09-18 23:29:47
initial_query_start_time_microseconds: 2023-09-18 23:29:47.677002
interface: 1
os_user:
client_hostname: DESKTOP-M5IG93K.localdomain
client_name: ClickHouse client
client_revision: 54455
client_version_major: 22
client_version_minor: 3
client_version_patch: 15
http_method: 0
http_user_agent:
http_referer:
forwarded_for:
quota_key:
distributed_depth: 1
revision: 54478
log_comment:
thread_ids: [3471,3473,3509,3501,3506,3479,3502,3491,3498,3475,3497,3492,3503,3477,3500,3504,3493,3505,3229]
ProfileEvents: {'Query':1,'SelectQuery':1,'QueriesWithSubqueries':4,'SelectQueriesWithSubqueries':4,'ReadCompressedBytes':158,'CompressedReadBufferBlocks':3,'CompressedReadBufferBytes':78,'IOBufferAllocs':2,'IOBufferAllocBytes':1048801,'ArenaAllocChunks':2,'ArenaAllocBytes':8192,'FunctionExecute':7646,'TableFunctionExecute':1,'NetworkReceiveElapsedMicroseconds':9,'NetworkSendElapsedMicroseconds':306,'NetworkReceiveBytes':110,'NetworkSendBytes':119,'SelectedRows':500051806,'SelectedBytes':4000414441,'ContextLock':118,'RWLockAcquiredReadLocks':8,'RealTimeMicroseconds':92858984,'UserTimeMicroseconds':4166238,'SystemTimeMicroseconds':1050727,'SoftPageFaults':1467834,'OSCPUWaitMicroseconds':2102,'OSCPUVirtualTimeMicroseconds':5216843,'OSWriteBytes':8192,'OSReadChars':121395,'OSWriteChars':43072,'QueryProfilerRuns':97,'ScalarSubqueriesGlobalCacheHit':1,'ScalarSubqueriesCacheMiss':3,'LogTrace':21,'LogDebug':1}
Settings: {'use_uncompressed_cache':'0','load_balancing':'random','http_receive_timeout':'240','max_memory_usage':'10000000000','prefer_localhost_replica':'0','enable_global_with_statement':'0','use_structure_from_insertion_table_in_table_functions':'0','allow_experimental_map_type':'1'}
used_aggregate_functions: ['groupArray','sum']
used_aggregate_function_combinators: []
used_database_engines: []
used_data_type_families: ['UInt32','UInt64','Nullable']
used_dictionaries: []
used_formats: []
used_functions: ['rand','identity','_CAST']
used_storages: []
used_table_functions: ['view']

Row 4:
──────
type: ExceptionWhileProcessing
event_date: 2023-09-18
event_time: 2023-09-18 23:29:55
event_time_microseconds: 2023-09-18 23:29:55.582003
query_start_time: 2023-09-18 23:29:47
query_start_time_microseconds: 2023-09-18 23:29:47.677002
query_duration_ms: 5853
read_rows: 500051806
read_bytes: 4000414441
written_rows: 0
written_bytes: 0
result_rows: 0
result_bytes: 0
memory_usage: 92976
current_database: default
query: SELECT sum(`sn`), groupArray(`sn`) FROM view(WITH (SELECT sum(rand()) FROM (SELECT `number` FROM `system`.`numbers` LIMIT 500000000)) AS `sn` SELECT `sn`)
formatted_query:
normalized_query_hash: 12510725402136903933
query_kind: Select
databases: ['_table_function','system']
tables: ['_table_function.view','system.numbers','system.one']
columns: ['_table_function.view.sn','system.numbers.number','system.one.dummy']
projections: []
views: []
exception_code: 210
exception: Code: 210. DB::NetException: I/O error: Broken pipe, while writing to socket (127.0.0.1:9002 -> 127.0.0.1:55694). (NETWORK_ERROR) (version 23.9.1.1)
stack_trace: 0. /home/wzb/ck_code/ck_master/ClickHouse/contrib/llvm-project/libcxx/include/exception:134: std::exception::capture() @ 0x000000000ad93d42 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
1. /home/wzb/ck_code/ck_master/ClickHouse/contrib/llvm-project/libcxx/include/exception:112: std::exception::exception[abi:v15000]() @ 0x000000000ad93d0d in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
2. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Foundation/src/Exception.cpp:27: Poco::Exception::Exception(String const&, int) @ 0x000000002326cd80 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
3. /home/wzb/ck_code/ck_master/ClickHouse/src/Common/Exception.cpp:96: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x00000000134ccbee in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
4. /home/wzb/ck_code/ck_master/ClickHouse/src/Common/Exception.h:73: DB::Exception::Exception(String&&, int, bool) @ 0x000000000ad85d8a in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
5. /home/wzb/ck_code/ck_master/ClickHouse/src/Common/NetException.h:26: DB::NetException::NetException(int, FormatStringHelperImpl::type, std::type_identity::type, std::type_identity::type>, String&&, String&&, String&&) @ 0x00000000138dcfa8 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
6. /home/wzb/ck_code/ck_master/ClickHouse/src/IO/WriteBufferFromPocoSocket.cpp:107: DB::WriteBufferFromPocoSocket::nextImpl() @ 0x00000000138dc5c4 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
7. /home/wzb/ck_code/ck_master/ClickHouse/src/IO/WriteBuffer.h:48: DB::WriteBuffer::next() @ 0x000000000adf50ed in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
8. /home/wzb/ck_code/ck_master/ClickHouse/src/Server/TCPHandler.cpp:2002: DB::TCPHandler::sendData(DB::Block const&) @ 0x000000001e09b60f in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
9. /home/wzb/ck_code/ck_master/ClickHouse/src/Server/TCPHandler.cpp:924: DB::TCPHandler::processOrdinaryQueryWithProcessors() @ 0x000000001e098c19 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
10. /home/wzb/ck_code/ck_master/ClickHouse/src/Server/TCPHandler.cpp:518: DB::TCPHandler::runImpl() @ 0x000000001e090f71 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
11. /home/wzb/ck_code/ck_master/ClickHouse/src/Server/TCPHandler.cpp:2124: DB::TCPHandler::run() @ 0x000000001e0a30c5 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
12. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Net/src/TCPServerConnection.cpp:43: Poco::Net::TCPServerConnection::start() @ 0x0000000023094ff9 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
13. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Net/src/TCPServerDispatcher.cpp:115: Poco::Net::TCPServerDispatcher::run() @ 0x000000002309583c in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
14. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Foundation/src/ThreadPool.cpp:188: Poco::PooledThread::run() @ 0x00000000232ea294 in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
15. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Foundation/src/Thread.cpp:46: Poco::(anonymous namespace)::RunnableHolder::run() @ 0x00000000232e703a in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
16. /home/wzb/ck_code/ck_master/ClickHouse/base/poco/Foundation/src/Thread_POSIX.cpp:335: Poco::ThreadImpl::runnableEntry(void*) @ 0x00000000232e5d3e in /home/wzb/ck_code/ck_master/ClickHouse/build_v2/programs/clickhouse
17. ? @ 0x00007f19f95cab43 in ?
18. ? @ 0x00007f19f965ca00 in ?

is_initial_query: 0
user: default
query_id: a04a111e-6deb-4295-a6cf-5b5779980e63
address: ::ffff:127.0.0.1
port: 55694
initial_user: default
initial_query_id: 989f9ec3-69b0-4dab-a5f7-7e2f88574386
initial_address: ::ffff:127.0.0.1
initial_port: 53342
initial_query_start_time: 2023-09-18 23:29:47
initial_query_start_time_microseconds: 2023-09-18 23:29:47.677002
interface: 1
os_user:
client_hostname: DESKTOP-M5IG93K.localdomain
client_name: ClickHouse client
client_revision: 54455
client_version_major: 22
client_version_minor: 3
client_version_patch: 15
http_method: 0
http_user_agent:
http_referer:
forwarded_for:
quota_key:
distributed_depth: 1
revision: 54478
log_comment:
thread_ids: [6159,3793,3771,3782,3765,6157,3792,3798,3775,3791,3764,3787,6156,3784,6158,3789,3778,3523]
ProfileEvents: {'Query':1,'SelectQuery':1,'QueriesWithSubqueries':4,'SelectQueriesWithSubqueries':4,'ReadCompressedBytes':158,'CompressedReadBufferBlocks':3,'CompressedReadBufferBytes':78,'IOBufferAllocs':2,'IOBufferAllocBytes':1048801,'ArenaAllocChunks':2,'ArenaAllocBytes':8192,'FunctionExecute':7646,'TableFunctionExecute':1,'NetworkReceiveElapsedMicroseconds':6,'NetworkSendElapsedMicroseconds':306,'NetworkReceiveBytes':110,'NetworkSendBytes':119,'CompileFunction':1,'CompileExpressionsMicroseconds':724174,'CompileExpressionsBytes':8192,'SelectedRows':500051806,'SelectedBytes':4000414441,'ContextLock':118,'RWLockAcquiredReadLocks':8,'RealTimeMicroseconds':92219373,'UserTimeMicroseconds':4829592,'SystemTimeMicroseconds':1034372,'SoftPageFaults':1468067,'OSCPUWaitMicroseconds':2210,'OSCPUVirtualTimeMicroseconds':5863834,'OSReadChars':119516,'OSWriteChars':43873,'QueryProfilerRuns':98,'ScalarSubqueriesGlobalCacheHit':1,'ScalarSubqueriesCacheMiss':3,'LogTrace':22,'LogDebug':1}
Settings: {'use_uncompressed_cache':'0','load_balancing':'random','http_receive_timeout':'240','max_memory_usage':'10000000000','prefer_localhost_replica':'0','enable_global_with_statement':'0','use_structure_from_insertion_table_in_table_functions':'0','allow_experimental_map_type':'1'}
used_aggregate_functions: ['groupArray','sum']
used_aggregate_function_combinators: []
used_database_engines: []
used_data_type_families: ['UInt32','UInt64','Nullable']
used_dictionaries: []
used_formats: []
used_functions: ['rand','identity','_CAST']
used_storages: []
used_table_functions: ['view']

Row 5:
──────
type: QueryFinish
event_date: 2023-09-18
event_time: 2023-09-18 23:29:53
event_time_microseconds: 2023-09-18 23:29:53.538997
query_start_time: 2023-09-18 23:29:47
query_start_time_microseconds: 2023-09-18 23:29:47.677002
query_duration_ms: 5815
read_rows: 500051806
read_bytes: 4000414441
written_rows: 0
written_bytes: 0
result_rows: 1
result_bytes: 512
memory_usage: 94288
current_database: default
query: SELECT sum(`sn`), groupArray(`sn`) FROM view(WITH (SELECT sum(rand()) FROM (SELECT `number` FROM `system`.`numbers` LIMIT 500000000)) AS `sn` SELECT `sn`)
formatted_query:
normalized_query_hash: 12510725402136903933
query_kind: Select
databases: ['_table_function','system']
tables: ['_table_function.view','system.numbers','system.one']
columns: ['_table_function.view.sn','system.numbers.number','system.one.dummy']
projections: []
views: []
exception_code: 0
exception:
stack_trace:
is_initial_query: 0
user: default
query_id: 86388ae9-88e9-45e4-b367-30efa99adc5c
address: ::ffff:127.0.0.1
port: 43920
initial_user: default
initial_query_id: 989f9ec3-69b0-4dab-a5f7-7e2f88574386
initial_address: ::ffff:127.0.0.1
initial_port: 53342
initial_query_start_time: 2023-09-18 23:29:47
initial_query_start_time_microseconds: 2023-09-18 23:29:47.677002
interface: 1
os_user:
client_hostname: DESKTOP-M5IG93K.localdomain
client_name: ClickHouse client
client_revision: 54455
client_version_major: 22
client_version_minor: 3
client_version_patch: 15
http_method: 0
http_user_agent:
http_referer:
forwarded_for:
quota_key:
distributed_depth: 1
revision: 54478
log_comment:
thread_ids: [4096,4069,4062,4085,4076,4078,4093,4086,4063,4066,4089,4072,4095,3812,4088,4082,4060,4081,4068]
ProfileEvents: {'Query':1,'SelectQuery':1,'QueriesWithSubqueries':4,'SelectQueriesWithSubqueries':4,'ReadCompressedBytes':158,'CompressedReadBufferBlocks':3,'CompressedReadBufferBytes':78,'IOBufferAllocs':2,'IOBufferAllocBytes':1048801,'ArenaAllocChunks':2,'ArenaAllocBytes':8192,'FunctionExecute':7646,'TableFunctionExecute':1,'NetworkReceiveElapsedMicroseconds':5,'NetworkSendElapsedMicroseconds':176,'NetworkReceiveBytes':110,'NetworkSendBytes':3145,'CompileFunction':1,'CompileExpressionsMicroseconds':676586,'CompileExpressionsBytes':8192,'SelectedRows':500051806,'SelectedBytes':4000414441,'ContextLock':118,'RWLockAcquiredReadLocks':8,'RealTimeMicroseconds':92318863,'UserTimeMicroseconds':4781693,'SystemTimeMicroseconds':1044195,'SoftPageFaults':1468177,'OSCPUWaitMicroseconds':2349,'OSCPUVirtualTimeMicroseconds':5825766,'OSWriteBytes':4096,'OSReadChars':120928,'OSWriteChars':44078,'QueryProfilerRuns':99,'ScalarSubqueriesGlobalCacheHit':1,'ScalarSubqueriesCacheMiss':3,'LogTrace':22,'LogDebug':1}
Settings: {'use_uncompressed_cache':'0','load_balancing':'random','http_receive_timeout':'240','max_memory_usage':'10000000000','prefer_localhost_replica':'0','enable_global_with_statement':'0','use_structure_from_insertion_table_in_table_functions':'0','allow_experimental_map_type':'1'}
used_aggregate_functions: ['groupArray','sum']
used_aggregate_function_combinators: []
used_database_engines: []
used_data_type_families: ['UInt32','UInt64','Nullable']
used_dictionaries: []
used_formats: []
used_functions: ['rand','identity','_CAST']
used_storages: []
used_table_functions: ['view']

5 rows in set. Elapsed: 0.667 sec.

## if remove with, there will be no receive data timeout issues
SELECT
sum(sn),
groupArray(sn)
FROM cluster(test_multiple_shards_multiple_replicas, view(
SELECT sum(rand()) AS sn
FROM
(
SELECT number
FROM system.numbers
LIMIT 500000000
)
))
SETTINGS prefer_localhost_replica = 0

Query id: 0f6b43f8-d471-4252-adf9-2ce97d8f71ec

┌─────────────sum(sn)─┬─groupArray(sn)────────────────────────────┐
│ 2147492372396279723 │ [1073701985878040216,1073790386518239507] │
└─────────────────────┴───────────────────────────────────────────┘

1 rows in set. Elapsed: 5.518 sec. Processed 1.00 billion rows, 8.00 GB (181.25 million rows/s., 1.45 GB/s.)

SELECT *
FROM system.query_log_dst
WHERE (event_date = today()) AND (type != 1) AND (initial_query_id = '0f6b43f8-d471-4252-adf9-2ce97d8f71ec')
ORDER BY is_initial_query DESC

Query id: 62933d84-d4a6-499e-bfc7-62cc44031ef5

Row 1:
──────
type: QueryFinish
event_date: 2023-09-18
event_time: 2023-09-18 23:33:57
event_time_microseconds: 2023-09-18 23:33:57.978010
query_start_time: 2023-09-18 23:33:52
query_start_time_microseconds: 2023-09-18 23:33:52.465992
query_duration_ms: 5511
read_rows: 1000103610
read_bytes: 8000828880
written_rows: 0
written_bytes: 0
result_rows: 1
result_bytes: 768
memory_usage: 8390728
current_database: default
query: SELECT
sum(sn),
groupArray(sn)
FROM cluster(test_multiple_shards_multiple_replicas, view(
SELECT sum(rand()) as sn
FROM
(
SELECT number
FROM system.numbers
LIMIT 500000000
)
))
SETTINGS prefer_localhost_replica = 0;
formatted_query:
normalized_query_hash: 9548703235636197289
query_kind: Select
databases: ['_table_function']
tables: ['_table_function.cluster']
columns: ['_table_function.cluster.sn']
projections: []
views: []
exception_code: 0
exception:
stack_trace:
is_initial_query: 1
user: default
query_id: 0f6b43f8-d471-4252-adf9-2ce97d8f71ec
address: ::ffff:127.0.0.1
port: 53342
initial_user: default
initial_query_id: 0f6b43f8-d471-4252-adf9-2ce97d8f71ec
initial_address: ::ffff:127.0.0.1
initial_port: 53342
initial_query_start_time: 2023-09-18 23:33:52
initial_query_start_time_microseconds: 2023-09-18 23:33:52.465992
interface: 1
os_user:
client_hostname: DESKTOP-M5IG93K.localdomain
client_name: ClickHouse client
client_revision: 54455
client_version_major: 22
client_version_minor: 3
client_version_patch: 15
http_method: 0
http_user_agent:
http_referer:
forwarded_for:
quota_key:
distributed_depth: 0
revision: 54478
log_comment:
thread_ids: [5097,5106,5080,6214,5134,5088,5069,5082,5132,5109,5076,5090,5094,5078,5100,5085,5108,4828]
ProfileEvents: {'Query':1,'SelectQuery':1,'QueriesWithSubqueries':1,'SelectQueriesWithSubqueries':1,'ReadCompressedBytes':526,'CompressedReadBufferBlocks':6,'CompressedReadBufferBytes':470,'IOBufferAllocs':8,'IOBufferAllocBytes':6292330,'ArenaAllocChunks':6,'ArenaAllocBytes':24576,'TableFunctionExecute':1,'NetworkReceiveElapsedMicroseconds':6175,'NetworkSendElapsedMicroseconds':4453,'NetworkReceiveBytes':123689,'NetworkSendBytes':183070,'DistributedConnectionTries':2,'DistributedConnectionUsable':2,'SuspendSendingQueryToShard':2,'SelectedRows':1000103610,'SelectedBytes':8000828880,'ContextLock':70,'RWLockAcquiredReadLocks':4,'RealTimeMicroseconds':98631039,'UserTimeMicroseconds':464608,'SystemTimeMicroseconds':51202,'SoftPageFaults':84,'OSCPUWaitMicroseconds':179,'OSCPUVirtualTimeMicroseconds':515713,'OSWriteBytes':139264,'OSReadChars':50326,'OSWriteChars':383320,'QueryProfilerRuns':100,'LogTrace':9,'LogDebug':1007}
Settings: {'use_uncompressed_cache':'0','load_balancing':'random','http_receive_timeout':'240','max_memory_usage':'10000000000','prefer_localhost_replica':'0','allow_experimental_map_type':'1'}
used_aggregate_functions: ['groupArray','sum']
used_aggregate_function_combinators: []
used_database_engines: []
used_data_type_families: ['Int64','Enum8','DateTime','AggregateFunction','String','UInt64']
used_dictionaries: []
used_formats: []
used_functions: ['rand']
used_storages: []
used_table_functions: ['view','cluster']

Row 2:
──────
type: QueryFinish
event_date: 2023-09-18
event_time: 2023-09-18 23:33:57
event_time_microseconds: 2023-09-18 23:33:57.968219
query_start_time: 2023-09-18 23:33:52
query_start_time_microseconds: 2023-09-18 23:33:52.465992
query_duration_ms: 5464
read_rows: 500051805
read_bytes: 4000414440
written_rows: 0
written_bytes: 0
result_rows: 1
result_bytes: 512
memory_usage: 101648
current_database: default
query: SELECT sum(`sn`), groupArray(`sn`) FROM view(SELECT sum(rand()) AS `sn` FROM (SELECT `number` FROM `system`.`numbers` LIMIT 500000000))
formatted_query:
normalized_query_hash: 12404500358034894967
query_kind: Select
databases: ['_table_function','system']
tables: ['_table_function.view','system.numbers']
columns: ['_table_function.view.sn','system.numbers.number']
projections: []
views: []
exception_code: 0
exception:
stack_trace:
is_initial_query: 0
user: default
query_id: c8d222f7-6420-4284-8ce7-801a80cb055c
address: ::ffff:127.0.0.1
port: 60718
initial_user: default
initial_query_id: 0f6b43f8-d471-4252-adf9-2ce97d8f71ec
initial_address: ::ffff:127.0.0.1
initial_port: 53342
initial_query_start_time: 2023-09-18 23:33:52
initial_query_start_time_microseconds: 2023-09-18 23:33:52.465992
interface: 1
os_user:
client_hostname: DESKTOP-M5IG93K.localdomain
client_name: ClickHouse client
client_revision: 54455
client_version_major: 22
client_version_minor: 3
client_version_patch: 15
http_method: 0
http_user_agent:
http_referer:
forwarded_for:
quota_key:
distributed_depth: 1
revision: 54478
log_comment:
thread_ids: [3499,3484,3472,3510,3471,3473,3501,3230,3506,3509,3477,3498,3475,3497,3502,3504,3503,3505]
ProfileEvents: {'Query':1,'SelectQuery':1,'QueriesWithSubqueries':3,'SelectQueriesWithSubqueries':3,'ReadCompressedBytes':158,'CompressedReadBufferBlocks':3,'CompressedReadBufferBytes':78,'IOBufferAllocs':2,'IOBufferAllocBytes':1048801,'ArenaAllocChunks':17,'ArenaAllocBytes':69632,'FunctionExecute':7646,'TableFunctionExecute':1,'NetworkReceiveElapsedMicroseconds':226,'NetworkSendElapsedMicroseconds':3044,'NetworkReceiveBytes':110,'NetworkSendBytes':61396,'CompileFunction':1,'CompileExpressionsMicroseconds':672793,'CompileExpressionsBytes':8192,'SelectedRows':500051805,'SelectedBytes':4000414440,'ContextLock':75,'RWLockAcquiredReadLocks':5,'RealTimeMicroseconds':86277837,'UserTimeMicroseconds':4698781,'SystemTimeMicroseconds':817231,'SoftPageFaults':1467984,'OSCPUWaitMicroseconds':2610,'OSCPUVirtualTimeMicroseconds':5515893,'OSWriteBytes':8192,'OSReadChars':112338,'OSWriteChars':50157,'QueryProfilerRuns':88,'LogTrace':48,'LogDebug':1}
Settings: {'use_uncompressed_cache':'0','load_balancing':'random','http_receive_timeout':'240','max_memory_usage':'10000000000','prefer_localhost_replica':'0','allow_experimental_map_type':'1'}
used_aggregate_functions: ['groupArray','sum']
used_aggregate_function_combinators: []
used_database_engines: []
used_data_type_families: ['UInt32']
used_dictionaries: []
used_formats: []
used_functions: ['rand']
used_storages: []
used_table_functions: ['view']

Row 3:
──────
type: QueryFinish
event_date: 2023-09-18
event_time: 2023-09-18 23:33:57
event_time_microseconds: 2023-09-18 23:33:57.362610
query_start_time: 2023-09-18 23:33:52
query_start_time_microseconds: 2023-09-18 23:33:52.465992
query_duration_ms: 4858
read_rows: 500051805
read_bytes: 4000414440
written_rows: 0
written_bytes: 0
result_rows: 1
result_bytes: 512
memory_usage: 99704
current_database: default
query: SELECT sum(`sn`), groupArray(`sn`) FROM view(SELECT sum(rand()) AS `sn` FROM (SELECT `number` FROM `system`.`numbers` LIMIT 500000000))
formatted_query:
normalized_query_hash: 12404500358034894967
query_kind: Select
databases: ['_table_function','system']
tables: ['_table_function.view','system.numbers']
columns: ['_table_function.view.sn','system.numbers.number']
projections: []
views: []
exception_code: 0
exception:
stack_trace:
is_initial_query: 0
user: default
query_id: 9c89c465-67de-4de0-b25f-fe4711badd44
address: ::ffff:127.0.0.1
port: 55704
initial_user: default
initial_query_id: 0f6b43f8-d471-4252-adf9-2ce97d8f71ec
initial_address: ::ffff:127.0.0.1
initial_port: 53342
initial_query_start_time: 2023-09-18 23:33:52
initial_query_start_time_microseconds: 2023-09-18 23:33:52.465992
interface: 1
os_user:
client_hostname: DESKTOP-M5IG93K.localdomain
client_name: ClickHouse client
client_revision: 54455
client_version_major: 22
client_version_minor: 3
client_version_patch: 15
http_method: 0
http_user_agent:
http_referer:
forwarded_for:
quota_key:
distributed_depth: 1
revision: 54478
log_comment:
thread_ids: [3792,3782,6159,12944,3778,12945,3791,3784,3789,3771,3793,6157,3765,3775,3798,3787,3764,3524]
ProfileEvents: {'Query':1,'SelectQuery':1,'QueriesWithSubqueries':3,'SelectQueriesWithSubqueries':3,'ReadCompressedBytes':158,'CompressedReadBufferBlocks':3,'CompressedReadBufferBytes':78,'IOBufferAllocs':2,'IOBufferAllocBytes':1048801,'ArenaAllocChunks':17,'ArenaAllocBytes':69632,'FunctionExecute':7646,'TableFunctionExecute':1,'NetworkReceiveElapsedMicroseconds':260,'NetworkSendElapsedMicroseconds':3590,'NetworkReceiveBytes':110,'NetworkSendBytes':62153,'SelectedRows':500051805,'SelectedBytes':4000414440,'ContextLock':75,'RWLockAcquiredReadLocks':5,'RealTimeMicroseconds':86799270,'UserTimeMicroseconds':3916668,'SystemTimeMicroseconds':996135,'SoftPageFaults':1467874,'OSCPUWaitMicroseconds':2516,'OSCPUVirtualTimeMicroseconds':4912689,'OSWriteBytes':4096,'OSReadChars':113420,'OSWriteChars':50347,'QueryProfilerRuns':90,'LogTrace':47,'LogDebug':1}
Settings: {'use_uncompressed_cache':'0','load_balancing':'random','http_receive_timeout':'240','max_memory_usage':'10000000000','prefer_localhost_replica':'0','allow_experimental_map_type':'1'}
used_aggregate_functions: ['groupArray','sum']
used_aggregate_function_combinators: []
used_database_engines: []
used_data_type_families: ['UInt32']
used_dictionaries: []
used_formats: []
used_functions: ['rand']
used_storages: []
used_table_functions: ['view']

3 rows in set. Elapsed: 0.532 sec.

```