background gc report error `Check iter != keyspace_id_to_used_bytes.end() failed, keyspace_id = 4294967295`

Bug Report

Please answer these questions before submitting your issue. Thanks!

1. Minimal reproduce step (Required)

2. What did you expect to see? (Required)

3. What did you see instead (Required)

[2024/08/20 18:04:33.268 +08:00] [DEBUG] [PageDirectory.cpp:2242] ["After MVCC gc in memory [lowest_seq=35] clean [invalid_snapshot_nums=0] [invalid_page_nums=1] [total_deref_counter=0] [all_del_entries=5]. Still exist [snapshot_nums=0], [page_nums=14]. Longest alive snapshot: [longest_alive_snapshot_time=0] [longest_alive_snapshot_seq=0] [stale_snapshot_nums=│0]"] [source=uni_write] [thread_id=155] 
[2024/08/20 18:04:33.269 +08:00] [DEBUG] [BlobStore.cpp:1311] ["BlobStore gc get status done. blob_ids details {ReadOnly: [null]} {Unchanged: [<id:11 rate:0.44>, <id:20 rate:0.56>]} {FullGC: [null]} {Truncated: [null]}"] [source=uni_write] [thread_id=155]
[2024/08/20 18:04:34.892 +08:00] [ERROR] [Exception.cpp:96] ["Code: 49, e.displayText() = DB::Exception: Check iter != keyspace_id_to_used_bytes.end() failed, keyspace_id = 4294967295, e.what() = DB::Exception, Stack trace:
  0x560e0b2be51e    StackTrace::StackTrace() [tiflash+72385822]
                    dbms/src/Common/StackTrace.cpp:23
  0x560e0b2addc2    DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, int) [tiflash+72318402]
                    dbms/src/Common/Exception.h:46
  0x560e148c6f32    DB::PathCapacityMetrics::freeRemoteUsedSize(unsigned int, unsigned long) [tiflash+229707570]
                    dbms/src/Storages/PathCapacityMetrics.cpp:137
  0x560e148dc253    DB::StableDiskDelegator::removeRemoteDTFile(unsigned long, bool) [tiflash+229794387]
                    dbms/src/Storages/PathPool.cpp:659
  0x560e1311be06    DB::DM::S3DMFileGcRemover::operator()(std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&) [tiflash+204889606]
                    dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp:261
  0x560e1311bbf5    decltype(std::declval<DB::DM::S3DMFileGcRemover&>()(std::declval<std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&>(), std::declval<std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&>())) std::__1::__invoke[abi:ue170006]<DB::DM::S3DMFileGcRemover&, std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&>(DB::DM::S3DMFileGcRemover&, std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&) [tiflash+204889077]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__type_traits/invoke.h:340
  0x560e1311bba5    void std::__1::__invoke_void_return_wrapper<void, true>::__call[abi:ue170006]<DB::DM::S3DMFileGcRemover&, std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&>(DB::DM::S3DMFileGcRemover&, std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&) [tiflash+204888997]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__type_traits/invoke.h:415
  0x560e1311bb6d    std::__1::__function::__alloc_func<DB::DM::S3DMFileGcRemover, std::__1::allocator<DB::DM::S3DMFileGcRemover>, void (std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&)>::operator()[abi:ue170006](std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&) [tiflash+204888941]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__functional/function.h:192
  0x560e1311afc9    std::__1::__function::__func<DB::DM::S3DMFileGcRemover, std::__1::allocator<DB::DM::S3DMFileGcRemover>, void (std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&)>::operator()(std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&) [tiflash+204885961]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__functional/function.h:363
  0x560e152460b2    std::__1::__function::__value_func<void (std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&)>::operator()[abi:ue170006](std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&) const [tiflash+239665330]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__functional/function.h:517
  0x560e152457d5    std::__1::function<void (std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&)>::operator()(std::__1::vector<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>>>>> const&, std::__1::set<unsigned long, std::__1::less<unsigned long>, std::__1::allocator<unsigned long>> const&) const [tiflash+239663061]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__functional/function.h:1168
  0x560e1537a733    DB::PS::V3::ExternalPageCallbacksManager<DB::PS::V3::universal::ExternalPageCallbacksManagerTrait>::cleanExternalPage(DB::PS::V3::PageDirectory<DB::PS::V3::universal::PageDirectoryTrait>&, Stopwatch&, DB::PS::V3::GCTimeStatistics&) [tiflash+240928563]
                    dbms/src/Storages/Page/V3/GCDefines.cpp:207
  0x560e15379cdf    DB::PS::V3::ExternalPageCallbacksManager<DB::PS::V3::universal::ExternalPageCallbacksManagerTrait>::doGC(DB::PS::V3::BlobStore<DB::PS::V3::universal::BlobStoreTrait>&, DB::PS::V3::PageDirectory<DB::PS::V3::universal::PageDirectoryTrait>&, std::__1::shared_ptr<DB::WriteLimiter> const&, std::__1::shared_ptr<DB::ReadLimiter> const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, unsigned long, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const, unsigned long>>>*) [tiflash+240925919]
                    dbms/src/Storages/Page/V3/GCDefines.cpp:289
  0x560e1537966c    DB::PS::V3::ExternalPageCallbacksManager<DB::PS::V3::universal::ExternalPageCallbacksManagerTrait>::gc(DB::PS::V3::BlobStore<DB::PS::V3::universal::BlobStoreTrait>&, DB::PS::V3::PageDirectory<DB::PS::V3::universal::PageDirectoryTrait>&, std::__1::shared_ptr<DB::WriteLimiter> const&, std::__1::shared_ptr<DB::ReadLimiter> const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, unsigned long, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const, unsigned long>>>*, std::__1::shared_ptr<DB::Logger>) [tiflash+240924268]
                    dbms/src/Storages/Page/V3/GCDefines.cpp:147
  0x560e153c8372    DB::UniversalPageStorage::gc(bool, std::__1::shared_ptr<DB::WriteLimiter> const&, std::__1::shared_ptr<DB::ReadLimiter> const&) [tiflash+241247090]
                    dbms/src/Storages/Page/V3/Universal/UniversalPageStorage.cpp:416
  0x560e153eac0a    DB::UniversalPageStorageService::gc() const [tiflash+241388554]
                    dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.cpp:302
  0x560e153ed2bb    DB::UniversalPageStorageService::create(DB::Context&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::PSDiskDelegator>, DB::PageStorageConfig const&)::$_1::operator()() const [tiflash+241398459]
                    dbms/src/Storages/Page/V3/Universal/UniversalPageStorageService.cpp:86
  0x560e153ed265    decltype(std::declval<DB::UniversalPageStorageService::create(DB::Context&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::PSDiskDelegator>, DB::PageStorageConfig const&)::$_1&>()()) std::__1::__invoke[abi:ue170006]<DB::UniversalPageStorageService::create(DB::Context&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::PSDiskDelegator>, DB::PageStorageConfig const&)::$_1&>(DB::UniversalPageStorageService::create(DB::Context&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::PSDiskDelegator>, DB::PageStorageConfig const&)::$_1&) [tiflash+241398373]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__type_traits/invoke.h:340
  0x560e153ed225    bool std::__1::__invoke_void_return_wrapper<bool, false>::__call[abi:ue170006]<DB::UniversalPageStorageService::create(DB::Context&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::PSDiskDelegator>, DB::PageStorageConfig const&)::$_1&>(DB::UniversalPageStorageService::create(DB::Context&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::PSDiskDelegator>, DB::PageStorageConfig const&)::$_1&) [tiflash+241398309]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__type_traits/invoke.h:407
  0x560e153ed1fd    std::__1::__function::__alloc_func<DB::UniversalPageStorageService::create(DB::Context&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::PSDiskDelegator>, DB::PageStorageConfig const&)::$_1, std::__1::allocator<DB::UniversalPageStorageService::create(DB::Context&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::PSDiskDelegator>, DB::PageStorageConfig const&)::$_1>, bool ()>::operator()[abi:ue170006]() [tiflash+241398269]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__functional/function.h:192
  0x560e153ec4e9    std::__1::__function::__func<DB::UniversalPageStorageService::create(DB::Context&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::PSDiskDelegator>, DB::PageStorageConfig const&)::$_1, std::__1::allocator<DB::UniversalPageStorageService::create(DB::Context&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::PSDiskDelegator>, DB::PageStorageConfig const&)::$_1>, bool ()>::operator()() [tiflash+241394921]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__functional/function.h:363
  0x560e0d76e2a2    std::__1::__function::__value_func<bool ()>::operator()[abi:ue170006]() const [tiflash+110854818]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__functional/function.h:517
  0x560e0d76ddf5    std::__1::function<bool ()>::operator()() const [tiflash+110853621]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__functional/function.h:1168
  0x560e1489f2f5    DB::BackgroundProcessingPool::threadFunction(unsigned long) [tiflash+229544693]
                    dbms/src/Storages/BackgroundProcessingPool.cpp:265
  0x560e148a005c    DB::BackgroundProcessingPool::BackgroundProcessingPool(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::shared_ptr<DB::JointThreadInfoJeallocMap>)::$_1::operator()() const [tiflash+229548124]
                    dbms/src/Storages/BackgroundProcessingPool.cpp:101
  0x560e148a0015    decltype(std::declval<DB::BackgroundProcessingPool::BackgroundProcessingPool(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::shared_ptr<DB::JointThreadInfoJeallocMap>)::$_1>()()) std::__1::__invoke[abi:ue170006]<DB::BackgroundProcessingPool::BackgroundProcessingPool(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::shared_ptr<DB::JointThreadInfoJeallocMap>)::$_1>(DB::BackgroundProcessingPool::BackgroundProcessingPool(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::shared_ptr<DB::JointThreadInfoJeallocMap>)::$_1&&) [tiflash+229548053]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__type_traits/invoke.h:340
  0x560e1489ffed    void std::__1::__thread_execute[abi:ue170006]<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, DB::BackgroundProcessingPool::BackgroundProcessingPool(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::shared_ptr<DB::JointThreadInfoJeallocMap>)::$_1>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, DB::BackgroundProcessingPool::BackgroundProcessingPool(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::shared_ptr<DB::JointThreadInfoJeallocMap>)::$_1>&, std::__1::__tuple_indices<>) [tiflash+229548013]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__thread/thread.h:221
  0x560e1489fe12    void* std::__1::__thread_proxy[abi:ue170006]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, DB::BackgroundProcessingPool::BackgroundProcessingPool(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::shared_ptr<DB::JointThreadInfoJeallocMap>)::$_1>>(void*) [tiflash+229547538]
                    /DATA/disk1/ra_common/tiflash-env-17/sysroot/bin/../include/c++/v1/__thread/thread.h:232
  0x7fb43d39b802    start_thread [libc.so.6+653314]
  0x7fb43d33b450    clone3 [libc.so.6+259152]"] [source="void DB::BackgroundProcessingPool::threadFunction(size_t)"] [thread_id=155]

4. What is your TiFlash version? (Required)

base on v7.5.3

Start at dbms/src/Storages/PathCapacityMetrics.cpp:137, then follow the callers in dbms/src/Storages/PathPool.cpp:659 and dbms/src/Storages/DeltaMerge/DeltaMergeStore_InternalBg.cpp:261. Trace how background GC produces keyspace_id 4294967295 and determine why freeRemoteUsedSize cannot find it. Done means background GC no longer reports this failed check; the issue provides no reproduction steps or named test.