"DB::Exception: Can not find path for DMFile" when doing manual compaction

Bug Report

Please answer these questions before submitting your issue. Thanks!

1. Minimal reproduce step (Required)

Enable PageStorageV3, then,

create table x(pk int, b int) engine = DeltaMerge(pk);
create table y(pk int, b int) engine = DeltaMerge(pk);
create table z(pk int, b int) engine = DeltaMerge(pk);
manage table x merge delta;
manage table y merge delta;  <--- Error raised

2. What did you expect to see? (Required)

No errors

3. What did you see instead (Required)

Code: 0. DB::Exception: Received from 127.0.0.1:9000. DB::Exception: Can not find path for DMFile [id=2].

With the following error log in server:

2022.04.29 16:45:30.022599 [ 7 ] <Error> void DB::AsynchronousMetrics::run(): Code: 0, e.displayText() = DB::Exception: Can not find path for DMFile [id=2], e.what() = DB::Exception, Stack trace:


     0x104af2038	StackTrace::StackTrace() [tiflash+4296024120]
                	dbms/src/Common/StackTrace.cpp:23
     0x104af2074	StackTrace::StackTrace() [tiflash+4296024180]
                	dbms/src/Common/StackTrace.cpp:22
     0x104d068e0	DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) [tiflash+4298205408]
                	dbms/src/Common/Exception.h:41
     0x104cf63c8	DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) [tiflash+4298138568]
                	dbms/src/Common/Exception.h:43
     0x10d8cf740	DB::StableDiskDelegator::getDTFilePath(unsigned long long, bool) const [tiflash+4444780352]
                	dbms/src/Storages/PathPool.cpp:403
     0x10dc6f0f4	DB::DM::StableValueSpace::restore(DB::DM::DMContext&, unsigned long long) [tiflash+4448579828]
                	dbms/src/Storages/DeltaMerge/StableValueSpace.cpp:109
     0x10dbf61c8	DB::DM::Segment::restoreSegment(DB::DM::DMContext&, unsigned long long) [tiflash+4448084424]
                	dbms/src/Storages/DeltaMerge/Segment.cpp:275
     0x10db42e98	DB::DM::DeltaMergeStore::DeltaMergeStore(DB::Context&, bool, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, long long, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> > const&, DB::DM::ColumnDefine const&, bool, unsigned long, DB::DM::DeltaMergeStore::Settings const&) [tiflash+4447350424]
                	dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp:267
     0x10db44eb4	DB::DM::DeltaMergeStore::DeltaMergeStore(DB::Context&, bool, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, long long, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> > const&, DB::DM::ColumnDefine const&, bool, unsigned long, DB::DM::DeltaMergeStore::Settings const&) [tiflash+4447358644]
                	dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp:212
     0x10d9af7c4	std::__1::__shared_ptr_emplace<DB::DM::DeltaMergeStore, std::__1::allocator<DB::DM::DeltaMergeStore> >::__shared_ptr_emplace<DB::Context&, bool const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, long long&, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> >, DB::DM::ColumnDefine, bool&, unsigned long&, DB::DM::DeltaMergeStore::Settings>(std::__1::allocator<DB::DM::DeltaMergeStore>, DB::Context&, bool const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, long long&, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> >&&, DB::DM::ColumnDefine&&, bool&, unsigned long&, DB::DM::DeltaMergeStore::Settings&&) [tiflash+4445697988]
                	/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include/c++/v1/memory:2627
     0x10d9af648	std::__1::__shared_ptr_emplace<DB::DM::DeltaMergeStore, std::__1::allocator<DB::DM::DeltaMergeStore> >::__shared_ptr_emplace<DB::Context&, bool const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, long long&, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> >, DB::DM::ColumnDefine, bool&, unsigned long&, DB::DM::DeltaMergeStore::Settings>(std::__1::allocator<DB::DM::DeltaMergeStore>, DB::Context&, bool const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, long long&, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> >&&, DB::DM::ColumnDefine&&, bool&, unsigned long&, DB::DM::DeltaMergeStore::Settings&&) [tiflash+4445697608]
                	/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include/c++/v1/memory:2621
     0x10d9af544	std::__1::shared_ptr<DB::DM::DeltaMergeStore> std::__1::allocate_shared<DB::DM::DeltaMergeStore, std::__1::allocator<DB::DM::DeltaMergeStore>, DB::Context&, bool const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, long long&, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> >, DB::DM::ColumnDefine, bool&, unsigned long&, DB::DM::DeltaMergeStore::Settings, void>(std::__1::allocator<DB::DM::DeltaMergeStore> const&, DB::Context&, bool const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, long long&, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> >&&, DB::DM::ColumnDefine&&, bool&, unsigned long&, DB::DM::DeltaMergeStore::Settings&&) [tiflash+4445697348]
                	/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include/c++/v1/memory:3385
     0x10d96cb6c	std::__1::shared_ptr<DB::DM::DeltaMergeStore> std::__1::make_shared<DB::DM::DeltaMergeStore, DB::Context&, bool const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, long long&, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> >, DB::DM::ColumnDefine, bool&, unsigned long&, DB::DM::DeltaMergeStore::Settings, void>(DB::Context&, bool const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&, long long&, std::__1::vector<DB::DM::ColumnDefine, std::__1::allocator<DB::DM::ColumnDefine> >&&, DB::DM::ColumnDefine&&, bool&, unsigned long&, DB::DM::DeltaMergeStore::Settings&&) [tiflash+4445424492]
                	/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include/c++/v1/memory:3394
     0x10d95e62c	DB::StorageDeltaMerge::getAndMaybeInitStore() [tiflash+4445365804]
                	dbms/src/Storages/StorageDeltaMerge.cpp:1528
     0x10d1eacbc	DB::StorageDeltaMerge::getStore() [tiflash+4437552316]
                	dbms/src/Storages/StorageDeltaMerge.h:137
     0x10d1ea590	DB::AsynchronousMetrics::update() [tiflash+4437550480]
                	dbms/src/Interpreters/AsynchronousMetrics.cpp:165
     0x10d1e9ff4	DB::AsynchronousMetrics::run() [tiflash+4437549044]
                	dbms/src/Interpreters/AsynchronousMetrics.cpp:103
     0x104c31278	DB::AsynchronousMetrics::AsynchronousMetrics(DB::Context&)::'lambda'()::operator()() const [tiflash+4297331320]
                	dbms/src/Interpreters/AsynchronousMetrics.h:38
     0x104c31214	decltype(std::__1::forward<DB::AsynchronousMetrics::AsynchronousMetrics(DB::Context&)::'lambda'()>(fp)()) std::__1::__invoke<DB::AsynchronousMetrics::AsynchronousMetrics(DB::Context&)::'lambda'()>(DB::AsynchronousMetrics::AsynchronousMetrics(DB::Context&)::'lambda'()&&) [tiflash+4297331220]
                	/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include/c++/v1/type_traits:3694
     0x104c311ac	void std::__1::__thread_execute<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, DB::AsynchronousMetrics::AsynchronousMetrics(DB::Context&)::'lambda'()>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, DB::AsynchronousMetrics::AsynchronousMetrics(DB::Context&)::'lambda'()>&, std::__1::__tuple_indices<>) [tiflash+4297331116]
                	/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include/c++/v1/thread:286
     0x104c309ac	void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, DB::AsynchronousMetrics::AsynchronousMetrics(DB::Context&)::'lambda'()> >(void*) [tiflash+4297329068]
                	/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include/c++/v1/thread:297
     0x1c7691240	__pthread_deallocate [libsystem_pthread.dylib+6445650496]
     0x1c768c024	_pthread_key_init_np [libsystem_pthread.dylib+6445629476]

4. What is your TiFlash version? (Required)

abb313953973dd955daad63724a2cf22bb94e264

(April 22, 2022 at 11:58:04 GMT+8)

Reproduce the PageStorageV3 case with the three DeltaMerge tables and manual compactions. Start at dbms/src/Storages/PathPool.cpp:getDTFilePath, then follow the restore paths in StableValueSpace.cpp and Segment.cpp. Done means the second MANAGE TABLE merge delta completes without the missing-DMFile error.