[QUESTION] pybind11 pickle intrusive_ptr

I tried to add the pickle support for sentencepiece cpp extension under pybind11. However, I always got the error as follow:

FAILED: /private/home/zhangguanheng/tmp/PyTorch/text/build/temp.linux-x86_64-3.7/private/home/zhangguanheng/tmp/PyTorch/text/torchtext/csrc/register_bindings.o
c++ -MMD -MF /private/home/zhangguanheng/tmp/PyTorch/text/build/temp.linux-x86_64-3.7/private/home/zhangguanheng/tmp/PyTorch/text/torchtext/csrc/register_bindings.o.d -pthread -B /private/home/zhangguanheng/anaconda3/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/private/home/zhangguanheng/tmp/PyTorch/text/torchtext/csrc -I/private/home/zhangguanheng/tmp/PyTorch/text/third_party/build/include -I/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include -I/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/torch/csrc/api/include -I/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/TH -I/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/THC -I/private/home/zhangguanheng/anaconda3/include/python3.7m -c -c /private/home/zhangguanheng/tmp/PyTorch/text/torchtext/csrc/register_bindings.cpp -o /private/home/zhangguanheng/tmp/PyTorch/text/build/temp.linux-x86_64-3.7/private/home/zhangguanheng/tmp/PyTorch/text/torchtext/csrc/register_bindings.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H '-DPYBIND11_COMPILER_TYPE="_gcc"' '-DPYBIND11_STDLIB="_libstdcpp"' '-DPYBIND11_BUILD_ABI="_cxxabi1011"' -DTORCH_EXTENSION_NAME=_torchtext -D_GLIBCXX_USE_CXX11_ABI=1 -std=c++14
cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++
In file included from /private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/pybind11/pybind11.h:48:0,
                 from /private/home/zhangguanheng/tmp/PyTorch/text/torchtext/csrc/register_bindings.cpp:1:
/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/pybind11/detail/init.h: In instantiation of ‘void pybind11::detail::initimpl::construct(...) [with Class = pybind11::class_<torchtext::SentencePiece>]’:
/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/pybind11/detail/init.h:288:21:   required from ‘void pybind11::detail::initimpl::setstate(pybind11::detail::value_and_holder&, T&&, bool) [with Class = pybind11::class_<torchtext::SentencePiece>; T = c10::intrusive_ptr<torchtext::SentencePiece>]’
/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/pybind11/detail/init.h:328:28:   required from ‘pybind11::detail::initimpl::pickle_factory<Get, Set, RetState(Self), NewInstance(ArgState)>::execute(Class&, const Extra& ...) &&::<lambda(pybind11::detail::value_and_holder&, ArgState)> [with Class = pybind11::class_<torchtext::SentencePiece>; Extra = {}; Get = torchtext::pybind11_init__torchtext(pybind11::module_&)::<lambda(const c10::intrusive_ptr<torchtext::SentencePiece>&)>; Set = torchtext::pybind11_init__torchtext(pybind11::module_&)::<lambda(std::__cxx11::string)>; RetState = std::__cxx11::basic_string<char>; Self = const c10::intrusive_ptr<torchtext::SentencePiece>&; NewInstance = c10::intrusive_ptr<torchtext::SentencePiece>; ArgState = std::__cxx11::basic_string<char>]’
/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/pybind11/detail/init.h:322:53:   required from ‘struct pybind11::detail::initimpl::pickle_factory<Get, Set, RetState(Self), NewInstance(ArgState)>::execute(Class&, const Extra& ...) && [with Class = pybind11::class_<torchtext::SentencePiece>; Extra = {}; Get = torchtext::pybind11_init__torchtext(pybind11::module_&)::<lambda(const c10::intrusive_ptr<torchtext::SentencePiece>&)>; Set = torchtext::pybind11_init__torchtext(pybind11::module_&)::<lambda(std::__cxx11::string)>; RetState = std::__cxx11::basic_string<char>; Self = const c10::intrusive_ptr<torchtext::SentencePiece>&; NewInstance = c10::intrusive_ptr<torchtext::SentencePiece>; ArgState = std::__cxx11::basic_string<char>]::<lambda(struct pybind11::detail::value_and_holder&, class std::__cxx11::basic_string<char>)>’
/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/pybind11/detail/init.h:322:9:   required from ‘void pybind11::detail::initimpl::pickle_factory<Get, Set, RetState(Self), NewInstance(ArgState)>::execute(Class&, const Extra& ...) && [with Class = pybind11::class_<torchtext::SentencePiece>; Extra = {}; Get = torchtext::pybind11_init__torchtext(pybind11::module_&)::<lambda(const c10::intrusive_ptr<torchtext::SentencePiece>&)>; Set = torchtext::pybind11_init__torchtext(pybind11::module_&)::<lambda(std::__cxx11::string)>; RetState = std::__cxx11::basic_string<char>; Self = const c10::intrusive_ptr<torchtext::SentencePiece>&; NewInstance = c10::intrusive_ptr<torchtext::SentencePiece>; ArgState = std::__cxx11::basic_string<char>]’
/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/pybind11/pybind11.h:1314:9:   required from ‘pybind11::class_<type_, options>& pybind11::class_<type_, options>::def(pybind11::detail::initimpl::pickle_factory<Args ...>&&, const Extra& ...) [with Args = {torchtext::pybind11_init__torchtext(pybind11::module_&)::<lambda(const c10::intrusive_ptr<torchtext::SentencePiece, c10::detail::intrusive_target_default_null_type<torchtext::SentencePiece> >&)>, torchtext::pybind11_init__torchtext(pybind11::module_&)::<lambda(std::__cxx11::string)>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >(const c10::intrusive_ptr<torchtext::SentencePiece, c10::detail::intrusive_target_default_null_type<torchtext::SentencePiece> >&), c10::intrusive_ptr<torchtext::SentencePiece, c10::detail::intrusive_target_default_null_type<torchtext::SentencePiece> >(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)}; Extra = {}; type_ = torchtext::SentencePiece; options = {}]’
/private/home/zhangguanheng/tmp/PyTorch/text/torchtext/csrc/register_bindings.cpp:51:13:   required from here
/private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/pybind11/detail/init.h:86:5: error: static assertion failed: pybind11::init(): init function must return a compatible pointer, holder, or value
     static_assert(!std::is_same<Class, Class>::value /* always false */,
     ^~~~~~~~~~~~~
In file included from /private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/ATen/core/ivalue.h:971:0,
                 from /private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/ATen/ATen.h:33,
                 from /private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/torch/csrc/api/include/torch/types.h:3,
                 from /private/home/zhangguanheng/tmp/PyTorch/pytorch/torch/include/torch/script.h:3,
                 from /private/home/zhangguanheng/tmp/PyTorch/text/torchtext/csrc/regex.h:3,
                 from /private/home/zhangguanheng/tmp/PyTorch/text/torchtext/csrc/register_bindings.cpp:3:

The original PR is https://github.com/pytorch/text/pull/1085/files.

Start with torchtext/csrc/register_bindings.cpp at line 51 and inspect the pybind11/detail/init.h diagnostics shown in the report. Compare the original pytorch/text pull request with pybind11's handling of c10::intrusive_ptr during pickle reconstruction. Done means establishing whether this usage is supported and documenting a confirmed resolution.