py::make_tuple defaulting std::optional members passed by const reference
Open
Nobody has claimed this yet.
- Dominant language
- C++
- Stars
- 18k
- Forks
- 2.3k
- Avg merge
- 5d 17h
- Merged PRs (30d)
- 10
Description
Issue description
I am having an issue when passing a const reference of an std::optional to py::make_tuple, the contents of the instance appear to be getting defaulted. There is a lot of boilerplate for printing the objects to the REPL, but I included it for a reproducible example.
Using v2.4.3, C++17, gcc 9.2.0 (musl-libc - alpine linux inside docker).
In [1]: import pickle
In [2]: from snmp_stream._snmp_stream import ObjectIdentityRange, ObjectIdentity
In [3]: x = ObjectIdentityRange(ObjectIdentity([1]), ObjectIdentity([2]))
In [4]: x
Out[4]: ObjectIdentityRange(start=ObjectIdentity([1]), stop=ObjectIdentity([2]))
In [5]: pickle.dumps(x)
Out[5]: b'\x80\x04\x95g\x00\x00\x00\x00\x00\x00\x00\x8c\x18snmp_stream._snmp_stream\x94\x8c\x13ObjectIdentityRange\x94\x93\x94)\x81\x94h\x00\x8c\x0eObjectIdentity\x94\x93\x94)\x81\x94]\x94K\x01a\x85\x94bh\x05)\x81\x94]\x94K\x02a\x85\x94b\x86\x94b.'
In [6]: x # vectors are getting erased, narrowed it down to py::make_tuple
Out[6]: ObjectIdentityRange(start=ObjectIdentity([]), stop=ObjectIdentity([]))
In [7]: x = ObjectIdentity([0])
In [8]: x
Out[8]: ObjectIdentity([0])
In [9]: pickle.dumps(x)
Out[9]: b'\x80\x04\x95:\x00\x00\x00\x00\x00\x00\x00\x8c\x18snmp_stream._snmp_stream\x94\x8c\x0eObjectIdentity\x94\x93\x94)\x81\x94]\x94K\x00a\x85\x94b.'
In [10]: x # does not occur in this case however
Out[10]: ObjectIdentity([0])
Reproducible example code
#include <algorithm>
#include <sstream>
#include <vector>
#include <pybind11/operators.h>
#include <pybind11/stl.h>
namespace py = pybind11;
namespace snmp_stream {
// BOILERPLATE FOR STR AND REPR ///////////////////////////////////////////////////
template <typename T>
inline auto join(T begin, T end, std::string const &sep) -> std::string {
std::ostringstream oss;
if (begin != end) {
oss << *begin++;
}
while (begin != end) {
oss << sep << *begin++;
}
return oss.str();
}
template <typename T, typename std::enable_if<std::is_integral<T>::value,
T>::type * = nullptr>
inline auto arg_to_string(T const &val) -> std::string {
return std::to_string(val);
}
template <typename, typename = void> struct has_repr : std::false_type {};
template <typename T>
struct has_repr<T, std::void_t<decltype(std::declval<T>().repr())>>
: std::is_same<decltype(std::declval<T>().repr()), std::string> {};
template <typename T,
typename std::enable_if<has_repr<T>::value, T>::type * = nullptr>
inline auto arg_to_string(T const &val) -> std::string {
return val.repr();
}
template <typename T> struct is_optional : std::false_type {};
template <typename T> struct is_optional<std::optional<T>> : std::true_type {};
template <typename T,
typename std::enable_if<is_optional<T>::value, T>::type * = nullptr>
inline auto arg_to_string(T const &val) -> std::string {
if (val.has_value()) {
return arg_to_string(*val);
}
return "None";
}
template <typename T, typename = void> struct is_iterable : std::false_type {};
template <typename T>
struct is_iterable<T, std::void_t<decltype(std::declval<T>().begin()),
decltype(std::declval<T>().end())>>
: std::true_type {};
template <typename T,
typename std::enable_if<is_iterable<T>::value, T>::type * = nullptr>
inline auto arg_to_string(T const &val) -> std::string {
std::vector<std::string> strings;
std::transform(val.begin(), val.end(), std::back_inserter(strings),
[](auto i) -> std::string { return arg_to_string(i); });
return "[" + join(strings.begin(), strings.end(), ", ") + "]";
}
#define DEFINE_REPR(T) \
auto str() const->std::string; \
inline auto str()->std::string { return std::as_const(*this).str(); }; \
inline auto repr() const->std::string { return this->str(); }; \
inline auto repr()->std::string { return std::as_const(*this).repr(); };
///////////////////////////////////////////////////////////////////////////////////
#define INLINE_CONST_GETTER(T, field) \
inline auto get_##field() const->decltype(T::field) const & { \
return this->field; \
} \
inline auto get_##field()->decltype(T::field) const & { \
return std::as_const(*this).get_##field(); \
}
class ObjectIdentity {
private:
std::vector<uint64_t> oid;
public:
ObjectIdentity(std::vector<uint64_t> v) : oid(std::move(v)) {}
INLINE_CONST_GETTER(ObjectIdentity, oid);
DEFINE_REPR(ObjectIdentity);
};
auto ObjectIdentity::str() const -> std::string {
return "ObjectIdentity(" + arg_to_string(oid) + ")";
}
class ObjectIdentityRange {
private:
std::optional<ObjectIdentity> start;
std::optional<ObjectIdentity> stop;
public:
ObjectIdentityRange(std::optional<ObjectIdentity> start,
std::optional<ObjectIdentity> stop)
: start(start.has_value() && !start->get_oid().empty() ? std::move(start)
: std::nullopt),
stop(stop.has_value() && !stop->get_oid().empty() ? std::move(stop)
: std::nullopt) {}
INLINE_CONST_GETTER(ObjectIdentityRange, start);
INLINE_CONST_GETTER(ObjectIdentityRange, stop);
DEFINE_REPR(ObjectIdentityRange);
};
auto ObjectIdentityRange::str() const -> std::string {
return "ObjectIdentityRange(start=" + arg_to_string(start) +
", stop=" + arg_to_string(stop) + ")";
}
PYBIND11_MODULE(_snmp_stream, m) { // NOLINT
py::class_<ObjectIdentity>(m, "ObjectIdentity")
.def(py::init<std::vector<uint64_t> const &>())
.def("__str__", [](ObjectIdentity const &oid) { return oid.str(); })
.def("__repr__", [](ObjectIdentity const &oid) { return oid.repr(); })
.def(py::pickle(
[](ObjectIdentity const &oid) {
return py::make_tuple(oid.get_oid());
},
[](py::tuple const &t) {
return (ObjectIdentity){t[0].cast<std::vector<uint64_t>>()};
}));
py::class_<ObjectIdentityRange>(m, "ObjectIdentityRange")
.def(py::init<std::optional<ObjectIdentity> const &,
std::optional<ObjectIdentity> const &>(),
py::arg("start") = std::nullopt, py::arg("stop") = std::nullopt)
.def("__str__",
[](ObjectIdentityRange const &range) { return range.str(); })
.def("__repr__",
[](ObjectIdentityRange const &range) { return range.repr(); })
.def(py::pickle(
[](ObjectIdentityRange const &range) {
return py::make_tuple(range.get_start(), range.get_stop());
},
[](py::tuple const &t) {
return (ObjectIdentityRange){
t[0].cast<std::optional<ObjectIdentity>>(),
t[1].cast<std::optional<ObjectIdentity>>()};
}));
}
} // namespace snmp_stream
Contributor guide
First steps
- Read the whole issue, then the project's contributing guide.
- Comment on the issue to say you are picking it up — it saves two people doing the same work.
- Fork the repository and make your change on a branch.
- Open a pull request that references the issue number.
Research direction
Use the provided C++17 reproducer and begin at the py::make_tuple call in ObjectIdentityRange's py::pickle serializer. Verify that the pickle round trip preserves the start and stop vector contents, including the [1]/[2] case, and use that as the done condition.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- cpp, python
- Domain
- api
- Issue type
- Bug
- Difficulty
- 3/5
- Estimated time
- 1-2 days
- Activity status
- Stale
- Clarity
- Clearly specified
- Newbie friendliness
- 42/100