Address sanitizer warnings in threaded `reduce_sum`

Description

If you build the reduce_sum tests with threading and the address sanitizer turned on you can get warnings about memory leaking.

Test is:

./runTests.py test/unit/math/rev/functor/reduce_sum_test

Get output like this:

Indirect leak of 8 byte(s) in 1 object(s) allocated from:
    #0 0x51baa0 in operator new(unsigned long) (/home/bbales2/math-benchmarks/math/test/unit/math/rev/functor/reduce_sum_test+0x51baa0)
    #1 0x53165b in __gnu_cxx::new_allocator<unsigned long>::allocate(unsigned long, void const*) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/ext/new_allocator.h:111:27
    #2 0x5315fb in std::allocator_traits<std::allocator<unsigned long> >::allocate(std::allocator<unsigned long>&, unsigned long) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/alloc_traits.h:436:20
    #3 0x5315a2 in std::_Vector_base<unsigned long, std::allocator<unsigned long> >::_M_allocate(unsigned long) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/stl_vector.h:172:20
    #4 0x547483 in void std::vector<unsigned long, std::allocator<unsigned long> >::_M_realloc_insert<unsigned long>(__gnu_cxx::__normal_iterator<unsigned long*, std::vector<unsigned long, std::allocator<unsigned long> > >, unsigned long&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/vector.tcc:406:33
    #5 0x5471a1 in void std::vector<unsigned long, std::allocator<unsigned long> >::emplace_back<unsigned long>(unsigned long&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/vector.tcc:105:4
    #6 0x546d7f in std::vector<unsigned long, std::allocator<unsigned long> >::push_back(unsigned long&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/bits/stl_vector.h:954:9
    #7 0x5233c4 in stan::math::start_nested() /home/bbales2/math-benchmarks/math/./stan/math/rev/core/start_nested.hpp:19:62
    #8 0x5445c0 in stan::math::nested_rev_autodiff::nested_rev_autodiff() /home/bbales2/math-benchmarks/math/./stan/math/rev/core/nested_rev_autodiff.hpp:29:27
    #9 0x562725 in stan::math::internal::reduce_sum_impl<stan::math::test::count_lpdf<stan::math::var_value<double, void> >, void, stan::math::var_value<double, void>, std::vector<int, std::allocator<int> >&, std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >&, std::vector<int, std::allocator<int> >&>::recursive_reducer::operator()(tbb::blocked_range<unsigned long> const&) /home/bbales2/math-benchmarks/math/./stan/math/rev/functor/reduce_sum.hpp:97:33
    #10 0x5621c8 in tbb::interface9::internal::start_reduce<tbb::blocked_range<unsigned long>, stan::math::internal::reduce_sum_impl<stan::math::test::count_lpdf<stan::math::var_value<double, void> >, void, stan::math::var_value<double, void>, std::vector<int, std::allocator<int> >&, std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >&, std::vector<int, std::allocator<int> >&>::recursive_reducer, tbb::auto_partitioner const>::run_body(tbb::blocked_range<unsigned long>&) /home/bbales2/math-benchmarks/math/lib/tbb_2019_U8/include/tbb/parallel_reduce.h:147:37
    #11 0x5618e0 in void tbb::interface9::internal::dynamic_grainsize_mode<tbb::interface9::internal::adaptive_mode<tbb::interface9::internal::auto_partition_type> >::work_balance<tbb::interface9::internal::start_reduce<tbb::blocked_range<unsigned long>, stan::math::internal::reduce_sum_impl<stan::math::test::count_lpdf<stan::math::var_value<double, void> >, void, stan::math::var_value<double, void>, std::vector<int, std::allocator<int> >&, std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >&, std::vector<int, std::allocator<int> >&>::recursive_reducer, tbb::auto_partitioner const>, tbb::blocked_range<unsigned long> >(tbb::interface9::internal::start_reduce<tbb::blocked_range<unsigned long>, stan::math::internal::reduce_sum_impl<stan::math::test::count_lpdf<stan::math::var_value<double, void> >, void, stan::math::var_value<double, void>, std::vector<int, std::allocator<int> >&, std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >&, std::vector<int, std::allocator<int> >&>::recursive_reducer, tbb::auto_partitioner const>&, tbb::blocked_range<unsigned long>&) /home/bbales2/math-benchmarks/math/lib/tbb_2019_U8/include/tbb/partitioner.h:435:23
    #12 0x5611ce in void tbb::interface9::internal::partition_type_base<tbb::interface9::internal::auto_partition_type>::execute<tbb::interface9::internal::start_reduce<tbb::blocked_range<unsigned long>, stan::math::internal::reduce_sum_impl<stan::math::test::count_lpdf<stan::math::var_value<double, void> >, void, stan::math::var_value<double, void>, std::vector<int, std::allocator<int> >&, std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >&, std::vector<int, std::allocator<int> >&>::recursive_reducer, tbb::auto_partitioner const>, tbb::blocked_range<unsigned long> >(tbb::interface9::internal::start_reduce<tbb::blocked_range<unsigned long>, stan::math::internal::reduce_sum_impl<stan::math::test::count_lpdf<stan::math::var_value<double, void> >, void, stan::math::var_value<double, void>, std::vector<int, std::allocator<int> >&, std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >&, std::vector<int, std::allocator<int> >&>::recursive_reducer, tbb::auto_partitioner const>&, tbb::blocked_range<unsigned long>&) /home/bbales2/math-benchmarks/math/lib/tbb_2019_U8/include/tbb/partitioner.h:253:16
    #13 0x560a32 in tbb::interface9::internal::start_reduce<tbb::blocked_range<unsigned long>, stan::math::internal::reduce_sum_impl<stan::math::test::count_lpdf<stan::math::var_value<double, void> >, void, stan::math::var_value<double, void>, std::vector<int, std::allocator<int> >&, std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >&, std::vector<int, std::allocator<int> >&>::recursive_reducer, tbb::auto_partitioner const>::execute() /home/bbales2/math-benchmarks/math/lib/tbb_2019_U8/include/tbb/parallel_reduce.h:187:22
    #14 0x7f420c406ef8 in tbb::internal::custom_scheduler<tbb::internal::IntelSchedulerTraits>::local_wait_for_all(tbb::task&, tbb::task*) (/home/bbales2/math-benchmarks/math/lib/tbb/libtbb.so.2+0x27ef8)

This is what I have for my make/local:

STAN_THREADS=TRUE
CXX=clang++
O=0
CXXFLAGS=-g -fsanitize=address -ftemplate-backtrace-limit=0 -fno-omit-frame-pointer
LDFLAGS=-g -fsanitize=address -fno-omit-frame-pointer

You can comment out all the tests below the one starting with:

TEST(StanMathRev_reduce_sum, gradient)

The errors are intermittent so you might have to run the code a bunch to get one.

@wds15 so you know. Not sure what this is.

Current Version:

v3.3.0

Run ./runTests.py test/unit/math/rev/functor/reduce_sum_test with the threaded AddressSanitizer settings from make/local, repeating the test as needed. Inspect stan/math/rev/functor/reduce_sum.hpp around line 97 and the nested_rev_autodiff/start_nested paths shown in the trace. Done means the intermittent leak warnings no longer appear in this test.