initialize universal vector on the host
- Dominant language
- C++
- Stars
- 2.5k
- Forks
- 487
- Avg merge
- 2d 7h
- Merged PRs (30d)
- 296
Description
We use `thrust:universal_vector` to store data that might be accessed from the CPU or the GPU. We do the processing on the CPU for smaller buffers, and use the GPU when the buffer is larger. However, when we construct the universal vector, it seems that the constructor will call `thrust::uninitialized_fill`, which performs the uninitialized fill on the GPU and migrate the memory to the GPU, and cause page faults for subsequent accesses on the CPU. Prefetching can help to mitigate the performance issue a bit, but it is still suboptimal.
I wonder if it is possible to provide some APIs that allows us to construct the universal vector on the CPU, without touching the GPU. Below is an example that shows how much faster we can get with such an API:
```c++
#include
#include
#include
constexpr bool prefetch = false;
int universal(int vec_length) {
auto t0 = std::chrono::high_resolution_clock::now();
thrust::universal_vector test_vector(vec_length);
if (prefetch)
cudaMemPrefetchAsync(test_vector.data().get(), vec_length * sizeof(int),
cudaCpuDeviceId);
for (int j = 0; j < vec_length; j++)
test_vector[j] = j + 1;
thrust::reduce(thrust::host, test_vector.begin(), test_vector.end(), 0,
thrust::plus{});
auto t1 = std::chrono::high_resolution_clock::now();
return std::chrono::duration_cast(t1 - t0).count();
}
int raw_managed_memory(int vec_length) {
auto t0 =
std::chrono::high_resolution_clock::now();
int *managed_ptr;
cudaMallocManaged(&managed_ptr, sizeof(int) * vec_length);
thrust::uninitialized_fill_n(thrust::host, managed_ptr, vec_length, 0);
for (int j = 0; j < vec_length; j++)
managed_ptr[j] = j + 1;
thrust::reduce(thrust::host, managed_ptr, managed_ptr + vec_length, 0,
thrust::plus{});
auto t1 = std::chrono::high_resolution_clock::now();
return std::chrono::duration_cast(t1 - t0).count();
}
int main() {
// warm up
for (int i = 0; i < 10; i++) {
raw_managed_memory(100);
universal(100);
}
constexpr int repeat = 1000;
constexpr int vec_length = 10000;
int results[2] = {0};
for (int i = 0; i < repeat; i++) {
results[0] += raw_managed_memory(vec_length);
results[1] += universal(vec_length);
}
for (int &r : results)
r /= repeat;
std::cout << "interleaved:" << std::endl;
std::cout << "raw managed memory: " << results[0] << "us"
<< std::endl;
std::cout << "universal vector: " << results[1] << "us"
<< std::endl;
results[0] = 0;
results[1] = 0;
for (int i = 0; i < repeat; i++)
results[0] += raw_managed_memory(vec_length);
for (int i = 0; i < repeat; i++)
results[1] += universal(vec_length);
for (int &r : results)
r /= repeat;
std::cout << std::endl;
std::cout << "grouped:" << std::endl;
std::cout << "raw managed memory: " << results[0] << "us"
<< std::endl;
std::cout << "universal vector: " << results[1] << "us"
<< std::endl;
std::cout << std::endl;
}
```
With prefetching disabled:
```
interleaved:
raw managed memory: 14us
universal vector: 199us
grouped:
raw managed memory: 14us
universal vector: 70us
```
With prefetching enabled:
```
interleaved:
raw managed memory: 8us
universal vector: 63us
grouped:
raw managed memory: 15us
universal vector: 65us
```
Contributor guide
Assessment
This issue has not been assessed yet.