Device mapping takes a very long time
- Dominant language
- Python
- Stars
- 636
- Forks
- 81
- Avg merge
- 1d 19h
- Merged PRs (30d)
- 7
Description
@mattwala
The code below (unintentionally) triggers many write-after-write dependencies in the writing of all the `f_new`. This leads to a lot of kernel splitting, and the device mapping stage takes a very long time. This leads to a poor user experience, and we should think of a way to fix that. If nothing else, this may serve as a good example that provokes performance degradation.
```
import loopy as lp
import pyopencl as cl
import pyopencl.array
import numpy as np
nx, ny, nv = 100, 100, 12
f = np.zeros((nx, ny, nv), dtype=np.float32)
f_new = np.zeros_like(f)
ctx = cl.create_some_context()
queue = cl.CommandQueue(ctx)
knl = lp.make_kernel(
"{[ii,jj]:0<=ii m[0] = + f[i-1, j, 0] + f[i, j-1, 1] + f[i+1, j, 2] + f[i, j+1, 3]
m[1] = + 4.*f[i-1, j, 0] - 4.*f[i+1, j, 2]
m[2] = + 4.*f[i, j-1, 1] - 4.*f[i, j+1, 3]
m[3] = + f[i-1, j, 0] - f[i, j-1, 1] + f[i+1, j, 2] - f[i, j+1, 3]
m[4] = + f[i-1, j, 4] + f[i, j-1, 5] + f[i+1, j, 6] + f[i, j+1, 7]
m[5] = + 4.*f[i-1, j, 4] - 4.*f[i+1, j, 6]
m[6] = + 4.*f[i, j-1, 5] - 4.*f[i, j+1, 7]
m[7] = + f[i-1, j, 4] - f[i, j-1, 5] + f[i+1, j, 6] - f[i, j+1, 7]
m[8] = + f[i-1, j, 8] + f[i, j-1, 9] + f[i+1, j, 10] + f[i, j+1, 11]
m[9] = + 4.*f[i-1, j, 8] - 4.*f[i+1, j, 10]
m[10] = + 4.*f[i, j-1, 9] - 4.*f[i, j+1, 11]
m[11] = + f[i-1, j, 8] - f[i, j-1, 9] + f[i+1, j, 10] - f[i, j+1, 11]
m[1] = m[1] + 2.*(m[4] - m[1])
m[2] = m[2] + 2.*(m[8] - m[2])
m[3] = m[3]*(1. - 1.5)
m[5] = m[5] + 1.5*(0.5*(m[0]*m[0]) + (m[4]*m[4])/m[0] - m[5])
m[6] = m[6] + 1.5*(m[4]*m[8]/m[0] - m[6])
m[7] = m[7]*(1. - 1.2000000000000000)
m[9] = m[9] + 1.5*(m[4]*m[8]/m[0] - m[9])
m[10] = m[10] + 1.5*(0.5*(m[0]*m[0]) + (m[8]*m[8])/m[0] - m[10])
m[11] = m[11]*(1. - 1.2)
# <>f_new[0] = + 0.25*m[0] + 0.125*m[1] + 0.25*m[3]
# f_new[1] = + 0.25*m[0] + 0.125*m[2] - 0.25*m[3]
# f_new[2] = + 0.25*m[0] - 0.125*m[1] + 0.25*m[3]
# f_new[3] = + 0.25*m[0] - 0.125*m[2] - 0.25*m[3]
# f_new[4] = + 0.25*m[4] + 0.125*m[5] + 0.25*m[7]
# f_new[5] = + 0.25*m[4] + 0.125*m[6] - 0.25*m[7]
# f_new[6] = + 0.25*m[4] - 0.125*m[5] + 0.25*m[7]
# f_new[7] = + 0.25*m[4] - 0.125*m[6] - 0.25*m[7]
# f_new[8] = + 0.25*m[8] + 0.125*m[9] + 0.25*m[11]
# f_new[9] = + 0.25*m[8] + 0.125*m[10] - 0.25*m[11]
# f_new[10] = + 0.25*m[8] - 0.125*m[9] + 0.25*m[11]
# f_new[11] = + 0.25*m[8] - 0.125*m[10] - 0.25*m[11]
f_new[i, j, 0] = + 0.25*m[0] + 0.125*m[1] + 0.25*m[3]
f_new[i, j, 1] = + 0.25*m[0] + 0.125*m[2] - 0.25*m[3]
f_new[i, j, 2] = + 0.25*m[0] - 0.125*m[1] + 0.25*m[3]
f_new[i, j, 3] = + 0.25*m[0] - 0.125*m[2] - 0.25*m[3]
f_new[i, j, 4] = + 0.25*m[4] + 0.125*m[5] + 0.25*m[7]
f_new[i, j, 5] = + 0.25*m[4] + 0.125*m[6] - 0.25*m[7]
f_new[i, j, 6] = + 0.25*m[4] - 0.125*m[5] + 0.25*m[7]
f_new[i, j, 7] = + 0.25*m[4] - 0.125*m[6] - 0.25*m[7]
f_new[i, j, 8] = + 0.25*m[8] + 0.125*m[9] + 0.25*m[11]
f_new[i, j, 9] = + 0.25*m[8] + 0.125*m[10] - 0.25*m[11]
f_new[i, j, 10] = + 0.25*m[8] - 0.125*m[9] + 0.25*m[11]
f_new[i, j, 11] = + 0.25*m[8] - 0.125*m[10] - 0.25*m[11]
end
""", seq_dependencies=True)
knl = lp.add_and_infer_dtypes(knl, {"f": np.float32})
#knl = lp.add_and_infer_dtypes(knl, {"f_new": np.float32})
ref_knl = knl
knl = lp.split_iname(knl, "ii", 16, outer_tag="g.1", inner_tag="l.1")
knl = lp.split_iname(knl, "jj", 16, outer_tag="g.0", inner_tag="l.0")
knl = lp.expand_subst(knl)
knl = lp.add_prefetch(knl, "f", "ii_inner,jj_inner", fetch_bounding_box=True)
knl = lp.set_options(knl, write_cl=True)
knl = lp.preprocess_kernel(knl)
print(knl)
f = cl.array.to_device(queue, f)
f_new = cl.array.zeros_like(f)
evt, f_new = knl(queue, f=f)#, f_new=f_new)
```
Contributor guide
No contributing guide indexed for this repository
Assessment
This issue has not been assessed yet.