E3SM-Project / E3SM-Project/scream
Some bugs and questions about running SCREAM on intel PVC GPU
- Dominant language
- No language data
- Stars
- 79
- Forks
- 54
- PR merge metrics
- No merged PRs in 30d
Description
I'm trying to run SCREAM on intel PVC GPU, however meet some errors, looking for help, thanks!
Here's my testing environment:
SCREAM code: lasted code of master version (https://github.com/E3SM-Project/scream), not sure whether this is the correct version for test on intel GPU?
Machine: intel CPU+ intel PVC GPU (1100)
Compiler and MPI: intel oneapi 2024 (icx, ifx, icpx; mpiicx, mpiifx, mpiicpx)
Here's my config files:
1. scream/cime_config/machines/config_machines.xml
```xml
HPC, 6430 CPU + 1100 PVC(56core, 48GB)
LINUX
oneapi-ifx,oneapi-ifxgpu,gnu
impi,openmpi, mpich
/home/lujingyu/E3SM/SCREAM/cases/scratch/$CASE
/home/lujingyu/E3SM/SCREAM/inputdata
/home/lujingyu/E3SM/SCREAM/inputdata/atm/datm7
/home/lujingyu/E3SM/SCREAM/cases/archive/$CASE
16
e3sm_developer
4
none
e3sm
56
208
56
56
64
56
FALSE
mpirun
-np {{ total_tasks }}
$CIME_OUTPUT_ROOT/$CASE/run
$CIME_OUTPUT_ROOT/$CASE/bld
/home/lujingyu/nc_pnc2023_intel2024
/home/lujingyu/nc_pnc2023_intel2024/lib:$ENV{LD_LIBRARY_PATH}
/home/lujingyu/nc_pnc2023_intel2024/bin:$ENV{PATH}
10
1
"opencl:gpu;level_zero:gpu"
1
4000MB
0
131072
20
0
DISABLED
131072
20
0
-1
```
2. scream/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake
```cmake
if (compile_threaded)
string(APPEND CMAKE_C_FLAGS " -qopenmp")
string(APPEND CMAKE_Fortran_FLAGS " -qopenmp")
string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp")
endif()
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_C_FLAGS " -traceback -fp-model precise -std=gnu99")
string(APPEND CMAKE_CXX_FLAGS " -traceback -fp-model precise")
string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise ")
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")
set(HAS_F2008_CONTIGUOUS "TRUE")
set(MPIFC "mpiifx")
set(MPICC "mpiicx")
set(MPICXX "mpiicpx")
set(SCC "icx")
set(SCXX "icpx")
set(SFC "ifx")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64 ")
set(USE_SYCL "TRUE")
set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "")
string(APPEND SYCL_FLAGS " -fsycl -fsycl-targets=spir64 ") #-linux-intel_gpu_pvc -Xsycl-target-backend Xe-MAX -sycl-std=121
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DCMAKE_CXX_STANDARD=17")
```
3. scream/components/eamxx/cmake/machine-files/PVC1100.cmake
```cmake
include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
common_setup()
# Load all kokkos settings from Ekat's mach file
include (${EKAT_MACH_FILES_PATH}/kokkos/intel-pvc.cmake)
```
4. scream/externals/ekat/cmake/machine-files/PVC1100.cmake
```cmake
# Load PVC arch with SYCL backend for kokkos
include (${CMAKE_CURRENT_LIST_DIR}/kokkos/intel-pvc.cmake)
```
Here's my case: (**is this F2000-SCREAM-SA @ ne30pg2_ne30pg2 the best test case?**)
```shell
./create_newcase --case test1 --compset F2000-SCREAM-SA --res ne30pg2_ne30pg2 --mach PVC1100 --compiler oneapi-ifxgpu --mpilib impi
```
And here are all my log files, [bld.zip](https://github.com/E3SM-Project/scream/files/14428057/bld.zip)
and the error is:
```shell
/opt/intel/oneapi/compiler/2024.0/bin/compiler/../../include/sycl/types.hpp:2382:17: error: ambiguous partial specializations of 'is_device_copyable, Kokkos::MemoryTraits<0>>, Kokkos::View, Kokkos::MemoryTraits<0>>, Kokkos::LayoutLeft, Kokkos::Experimental::SYCL, 5, long>, Kokkos::Experimental::Impl::SYCLInternal::USMObjectMem>>'
2382 | static_assert(is_device_copyable::value ||
```
I'm fresh on kokkos and SYCL, so I'm confused to the link between kokkos and SYCL, and the backend target with intel PVC GPU, **_is there anything wrong in my config files and lead to this error?_**
Looking forward to the reply, thanks again!
**Also, I have found some small bugs:**
1. in scream/externals/ekat/extern/kokkos/core/src/../../tpls/desul/include/desul/atomics/SYCLConversions.hpp, line 23:
the name space seems like to be in `::sycl` instead of `::sycl::ext::oneapi` using intel oneapi 2024;
2. in scream/components/homme/src/share/compose/cedr_kokkos.hpp: line 21:
an unexpected `>` appeared here `typedef Kokkos::Experimental::SYCL> CedrGpuSpace;`
Contributor guide
No contributing guide indexed for this repository
Assessment
This issue has not been assessed yet.