E3SM-Project / E3SM-Project/scream

Some bugs and questions about running SCREAM on intel PVC GPU

Open
#2,736 23 comments 0 reactions 0 assignees View on GitHub
Dominant language
No language data
Stars
79
Forks
54
PR merge metrics
No merged PRs in 30d

Description

I'm trying to run SCREAM on intel PVC GPU, however meet some errors, looking for help, thanks!
Here's my testing environment:
SCREAM code: lasted code of master version (https://github.com/E3SM-Project/scream), not sure whether this is the correct version for test on intel GPU?
Machine: intel CPU+ intel PVC GPU (1100)
Compiler and MPI: intel oneapi 2024 (icx, ifx, icpx; mpiicx, mpiifx, mpiicpx)

Here's my config files:
1. scream/cime_config/machines/config_machines.xml
```xml

HPC, 6430 CPU + 1100 PVC(56core, 48GB)

LINUX
oneapi-ifx,oneapi-ifxgpu,gnu
impi,openmpi, mpich

/home/lujingyu/E3SM/SCREAM/cases/scratch/$CASE
/home/lujingyu/E3SM/SCREAM/inputdata
/home/lujingyu/E3SM/SCREAM/inputdata/atm/datm7
/home/lujingyu/E3SM/SCREAM/cases/archive/$CASE


16
e3sm_developer
4
none
e3sm
56
208
56
56
64
56
FALSE

mpirun

-np {{ total_tasks }}



$CIME_OUTPUT_ROOT/$CASE/run
$CIME_OUTPUT_ROOT/$CASE/bld

/home/lujingyu/nc_pnc2023_intel2024


/home/lujingyu/nc_pnc2023_intel2024/lib:$ENV{LD_LIBRARY_PATH}
/home/lujingyu/nc_pnc2023_intel2024/bin:$ENV{PATH}


10
1





"opencl:gpu;level_zero:gpu"




1
4000MB
0

131072
20


0
DISABLED
131072
20
0



-1


```

2. scream/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake
```cmake
if (compile_threaded)
string(APPEND CMAKE_C_FLAGS " -qopenmp")
string(APPEND CMAKE_Fortran_FLAGS " -qopenmp")
string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp")
endif()
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_C_FLAGS " -traceback -fp-model precise -std=gnu99")
string(APPEND CMAKE_CXX_FLAGS " -traceback -fp-model precise")
string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise ")
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")
set(HAS_F2008_CONTIGUOUS "TRUE")
set(MPIFC "mpiifx")
set(MPICC "mpiicx")
set(MPICXX "mpiicpx")
set(SCC "icx")
set(SCXX "icpx")
set(SFC "ifx")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64 ")
set(USE_SYCL "TRUE")
set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "")
string(APPEND SYCL_FLAGS " -fsycl -fsycl-targets=spir64 ") #-linux-intel_gpu_pvc -Xsycl-target-backend Xe-MAX -sycl-std=121
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DCMAKE_CXX_STANDARD=17")
```

3. scream/components/eamxx/cmake/machine-files/PVC1100.cmake
```cmake
include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
common_setup()
# Load all kokkos settings from Ekat's mach file
include (${EKAT_MACH_FILES_PATH}/kokkos/intel-pvc.cmake)
```
4. scream/externals/ekat/cmake/machine-files/PVC1100.cmake
```cmake
# Load PVC arch with SYCL backend for kokkos
include (${CMAKE_CURRENT_LIST_DIR}/kokkos/intel-pvc.cmake)
```
Here's my case: (**is this F2000-SCREAM-SA @ ne30pg2_ne30pg2 the best test case?**)
```shell
./create_newcase --case test1 --compset F2000-SCREAM-SA --res ne30pg2_ne30pg2 --mach PVC1100 --compiler oneapi-ifxgpu --mpilib impi
```

And here are all my log files, [bld.zip](https://github.com/E3SM-Project/scream/files/14428057/bld.zip)
and the error is:
```shell
/opt/intel/oneapi/compiler/2024.0/bin/compiler/../../include/sycl/types.hpp:2382:17: error: ambiguous partial specializations of 'is_device_copyable, Kokkos::MemoryTraits<0>>, Kokkos::View, Kokkos::MemoryTraits<0>>, Kokkos::LayoutLeft, Kokkos::Experimental::SYCL, 5, long>, Kokkos::Experimental::Impl::SYCLInternal::USMObjectMem>>'
2382 | static_assert(is_device_copyable::value ||
```

I'm fresh on kokkos and SYCL, so I'm confused to the link between kokkos and SYCL, and the backend target with intel PVC GPU, **_is there anything wrong in my config files and lead to this error?_**
Looking forward to the reply, thanks again!

**Also, I have found some small bugs:**
1. in scream/externals/ekat/extern/kokkos/core/src/../../tpls/desul/include/desul/atomics/SYCLConversions.hpp, line 23:
the name space seems like to be in `::sycl` instead of `::sycl::ext::oneapi` using intel oneapi 2024;
2. in scream/components/homme/src/share/compose/cedr_kokkos.hpp: line 21:
an unexpected `>` appeared here `typedef Kokkos::Experimental::SYCL> CedrGpuSpace;`

Contributor guide

No contributing guide indexed for this repository

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.