trixi-framework / trixi-framework/Trixi.jl

Add MPI implementation for nonconservative equations on `TreeMesh` and `P4estMesh`

Open
#1,423 7 comments 0 reactions 0 assignees View on GitHub
bug parallelization
Dominant language
Julia
Stars
731
Forks
167
Avg merge
2d 18h
Merged PRs (30d)
25

Description

When I run the elixir under examples/tree_2d_dgsem/elixir_shallowwater_source_terms.jl with MPI I get a `MethodError: no method matching calc_mpi_interface_flux!`. See the top of the stacktrace below (the whole stacktrace is just too long and imho the top is most interesting).

Stacktrace (on each rank)

```julia
ERROR: LoadError: MethodError: no method matching calc_mpi_interface_flux!(::Array{Float64, 4}, ::TreeMesh{2, Trixi.ParallelTree{2}}, ::Static.True, ::ShallowWaterEquations2D{Float64}, ::SurfaceIntegralWeakForm{Tuple{FluxLaxFri[462/482]
ypeof(max_abs_speed_naive)}, typeof(flux_nonconservative_fjordholm_etal)}}, ::DGSEM{LobattoLegendreBasis{Float64, 4, SVector{4, Float64}, Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}, Trixi.LobattoLegendreMortarL2{Float64, 4, Matr
ix{Float64}, Matrix{Float64}}, SurfaceIntegralWeakForm{Tuple{FluxLaxFriedrichs{typeof(max_abs_speed_naive)}, typeof(flux_nonconservative_fjordholm_etal)}}, VolumeIntegralFluxDifferencing{Tuple{typeof(flux_wintermeyer_etal), typeof(flux_
nonconservative_wintermeyer_etal)}}}, ::NamedTuple{(:elements, :interfaces, :mpi_interfaces, :boundaries, :mortars, :mpi_mortars, :mpi_cache, :fstar_upper_threaded, :fstar_lower_threaded), Tuple{Trixi.ElementContainer2D{Float64, Float64
}, Trixi.InterfaceContainer2D{Float64}, Trixi.MPIInterfaceContainer2D{Float64}, Trixi.BoundaryContainer2D{Float64, Float64}, Trixi.L2MortarContainer2D{Float64}, Trixi.MPIL2MortarContainer2D{Float64}, Trixi.MPICache, Vector{StaticArraysC
ore.MMatrix{4, 4, Float64, 16}}, Vector{StaticArraysCore.MMatrix{4, 4, Float64, 16}}}})

Closest candidates are:
calc_mpi_interface_flux!(::Any, ::TreeMesh{2, <:Trixi.ParallelTree{2}}, ::Static.False, ::Any, ::Any, ::DG, ::Any)
@ Trixi ~/work/julia/Trixi.jl/src/solvers/dgsem_tree/dg_2d_parallel.jl:682
calc_mpi_interface_flux!(::Any, ::P4estMesh{2, <:Real, <:Static.True}, ::Any, ::Any, ::Any, ::DG, ::Any)
@ Trixi ~/work/julia/Trixi.jl/src/solvers/dgsem_p4est/dg_2d_parallel.jl:42
calc_mpi_interface_flux!(::Any, ::P4estMesh{3, <:Real, <:Static.True}, ::Any, ::Any, ::Any, ::DG, ::Any)
@ Trixi ~/work/julia/Trixi.jl/src/solvers/dgsem_p4est/dg_3d_parallel.jl:141
...

Stacktrace:
[1] macro expansion
@ ~/work/julia/Trixi.jl/src/auxiliary/auxiliary.jl:272 [inlined]
[2] rhs!(du::StrideArraysCore.PtrArray{Float64, 4, (1, 2, 3, 4), Tuple{Static.StaticInt{4}, Static.StaticInt{4}, Static.StaticInt{4}, Int64}, NTuple{4, Nothing}, NTuple{4, Static.StaticInt{1}}}, u::StrideArraysCore.PtrArray{Float64, 4
, (1, 2, 3, 4), Tuple{Static.StaticInt{4}, Static.StaticInt{4}, Static.StaticInt{4}, Int64}, NTuple{4, Nothing}, NTuple{4, Static.StaticInt{1}}}, t::Float64, mesh::TreeMesh{2, Trixi.ParallelTree{2}}, equations::ShallowWaterEquations2D{F
loat64}, initial_condition::Function, boundary_conditions::Trixi.BoundaryConditionPeriodic, source_terms::Function, dg::DGSEM{LobattoLegendreBasis{Float64, 4, SVector{4, Float64}, Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}, Trix
i.LobattoLegendreMortarL2{Float64, 4, Matrix{Float64}, Matrix{Float64}}, SurfaceIntegralWeakForm{Tuple{FluxLaxFriedrichs{typeof(max_abs_speed_naive)}, typeof(flux_nonconservative_fjordholm_etal)}}, VolumeIntegralFluxDifferencing{Tuple{t
ypeof(flux_wintermeyer_etal), typeof(flux_nonconservative_wintermeyer_etal)}}}, cache::NamedTuple{(:elements, :interfaces, :mpi_interfaces, :boundaries, :mortars, :mpi_mortars, :mpi_cache, :fstar_upper_threaded, :fstar_lower_threaded),
Tuple{Trixi.ElementContainer2D{Float64, Float64}, Trixi.InterfaceContainer2D{Float64}, Trixi.MPIInterfaceContainer2D{Float64}, Trixi.BoundaryContainer2D{Float64, Float64}, Trixi.L2MortarContainer2D{Float64}, Trixi.MPIL2MortarContainer2D
{Float64}, Trixi.MPICache, Vector{StaticArraysCore.MMatrix{4, 4, Float64, 16}}, Vector{StaticArraysCore.MMatrix{4, 4, Float64, 16}}}})
@ Trixi ~/work/julia/Trixi.jl/src/solvers/dgsem_tree/dg_2d_parallel.jl:507
[3] macro expansion
@ ~/work/julia/Trixi.jl/src/auxiliary/auxiliary.jl:272 [inlined] [434/482]
[4] rhs!(du_ode::Vector{Float64}, u_ode::Vector{Float64}, semi::SemidiscretizationHyperbolic{TreeMesh{2, Trixi.ParallelTree{2}}, ShallowWaterEquations2D{Float64}, typeof(initial_condition_convergence_test), Trixi.BoundaryConditionPeri
odic, typeof(source_terms_convergence_test), DGSEM{LobattoLegendreBasis{Float64, 4, SVector{4, Float64}, Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}, Trixi.LobattoLegendreMortarL2{Float64, 4, Matrix{Float64}, Matrix{Float64}}, Su
rfaceIntegralWeakForm{Tuple{FluxLaxFriedrichs{typeof(max_abs_speed_naive)}, typeof(flux_nonconservative_fjordholm_etal)}}, VolumeIntegralFluxDifferencing{Tuple{typeof(flux_wintermeyer_etal), typeof(flux_nonconservative_wintermeyer_etal)
}}}, NamedTuple{(:elements, :interfaces, :mpi_interfaces, :boundaries, :mortars, :mpi_mortars, :mpi_cache, :fstar_upper_threaded, :fstar_lower_threaded), Tuple{Trixi.ElementContainer2D{Float64, Float64}, Trixi.InterfaceContainer2D{Float
64}, Trixi.MPIInterfaceContainer2D{Float64}, Trixi.BoundaryContainer2D{Float64, Float64}, Trixi.L2MortarContainer2D{Float64}, Trixi.MPIL2MortarContainer2D{Float64}, Trixi.MPICache, Vector{StaticArraysCore.MMatrix{4, 4, Float64, 16}}, Ve
ctor{StaticArraysCore.MMatrix{4, 4, Float64, 16}}}}}, t::Float64)
@ Trixi ~/work/julia/Trixi.jl/src/semidiscretization/semidiscretization_hyperbolic.jl:297
[5] (::SciMLBase.Void{typeof(Trixi.rhs!)})(::Vector{Float64}, ::Vararg{Any})
@ SciMLBase ~/.julia/packages/SciMLBase/VdcHg/src/utils.jl:468
[6] (::FunctionWrappers.CallWrapper{Nothing})(f::SciMLBase.Void{typeof(Trixi.rhs!)}, arg1::Vector{Float64}, arg2::Vector{Float64}, arg3::SemidiscretizationHyperbolic{TreeMesh{2, Trixi.ParallelTree{2}}, ShallowWaterEquations2D{Float64}
, typeof(initial_condition_convergence_test), Trixi.BoundaryConditionPeriodic, typeof(source_terms_convergence_test), DGSEM{LobattoLegendreBasis{Float64, 4, SVector{4, Float64}, Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}, Trixi.
LobattoLegendreMortarL2{Float64, 4, Matrix{Float64}, Matrix{Float64}}, SurfaceIntegralWeakForm{Tuple{FluxLaxFriedrichs{typeof(max_abs_speed_naive)}, typeof(flux_nonconservative_fjordholm_etal)}}, VolumeIntegralFluxDifferencing{Tuple{typ
eof(flux_wintermeyer_etal), typeof(flux_nonconservative_wintermeyer_etal)}}}, NamedTuple{(:elements, :interfaces, :mpi_interfaces, :boundaries, :mortars, :mpi_mortars, :mpi_cache, :fstar_upper_threaded, :fstar_lower_threaded), Tuple{Tri
xi.ElementContainer2D{Float64, Float64}, Trixi.InterfaceContainer2D{Float64}, Trixi.MPIInterfaceContainer2D{Float64}, Trixi.BoundaryContainer2D{Float64, Float64}, Trixi.L2MortarContainer2D{Float64}, Trixi.MPIL2MortarContainer2D{Float64}
, Trixi.MPICache, Vector{StaticArraysCore.MMatrix{4, 4, Float64, 16}}, Vector{StaticArraysCore.MMatrix{4, 4, Float64, 16}}}}}, arg4::Float64)
@ FunctionWrappers ~/.julia/packages/FunctionWrappers/Q5cBx/src/FunctionWrappers.jl:65
[7] macro expansion
@ ~/.julia/packages/FunctionWrappers/Q5cBx/src/FunctionWrappers.jl:137 [inlined]
[8] do_ccall
@ ~/.julia/packages/FunctionWrappers/Q5cBx/src/FunctionWrappers.jl:125 [inlined]
[9] FunctionWrapper
@ ~/.julia/packages/FunctionWrappers/Q5cBx/src/FunctionWrappers.jl:144 [inlined]
[10] _call
@ ~/.julia/packages/FunctionWrappersWrappers/9XR0m/src/FunctionWrappersWrappers.jl:12 [inlined]
[11] FunctionWrappersWrapper
@ ~/.julia/packages/FunctionWrappersWrappers/9XR0m/src/FunctionWrappersWrappers.jl:10 [inlined]
[12] ODEFunction
@ ~/.julia/packages/SciMLBase/VdcHg/src/scimlfunctions.jl:2126 [inlined] [406/482]
[13] macro expansion
@ ~/.julia/packages/TimerOutputs/RsWnF/src/TimerOutput.jl:391 [inlined]
[14] macro expansion
@ ~/work/julia/Trixi.jl/src/callbacks_step/analysis.jl:292 [inlined]
[15] macro expansion
@ ~/work/julia/Trixi.jl/src/auxiliary/auxiliary.jl:272 [inlined]
[16] (::AnalysisCallback{Trixi.LobattoLegendreAnalyzer{Float64, 7, SVector{7, Float64}, Matrix{Float64}}, Matrix{Float64}}, Tuple{typeof(Trixi.entropy_timederivative)}, SVector{4, Float64}, NamedTuple{(:u_local, :u_tmp1, :x_local, :x_tmp1), Tuple{Stride
ArraysCore.StaticStrideArray{Float64, 3, (1, 2, 3), Tuple{Static.StaticInt{4}, Static.StaticInt{7}, Static.StaticInt{7}}, Tuple{Nothing, Nothing, Nothing}, Tuple{Static.StaticInt{1}, Static.StaticInt{1}, Static.StaticInt{1}}, 196}, Stri
deArraysCore.StaticStrideArray{Float64, 3, (1, 2, 3), Tuple{Static.StaticInt{4}, Static.StaticInt{7}, Static.StaticInt{4}}, Tuple{Nothing, Nothing, Nothing}, Tuple{Static.StaticInt{1}, Static.StaticInt{1}, Static.StaticInt{1}}, 112}, St
rideArraysCore.StaticStrideArray{Float64, 3, (1, 2, 3), Tuple{Static.StaticInt{2}, Static.StaticInt{7}, Static.StaticInt{7}}, Tuple{Nothing, Nothing, Nothing}, Tuple{Static.StaticInt{1}, Static.StaticInt{1}, Static.StaticInt{1}}, 98}, S
trideArraysCore.StaticStrideArray{Float64, 3, (1, 2, 3), Tuple{Static.StaticInt{2}, Static.StaticInt{7}, Static.StaticInt{4}}, Tuple{Nothing, Nothing, Nothing}, Tuple{Static.StaticInt{1}, Static.StaticInt{1}, Static.StaticInt{1}}, 56}}}
})
[...]
```

Contributor guide

Open the contributing guide

Research direction

Start with calc_mpi_interface_flux! in src/solvers/dgsem_tree/dg_2d_parallel.jl and src/solvers/dgsem_p4est/dg_2d_parallel.jl, comparing the existing conservative and nonconservative dispatches. Re-run examples/tree_2d_dgsem/elixir_shallowwater_source_terms.jl with MPI; done means the nonconservative TreeMesh and P4estMesh cases no longer raise the reported MethodError.

Written by the indexing model from the issue text.

Assessment

Tech stack
julia
Domain
hpc
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
45/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.