Loving the new @launch
macro! But I noticed my kernels for Oceananigans.jl are taking much longer to compile once I upgraded to GPUifyLoops v0.2.0 and one kernel in particular would never finish compiling as it somehow got stuck in an infinite recursive call ending with a StackOverflowError
.
I was able to track this down to the following line
|
kernel = cufunction(contextualize(f), kernel_tt; compiler_kwargs...) |
which I changed to
kernel = cufunction(f, kernel_tt; compiler_kwargs...)
which got me up and running again as none of my kernels require contextualize
and brought compilation time back down to 3.5 minutes from 8.5 minutes with contextualize
. No idea why the compiler got stuck when contextualize
was turned on as I didn't have any function calls that could be converted.
I was wondering if it would be a good idea to make contextualize
an option when launching a kernel? From my understanding, it lets you write exp(x)
in your kernel which gets converted to CUDAnative.exp(x)
when called via @launch CUDA()
. I can see contextualize
being useful but might be nice to turn it on for certain kernels that need it to speed up compilation.
I'm happy work on this. In case contextualize
should always be on I can just keep this on my fork.
I was able to reproduce this with Julia 1.1 and the latest nightly [Version 1.2.0-DEV.562 (2019-03-28) Commit 11156024da].
Unfortunately I'm not sure what the issue was so I can't provide a minimal working example.
Verbose details included below.
Problematic kernel (I'm working on making it less ugly!)
function calculate_interior_source_terms!(grid::Grid, constants, eos, cfg, u, v, w, T, S, pHY′, Gu, Gv, Gw, GT, GS, F)
Nx, Ny, Nz = grid.Nx, grid.Ny, grid.Nz
Δx, Δy, Δz = grid.Δx, grid.Δy, grid.Δz
fCor = constants.f
ρ₀ = eos.ρ₀
𝜈h, 𝜈v, κh, κv = cfg.𝜈h, cfg.𝜈v, cfg.κh, cfg.κv
@loop for k in (1:grid.Nz; blockIdx().z)
@loop for j in (1:grid.Ny; (blockIdx().y - 1) * blockDim().y + threadIdx().y)
@loop for i in (1:grid.Nx; (blockIdx().x - 1) * blockDim().x + threadIdx().x)
# u-momentum equation
@inbounds Gu[i, j, k] = (-u∇u(u, v, w, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ fCor*avg_xy(v, Nx, Ny, i, j, k)
- δx_c2f(pHY′, Nx, i, j, k) / (Δx * ρ₀)
+ 𝜈∇²u(u, 𝜈h, 𝜈v, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ F.u(u, v, w, T, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k))
# v-momentum equation
@inbounds Gv[i, j, k] = (-u∇v(u, v, w, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
- fCor*avg_xy(u, Nx, Ny, i, j, k)
- δy_c2f(pHY′, Ny, i, j, k) / (Δy * ρ₀)
+ 𝜈∇²v(v, 𝜈h, 𝜈v, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ F.v(u, v, w, T, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k))
# w-momentum equation: comment about how pressure and buoyancy are handled
@inbounds Gw[i, j, k] = (-u∇w(u, v, w, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ 𝜈∇²w(w, 𝜈h, 𝜈v, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ F.w(u, v, w, T, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k))
# temperature equation
@inbounds GT[i, j, k] = (-div_flux(u, v, w, T, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ κ∇²(T, κh, κv, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ F.T(u, v, w, T, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k))
# salinity equation
@inbounds GS[i, j, k] = (-div_flux(u, v, w, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ κ∇²(S, κh, κv, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ F.S(u, v, w, T, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k))
end
end
end
@synchronize
end
Launching the kernel
@launch CUDA() calculate_interior_source_terms!(grid, constants, eos, cfg, uvw..., TS..., pr.pHY′.data, Gⁿ..., forcing, threads=(Tx, Ty), blocks=(Bx, By, Bz))
Error
ERROR: StackOverflowError:
Stacktrace:
[1] snca_compress!(::Array{Core.Compiler.Node,1}, ::Array{UInt64,1}, ::UInt64, ::UInt64) at ./compiler/ssair/domtree.jl:204
[2] snca_compress!(::Array{Core.Compiler.Node,1}, ::Array{UInt64,1}, ::UInt64, ::UInt64) at ./compiler/ssair/domtree.jl:207 (repeats 25909 times)
[3] SNCA(::Core.Compiler.CFG) at ./compiler/ssair/domtree.jl:258
[4] construct_domtree(::Core.Compiler.CFG) at ./compiler/ssair/domtree.jl:104
[5] run_passes(::Core.CodeInfo, ::Int64, ::Core.Compiler.OptimizationState) at ./compiler/ssair/driver.jl:120
[6] optimize(::Core.Compiler.OptimizationState, ::Any) at ./compiler/optimize.jl:167
[7] typeinf(::Core.Compiler.InferenceState) at ./compiler/typeinfer.jl:33
[8] typeinf_edge(::Method, ::Any, ::Core.SimpleVector, ::Core.Compiler.InferenceState) at ./compiler/typeinfer.jl:503
[9] abstract_call_method(::Method, ::Any, ::Core.SimpleVector, ::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:363
[10] abstract_call_gf_by_type(::Any, ::Array{Any,1}, ::Any, ::Core.Compiler.InferenceState, ::Int64) at ./compiler/abstractinterpretation.jl:92
[11] abstract_call(::Any, ::Nothing, ::Array{Any,1}, ::Array{Any,1}, ::Core.Compiler.InferenceState, ::Int64) at ./compiler/abstractinterpretation.jl:802
[12] abstract_apply(::Any, ::Array{Any,1}, ::Array{Any,1}, ::Core.Compiler.InferenceState, ::Int64) at ./compiler/abstractinterpretation.jl:545
[13] abstract_call(::Any, ::Array{Any,1}, ::Array{Any,1}, ::Array{Any,1}, ::Core.Compiler.InferenceState, ::Int64) at ./compiler/abstractinterpretation.jl:593
[14] abstract_call(::Any, ::Array{Any,1}, ::Array{Any,1}, ::Array{Any,1}, ::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:592
1 GPUifyLoops.jl + X
[15] abstract_eval_call(::Array{Any,1}, ::Array{Any,1}, ::Array{Any,1}, ::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:831
[16] abstract_eval(::Any, ::Array{Any,1}, ::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:901
[17] typeinf_local(::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:1159
[18] typeinf_nocycle(::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:1215
[19] typeinf(::Core.Compiler.InferenceState) at ./compiler/typeinfer.jl:12
[20] typeinf at ./compiler/typeinfer.jl:8 [inlined]
[21] typeinf_type(::Method, ::Any, ::Core.SimpleVector, ::Core.Compiler.Params) at ./compiler/typeinfer.jl:614
[22] return_types(::Any, ::Any) at ./reflection.jl:1068
[23] check_method(::CUDAnative.CompilerContext) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/validation.jl:12
[24] compile(::CUDAnative.CompilerContext) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:67
[25] #compile#109(::Bool, ::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(CUDAnative.compile), ::VersionNumber, ::Any, ::Any) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:45
[26] compile at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:43 [inlined]
[27] #compile#108(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(CUDAnative.compile), ::CUDAdrv.CuDevice, ::Function, ::Any) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:18
[28] compile at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:16 [inlined]
[29] macro expansion at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/execution.jl:269 [inlined]
[30] #cufunction#123(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(CUDAnative.cufunction), ::getfield(GPUifyLoops, Symbol("##16#17")){typeof(Oceananigans.calculate_interior_source_terms!)}, ::Type{Tuple{RegularCartesianGrid{Float64,StepRangeLen{Float64,Base.TwicePrecision{Float64},Base.TwicePrecision{Float64}}},PlanetaryConstants{Float64},LinearEquationOfState{Float64},ModelConfiguration{Float64},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},Forcing{typeof(Oceananigans.zero_func),typeof(Oceananigans.zero_func),typeof(Oceananigans.zero_func),typeof(Oceananigans.zero_func),typeof(Oceananigans.zero_func)}}}) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/execution.jl:240
[31] cufunction(::Function, ::Type) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/execution.jl:240
[32] macro expansion at /home/alir_mit_edu/.julia/packages/GPUifyLoops/xOg1y/src/GPUifyLoops.jl:109 [inlined]
[33] macro expansion at ./gcutils.jl:87 [inlined]
[34] #launch#34(::Base.Iterators.Pairs{Symbol,Tuple{Int64,Int64,Vararg{Int64,N} where N},Tuple{Symbol,Symbol},NamedTuple{(:threads, :blocks),Tuple{Tuple{Int64,Int64},Tuple{Int64,Int64,Int64}}}}, ::typeof(GPUifyLoops.launch), ::GPUifyLoops.CUDA, ::typeof(Oceananigans.calculate_interior_source_terms!), ::RegularCartesianGrid{Float64,StepRangeLen{Float64,Base.TwicePrecision{Float64},Base.TwicePrecision{Float64}}}, ::Vararg{Any,N} where N) at /home/alir_mit_edu/.julia/packages/GPUifyLoops/xOg1y/src/GPUifyLoops.jl:106
[35] (::getfield(GPUifyLoops, Symbol("#kw##launch")))(::NamedTuple{(:threads, :blocks),Tuple{Tuple{Int64,Int64},Tuple{Int64,Int64,Int64}}}, ::typeof(GPUifyLoops.launch), ::GPUifyLoops.CUDA, ::typeof(Oceananigans.calculate_interior_source_terms!), ::RegularCartesianGrid{Float64,StepRangeLen{Float64,Base.TwicePrecision{Float64},Base.TwicePrecision{Float64}}}, ::Vararg{Any,N} where N) at ./none:0
[36] macro expansion at /home/alir_mit_edu/.julia/packages/GPUifyLoops/xOg1y/src/GPUifyLoops.jl:50 [inlined]
[37] time_step!(::Model{GPU}, ::Int64, ::Int64) at /home/alir_mit_edu/gpu_test/Oceananigans.jl/src/time_steppers.jl:79
[38] top-level scope at util.jl:156