Loving the new @launch macro! But I noticed my kernels for Oceananigans.jl are taking much longer to compile once I upgraded to GPUifyLoops v0.2.0 and one kernel in particular would never finish compiling as it somehow got stuck in an infinite recursive call ending with a StackOverflowError.
I was able to track this down to the following line
|
kernel = cufunction(contextualize(f), kernel_tt; compiler_kwargs...) |
which I changed to
kernel = cufunction(f, kernel_tt; compiler_kwargs...)
which got me up and running again as none of my kernels require contextualize and brought compilation time back down to 3.5 minutes from 8.5 minutes with contextualize. No idea why the compiler got stuck when contextualize was turned on as I didn't have any function calls that could be converted.
I was wondering if it would be a good idea to make contextualize an option when launching a kernel? From my understanding, it lets you write exp(x) in your kernel which gets converted to CUDAnative.exp(x) when called via @launch CUDA(). I can see contextualize being useful but might be nice to turn it on for certain kernels that need it to speed up compilation.
I'm happy work on this. In case contextualize should always be on I can just keep this on my fork.
I was able to reproduce this with Julia 1.1 and the latest nightly [Version 1.2.0-DEV.562 (2019-03-28) Commit 11156024da].
Unfortunately I'm not sure what the issue was so I can't provide a minimal working example.
Verbose details included below.
Problematic kernel (I'm working on making it less ugly!)
function calculate_interior_source_terms!(grid::Grid, constants, eos, cfg, u, v, w, T, S, pHY′, Gu, Gv, Gw, GT, GS, F)
Nx, Ny, Nz = grid.Nx, grid.Ny, grid.Nz
Δx, Δy, Δz = grid.Δx, grid.Δy, grid.Δz
fCor = constants.f
ρ₀ = eos.ρ₀
𝜈h, 𝜈v, κh, κv = cfg.𝜈h, cfg.𝜈v, cfg.κh, cfg.κv
@loop for k in (1:grid.Nz; blockIdx().z)
@loop for j in (1:grid.Ny; (blockIdx().y - 1) * blockDim().y + threadIdx().y)
@loop for i in (1:grid.Nx; (blockIdx().x - 1) * blockDim().x + threadIdx().x)
# u-momentum equation
@inbounds Gu[i, j, k] = (-u∇u(u, v, w, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ fCor*avg_xy(v, Nx, Ny, i, j, k)
- δx_c2f(pHY′, Nx, i, j, k) / (Δx * ρ₀)
+ 𝜈∇²u(u, 𝜈h, 𝜈v, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ F.u(u, v, w, T, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k))
# v-momentum equation
@inbounds Gv[i, j, k] = (-u∇v(u, v, w, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
- fCor*avg_xy(u, Nx, Ny, i, j, k)
- δy_c2f(pHY′, Ny, i, j, k) / (Δy * ρ₀)
+ 𝜈∇²v(v, 𝜈h, 𝜈v, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ F.v(u, v, w, T, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k))
# w-momentum equation: comment about how pressure and buoyancy are handled
@inbounds Gw[i, j, k] = (-u∇w(u, v, w, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ 𝜈∇²w(w, 𝜈h, 𝜈v, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ F.w(u, v, w, T, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k))
# temperature equation
@inbounds GT[i, j, k] = (-div_flux(u, v, w, T, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ κ∇²(T, κh, κv, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ F.T(u, v, w, T, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k))
# salinity equation
@inbounds GS[i, j, k] = (-div_flux(u, v, w, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ κ∇²(S, κh, κv, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k)
+ F.S(u, v, w, T, S, Nx, Ny, Nz, Δx, Δy, Δz, i, j, k))
end
end
end
@synchronize
endLaunching the kernel
@launch CUDA() calculate_interior_source_terms!(grid, constants, eos, cfg, uvw..., TS..., pr.pHY′.data, Gⁿ..., forcing, threads=(Tx, Ty), blocks=(Bx, By, Bz))
Error
ERROR: StackOverflowError:
Stacktrace:
[1] snca_compress!(::Array{Core.Compiler.Node,1}, ::Array{UInt64,1}, ::UInt64, ::UInt64) at ./compiler/ssair/domtree.jl:204
[2] snca_compress!(::Array{Core.Compiler.Node,1}, ::Array{UInt64,1}, ::UInt64, ::UInt64) at ./compiler/ssair/domtree.jl:207 (repeats 25909 times)
[3] SNCA(::Core.Compiler.CFG) at ./compiler/ssair/domtree.jl:258
[4] construct_domtree(::Core.Compiler.CFG) at ./compiler/ssair/domtree.jl:104
[5] run_passes(::Core.CodeInfo, ::Int64, ::Core.Compiler.OptimizationState) at ./compiler/ssair/driver.jl:120
[6] optimize(::Core.Compiler.OptimizationState, ::Any) at ./compiler/optimize.jl:167
[7] typeinf(::Core.Compiler.InferenceState) at ./compiler/typeinfer.jl:33
[8] typeinf_edge(::Method, ::Any, ::Core.SimpleVector, ::Core.Compiler.InferenceState) at ./compiler/typeinfer.jl:503
[9] abstract_call_method(::Method, ::Any, ::Core.SimpleVector, ::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:363
[10] abstract_call_gf_by_type(::Any, ::Array{Any,1}, ::Any, ::Core.Compiler.InferenceState, ::Int64) at ./compiler/abstractinterpretation.jl:92
[11] abstract_call(::Any, ::Nothing, ::Array{Any,1}, ::Array{Any,1}, ::Core.Compiler.InferenceState, ::Int64) at ./compiler/abstractinterpretation.jl:802
[12] abstract_apply(::Any, ::Array{Any,1}, ::Array{Any,1}, ::Core.Compiler.InferenceState, ::Int64) at ./compiler/abstractinterpretation.jl:545
[13] abstract_call(::Any, ::Array{Any,1}, ::Array{Any,1}, ::Array{Any,1}, ::Core.Compiler.InferenceState, ::Int64) at ./compiler/abstractinterpretation.jl:593
[14] abstract_call(::Any, ::Array{Any,1}, ::Array{Any,1}, ::Array{Any,1}, ::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:592
1 GPUifyLoops.jl + X
[15] abstract_eval_call(::Array{Any,1}, ::Array{Any,1}, ::Array{Any,1}, ::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:831
[16] abstract_eval(::Any, ::Array{Any,1}, ::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:901
[17] typeinf_local(::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:1159
[18] typeinf_nocycle(::Core.Compiler.InferenceState) at ./compiler/abstractinterpretation.jl:1215
[19] typeinf(::Core.Compiler.InferenceState) at ./compiler/typeinfer.jl:12
[20] typeinf at ./compiler/typeinfer.jl:8 [inlined]
[21] typeinf_type(::Method, ::Any, ::Core.SimpleVector, ::Core.Compiler.Params) at ./compiler/typeinfer.jl:614
[22] return_types(::Any, ::Any) at ./reflection.jl:1068
[23] check_method(::CUDAnative.CompilerContext) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/validation.jl:12
[24] compile(::CUDAnative.CompilerContext) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:67
[25] #compile#109(::Bool, ::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(CUDAnative.compile), ::VersionNumber, ::Any, ::Any) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:45
[26] compile at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:43 [inlined]
[27] #compile#108(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(CUDAnative.compile), ::CUDAdrv.CuDevice, ::Function, ::Any) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:18
[28] compile at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:16 [inlined]
[29] macro expansion at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/execution.jl:269 [inlined]
[30] #cufunction#123(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(CUDAnative.cufunction), ::getfield(GPUifyLoops, Symbol("##16#17")){typeof(Oceananigans.calculate_interior_source_terms!)}, ::Type{Tuple{RegularCartesianGrid{Float64,StepRangeLen{Float64,Base.TwicePrecision{Float64},Base.TwicePrecision{Float64}}},PlanetaryConstants{Float64},LinearEquationOfState{Float64},ModelConfiguration{Float64},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},CUDAnative.CuDeviceArray{Float64,3,CUDAnative.AS.Global},Forcing{typeof(Oceananigans.zero_func),typeof(Oceananigans.zero_func),typeof(Oceananigans.zero_func),typeof(Oceananigans.zero_func),typeof(Oceananigans.zero_func)}}}) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/execution.jl:240
[31] cufunction(::Function, ::Type) at /home/alir_mit_edu/.julia/packages/CUDAnative/PFgO3/src/execution.jl:240
[32] macro expansion at /home/alir_mit_edu/.julia/packages/GPUifyLoops/xOg1y/src/GPUifyLoops.jl:109 [inlined]
[33] macro expansion at ./gcutils.jl:87 [inlined]
[34] #launch#34(::Base.Iterators.Pairs{Symbol,Tuple{Int64,Int64,Vararg{Int64,N} where N},Tuple{Symbol,Symbol},NamedTuple{(:threads, :blocks),Tuple{Tuple{Int64,Int64},Tuple{Int64,Int64,Int64}}}}, ::typeof(GPUifyLoops.launch), ::GPUifyLoops.CUDA, ::typeof(Oceananigans.calculate_interior_source_terms!), ::RegularCartesianGrid{Float64,StepRangeLen{Float64,Base.TwicePrecision{Float64},Base.TwicePrecision{Float64}}}, ::Vararg{Any,N} where N) at /home/alir_mit_edu/.julia/packages/GPUifyLoops/xOg1y/src/GPUifyLoops.jl:106
[35] (::getfield(GPUifyLoops, Symbol("#kw##launch")))(::NamedTuple{(:threads, :blocks),Tuple{Tuple{Int64,Int64},Tuple{Int64,Int64,Int64}}}, ::typeof(GPUifyLoops.launch), ::GPUifyLoops.CUDA, ::typeof(Oceananigans.calculate_interior_source_terms!), ::RegularCartesianGrid{Float64,StepRangeLen{Float64,Base.TwicePrecision{Float64},Base.TwicePrecision{Float64}}}, ::Vararg{Any,N} where N) at ./none:0
[36] macro expansion at /home/alir_mit_edu/.julia/packages/GPUifyLoops/xOg1y/src/GPUifyLoops.jl:50 [inlined]
[37] time_step!(::Model{GPU}, ::Int64, ::Int64) at /home/alir_mit_edu/gpu_test/Oceananigans.jl/src/time_steppers.jl:79
[38] top-level scope at util.jl:156
![bors[bot] avatar](https://avatars.githubusercontent.com/in/1847?v=4 "bors[bot]")
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent