BinnedPool: key error

[maleadt]

From Slack:

KeyError: key CUDAdrv.CuPtr{Nothing}(0x00007f4e6ebe8000) not found
getindex at dict.jl:477 [inlined]
actual_free(::CUDAdrv.CuPtr{Nothing}) at memory.jl:73
actual_free at binned.jl:58 [inlined]
macro expansion at binned.jl:173 [inlined]
macro expansion at TimerOutput.jl:228 [inlined]
reclaim(::Bool, ::Int64) at binned.jl:163
reclaim at binned.jl:137 [inlined]
macro expansion at train.jl:38 [inlined]
macro expansion at progress.jl:119 [inlined]
macro expansion at train.jl:29 [inlined]
macro expansion at progress.jl:119 [inlined]

[baggepinnen]

The error appears randomly throughout training sometimes once every 2 hours, sometimes with 5 minutes interval. I also get errors like
error in running finalizer: ErrorException("task switch not allowed from inside gc finalizer") every noew and then, but they don't stop julia and training just continues through them. The error above, however, is a julia exception.

julia> versioninfo()
Julia Version 1.3.0
Commit 46ce4d7933 (2019-11-26 06:09 UTC)
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-6.0.1 (ORCJIT, skylake)
Environment:
  JULIA_NUM_THREADS = 6
  JULIA_EDITOR = atom  -a

(v1.3) pkg> st CuArrays
    Status `~/.julia/environments/v1.3/Project.toml`
  [3a865a2d] CuArrays v1.4.7 [`~/.julia/dev/CuArrays`]
  [864edb3b] DataStructures v0.17.6
  [1914dd2f] MacroTools v0.5.2
  [872c559c] NNlib v0.6.0
  [a759f4b9] TimerOutputs v0.5.3
  [9a3f8284] Random 
  [2f01184e] SparseArrays 

fredrikb@billman:~$ cd .julia/dev/CuArrays/
fredrikb@billman:~/.julia/dev/CuArrays$ git branch
  master
* tb/timeit_debug

I'm training using Flux and Tracker

[baggepinnen]

I now got a similar but different error

AssertionError: !(haskey(requested, ptr))
alloc at memory.jl:132 [inlined]
CuArray{Float32,4,P} where P(::UndefInitializer, ::NTuple{4,Int64}) at array.jl:90
similar at array.jl:122 [inlined]
similar at abstractarray.jl:629 [inlined]
macro expansion at conv.jl:112 [inlined]
#conv#97(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(NNlib.conv), ::CuArray{Float32,4,Nothing}, ::CuArray{Float32,4,Nothing}, ::DenseConvDims{2,(11, 1),4,4,(1, 1),(0, 0, 0, 0),(1, 1),false}) at TimerOutput.jl:190
#_forward#524 at TimerOutput.jl:197 [inlined]
_forward(::typeof(NNlib.conv), ::TrackedArray{…,CuArray{Float32,4,Nothing}}, ::TrackedArray{…,CuArray{Float32,4,Nothing}}, ::DenseConvDims{2,(11, 1),4,4,(1, 1),(0, 0, 0, 0),(1, 1),false}) at none:0
#track#1(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(Tracker.track), ::typeof(NNlib.conv), ::TrackedArray{…,CuArray{Float32,4,Nothing}}, ::Vararg{Any,N} where N) at Tracker.jl:52
track at Tracker.jl:52 [inlined]
#conv#521 at array.jl:443 [inlined]
conv at array.jl:443 [inlined]
(::Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}})(::TrackedArray{…,CuArray{Float32,4,Nothing}}) at conv.jl:55
applychain(::Tuple{Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(identity),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}}}, ::TrackedArray{…,CuArray{Float32,4,Nothing}}) at basic.jl:31
applychain(::Tuple{Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(identity),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}}}, ::CuArray{Float32,4,CuArray{Float32,4,Nothing}}) at basic.jl:31
(::Chain{Tuple{Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(identity),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}}}})(::CuArray{Float32,4,CuArray{Float32,4,Nothing}}) at basic.jl:33
encode(::Detector.MixtureAutoencoder{Chain{Tuple{Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Conv{2,2,typeof(identity),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}}}},Chain{Tuple{ConvTranspose{2,4,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},ConvTranspose{2,4,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},ConvTranspose{2,4,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},ConvTranspose{2,4,typeof(leakyrelu),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},ConvTranspose{2,4,typeof(identity),TrackedArray{…,CuArray{Float32,4,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}}}},Chain{Tuple{typeof(Detector.preM),Dense{typeof(relu),TrackedArray{…,CuArray{Float32,2,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Dense{typeof(tanh),TrackedArray{…,CuArray{Float32,2,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},Dense{typeof(identity),TrackedArray{…,CuArray{Float32,2,Nothing}},TrackedArray{…,CuArray{Float32,1,Nothing}}},typeof(softmax)}},TrackedArray{…,CuArray{Float32,1,Nothing}},Tuple{Float32,Float32}}, ::CuArray{Float32,4,Nothing}, ::Bool) at define_model.jl:44
encode(::Detector.Mi...