Hi guys
I have no experience using RETURNN. While trying to run the go.sh in IAM folder i get some errors which i am not sure why they occur. I am using Theano 0.9 and cudnn 5. It would be great if someone could leave the exact versions of the dependencies they have used and also worked fine. Anyways here i leave the error, I will appreciate it if you can guide me.
arman@arman-N551JW:~/returnn/demos/mdlstm/IAM$ ./go.sh
('converting IAM_lines to', 'features/raw/demo.h5')
features/raw/demo.h5
(0, '/', 3)
RETURNN starting up, version 20180405.225130--git-538ed96-dirty, date/time 2018-04-09-11-35-51 (UTC+0200), pid 11244, cwd /home/arman/returnn/demos/mdlstm/IAM, Python /usr/bin/python
RETURNN command line options: ['config_demo']
faulthandler import error. No module named faulthandler
Theano: 0.9.0 ( in /usr/local/lib/python2.7/dist-packages/theano)
pynvml not available, memory information missing
WARNING (theano.sandbox.cuda): The cuda backend is deprecated and will be removed in the next release (v0.10). Please switch to the gpuarray backend. You can get more information about how to switch at this URL:
https://github.com/Theano/Theano/wiki/Converting-to-the-new-gpu-back-end%28gpuarray%29
/usr/local/lib/python2.7/dist-packages/theano/sandbox/cuda/init.py:556: UserWarning: Theano flag device=gpu* (old gpu back-end) only support floatX=float32. You have floatX=float64. Use the new gpu back-end with device=cuda* for that value of floatX.
warnings.warn(msg)
Using gpu device 0: GeForce GTX 960M (CNMeM is disabled, cuDNN Mixed dnn version. The header is from one version, but we link with a different version (5110, 6021))
Device gpuX proc starting up, pid 11270
Device gpuX proc: THEANO_FLAGS = 'compiledir_format=compiledir_%(platform)s-%(processor)s-%(python_version)s-%(python_bitwidth)s--dev-gpuZ,device=gpu,force_device=True'
faulthandler import error. No module named faulthandler
Device gpuX proc exception: CudaNdarrayType only supports dtype float32 for now. Tried using dtype float64 for variable None
Unhandled exception <type 'exceptions.TypeError'> in thread <_MainThread(MainThread, started 140321675474688)>, proc 11270.
Thread current, main, <_MainThread(MainThread, started 140321675474688)>:
(Excluded thread.)
That were all threads.
EXCEPTION
Traceback (most recent call last):
File "/home/arman/returnn/Device.py", line 1027, in process
line: self.process_inner(device, config, self.update_specs, asyncTask)
locals:
self = <Device.Device object at 0x7f9f002200d0>
self.process_inner = <bound method Device.process_inner of <Device.Device object at 0x7f9f002200d0>>
device = 'gpuX'
config = <Config.Config instance at 0x7f9f00228128>
self.update_specs = {'layers': [], 'block_size': 0, 'update_params': {}, 'update_rule': 'global'}
asyncTask = <TaskSystem.AsyncTask instance at 0x7f9f2b563680>
File "/home/arman/returnn/Device.py", line 1080, in process_inner
line: self.initialize(config, update_specs=update_specs)
locals:
self = <Device.Device object at 0x7f9f002200d0>
self.initialize = <bound method Device.initialize of <Device.Device object at 0x7f9f002200d0>>
config = <Config.Config instance at 0x7f9f00228128>
update_specs = {'layers': [], 'block_size': 0, 'update_params': {}, 'update_rule': 'global'}
File "/home/arman/returnn/Device.py", line 457, in initialize
line: self.trainnet = LayerNetwork.from_config_topology(config, train_flag=True, eval_flag=False)
locals:
self = <Device.Device object at 0x7f9f002200d0>
self.trainnet = !AttributeError: 'Device' object has no attribute 'trainnet'
LayerNetwork = <class 'Network.LayerNetwork'>
LayerNetwork.from_config_topology = <bound method type.from_config_topology of <class 'Network.LayerNetwork'>>
config = <Config.Config instance at 0x7f9f00228128>
train_flag =
eval_flag = False
File "/home/arman/returnn/Network.py", line 119, in from_config_topology
line: return cls.from_json_and_config(json_content, config, mask=mask, **kwargs)
locals:
cls = <class 'Network.LayerNetwork'>
cls.from_json_and_config = <bound method type.from_json_and_config of <class 'Network.LayerNetwork'>>
json_content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
config = <Config.Config instance at 0x7f9f00228128>
mask = None
kwargs = {'train_flag': True, 'eval_flag': False}
File "/home/arman/returnn/Network.py", line 204, in from_json_and_config
line: network = cls.from_json(json_content, **dict_joined(kwargs, cls.init_args_from_config(config)))
locals:
network =
cls = <class 'Network.LayerNetwork'>
cls.from_json = <bound method type.from_json of <class 'Network.LayerNetwork'>>
json_content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
dict_joined = <function dict_joined at 0x7f9f0802a0c8>
kwargs = {'train_flag': True, 'mask': None, 'eval_flag': False}
cls.init_args_from_config = <bound method type.init_args_from_config of <class 'Network.LayerNetwork'>>
config = <Config.Config instance at 0x7f9f00228128>
File "/home/arman/returnn/Network.py", line 422, in from_json
line: traverse(json_content, layer_name, trg, index)
locals:
traverse = <function traverse at 0x7f9f00109410>
json_content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
layer_name = 'output', len = 6
trg = 'classes', len = 7
index = j_classes
File "/home/arman/returnn/Network.py", line 350, in traverse
line: index = traverse(content, prev, target, index)
locals:
index = j_classes
traverse = <function traverse at 0x7f9f00109410>
content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
prev = 'mdlstm4', len = 7
target = 'classes', len = 7
File "/home/arman/returnn/Network.py", line 350, in traverse
line: index = traverse(content, prev, target, index)
locals:
index = j_classes
traverse = <function traverse at 0x7f9f00109410>
content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
prev = 'conv4'
target = 'classes', len = 7
File "/home/arman/returnn/Network.py", line 350, in traverse
line: index = traverse(content, prev, target, index)
locals:
index = j_classes
traverse = <function traverse at 0x7f9f00109410>
content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
prev = 'mdlstm3', len = 7
target = 'classes', len = 7
File "/home/arman/returnn/Network.py", line 350, in traverse
line: index = traverse(content, prev, target, index)
locals:
index = j_classes
traverse = <function traverse at 0x7f9f00109410>
content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
prev = 'conv3'
target = 'classes', len = 7
File "/home/arman/returnn/Network.py", line 350, in traverse
line: index = traverse(content, prev, target, index)
locals:
index = j_classes
traverse = <function traverse at 0x7f9f00109410>
content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
prev = 'mdlstm2', len = 7
target = 'classes', len = 7
File "/home/arman/returnn/Network.py", line 350, in traverse
line: index = traverse(content, prev, target, index)
locals:
index = j_classes
traverse = <function traverse at 0x7f9f00109410>
content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
prev = 'conv2'
target = 'classes', len = 7
File "/home/arman/returnn/Network.py", line 350, in traverse
line: index = traverse(content, prev, target, index)
locals:
index = j_classes
traverse = <function traverse at 0x7f9f00109410>
content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
prev = 'mdlstm1', len = 7
target = 'classes', len = 7
File "/home/arman/returnn/Network.py", line 350, in traverse
line: index = traverse(content, prev, target, index)
locals:
index = j_classes
traverse = <function traverse at 0x7f9f00109410>
content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
prev = 'conv1'
target = 'classes', len = 7
File "/home/arman/returnn/Network.py", line 350, in traverse
line: index = traverse(content, prev, target, index)
locals:
index = j_classes
traverse = <function traverse at 0x7f9f00109410>
content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
prev = 'mdlstm0', len = 7
target = 'classes', len = 7
File "/home/arman/returnn/Network.py", line 350, in traverse
line: index = traverse(content, prev, target, index)
locals:
index = j_classes
traverse = <function traverse at 0x7f9f00109410>
content = {'mdlstm0': {'dropout': 0.25, 'from': ['conv0'], 'class': 'mdlstm', 'n_out': 30}, 'conv2': {'from': ['mdlstm1'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_features': 75, 'class': 'conv2'}, 'conv1': {'from': ['mdlstm0'], 'dropout': 0.25, 'pool_size': [2, 2], 'filter': [3, 3], 'n_f..., len = 13
prev = 'conv0'
target = 'classes', len = 7
File "/home/arman/returnn/Network.py", line 408, in traverse
line: return network.add_layer(layer_class(**params)).index
locals:
network = <Network.LayerNetwork object at 0x7f9f0017b310>
network.add_layer = <bound method LayerNetwork.add_layer of <Network.LayerNetwork object at 0x7f9f0017b310>>
layer_class = <class 'NetworkTwoDLayer.ConvPoolLayer2'>
params = {'index': j_sizes, 'network': <Network.LayerNetwork object at 0x7f9f0017b310>, 'dropout': 0.0, 'mask': None, 'pool_size': [2, 2], 'filter': [3, 3], 'sources': [<<class 'NetworkTwoDLayer.OneDToTwoDLayer'> class:1Dto2D name:1Dto2D>], 'y_in': {'classes': y_classes, 'data': x, 'sizes': y_sizes}, 'tra..., len = 12
index = j_sizes
File "/home/arman/returnn/NetworkTwoDLayer.py", line 460, in init
line: Z = conv_crop_pool_op(self.X, sizes, self.output_sizes, self.W, self.b, self.n_in, self.n_features, self.filter_height,
self.filter_width, filter_dilation, pool_size)
locals:
Z =
conv_crop_pool_op = <function conv_crop_pool_op at 0x7f9f002de9b0>
self = <<class 'NetworkTwoDLayer.ConvPoolLayer2'> class:conv2 name:conv0>
self.X = if{}.0
sizes = if{}.0
self.output_sizes = Join.0
self.W = W_conv0
self.b = b_conv0
self.n_in = 1
self.n_features = 15
self.filter_height = 3
self.filter_width = 3
filter_dilation = [1, 1]
pool_size = [2, 2]
File "/home/arman/returnn/NetworkTwoDLayer.py", line 358, in conv_crop_pool_op
line: conv_out = conv_op(X, W, b) if filter_height * filter_width > 0 else X
locals:
conv_out =
conv_op = <cuda_implementation.CuDNNConvHWBCOp.CuDNNConvHWBCOp object at 0x7f9f002d7f50>
X = if{}.0
W = W_conv0
b = b_conv0
filter_height = 3
filter_width = 3
File "/usr/local/lib/python2.7/dist-packages/theano/gof/op.py", line 615, in call
line: node = self.make_node(*inputs, **kwargs)
locals:
node =
self = <cuda_implementation.CuDNNConvHWBCOp.CuDNNConvHWBCOp object at 0x7f9f002d7f50>
self.make_node = <bound method CuDNNConvHWBCOp.make_node of <cuda_implementation.CuDNNConvHWBCOp.CuDNNConvHWBCOp object at 0x7f9f002d7f50>>
inputs = (if{}.0, W_conv0, b_conv0)
kwargs = {}
File "/home/arman/returnn/cuda_implementation/CuDNNConvHWBCOp.py", line 217, in make_node
line: W = gpu_contiguous(as_cuda_ndarray_variable(W))
locals:
W = W_conv0
gpu_contiguous = <theano.sandbox.cuda.basic_ops.GpuContiguous object at 0x7f9f098dced0>
as_cuda_ndarray_variable = <function as_cuda_ndarray_variable at 0x7f9f0d943848>
File "/usr/local/lib/python2.7/dist-packages/theano/sandbox/cuda/basic_ops.py", line 46, in as_cuda_ndarray_variable
line: return gpu_from_host(tensor_x)
locals:
gpu_from_host = <theano.sandbox.cuda.basic_ops.GpuFromHost object at 0x7f9f098dc690>
tensor_x = W_conv0
File "/usr/local/lib/python2.7/dist-packages/theano/gof/op.py", line 615, in call
line: node = self.make_node(*inputs, **kwargs)
locals:
node =
self = <theano.sandbox.cuda.basic_ops.GpuFromHost object at 0x7f9f098dc690>
self.make_node = <bound method GpuFromHost.make_node of <theano.sandbox.cuda.basic_ops.GpuFromHost object at 0x7f9f098dc690>>
inputs = (W_conv0,)
kwargs = {}
File "/usr/local/lib/python2.7/dist-packages/theano/sandbox/cuda/basic_ops.py", line 132, in make_node
line: return Apply(self, [x], [CudaNdarrayType(broadcastable=x.broadcastable,
dtype=x.dtype)()])
locals:
Apply = <class 'theano.gof.graph.Apply'>
self = <theano.sandbox.cuda.basic_ops.GpuFromHost object at 0x7f9f098dc690>
x = W_conv0
CudaNdarrayType = <class 'theano.sandbox.cuda.type.CudaNdarrayType'>
broadcastable =
x.broadcastable = (False, False, False, False)
dtype =
x.dtype = 'float64', len = 7
File "/usr/local/lib/python2.7/dist-packages/theano/sandbox/cuda/type.py", line 95, in init
line: raise TypeError('%s only supports dtype float32 for now. Tried '
'using dtype %s for variable %s' %
(self.class.name, dtype, name))
locals:
TypeError = <type 'exceptions.TypeError'>
self = !AttributeError: 'CudaNdarrayType' object has no attribute 'name'
self.class = <class 'theano.sandbox.cuda.type.CudaNdarrayType'>
self.class.name = 'CudaNdarrayType', len = 15
dtype = 'float64', len = 7
name = None
TypeError: CudaNdarrayType only supports dtype float32 for now. Tried using dtype float64 for variable None
Device proc gpuX (gpuZ) died: ProcConnectionDied('recv_bytes EOFError: ',)
Theano flags: compiledir_format=compiledir_%(platform)s-%(processor)s-%(python_version)s-%(python_bitwidth)s--dev-gpuZ,device=gpu,force_device=True
EXCEPTION
Traceback (most recent call last):
File "/home/arman/returnn/Device.py", line 347, in startProc
line: self._startProc(*args, **kwargs)
locals:
self = <Device.Device object at 0x7f7a861e7b10>
self._startProc = <bound method Device._startProc of <Device.Device object at 0x7f7a861e7b10>>
args = ('gpuZ',)
kwargs = {}
File "/home/arman/returnn/Device.py", line 401, in _startProc
line: interrupt_main()
locals:
interrupt_main = <function interrupt_main at 0x7f7a86e5a5f0>
File "/home/arman/returnn/Util.py", line 665, in interrupt_main
line: sys.exit(1) # And exit the thread.
locals:
sys = <module 'sys' (built-in)>
sys.exit =
SystemExit: 1
KeyboardInterrupt
EXCEPTION
Traceback (most recent call last):
File "../../../rnn.py", line 546, in main
line: init(commandLineOptions=argv[1:])
locals:
init = <function init at 0x7f7a861dda28>
commandLineOptions =
argv = ['../../../rnn.py', 'config_demo'], _[0]: {len = 15}
File "../../../rnn.py", line 343, in init
line: devices = initDevices()
locals:
devices =
initDevices = <function initDevices at 0x7f7a861dd668>
File "../../../rnn.py", line 154, in initDevices
line: time.sleep(0.25)
locals:
time = <module 'time' (built-in)>
time.sleep =
KeyboardInterrupt
Quitting