csingh27 / trident_primitives Goto Github PK

raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(

RuntimeError: Error(s) in loading state_dict for MAML:
Missing key(s) in state_dict: "module.encoder.net.0.weight", "module.encoder.net.0.bias", "module.encoder.net.1.weight", "module.encoder.net.1.bias", "module.encoder.net.1.running_mean", "module.encoder.net.1.running_var", "module.encoder.net.4.weight", "module.encoder.net.4.bias", "module.encoder.net.5.weight", "module.encoder.net.5.bias", "module.encoder.net.5.running_mean", "module.encoder.net.5.running_var", "module.encoder.net.8.weight", "module.encoder.net.8.bias", "module.encoder.net.9.weight", "module.encoder.net.9.bias", "module.encoder.net.9.running_mean", "module.encoder.net.9.running_var", "module.encoder.net.12.weight", "module.encoder.net.12.bias", "module.encoder.net.13.weight", "module.encoder.net.13.bias", "module.encoder.net.13.running_mean", "module.encoder.net.13.running_var", "module.decoder.linear.0.weight", "module.decoder.linear.0.bias", "module.decoder.net.1.weight", "module.decoder.net.1.bias", "module.decoder.net.4.weight", "module.decoder.net.4.bias", "module.decoder.net.7.weight", "module.decoder.net.7.bias", "module.decoder.net.10.weight", "module.decoder.net.10.bias", "module.classifier_vae.encoder.net.0.weight", "module.classifier_vae.encoder.net.0.bias", "module.classifier_vae.encoder.net.1.weight", "module.classifier_vae.encoder.net.1.bias", "module.classifier_vae.encoder.net.1.running_mean", "module.classifier_vae.encoder.net.1.running_var", "module.classifier_vae.encoder.net.4.weight", "module.classifier_vae.encoder.net.4.bias", "module.classifier_vae.encoder.net.5.weight", "module.classifier_vae.encoder.net.5.bias", "module.classifier_vae.encoder.net.5.running_mean", "module.classifier_vae.encoder.net.5.running_var", "module.classifier_vae.encoder.net.8.weight", "module.classifier_vae.encoder.net.8.bias", "module.classifier_vae.encoder.net.9.weight", "module.classifier_vae.encoder.net.9.bias", "module.classifier_vae.encoder.net.9.running_mean", "module.classifier_vae.encoder.net.9.running_var", "module.classifier_vae.encoder.net.12.weight", "module.classifier_vae.encoder.net.12.bias", "module.classifier_vae.encoder.net.13.weight", "module.classifier_vae.encoder.net.13.bias", "module.classifier_vae.encoder.net.13.running_mean", "module.classifier_vae.encoder.net.13.running_var", "module.classifier_vae.encoder.fe.0.weight", "module.classifier_vae.encoder.fe.2.weight", "module.classifier_vae.encoder.f_q.weight", "module.classifier_vae.encoder.f_k.weight", "module.classifier_vae.encoder.f_v.weight", "module.classifier_vae.gaussian_parametrizer.h1.weight", "module.classifier_vae.gaussian_parametrizer.h1.bias", "module.classifier_vae.gaussian_parametrizer.h2.weight", "module.classifier_vae.gaussian_parametrizer.h2.bias", "module.classifier_vae.classifier.0.weight", "module.classifier_vae.classifier.0.bias", "module.classifier_vae.classifier.2.weight", "module.classifier_vae.classifier.2.bias", "module.gaussian_parametrizer.h1.weight", "module.gaussian_parametrizer.h1.bias", "module.gaussian_parametrizer.h2.weight", "module.gaussian_parametrizer.h2.bias".
Unexpected key(s) in state_dict: "state", "param_groups".
csingh@CSINGH-X1-U:~/DFKI/PhysWM/trident_primitives$

File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/src/trident_train.py", line 80, in
train_tasks, valid_tasks, _, learner = setup(
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/src/zoo/trident_utils.py", line 62, in setup
learner = learner.to(device)

RuntimeError: CUDA error: out of memory
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with TORCH_USE_CUDA_DSA to enable device-side asserti

trident_primitives/src/trident_train.py", line 100, in
ttask = train_tasks.sample()

raise ValueError("Sample larger than population or is negative")

ValueError: Sample larger than population or is negative

size mismatch for module.classifier_vae.encoder.fe.0.weight: copying a param with shape torch.Size([64, 1, 30, 1]) from checkpoint, the shape in current model is torch.Size([64, 1, 10, 1]).

Traceback (most recent call last):
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/src/trident_train.py", line 104, in
evaluation_loss, evaluation_accuracy = inner_adapt_trident(
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/src/zoo/trident_utils.py", line 138, in inner_adapt_trident
reconst_image, logits, mu_l, log_var_l, mu_s, log_var_s = learner(
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/learn2learn/algorithms/maml.py", line 107, in forward
return self.module(*args, **kwargs)
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/src/zoo/archs.py", line 810, in forward
mu_s, log_var_s = self.gaussian_parametrizer(xs)
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/src/zoo/archs.py", line 404, in forward
mu = self.h1(x)
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/torch/nn/modules/linear.py", line 114, in forward
return F.linear(input, self.weight, self.bias)
RuntimeError: mat1 and mat2 shapes cannot be multiplied (1x38400 and 800x64)

2153] Execution of replica 0 failed: INTERNAL: Could not find the corresponding function
Traceback (most recent call last):
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/src/trident_train.py", line 80, in
train_tasks, valid_tasks, _, learner = setup(
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/src/zoo/trident_utils.py", line 40, in setup
train_tasks = gen_tasks(dataset, root, download=download, mode='train',
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/data/taskers.py", line 104, in gen_tasks
cfs = Primitives(root, mode, transform=image_transforms,
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/data/loaders.py", line 138, in init
shots = [load_shot(path, shape) for path in load_paths(concept_path)]
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/data/loaders.py", line 138, in
shots = [load_shot(path, shape) for path in load_paths(concept_path)]
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/data/primitives.py", line 31, in load_shot
image = load_image(os.path.join(directory, 'image.png'), shape)
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/data/primitives.py", line 12, in load_image
image = jax.image.resize(image, (*shape, 3), 'bilinear')
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/jax/_src/image/scale.py", line 339, in resize
return _resize(image, core.canonicalize_shape(shape), method, antialias,
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
return fun(*args, **kwargs)
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/jax/_src/api.py", line 623, in cache_miss
out_flat = call_bind_continuation(execute(*args_flat))
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/jax/_src/dispatch.py", line 895, in _execute_compiled
out_flat = compiled.execute(in_flat)
jax._src.traceback_util.UnfilteredStackTrace: jaxlib.xla_extension.XlaRuntimeError: INTERNAL: Could not find the corresponding function

The stack trace below excludes JAX-internal frames.
The preceding is the original exception that occurred, unmodified.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/src/trident_train.py", line 80, in
train_tasks, valid_tasks, _, learner = setup(
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/src/zoo/trident_utils.py", line 40, in setup
train_tasks = gen_tasks(dataset, root, download=download, mode='train',
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/data/taskers.py", line 104, in gen_tasks
cfs = Primitives(root, mode, transform=image_transforms,
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/data/loaders.py", line 138, in init
shots = [load_shot(path, shape) for path in load_paths(concept_path)]
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/data/loaders.py", line 138, in
shots = [load_shot(path, shape) for path in load_paths(concept_path)]
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/data/primitives.py", line 31, in load_shot
image = load_image(os.path.join(directory, 'image.png'), shape)
File "/home/dfki.uni-bremen.de/csingh/DFKI/PhysWM/trident_primitives/data/primitives.py", line 12, in load_image
image = jax.image.resize(image, (*shape, 3), 'bilinear')
File "/home/dfki.uni-bremen.de/csingh/anaconda3/lib/python3.9/site-packages/jax/_src/image/scale.py", line 339, in resize
return _resize(image, core.canonicalize_shape(shape), method, antialias,
jaxlib.xla_extension.XlaRuntimeError: INTERNAL: Could not find the corresponding functio

ttask = train_tasks.sample()

AttributeError: 'NoneType' object has no attribute 'sample'