trouble in run_bart_test.py or run_bart_val.py about fastmri HOT 6 CLOSED

Dear Jure Zbontar, I run run_bart_val.py when changing the number of processes to 2. However, I got the following error.  Traceback (most recent call last):   File "run_bart_val.py", line 144, in <module>     main()   File "run_bart_val.py", line 111, in main     outputs = pool.map(run_model, range(len(data)))   File "/mnt/sdb/anaconda3/lib/python3.6/multiprocessing/pool.py", line 266, in map     return self._map_async(func, iterable, mapstar, chunksize).get()   File "/mnt/sdb/anaconda3/lib/python3.6/multiprocessing/pool.py", line 644, in get     raise self._value multiprocessing.pool.MaybeEncodingError: Error sending result: ... I attached my run_bart_val.py and run_bart_val_my.py to this e-mail. The latter does not use multiprocessing, and it works fine in my PC.  Sincerely yours, Mizuho Nishio

""" Copyright (c) Facebook, Inc. and its affiliates. This source code is licensed under the MIT license found in the LICENSE file in the root directory of this source tree. """ import logging import multiprocessing import pathlib import random import time from collections import defaultdict import numpy as np import torch import bart from common import utils from common.args import Args from common.subsample import MaskFunc from common.utils import tensor_to_complex_np from data import transforms from data.mri_data import SliceData logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class DataTransform: """ Data Transformer that masks input k-space. """ def __init__(self, mask_func): """ Args: mask_func (common.subsample.MaskFunc): A function that can create a mask of appropriate shape. """ self.mask_func = mask_func def __call__(self, kspace, target, attrs, fname, slice): """ Args: kspace (numpy.array): Input k-space of shape (num_coils, rows, cols, 2) for multi-coil data or (rows, cols, 2) for single coil data. target (numpy.array, optional): Target image attrs (dict): Acquisition related information stored in the HDF5 object. fname (str): File name slice (int): Serial number of the slice. Returns: (tuple): tuple containing: masked_kspace (torch.Tensor): Sub-sampled k-space with the same shape as kspace. fname (str): File name containing the current data item slice (int): The index of the current slice in the volume """ kspace = transforms.to_tensor(kspace) seed = tuple(map(ord, fname)) # Apply mask to raw k-space masked_kspace, mask = transforms.apply_mask(kspace, self.mask_func, seed) return masked_kspace, fname, slice def create_data_loader(args): dev_mask = MaskFunc(args.center_fractions, args.accelerations) data = SliceData( root=args.data_path / f'{args.challenge}_val', transform=DataTransform(dev_mask), challenge=args.challenge, sample_rate=args.sample_rate ) return data def cs_total_variation(args, kspace): """ Run ESPIRIT coil sensitivity estimation and Total Variation Minimization based reconstruction algorithm using the BART toolkit. """ if args.challenge == 'singlecoil': kspace = kspace.unsqueeze(0) kspace = kspace.permute(1, 2, 0, 3).unsqueeze(0) kspace = tensor_to_complex_np(kspace) # Estimate sensitivity maps #sens_maps = bart.bart(1, f'ecalib -d0 -m1', kspace) sens_maps = bart.bart(1, f'ecalib -m1', kspace) # Use Total Variation Minimization to reconstruct the image pred = bart.bart( 1, f'pics -d0 -S -R T:7:0:{args.reg_wt} -i {args.num_iters}', kspace, sens_maps ) pred = torch.from_numpy(np.abs(pred[0])) # Crop the predicted image to the correct size return transforms.center_crop(pred, (args.resolution, args.resolution)) def run_model(i): masked_kspace, fname, slice = data[i] prediction = cs_total_variation(args, masked_kspace) return fname, slice, prediction def main(): #with multiprocessing.Pool(20) as pool: with multiprocessing.Pool(2) as pool: start_time = time.perf_counter() outputs = pool.map(run_model, range(len(data))) time_taken = time.perf_counter() - start_time logging.info(f'Run Time = {time_taken:}s') save_outputs(outputs, args.output_path) def save_outputs(outputs, output_path): reconstructions = defaultdict(list) for fname, slice, pred in outputs: reconstructions[fname].append((slice, pred)) reconstructions = { fname: np.stack([pred for _, pred in sorted(slice_preds)]) for fname, slice_preds in reconstructions.items() } utils.save_reconstructions(reconstructions, output_path) if __name__ == '__main__': parser = Args() parser.add_argument('--output-path', type=pathlib.Path, default=None, help='Path to save the reconstructions to') parser.add_argument('--num-iters', type=int, default=200, help='Number of iterations to run the reconstruction algorithm') parser.add_argument('--reg-wt', type=float, default=0.01, help='Regularization weight parameter') args = parser.parse_args() random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) print("starting ...") data = create_data_loader(args) main() """ Copyright (c) Facebook, Inc. and its affiliates. This source code is licensed under the MIT license found in the LICENSE file in the root directory of this source tree. """ import logging import multiprocessing import pathlib import random import time from collections import defaultdict import numpy as np import torch import bart from common import utils from common.args import Args from common.subsample import MaskFunc from common.utils import tensor_to_complex_np from data import transforms from data.mri_data import SliceData logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class DataTransform: """ Data Transformer that masks input k-space. """ def __init__(self, mask_func): """ Args: mask_func (common.subsample.MaskFunc): A function that can create a mask of appropriate shape. """ self.mask_func = mask_func def __call__(self, kspace, target, attrs, fname, slice): """ Args: kspace (numpy.array): Input k-space of shape (num_coils, rows, cols, 2) for multi-coil data or (rows, cols, 2) for single coil data. target (numpy.array, optional): Target image attrs (dict): Acquisition related information stored in the HDF5 object. fname (str): File name slice (int): Serial number of the slice. Returns: (tuple): tuple containing: masked_kspace (torch.Tensor): Sub-sampled k-space with the same shape as kspace. fname (str): File name containing the current data item slice (int): The index of the current slice in the volume """ kspace = transforms.to_tensor(kspace) seed = tuple(map(ord, fname)) # Apply mask to raw k-space masked_kspace, mask = transforms.apply_mask(kspace, self.mask_func, seed) return masked_kspace, fname, slice def create_data_loader(args): dev_mask = MaskFunc(args.center_fractions, args.accelerations) data = SliceData( root=args.data_path / f'{args.challenge}_val', transform=DataTransform(dev_mask), challenge=args.challenge, sample_rate=args.sample_rate ) return data def cs_total_variation(args, kspace): """ Run ESPIRIT coil sensitivity estimation and Total Variation Minimization based reconstruction algorithm using the BART toolkit. """ if args.challenge == 'singlecoil': kspace = kspace.unsqueeze(0) kspace = kspace.permute(1, 2, 0, 3).unsqueeze(0) kspace = tensor_to_complex_np(kspace) # Estimate sensitivity maps sens_maps = bart.bart(1, f'ecalib -m1', kspace) # Use Total Variation Minimization to reconstruct the image pred = bart.bart( 1, f'pics -d0 -S -R T:7:0:{args.reg_wt} -i {args.num_iters}', kspace, sens_maps ) pred = torch.from_numpy(np.abs(pred[0])) # Crop the predicted image to the correct size return transforms.center_crop(pred, (args.resolution, args.resolution)) def run_model(i): masked_kspace, fname, slice = data[i] prediction = cs_total_variation(args, masked_kspace) return fname, slice, prediction """ def main(): with multiprocessing.Pool(20) as pool: start_time = time.perf_counter() outputs = pool.map(run_model, range(len(data))) time_taken = time.perf_counter() - start_time logging.info(f'Run Time = {time_taken:}s') save_outputs(outputs, args.output_path) """ def main_hdf(): outputs = [] N = len(data) #N = 40 print("number of slices:", N) for i in range(N): fname, slice, pred = run_model(i) print("finished: index=%s, filename=%s, slice=%s" % (i,fname, slice)) outputs.append( (fname, slice, pred) ) save_outputs(outputs, args.output_path) return def save_outputs(outputs, output_path): reconstructions = defaultdict(list) for fname, slice, pred in outputs: reconstructions[fname].append((slice, pred)) reconstructions = { fname: np.stack([pred for _, pred in sorted(slice_preds)]) for fname, slice_preds in reconstructions.items() } utils.save_reconstructions(reconstructions, output_path) return def main_numpy(): for i in range(len(data)): print("starting index =", i) start_time = time.perf_counter() fname, slice, pred = run_model(i) print(fname, slice, pred.shape) time_taken = time.perf_counter() - start_time logging.info(f'Run Time = {time_taken:}s') save_as_numpy(fname, slice, pred, args.output_path) def save_as_numpy(fname, slice_num, pred, output_path): path = str(output_path) + "/val_%s___%05d.npy" % (str(fname.split(".")[0]), slice_num) print("saving ... ", path) np.save(path, pred.numpy()) return if __name__ == '__main__': parser = Args() parser.add_argument('--output-path', type=pathlib.Path, default=None, help='Path to save the reconstructions to') parser.add_argument('--num-iters', type=int, default=200, help='Number of iterations to run the reconstruction algorithm') parser.add_argument('--reg-wt', type=float, default=0.01, help='Regularization weight parameter') args = parser.parse_args() random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) data = create_data_loader(args) main_hdf() #main_numpy()

Thank you for your response.  I tried to run run_bart_val.py again with reducing number of cpu (2) in multiprocessing. But, the same error was obtained.   My environment is as follows: $ cat /etc/os-releaseNAME="Ubuntu"VERSION="18.04.1 LTS (Bionic Beaver)"ID=ubuntuID_LIKE=debianPRETTY_NAME="Ubuntu 18.04.1 LTS"VERSION_ID="18.04"HOME_URL="https://www.ubuntu.com/"SUPPORT_URL="https://help.ubuntu.com/"BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"VERSION_CODENAME=bionicUBUNTU_CODENAME=bionic $ uname -aLinux dlbox01 4.15.0-39-generic #42-Ubuntu SMP Tue Oct 23 15:48:01 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux $ ./python --versionPython 3.6.5 :: Anaconda, Inc.

Thank you for your response. In my environment, the following patch are necessary for run_bart_val.py My bart version is v0.4.03. --- run_bart_val2.py.orig       2019-01-28 16:52:15.998883999 +0900+++ run_bart_val2.py    2019-01-28 16:53:04.565973851 +0900@@ -85,7 +85,7 @@     kspace = tensor_to_complex_np(kspace)      # Estimate sensitivity maps-    sens_maps = bart.bart(1, f'ecalib -d0 -m1', kspace)+    sens_maps = bart.bart(1, f'ecalib -m1', kspace)      # Use Total Variation Minimization to reconstruct the image     pred = bart.bart( After applying the patch, run_bart_val.py was finished without error. I will check the outputs of run_bart_val.py. Again, thank you for your update.