profjsb / deepcr Goto Github PK

cori-2151-4: ERROR: step 129 exited with 1
Traceback (most recent call last):
  File "one_hdu.py", line 23, in <module>
    mask = model.clean(im, binary=False, n_jobs=4)
  File "/global/homes/k/kemingz/.conda/envs/deepcr/lib/python3.7/site-packages/deepCR/model.py", line 132, in clean
    n_jobs=n_jobs)
  File "/global/homes/k/kemingz/.conda/envs/deepcr/lib/python3.7/site-packages/deepCR/model.py", line 240, in clean_large_parallel
    shape=im_shape, mode='w+')
Remaining jobs: 210
  File "/global/homes/k/kemingz/.conda/envs/deepcr/lib/python3.7/site-packages/numpy/core/memmap.py", line 264, in __new__
    mm = mmap.mmap(fid.fileno(), bytes, access=acc, offset=start)
OSError: [Errno 38] Function not implemented

import sys
from deepCR import deepCR
from astropy.io import fits
import numpy as np
import os

file = sys.argv[1]

out_dir = '/global/cscratch1/sd/kemingz/decals_dr9_deepcr/'

model = deepCR('decam')
path, f_nopath = os.path.split(file)
try:
    os.mkdir(out_dir + path[-10:])
except:
    pass
save_path = out_dir + path[-10:] + '/'
hdu = fits.open(file)
for i in range(len(hdu)-1):
    im = hdu[i + 1].data
    mask = model.clean(im, parallel=True, n_jobs=2, binary=False)
    hdu[i + 1].data = (mask * 255).astype(np.uint8)
output_f = save_path + f_nopath[:-8] + '_deepcr_prob.fits.fz'
hdu.writeto(output_f, overwrite=True)

    def clean_large_parallel(self, img0, threshold=0.5, inpaint=True, binary=True,
                    patch=256, n_jobs=-1):
        """
            given input image
            return cosmic ray mask and (optionally) clean image
            mask could be binary or probabilistic
        :param img0: (np.ndarray) 2D input image
        :param threshold: for creating binary mask from probabilistic mask
        :param inpaint: return clean image only if True
        :param binary: return binary mask if True. probabilistic mask otherwise.
        :param patch: (int) Use 256 unless otherwise required. patch size to run deepCR on.
        :param n_jobs: (int) number of jobs to run in parallel, passed to `joblib.` Beware of memory overflow for
          larger n_jobs.
        :return: CR mask and (optionally) clean inpainted image
        """
        folder = './joblib_memmap_' + secrets.token_hex(3)
        try:
            mkdir(folder)
        except FileExistsError:
            folder = './joblib_memmap_' + secrets.token_hex(3)
            mkdir(folder)

        im_shape = img0.shape
        img0_dtype = img0.dtype
        hh = int(math.ceil(im_shape[0]/patch))
        ww = int(math.ceil(im_shape[1]/patch))

        img0_filename_memmap = path.join(folder, 'img0_memmap')
        dump(img0, img0_filename_memmap)
        img0 = load(img0_filename_memmap, mmap_mode='r')

        if inpaint:
            img1_filename_memmap = path.join(folder, 'img1_memmap')
            img1 = np.memmap(img1_filename_memmap, dtype=img0.dtype,
                            shape=im_shape, mode='w+')
        else:
            img1 = None

        mask_filename_memmap = path.join(folder, 'mask_memmap')
        mask = np.memmap(mask_filename_memmap, dtype=np.int8 if binary else img0_dtype,
                           shape=im_shape, mode='w+')

        @wrap_non_picklable_objects
        def fill_values(i, j, img0, img1, mask, patch, inpaint, threshold, binary):
            img = img0[i * patch: min((i + 1) * patch, im_shape[0]), j * patch: min((j + 1) * patch, im_shape[1])]
            if inpaint:
                mask_, clean_ = self.clean_(img, threshold=threshold, inpaint=True, binary=binary)
                mask[i * patch: min((i + 1) * patch, im_shape[0]), j * patch: min((j + 1) * patch, im_shape[1])] = mask_
                img1[i * patch: min((i + 1) * patch, im_shape[0]), j * patch: min((j + 1) * patch, im_shape[1])] = clean_
            else:
                mask_ = self.clean_(img, threshold=threshold, inpaint=False, binary=binary)
                mask[i * patch: min((i + 1) * patch, im_shape[0]), j * patch: min((j + 1) * patch, im_shape[1])] = mask_

        results = Parallel(n_jobs=n_jobs, verbose=0)\
                   (delayed(fill_values)(i, j, img0, img1, mask, patch, inpaint, threshold, binary)
                    for i in range(hh) for j in range(ww))

        mask = np.array(mask)
        if inpaint:
            img1 = np.array(img1)
        try:
            shutil.rmtree(folder)
        except:
            print('Could not clean-up automatically.')

        if inpaint:
            return mask, img1
        else:
            return mask