bjascob / amr_coref Goto Github PK

import re
import logging
from   multiprocessing import Pool
from   tqdm import tqdm
import numpy
from functools import partial

gfeaturizer, gmax_dist = None, None    # for multiprocessing
def build_coref_features(mdata, model, **kwargs):
    chunksize = kwargs.get('feat_chunksize',          200)
    maxtpc    = kwargs.get('feat_maxtasksperchild',   200)
    processes = kwargs.get('feat_processes',         None)    # None = use os.cpu_count()
    show_prog = kwargs.get('show_prog',              True)
    global gfeaturizer, gmax_dist

    gfeaturizer = CorefFeaturizer(mdata, model)
    gmax_dist   = model.config.max_dist if model.config.max_dist is not None else 999999999
    # Build the list of doc_names and mention indexes for multiprocessing and the output container
    idx_keys = [(dn, idx) for dn, mlist in gfeaturizer.mdata.mentions.items() for idx in range(len(mlist))]
    feat_data = {}
    for dn, mlist in gfeaturizer.mdata.mentions.items():
        feat_data[dn] = [None]*len(mlist)
    # Loop through and get the pair features for all antecedents
    pbar = tqdm(total=len(idx_keys), ncols=100, disable=not show_prog)
    with Pool(processes=processes, maxtasksperchild=maxtpc) as pool:
        worker_with_args = partial(worker, gfeaturizer=gfeaturizer, gmax_dist=gmax_dist)
        for fdata in pool.imap_unordered(worker_with_args, idx_keys, chunksize=chunksize):
            dn, midx, sspans, dspans, words, sfeats, pfeats, slabels, plabels = fdata
            feat_data[dn][midx] = {'sspans':sspans,   'dspans':dspans, 'words':words,
                                   'sfeats':sfeats,   'pfeats':pfeats,
                                   'slabels':slabels, 'plabels':plabels}
            pbar.update(1)
    pbar.close()
    # Error check
    for dn, feat_list in feat_data.items():
        assert None not in feat_list
    return feat_data

def worker(idx_key, gfeaturizer, gmax_dist):
    global gfrozen_embeds
    doc_name, midx = idx_key
    mlist       = gfeaturizer.mdata.mentions[doc_name]
    mention     = mlist[midx]               # the head mention
    antecedents = mlist[:midx]              # all antecedents up to (not including) head mention
    antecedents = antecedents[-gmax_dist:]  # truncate earlier value so list is only max_dist long
    # Process the single and pair data data
    sspan_vector = gfeaturizer.get_sentence_span_vector(mention)
    dspan_vector = gfeaturizer.get_document_span_vector(mention)
    word_indexes = gfeaturizer.get_word_indexes(mention)
    sfeats       = gfeaturizer.get_single_features(mention)
    pfeats       = gfeaturizer.get_pair_features(mention, antecedents)
    # Build target labels.  Note that if there are no clusters in the mention data this will still
    # return a list of targets, though all singles will be 1 and pairs 0
    slabels, plabels = gfeaturizer.build_targets(mention, antecedents)
    return doc_name, midx, sspan_vector, dspan_vector, word_indexes, sfeats, pfeats, slabels, plabels

Loading model from data/model_coref-v0.1.0/
Loading test data
ignoring epigraph data for duplicate triple: ('w', ':polarity', '-')
Clustering
multiprocessing.pool.RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
  File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/pool.py", line 48, in mapstar
    return list(map(*args))
  File "/work/third/amr_coref/amr_coref/coref/coref_featurizer.py", line 259, in worker
    mlist       = gfeaturizer.mdata.mentions[doc_name]
AttributeError: 'NoneType' object has no attribute 'mdata'
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/work/third/amr_coref/40_Run_Inference.py", line 67, in <module>
    cluster_dict = inference.coreference(ordered_pgraphs)
  File "/work/third/amr_coref/amr_coref/coref/inference.py", line 47, in coreference
    self.test_dloader = get_data_loader_from_data(tdata_dict, self.model, show_prog=self.show_prog, shuffle=False)
  File "/work/third/amr_coref/amr_coref/coref/coref_data_loader.py", line 41, in get_data_loader_from_data
    fdata   = build_coref_features(mdata, model, **kwargs)
  File "/work/third/amr_coref/amr_coref/coref/coref_featurizer.py", line 243, in build_coref_features
    for fdata in pool.imap_unordered(worker, idx_keys, chunksize=chunksize):
  File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/pool.py", line 448, in <genexpr>
    return (item for chunk in result for item in chunk)
  File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/pool.py", line 870, in next
    raise value
AttributeError: 'NoneType' object has no attribute 'mdata'

Compilation exited abnormally with code 1 at Fri Aug  5 18:19:19