Train data description for NER training, how to train NER model about emnlp2017-bilstm-cnn-crf HOT 4 OPEN

"""
Converts the IOB encoding from CoNLL 2003 to BIO encoding
"""


filenames = ['train.txt', 'dev.txt', 'test.txt']

for filename in filenames:
    fOut = open(filename+'.bio', 'w')
    fIn = open(filename, 'r')
    
    for line in fIn:
        if line.startswith('-DOCSTART-'):
            lastChunk = 'O'
            lastNER = 'O'
            continue
        
        if len(line.strip()) == 0:
            lastChunk = 'O'
            lastNER = 'O'
            fOut.write("\n")
            continue
            
        
        splits = line.strip().split()
        
        chunk = splits[2]
        ner = splits[3]
        
        if chunk[0] == 'I':
            if chunk[1:] != lastChunk[1:]:
                chunk = 'B'+chunk[1:]
                
        if ner[0] == 'I':
            if ner[1:] != lastNER[1:]:
                ner = 'B'+ner[1:]
                
        splits[2] = chunk 
        splits[3] = ner
        
        fOut.write("\t".join(splits))
        fOut.write("\n")
        
        lastChunk = chunk
        lastNER = ner

datasets = {
    'conll2003_ner':                            
        {'columns': {0:'tokens', 3:'NER_BIO'},   
         'label': 'NER_BIO',                     
         'evaluate': True,                   
         'commentSymbol': None}              
}