关于预处理 about spammessage HOT 2 OPEN

if __name__ == '__main__':
    data, target = read_train_data()
    #data = Pool().map(jieba.lcut, data)
    data2words = []
    for words in data:
        temp = jieba.cut(words)
        data2words.append(temp)
    save_tokenlization_result(data2words, target)

    with codecs.open('./data/tags_token_results', 'r', 'utf-8') as f:
        data = [line.strip().split() for line in f.read().split('\n')]
        if not data[-1]: data.pop()
        t = [Counter(d) for d in data]  # 每一行为一个短信， 值就是TF
        v = DictVectorizer()
        t = v.fit_transform(t)  # 稀疏矩阵表示sparse matrix,词编好号
        TrainData.save(t)