# windows: tensorflow is not supported on windows right now, so is deepnlp
    # linux, run the script:
    
    pip install deepnlp
    
    # Due to pkg size restriction, english pos model, ner model files are not distributed on pypi
    # You can download the pre-trained model files from github and put in your installation directory .../site-packages/.../deepnlp/...
    # model files: ../pos/ckpt/en/pos.ckpt  ; ../ner/ckpt/zh/ner.ckpt
    

    # linux, run the script:
    tar zxvf deepnlp-0.1.5.tar.gz
    cd deepnlp-0.1.5
    python setup.py install

    # linux, run the script:
    cd test
    python test_pos_en.py
    python test_segmenter.py
    python test_pos_zh.py
    # Check if output is correct

#coding=utf-8
from __future__ import unicode_literals # compatible with python3 unicode

#coding=utf-8
from __future__ import unicode_literals

from deepnlp import segmenter

text = "我爱吃北京烤鸭"
segList = segmenter.seg(text)
text_seg = " ".join(segList)

print (text.encode('utf-8'))
print (text_seg.encode('utf-8'))

#coding:utf-8
from __future__ import unicode_literals

## English Model
from deepnlp import pos_tagger
tagger = pos_tagger.load_model(lang = 'en')  # Loading English model, lang code 'en', English Model Brown Corpus

#Segmentation
text = "I will see a funny movie"
words = text.split(" ")     # unicode
print (" ".join(words).encode('utf-8'))

#POS Tagging
tagging = tagger.predict(words)
for (w,t) in tagging:
    str = w + "/" + t
    print (str.encode('utf-8'))


#coding:utf-8
from __future__ import unicode_literals

## Chinese Model
from deepnlp import segmenter
from deepnlp import pos_tagger
tagger = pos_tagger.load_model(lang = 'zh') # Loading Chinese model, lang code 'zh', China Daily Corpus

#Segmentation
text = "我爱吃北京烤鸭"
words = segmenter.seg(text) # words in unicode coding
print (" ".join(words).encode('utf-8'))

#POS Tagging
tagging = tagger.predict(words)  # input: unicode coding
for (w,t) in tagging:
    str = w + "/" + t
    print (str.encode('utf-8'))

#Results
#我/r 爱/v 吃/v 北京/ns 烤鸭/n

#coding:utf-8
from __future__ import unicode_literals

from deepnlp import segmenter
from deepnlp import ner_tagger
tagger = ner_tagger.load_model(lang = 'zh') # Loading Chinese NER model

#Segmentation
text = "我爱吃北京烤鸭"
words = segmenter.seg(text)
print (" ".join(words).encode('utf-8'))

#NER tagging
tagging = tagger.predict(words)
for (w,t) in tagging:
    str = w + "/" + t
    print (str.encode('utf-8'))

#Results
#我/nt 爱/nt 吃/nt 北京/p 烤鸭/nt

#coding:utf-8
from __future__ import unicode_literals

from deepnlp import pipeline
p = pipeline.load_model('zh')

#Segmentation
text = "我爱吃北京烤鸭"
res = p.analyze(text)

print (res[0].encode('utf-8'))
print (res[1].encode('utf-8'))
print (res[2].encode('utf-8'))

words = p.segment(text)
pos_tagging = p.tag_pos(words)
ner_tagging = p.tag_ner(words)

print (pos_tagging.encode('utf-8'))
print (ner_tagging.encode('utf-8'))

python data_util.py train.txt train_word_tag.txt

#train.txt
#English:
POS/NN tagging/NN is/VBZ now/RB done/VBN in/IN the/DT context/NN of/IN computational/JJ linguistics/NNS ./.

#Chinese:
充满/v  希望/n  的/u  新/a  世纪/n  ——/w  一九九八年/t  新年/t  讲话/n  （/w  附/v  图片/n  １/m  张/q  ）/w  

python pos_model.py en # LSTM model English

python pos_model.py zh # LSTM model Chinese

python pos_model_bilstm.py en # Bi-LSTM model English

python pos_model_bilstm.py zh # Bi-LSTM model Chinese

python ner_model.py zh # training Chinese model

    # windows: tensorflow is not supported on windows right now, so is deepnlp
    # linux, run the script:
    pip install deepnlp

    # linux, run the script:
    tar zxvf deepnlp-0.1.5.tar.gz
    cd deepnlp-0.1.5
    python setup.py install