def get_part_of_speech(word):
  probable_part_of_speech = wordnet.synsets(word)
  pos_counts = Counter()
  pos_counts["n"] = len(  [ item for item in probable_part_of_speech if item.pos()=="n"]  )
  pos_counts["v"] = len(  [ item for item in probable_part_of_speech if item.pos()=="v"]  )
  pos_counts["a"] = len(  [ item for item in probable_part_of_speech if item.pos()=="a"]  )
  pos_counts["r"] = len(  [ item for item in probable_part_of_speech if item.pos()=="r"]  )
  most_likely_part_of_speech = pos_counts.most_common(1)[0][0]
  return most_likely_part_of_speech

def preprocess_text(text):
    
  cleaned = re.sub(r'\W+', ' ', text).lower()
  tokenized = word_tokenize(cleaned)
  lemmatized = [normalizer.lemmatize(token, get_part_of_speech(token)) for token in tokenized if token not in stopwords.words('english')]
  normalized = ' '.join(lemmatized)
  
  return normalized

pos = open('NLTK/short_reviews/positive.txt', 'r', encoding='latin-1').read()
neg = open('NLTK/short_reviews/negative.txt', 'r', encoding='latin-1').read()

pos_clean = [preprocess_text(sen) for sen in pos.split('\n') if sen != '']
neg_clean = [preprocess_text(sen) for sen in neg.split('\n') if sen != '']

x_clean = pos_clean + neg_clean
labels = [1] * len(pos_clean) + [0] * len(neg_clean)

vocab = []

for sentence in x_clean:
  for word in sentence.split(' '):
    if word not in vocab:
      vocab.append(word)

x_train, x_test, y_train, y_test = train_test_split(
  x_clean, labels, test_size=0.2, random_state=42
)

vectorizer = CountVectorizer(vocabulary=vocab)

x_vec = vectorizer.fit_transform(x_train).toarray()

# xt_vec = vectorizer.transform(x_test).toarray()

with numpy.printoptions(threshold=numpy.inf):
    print(x_vec[0])