import tiktoken
encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")
token = encoding.encode(text)
print(len(token))

def jaccard_similarity(text1, text2, somajo_tokenizer):

    def get_token_set(text):
        sentences = somajo_tokenizer.tokenize_text([text])
        tokens = [t.text.lower() for sentence in sentences for t in sentence]
        #token_ngrams = list(ngrams(sequence=tokens, n=2))
        #token_set = set(tokens + token_ngrams)
        token_set = set(tokens)
        return token_set

    token_set1 = get_token_set(text1)
    token_set2 = get_token_set(text2)
    intersection = token_set1.intersection(token_set2)
    union = token_set1.union(token_set2)
    jaccard_similarity = float(len(intersection)) / len(union)
    return jaccard_similarity

class LanguageIdentification:

    def __init__(self):
        pretrained_lang_model = "/home/phmay/models/lid.176.bin"
        self.model = fasttext.load_model(pretrained_lang_model)

    def predict_lang(self, text):
        predictions = self.model.predict(text, k=2) # returns top 2 matching languages
        #print(predictions)
        if '__label__de' in predictions[0]:
            return predictions[1][predictions[0].index('__label__de')]
        else:
            return 0.0

li = LanguageIdentification()

sentences = [s for s in sentences if li.predict_lang(s) > 0.9]

with gzip.open(write_filename, 'w') as fout:
    fout.write(json.dumps(some_data, indent=2).encode('utf-8')) 

en2de = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.en-de',
                       checkpoint_file='model1.pt:model2.pt:model3.pt:model4.pt',
                       tokenizer='moses', bpe='fastbpe')
_ = en2de.eval()  # disable dropout
_ = en2de.cuda()  # use GPU