To implement Information Retrieval Using Vector Space Model in Python.
- Define sample documents.
- Preprocess text data by tokenizing, removing stopwords, and punctuation.
- Construct a TF-IDF matrix using TfidfVectorizer from sklearn.
- Define a search function that calculates cosine similarity between a query and documents based on the TF-IDF matrix.
- Execute a sample query and display the search results along with similarity scores.
import nltk
nltk.download('punkt')
nltk.download('stopwords')
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import string
# Sample documents
documents = [
"This is the first document.",
"This document is the second document.",
"And this is the third one.",
"Is this the first document?",
]
# Preprocessing function to tokenize and remove stopwords/punctuation
def preprocess_text(text):
tokens = word_tokenize(text.lower())
tokens = [token for token in tokens if token not in stopwords.words("english") and token not in string.punctuation]
return " ".join(tokens)
print(tokens)
# Preprocess documents
preprocessed_docs = [preprocess_text(doc) for doc in documents]
# Construct TF-IDF matrix
tfidf_vectorizer = TfidfVectorizer()
tfidf_matrix = tfidf_vectorizer.fit_transform(preprocessed_docs)
# Calculate cosine similarity between query and documents
def search(query, tfidf_matrix, tfidf_vectorizer):
preprocessed_query = preprocess_text(query)
query_vector = tfidf_vectorizer.transform([preprocessed_query])
# Calculate cosine similarity between query and documents
similarity_scores = cosine_similarity(query_vector, tfidf_matrix)
# Sort documents based on similarity scores
sorted_indexes = similarity_scores.argsort()[0][::-1]
# Return sorted documents along with their similarity scores
results = [(documents[i], similarity_scores[0, i]) for i in sorted_indexes]
return results
# Example query
query =input("Enter query: ")
# Perform search
search_results = search(query, tfidf_matrix, tfidf_vectorizer)
# Display search results
i=1
for result in search_results:
print("----------------------")
print("\nRank: ",i)
print("Document:", result[0])
print("Similarity Score:", result[1])
i=i+1
Thus, the implementation of Information Retrieval Using Vector Space Model in Python is executed successfully.