added files to run

extract.py

@@ -0,0 +1,21 @@
+import pdfplumber
+
+
+def extract_text_from_pdfs(pdf_files):
+    """
+    Extracts text from a list of PDF files.
+
+    Args:
+    pdf_files (list): List of paths to PDF files.
+
+    Returns:
+    list: List of extracted text from each PDF.
+    """
+    all_texts = []
+    for pdf_file in pdf_files:
+        with pdfplumber.open(pdf_file) as pdf:
+            text = ""
+            for page in pdf.pages:
+                text += page.extract_text()
+        all_texts.append(text)
+    return all_texts

preprocess.py

@@ -0,0 +1,25 @@
+import string
+
+import nltk
+
+nltk.download('punkt')
+from nltk.tokenize import word_tokenize
+
+
+def preprocess_text(texts):
+    """
+    Preprocesses a list of texts by converting to lowercase, removing punctuation, and tokenizing.
+
+    Args:
+    texts (list): List of text strings to preprocess.
+
+    Returns:
+    list: List of preprocessed and tokenized texts.
+    """
+    processed_texts = []
+    for text in texts:
+        text = text.lower()
+        text = text.translate(str.maketrans('', '', string.punctuation))
+        tokens = word_tokenize(text)
+        processed_texts.append(tokens)
+    return processed_texts

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+PyMuPDF
+nltk
+scikit-learn
+openai

retrieve.py

@@ -0,0 +1,35 @@
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+
+def create_vectorizer(processed_texts):
+    """
+    Creates a TF-IDF vectorizer and transforms the texts.
+
+    Args:
+    processed_texts (list): List of preprocessed and tokenized texts.
+
+    Returns:
+    tuple: TF-IDF vectorizer and transformed text matrix.
+    """
+    vectorizer = TfidfVectorizer()
+    X = vectorizer.fit_transform([' '.join(text) for text in processed_texts])
+    return vectorizer, X
+
+def retrieve(query, X, vectorizer, top_k=5):
+    """
+    Retrieves the top-k most relevant texts for a given query.
+
+    Args:
+    query (str): Query string.
+    X (matrix): TF-IDF transformed text matrix.
+    vectorizer (TfidfVectorizer): TF-IDF vectorizer.
+    top_k (int): Number of top results to retrieve.
+
+    Returns:
+    list: Indices of the top-k most relevant texts.
+    """
+    query_vec = vectorizer.transform([query])
+    scores = np.dot(X, query_vec.T).toarray()
+    top_indices = np.argsort(scores, axis=0)[-top_k:][::-1]
+    return top_indices.flatten()