rombak

app.py

@@ -1,279 +1,241 @@
-import os
+# app.py
 import gradio as gr
-from datasets import Dataset
+import os
+import re
+import shutil
+import torch
+
+# LangChain imports
+from langchain_community.document_loaders import ArxivLoader, PyPDFLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain_community.vectorstores import FAISS
-from langchain_community.llms import HuggingFaceHub
 from langchain.chains import RetrievalQA
-from langchain.agents import AgentExecutor, create_react_agent, Tool
-from langchain_core.prompts import PromptTemplate
-from langchain_core.messages import SystemMessage
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_core.documents import Document
-import arxiv # Import library arxiv
-
-# --- Konfigurasi dan Setup Awal ---
-# Pastikan Anda memiliki token API Hugging Face.
-# Anda bisa mendapatkannya dari https://huggingface.co/settings/tokens
-# Simpan sebagai variabel lingkungan di Hugging Face Spaces (Settings -> Repository secrets)
-# dengan nama HUGGINGFACEHUB_API_TOKEN.
-os.environ["HUGGINGFACEHUB_API_TOKEN"] = "HUGGINGFACEHUB_API_TOKEN" # Jangan hardcode di sini!
-
-# Periksa apakah token API telah disetel
-if "HUGGINGFACEHUB_API_TOKEN" not in os.environ:
-    print("WARNING: Variabel lingkungan 'HUGGINGFACEHUB_API_TOKEN' tidak disetel.")
-    print("Silakan setel token Anda sebagai 'Repository secret' di Hugging Face Spaces.")
-    # Jika token tidak disetel, kita akan menggunakan placeholder untuk LLM
-    # Untuk demo ini, jika token tidak ada, LLM akan menghasilkan pesan peringatan.
-
-# --- Fungsi untuk Mengambil Paper dari ArXiv ---
-def fetch_papers_from_arxiv(query: str, max_results: int = 5) -> list[Document]:
-    """
-    Mengambil paper dari arXiv berdasarkan query pencarian.
-    Mengembalikan list objek Document Langchain.
-    """
-    client = arxiv.Client()
-    search_query = arxiv.Search(
-        query=query,
-        max_results=max_results,
-        sort_by=arxiv.SortCriterion.Relevance,
-        sort_order=arxiv.SortOrder.Descending
-    )
-    
-    papers = []
-    try:
-        for result in client.results(search_query):
-            # Menggabungkan judul, abstrak, dan penulis sebagai konten dokumen
-            # Anda bisa memilih untuk mengunduh full text jika diperlukan,
-            # namun abstrak biasanya cukup untuk RAG awal.
-            content = f"Title: {result.title}\nAuthors: {', '.join([a.name for a in result.authors])}\nAbstract: {result.summary}"
+from langchain_community.llms import HuggingFacePipeline
+from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
+
+# --- Configuration ---
+ARXIV_DIR = "./arxiv_papers" # Directory to save downloaded papers
+CHUNK_SIZE = 500 # Characters per chunk
+CHUNK_OVERLAP = 50 # Overlap between chunks
+EMBEDDING_MODEL_NAME = 'all-MiniLM-L6-v2'
+LLM_MODEL_NAME = "google/flan-t5-small"
+
+# --- RAGAgent Class ---
+
+class RAGAgent:
+    def __init__(self):
+        self.embedding_model = None
+        self.llm = None
+        self.vectorstore = None
+        self.qa_chain = None
+        self.is_initialized = False
+
+    def _load_models(self):
+        """Loads the embedding and generation models if not already loaded."""
+        if self.embedding_model is None:
+            print(f"Loading Embedding Model: {EMBEDDING_MODEL_NAME}...")
+            self.embedding_model = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL_NAME)
+        
+        if self.llm is None:
+            print(f"Loading LLM Model: {LLM_MODEL_NAME}...")
+            tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL_NAME)
+            model = AutoModelForSeq2SeqLM.from_pretrained(LLM_MODEL_NAME)
             
-            # Menambahkan metadata seperti URL dan ID ArXiv
-            metadata = {
-                "title": result.title,
-                "authors": [a.name for a in result.authors],
-                "published": result.published.strftime("%Y-%m-%d"),
-                "arxiv_url": result.entry_id,
-                "pdf_url": result.pdf_url
-            }
-            papers.append(Document(page_content=content, metadata=metadata))
-        print(f"Berhasil mengambil {len(papers)} paper dari arXiv untuk query: '{query}'")
-    except Exception as e:
-        print(f"Gagal mengambil paper dari arXiv: {e}")
-        print("Pastikan ada koneksi internet dan query valid.")
-    return papers
-
-# --- 1. Memuat Data Paper AI dari ArXiv ---
-# Ganti dummy data dengan paper nyata dari arXiv.
-# Anda bisa menyesuaikan query dan jumlah hasil yang diinginkan.
-arxiv_search_query = "large language models" # Contoh query pencarian
-num_arxiv_papers = 5 # Jumlah paper yang ingin diambil
-documents = fetch_papers_from_arxiv(arxiv_search_query, num_arxiv_papers)
-
-# Jika tidak ada dokumen yang diambil, gunakan dummy data sebagai fallback
-if not documents:
-    print("Tidak ada paper yang diambil dari arXiv. Menggunakan dummy data sebagai fallback.")
-    dummy_ai_papers_content = [
-        """
-        Paper Title: Deep Learning for Natural Language Processing: A Review
-        Abstract: This paper reviews the advancements in deep learning techniques applied to Natural Language Processing (NLP). We discuss various architectures such as Recurrent Neural Networks (RNNs), Convolutional Neural Networks (CNNs), and Transformers, highlighting their impact on tasks like machine translation, sentiment analysis, and text summarization. The rise of large language models (LLMs) has significantly pushed the boundaries of what's possible in NLP.
-        Keywords: Deep Learning, NLP, Transformers, RNN, CNN, LLM, Machine Translation, Sentiment Analysis.
-        """,
-        """
-        Paper Title: Reinforcement Learning in Robotics: Challenges and Future Directions
-        Abstract: Reinforcement Learning (RL) has shown promise in enabling robots to learn complex behaviors through interaction with their environment. This paper explores the current challenges in applying RL to robotics, including sample efficiency, sim-to-real transfer, and safety. We also discuss potential future directions and the integration of RL with other AI paradigms like computer vision.
-        Keywords: Reinforcement Learning, Robotics, AI, Sample Efficiency, Sim-to-Real, Computer Vision.
-        """,
+            # Determine device for pipeline
+            device = 0 if torch.cuda.is_available() else -1
+
+            # Create a Hugging Face pipeline for text generation
+            text_generation_pipeline = pipeline(
+                "text2text-generation",
+                model=model,
+                tokenizer=tokenizer,
+                max_new_tokens=150, # Set a default max_new_tokens for the pipeline
+                min_length=20,
+                num_beams=5,
+                early_stopping=True,
+                device=device
+            )
+            self.llm = HuggingFacePipeline(pipeline=text_generation_pipeline)
+        
+        self.is_initialized = True
+
+    def initialize_knowledge_base(self, arxiv_query: str, max_papers: int = 5) -> str:
         """
-        Paper Title: Explainable AI (XAI): Methods and Applications
-        Abstract: As AI models become more complex, the need for Explainable AI (XAI) grows. This paper surveys various XAI methods, including LIME, SHAP, and attention mechanisms, which aim to make AI decisions more transparent and understandable to humans. We examine their applications in critical domains like healthcare and finance, where interpretability is paramount.
-        Keywords: Explainable AI, XAI, Interpretability, LIME, SHAP, Healthcare AI, Finance AI.
-        """,
+        Initializes the knowledge base by downloading, extracting, and chunking
+        arXiv papers using LangChain components, then building a FAISS vectorstore.
         """
-        Paper Title: Federated Learning: A Privacy-Preserving Machine Learning Paradigm
-        Abstract: Federated Learning (FL) is an emerging machine learning approach that enables collaborative model training across decentralized devices or organizations while keeping raw data localized. This paper discusses the architectural principles of FL, its privacy benefits, and challenges such as communication overhead, heterogeneity, and security vulnerabilities.
-        Keywords: Federated Learning, FL, Privacy, Decentralized AI, Machine Learning, Security.
-        """,
+        self._load_models() # Ensure models are loaded first
+
+        # Clear existing papers before downloading new ones
+        if os.path.exists(ARXIV_DIR):
+            shutil.rmtree(ARXIV_DIR)
+        os.makedirs(ARXIV_DIR, exist_ok=True)
+
+        self.vectorstore = None
+        self.qa_chain = None
+
+        print(f"Searching arXiv for '{arxiv_query}' and downloading up to {max_papers} papers...")
+        try:
+            # Use LangChain's ArxivLoader
+            # ArxivLoader downloads PDFs to a temporary directory by default,
+            # but we can specify a custom path to ensure cleanup.
+            # For simplicity, we'll let it download to its default temp dir
+            # and then process. Or, we can manually download and use PyPDFLoader.
+            # Let's stick to manual download for better control and consistency with previous code.
+
+            # Manual download using arxiv library (as it offers more control over filenames)
+            search_results = arxiv.Search(
+                query=arxiv_query,
+                max_results=max_papers,
+                sort_by=arxiv.SortCriterion.Relevance,
+                sort_order=arxiv.SortOrder.Descending
+            )
+            pdf_paths = []
+            for i, result in enumerate(search_results.results()):
+                try:
+                    safe_title = re.sub(r'[\\/:*?"<>|]', '', result.title)
+                    filename = f"{ARXIV_DIR}/{safe_title[:100]}_{result.arxiv_id}.pdf"
+                    print(f"Downloading paper {i+1}/{max_papers}: {result.title}")
+                    result.download_pdf(filename=filename)
+                    pdf_paths.append(filename)
+                except Exception as e:
+                    print(f"Could not download {result.title}: {e}")
+            
+            if not pdf_paths:
+                return "No papers found or downloaded for the given query. Please try a different query."
+
+            # Load documents from downloaded PDFs using PyPDFLoader
+            all_documents = []
+            for pdf_path in pdf_paths:
+                try:
+                    loader = PyPDFLoader(pdf_path)
+                    all_documents.extend(loader.load())
+                except Exception as e:
+                    print(f"Error loading PDF {pdf_path}: {e}")
+
+            if not all_documents:
+                return "Could not load any documents from downloaded PDFs. Please try a different query or fewer papers."
+
+            print(f"Loaded {len(all_documents)} raw documents from PDFs.")
+
+            # Split documents into chunks using RecursiveCharacterTextSplitter
+            text_splitter = RecursiveCharacterTextSplitter(
+                chunk_size=CHUNK_SIZE,
+                chunk_overlap=CHUNK_OVERLAP,
+                length_function=len,
+                is_separator_regex=False,
+            )
+            self.knowledge_base_chunks = text_splitter.split_documents(all_documents)
+            
+            if not self.knowledge_base_chunks:
+                return "No meaningful text chunks could be created from the papers after splitting."
+
+            print(f"Total chunks created: {len(self.knowledge_base_chunks)}")
+
+            # Create FAISS vectorstore from chunks and embeddings
+            print("Creating FAISS vectorstore from chunks...")
+            self.vectorstore = FAISS.from_documents(self.knowledge_base_chunks, self.embedding_model)
+            print(f"FAISS vectorstore created with {len(self.knowledge_base_chunks)} documents.")
+
+            # Create RetrievalQA chain
+            self.qa_chain = RetrievalQA.from_chain_type(
+                llm=self.llm,
+                chain_type="stuff", # "stuff" puts all retrieved docs into one prompt
+                retriever=self.vectorstore.as_retriever(search_kwargs={"k": 3}), # Retrieve top 3 docs
+                return_source_documents=False # Set to True if you want to return source docs
+            )
+            
+            return f"Knowledge base loaded with {len(self.knowledge_base_chunks)} chunks from {len(pdf_paths)} arXiv papers on '{arxiv_query}'."
+
+        except Exception as e:
+            print(f"Error during knowledge base initialization: {e}")
+            return f"An error occurred during knowledge base initialization: {e}"
+
+    def query_agent(self, query: str) -> str:
         """
-        Paper Title: Generative Adversarial Networks (GANs): Architectures and Applications
-        Abstract: Generative Adversarial Networks (GANs) are a class of AI algorithms used in unsupervised machine learning, implemented by a system of two neural networks contesting with each other in a zero-sum game framework. This paper reviews the foundational architectures of GANs, including DCGAN, CycleGAN, and StyleGAN, and explores their diverse applications in image synthesis, data augmentation, and anomaly detection.
-        Keywords: GANs, Generative Models, Image Synthesis, Anomaly Detection, Neural Networks.
+        Retrieves relevant information from the knowledge base and generates an answer
+         using the LangChain RetrievalQA chain.
         """
-    ]
-    documents = [Document(page_content=text) for text in dummy_ai_papers_content]
-
-
-print(f"Jumlah dokumen awal: {len(documents)}")
-
-# --- 2. Pemisahan Teks (Text Splitting) ---
-# Memecah dokumen menjadi chunks yang lebih kecil untuk retrieval yang lebih baik.
-text_splitter = RecursiveCharacterTextSplitter(
-    chunk_size=1000,
-    chunk_overlap=200,
-    length_function=len,
-    add_start_index=True,
-)
-chunks = text_splitter.split_documents(documents)
-print(f"Jumlah chunks setelah pemisahan: {len(chunks)}")
-
-# --- 3. Embeddings ---
-# Menginisialisasi model embedding dari Hugging Face.
-# Ini akan mengunduh model jika belum ada.
-embedding_model_name = "sentence-transformers/all-MiniLM-L6-v2"
-embeddings = HuggingFaceEmbeddings(model_name=embedding_model_name)
-
-# --- 4. Vector Store (FAISS) ---
-# Membuat indeks FAISS dari chunks dan embeddings.
-print("Membuat indeks FAISS... Ini mungkin butuh waktu tergantung ukuran data.")
-vectorstore = FAISS.from_documents(chunks, embeddings)
-print("Indeks FAISS berhasil dibuat.")
-
-# --- 5. Inisialisasi LLM ---
-# Menggunakan HuggingFaceHub untuk LLM.
-# Anda bisa menggantinya dengan model lokal atau API LLM lainnya jika diinginkan.
-if "HUGGINGFACEHUB_API_TOKEN" in os.environ:
-    llm = HuggingFaceHub(
-        repo_id="google/gemma-7b-it",  # Contoh model yang bagus
-        model_kwargs={"temperature": 0.1, "max_length": 512}
+        if not query.strip():
+            return "Please enter a question."
+        if not self.is_initialized or self.qa_chain is None:
+            return "Knowledge base not loaded. Please initialize it by providing an arXiv query."
+
+        print(f"\n--- Querying LLM with LangChain QA Chain ---\nQuestion: {query}\n----------------------")
+
+        try:
+            # Use the RetrievalQA chain to get the answer
+            result = self.qa_chain.invoke({"query": query})
+            answer = result["result"].strip()
+        except Exception as e:
+            print(f"Error during generation: {e}")
+            answer = "I apologize, but I encountered an error while generating the answer. Please try again or rephrase your question."
+
+        return answer
+
+# --- Gradio Interface ---
+
+# Instantiate the RAGAgent
+rag_agent_instance = RAGAgent()
+
+print("Setting up Gradio interface...")
+
+with gr.Blocks() as demo:
+    gr.Markdown("# 📚 Educational RAG Agent with arXiv Knowledge Base (LangChain)")
+    gr.Markdown("First, load a knowledge base by specifying an arXiv search query. Then, ask questions!")
+
+    with gr.Row():
+        arxiv_input = gr.Textbox(
+            label="arXiv Search Query (e.g., 'Large Language Models', 'Reinforcement Learning')",
+            placeholder="Enter a topic to search for papers on arXiv...",
+            lines=1
+        )
+        max_papers_slider = gr.Slider(
+            minimum=1,
+            maximum=10,
+            step=1,
+            value=3,
+            label="Max Papers to Download"
+        )
+        load_kb_button = gr.Button("Load Knowledge Base from arXiv")
+    
+    kb_status_output = gr.Textbox(label="Knowledge Base Status", interactive=False)
+
+    with gr.Row():
+        question_input = gr.Textbox(
+            lines=3,
+            placeholder="Ask a question based on the loaded arXiv papers...",
+            label="Your Question"
+        )
+        answer_output = gr.Textbox(label="Answer", lines=7, interactive=False)
+    
+    submit_button = gr.Button("Get Answer")
+
+    load_kb_button.click(
+        fn=rag_agent_instance.initialize_knowledge_base, # Call method of instance
+        inputs=[arxiv_input, max_papers_slider],
+        outputs=kb_status_output
     )
-else:
-    # Placeholder LLM jika token tidak disetel
-    class DummyLLM:
-        def invoke(self, prompt, **kwargs):
-            return "Maaf, LLM tidak dapat diinisialisasi karena HUGGINGFACEHUB_API_TOKEN tidak disetel. Silakan setel token Anda."
-        def __call__(self, prompt, **kwargs):
-            return self.invoke(prompt, **kwargs)
-    llm = DummyLLM()
-    print("Menggunakan DummyLLM karena HUGGINGFACEHUB_API_TOKEN tidak disetel.")
-
-
-# --- 6. Membuat Chain RAG (RetrievalQA) ---
-# Chain ini akan mengambil dokumen yang relevan dan meneruskannya ke LLM untuk menghasilkan jawaban.
-qa_chain = RetrievalQA.from_chain_type(
-    llm=llm,
-    chain_type="stuff",  # "stuff" menggabungkan semua dokumen ke dalam satu prompt
-    retriever=vectorstore.as_retriever(),
-    return_source_documents=True, # Untuk melihat dokumen sumber yang diambil
-)
-
-# --- 7. Membuat Agentic RAG Logic ---
-# Kita akan membuat "tool" yang membungkus chain RAG kita.
-# Agent kemudian dapat memutuskan kapan harus menggunakan tool ini.
-tool_description = (
-    "Berguna untuk menjawab pertanyaan tentang paper Artificial Intelligence, "
-    "termasuk konsep, metodologi, tantangan, dan aplikasi yang dibahas dalam paper."
-    "Gunakan tool ini untuk mengambil informasi dari basis pengetahuan paper AI."
-)
-
-tools = [
-    Tool(
-        name="AI_Paper_Retriever",
-        func=qa_chain.invoke, # Menggunakan .invoke() untuk Langchain Expression Language
-        description=tool_description,
-        return_direct=False # Agent akan memproses output tool ini
+
+    submit_button.click(
+        fn=rag_agent_instance.query_agent, # Call method of instance
+        inputs=question_input,
+        outputs=answer_output
     )
-]
-
-# Prompt untuk Agent
-# Agent akan memutuskan tool mana yang akan digunakan berdasarkan prompt ini.
-# Kita menggunakan ReAct (Reasoning and Acting) prompt.
-agent_prompt = ChatPromptTemplate.from_messages(
-    [
-        SystemMessage(
-            content=(
-                "Anda adalah asisten AI yang sangat membantu dan berpengetahuan luas, "
-                "khususnya dalam bidang Artificial Intelligence dan paper penelitian terkait."
-                "Tugas utama Anda adalah membantu pengguna memahami dan menavigasi informasi dari paper AI."
-                "Gunakan tool 'AI_Paper_Retriever' jika pertanyaan pengguna berkaitan dengan konten paper AI "
-                "atau memerlukan informasi spesifik dari basis pengetahuan Anda."
-                "Jika pertanyaan bersifat umum atau tidak memerlukan pengambilan data, jawablah langsung."
-                "Selalu berikan jawaban yang komprehensif dan relevan."
-            )
-        ),
-        MessagesPlaceholder(variable_name="chat_history"),
-        ("human", "{input}"),
-        MessagesPlaceholder(variable_name="tools"), 
-        MessagesPlaceholder(variable_name="agent_scratchpad"),
-        MessagesPlaceholder(variable_name="tool_names"),
-    ]
-)
-
-# Membuat Agent
-# Menggunakan create_react_agent untuk agent yang berbasis ReAct
-agent = create_react_agent(llm, tools, agent_prompt)
-
-# Membuat Agent Executor
-# Ini adalah runtime untuk agent, yang menjalankan loop pemikiran-aksi.
-agent_executor = AgentExecutor(
-    agent=agent,
-    tools=tools,
-    verbose=True, # Untuk melihat langkah-langkah pemikiran agent
-    handle_parsing_errors=True, # Menangani kesalahan parsing agent
-    max_iterations=5 # Batasi iterasi untuk menghindari loop tak terbatas
-)
-
-# --- 8. Gradio Interface ---
-# Fungsi untuk memproses pertanyaan pengguna menggunakan agent
-def process_query(query, chat_history_tuples):
-    # Mengubah riwayat chat dari Gradio ke format Langchain
-    from langchain_core.messages import HumanMessage, AIMessage
-    formatted_chat_history = []
-    for human_msg, ai_msg in chat_history_tuples:
-        formatted_chat_history.append(HumanMessage(content=human_msg))
-        formatted_chat_history.append(AIMessage(content=ai_msg))
-
-    try:
-        # Panggil agent executor
-        response = agent_executor.invoke({
-            "input": query,
-            "chat_history": formatted_chat_history
-        })
-        answer = response["output"]
-
-        # Jika agent menggunakan tool RAG, kita bisa menampilkan sumbernya juga
-        # Ini memerlukan sedikit modifikasi jika Anda ingin menampilkan sumber
-        # secara eksplisit dari dalam agent_executor.invoke().
-        # Untuk kesederhanaan, kita hanya akan menampilkan jawaban agent.
-        # Jika Anda ingin menampilkan sumber, Anda perlu memodifikasi tool
-        # atau agent_executor untuk mengembalikan informasi sumber secara eksplisit.
-
-    except Exception as e:
-        answer = f"Terjadi kesalahan saat memproses pertanyaan: {e}"
-        if "HUGGINGFACEHUB_API_TOKEN" not in os.environ:
-             answer += "\nPastikan HUGGINGFACEHUB_API_TOKEN Anda disetel dengan benar."
-
-    return answer
-
-# Membuat antarmuka Gradio
-with gr.Blocks() as demo:
-    gr.Markdown("# Agentic RAG untuk Literasi Paper AI")
-    gr.Markdown("Tanyakan apa pun tentang paper AI yang diambil dari arXiv.")
-
-    chatbot = gr.Chatbot(label="Percakapan")
-    msg = gr.Textbox(label="Pertanyaan Anda")
-    clear = gr.Button("Clear")
-
-    def user_message(user_message, history):
-        return "", history + [[user_message, None]]
-
-    def bot_response(history):
-        query = history[-1][0]
-        # Mengirimkan riwayat chat tanpa respons bot terakhir yang masih None
-        chat_history_for_agent = history[:-1]
-        response = process_query(query, chat_history_for_agent)
-        history[-1][1] = response
-        return history
-
-    msg.submit(user_message, [msg, chatbot], [msg, chatbot], queue=False).then(
-        bot_response, chatbot, chatbot
+
+    gr.Examples(
+        examples=[
+            ["What is the transformer architecture?"],
+            ["Explain attention mechanisms in deep learning."],
+            ["What are the challenges in reinforcement learning?"],
+        ],
+        inputs=question_input
     )
-    clear.click(lambda: None, None, chatbot, queue=False)
 
-# Jalankan aplikasi Gradio
-# Untuk menjalankan di Hugging Face Spaces, Anda mungkin perlu mengatur share=True
-# atau cukup jalankan tanpa share=True jika sudah di dalam lingkungan Space.
+# Launch the Gradio app
 if __name__ == "__main__":
-    print("\nAplikasi Gradio siap dijalankan. Buka URL yang diberikan setelah 'Running on local URL:'.")
-    print("Jika Anda melihat pesan peringatan tentang HUGGINGFACEHUB_API_TOKEN, pastikan Anda menyetelnya.")
-    demo.launch(debug=True) # debug=True untuk melihat log di konsol
+    print("Launching Gradio app...")
+    demo.launch(share=False)
+```
+```text

requirements.txt

@@ -1,11 +1,10 @@
 huggingface_hub==0.25.2
-datasets
+gradio
 transformers
-torch
-langchain
+sentence-transformers
 faiss-cpu
-gradio
-huggingface_hub
+torch
 arxiv
-langchain_community
-sentence-transformers
+pypdf2
+langchain
+langchain-community