Update app.py

app.py

@@ -5,6 +5,7 @@ import inspect
 import pandas as pd
 import logging
 from smolagents import CodeAgent, DuckDuckGoSearchTool, HfApiModel, OpenAIServerModel
+from pathlib import Path
 
 from llama_index.core import (
     VectorStoreIndex,
@@ -49,7 +50,7 @@ class BasicAgent:
         # Prepara il query engine
         self.query_engine = self.index.as_query_engine()
         print("Agent ready.")
-
+'''
     def __call__(self, question: str) -> str:
         print(f"Received question: {question[:50]}...")
         response = self.query_engine.query(question)
@@ -59,6 +60,83 @@ class BasicAgent:
         print("Response text:", str(response))
         
         return str(response)
+'''
+
+    def __call__(self, question: str, context=None) -> str:
+        if isinstance(context, str):
+            #prompt = f"{context}\n\nDomanda: {question}"
+            response = self.query_engine.query(question)
+            # Stampa ragionamento interno
+            print("Query response object:", response)
+            print("Response text:", str(response))
+            return response.text
+
+        elif context and hasattr(context, "read"):  # file immagine o audio
+            file_bytes = context.read()
+            filename = context.name
+
+            if filename.endswith((".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif")):
+                print("coso entrato in video file:", filename)
+                image = Image.open(io.BytesIO(file_bytes))
+                response = self.query_engine.query(
+                    messages=[
+                        {"role": "user", "content": [
+                            {"type": "text", "text": question},
+                            {"type": "image_url", "image_url": image}
+                        ]}
+                    ]
+                )
+                return response.choices[0].message.content
+
+            elif filename.endswith((".mp3", ".wav", ".ogg")):
+                print("coso entrato in audio file:", filename)
+                response = self.query_engine.query(
+                    messages=[
+                        {"role": "user", "content": [
+                            {"type": "text", "text": question},
+                            {"type": "audio_url", "audio_url": file_bytes}
+                        ]}
+                    ]
+                )
+                return response.choices[0].message.content
+
+        else:
+            print("coso entrato in else")
+            response = self.llm.complete(question)
+            return response.text
+
+
+
+def answer_with_media(question, uploaded_file):
+    file_content = ""
+    file_path = None
+
+    if uploaded_file is not None:
+        file_path = Path(uploaded_file.name)
+        suffix = file_path.suffix.lower()
+
+        if suffix in [".txt", ".md", ".csv"]:
+            with open(file_path, "r", encoding="utf-8") as f:
+                file_content = f.read()
+        elif suffix in [".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif", ".mp3", ".wav", ".ogg"]:
+            # immagini/audio li gestiamo come path da passare al modello
+            file_content = uploaded_file
+        else:
+            return "Formato file non supportato."
+
+    response = agent(question, context=file_content)
+    return response
+
+gr.Interface(
+    fn=answer_with_media,
+    inputs=[
+        gr.Textbox(label="Domanda"),
+        gr.File(label="Carica file (testo, immagine o audio)")
+    ],
+    outputs=gr.Textbox(label="Risposta")
+).launch()
+
+