All models and query

Dockerfile

@@ -1,13 +1,32 @@
 FROM python:3.12-slim
 
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    libpq-dev \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Add a non-root user
 RUN useradd -m -u 1000 user
 USER user
 ENV PATH="/home/user/.local/bin:$PATH"
 
 WORKDIR /app
 
+# Copy and install Python dependencies
 COPY --chown=user ./requirements.txt requirements.txt
-RUN pip install --no-cache-dir --upgrade -r requirements.txt
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Download Spacy model
+RUN python -m spacy download pt_core_news_md
 
+# Copy application code
 COPY --chown=user . /app
+
+# Expose the application port
+EXPOSE 7860
+
+# Run the application
 CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -1,25 +1,42 @@
-# main.py
-import logging
 from fastapi import FastAPI, HTTPException, BackgroundTasks
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
-from typing import Dict, Optional
+from typing import Dict, Optional, List
 import uuid
-from datetime import datetime, timedelta
-import asyncio
-import random
-# from sentence_transformers import SentenceTransformer
-# from transformers import T5Tokenizer, T5ForConditionalGeneration
-# from LexRank import degree_centrality_scores
-# import torch
-# import nltk
-# import spacy
-# from psycopg2 import sql
+from datetime import datetime
+from contextlib import asynccontextmanager
 
+from models.embedding import EmbeddingModel
+from models.summarization import SummarizationModel
+from models.nlp import NLPModel
+from database.query import DatabaseService
+from database.query_processor import QueryProcessor
 
-app = FastAPI(title="Kairos News API", version="1.0")
 
-# Enable CORS
+# Initialize models
+embedding_model = None
+summarization_model = None
+nlp_model = None
+db_service = None
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Load models when app starts
+    global embedding_model, summarization_model, nlp_model, db_service
+    embedding_model = EmbeddingModel()
+    summarization_model = SummarizationModel()
+    nlp_model = NLPModel()
+    db_service = DatabaseService()
+    yield
+    # Clean up when app stops
+    await db_service.close()
+
+app = FastAPI(
+    title="Kairos News API",
+    version="1.0",
+    lifespan=lifespan
+)
+
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -27,13 +44,24 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-# In-memory database simulation
+# In-memory job storage
 jobs_db: Dict[str, Dict] = {}
 
 class PostRequest(BaseModel):
     query: str
+    topic: Optional[str] = None
+    start_date: Optional[str] = None  # Format: "YYYY-MM-DD"
+    end_date: Optional[str] = None    # Format: "YYYY-MM-DD"
+
+class ArticleResult(BaseModel):
+    url: str
+    content: str
+    distance: float
+    date: str
     topic: str
-    date: str  # Format: "YYYY/MM to YYYY/MM"
+
+class SummaryResult(BaseModel):
+    summary: str
 
 class JobStatus(BaseModel):
     id: str
@@ -45,10 +73,8 @@ class JobStatus(BaseModel):
 
 @app.post("/index", response_model=JobStatus)
 async def create_job(request: PostRequest, background_tasks: BackgroundTasks):
-    """Create a new processing job"""
     job_id = str(uuid.uuid4())
     
-    # Store initial job data
     jobs_db[job_id] = {
         "status": "processing",
         "created_at": datetime.now(),
@@ -56,9 +82,16 @@ async def create_job(request: PostRequest, background_tasks: BackgroundTasks):
         "request": request.dict(),
         "result": None
     }
-    logging.info(f"Job {job_id} created with request: {request.query}")
-    # Simulate background processing
-    background_tasks.add_task(process_job, job_id)
+
+    background_tasks.add_task(
+        process_job,
+        job_id,
+        request,
+        embedding_model,
+        summarization_model,
+        nlp_model,
+        db_service
+    )
     
     return {
         "id": job_id,
@@ -71,48 +104,42 @@ async def create_job(request: PostRequest, background_tasks: BackgroundTasks):
 
 @app.get("/loading", response_model=JobStatus)
 async def get_job_status(id: str):
-    """Check job status with timeout simulation"""
     if id not in jobs_db:
         raise HTTPException(status_code=404, detail="Job not found")
     
-    job = jobs_db[id]
-    
-    # Simulate random processing time (3-25 seconds)
-    elapsed = datetime.now() - job["created_at"]
-    if elapsed < timedelta(seconds=3):
-        await asyncio.sleep(1)  # Artificial delay
-        
-    # 10% chance of failure for demonstration
-    if random.random() < 0.1 and job["status"] == "processing":
-        job["status"] = "failed"
-        job["result"] = {"error": "Random processing failure"}
-    
-    return {
-        "id": id,
-        "status": job["status"],
-        "created_at": job["created_at"],
-        "completed_at": job["completed_at"],
-        "request": job["request"],
-        "result": job["result"]
-    }
+    return jobs_db[id]
 
-async def process_job(job_id: str):
-    """Background task to simulate processing"""
-    await asyncio.sleep(random.uniform(3, 10))  # Random processing time
-    
-    if job_id in jobs_db:
-        jobs_db[job_id]["status"] = "completed"
-        jobs_db[job_id]["completed_at"] = datetime.now()
-        jobs_db[job_id]["result"] = {
-            "query": jobs_db[job_id]["request"]["query"],
-            "topic": jobs_db[job_id]["request"]["topic"],
-            "date_range": jobs_db[job_id]["request"]["date"],
-            "analysis": f"Processed results for {jobs_db[job_id]['request']['query']}",
-            "sources": ["Source A", "Source B", "Source C"],
-            "summary": "This is a generated summary based on your query."
-        }
-
-@app.get("/jobs")
-async def list_jobs():
-    """Debug endpoint to view all jobs"""
-    return jobs_db
+async def process_job(
+    job_id: str,
+    request: PostRequest,
+    embedding_model: EmbeddingModel,
+    summarization_model: SummarizationModel,
+    nlp_model: NLPModel,
+    db_service: DatabaseService
+):
+    try:
+        processor = QueryProcessor(
+            embedding_model=embedding_model,
+            summarization_model=summarization_model,
+            nlp_model=nlp_model,
+            db_service=db_service
+        )
+        
+        result = await processor.process(
+            query=request.query,
+            topic=request.topic,
+            start_date=request.start_date,
+            end_date=request.end_date
+        )
+        
+        jobs_db[job_id].update({
+            "status": "completed",
+            "completed_at": datetime.now(),
+            "result": result
+        })
+    except Exception as e:
+        jobs_db[job_id].update({
+            "status": "failed",
+            "completed_at": datetime.now(),
+            "result": {"error": str(e)}
+        })

app_antiga.py

@@ -0,0 +1,136 @@
+# main.py
+import logging
+from fastapi import FastAPI, HTTPException, BackgroundTasks
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import Dict, Optional
+import uuid
+from datetime import datetime, timedelta
+import asyncio
+import random
+from sentence_transformers import SentenceTransformer
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+from models.LexRank import degree_centrality_scores
+import torch
+import nltk
+import spacy
+from psycopg2 import sql
+
+
+
+app = FastAPI(title="Kairos News API", version="1.0")
+
+# Enable CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+#Database connection setup
+url = "https://daxquaudqidyeirypexa.supabase.co"
+key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImRheHF1YXVkcWlkeWVpcnlwZXhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzOTIzNzcsImV4cCI6MjA1OTk2ODM3N30.3qB-GfiCoqXEpbNfqV3iHiqOLr8Ex9nPVr6p9De5Hdc"
+opts = ClientOptions().replace(schema="articles")
+supabase = create_client(url, key, options=opts)
+
+# Loading models
+nlp = spacy.load("pt_core_news_md")
+model_embedding = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
+token_name = 'unicamp-dl/ptt5-base-portuguese-vocab'
+model_name = 'recogna-nlp/ptt5-base-summ'
+tokenizer = T5Tokenizer.from_pretrained(token_name)
+model_summ = T5ForConditionalGeneration.from_pretrained(model_name).to('cuda')
+
+# In-memory database simulation
+jobs_db: Dict[str, Dict] = {}
+
+class PostRequest(BaseModel):
+    query: str
+    topic: str
+    start_date: str  # Format: "YYYY/MM to YYYY/MM"
+    end_date: str  # Format: "YYYY/MM to YYYY/MM"
+
+class JobStatus(BaseModel):
+    id: str
+    status: str  # "processing", "completed", "failed"
+    created_at: datetime
+    completed_at: Optional[datetime]
+    request: PostRequest
+    result: Optional[Dict]
+
+@app.post("/index", response_model=JobStatus)
+async def create_job(request: PostRequest, background_tasks: BackgroundTasks):
+    """Create a new processing job"""
+    job_id = str(uuid.uuid4())
+    
+    # Store initial job data
+    jobs_db[job_id] = {
+        "status": "processing",
+        "created_at": datetime.now(),
+        "completed_at": None,
+        "request": request.dict(),
+        "result": None
+    }
+
+    logging.info(f"Job {job_id} created with request: {request.query}")
+    # Simulate background processing
+    background_tasks.add_task(process_job, job_id)
+    
+    return {
+        "id": job_id,
+        "status": "processing",
+        "created_at": jobs_db[job_id]["created_at"],
+        "completed_at": None,
+        "request": request,
+        "result": None
+    }
+
+
+@app.get("/loading", response_model=JobStatus)
+async def get_job_status(id: str):
+    """Check job status with timeout simulation"""
+    if id not in jobs_db:
+        raise HTTPException(status_code=404, detail="Job not found")
+    
+    job = jobs_db[id]
+    
+    # Simulate random processing time (3-25 seconds)
+    elapsed = datetime.now() - job["created_at"]
+    if elapsed < timedelta(seconds=3):
+        await asyncio.sleep(1)  # Artificial delay
+        
+    # 10% chance of failure for demonstration
+    if random.random() < 0.1 and job["status"] == "processing":
+        job["status"] = "failed"
+        job["result"] = {"error": "Random processing failure"}
+    
+    return {
+        "id": id,
+        "status": job["status"],
+        "created_at": job["created_at"],
+        "completed_at": job["completed_at"],
+        "request": job["request"],
+        "result": job["result"]
+    }
+
+async def process_job(job_id: str):
+    """Background task to simulate processing"""
+    await asyncio.sleep(random.uniform(3, 10))  # Random processing time
+    
+    if job_id in jobs_db:
+        jobs_db[job_id]["status"] = "completed"
+        jobs_db[job_id]["completed_at"] = datetime.now()
+        jobs_db[job_id]["result"] = {
+            "query": jobs_db[job_id]["request"]["query"],
+            "topic": jobs_db[job_id]["request"]["topic"],
+            "date_range": jobs_db[job_id]["request"]["date"],
+            "analysis": f"Processed results for {jobs_db[job_id]['request']['query']}",
+            "sources": ["Source A", "Source B", "Source C"],
+            "summary": "This is a generated summary based on your query."
+        }
+
+@app.get("/jobs")
+async def list_jobs():
+    """Debug endpoint to view all jobs"""
+    return jobs_db

database/query.py

@@ -0,0 +1,90 @@
+import os
+from typing import List, Dict, Optional
+import vecs
+from datetime import datetime
+
+class DatabaseService:
+    def __init__(self):
+        # Connection parameters
+        self.DB_HOST = os.getenv("SUPABASE_HOST", "db.daxquaudqidyeirypexa.supabase.co")
+        self.DB_PORT = os.getenv("DB_PORT", "5432")
+        self.DB_NAME = os.getenv("DB_NAME", "postgres")
+        self.DB_USER = os.getenv("DB_USER", "postgres")
+        self.DB_PASSWORD = os.getenv("DB_PASSWORD", "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImRheHF1YXVkcWlkeWVpcnlwZXhhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDQzOTIzNzcsImV4cCI6MjA1OTk2ODM3N30.3qB-GfiCoqXEpbNfqV3iHiqOLr8Ex9nPVr6p9De5Hdc")
+        
+        # Create vecs client
+        self.vx = vecs.create_client(
+            f"postgresql://{self.DB_USER}:{self.DB_PASSWORD}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_NAME}"
+        )
+        
+        # Get or create the collection
+        self.articles = self.vx.get_or_create_collection(
+            name="articles",
+            dimension=384  # Match your embedding model's output dimension
+        )
+
+    async def semantic_search(
+        self,
+        query_embedding: List[float],
+        start_date: Optional[datetime] = None,
+        end_date: Optional[datetime] = None,
+        topic: Optional[str] = None,
+        entities: Optional[List[str]] = None,  # Add entities parameter
+        limit: int = 10
+    ) -> List[Dict[str, any]]:
+        try:
+            # Base vector search
+            filters = self._build_filters(start_date, end_date, topic)
+            
+            # Add entity filter if entities are provided
+            if entities:
+                filters["entities"] = {"$in": entities}
+            
+            results = self.articles.query(
+                data=query_embedding,
+                limit=limit,
+                filters=filters,
+                measure="cosine_distance"  # or "inner_product", "l2_distance"
+            )
+            
+            # Format results with metadata
+            formatted_results = []-
+            for article_id, distance in results:
+                metadata = self.articles.fetch(ids=[article_id])[0]["metadata"]
+                formatted_results.append({
+                    "id": article_id,
+                    "url": metadata.get("url"),
+                    "content": metadata.get("content"),
+                    "date": metadata.get("date"),
+                    "topic": metadata.get("topic"),
+                    "distance": float(distance),
+                    "similarity": 1 - float(distance)  # Convert to similarity score
+                })
+            
+            return formatted_results
+        
+        except Exception as e:
+            print(f"Vector search error: {e}")
+            return []
+
+    def _build_filters(
+        self,
+        start_date: Optional[datetime],
+        end_date: Optional[datetime],
+        topic: Optional[str]
+    ) -> Dict[str, any]:
+        filters = {}
+        
+        if start_date and end_date:
+            filters["date"] = {
+                "$gte": start_date.isoformat(),
+                "$lte": end_date.isoformat()
+            }
+        
+        if topic:
+            filters["topic"] = {"$eq": topic}
+            
+        return filters
+
+    async def close(self):
+        self.vx.disconnect()

database/query_processor.py

@@ -0,0 +1,67 @@
+from typing import List, Dict, Any
+import numpy as np
+from LexRank import degree_centrality_scores
+
+class QueryProcessor:
+    def __init__(self, embedding_model, summarization_model, nlp_model, db_service):
+        self.embedding_model = embedding_model
+        self.summarization_model = summarization_model
+        self.nlp_model = nlp_model
+        self.db_service = db_service
+    
+    async def process(
+        self,
+        query: str,
+        topic: Optional[str] = None,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None
+    ) -> Dict[str, Any]:
+        # Convert string dates to datetime objects
+        start_dt = datetime.strptime(start_date, "%Y-%m-%d") if start_date else None
+        end_dt = datetime.strptime(end_date, "%Y-%m-%d") if end_date else None
+        
+        # Get query embedding
+        query_embedding = self.embedding_model.encode(query).tolist()
+
+        # Get entities from the query
+        doc = self.nlp_model(query)
+        entities = [ent.text.lower() for ent in doc.ents]  # Extract entity texts
+
+        # Semantic search with entities
+        articles = await self.db_service.semantic_search(
+            query_embedding=query_embedding,
+            start_date=start_dt,
+            end_date=end_dt,
+            topic=topic,
+            entities=entities  # Pass entities to the search
+        )
+        
+        if not articles:
+            return {"error": "No articles found matching the criteria"}
+        
+        # Step 3: Process results
+        contents = [article["content"] for article in articles]
+        sentences = []
+        for content in contents:
+            sentences.extend(self.nlp_model.tokenize_sentences(content))
+        
+        # Step 4: Generate summary
+        if sentences:
+            embeddings = self.embedding_model.encode(sentences)
+            similarity_matrix = np.inner(embeddings, embeddings)
+            centrality_scores = degree_centrality_scores(similarity_matrix, threshold=None)
+            
+            top_indices = np.argsort(-centrality_scores)[0:10]
+            key_sentences = [sentences[idx].strip() for idx in top_indices]
+            combined_text = ' '.join(key_sentences)
+            
+            summary = self.summarization_model.summarize(combined_text)
+        else:
+            key_sentences = []
+            summary = "No content available for summarization"
+        
+        return {
+            "summary": summary,
+            "key_sentences": key_sentences,
+            "articles": articles
+        }

models/embedding.py

@@ -0,0 +1,10 @@
+from sentence_transformers import SentenceTransformer
+import torch
+
+class EmbeddingModel:
+    def __init__(self):
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
+    
+    def encode(self, text: str):
+        return self.model.encode(text, device=self.device)

models/nlp.py

@@ -0,0 +1,14 @@
+import spacy
+import nltk
+
+class NLPModel:
+    def __init__(self):
+        self.nlp = spacy.load("pt_core_news_md")
+        nltk.download('punkt')
+    
+    def extract_entities(self, text: str):
+        doc = self.nlp(text)
+        return [(ent.text.lower(), ent.label_) for ent in doc.ents]
+    
+    def tokenize_sentences(self, text: str):
+        return nltk.sent_tokenize(text)

models/summarization.py

@@ -0,0 +1,27 @@
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+import torch
+
+class SummarizationModel:
+    def __init__(self):
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.tokenizer = T5Tokenizer.from_pretrained('unicamp-dl/ptt5-base-portuguese-vocab')
+        self.model = T5ForConditionalGeneration.from_pretrained('recogna-nlp/ptt5-base-summ').to(self.device)
+    
+    def summarize(self, text: str, max_length: int = 256, min_length: int = 128) -> str:
+        inputs = self.tokenizer.encode(
+            text,
+            max_length=512,
+            truncation=True,
+            return_tensors='pt'
+        ).to(self.device)
+        
+        summary_ids = self.model.generate(
+            inputs,
+            max_length=max_length,
+            min_length=min_length,
+            num_beams=5,
+            no_repeat_ngram_size=3,
+            early_stopping=False
+        )
+        
+        return self.tokenizer.decode(summary_ids[0], skip_special_tokens=True)

requirements.txt

@@ -1,12 +1,13 @@
 fastapi
 uvicorn[standard]
 logging
-# transformers
-# torch
-# sentence_transformers
-# nltk
-# spacy
-# numpy
-# pandas
-# scipy
-# psycopg2
+transformers
+torch
+sentence_transformers
+nltk
+spacy
+numpy
+pandas
+scipy
+psycopg2
+vecs