Fix TDocs downloading not working

app.py

@@ -1,20 +1,24 @@
-from typing import Literal
-from bs4 import BeautifulSoup
+import asyncio
+import logging
+import nltk
+import string
 import warnings
 import io
+import traceback
 import zipfile
-from lxml import etree
+import json
 import os
-from dotenv import load_dotenv
 import requests
 import subprocess
-import string
+import pandas as pd
+import re
+from lxml import etree
+from typing import Literal
+from dotenv import load_dotenv
 from nltk.tokenize import word_tokenize
+from bs4 import BeautifulSoup
 from nltk.corpus import stopwords
 from nltk.stem import WordNetLemmatizer
-from concurrent.futures import ThreadPoolExecutor, as_completed
-import json
-import traceback
 from fastapi import FastAPI, BackgroundTasks, HTTPException, Request
 from fastapi.staticfiles import StaticFiles
 from schemas import *
@@ -22,11 +26,6 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse, StreamingResponse
 from litellm.router import Router
 from aiolimiter import AsyncLimiter
-import pandas as pd
-import asyncio
-import logging
-import re
-import nltk
 
 load_dotenv()
 
@@ -36,6 +35,7 @@ logging.basicConfig(
     datefmt='%Y-%m-%d %H:%M:%S'
 )
 
+# Download required packages for NLTK
 nltk.download('stopwords')
 nltk.download('punkt_tab')
 nltk.download('wordnet')
@@ -227,11 +227,14 @@ def get_meetings(req: MeetingsRequest):
     working_group = req.working_group
     tsg = re.sub(r"\d+", "", working_group)
     wg_number = re.search(r"\d", working_group).group(0)
+
     logging.debug(tsg, wg_number)
     url = "https://www.3gpp.org/ftp/tsg_" + tsg
     logging.debug(url)
+
     resp = requests.get(url, verify=False)
     soup = BeautifulSoup(resp.text, "html.parser")
+
     meeting_folders = []
     all_meetings = []
     wg_folders = [item.get_text() for item in soup.select("tr td a")]
@@ -309,15 +312,18 @@ def download_tdocs(req: DownloadRequest):
             data=json.dumps({"doc_id": doc_id}),
             verify=False
         )
-        print(url.status_code)
+        logging.info(
+            f"Retrieving URL for doc {doc_id} returned http status {url.status_code}")
         url = url.json()['url']
-        print(url)
+        logging.debug(f"Doc URL for {doc_id} is {url}")
+
         try:
             txt = "\n".join(docx_to_txt(doc_id, url))
         except Exception as e:
             txt = f"Document {doc_id} text extraction failed: {e}"
         return doc_id, txt.encode("utf-8")
 
+    # PERF: use asyncio?
     def process_batch(batch):
         results = {}
         for doc in batch:
@@ -420,8 +426,6 @@ async def gen_reqs(req: RequirementsRequest, background_tasks: BackgroundTasks):
 
 # ======================================================================================================================================================================================
 
-SUBPROCESS_SEMAPHORE = asyncio.Semaphore(32)
-
 
 class ProgressUpdate(BaseModel):
     """Defines the structure of a single SSE message."""
@@ -431,7 +435,7 @@ class ProgressUpdate(BaseModel):
     processed_docs: int
 
 
-@app.post("/generate_requirements/v2")
+@app.post("/generate_requirements/sse")
 async def gen_reqs(req: RequirementsRequest, con: Request):
     """Extract requirements from the specified TDocs using a LLM and returns SSE events about the progress of ongoing operations"""
 

static/script.js

@@ -520,7 +520,7 @@ async function downloadTDocs() {
         const blob = await response.blob();
         downloadBlob(blob, generateDownloadFilename());
     } catch (error) {
-        console.error('Erreur lors du téléchargement:', error);
+        console.error(error);
         alert('Erreur lors du téléchargement des TDocs');
     } finally {
         hideLoadingOverlay();
@@ -535,19 +535,29 @@ async function downloadTDocs() {
 function generateDownloadFilename() {
     let filename = document.getElementById('meeting-select').value || 'documents';
 
-    const agendaItem = document.getElementById('agenda-item-filter').value;
-    const docStatus = document.getElementById('doc-status-filter').value;
-    const docType = document.getElementById('doc-type-filter').value;
+    const agendaItems = selectedAgenda;
+    const docStatuses = selectedStatus
+    const docType = selectedType;
 
-    if (agendaItem && agendaItem !== 'Tous') {
-        filename += `_${agendaItem}`;
+    // empty set means "Tous" is selected
+    if (agendaItems) {
+        for (aItem of agendaItems) {
+            filename += `_${aItem}`;
+        }
     }
-    if (docStatus && docStatus !== 'Tous') {
-        filename += `_${docStatus}`;
+
+    // empty set means "Tous" is selected
+    if (docStatuses) {
+        for (docStatus of docStatuses) {
+            filename += `_${docStatus}`;
+        }
     }
-    if (docType && docType !== 'Tous') {
+
+    // empty means "Tous"
+    if (docType && docType !== "") {
         filename = `${docType}_${filename}`;
     }
+
     if (hasRequirementsExtracted) {
         filename = `requirements_${filename}`;
     }
@@ -585,7 +595,7 @@ async function extractRequirements() {
     toggleElementsEnabled(['extract-requirements-btn'], false);
 
     try {
-        const response = await postWithSSE('/generate_requirements/v2', { documents: selectedData }, {
+        const response = await postWithSSE('/generate_requirements/sse', { documents: selectedData }, {
             onMessage: (msg) => {
                 console.log("SSE message:");
                 console.log(msg);
@@ -599,11 +609,11 @@ async function extractRequirements() {
         });
 
 
-    //     const response = await fetch('/generate_requirements/', {
-    //         method: 'POST',
-    //         headers: { 'Content-Type': 'application/json' },
-    //         body: req
-    //     });
+        //     const response = await fetch('/generate_requirements/', {
+        //         method: 'POST',
+        //         headers: { 'Content-Type': 'application/json' },
+        //         body: req
+        //     });
 
         const data = response.data; // data in the SSE message contains the requirements response
         requirements = data.requirements;
@@ -619,7 +629,7 @@ async function extractRequirements() {
                 req_id++;
             })
         })
-        
+
         displayRequirements(requirements);
 
         toggleContainersVisibility(['requirements-container', 'query-requirements-container'], true);