Update Dockerfile to use new app entry point and enhance requirements.txt with additional dependencies. Remove obsolete streamlit_app.py file.

.env

@@ -0,0 +1,7 @@
+AZURE_DI_ENDPOINT=https://luisvalenciadi.cognitiveservices.azure.com/
+AZURE_DI_KEY=2MMHWymCHlykFwMGtaDHKGcbcWFpMfF29vqWKV6C584Zi9EJj5m4JQQJ99BEACfhMk5XJ3w3AAALACOGUYF3
+OPENAI_API_KEY=1bW5tUhsr8yHXQe0Pra6YdM43fWCUr06KtbGy9gKxVOd7ut5vL9GJQQJ99BDACfhMk5XJ3w3AAAAACOGTQtA
+AZURE_OPENAI_ENDPOINT=https://lvaaifoundry1567796533.openai.azure.com/
+AZURE_OPENAI_DEPLOYMENT=gpt-4.1
+AZURE_OPENAI_API_VERSION=2025-03-01-preview 
+AZURE_OPENAI_EMBEDDING_MODEL=text-embedding-3-small

.vscode/launch.json

@@ -0,0 +1,13 @@
+{
+    "configurations": [
+      {
+        "name": "Streamlit (debug)",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "streamlit",
+        "args": ["run", "src/app.py"],
+        "envFile": "${workspaceFolder}/.env",
+      }
+    ]
+  }
+  

Dockerfile

@@ -18,4 +18,4 @@ EXPOSE 8501
 
 HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
 
-ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"]
+ENTRYPOINT ["streamlit", "run", "src/app.py", "--server.port=8501", "--server.address=0.0.0.0"]

requirements.txt

@@ -1,3 +1,9 @@
 altair
 pandas
-streamlit
+streamlit
+pyyaml
+python-dotenv
+openai
+pydantic-settings
+PyMuPDF
+azure-ai-documentintelligence

src/README.md

@@ -0,0 +1,167 @@
+# Deep-Research PDF Field Extractor
+
+A powerful tool for extracting structured data from PDF documents, designed to handle various document types and extract specific fields of interest.
+
+## For End Users
+
+### Overview
+The PDF Field Extractor helps you extract specific information from PDF documents. It can extract any fields you specify, such as dates, names, values, locations, and more. The tool is particularly useful for converting unstructured PDF data into structured, analyzable formats.
+
+### How to Use
+
+1. **Upload Your PDF**
+   - Click the "Upload PDF" button
+   - Select your PDF file from your computer
+
+2. **Specify Fields to Extract**
+   - Enter the fields you want to extract, separated by commas
+   - Example: `Date, Name, Value, Location, Page, FileName`
+
+3. **Optional: Add Field Descriptions**
+   - You can provide additional context about the fields.
+   - This helps the system better understand what to look for
+
+4. **Run Extraction**
+   - Click the "Run extraction" button
+   - Wait for the process to complete
+   - View your results in a table format
+
+5. **Download Results**
+   - Download your extracted data as a CSV file
+   - View execution traces and logs if needed
+
+### Features
+- Automatic document type detection
+- Smart field extraction
+- Support for tables and text
+- Detailed execution traces
+- Downloadable results and logs
+
+## For Developers
+
+### Architecture Overview
+
+The application is built using a multi-agent architecture with the following components:
+
+#### Core Components
+
+1. **Planner (`orchestrator/planner.py`)**
+   - Generates execution plans using Azure OpenAI
+
+
+2. **Executor (`orchestrator/executor.py`)**
+   - Executes the generated plan
+   - Manages agent execution flow
+   - Handles context and result management
+
+3. **Agents**
+   - `PDFAgent`: Extracts text from PDFs
+   - `TableAgent`: Extracts tables from PDFs
+   - `FieldMapper`: Maps fields to values
+   - `ForEachField`: Control flow for field iteration
+
+### Agent Pipeline
+
+1. **Document Processing**
+   ```python
+   # Document is processed in stages:
+   1. PDF text extraction
+   2. Table extraction
+   3. Field mapping
+   ```
+
+2. **Field Extraction Process**
+   - Document type inference
+   - User profile determination
+   - Page-by-page scanning
+   - Value extraction and validation
+
+3. **Context Building**
+   - Document metadata
+   - Field descriptions
+   - User context
+   - Execution history
+
+### Key Features
+
+#### Document Type Inference
+The system automatically infers document type and user profile:
+```python
+# Example inference:
+"Document type: Analytical report
+User profile: Data analysts or researchers working with document analysis"
+```
+
+#### Field Mapping
+The FieldMapper agent uses a sophisticated approach:
+1. Document context analysis
+2. Page-by-page scanning
+3. Value extraction using LLM
+4. Result validation
+
+#### Execution Traces
+The system maintains detailed execution traces:
+- Tool execution history
+- Success/failure status
+- Detailed logs
+- Result storage
+
+### Technical Setup
+
+1. **Dependencies**
+   ```python
+   # Key dependencies:
+   - streamlit
+   - pandas
+   - PyMuPDF (fitz)
+   - Azure OpenAI
+   - Azure Document Intelligence
+   ```
+
+2. **Configuration**
+   - Environment variables for API keys
+   - Prompt templates in `config/prompts.yaml`
+   - Settings in `config/settings.py`
+
+3. **Logging System**
+   ```python
+   # Custom logging setup:
+   - LogCaptureHandler for UI display
+   - Structured logging format
+   - Execution history storage
+   ```
+
+### Development Guidelines
+
+1. **Adding New Agents**
+   - Inherit from base agent class
+   - Implement required methods
+   - Add to planner configuration
+
+2. **Modifying Extraction Logic**
+   - Update prompt templates
+   - Modify field mapping logic
+   - Adjust validation rules
+
+3. **Extending Functionality**
+   - Add new field types
+   - Implement custom validators
+   - Create new output formats
+
+### Testing
+- Unit tests for agents
+- Integration tests for pipeline
+- End-to-end testing with sample PDFs
+
+### Deployment
+- Streamlit app deployment
+- Environment configuration
+- API key management
+- Logging setup
+
+### Future Improvements
+- Enhanced error handling
+- Additional field types
+- Improved validation
+- Performance optimization
+- Extended documentation

src/agents/__init__.py

@@ -0,0 +1 @@
+ 

src/agents/base_agent.py

@@ -0,0 +1,8 @@
+"""Abstract base class for every agent."""
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+
+class BaseAgent(ABC):
+    @abstractmethod
+    def execute(self, ctx: Dict[str, Any]):
+        """Mutate / consume ctx and return a value."""

src/agents/confidence_scorer.py

@@ -0,0 +1,6 @@
+from typing import Dict, Any
+from .base_agent import BaseAgent
+
+class ConfidenceScorer(BaseAgent):
+    def execute(self, ctx: Dict[str, Any]):
+        return 1.0  # always confident for stub

src/agents/field_mapper_agent.py

@@ -0,0 +1,311 @@
+"""Map a single field to a candidate value using page-by-page analysis and LLM-based extraction."""
+from typing import Dict, Any, Optional, List
+import logging
+import re
+import json
+from .base_agent import BaseAgent
+from services.llm_client import LLMClient
+from services.embedding_client import EmbeddingClient
+from config.settings import settings
+
+# Configure logging to disable verbose Azure HTTP logs
+logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.WARNING)
+logging.getLogger('azure.core.pipeline').setLevel(logging.WARNING)
+logging.getLogger('azure').setLevel(logging.WARNING)
+
+class FieldMapperAgent(BaseAgent):
+    def __init__(self):
+        self.logger = logging.getLogger(__name__)
+        self.llm = LLMClient(settings)
+        self.embedding_client = EmbeddingClient()
+
+    def _infer_document_context(self, text: str) -> str:
+        """Use LLM to infer document context and user profile."""
+        prompt = f"""Given this document text, describe the document type and typical user profile in 1-2 sentences.
+        Focus on the domain, purpose, and who would use this document.
+        
+        Document text:
+        {text[:2000]}  # First 2000 chars for context
+        
+        Response format:
+        Document type: [type]
+        User profile: [profile]
+        """
+        
+        try:
+            self.logger.info("Inferring document context...")
+            self.logger.debug(f"Using text preview: {text[:500]}...")
+            context = self.llm.responses(prompt, temperature=0.0)
+            self.logger.info(f"Inferred context: {context}")
+            return context
+        except Exception as e:
+            self.logger.error(f"Error inferring context: {str(e)}")
+            return "Generic document user"
+
+    def _find_similar_chunks_search(self, query: str, index: Dict[str, Any], top_k: int = 3) -> List[Dict[str, Any]]:
+        """Find chunks semantically similar to the query using cosine similarity."""
+        try:
+            self.logger.info(f"Finding similar chunks for query: {query}")
+            self.logger.debug(f"Index contains {len(index['chunks'])} chunks and {len(index['embeddings'])} embeddings")
+            
+            # Get query embedding
+            self.logger.debug("Generating embedding for query...")
+            query_embedding = self.embedding_client.embed([query])[0]
+            self.logger.debug(f"Query embedding generated, length: {len(query_embedding)}")
+            
+            # Calculate similarities
+            similarities = []
+            for i, (chunk, embedding) in enumerate(zip(index["chunks"], index["embeddings"])):
+                similarity = self._cosine_similarity(query_embedding, embedding)
+                similarities.append((similarity, chunk))
+                self.logger.debug(f"Chunk {i} similarity: {similarity:.3f}")
+                self.logger.debug(f"Chunk {i} preview: {chunk['text'][:100]}...")
+            
+            # Sort by similarity and return top k
+            similarities.sort(reverse=True)
+            results = [chunk for _, chunk in similarities[:top_k]]
+            
+            # Log top results
+            self.logger.info(f"Found {len(results)} similar chunks")
+            for i, (sim, chunk) in enumerate(similarities[:top_k]):
+                self.logger.info(f"Top {i+1} match (similarity: {sim:.3f}): {chunk['text'][:200]}...")
+            
+            return results
+            
+        except Exception as e:
+            self.logger.error(f"Error finding similar chunks: {str(e)}", exc_info=True)
+            return []
+
+    def _cosine_similarity(self, a: List[float], b: List[float]) -> float:
+        """Calculate cosine similarity between two vectors."""
+        import numpy as np
+        try:
+            # Check for zero vectors
+            if not a or not b or all(x == 0 for x in a) or all(x == 0 for x in b):
+                self.logger.warning("Zero vector detected in cosine similarity calculation")
+                return 0.0
+                
+            # Convert to numpy arrays
+            a_np = np.array(a)
+            b_np = np.array(b)
+            
+            # Calculate norms
+            norm_a = np.linalg.norm(a_np)
+            norm_b = np.linalg.norm(b_np)
+            
+            # Check for zero norms
+            if norm_a == 0 or norm_b == 0:
+                self.logger.warning("Zero norm detected in cosine similarity calculation")
+                return 0.0
+                
+            # Calculate similarity
+            similarity = np.dot(a_np, b_np) / (norm_a * norm_b)
+            
+            # Check for NaN
+            if np.isnan(similarity):
+                self.logger.warning("NaN detected in cosine similarity calculation")
+                return 0.0
+                
+            return float(similarity)
+        except Exception as e:
+            self.logger.error(f"Error calculating cosine similarity: {str(e)}")
+            return 0.0
+
+    def _extract_field_value_search(self, field: str, chunks: List[Dict[str, Any]], context: str) -> Optional[str]:
+        """Use LLM to extract field value from relevant chunks."""
+        # Combine chunks into context
+        chunk_texts = [chunk["text"] for chunk in chunks]
+        combined_context = "\n".join(chunk_texts)
+        
+        self.logger.info(f"Extracting value for field '{field}' from {len(chunks)} chunks")
+        self.logger.debug(f"Combined context preview: {combined_context[:500]}...")
+        
+        # Get filename from context if available
+        filename = self.ctx.get("pdf_meta", {}).get("filename", "")
+        filename_context = f"\nDocument filename: {filename}" if filename else ""
+        
+        prompt = f"""You are an expert in {context}
+        
+        Your task is to extract the value for the field: {field}{filename_context}
+        
+        Consider the following context from the document:
+        {combined_context}
+        
+        Instructions:
+        1. Look for the field value in the context
+        2. If you find multiple potential values, choose the most relevant one
+        3. If you're not sure, return None
+        4. Return ONLY the value, no explanations
+        
+        Field value:"""
+        
+        try:
+            self.logger.info(f"Calling LLM to extract value for field '{field}'")
+            self.logger.debug(f"Using prompt: {prompt}")
+            value = self.llm.responses(prompt, temperature=0.0)
+            self.logger.debug(f"Raw LLM response: {value}")
+            
+            if value and value.lower() not in ["none", "null", "n/a"]:
+                self.logger.info(f"Successfully extracted value: {value}")
+                return value.strip()
+            else:
+                self.logger.warning(f"LLM returned no valid value for field '{field}'")
+            return None
+        except Exception as e:
+            self.logger.error(f"Error extracting field value: {str(e)}", exc_info=True)
+            return None
+
+    def _extract_field_value_from_page(self, field: str, page_text: str, context: str) -> Optional[str]:
+        """Use LLM to extract field value from a single page."""
+        self.logger.info(f"Extracting value for field '{field}' from page")
+        self.logger.debug(f"Page text preview: {page_text[:500]}...")
+        
+        # Get filename from context if available
+        filename = self.ctx.get("pdf_meta", {}).get("filename", "")
+        filename_context = f"\nDocument filename: {filename}" if filename else ""
+        
+        prompt = f"""You are an expert in {context}
+        
+        Your task is to extract the value for the field: {field}{filename_context}
+        
+        Consider the following page from the document:
+        {page_text}
+        
+        Instructions:
+        1. Look for the field values in this page
+        2. Return the data in a tabular format where each field is a column
+        3. Each field should have an array of values
+        4. The arrays must be aligned (same length) to represent rows
+        5. Return ONLY the JSON value, no explanations
+        6. Format the response as a valid JSON object with field names as keys
+        7. Keep the structure flat - do not nest values under 'details' or other keys
+        
+        Example response format:
+        {{
+            "field1": ["value1", "value2", "value3"],
+            "field2": ["value4", "value5", "value6"],
+            "field3": ["value7", "value8", "value9"]
+        }}
+        
+        Field value:"""
+        
+        try:
+            self.logger.info(f"Calling LLM to extract value for field '{field}' from page")
+            value = self.llm.responses(prompt, temperature=0.0)
+            self.logger.debug(f"Raw LLM response: {value}")
+            
+            if value and value.lower() not in ["none", "null", "n/a"]:
+                # Try to parse as JSON to ensure it's valid
+                try:
+                    json_value = json.loads(value)
+                    self.logger.info(f"Successfully extracted value: {json.dumps(json_value, indent=2)}")
+                    return json.dumps(json_value, indent=2)
+                except json.JSONDecodeError:
+                    # If not valid JSON, wrap it in a JSON object
+                    json_value = {field: value.strip()}
+                    self.logger.info(f"Wrapped non-JSON value in JSON object: {json.dumps(json_value, indent=2)}")
+                    return json.dumps(json_value, indent=2)
+            else:
+                self.logger.warning(f"LLM returned no valid value for field '{field}'")
+            return None
+        except Exception as e:
+            self.logger.error(f"Error extracting field value from page: {str(e)}", exc_info=True)
+            return None
+
+    def execute(self, ctx: Dict[str, Any]):  # noqa: D401
+        field = ctx.get("current_field")
+        self.logger.info(f"Starting field mapping for: {field}")
+        
+        # Store context for use in extraction methods
+        self.ctx = ctx
+        
+        # Get text and index
+        text = ""
+        index = None
+        if "index" in ctx and isinstance(ctx["index"], dict):
+            index = ctx["index"]
+            text = index.get("text", "")
+            self.logger.info(f"Using text from index (length: {len(text)})")
+            self.logger.debug(f"Index contains {len(index.get('chunks', []))} chunks")
+            self.logger.debug(f"Index contains {len(index.get('embeddings', []))} embeddings")
+        elif "text" in ctx:
+            text = ctx["text"]
+            self.logger.info(f"Using text from direct context (length: {len(text)})")
+        
+        if not field:
+            self.logger.warning("No field provided in context")
+            return None
+        if not text:
+            self.logger.warning("No text content found in context or index")
+            return None
+
+        # Infer document context if not already present
+        if "document_context" not in ctx:
+            ctx["document_context"] = self._infer_document_context(text)
+        
+        self.logger.info(f"Processing field: {field}")
+        self.logger.info(f"Using document context: {ctx['document_context']}")
+        
+        # Split text into pages using the same markers as IndexAgent
+        page_markers = [
+            '<!-- PageBreak -->',
+            'Page',
+            '---',
+            '\n\n',  # Double newline as fallback
+        ]
+        
+        # First try to find which marker is used
+        used_marker = None
+        for marker in page_markers:
+            if marker in text:
+                used_marker = marker
+                self.logger.info(f"Found page marker: {marker}")
+                break
+        
+        if not used_marker:
+            self.logger.warning("No page markers found, falling back to double newline")
+            used_marker = '\n\n'
+        
+        # Split the text
+        if used_marker == '\n\n':
+            pages = [p.strip() for p in text.split(used_marker) if p.strip()]
+        else:
+            pages = []
+            current_page = []
+            
+            for line in text.split('\n'):
+                if used_marker in line:
+                    if current_page:
+                        pages.append('\n'.join(current_page))
+                    current_page = [line]
+                else:
+                    current_page.append(line)
+            
+            if current_page:
+                pages.append('\n'.join(current_page))
+        
+        self.logger.info(f"Split document into {len(pages)} pages")
+        
+        # Process each page
+        for i, page in enumerate(pages, 1):
+            self.logger.info(f"Processing page {i}/{len(pages)}")
+            value = self._extract_field_value_from_page(field, page, ctx["document_context"])
+            if value:
+                return value
+        
+        # If no value found in any page, try the search-based approach as fallback
+        self.logger.warning("No value found in page-by-page analysis, falling back to search-based approach")
+        
+        if index and "embeddings" in index:
+            self.logger.info("Using semantic search with embeddings")
+            search_query = f"{field} in {ctx['document_context']}"
+            similar_chunks = self._find_similar_chunks_search(search_query, index)
+            
+            if similar_chunks:
+                self.logger.info(f"Found {len(similar_chunks)} relevant chunks, attempting value extraction")
+                value = self._extract_field_value_search(field, similar_chunks, ctx["document_context"])
+                if value:
+                    return value
+        
+        self.logger.warning(f"No candidate found for field: {field}")
+        return f"<no candidate for {field}>"

src/agents/index_agent.py

@@ -0,0 +1,135 @@
+"""Create a semantic index of document content using embeddings."""
+from typing import Dict, Any, List, Tuple
+import logging
+import numpy as np
+from .base_agent import BaseAgent
+from services.embedding_client import EmbeddingClient
+
+class IndexAgent(BaseAgent):
+    def __init__(self):
+        self.logger = logging.getLogger(__name__)
+        self.embedding_client = EmbeddingClient()
+        self.logger.info("IndexAgent initialized")
+
+    def execute(self, ctx: Dict[str, Any]):
+        """Create a semantic index of document content."""
+        try:
+            self.logger.info("Starting index creation")
+            
+            # Get text from PDF agent
+            text = ctx.get("text", "")
+            if not text:
+                self.logger.warning("No text content found in context")
+                return {}
+            self.logger.info(f"Found text content of length {len(text)}")
+            
+            # Get tables from Table agent
+            tables = ctx.get("tables", [])
+            self.logger.info(f"Found {len(tables)} tables in context")
+            
+            # Combine all content
+            all_content = text
+            if tables:
+                all_content += "\n".join(tables)
+                self.logger.info(f"Combined content length: {len(all_content)}")
+            
+            # Create chunks with metadata
+            chunks = self._create_chunks(all_content)
+            self.logger.info(f"Created {len(chunks)} content chunks")
+            for i, chunk in enumerate(chunks):
+                self.logger.debug(f"Chunk {i}: {chunk['text'][:100]}...")
+            
+            # Get embeddings for chunks
+            chunk_texts = [chunk["text"] for chunk in chunks]
+            self.logger.info(f"Getting embeddings for {len(chunk_texts)} chunks")
+            embeddings = self.embedding_client.embed(chunk_texts)
+            self.logger.info(f"Generated {len(embeddings)} embeddings")
+            
+            # Create semantic index
+            index = {
+                "chunks": chunks,
+                "embeddings": embeddings,
+                "text": all_content,  # Keep full text for non-semantic search
+            }
+            
+            # Store in context
+            ctx["index"] = index
+            self.logger.info(f"Created semantic index with {len(chunks)} chunks")
+            return index
+            
+        except Exception as e:
+            self.logger.error(f"Error in IndexAgent: {str(e)}", exc_info=True)
+            return {}
+
+    def _create_chunks(self, text: str, chunk_size: int = 1000) -> List[Dict[str, Any]]:
+        """Split text into chunks with metadata."""
+        self.logger.info(f"Creating chunks from text of length {len(text)}")
+        chunks = []
+        sentences = text.split(". ")
+        self.logger.info(f"Split into {len(sentences)} sentences")
+        current_chunk = []
+        current_size = 0
+        total_length = 0
+        
+        for sentence in sentences:
+            sentence = sentence.strip() + ". "
+            sentence_size = len(sentence)
+            
+            if current_size + sentence_size > chunk_size and current_chunk:
+                # Save current chunk
+                chunk_text = "".join(current_chunk)
+                chunks.append({
+                    "text": chunk_text,
+                    "start": total_length,
+                    "end": total_length + len(chunk_text),
+                    "type": "text"
+                })
+                total_length += len(chunk_text)
+                self.logger.debug(f"Created chunk of size {len(chunk_text)}")
+                current_chunk = []
+                current_size = 0
+            
+            current_chunk.append(sentence)
+            current_size += sentence_size
+        
+        # Add last chunk if any
+        if current_chunk:
+            chunk_text = "".join(current_chunk)
+            chunks.append({
+                "text": chunk_text,
+                "start": total_length,
+                "end": total_length + len(chunk_text),
+                "type": "text"
+            })
+            self.logger.debug(f"Created final chunk of size {len(chunk_text)}")
+        
+        self.logger.info(f"Created {len(chunks)} total chunks")
+        return chunks
+
+    def find_similar_chunks(self, query: str, index: Dict[str, Any], top_k: int = 3) -> List[Dict[str, Any]]:
+        """Find chunks semantically similar to the query."""
+        try:
+            self.logger.info(f"Finding similar chunks for query: {query}")
+            # Get query embedding
+            query_embedding = self.embedding_client.embed([query])[0]
+            
+            # Calculate similarities
+            similarities = []
+            for chunk, embedding in zip(index["chunks"], index["embeddings"]):
+                similarity = self._cosine_similarity(query_embedding, embedding)
+                similarities.append((similarity, chunk))
+                self.logger.debug(f"Chunk similarity: {similarity:.3f}")
+            
+            # Sort by similarity and return top k
+            similarities.sort(reverse=True)
+            results = [chunk for _, chunk in similarities[:top_k]]
+            self.logger.info(f"Found {len(results)} similar chunks")
+            return results
+            
+        except Exception as e:
+            self.logger.error(f"Error finding similar chunks: {str(e)}", exc_info=True)
+            return []
+
+    def _cosine_similarity(self, a: List[float], b: List[float]) -> float:
+        """Calculate cosine similarity between two vectors."""
+        return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

src/agents/pdf_agent.py

@@ -0,0 +1,28 @@
+"""Extract raw text from the uploaded PDF using PyMuPDF.
+This keeps the implementation minimal for a POC while remaining easy to extend.
+"""
+from typing import Dict, Any, List
+
+import fitz  # PyMuPDF
+
+from .base_agent import BaseAgent
+
+
+class PDFAgent(BaseAgent):
+    """Reads the PDF, concatenates all page text and stores it under ``ctx['text']``."""
+
+    def _extract_text(self, pdf_bytes: bytes) -> str:
+        doc = fitz.open(stream=pdf_bytes, filetype="pdf")  # type: ignore[arg-type]
+        pages: List[str] = [page.get_text() for page in doc]  # list-comp for clarity
+        return "\n".join(pages)
+
+    # -----------------------------------------------------
+    def execute(self, ctx: Dict[str, Any]):  # noqa: D401
+        pdf_file = ctx.get("pdf_file")
+        if pdf_file is None:
+            raise ValueError("PDFAgent expected 'pdf_file' in context but none provided.")
+
+        pdf_bytes = pdf_file.read()
+        text = self._extract_text(pdf_bytes)
+        ctx["text"] = text
+        return text

src/agents/query_generator.py

@@ -0,0 +1,7 @@
+from typing import Dict, Any
+from .base_agent import BaseAgent
+
+class QueryGenerator(BaseAgent):
+    def execute(self, ctx: Dict[str, Any]):
+        field = ctx.get("current_field")
+        return f"Follow‑up query for {field}"

src/agents/semantic_reasoner.py

@@ -0,0 +1,8 @@
+from typing import Dict, Any
+from .base_agent import BaseAgent
+
+class SemanticReasonerAgent(BaseAgent):
+    def execute(self, ctx: Dict[str, Any]):
+        field = ctx.get("current_field")
+        candidate = ctx.get("candidates", {}).get(field)
+        return candidate or f"<unresolved {field}>"

src/agents/table_agent.py

@@ -0,0 +1,46 @@
+"""Extract tables from PDF using Azure Document Intelligence."""
+from typing import Dict, Any
+import logging
+from .base_agent import BaseAgent
+from services.azure_di_service import AzureDIService
+
+class TableAgent(BaseAgent):
+    def __init__(self, settings):
+        self.service = AzureDIService(settings.AZURE_DI_ENDPOINT, settings.AZURE_DI_KEY)
+        self.logger = logging.getLogger(__name__)
+
+    def execute(self, ctx: Dict[str, Any]):
+        """Extract tables from PDF."""
+        try:
+            pdf_file = ctx.get("pdf_file")
+            if not pdf_file:
+                self.logger.error("No PDF file found in context")
+                return {}
+
+            # Get the current position of the file pointer
+            current_pos = pdf_file.tell()
+            self.logger.info(f"Current file position: {current_pos}")
+
+            # Reset to beginning if not at start
+            if current_pos != 0:
+                self.logger.info("Resetting file pointer to beginning")
+                pdf_file.seek(0)
+
+            # Read the file
+            pdf_bytes = pdf_file.read()
+            self.logger.info(f"Read {len(pdf_bytes)} bytes from PDF")
+
+            # Extract content using Azure DI
+            result = self.service.extract_tables(pdf_bytes)
+            
+            # Store both text and tables in context
+            ctx["text"] = result["text"]
+            ctx["tables"] = result["tables"]
+            
+            self.logger.info(f"Extracted {len(result['text'])} characters of text and {len(result['tables'])} tables")
+            return result
+            
+        except Exception as e:
+            self.logger.error(f"Error in TableAgent: {str(e)}")
+            self.logger.exception("Full traceback:")
+            return {}

src/app.py

@@ -0,0 +1,364 @@
+"""Streamlit front‑end entry‑point."""
+import yaml
+import json
+import streamlit as st
+import logging
+from dotenv import load_dotenv
+from orchestrator.planner import Planner
+from orchestrator.executor import Executor
+from config.settings import settings
+import fitz  # PyMuPDF local import to avoid heavy load on startup
+import pandas as pd
+from datetime import datetime
+import io
+import sys
+from io import StringIO
+
+# Create a custom stream handler to capture logs
+class LogCaptureHandler(logging.StreamHandler):
+    def __init__(self):
+        super().__init__()
+        self.logs = []
+        
+    def emit(self, record):
+        try:
+            msg = self.format(record)
+            self.logs.append(msg)
+        except Exception:
+            self.handleError(record)
+            
+    def get_logs(self):
+        return "\n".join(self.logs)
+        
+    def clear(self):
+        self.logs = []
+
+# Initialize session state for storing execution history
+if 'execution_history' not in st.session_state:
+    st.session_state.execution_history = []
+
+# Set up logging capture
+log_capture = LogCaptureHandler()
+log_capture.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+
+# Configure root logger
+root_logger = logging.getLogger()
+root_logger.setLevel(logging.INFO)
+root_logger.addHandler(log_capture)
+
+# Configure specific loggers
+for logger_name in ['orchestrator', 'agents', 'services']:
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(logging.INFO)
+    logger.addHandler(log_capture)
+
+load_dotenv()
+
+st.set_page_config(page_title="PDF Field Extractor", layout="wide")
+
+# Sidebar navigation
+st.sidebar.title("Navigation")
+page = st.sidebar.radio("Go to", ["Documentation", "Traces", "Execution"])
+
+# Documentation Page
+if page == "Documentation":
+    st.title("Deep‑Research PDF Field Extractor (POC)")
+    
+    st.markdown("""
+    This system uses a multi-step pipeline to extract fields from PDFs:
+    1. **Document Intelligence**: Extracts text and tables from PDFs using Azure Document Intelligence
+    2. **Semantic Indexing**: Creates searchable chunks with embeddings for semantic search
+    3. **Field Extraction**: Uses a two-step approach:
+       - First attempts page-by-page scanning for precise extraction
+       - Falls back to semantic search if no value is found
+    4. **Validation**: Ensures extracted values are correct and properly formatted
+    5. **Confidence Scoring**: Identifies which extractions need review
+    """)
+
+    st.markdown("""
+    ### Agent Descriptions
+    #### DocumentIntelligenceAgent
+    - Uses Azure Document Intelligence to extract text and tables
+    - Preserves document layout and structure
+    - Outputs both raw text and formatted tables as HTML
+    - Handles complex document layouts and tables
+
+    #### IndexAgent
+    - Creates semantic search index from extracted content
+    - Splits document into manageable chunks with metadata
+    - Generates embeddings for semantic search
+    - Provides both chunk-based and full-text search capabilities
+    - Includes chunk statistics and visualization
+
+    #### FieldMapper
+    - Implements a two-step field extraction strategy:
+      1. Page-by-page scanning for precise extraction
+      2. Semantic search fallback if no value found
+    - Uses document context to improve extraction accuracy
+    - Handles multiple potential values with confidence scoring
+    - Returns structured JSON responses with value details
+
+    #### SemanticReasoner
+    - Validates and cleans up candidate values
+    - Uses domain knowledge to ensure values make sense
+    - Can reformat values to standard format
+    - Returns `<unresolved>` if value is wrong/missing
+
+    #### ConfidenceScorer
+    - Assigns confidence score (0-1) to each extraction
+    - Helps identify which extractions need review
+    - Can trigger follow-up queries when confidence is low
+
+    #### QueryGenerator
+    - Generates follow-up questions when confidence is low
+    - Creates concise questions (≤12 words)
+    - Helps guide system to better extractions
+    """)
+
+# Traces Page
+elif page == "Traces":
+    st.title("Execution Traces")
+    
+    if not st.session_state.execution_history:
+        st.info("No execution traces available yet. Run an extraction to see traces here.")
+    else:
+        # Create a DataFrame from the execution history
+        history_data = []
+        for record in st.session_state.execution_history:
+            history_data.append({
+                "filename": record["filename"],
+                "datetime": record["datetime"],
+                "fields": ", ".join(record.get("fields", [])),
+                "logs": record.get("logs", []),
+                "results": record.get("results", None)
+            })
+        
+        history_df = pd.DataFrame(history_data)
+        
+        # Display column headers
+        col1, col2, col3, col4, col5 = st.columns([2, 2, 3, 1, 1])
+        with col1:
+            st.markdown("**Filename**")
+        with col2:
+            st.markdown("**Timestamp**")
+        with col3:
+            st.markdown("**Fields**")
+        with col4:
+            st.markdown("**Logs**")
+        with col5:
+            st.markdown("**Results**")
+        
+        st.markdown("---")  # Add a separator line
+        
+        # Display the table with download buttons
+        for idx, row in history_df.iterrows():
+            col1, col2, col3, col4, col5 = st.columns([2, 2, 3, 1, 1])
+            with col1:
+                st.write(row["filename"])
+            with col2:
+                st.write(row["datetime"])
+            with col3:
+                st.write(row["fields"])
+            with col4:
+                if row["logs"]:  # Check if we have any logs
+                    st.download_button(
+                        "Download Logs",
+                        row["logs"],  # Use the stored logs
+                        file_name=f"logs_{row['filename']}_{row['datetime']}.txt",
+                        key=f"logs_dl_{idx}"
+                    )
+                else:
+                    st.write("No Logs")
+            with col5:
+                if row["results"] is not None:
+                    results_df = pd.DataFrame(row["results"])
+                    st.download_button(
+                        "Download Results",
+                        results_df.to_csv(index=False),
+                        file_name=f"results_{row['filename']}_{row['datetime']}.csv",
+                        key=f"results_dl_{idx}"
+                    )
+                else:
+                    st.write("No Results")
+            st.markdown("---")  # Add a separator line between rows
+
+# Execution Page
+else:  # page == "Execution"
+    st.title("Deep‑Research PDF Field Extractor (POC)")
+
+    pdf_file = st.file_uploader("Upload PDF", type=["pdf"])
+    fields_str = st.text_input("Fields (comma‑separated)", "Protein Lot, Chain, Residue")
+    desc_blob = st.text_area("Field descriptions / rules (YAML, optional)")
+
+    def flatten_json_response(json_data, fields):
+        """Flatten the nested JSON response into a tabular structure with dynamic columns."""
+        logger = logging.getLogger(__name__)
+        logger.info("Starting flatten_json_response")
+        logger.info(f"Input fields: {fields}")
+        
+        # Handle the case where the response is a string
+        if isinstance(json_data, str):
+            logger.info("Input is a string, attempting to parse as JSON")
+            try:
+                json_data = json.loads(json_data)
+                logger.info("Successfully parsed JSON string")
+            except json.JSONDecodeError as e:
+                logger.error(f"Failed to parse JSON string: {e}")
+                return pd.DataFrame(columns=fields)
+        
+        # If the data is wrapped in an array, get the first item
+        if isinstance(json_data, list) and len(json_data) > 0:
+            logger.info("Data is wrapped in an array, extracting first item")
+            json_data = json_data[0]
+        
+        # If the data is a dictionary with numeric keys, get the first value
+        if isinstance(json_data, dict):
+            keys = list(json_data.keys())
+            logger.info(f"Checking dictionary keys: {keys}")
+            # Check if all keys are integers or string representations of integers
+            if all(isinstance(k, int) or (isinstance(k, str) and k.isdigit()) for k in keys):
+                logger.info("Data has numeric keys, extracting first value")
+                first_key = sorted(keys, key=lambda x: int(x) if isinstance(x, str) else x)[0]
+                json_data = json_data[first_key]
+                logger.info(f"Extracted data from key '{first_key}'")
+        
+        logger.info(f"JSON data keys: {list(json_data.keys()) if isinstance(json_data, dict) else 'Not a dict'}")
+        
+        # Create a list to store rows
+        rows = []
+        
+        # Get the length of the first array to determine number of rows
+        if isinstance(json_data, dict) and len(json_data) > 0:
+            first_field = list(json_data.keys())[0]
+            num_rows = len(json_data[first_field]) if isinstance(json_data[first_field], list) else 1
+            logger.info(f"Number of rows to process: {num_rows}")
+            
+            # Create a row for each index
+            for i in range(num_rows):
+                logger.debug(f"Processing row {i}")
+                row = {}
+                for field in fields:
+                    if field in json_data and isinstance(json_data[field], list) and i < len(json_data[field]):
+                        row[field] = json_data[field][i]
+                        logger.debug(f"Field '{field}' value at index {i}: {json_data[field][i]}")
+                    else:
+                        row[field] = None
+                        logger.debug(f"Field '{field}' not found or index {i} out of bounds")
+                rows.append(row)
+        else:
+            logger.error(f"Unexpected data structure: {type(json_data)}")
+            return pd.DataFrame(columns=fields)
+        
+        # Create DataFrame with all requested fields as columns
+        df = pd.DataFrame(rows)
+        logger.info(f"Created DataFrame with shape: {df.shape}")
+        logger.info(f"DataFrame columns: {df.columns.tolist()}")
+        
+        # Ensure columns are in the same order as the fields list
+        df = df[fields]
+        logger.info(f"Final DataFrame columns after reordering: {df.columns.tolist()}")
+        
+        return df
+
+    if st.button("Run extraction") and pdf_file:
+        field_list = [f.strip() for f in fields_str.split(",") if f.strip()]
+        field_descs = yaml.safe_load(desc_blob) if desc_blob.strip() else {}
+
+        try:
+            with st.spinner("Planning …"):
+                # quick first-page text preview to give LLM document context
+                doc = fitz.open(stream=pdf_file.getvalue(), filetype="pdf")  # type: ignore[arg-type]
+                preview = "\n".join(page.get_text() for page in doc[:10])[:20000]  # first 2 pages, 2k chars
+
+                planner = Planner()
+                plan = planner.build_plan(
+                    pdf_meta={"filename": pdf_file.name},
+                    doc_preview=preview,
+                    fields=field_list,
+                    field_descs=field_descs,
+                )
+                
+                # Add a visual separator
+                st.markdown("---")
+
+            with st.spinner("Executing …"):
+                executor = Executor(settings=settings)
+                results, logs = executor.run(plan, pdf_file)
+
+                # Add detailed logging about what executor returned
+                logger.info(f"Executor returned results of type: {type(results)}")
+                logger.info(f"Results content: {results}")
+                
+                # Check if results is already a DataFrame
+                if isinstance(results, pd.DataFrame):
+                    logger.info(f"Results is already a DataFrame with shape: {results.shape}")
+                    logger.info(f"DataFrame columns: {results.columns.tolist()}")
+                    logger.info(f"DataFrame head: {results.head()}")
+                    df = results
+                else:
+                    logger.info("Results is not a DataFrame, calling flatten_json_response")
+                    # Process results using flatten_json_response
+                    df = flatten_json_response(results, field_list)
+                
+                # Log final DataFrame info
+                logger.info(f"Final DataFrame shape: {df.shape}")
+                logger.info(f"Final DataFrame columns: {df.columns.tolist()}")
+                if not df.empty:
+                    logger.info(f"Final DataFrame sample: {df.head()}")
+
+                # Store execution in history
+                execution_record = {
+                    "filename": pdf_file.name,
+                    "datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                    "fields": field_list,
+                    "logs": log_capture.get_logs(),  # Store the actual logs
+                    "results": df.to_dict() if not df.empty else None
+                }
+                st.session_state.execution_history.append(execution_record)
+                log_capture.clear()  # Clear logs after storing them
+
+            # ----------------- UI: show execution tree -----------------
+            st.subheader("Execution trace")
+            for log in logs:
+                indent = "&nbsp;" * 4 * log["depth"]
+                # Add error indicator if there was an error
+                error_indicator = "❌ " if log.get("error") else "✓ "
+                # Use a fixed preview text instead of the result
+                with st.expander(f"{indent}{error_indicator}{log['tool']} – Click to view result"):
+                    st.markdown(f"**Args**: `{log['args']}`", unsafe_allow_html=True)
+                    if log.get("error"):
+                        st.error(f"Error: {log['error']}")
+                    
+                    # Special handling for IndexAgent output
+                    if log['tool'] == "IndexAgent" and isinstance(log["result"], dict):
+                        # Display chunk statistics if available
+                        if "chunk_stats" in log["result"]:
+                            st.markdown("### Chunk Statistics")
+                            # Create a DataFrame for better visualization
+                            stats_df = pd.DataFrame(log["result"]["chunk_stats"])
+                            st.dataframe(stats_df)
+                            
+                            # Add summary statistics
+                            st.markdown("### Summary")
+                            st.markdown(f"""
+                            - Total chunks: {len(stats_df)}
+                            - Average chunk length: {stats_df['length'].mean():.0f} characters
+                            - Shortest chunk: {stats_df['length'].min()} characters
+                            - Longest chunk: {stats_df['length'].max()} characters
+                            """)
+                            
+                            # Add a bar chart of chunk lengths
+                            st.markdown("### Chunk Length Distribution")
+                            st.bar_chart(stats_df.set_index('chunk_number')['length'])
+                    else:
+                        st.code(log["result"])
+
+            if not df.empty:
+                st.success("Done ✓")
+                st.dataframe(df)
+                st.download_button("Download CSV", df.to_csv(index=False), "results.csv")
+            else:
+                st.warning("No results were extracted. Check the execution trace for errors.")
+        except Exception as e:
+            logging.exception("App error:")
+            st.error(f"An error occurred: {e}")

src/config/__init__.py

@@ -0,0 +1 @@
+ 

src/config/field_rules.yaml

@@ -0,0 +1,4 @@
+Chain:
+  description: Heavy vs Light chain based on Seq Loc prefix.
+  rules:
+    starts_with: {L: "Light", H: "Heavy"}

src/config/prompts.yaml

@@ -0,0 +1,53 @@
+planner: |
+  You are "Doc-to-Record Planner v1" – an expert at designing multi-step
+  extraction pipelines that convert an arbitrary document into a flat record
+  of user-requested fields.
+
+  You will be given:
+  – doc_preview: a few kB of raw text from the uploaded document (may include table HTML).
+  – fields: the list of field names the user wants extracted.
+  – pdf_meta / field_descriptions for extra context.
+
+  Available tools (use exactly these names in the JSON):
+    PDFAgent            → extracts raw text from the full PDF.
+    TableAgent          → calls Azure Document Intelligence to get HTML tables.
+    FieldMapper         → maps one field name to a candidate value.
+
+  Control-flow helper:
+    ForEachField – loops over every requested field and executes the nested "loop" array.
+
+  Output JSON **only** with this schema (no markdown):
+  {
+    "fields": [<same list you received>],
+    "steps": [
+       {"tool": "PDFAgent", "args": {}},
+       {"tool": "TableAgent", "args": {}},
+       {"tool": "ForEachField",
+        "loop": [
+           {"tool": "FieldMapper", "args": {"field": "$field"}}
+        ]}
+    ]
+  }
+
+  Always include PDFAgent and TableAgent first, then the ForEachField loop. Keep plans short and deterministic.
+
+field_mapper: |
+  You are "FieldMapper v1" – a precision extractor.
+  Given:
+    • field (target field name)
+    • context (snippet of raw PDF text or table row)
+  Return **only** the best candidate value (no extra words).
+
+semantic_reasoner: |
+  You are "Semantic Reasoner v1".
+  Validate the candidate value for a field using domain knowledge and the surrounding context.
+  If the candidate is obviously wrong / absent output <unresolved FIELDNAME> (same token as placeholder).
+  Otherwise output a cleaned, final value – no explanation text.
+
+confidence_scorer: |
+  You are "Confidence Scorer". For the given field and candidate value assign a confidence between 0 and 1.
+  Output **only** the float.
+
+query_generator: |
+  You are "Follow-up Query Generator". The previous candidate for a field was low-confidence.
+  Formulate a concise follow-up question (<=12 words) that, when answered, would help identify the field value.

src/config/settings.py

@@ -0,0 +1,18 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+class Settings(BaseSettings):
+    OPENAI_API_KEY: str = Field("", env="OPENAI_API_KEY")
+    AZURE_DI_ENDPOINT: str = Field("", env="AZURE_DI_ENDPOINT")
+    AZURE_DI_KEY: str = Field("", env="AZURE_DI_KEY")
+
+    # Azure OpenAI
+    AZURE_OPENAI_ENDPOINT: str = Field("", env="AZURE_OPENAI_ENDPOINT")
+    AZURE_OPENAI_DEPLOYMENT: str = Field("", env="AZURE_OPENAI_DEPLOYMENT")
+    AZURE_OPENAI_API_VERSION: str = Field("2024-02-15-preview", env="AZURE_OPENAI_API_VERSION")
+    AZURE_OPENAI_API_KEY: str = Field("", env="AZURE_OPENAI_API_KEY")
+    AZURE_OPENAI_EMBEDDING_MODEL: str = Field("text-embedding-3-small", env="AZURE_OPENAI_EMBEDDING_MODEL")
+
+    model_config: SettingsConfigDict = {"env_file": ".env"}
+
+settings = Settings()

src/docker/Dockerfile

@@ -0,0 +1,21 @@
+# ---------- builder -------------
+    FROM python:3.11-slim AS builder
+    RUN pip install --no-cache-dir uv
+    
+    WORKDIR /app
+    COPY pyproject.toml ./
+    RUN uv pip install -r <(uv pip compile --quiet) \
+        && uv pip freeze > /installed.txt          # layer cache
+    
+    COPY . .
+    
+    # ---------- runtime -------------
+    FROM python:3.11-slim
+    ENV PYTHONUNBUFFERED=1
+    WORKDIR /app
+    COPY --from=builder /installed.txt /installed.txt
+    RUN xargs -a /installed.txt pip install --no-cache-dir
+    
+    COPY . .
+    
+    ENTRYPOINT ["bash", "docker/entrypoint.sh"]

src/docker/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+source .env              # load secrets (mounted at run)
+streamlit run app.py --server.port ${PORT:-8501}

src/main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from filetorecord!")
+
+
+if __name__ == "__main__":
+    main()

src/orchestrator/__init__.py

@@ -0,0 +1 @@
+