Update app.py

app.py

@@ -4,64 +4,109 @@ import os
 from pathlib import Path
 
 def create_reranking_interface(task_data):
-    """Create a Gradio interface for reranking evaluation."""
+    """Create a Gradio interface for reranking evaluation using drag and drop."""
     samples = task_data["samples"]
     results = {"task_name": task_data["task_name"], "task_type": "reranking", "annotations": []}
     completed_samples = {s["id"]: False for s in samples}
     
-    def save_ranking(rankings, sample_id):
-        """Save the current set of rankings."""
+    # Define helper functions before UI elements are created
+    def generate_sortable_html(candidates, existing_ranks=None):
+        """Generate the HTML for the sortable list."""
+        if existing_ranks and len(existing_ranks) == len(candidates):
+            order = sorted(range(len(candidates)), key=lambda i: existing_ranks[i])
+        else:
+            order = list(range(len(candidates)))
+        
+        html = '<div id="sortable-container" class="sortable-container">'
+        for rank_minus_1, idx in enumerate(order):
+            if idx < len(candidates):
+                doc = candidates[idx]
+                rank = rank_minus_1 + 1
+                import html as html_escaper
+                escaped_doc = html_escaper.escape(doc)
+                html += f'''\
+                <div class="sortable-item rank-bg-{rank}" data-doc-id="{idx}">
+                    <div class="rank-badge">{rank}</div>
+                    <div class="doc-content">{escaped_doc}</div>
+                </div>
+                '''
+        html += '</div>'
+        return html
+    
+    def save_ranking(order_json, sample_id):
+        """Save the current ranking to results."""
         try:
-            # Convert to integers with error handling
-            processed_rankings = []
-            for r in rankings:
-                if r is None or r == "":
-                    processed_rankings.append(None)
+            if not order_json or order_json == "[]":
+                return "⚠️ Drag documents to set the ranking before submitting.", progress_text.value
+            order = json.loads(order_json)
+            num_candidates = len(next(s["candidates"] for s in samples if s["id"] == sample_id))
+            if len(order) != num_candidates:
+                return f"⚠️ Ranking order length mismatch. Expected {num_candidates}, got {len(order)}.", progress_text.value
+            rankings = [0] * num_candidates
+            for rank_minus_1, doc_idx in enumerate(order):
+                if doc_idx < num_candidates:
+                    rankings[doc_idx] = rank_minus_1 + 1
                 else:
-                    try:
-                        processed_rankings.append(int(r))
-                    except ValueError:
-                        return "⚠️ Invalid ranking value. Please use only numbers.", f"Progress: {sum(completed_samples.values())}/{len(samples)}"
-            
-            # Check if all documents have rankings
-            if None in processed_rankings:
-                return "⚠️ Please assign a rank to all documents before submitting", f"Progress: {sum(completed_samples.values())}/{len(samples)}"
-            
-            # Check for duplicate rankings
-            if len(set(processed_rankings)) != len(processed_rankings):
-                return "⚠️ Each document must have a unique rank. Please review your rankings.", f"Progress: {sum(completed_samples.values())}/{len(samples)}"
-                
-            # Store this annotation in memory
+                    raise ValueError(f"Invalid document index {doc_idx} found in order.")
+            if sorted(rankings) != list(range(1, num_candidates + 1)):
+                return "⚠️ Ranking validation failed. Ranks are not 1 to N.", progress_text.value
+            annotation = {"sample_id": sample_id, "rankings": rankings}
             existing_idx = next((i for i, a in enumerate(results["annotations"]) if a["sample_id"] == sample_id), None)
             if existing_idx is not None:
-                results["annotations"][existing_idx] = {
-                    "sample_id": sample_id,
-                    "rankings": processed_rankings
-                }
+                results["annotations"][existing_idx] = annotation
             else:
-                results["annotations"].append({
-                    "sample_id": sample_id,
-                    "rankings": processed_rankings
-                })
-            
+                results["annotations"].append(annotation)
             completed_samples[sample_id] = True
-            
-            # Try to save to file, but continue even if it fails
-            try:
-                output_path = f"{task_data['task_name']}_human_results.json"
-                with open(output_path, "w") as f:
-                    json.dump(results, f, indent=2)
-                return f"✅ Rankings saved successfully (in memory and to file)", f"Progress: {sum(completed_samples.values())}/{len(samples)}"
-            except:
-                # If file saving fails, still mark as success since we saved in memory
-                return f"✅ Rankings saved in memory (file save failed)", f"Progress: {sum(completed_samples.values())}/{len(samples)}"
+            output_path = f"{task_data['task_name']}_human_results.json"
+            with open(output_path, "w") as f:
+                json.dump(results, f, indent=2)
+            return f"✅ Rankings saved successfully ({len(results['annotations'])}/{len(samples)} completed)", f"Progress: {sum(completed_samples.values())}/{len(samples)}"
+        except json.JSONDecodeError:
+            return "⚠️ Error decoding ranking order. Please try again.", progress_text.value
         except Exception as e:
-            # Return specific error message
-            return f"Error: {str(e)}", f"Progress: {sum(completed_samples.values())}/{len(samples)}"
+            import traceback
+            print(traceback.format_exc())
+            return f"Error saving ranking: {str(e)}", progress_text.value
+    
+    def load_sample(sample_id):
+        """Load a sample into the interface."""
+        try:
+            sample = next((s for s in samples if s["id"] == sample_id), None)
+            if not sample:
+                return gr.update(), gr.update(value="[]"), gr.update(), gr.update()
+            existing_ranking = next((anno["rankings"] for anno in results["annotations"] if anno["sample_id"] == sample_id), None)
+            new_html = generate_sortable_html(sample["candidates"], existing_ranking)
+            status = "Ready to rank" if not completed_samples.get(sample_id, False) else "Already ranked"
+            progress = f"Progress: {sum(completed_samples.values())}/{len(samples)}"
+            return sample["query"], new_html, "[]", progress, status
+        except Exception as e:
+            return gr.update(), gr.update(value="[]"), gr.update(), gr.update(value=f"Error loading sample: {str(e)}")
+    
+    def next_sample_id(current_id):
+        current_idx = next((i for i, s in enumerate(samples) if s["id"] == current_id), -1)
+        if current_idx == -1:
+            return current_id
+        next_idx = min(current_idx + 1, len(samples) - 1)
+        return samples[next_idx]["id"]
+    
+    def prev_sample_id(current_id):
+        current_idx = next((i for i, s in enumerate(samples) if s["id"] == current_id), -1)
+        if current_idx == -1:
+            return current_id
+        prev_idx = max(current_idx - 1, 0)
+        return samples[prev_idx]["id"]
+    
+    def save_results():
+        output_path = f"{task_data['task_name']}_human_results.json"
+        try:
+            with open(output_path, "w") as f:
+                json.dump(results, f, indent=2)
+            return f"✅ Results saved to {output_path} ({len(results['annotations'])} annotations)"
+        except Exception as e:
+            return f"⚠️ Error saving results file: {str(e)}"
     
     with gr.Blocks(theme=gr.themes.Soft()) as demo:
         gr.Markdown(f"# {task_data['task_name']} - Human Reranking Evaluation")
-        
         with gr.Accordion("Instructions", open=True):
             gr.Markdown("""
             ## Task Instructions
@@ -70,12 +115,11 @@ def create_reranking_interface(task_data):
             
             ### How to use this interface:
             1. Read the query at the top
-            2. Review each document carefully
-            3. Assign a rank to each document (1 = most relevant, higher numbers = less relevant)
-            4. Each document must have a unique rank
-            5. Click "Submit Rankings" when you're done with the current query
-            6. Use "Previous" and "Next" to navigate between queries
-            7. Click "Save All Results" periodically to ensure your work is saved
+            2. Drag and drop documents to reorder them based on relevance
+            3. Top document = Rank 1, Second = Rank 2, etc.
+            4. Click "Submit Rankings" when you're done with the current query
+            5. Use "Previous" and "Next" to navigate between queries
+            6. Click "Save All Results" periodically to ensure your work is saved
             """.format(instructions=task_data["instructions"]))
         
         current_sample_id = gr.State(value=samples[0]["id"])
@@ -87,200 +131,149 @@ def create_reranking_interface(task_data):
         with gr.Group():
             gr.Markdown("## Query:")
             query_text = gr.Textbox(value=samples[0]["query"], label="", interactive=False)
-            
-            gr.Markdown("## Documents to Rank:")
-            
-            # Create document displays and ranking inputs in synchronized pairs
-            doc_containers = []
-            ranking_inputs = []
-            
-            with gr.Column():
-                for i, doc in enumerate(samples[0]["candidates"]):
-                    with gr.Box():
-                        gr.Markdown(f"### Document {i+1}")
-                        doc_box = gr.Textbox(
-                            value=doc, 
-                            label=None,
-                            interactive=False,
-                            lines=4
-                        )
-                        doc_containers.append(doc_box)
-                        
-                        rank_input = gr.Number(
-                            value=None,
-                            label=f"Rank (1 = highest, {len(samples[0]['candidates'])} = lowest)",
-                            minimum=1,
-                            maximum=len(samples[0]['candidates']),
-                            step=1
-                        )
-                        ranking_inputs.append(rank_input)
-            
-            # Add simple instructions for ranking
-            with gr.Accordion("Ranking Instructions", open=True):
-                gr.Markdown("""
-                ### Ranking Documents:
-                - Enter a number from 1 to {max_rank} for each document
-                - 1 = most relevant document
-                - Higher numbers = less relevant documents
-                - Each document must have a unique rank
-                """.format(max_rank=len(samples[0]["candidates"])))
-            
+            gr.Markdown("## Documents to Rank (Drag to Reorder):")
+            sortable_list = gr.HTML(generate_sortable_html(samples[0]["candidates"], []), elem_id="sortable-list-container")
+            order_state = gr.Textbox(value="[]", visible=False, elem_id="current-order")
             with gr.Row():
                 prev_btn = gr.Button("← Previous Query", size="sm", elem_id="prev-btn")
                 submit_btn = gr.Button("Submit Rankings", size="lg", variant="primary", elem_id="submit-btn")
                 next_btn = gr.Button("Next Query →", size="sm", elem_id="next-btn")
-            
             save_btn = gr.Button("💾 Save All Results", variant="secondary")
         
-        def load_sample(sample_id):
-            """Load a specific sample into the interface."""
-            sample = next((s for s in samples if s["id"] == sample_id), None)
-            if not sample:
-                return [query_text.value] + [d.value for d in doc_containers] + [None] * len(ranking_inputs) + [current_sample_id.value, progress_text.value, status_box.value]
-            
-            # Update query
-            new_query = sample["query"]
-            
-            # Update documents
-            new_docs = []
-            for i, doc in enumerate(sample["candidates"]):
-                if i < len(doc_containers):
-                    new_docs.append(doc)
-                    
-            # Initialize rankings
-            new_rankings = [None] * len(ranking_inputs)
-            
-            # Check if this sample has already been annotated
-            existing_annotation = next((a for a in results["annotations"] if a["sample_id"] == sample_id), None)
-            if existing_annotation:
-                # Restore previous rankings
-                for i, rank in enumerate(existing_annotation["rankings"]):
-                    if i < len(new_rankings) and rank is not None:
-                        new_rankings[i] = rank
-            
-            # Update progress
-            current_idx = samples.index(sample)
-            new_progress = f"Progress: {sum(completed_samples.values())}/{len(samples)}"
-            
-            new_status = f"Viewing query {current_idx + 1} of {len(samples)}"
-            if completed_samples[sample_id]:
-                new_status += " (already completed)"
-            
-            return [new_query] + new_docs + new_rankings + [sample["id"], new_progress, new_status]
-        
-        def next_sample(current_id):
-            """Load the next sample."""
-            current_sample = next((s for s in samples if s["id"] == current_id), None)
-            if not current_sample:
-                return current_id
-            
-            current_idx = samples.index(current_sample)
-            if current_idx < len(samples) - 1:
-                next_sample = samples[current_idx + 1]
-                return next_sample["id"]
-            return current_id
+        js_code = """
+        <script src=\"https://cdn.jsdelivr.net/npm/sortablejs@1.15.0/Sortable.min.js\"></script>
+        <script>
+        function initializeSortableForApp() {
+            const container = document.getElementById('sortable-container');
+            if (!container) {
+                setTimeout(initializeSortableForApp, 200);
+                return;
+            }
+            if (typeof Sortable === 'undefined') {
+                setTimeout(initializeSortableForApp, 200);
+                return;
+            }
+            if (container.sortableInstance) {
+                return;
+            }
+            container.sortableInstance = new Sortable(container, {
+                animation: 150,
+                ghostClass: "sortable-ghost",
+                onEnd: function() {
+                    updateRanksAndSyncState();
+                }
+            });
+            updateRanksAndSyncState();
+        }
         
-        def prev_sample(current_id):
-            """Load the previous sample."""
-            current_sample = next((s for s in samples if s["id"] == current_id), None)
-            if not current_sample:
-                return current_id
-            
-            current_idx = samples.index(current_sample)
-            if current_idx > 0:
-                prev_sample = samples[current_idx - 1]
-                return prev_sample["id"]
-            return current_id
+        function updateRanksAndSyncState() {
+            const container = document.getElementById('sortable-container');
+            if (!container) return;
+            
+            const items = container.querySelectorAll('.sortable-item');
+            const orderInput = document.querySelector('#current-order textarea');
+            if (!orderInput) {
+                return;
+            }
+            
+            const order = [];
+            items.forEach((item, index) => {
+                const rankBadge = item.querySelector('.rank-badge');
+                if (rankBadge) rankBadge.textContent = (index + 1);
+                item.className = item.className.replace(/rank-bg-\d+/g, '').trim();
+                item.classList.add(`rank-bg-${index + 1}`);
+                const docId = parseInt(item.getAttribute('data-doc-id'));
+                order.push(docId);
+            });
+            
+            const newOrderValue = JSON.stringify(order);
+            if (orderInput.value !== newOrderValue) {
+                orderInput.value = newOrderValue;
+                const event = new Event('input', { bubbles: true });
+                orderInput.dispatchEvent(event);
+            }
+        }
         
-        def save_results():
-            """Save all collected results to a file."""
-            output_path = f"{task_data['task_name']}_human_results.json"
-            with open(output_path, "w") as f:
-                json.dump(results, f, indent=2)
-            return f"✅ Results saved to {output_path} ({len(results['annotations'])} annotations)"
+        const targetNode = document.getElementById('sortable-list-container');
+        if (targetNode) {
+            const config = { childList: true, subtree: true };
+            const callback = function(mutationsList, observer) {
+                for(const mutation of mutationsList) {
+                    if (mutation.type === 'childList') {
+                        if (document.getElementById('sortable-container')){
+                            initializeSortableForApp();
+                        }
+                    }
+                }
+            };
+            const observer = new MutationObserver(callback);
+            observer.observe(targetNode, config);
+        }
         
-        # Resolve rank conflicts automatically
-        def resolve_rank_conflicts(*ranks):
-            ranks = list(ranks)
-            # Convert to integers with validation
-            int_ranks = []
-            for r in ranks:
-                try:
-                    r_int = int(r) if r is not None else None
-                    if r_int is not None and (r_int < 1 or r_int > len(ranks)):
-                        r_int = None
-                    int_ranks.append(r_int)
-                except:
-                    int_ranks.append(None)
-            
-            # Find duplicates
-            seen = set()
-            duplicates = set()
-            for i, r in enumerate(int_ranks):
-                if r is None:
-                    continue
-                if r in seen:
-                    duplicates.add(r)
-                seen.add(r)
-            
-            # Resolve duplicates by incrementing/shifting
-            for dup in sorted(duplicates):
-                indices = [i for i, r in enumerate(int_ranks) if r == dup]
-                # Keep the first occurrence, shift others
-                for idx in indices[1:]:
-                    # Find the next available rank
-                    next_rank = dup + 1
-                    while next_rank in int_ranks and next_rank <= len(ranks):
-                        next_rank += 1
-                    if next_rank <= len(ranks):
-                        int_ranks[idx] = next_rank
-                    else:
-                        # If no ranks available, find the first empty spot
-                        for j in range(1, len(ranks) + 1):
-                            if j not in int_ranks:
-                                int_ranks[idx] = j
-                                break
-            
-            # Convert back to original type
-            return int_ranks
+        setTimeout(initializeSortableForApp, 500);
         
-        # Connect events
-        for i, rank_input in enumerate(ranking_inputs):
-            rank_input.change(
-                resolve_rank_conflicts,
-                inputs=ranking_inputs,
-                outputs=ranking_inputs
-            )
+        </script>
+        <style>
+        .sortable-container { display: flex; flex-direction: column; gap: 8px; }
+        .sortable-item {
+            padding: 10px 15px; background-color: #fff; border: 1px solid #e0e0e0;
+            border-radius: 6px; cursor: grab; display: flex; align-items: center;
+            transition: background-color 0.2s ease;
+        }
+        .sortable-item:hover { background-color: #f8f9fa; }
+        .sortable-ghost { background-color: #d3e3fd !important; border-style: dashed !important; opacity: 0.6; }
+        .sortable-chosen { cursor: grabbing; }
+        .rank-badge {
+            display: flex; align-items: center; justify-content: center;
+            width: 28px; height: 28px; border-radius: 50%;
+            background-color: #6c757d; color: white; font-weight: bold; margin-right: 15px;
+            flex-shrink: 0;
+        }
+        .doc-content { flex: 1; line-height: 1.4; }
+        .rank-bg-1 .rank-badge { background-color: #198754; }
+        .rank-bg-2 .rank-badge { background-color: #20c997; }
+        .rank-bg-3 .rank-badge { background-color: #ffc107; color: #333; }
+        .rank-bg-4 .rank-badge { background-color: #fd7e14; }
+        .rank-bg-5 .rank-badge { background-color: #dc3545; }
+        .rank-bg-6 .rank-badge, .rank-bg-7 .rank-badge { background-color: #6f42c1; }
+        .rank-bg-8 .rank-badge, .rank-bg-9 .rank-badge { background-color: #d63384; }
+        .rank-bg-10 .rank-badge, .rank-bg-11 .rank-badge, .rank-bg-12 .rank-badge, 
+        .rank-bg-13 .rank-badge, .rank-bg-14 .rank-badge, .rank-bg-15 .rank-badge, 
+        .rank-bg-16 .rank-badge, .rank-bg-17 .rank-badge, .rank-bg-18 .rank-badge, 
+        .rank-bg-19 .rank-badge, .rank-bg-20 .rank-badge { 
+            background-color: #6c757d;
+        }
+        </style>
+        """
+        gr.HTML(js_code)
         
-        # Connect events
         submit_btn.click(
             save_ranking,
-            inputs=ranking_inputs + [current_sample_id],
+            inputs=[order_state, current_sample_id],
             outputs=[status_box, progress_text]
         )
         
         next_btn.click(
-            next_sample,
-            inputs=[current_sample_id],
-            outputs=[current_sample_id]
+            next_sample_id, inputs=[current_sample_id], outputs=[current_sample_id]
         ).then(
             load_sample,
             inputs=[current_sample_id],
-            outputs=[query_text] + doc_containers + ranking_inputs + [current_sample_id, progress_text, status_box]
+            outputs=[query_text, sortable_list, order_state, progress_text, status_box]
         )
         
         prev_btn.click(
-            prev_sample,
-            inputs=[current_sample_id],
-            outputs=[current_sample_id]
+            prev_sample_id, inputs=[current_sample_id], outputs=[current_sample_id]
         ).then(
             load_sample,
             inputs=[current_sample_id],
-            outputs=[query_text] + doc_containers + ranking_inputs + [current_sample_id, progress_text, status_box]
+            outputs=[query_text, sortable_list, order_state, progress_text, status_box]
         )
         
         save_btn.click(save_results, outputs=[status_box])
-    
+        
+        demo.load(lambda: load_sample(samples[0]['id']), 
+                  outputs=[query_text, sortable_list, order_state, progress_text, status_box])
+        
     return demo
 
 # Main app with file upload capability