Add open PR checking & Add prompt injection in translation step

agent/handler.py

@@ -13,6 +13,7 @@ from agent.workflow import (
     generate_github_pr,
 )
 from pr_generator.searcher import find_reference_pr_simple_stream
+from translator.content import get_full_prompt, get_content, preprocess_content
 
 
 # State management
@@ -22,6 +23,7 @@ class ChatState:
         self.target_language = "ko"
         self.k_files = 10
         self.files_to_translate = []
+        self.additional_instruction = ""
         self.current_file_content = {"translated": ""}
         self.pr_result = None  # Store PR creation result
         # GitHub configuration
@@ -127,8 +129,8 @@ def start_translation_process():
 
     # Call translation function (simplified for demo)
     try:
-        status, translated = translate_docs_interactive(
-            state.target_language, [[current_file]]
+        translated = translate_docs_interactive(
+            state.target_language, [[current_file]], state.additional_instruction
         )
 
         state.current_file_content = {"translated": translated}
@@ -152,10 +154,15 @@ def start_translation_process():
             ""
             f"{original_file_link}\n"
             "**🌐 Translated Content:**\n"
-            f"\n```\n\n{_extract_content_for_display(translated)}\n```"
+            # f"\n```\n\n{_extract_content_for_display(translated)}\n```"
+            # "\n```\n\n"
+            # f"\n{translated}\n"
+            # f"```"
             # f"{status}\n"
             # "✅ Translation completed. The code block will be added when generating PR."
         )
+        return response, translated
+
 
     except Exception as e:
         response = f"❌ Translation failed: {str(e)}"
@@ -211,12 +218,14 @@ def handle_user_message(message, history):
         # User wants to start translation
         if state.files_to_translate:
             state.step = "translate"
-            response = start_translation_process()
+            response, translated = start_translation_process()
+            history.append([message, response])
+            history.append(["", translated])
+            return history, ""
         else:
             response = (
                 "❌ No files available for translation. Please search for files first."
             )
-
     # Handle GitHub PR creation - This part is removed as approve_handler is the main entry point
     else:
         # General response
@@ -308,15 +317,43 @@ def update_github_config(token, owner, repo, reference_pr_url):
     return f"✅ GitHub configuration updated: {owner}/{repo}"
 
 
+def update_prompt_preview(language, file_path, additional_instruction):
+    """Update prompt preview based on current settings"""
+    if not file_path.strip():
+        return "Select a file to see the prompt preview..."
+    
+    try:
+        # Get language name
+        if language == "ko":
+            translation_lang = "Korean"
+        else:
+            translation_lang = language
+        
+        # Get sample content (first 500 characters)
+        content = get_content(file_path)
+        to_translate = preprocess_content(content)
+        
+        # Truncate for preview
+        sample_content = to_translate[:500] + ("..." if len(to_translate) > 500 else "")
+        
+        # Generate prompt
+        prompt = get_full_prompt(translation_lang, sample_content, additional_instruction)
+        
+        return prompt
+    except Exception as e:
+        return f"Error generating prompt preview: {str(e)}"
+
+
 def send_message(message, history):
     new_history, cleared_input = handle_user_message(message, history)
     return new_history, cleared_input, update_status()
 
 
 # Button handlers with tab switching
-def start_translate_handler(history, anthropic_key, file_to_translate):
+def start_translate_handler(history, anthropic_key, file_to_translate, additional_instruction=""):
     os.environ["ANTHROPIC_API_KEY"] = anthropic_key
-
+    
+    state.additional_instruction = additional_instruction
     state.files_to_translate = [file_to_translate]
     new_hist, cleared_input = handle_user_message("start translation", history)
     selected_tabs = 2 if state.current_file_content["translated"] else 0

agent/workflow.py

@@ -51,7 +51,7 @@ def report_in_translation_status_files(translate_lang: str) -> tuple[str, list[s
     return status_report, docs
 
 
-def translate_docs(lang: str, file_path: str) -> tuple[str, str]:
+def translate_docs(lang: str, file_path: str, additional_instruction: str = "") -> tuple[str, str]:
     """Translate documentation."""
     # step 1. Get content from file path
     content = get_content(file_path)
@@ -60,7 +60,7 @@ def translate_docs(lang: str, file_path: str) -> tuple[str, str]:
     # step 2. Prepare prompt with docs content
     if lang == "ko":
         translation_lang = "Korean"
-    to_translate_with_prompt = get_full_prompt(translation_lang, to_translate)
+    to_translate_with_prompt = get_full_prompt(translation_lang, to_translate, additional_instruction)
 
     print("to_translate_with_prompt:\n", to_translate_with_prompt)
 
@@ -77,8 +77,8 @@ def translate_docs(lang: str, file_path: str) -> tuple[str, str]:
 
 
 def translate_docs_interactive(
-    translate_lang: str, selected_files: list[list[str]]
-) -> tuple[str, str, str]:
+    translate_lang: str, selected_files: list[list[str]], additional_instruction: str = ""
+) -> tuple[str, str]:
     """Interactive translation function that processes files one by one.
 
     Args:
@@ -87,27 +87,17 @@ def translate_docs_interactive(
     """
     # Extract file paths from the dataframe format
     file_paths = [row[0] for row in selected_files if row and len(row) > 0]
-    if not file_paths:
-        return (
-            "No files selected for translation.",
-            gr.update(visible=False),
-            gr.update(visible=False),
-            gr.update(visible=False),
-            [],
-            0,
-        )
 
     # Start with the first file
     current_file = file_paths[0]
 
     status = f"✅ Translation completed: `{current_file}` → `{translate_lang}`\n\n"
-    callback_result, translated_content = translate_docs(translate_lang, current_file)
+    callback_result, translated_content = translate_docs(translate_lang, current_file, additional_instruction)
     status += f"💰 Used token and cost: \n```\n{callback_result}\n```"
 
-    if len(file_paths) > 1:
-        status += f"\n### 📝 Note: Currently, only the first file has been translated.\n> The remaining {len(file_paths) - 1} files have not been processed yet, as the system is in its beta version"
+    print(status)
 
-    return status, translated_content
+    return translated_content
 
 
 def generate_github_pr(

app.py

@@ -14,6 +14,7 @@ from agent.handler import (
     send_message,
     start_translate_handler,
     sync_language_displays,
+    update_prompt_preview,
     update_status,
     update_github_config,
 )
@@ -30,7 +31,7 @@ css = """
     background: rgba(255, 255, 180, 0.25);
     border-radius: 18px;
     box-shadow: 0 4px 24px rgba(0,0,0,0.08);
-    padding: 1.5em;
+    padding: 1.0em;
     backdrop-filter: blur(8px);
     border: 1px solid rgba(255,255,180,0.25);
     width: 100%;
@@ -40,7 +41,7 @@ css = """
     background: rgba(255, 255, 180, 0.25);
     border-radius: 18px;
     box-shadow: 0 4px 24px rgba(0,0,0,0.08);
-    padding: 1.5em;
+    padding: 1.0em;
     backdrop-filter: blur(8px);
     border: 1px solid rgba(255,255,180,0.25);
     width: 100%;
@@ -106,11 +107,12 @@ with gr.Blocks(
     # Content
     with gr.Row():
         # Chat interface
-        with gr.Column(scale=4, elem_classes=["chat-container"]):
+        with gr.Column(scale=3, elem_classes=["chat-container"]):
             gr.Markdown("### 🌐 Hugging Face i18n Agent")
 
             chatbot = gr.Chatbot(
-                value=[[None, get_welcome_message()]], scale=1, height=585
+                value=[[None, get_welcome_message()]], scale=1, height=585,
+                show_copy_button=True
             )
 
         # Controller interface
@@ -144,7 +146,7 @@ with gr.Blocks(
                                 choices=[],
                                 label="📄 Select a file to translate",
                                 interactive=True,
-                                value=[],
+                                value=None,
                             )
                             file_to_translate_input = gr.Textbox(
                                 label="🌍 Select in the dropdown or write the file path to translate",
@@ -161,6 +163,21 @@ with gr.Blocks(
                                 label="🔑 Anthropic API key for translation generation",
                                 type="password",
                             )
+                            additional_instruction = gr.Textbox(
+                                label="📝 Additional instructions (Optional - e.g., custom glossary)",
+                                placeholder="Example: Translate 'model' as '모델' consistently",
+                                lines=2,
+                            )
+                            
+                            with gr.Accordion("🔍 Preview Prompt", open=False):
+                                prompt_preview = gr.Textbox(
+                                    label="Current Translation Prompt",
+                                    lines=8,
+                                    interactive=False,
+                                    placeholder="Select a file and language to see the prompt preview...",
+                                    show_copy_button=True,
+                                )
+                            
                             start_translate_btn = gr.Button(
                                 "🚀 Start Translation", elem_classes="action-button"
                             )
@@ -230,7 +247,7 @@ with gr.Blocks(
     # Button event handlers
     start_translate_btn.click(
         fn=start_translate_handler,
-        inputs=[chatbot, anthropic_key, file_to_translate_input],
+        inputs=[chatbot, anthropic_key, file_to_translate_input, additional_instruction],
         outputs=[chatbot, msg_input, status_display, control_tabs],
     )
 
@@ -265,5 +282,13 @@ with gr.Blocks(
         outputs=[chatbot, msg_input, status_display],
     )
 
+    # Update prompt preview when inputs change
+    for input_component in [translate_lang_display, file_to_translate_input, additional_instruction]:
+        input_component.change(
+            fn=update_prompt_preview,
+            inputs=[translate_lang_display, file_to_translate_input, additional_instruction],
+            outputs=[prompt_preview],
+        )
+
 root_path = os.environ.get("GRADIO_ROOT_PATH")
 demo.launch(root_path=root_path)

pr_generator/agent.py

@@ -518,7 +518,7 @@ Please return only the commit message. No other explanation is needed."""
                     "status": "partial_success",
                     "branch": branch_name,
                     "file_path": target_filepath,
-                    "message": f"File was saved but PR creation failed: {pr_result}",
+                    "message": f"File was saved and commit was successful.\nPR creation failed: {pr_result}",
                     "error_details": pr_result,
                 }
             elif "successful" in pr_result and "http" in pr_result:

translator/content.py

@@ -9,6 +9,9 @@ from translator.prompt_glossary import PROMPT_WITH_GLOSSARY
 
 
 def get_content(filepath: str) -> str:
+    if filepath == "":
+        raise ValueError("No files selected for translation.")
+
     url = string.Template(
         "https://raw.githubusercontent.com/huggingface/" "transformers/main/$filepath"
     ).safe_substitute(filepath=filepath)
@@ -26,25 +29,31 @@ def preprocess_content(content: str) -> str:
     ## ignore top license comment
     to_translate = content[content.find("#") :]
     ## remove code blocks from text
-    to_translate = re.sub(r"```.*?```", "", to_translate, flags=re.DOTALL)
+    # to_translate = re.sub(r"```.*?```", "", to_translate, flags=re.DOTALL)
     ## remove markdown tables from text
-    to_translate = re.sub(r"^\|.*\|$\n?", "", to_translate, flags=re.MULTILINE)
+    # to_translate = re.sub(r"^\|.*\|$\n?", "", to_translate, flags=re.MULTILINE)
     ## remove empty lines from text
     to_translate = re.sub(r"\n\n+", "\n\n", to_translate)
-
     return to_translate
 
 
-def get_full_prompt(language: str, to_translate: str) -> str:
-    prompt = string.Template(
+def get_full_prompt(language: str, to_translate: str, additional_instruction: str = "") -> str:
+    base_prompt = string.Template(
         "What do these sentences about Hugging Face Transformers "
         "(a machine learning library) mean in $language? "
         "Please do not translate the word after a 🤗 emoji "
         "as it is a product name. Output the complete markdown file**, with prose translated and all other content intact"
-        "No explanations or extras—only the translated markdown"
-        "\n\n```md"
+        "No explanations or extras—only the translated markdown. Also translate all comments within code blocks as well."
     ).safe_substitute(language=language)
-    return "\n".join([prompt, to_translate.strip(), "```", PROMPT_WITH_GLOSSARY])
+    
+    base_prompt += "\n\n```md"
+    
+    full_prompt = "\n".join([base_prompt, to_translate.strip(), "```", PROMPT_WITH_GLOSSARY])
+    
+    if additional_instruction.strip():
+        full_prompt += f"\n\n🗒️ Additional instructions: {additional_instruction.strip()}"
+    
+    return full_prompt
 
 
 def split_markdown_sections(markdown: str) -> list:
@@ -72,36 +81,84 @@ def make_scaffold(content: str, to_translate: str) -> string.Template:
     return string.Template(scaffold)
 
 
+def is_in_code_block(text: str, position: int) -> bool:
+    """Check if a position in text is inside a code block"""
+    text_before = text[:position]
+    code_block_starts = text_before.count("```")
+    return code_block_starts % 2 == 1
+
+
 def fill_scaffold(content: str, to_translate: str, translated: str) -> str:
     scaffold = make_scaffold(content, to_translate)
     print("scaffold:")
     print(scaffold.template)
+    
+    # Get original text sections to maintain structure
+    original_sections = to_translate.split("\n\n")
+    
+    # Split markdown sections to get headers and anchors
     divided = split_markdown_sections(to_translate)
     print("divided:")
     print(divided)
     anchors = get_anchors(divided)
-
-    translated = split_markdown_sections(translated)
-    print("translated:")
-    print(translated)
-
-    translated[1::3] = [
-        f"{korean_title} {anchors[i]}"
-        for i, korean_title in enumerate(translated[1::3])
-    ]
-    translated = "".join(
-        ["".join(translated[i * 3 : i * 3 + 3]) for i in range(len(translated) // 3)]
-    ).split("\n\n")
-    if newlines := scaffold.template.count("$hf_i18n_placeholder") - len(translated):
-        return str(
-            [
-                f"Please {'recover' if newlines > 0 else 'remove'} "
-                f"{abs(newlines)} incorrectly inserted double newlines."
-            ]
-        )
-
+    
+    # Split translated content by markdown sections
+    translated_divided = split_markdown_sections(translated)
+    print("translated divided:")
+    print(translated_divided)
+    
+    # Ensure we have the same number of headers as the original
+    if len(translated_divided[1::3]) != len(anchors):
+        print(f"Warning: Header count mismatch. Original: {len(anchors)}, Translated: {len(translated_divided[1::3])}")
+        # Adjust anchors list to match translated headers
+        if len(translated_divided[1::3]) < len(anchors):
+            anchors = anchors[:len(translated_divided[1::3])]
+        else:
+            # Add empty anchors for extra headers
+            anchors.extend([""] * (len(translated_divided[1::3]) - len(anchors)))
+    
+    # Add anchors to translated headers only if they're not in code blocks
+    for i, korean_title in enumerate(translated_divided[1::3]):
+        if i < len(anchors):
+            # Find the position of this header in the original translated text
+            header_pos = translated.find(korean_title.strip())
+            if header_pos != -1 and not is_in_code_block(translated, header_pos):
+                translated_divided[1 + i * 3] = f"{korean_title} {anchors[i]}"
+            else:
+                translated_divided[1 + i * 3] = korean_title
+    
+    # Reconstruct translated content with proper structure
+    reconstructed_translated = "".join([
+        "".join(translated_divided[i * 3 : i * 3 + 3]) 
+        for i in range(len(translated_divided) // 3)
+    ])
+    
+    # Split by double newlines to match original structure
+    translated_sections = reconstructed_translated.split("\n\n")
+    
+    print("scaffold template count:")
+    print(scaffold.template.count("$hf_i18n_placeholder"))
+    print("original sections length:")
+    print(len(original_sections))
+    print("translated sections length:")
+    print(len(translated_sections))
+    
+    # Ensure section counts match
+    placeholder_count = scaffold.template.count("$hf_i18n_placeholder")
+    
+    if len(translated_sections) < placeholder_count:
+        # Add empty sections if translated has fewer sections
+        translated_sections.extend([""] * (placeholder_count - len(translated_sections)))
+    elif len(translated_sections) > placeholder_count:
+        # Truncate if translated has more sections
+        translated_sections = translated_sections[:placeholder_count]
+    
+    # Final check
+    if len(translated_sections) != placeholder_count:
+        return f"Error: Section count mismatch. Expected: {placeholder_count}, Got: {len(translated_sections)}"
+    
     translated_doc = scaffold.safe_substitute(
-        {f"hf_i18n_placeholder{i}": text for i, text in enumerate(translated)}
+        {f"hf_i18n_placeholder{i}": text for i, text in enumerate(translated_sections)}
     )
     return translated_doc
 

translator/retriever.py

@@ -37,18 +37,33 @@ def get_github_issue_open_pr(lang: str = "ko"):
             "No Github issue has been registered to the server. (Only 'ko' is supported - please contact us to support this.)"
         )
 
-    url = f"https://api.github.com/repos/huggingface/transformers/pulls?state=open"
-
     headers = {
         "Accept": "application/vnd.github+json",
     }
-    response = requests.get(url, headers=headers)
-
-    if response.status_code != 200:
-        raise Exception(f"GitHub API error: {response.status_code} {response.text}")
-
-    open_prs = response.json()
-    filtered_prs = [pr for pr in open_prs if pr["title"].startswith("🌐 [i18n-KO]")]
+    
+    all_open_prs = []
+    page = 1
+    per_page = 100  # Maximum allowed by GitHub API
+    
+    while True:
+        url = f"https://api.github.com/repos/huggingface/transformers/pulls?state=open&page={page}&per_page={per_page}"
+        response = requests.get(url, headers=headers)
+        
+        if response.status_code != 200:
+            raise Exception(f"GitHub API error: {response.status_code} {response.text}")
+        
+        page_prs = response.json()
+        if not page_prs:  # No more PRs
+            break
+            
+        all_open_prs.extend(page_prs)
+        page += 1
+        
+        # Break if we got less than per_page results (last page)
+        if len(page_prs) < per_page:
+            break
+
+    filtered_prs = [pr for pr in all_open_prs if pr["title"].startswith("🌐 [i18n-KO]")]
 
     pattern = re.compile(r"`([^`]+\.md)`")