feat: Enhance embeddings and LLM interpretation

app.py

@@ -65,10 +65,14 @@ def main_interface():
                     )
                     
                     model_dropdown = gr.Dropdown(
-                        choices=["Facebook FastText (Pre-trained)"],
+                        choices=[
+                            "sentence-transformers/LaBSE", 
+                            "intfloat/e5-base-v2",
+                            "Facebook FastText (Pre-trained)"
+                        ],
                         label="Select Embedding Model",
-                        value="Facebook FastText (Pre-trained)",
-                        info="Using Facebook's pre-trained FastText model for semantic similarity. Other model options have been removed."
+                        value="sentence-transformers/LaBSE",
+                        info="Select the embedding model to use for semantic similarity analysis."
                     )
                     
                     stopwords_dropdown = gr.Dropdown(

pipeline/fasttext_embedding.py

@@ -136,7 +136,7 @@ def train_fasttext_model(
     return model
 
 
-def load_facebook_official_tibetan_model() -> Optional[fasttext.FastText._FastText]:
+def _load_facebook_official_tibetan_model() -> Optional[fasttext.FastText._FastText]:
     """
     Downloads (if necessary) and loads the official Facebook FastText Tibetan model.
 
@@ -424,6 +424,38 @@ def get_batch_embeddings(
     return np.array(embeddings)
 
 
+def get_model(model_id: str):
+    """
+    Loads a FastText model based on the provided model ID.
+
+    Args:
+        model_id (str): The identifier for the model to load.
+
+    Returns:
+        Tuple[Optional[Any], Optional[str]]: A tuple containing the loaded model and its type ('fasttext'),
+                                              or (None, None) if loading fails.
+    """
+    logger.info(f"Attempting to load FastText model: {model_id}")
+
+    if model_id == "facebook-fasttext-pretrained":
+        try:
+            model = _load_facebook_official_tibetan_model()
+            if model:
+                logger.info(f"FastText model '{model_id}' loaded successfully.")
+                return model, "fasttext"
+            else:
+                logger.error(f"Model loading for '{model_id}' returned None.")
+                return None, None
+        except Exception as e:
+            logger.error(f"Failed to load FastText model '{model_id}': {e}", exc_info=True)
+            return None, None
+    # Add logic for other custom models here if needed
+    # elif model_id == "custom-model-name":
+    #     ...
+    else:
+        logger.error(f"Unsupported model_id for get_model: '{model_id}'.")
+        return None, None
+
 def generate_embeddings(
     texts: List[str],
     model: fasttext.FastText._FastText,

pipeline/hf_embedding.py

@@ -0,0 +1,58 @@
+import logging
+from typing import List, Any, Optional, Tuple
+import numpy as np
+from sentence_transformers import SentenceTransformer
+
+logger = logging.getLogger(__name__)
+
+# Cache for loaded models
+_model_cache = {}
+
+def get_model(model_id: str) -> Tuple[Optional[SentenceTransformer], Optional[str]]:
+    """
+    Loads a SentenceTransformer model from the Hugging Face Hub.
+
+    Args:
+        model_id (str): The identifier for the model to load (e.g., 'sentence-transformers/LaBSE').
+
+    Returns:
+        Tuple[Optional[SentenceTransformer], Optional[str]]: A tuple containing the loaded model and its type ('sentence-transformer'),
+                                                              or (None, None) if loading fails.
+    """
+    if model_id in _model_cache:
+        logger.info(f"Returning cached model: {model_id}")
+        return _model_cache[model_id], "sentence-transformer"
+
+    logger.info(f"Loading SentenceTransformer model: {model_id}")
+    try:
+        model = SentenceTransformer(model_id)
+        _model_cache[model_id] = model
+        logger.info(f"Model '{model_id}' loaded successfully.")
+        return model, "sentence-transformer"
+    except Exception as e:
+        logger.error(f"Failed to load SentenceTransformer model '{model_id}': {e}", exc_info=True)
+        return None, None
+
+def generate_embeddings(texts: List[str], model: SentenceTransformer) -> Optional[np.ndarray]:
+    """
+    Generates embeddings for a list of texts using a SentenceTransformer model.
+
+    Args:
+        texts (list[str]): A list of texts to embed.
+        model (SentenceTransformer): The loaded SentenceTransformer model.
+
+    Returns:
+        Optional[np.ndarray]: A numpy array containing the embeddings. Returns None if generation fails.
+    """
+    if not texts or not isinstance(model, SentenceTransformer):
+        logger.warning("Invalid input for generating embeddings. Texts list is empty or model is not a SentenceTransformer.")
+        return None
+
+    logger.info(f"Generating embeddings for {len(texts)} texts with {type(model).__name__}...")
+    try:
+        embeddings = model.encode(texts, convert_to_numpy=True, show_progress_bar=False)
+        logger.info(f"Embeddings generated with shape: {embeddings.shape}")
+        return embeddings
+    except Exception as e:
+        logger.error(f"An unexpected error occurred during embedding generation: {e}", exc_info=True)
+        return None

pipeline/llm_service.py

@@ -26,7 +26,7 @@ except ImportError:
 
 # Constants
 DEFAULT_MAX_TOKENS = 4000
-DEFAULT_MODEL = "mistralai/mistral-7b-instruct"
+DEFAULT_MODEL = "moonshotai/kimi-k2:free"
 DEFAULT_TEMPERATURE = 0.3
 DEFAULT_TOP_P = 0.9
 
@@ -373,58 +373,17 @@ class LLMService:
         csv_data = df.to_csv(index=False)
         
         # Create the prompt using the user's template
-        prompt = """You are a specialized text analysis interpreter with expertise in Tibetan textual studies. Your task is to analyze text similarity data from a CSV file and create a clear, narrative explanation for scholars who may not have technical expertise.
+        prompt = """Please analyze the following CSV data, which contains text similarity metrics for Tibetan texts.
 
-<CONTEXT>
-This data comes from a text similarity analysis tool designed for various genres of Tibetan sources including historical, religious, literary, and philosophical texts. The tool compares texts using multiple linguistic metrics:
-- Jaccard Similarity (%): Measures word overlap between texts (higher % = more similar)
-- Normalized LCS: Longest Common Subsequence, measuring sequential text patterns
-- Semantic Similarity: Deep meaning comparison using sentence transformers or fasttext
-- TF-IDF Cosine Similarity: Term frequency-inverse document frequency comparison
-The "Chapter" column indicates which chapter/section of the texts is being compared.
-</CONTEXT>
+Your analysis should be a clear, narrative explanation (800-1000 words) suitable for scholars of Tibetan studies. The analysis should cover:
+1.  An introduction to the texts being compared and a high-level overview of the similarity patterns.
+2.  A detailed examination of the most and least similar chapters, explaining whether the similarities are more lexical (Jaccard, LCS) or conceptual (Semantic).
+3.  An interpretation of what these patterns suggest about the texts' relationships, such as their transmission history or potential shared sources.
+4.  A conclusion that summarizes your findings and raises 2-3 questions for further scholarly investigation.
 
-<INSTRUCTIONS>
-1. Begin by identifying the specific texts being compared in the data (e.g., "Japan13.txt vs Dolanji.txt").
+**Important:** If you find chapters with perfect (1.0 or 100%) or zero similarity, clarify that this may be due to empty or missing text segments, rather than a true textual match or mismatch.
 
-2. Create a dual-layer narrative analysis (800-1000 words) that includes:
-   a) A high-level overview of text similarity patterns accessible to non-technical readers
-   b) A more detailed analysis for scholars interested in specific textual relationships
-
-3. In your analysis:
-   - Summarize overall similarity patterns between the texts across all chapters
-   - Identify which chapters show strongest similarities and differences
-   - Explain whether similarities appear to be more lexical (Jaccard, LCS) or conceptual (Semantic)
-   - Interpret what these patterns might suggest about textual relationships, transmission, or variant histories
-   - Note any interesting anomalies (e.g., chapters with high semantic but low lexical similarity)
-
-4. Structure your analysis with:
-   - An introduction explaining the texts compared and general observations
-   - A section on overall patterns across all chapters with visualized trends
-   - A detailed examination of 2-3 notable chapters (highest/lowest similarity)
-   - A discussion of what different metrics reveal about textual relationships
-   - A conclusion suggesting what these patterns might mean for Tibetan textual scholarship
-   - 2-3 specific questions these findings raise for further investigation
-
-5. Connect your analysis to common interests in Tibetan textual studies such as:
-   - Textual transmission and lineages
-   - Regional variants and dialectical differences
-   - Potential historical relationships between texts
-   - Original vs. commentary material identification
-
-6. Consider using a "family tree" analogy to make the textual relationships more intuitive. For example:
-   - Texts with very high similarity (>80%) might be described as "siblings" from the same direct source
-   - Texts with moderate similarity (50-80%) could be "cousins" sharing a common ancestor but with separate development
-   - Texts with low similarity (<50%) might be "distant relatives" with only fundamental connections
-   Use this metaphor if it helps clarify the relationships, but don't force it if another explanation would be clearer.
-
-7. **Important note on perfect or zero similarity matches:**  
-   If you notice that all metrics indicate perfect or near-perfect similarity (for example, scores of 1.0/100 across all metrics for a chapter) or 0 for a complete mismatch, this may not indicate true textual identity or lack thereof. Instead, it likely means both corresponding text cells were empty or contained no content. In these cases, be sure to clarify in your narrative that such results are *artifacts of missing data, not genuine textual matches*, and should be interpreted with caution.
-
-8. Balance scholarly precision with accessibility, explaining technical concepts when necessary while keeping the overall narrative engaging for non-technical readers.
-</INSTRUCTIONS>
-
-Here is the CSV data to analyze:
+Here is the CSV data:
 [CSV_DATA]
 """
         
@@ -435,33 +394,7 @@ Here is the CSV data to analyze:
     
     def _get_system_prompt(self) -> str:
         """Get the system prompt for the LLM."""
-        return """
-        You are a senior scholar of Tibetan Buddhist texts with expertise in textual criticism and 
-        comparative analysis. Your task is to analyze the provided similarity metrics and provide 
-        expert-level insights into the relationships between these Tibetan texts.
-        
-        CRITICAL INSTRUCTIONS:
-        1. Your analysis MUST be grounded in the specific metrics provided
-        2. Always reference actual text names and metric values when making claims
-        3. Focus on what the data shows, not what it might show
-        4. Be precise and avoid vague or generic statements
-        
-        ANALYSIS APPROACH:
-        1. Begin with a brief executive summary of the most significant findings
-        2. Group similar text pairs and explain their relationships
-        3. Highlight any patterns that suggest textual transmission or common sources
-        4. Note any anomalies or unexpected results that merit further investigation
-        5. Provide specific examples from the data to support your analysis
-        
-        TIBETAN TEXT-SPECIFIC GUIDANCE:
-        - Consider the implications of shared vocabulary in the context of Tibetan Buddhist literature
-        - Be aware that high LCS scores might indicate shared liturgical or formulaic language
-        - Note that texts with similar Jaccard but different LCS scores might share content but differ in structure
-        - Consider the possibility of text reuse, commentary traditions, or shared sources
-        
-        Your analysis should be scholarly but accessible, providing clear insights that would be 
-        valuable to researchers studying these texts.
-        """
+        return """You are a senior scholar of Tibetan Buddhist texts, specializing in textual criticism. Your task is to analyze the provided similarity metrics and provide expert insights into the relationships between these texts. Ground your analysis in the data, be precise, and focus on what the metrics reveal about the texts' transmission and history."""
     
     def _call_openrouter_api(
         self,

pipeline/metrics.py

@@ -3,7 +3,9 @@ import pandas as pd
 from typing import List, Dict, Union
 from itertools import combinations
 from sklearn.metrics.pairwise import cosine_similarity
-from .semantic_embedding import generate_embeddings
+from .fasttext_embedding import generate_embeddings as generate_fasttext_embeddings
+from .hf_embedding import generate_embeddings as generate_hf_embeddings
+
 from .tokenize import tokenize_texts
 import logging
 from sklearn.feature_extraction.text import TfidfVectorizer
@@ -99,20 +101,19 @@ def compute_semantic_similarity(
             )
             return None
             
-        embedding = generate_embeddings(
-            texts=[raw_text_segment], 
-            model=model_obj, 
-            tokenize_fn=tokenize_fn_param,
-            use_stopwords=use_stopwords_param,
-            use_lite_stopwords=use_lite_stopwords_param,
-            corpus_token_freq=term_freq_corpus_param,
-            doc_freq_map=doc_freq_map_param,
-            total_docs_in_corpus=total_docs_in_corpus_param
-        )
+        if model_type == "fasttext":
+            embedding = generate_fasttext_embeddings(
+                texts=[raw_text_segment], 
+                model=model_obj, 
+                use_stopwords=use_stopwords_param,
+                use_lite_stopwords=use_lite_stopwords_param
+            )
+        elif model_type == "sentence-transformer":
+            embedding = generate_hf_embeddings(texts=[raw_text_segment], model=model_obj)
         
         if embedding is None or embedding.size == 0: 
             logger.error(
-                f"Failed to generate FastText embedding for text: {raw_text_segment[:100]}..."
+                f"Failed to generate embedding for text: {raw_text_segment[:100]}..."
             )
             return None
         return embedding

pipeline/process.py

@@ -1,8 +1,9 @@
 import pandas as pd
 from typing import Dict, List, Tuple
 from .metrics import compute_all_metrics
-from .semantic_embedding import get_model_and_device
-from .fasttext_embedding import load_fasttext_model # Added for custom fasttext
+from .fasttext_embedding import get_model as get_fasttext_model
+from .hf_embedding import get_model as get_hf_model
+from .fasttext_embedding import load_fasttext_model
 from .tokenize import tokenize_texts
 import logging
 from itertools import combinations
@@ -51,7 +52,7 @@ def process_texts(
     text_data: Dict[str, str], 
     filenames: List[str], 
     enable_semantic: bool = True,
-    model_name: str = "buddhist-nlp/buddhist-sentence-similarity",
+    model_name: str = "facebook-fasttext-pretrained",
     use_stopwords: bool = True,
     use_lite_stopwords: bool = False,
     progress_callback = None
@@ -65,7 +66,7 @@ def process_texts(
         enable_semantic (bool, optional): Whether to compute semantic similarity metrics. 
             Requires loading a sentence transformer model, which can be time-consuming. Defaults to True.
         model_name (str, optional): The name of the sentence transformer model to use for semantic similarity.
-            Must be a valid model identifier on Hugging Face. Defaults to "buddhist-nlp/buddhist-sentence-similarity".
+            Must be a valid model identifier on Hugging Face. Defaults to "facebook-fasttext-pretrained".
         use_stopwords (bool, optional): Whether to use stopwords in the metrics calculation. Defaults to True.
         use_lite_stopwords (bool, optional): Whether to use the lite stopwords list (common particles only)
             instead of the comprehensive list. Only applies if use_stopwords is True. Defaults to False.
@@ -101,57 +102,20 @@ def process_texts(
     if enable_semantic:
         logger.info("Semantic similarity enabled. Loading embedding model...")
         try:
-            logger.info("Using model: %s", model_name)
+            logger.info(f"Using model: {model_name}")
+            if 'e5-base' in model_name or 'LaBSE' in model_name:
+                model, model_type = get_hf_model(model_id=model_name)
+            else:
+                model, model_type = get_fasttext_model(model_id=model_name)
 
-            if model_name == FASTTEXT_MODEL_ID:  # FASTTEXT_MODEL_ID is 'fasttext-tibetan'
-                logger.info(f"Attempting to load custom FastText model: {model_name}")
-                if progress_callback is not None:
-                    try:
-                        progress_callback(0.25, desc=f"Loading custom FastText model: {model_name}...")
-                    except Exception as e:
-                        logger.warning(f"Progress callback error (non-critical): {e}")
-                
-                loaded_custom_model = load_fasttext_model(model_id=model_name) # model_id is expected to be path or key by this func
-                if loaded_custom_model:
-                    model = loaded_custom_model
-                    model_type = "fasttext"
-                    logger.info(f"Custom FastText model '{model_name}' loaded successfully.")
-                    if progress_callback is not None:
-                        try:
-                            progress_callback(0.3, desc=f"Custom FastText model '{model_name}' loaded.")
-                        except Exception as e:
-                            logger.warning(f"Progress callback error (non-critical): {e}")
-                else:
-                    model_warning = f"Custom FastText model ('{model_name}') failed to load. Semantic similarity will be disabled."
-                    logger.warning(model_warning)
-                    enable_semantic = False
-            
-            elif model_name == "facebook-fasttext-pretrained":
-                logger.info(f"Attempting to load Facebook FastText model: {model_name}")
+            if model:
+                logger.info(f"Model '{model_name}' (type: {model_type}) loaded successfully.")
                 if progress_callback is not None:
-                    try:
-                        progress_callback(0.25, desc=f"Loading Facebook FastText model: {model_name}...")
-                    except Exception as e:
-                        logger.warning(f"Progress callback error (non-critical): {e}")
-                
-                fb_model, fb_model_type = get_model_and_device(model_id=model_name) # from semantic_embedding
-                if fb_model:
-                    model = fb_model
-                    model_type = fb_model_type  # Should be "fasttext"
-                    logger.info(f"Facebook FastText model '{model_name}' (type: {model_type}) loaded successfully.")
-                    if progress_callback is not None:
-                        try:
-                            progress_callback(0.3, desc=f"Facebook FastText model '{model_name}' loaded.")
-                        except Exception as e:
-                            logger.warning(f"Progress callback error (non-critical): {e}")
-                else:
-                    model_warning = f"Facebook FastText model ('{model_name}') failed to load. Semantic similarity will be disabled."
-                    logger.warning(model_warning)
-                    enable_semantic = False
-            
-            else:  # Any other model_name is unsupported
-                model_warning = f"Unsupported model_name: '{model_name}'. Semantic similarity will be disabled. Supported models are '{FASTTEXT_MODEL_ID}' and 'facebook-fasttext-pretrained'."
+                    progress_callback(0.3, desc=f"Model '{model_name}' loaded.")
+            else:
+                model_warning = f"Model ('{model_name}') failed to load. Semantic similarity will be disabled."
                 logger.warning(model_warning)
+                warning += f" {model_warning}"
                 enable_semantic = False
                 if progress_callback is not None:
                     try:
@@ -320,8 +284,7 @@ def process_texts(
                     enable_semantic=enable_semantic,
                     model_type=model_type,
                     use_stopwords=use_stopwords,
-                    use_lite_stopwords=use_lite_stopwords,
-                    fasttext_tokenize_fn=tokenizer_for_fasttext
+                    use_lite_stopwords=use_lite_stopwords
                 )
                 
                 # Rename 'Text Pair' to show file stems and chapter number

pipeline/semantic_embedding.py

@@ -1,141 +0,0 @@
-import logging
-from typing import List, Any, Optional
-import numpy as np # Added for type hinting Optional[np.ndarray]
-
-# Configure logging
-logging.basicConfig(
-    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
-)
-logger = logging.getLogger(__name__)
-
-# Define the model ID for the Facebook FastText pretrained model
-DEFAULT_MODEL_NAME = "facebook-fasttext-pretrained"
-
-# FASTTEXT_MODEL_ID = "fasttext-tibetan" # Removed: Custom model loading to be handled in process.py directly
-
-
-def get_model_and_device(model_id: str = DEFAULT_MODEL_NAME):
-    """
-    Loads the Facebook official pre-trained FastText model for Tibetan.
-
-    Args:
-        model_id (str): The model ID. Must be 'facebook-fasttext-pretrained' (DEFAULT_MODEL_NAME).
-
-    Returns:
-        Tuple[Optional[Any], Optional[str]]: 
-            A tuple containing the loaded FastText model and its type ("fasttext"), 
-            or (None, None) if loading fails or model_id is unsupported.
-    """
-    logger.info("Attempting to load FastText model via semantic_embedding.get_model_and_device: %s", model_id)
-
-    if model_id == DEFAULT_MODEL_NAME:  # DEFAULT_MODEL_NAME is "facebook-fasttext-pretrained"
-        try:
-            # Importing here to minimize issues if fasttext_embedding also imports from semantic_embedding
-            from .fasttext_embedding import load_facebook_official_tibetan_model
-            
-            model = load_facebook_official_tibetan_model()
-            
-            if model:
-                logger.info(f"FastText model object received in get_model_and_device. Type: {type(model)}.")
-                try:
-                    logger.info(f"Model dimensions: {model.get_dimension()}")
-                    # Basic check for model validity via an expected attribute/method
-                    if hasattr(model, 'get_word_vector'):
-                        logger.info("Model has 'get_word_vector' method (Python API expected for fasttext.load_model results).")
-                except Exception as diag_e:
-                    logger.error(f"Error during diagnostic check of FastText model '{model_id}': {diag_e}", exc_info=True)
-                return model, "fasttext"
-            else:
-                # This case implies load_facebook_official_tibetan_model returned None without raising an error.
-                logger.error(f"Model loading for '{model_id}' via load_facebook_official_tibetan_model() returned None unexpectedly.")
-                return None, None
-        except Exception as e:
-            logger.error(f"Failed to load or initialize FastText model '{model_id}': {e}. Semantic similarity will not be available.", exc_info=True)
-            return None, None
-    else:
-        logger.error(f"Unsupported model_id for get_model_and_device in semantic_embedding.py: '{model_id}'. Only '{DEFAULT_MODEL_NAME}' is supported by this function.")
-        return None, None
-
-
-def generate_embeddings(texts: List[str], model: Any, tokenize_fn=None, use_stopwords: bool = True, use_lite_stopwords: bool = False, corpus_token_freq=None, doc_freq_map=None, total_docs_in_corpus=0) -> Optional[np.ndarray]:
-    """
-    Generates FastText embeddings for a list of texts.
-
-    Args:
-        texts (list[str]): A list of texts to embed.
-        model: The loaded FastText model.
-        tokenize_fn: Optional tokenization function for FastText (if different from default botok based).
-        use_stopwords (bool): Whether to filter out stopwords for FastText embeddings.
-        use_lite_stopwords (bool): Whether to use the 'lite' stopwords list.
-        corpus_token_freq: Corpus-wide term frequencies for TF-IDF weighted FastText.
-        doc_freq_map: Document frequency map for TF-IDF weighted FastText.
-        total_docs_in_corpus: Total documents in corpus for TF-IDF weighted FastText.
-
-    Returns:
-        Optional[np.ndarray]: A numpy array containing the embeddings. Returns None if generation fails.
-    """
-    if not texts:
-        logger.warning(
-            "No texts provided to generate_embeddings. Returning None."
-        )
-        return None
-
-    logger.info(f"Generating FastText embeddings for {len(texts)} texts...")
-
-    try:
-        from .fasttext_embedding import get_batch_embeddings
-        
-        stopwords_set = None
-        if use_stopwords:
-            if use_lite_stopwords:
-                from .stopwords_lite_bo import TIBETAN_STOPWORDS_LITE_SET
-                stopwords_set = TIBETAN_STOPWORDS_LITE_SET
-            else:
-                from .stopwords_bo import TIBETAN_STOPWORDS_SET
-                stopwords_set = TIBETAN_STOPWORDS_SET
-        
-        embeddings = get_batch_embeddings(
-            texts, 
-            model, 
-            tokenize_fn=tokenize_fn, 
-            use_stopwords=use_stopwords, 
-            stopwords_set=stopwords_set,
-            corpus_token_freq=corpus_token_freq,
-            doc_freq_map=doc_freq_map,
-            total_docs_in_corpus=total_docs_in_corpus
-        )
-        if embeddings is None:
-             logger.error(f"get_batch_embeddings returned None for {len(texts)} texts. First few: {texts[:2]}")
-             return None
-
-        logger.info("FastText embeddings generated with shape: %s", str(embeddings.shape))
-        return embeddings
-    except ImportError:
-        logger.error("Required FastText modules not found. Please ensure 'fasttext' and its dependencies are correctly installed.")
-        return None
-    except Exception as e:
-        logger.error(f"An unexpected error occurred during FastText embedding generation: {e}", exc_info=True)
-        return None
-
-
-def train_fasttext_model(corpus_texts: List[str], **kwargs):
-    """
-    Train a FastText model on the provided corpus texts.
-    
-    Args:
-        corpus_texts: List of texts to use for training
-        **kwargs: Additional parameters for training (dim, epoch, etc.)
-        
-    Returns:
-        Trained model and path to the model file (Note: current implementation returns only model object)
-    """ # Docstring updated for return type
-    try:
-        from .fasttext_embedding import prepare_corpus_file, train_fasttext_model as train_ft
-        
-        corpus_path = prepare_corpus_file(corpus_texts)
-        model = train_ft(corpus_path=corpus_path, **kwargs)
-        
-        return model # Returns model object, not path as previously suggested by older docstring
-    except ImportError:
-        logger.error("FastText module not found. Please install it with 'pip install fasttext'.")
-        raise # Re-raising to signal critical failure if training components are missing

requirements.txt

@@ -12,6 +12,7 @@ numpy==1.26.4
 scikit-learn==1.6.1
 numba==0.61.2
 fasttext==0.9.2
+sentence-transformers==3.0.1
 
 # Tibetan language processing
 botok