Initial commit

.gitignore

@@ -0,0 +1 @@
+gsw_tokenizer/sentencepiece.bpe.model

app.py

@@ -0,0 +1,142 @@
+import gradio as gr
+import utils
+import os
+
+# Create a custom theme
+theme = gr.themes.Base(
+    text_size="lg",
+    radius_size="none",
+    font=[gr.themes.GoogleFont('Source Sans 3'), 'ui-sans-serif', 'system-ui', 'sans-serif'],
+)
+
+# Load tokenizers only once during development
+if gr.NO_RELOAD:
+    print("Loading tokenizers...")
+    all_tokenizers = utils.load_tokenizers()
+    all_tokenizer_names = list(all_tokenizers.keys())
+    print("Tokenizers loaded!")
+
+def read_svg_file(name: str) -> str:
+    """Read SVG file content."""
+    icon_map = {
+        "meta-llama/Llama-4-Scout-17B-16E-Instruct": "llama.svg",
+        "deepseek-ai/DeepSeek-V3-0324": "deepseek.svg",
+        "ZurichNLP/swissbert": "swissbert.svg",
+        "mistralai/Mistral-Nemo-Instruct-2407": "mistral.svg",
+        "google/gemma-3-27b-it": "gemma.svg",
+        "gpt-4o": "chatgpt.svg"
+    }
+    icon_path = os.path.join("icons", icon_map.get(name, "chatgpt.svg"))
+    try:
+        with open(icon_path, 'r') as f:
+            return f.read()
+    except Exception as e:
+        print(f"Error reading SVG file {icon_path}: {e}")
+        return ""
+
+def get_model_icon(name: str) -> str:
+    """Get the HTML for the model icon."""
+    # Skip icons for collapsed models
+    if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b"]:
+        return ""
+        
+    svg_content = read_svg_file(name)
+    if svg_content:
+        # Add viewBox and preserve aspect ratio to the SVG element
+        svg_content = svg_content.replace('<svg', '<svg preserveAspectRatio="xMidYMid meet" style="height: 24px; width: 24px;"')
+        # Wrap in a container that maintains aspect ratio
+        return f'<div style="display: inline-block; vertical-align: middle; margin-right: 8px; height: 24px; width: 24px; overflow: hidden;">{svg_content}</div>'
+    return ""
+
+def process_text(text):
+    """Process the input text and return visualizations for all tokenizers."""
+    # Use the pre-loaded tokenizers
+    visualizations = utils.visualize_tokens(text, all_tokenizers)
+    return list(visualizations.values()) + [gr.update(visible=True)]
+
+# Create the Gradio interface
+with gr.Blocks(title="Tokens matter.", theme=theme, css="""
+    .tokenizer-panel > div { background: var(--input-background-fill); }
+    .no-padding { padding: 0 !important; }
+    .form { border: 0 !important; }
+    .html-container { line-height: 2em; !important; }
+    .pending { opacity: 1; }
+""") as demo:
+    gr.Markdown("# Tokens matter.")
+    
+    with gr.Row():
+        # Left column for inputs
+        with gr.Column(scale=1):
+            input_text = gr.Textbox(
+                label="Input Text:",
+                placeholder="Enter text to tokenize ...",
+                value="Als Zürcher bini nöd so Fan vom FC Basel.",
+                lines=3,
+                elem_classes="no-padding",
+                interactive=True,
+                every=True,  # This enables real-time updates
+            )
+        
+        # Right column for outputs
+        with gr.Column(scale=2):
+            # Create output boxes for main tokenizers
+            main_output_boxes = []
+            more_output_boxes = []
+            
+            # Create 2x2 grid for main tokenizers
+            with gr.Row():
+                with gr.Column():
+                    for name in all_tokenizer_names[:2]:
+                        if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b"]:
+                            continue
+                        display_name = utils.MODEL_DISPLAY_NAMES.get(name, name)
+                        with gr.Group(elem_classes="tokenizer-panel"):
+                            gr.HTML(value=f'<div style="display: flex; align-items: center; margin-bottom: 8px;">{get_model_icon(name)}<span style="font-weight: bold;">{display_name}</span></div>')
+                            box = gr.HTML()
+                            main_output_boxes.append(box)
+                with gr.Column():
+                    for name in all_tokenizer_names[2:4]:
+                        if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b"]:
+                            continue
+                        display_name = utils.MODEL_DISPLAY_NAMES.get(name, name)
+                        with gr.Group(elem_classes="tokenizer-panel"):
+                            gr.HTML(value=f'<div style="display: flex; align-items: center; margin-bottom: 8px;">{get_model_icon(name)}<span style="font-weight: bold;">{display_name}</span></div>')
+                            box = gr.HTML()
+                            main_output_boxes.append(box)
+            
+            # Display more tokenizers in accordion
+            more_models = gr.Accordion("More Models", open=False, visible=False)
+            with more_models:
+                for name in all_tokenizer_names:
+                    if name in ["google/gemma-3-27b-it", "mistralai/Mistral-Nemo-Instruct-2407", "CohereLabs/aya-expanse-8b"]:
+                        display_name = utils.MODEL_DISPLAY_NAMES.get(name, name)
+                        with gr.Group(elem_classes="tokenizer-panel"):
+                            gr.HTML(value=f'<div style="display: flex; align-items: center; margin-bottom: 8px;">{get_model_icon(name)}<span style="font-weight: bold;">{display_name}</span></div>')
+                            box = gr.HTML()
+                            more_output_boxes.append(box)
+    
+    all_outputs = main_output_boxes + more_output_boxes + [more_models]
+    
+    # Use change event for real-time updates
+    input_text.change(
+        fn=process_text,
+        inputs=[input_text],
+        outputs=all_outputs,
+        show_progress="hidden",
+    )
+
+    # Add examples
+    gr.Examples(
+        examples=[
+            ["Als Zürcher bini nöd so Fan vom FC Basel."],
+            ["Als Zürcher bin ich nicht sonderlich Fan des FC Basel."],
+            ["En tant que Zurichois, je ne suis pas un grand fan du FC Bâle."],
+            ["Come Zurighese, non sono un grande fan del FC Basilea."],
+            ["Sco Turitgais na sun jau betg in grond fan da l'FC Basilea."],
+            ["As a Zurich resident, I am not a big fan of FC Basel."],
+        ],
+        inputs=input_text
+    )
+
+if __name__ == "__main__":
+    demo.launch()

gsw_tokenizer/special_tokens_map.json

@@ -0,0 +1,40 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": "<unk>"
+}
+

gsw_tokenizer/tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}

requirements.txt

@@ -0,0 +1,3 @@
+gradio==5.29.0
+transformers[sentencepiece]==4.51.3
+tiktoken==0.9.0

tests/test_utils.py

@@ -0,0 +1,136 @@
+import unittest
+from utils import load_hf_tokenizer, load_openai_tokenizer, load_tokenizers, tokenize, load_gsw_tokenizer
+
+class TestTokenizerLoading(unittest.TestCase):
+    def test_load_hf_tokenizer(self):
+        """Test loading a single HuggingFace tokenizer."""
+        name, tokenizer = load_hf_tokenizer("meta-llama/Llama-4-Scout-17B-16E-Instruct")
+        self.assertEqual(name, "meta-llama/Llama-4-Scout-17B-16E-Instruct")
+        self.assertIsNotNone(tokenizer)
+
+    def test_load_openai_tokenizer(self):
+        """Test loading a single OpenAI tokenizer."""
+        name, tokenizer = load_openai_tokenizer("gpt-4o")
+        self.assertEqual(name, "gpt-4o")
+        self.assertIsNotNone(tokenizer)
+
+    def test_load_tokenizers(self):
+        """Test loading all tokenizers."""
+        tokenizers = load_tokenizers()
+        
+        # Check that we have the expected number of tokenizers
+        expected_count = 6  # 5 HF + 1 OpenAI
+        self.assertEqual(len(tokenizers), expected_count)
+        
+        # Check that all expected tokenizers are present
+        expected_names = {
+            "meta-llama/Llama-4-Scout-17B-16E-Instruct",
+            "deepseek-ai/DeepSeek-V3-0324",
+            "ZurichNLP/swissbert",
+            "mistralai/Mistral-Nemo-Instruct-2407",
+            "google/gemma-3-27b-it",
+            "gpt-4o"
+        }
+        self.assertEqual(set(tokenizers.keys()), expected_names)
+        
+        # Check that all tokenizers are valid
+        for name, tokenizer in tokenizers.items():
+            self.assertIsNotNone(tokenizer)
+
+    def test_load_gsw_tokenizer(self):
+        """Test loading the Swiss German tokenizer from local files."""
+        name, tokenizer = load_gsw_tokenizer()
+        self.assertEqual(name, "swissbert-gsw")
+        self.assertIsNotNone(tokenizer)
+        
+        # Test basic tokenization functionality
+        test_text = "nöd"
+        tokens = tokenize(test_text, tokenizer)
+        self.assertIsInstance(tokens, list)
+        self.assertTrue(all(isinstance(t, str) for t in tokens))
+        self.assertTrue(len(tokens) > 0)
+
+class TestTokenizerFunctionality(unittest.TestCase):
+    def setUp(self):
+        """Set up tokenizers for testing."""
+        self.tokenizers = load_tokenizers()
+        self.test_text = "Dies ist ein Test."
+
+    def test_tokenize_llama(self):
+        """Test tokenization with Llama tokenizer."""
+        tokenizer = self.tokenizers["meta-llama/Llama-4-Scout-17B-16E-Instruct"]
+        tokens = tokenize(self.test_text, tokenizer)
+        self.assertIsInstance(tokens, list)
+        self.assertTrue(all(isinstance(t, str) for t in tokens))
+        self.assertTrue(len(tokens) > 0)
+        print(tokens)
+
+    def test_tokenize_deepseek(self):
+        """Test tokenization with DeepSeek tokenizer."""
+        tokenizer = self.tokenizers["deepseek-ai/DeepSeek-V3-0324"]
+        tokens = tokenize(self.test_text, tokenizer)
+        self.assertIsInstance(tokens, list)
+        self.assertTrue(all(isinstance(t, str) for t in tokens))
+        self.assertTrue(len(tokens) > 0)
+        print(tokens)
+
+    def test_tokenize_swissbert(self):
+        """Test tokenization with SwissBERT tokenizer."""
+        tokenizer = self.tokenizers["ZurichNLP/swissbert"]
+        tokens = tokenize(self.test_text, tokenizer)
+        self.assertIsInstance(tokens, list)
+        self.assertTrue(all(isinstance(t, str) for t in tokens))
+        self.assertTrue(len(tokens) > 0)
+        print(tokens)
+
+    def test_tokenize_gpt4(self):
+        """Test tokenization with GPT-4 tokenizer."""
+        tokenizer = self.tokenizers["gpt-4o"]
+        tokens = tokenize(self.test_text, tokenizer)
+        self.assertIsInstance(tokens, list)
+        self.assertTrue(all(isinstance(t, str) for t in tokens))
+        self.assertTrue(len(tokens) > 0)
+        print(tokens)
+
+    def test_tokenize_swissbert_comparison(self):
+        """Test that SwissBERT tokenization compares both tokenizers and returns shorter result."""
+        tokenizer = self.tokenizers["ZurichNLP/swissbert"]
+        
+        # Test with a Swiss German word
+        test_text = "nöd"
+        tokens = tokenize(test_text, tokenizer)
+        
+        # Verify we get a valid tokenization
+        self.assertIsInstance(tokens, list)
+        self.assertTrue(all(isinstance(t, str) for t in tokens))
+        self.assertTrue(len(tokens) > 0)
+        
+        # Get both tokenizations directly to verify comparison
+        _, gsw_tokenizer = load_gsw_tokenizer()
+        swissbert_tokens = tokenize(test_text, tokenizer)
+        gsw_tokens = tokenize(test_text, gsw_tokenizer)
+        
+        # Verify that the returned tokenization is the shorter one
+        expected_tokens = swissbert_tokens if len(swissbert_tokens) <= len(gsw_tokens) else gsw_tokens
+        self.assertEqual(tokens, expected_tokens)
+
+    def test_tokenize_mistral(self):
+        """Test tokenization with Mistral NeMo tokenizer."""
+        tokenizer = self.tokenizers["mistralai/Mistral-Nemo-Instruct-2407"]
+        tokens = tokenize(self.test_text, tokenizer)
+        self.assertIsInstance(tokens, list)
+        self.assertTrue(all(isinstance(t, str) for t in tokens))
+        self.assertTrue(len(tokens) > 0)
+        print(tokens)
+
+    def test_tokenize_gemma(self):
+        """Test tokenization with Gemma 3 tokenizer."""
+        tokenizer = self.tokenizers["google/gemma-3-27b-it"]
+        tokens = tokenize(self.test_text, tokenizer)
+        self.assertIsInstance(tokens, list)
+        self.assertTrue(all(isinstance(t, str) for t in tokens))
+        self.assertTrue(len(tokens) > 0)
+        print(tokens)
+
+if __name__ == '__main__':
+    unittest.main() 

theme.py

@@ -0,0 +1,3 @@
+import gradio as gr
+
+gr.themes.builder()

utils.py

@@ -0,0 +1,235 @@
+from typing import Dict, List, Tuple
+from pathlib import Path
+
+from transformers import AutoTokenizer
+import tiktoken
+
+# UZH color palette
+UZH_COLORS = [
+    "#BACBFF",  # UZH Blue V1
+    "#DBF4F9",  # UZH Cyan V1
+    "#ECF6D6",  # UZH Apple V1
+    "#FFF4DA",  # UZH Gold V1
+    "#FFDBCC",  # UZH Orange V1
+    "#FBC6D4",  # UZH Berry V1
+    "#C2C2C2",  # UZH Grey V1
+    "#FAFAFA",  # UZH Light Grey V1
+    "#7596FF",  # UZH Blue V2
+    "#B7E9F4",  # UZH Cyan V2
+    "#DBEDAD",  # UZH Apple V2
+    "#FFE9B5",  # UZH Gold V2
+    "#FEB799",  # UZH Orange V2
+    "#F78CAA",  # UZH Berry V2
+    "#A3A3A3",  # UZH Grey V2
+    "#EFEFEF",  # UZH Light Grey V2
+]
+
+def load_hf_tokenizer(name: str) -> Tuple[str, object]:
+    """
+    Load a single HuggingFace tokenizer.
+    
+    Args:
+        name: The name of the tokenizer to load
+        
+    Returns:
+        Tuple of (tokenizer_name, tokenizer_object)
+    """
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(
+            name,
+            use_fast=True,
+            model_max_length=1000000,
+            clean_up_tokenization_spaces=True,
+            legacy=False
+        )
+    except Exception as e:
+        tokenizer = AutoTokenizer.from_pretrained(
+            name,
+            model_max_length=1000000,
+            clean_up_tokenization_spaces=True,
+            legacy=False
+        )
+    return name, tokenizer
+
+def load_openai_tokenizer(name: str) -> Tuple[str, object]:
+    """
+    Load a single OpenAI tokenizer.
+    
+    Args:
+        name: The name of the tokenizer to load
+        
+    Returns:
+        Tuple of (tokenizer_name, tokenizer_object)
+    """
+    return name, tiktoken.encoding_for_model(name)
+
+def load_gsw_tokenizer() -> Tuple[str, object]:
+    """
+    Load a Swiss German (GSW) tokenizer from local vocabulary files in gsw_tokenizer directory.
+    
+    Returns:
+        Tuple of (tokenizer_name, tokenizer_object)
+    """
+    tokenizer_path = Path(__file__).parent / "gsw_tokenizer"
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
+    return "swissbert-gsw", tokenizer
+
+def load_tokenizers() -> Dict[str, object]:
+    """
+    Load all tokenizers.
+    
+    Returns:
+        Dictionary mapping tokenizer names to tokenizer objects
+    """
+    tokenizers = {}
+    
+    # Load OpenAI tokenizers first
+    openai_names = ["gpt-4o"]
+    for name in openai_names:
+        tokenizer_name, tokenizer = load_openai_tokenizer(name)
+        tokenizers[tokenizer_name] = tokenizer
+    
+    # Load HuggingFace tokenizers in specified order
+    hf_names = [
+        "meta-llama/Llama-4-Scout-17B-16E-Instruct",
+        "deepseek-ai/DeepSeek-V3-0324",
+        "ZurichNLP/swissbert",
+        "google/gemma-3-27b-it",
+        "mistralai/Mistral-Nemo-Instruct-2407",
+        "CohereLabs/aya-expanse-8b",
+    ]
+    for name in hf_names:
+        tokenizer_name, tokenizer = load_hf_tokenizer(name)
+        tokenizers[tokenizer_name] = tokenizer
+    
+    return tokenizers
+
+# Mapping of model names to display names
+MODEL_DISPLAY_NAMES = {
+    "meta-llama/Llama-4-Scout-17B-16E-Instruct": "Llama 4",
+    "deepseek-ai/DeepSeek-V3-0324": "DeepSeek V3",
+    "ZurichNLP/swissbert": "SwissBERT 🇨🇭",
+    "mistralai/Mistral-Nemo-Instruct-2407": "Mistral NeMo",
+    "google/gemma-3-27b-it": "Gemma 3",
+    "gpt-4o": "ChatGPT (gpt-4o)",
+    "CohereLabs/aya-expanse-8b": "Aya Expanse"
+}
+
+def tokenize(s: str, tokenizer) -> List[str]:
+    """
+    Tokenize a string using any tokenizer from load_hf_tokenizers() or load_openai_tokenizers().
+    For SwissBERT tokenizer, compares both SwissBERT and SwissBERT-GSW tokenizations and returns the shorter one.
+    
+    Args:
+        s: The string to tokenize
+        tokenizer: A tokenizer from load_hf_tokenizers() or load_openai_tokenizers()
+        
+    Returns:
+        A list of tokens, with special tokens removed and any tail token markers (## or @@) removed
+    """
+    # Special handling for SwissBERT tokenizer
+    if hasattr(tokenizer, "name_or_path") and "swissbert" in tokenizer.name_or_path.lower():
+        # Get SwissBERT-GSW tokenizer
+        _, gsw_tokenizer = load_gsw_tokenizer()
+        
+        # Get tokenizations from both tokenizers§
+        swissbert_tokens = _tokenize_with_tokenizer(s, tokenizer)
+        gsw_tokens = _tokenize_with_tokenizer(s, gsw_tokenizer)
+        
+        # Return the shorter tokenization
+        shorter_tokens = swissbert_tokens if len(swissbert_tokens) <= len(gsw_tokens) else gsw_tokens
+        if len(shorter_tokens) > 0 and shorter_tokens[0].startswith(" "):
+            shorter_tokens[0] = shorter_tokens[0][1:]
+        return shorter_tokens
+    
+    return _tokenize_with_tokenizer(s, tokenizer)
+
+def _tokenize_with_tokenizer(s: str, tokenizer) -> List[str]:
+    """
+    Internal helper function to tokenize a string with a given tokenizer.
+    
+    Args:
+        s: The string to tokenize
+        tokenizer: A tokenizer object
+        
+    Returns:
+        A list of tokens, with special tokens removed and any tail token markers (## or @@) removed
+    """
+    if hasattr(tokenizer, "tokenize"):
+        encoded = tokenizer.encode(s, add_special_tokens=False)
+        if hasattr(tokenizer, "name_or_path") and any(name in tokenizer.name_or_path.lower() for name in ["llama", "deepseek", "mistral", "aya"]):
+            tokens = [tokenizer.decode([token_id], skip_special_tokens=False) for token_id in encoded]
+        else:
+            tokens = tokenizer.convert_ids_to_tokens(encoded)
+        
+        filtered_tokens = []
+        for t in tokens:
+            if t.startswith("<") or t.startswith("["):
+                continue
+            elif "Ġ" in t:
+                filtered_tokens.append(t.replace("Ġ", " "))
+            elif "Ċ" in t:
+                filtered_tokens.append(t.replace("Ċ", " "))
+            elif t.startswith("▁"):
+                filtered_tokens.append(" " + t[1:])
+            else:
+                filtered_tokens.append(t)
+        
+        return [t.rstrip("##").rstrip("@@") for t in filtered_tokens]
+    
+    elif hasattr(tokenizer, "encode"):
+        token_ids = tokenizer.encode(s)
+        return [tokenizer.decode([token_id]) for token_id in token_ids]
+    
+    else:
+        raise ValueError("Unsupported tokenizer type")
+
+def get_uzh_color(index):
+    """Get a color from the UZH color palette based on index."""
+    return UZH_COLORS[index % len(UZH_COLORS)]
+
+def visualize_tokens(text: str, tokenizers: Dict[str, object]):
+    """
+    Tokenize text with each tokenizer and visualize the tokens with colors.
+    Colors are consistent across tokenizers for the same token sequences.
+    Colors are deterministic based on token content.
+    
+    Args:
+        text: The input text to tokenize
+        tokenizers: Dictionary of tokenizers
+        
+    Returns:
+        Dictionary mapping tokenizer names to HTML visualizations
+    """
+    results = {}
+    
+    # First pass: collect all unique tokens across all tokenizers
+    all_tokens = set()
+    for tokenizer in tokenizers.values():
+        tokens = tokenize(text, tokenizer)
+        all_tokens.update(tokens)
+    
+    # Generate colors for all unique tokens using hash-based approach
+    token_colors = {}
+    for token in all_tokens:
+        # Use hash of token to get a deterministic index
+        token_hash = hash(token)
+        # Ensure positive index and wrap around to color list length
+        index = abs(token_hash) % len(UZH_COLORS)
+        token_colors[token] = get_uzh_color(index)
+    
+    # Second pass: create visualizations using the consistent colors
+    for name, tokenizer in tokenizers.items():
+        tokens = tokenize(text, tokenizer)
+        
+        # Create a colored visualization
+        html = ""
+        
+        # Build the HTML with colored spans for each token
+        for token in tokens:
+            color = token_colors[token]
+            html += f'<span style="background-color: {color}; padding: 2px; margin: 1px; border-radius: 3px;">{token}</span>'
+        
+        results[name] = html
+    
+    return results