Spaces:
Sleeping
Sleeping
| import streamlit as st | |
| import requests | |
| import pandas as pd | |
| import json | |
| import os | |
| from datasets import load_dataset | |
| # Set page configuration | |
| st.set_page_config( | |
| page_title="Huggingface Repository Explorer", | |
| page_icon="🤗", | |
| layout="wide", | |
| initial_sidebar_state="expanded" | |
| ) | |
| # Title and description | |
| st.title("🤗 Huggingface Repository Explorer") | |
| st.markdown(""" | |
| This dashboard showcases our models and datasets on Huggingface. | |
| Select a dataset to view sample data. | |
| """) | |
| # Access token will be set up via environment variable in the Huggingface Space | |
| # This way it's not exposed in the code and users don't need to enter it | |
| AUTH_TOKEN = os.environ.get("HF_TOKEN", "") | |
| # HF API endpoints | |
| HF_API_BASE = "https://huggingface.co/api" | |
| # Function to fetch dataset samples using the pre-configured token | |
| def fetch_dataset_samples(dataset_id, n=10): | |
| try: | |
| # Load the dataset in streaming mode | |
| dataset = load_dataset(dataset_id, | |
| split="train", | |
| streaming=True, | |
| token=AUTH_TOKEN) | |
| # Get the first n examples | |
| samples = [] | |
| for i, example in enumerate(dataset): | |
| if i >= n: | |
| break | |
| samples.append(example) | |
| return samples | |
| except Exception as e: | |
| st.error(f"Error loading dataset samples: {e}") | |
| return None | |
| # Hard-coded model list | |
| model_data = { | |
| "Model Name": [ | |
| "TitanCAProject/Qwen2.5-Coder-7B-Instruct_lora_r16a32-c_sharp", | |
| "TitanCAProject/Qwen2.5-Coder-7B-Instruct_lora_r16a32-python", | |
| "TitanCAProject/Qwen2.5-Coder-7B-Instruct_lora_r16a32-C", | |
| "TitanCAProject/Qwen2.5-Coder-7B-Instruct_lora_r16a32-java", | |
| "TitanCAProject/CodeBERT-javascript" | |
| ], | |
| "Description": [ | |
| "Qwen2.5 model for the Csharp language", | |
| "Qwen2.5 model for the Python language", | |
| "Qwen2.5 model for the C language", | |
| "Qwen2.5 model for the Jave language", | |
| "CodeBERT model for the Javascript language" | |
| ], | |
| "Size (GB)": [0.4, 0.5, 0.9, 1.3, 0.3], | |
| "Last Updated": [ | |
| "2024-11-15", | |
| "2024-10-30", | |
| "2024-12-05", | |
| "2024-11-20", | |
| "2024-12-10" | |
| ] | |
| } | |
| # Convert to DataFrames | |
| df_models = pd.DataFrame(model_data) | |
| # Function to fetch dataset info including size and sample count | |
| def fetch_dataset_info(dataset_id): | |
| headers = {"Authorization": f"Bearer {AUTH_TOKEN}"} | |
| size_url = f"https://datasets-server.huggingface.co/size?dataset={dataset_id}" | |
| url = f"{HF_API_BASE}/datasets/{dataset_id}" | |
| try: | |
| response = requests.get(size_url, headers=headers) | |
| if response.status_code != 200: | |
| st.warning(f"Error fetching dataset size info: {response.status_code}") | |
| return None | |
| dataset_info = response.json() | |
| # Get size information - need to calculate | |
| size_bytes = dataset_info['size']['dataset'].get('num_bytes_original_files', 0) | |
| # Convert to MB for display | |
| size_mb = round(size_bytes / (1024 * 1024), 2) if size_bytes else None | |
| # Get row count information | |
| sample_count = dataset_info['size']['dataset'].get('num_rows', 0) | |
| response = requests.get(url, headers=headers) | |
| if response.status_code != 200: | |
| st.warning(f"Error fetching dataset info: {response.status_code}") | |
| return None | |
| dataset_info = response.json() | |
| result = { | |
| 'id': dataset_id, | |
| 'description': dataset_info.get('description', 'No description available'), | |
| 'size_mb': size_mb, | |
| 'sample_count': sample_count, | |
| 'last_modified': dataset_info.get('lastModified', 'Unknown') | |
| } | |
| return result | |
| except Exception as e: | |
| st.error(f"Error processing dataset info: {e}") | |
| return None | |
| # Main tabs | |
| tab1, tab2 = st.tabs(["Models", "Datasets"]) | |
| # Models Tab | |
| with tab1: | |
| st.header("Models") | |
| # Display models table | |
| st.dataframe(df_models, use_container_width=True) | |
| # Selected model details | |
| st.subheader("Model Details") | |
| selected_model = st.selectbox("Select a model for details", df_models["Model Name"], key="model_select") | |
| if selected_model: | |
| model_details = df_models[df_models["Model Name"] == selected_model].iloc[0] | |
| st.markdown("### " + model_details["Model Name"]) | |
| st.markdown(f"**Description**: {model_details['Description']}") | |
| st.markdown(f"**Size**: {model_details['Size (GB)']} GB") | |
| st.markdown(f"**Last Updated**: {model_details['Last Updated']}") | |
| with tab2: | |
| st.header("Datasets") | |
| # List of dataset IDs to display | |
| dataset_ids = [ | |
| "YChang1112/test-dataset", | |
| "Anthropic/EconomicIndex" | |
| ] | |
| # Get actual dataset info from API | |
| dataset_info_list = [] | |
| if AUTH_TOKEN: | |
| with st.spinner("Loading dataset information..."): | |
| for dataset_id in dataset_ids: | |
| info = fetch_dataset_info(dataset_id) | |
| if info: | |
| dataset_info_list.append(info) | |
| else: | |
| st.warning("Authentication token not configured. Unable to fetch dataset information.") | |
| # Create a DataFrame from the collected information | |
| if dataset_info_list: | |
| df_datasets = pd.DataFrame({ | |
| "Dataset Name": [info['id'] for info in dataset_info_list], | |
| "Description": [info['description'] for info in dataset_info_list], | |
| "Size (MB)": [info['size_mb'] for info in dataset_info_list], | |
| "Samples": [info['sample_count'] for info in dataset_info_list], | |
| "Last Modified": [info['last_modified'] for info in dataset_info_list] | |
| }) | |
| # Display datasets table | |
| st.dataframe(df_datasets, use_container_width=True) | |
| else: | |
| st.error("No dataset information available. Please check your dataset IDs and authentication token.") | |
| # Dataset details with sample preview | |
| st.subheader("Dataset Preview") | |
| if dataset_info_list: | |
| selected_dataset = st.selectbox("Select a dataset to preview", | |
| [info['id'] for info in dataset_info_list], | |
| key="dataset_select") | |
| if selected_dataset: | |
| # Find the dataset info | |
| dataset_info = next((info for info in dataset_info_list if info['id'] == selected_dataset), None) | |
| if dataset_info: | |
| st.markdown(f"### {dataset_info['id']}") | |
| st.markdown(f"**Description**: {dataset_info['description']}") | |
| st.markdown(f"**Size**: {dataset_info['size_mb']} MB") | |
| st.markdown(f"**Total Samples**: {dataset_info['sample_count']:,}") | |
| st.markdown(f"**Last Modified**: {dataset_info['last_modified']}") | |
| # Show dataset samples | |
| st.markdown("### Sample Train Data") | |
| with st.spinner("Fetching dataset samples..."): | |
| samples = fetch_dataset_samples(selected_dataset) | |
| if samples: | |
| # Convert samples to DataFrame if possible | |
| try: | |
| # If it's a list of samples | |
| if isinstance(samples, list) and len(samples) > 0: | |
| # Try to normalize to handle nested structures | |
| df_sample = pd.json_normalize(samples) | |
| st.dataframe(df_sample, use_container_width=True) | |
| # If it's a single sample object | |
| elif isinstance(samples, dict): | |
| df_sample = pd.DataFrame([samples]) | |
| st.dataframe(df_sample, use_container_width=True) | |
| else: | |
| st.json(samples) | |
| except Exception as e: | |
| st.error(f"Error displaying samples: {e}") | |
| st.json(samples) # Fallback to raw JSON display | |
| else: | |
| st.warning("Could not fetch dataset samples.") | |
| # Footer | |
| st.markdown("---") | |
| st.markdown("Repository Explorer | Last updated: April 2025") | |