Spaces:
Running
on
Zero
Running
on
Zero
Upload 2 files
Browse files- app.py +597 -599
- requirements.txt +10 -9
app.py
CHANGED
|
@@ -1,600 +1,598 @@
|
|
| 1 |
-
import gradio as gr
|
| 2 |
-
import spaces
|
| 3 |
-
import onnxruntime as ort
|
| 4 |
-
import numpy as np
|
| 5 |
-
from PIL import Image, ImageDraw, ImageFont
|
| 6 |
-
import json
|
| 7 |
-
import os
|
| 8 |
-
import io
|
| 9 |
-
import requests
|
| 10 |
-
import matplotlib.pyplot as plt
|
| 11 |
-
import matplotlib
|
| 12 |
-
from huggingface_hub import hf_hub_download
|
| 13 |
-
from dataclasses import dataclass
|
| 14 |
-
from typing import List, Dict, Optional, Tuple
|
| 15 |
-
|
| 16 |
-
# MatplotlibのバックエンドをAggに設定 (GUIなし環境用)
|
| 17 |
-
matplotlib.use('Agg')
|
| 18 |
-
|
| 19 |
-
# --- onnx_predict.pyからの移植 ---
|
| 20 |
-
|
| 21 |
-
@dataclass
|
| 22 |
-
class LabelData:
|
| 23 |
-
names: list[str]
|
| 24 |
-
rating: list[np.int64]
|
| 25 |
-
general: list[np.int64]
|
| 26 |
-
artist: list[np.int64]
|
| 27 |
-
character: list[np.int64]
|
| 28 |
-
copyright: list[np.int64]
|
| 29 |
-
meta: list[np.int64]
|
| 30 |
-
quality: list[np.int64]
|
| 31 |
-
|
| 32 |
-
def pil_ensure_rgb(image: Image.Image) -> Image.Image:
|
| 33 |
-
if image.mode not in ["RGB", "RGBA"]:
|
| 34 |
-
image = image.convert("RGBA") if "transparency" in image.info else image.convert("RGB")
|
| 35 |
-
if image.mode == "RGBA":
|
| 36 |
-
background = Image.new("RGB", image.size, (255, 255, 255))
|
| 37 |
-
background.paste(image, mask=image.split()[3])
|
| 38 |
-
image = background
|
| 39 |
-
return image
|
| 40 |
-
|
| 41 |
-
def pil_pad_square(image: Image.Image) -> Image.Image:
|
| 42 |
-
width, height = image.size
|
| 43 |
-
if width == height:
|
| 44 |
-
return image
|
| 45 |
-
new_size = max(width, height)
|
| 46 |
-
new_image = Image.new("RGB", (new_size, new_size), (255, 255, 255))
|
| 47 |
-
paste_position = ((new_size - width) // 2, (new_size - height) // 2)
|
| 48 |
-
new_image.paste(image, paste_position)
|
| 49 |
-
return new_image
|
| 50 |
-
|
| 51 |
-
def load_tag_mapping(mapping_path):
|
| 52 |
-
with open(mapping_path, 'r', encoding='utf-8') as f:
|
| 53 |
-
tag_mapping_data = json.load(f)
|
| 54 |
-
|
| 55 |
-
# 新旧フォーマット対応
|
| 56 |
-
if isinstance(tag_mapping_data, dict) and "idx_to_tag" in tag_mapping_data:
|
| 57 |
-
# 旧フォーマット (辞書の中にidx_to_tagとtag_to_categoryがある)
|
| 58 |
-
idx_to_tag_dict = tag_mapping_data["idx_to_tag"]
|
| 59 |
-
tag_to_category_dict = tag_mapping_data["tag_to_category"]
|
| 60 |
-
# tag_mapping_dataが文字列キーになっている可能性があるのでintに変換
|
| 61 |
-
idx_to_tag = {int(k): v for k, v in idx_to_tag_dict.items()}
|
| 62 |
-
tag_to_category = tag_to_category_dict
|
| 63 |
-
elif isinstance(tag_mapping_data, dict):
|
| 64 |
-
# 新フォーマット (キーがインデックスの辞書)
|
| 65 |
-
tag_mapping_data = {int(k): v for k, v in tag_mapping_data.items()}
|
| 66 |
-
idx_to_tag = {}
|
| 67 |
-
tag_to_category = {}
|
| 68 |
-
for idx, data in tag_mapping_data.items():
|
| 69 |
-
tag = data['tag']
|
| 70 |
-
category = data['category']
|
| 71 |
-
idx_to_tag[idx] = tag
|
| 72 |
-
tag_to_category[tag] = category
|
| 73 |
-
else:
|
| 74 |
-
raise ValueError("Unsupported tag mapping format")
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
names = [None] * (max(idx_to_tag.keys()) + 1)
|
| 78 |
-
rating = []
|
| 79 |
-
general = []
|
| 80 |
-
artist = []
|
| 81 |
-
character = []
|
| 82 |
-
copyright = []
|
| 83 |
-
meta = []
|
| 84 |
-
quality = []
|
| 85 |
-
|
| 86 |
-
for idx, tag in idx_to_tag.items():
|
| 87 |
-
if idx >= len(names): # namesリストのサイズが足りない場合拡張
|
| 88 |
-
names.extend([None] * (idx - len(names) + 1))
|
| 89 |
-
names[idx] = tag
|
| 90 |
-
category = tag_to_category.get(tag, 'Unknown') # カテゴリが見つからない場合
|
| 91 |
-
|
| 92 |
-
if category == 'Rating':
|
| 93 |
-
rating.append(idx)
|
| 94 |
-
elif category == 'General':
|
| 95 |
-
general.append(idx)
|
| 96 |
-
elif category == 'Artist':
|
| 97 |
-
artist.append(idx)
|
| 98 |
-
elif category == 'Character':
|
| 99 |
-
character.append(idx)
|
| 100 |
-
elif category == 'Copyright':
|
| 101 |
-
copyright.append(idx)
|
| 102 |
-
elif category == 'Meta':
|
| 103 |
-
meta.append(idx)
|
| 104 |
-
elif category == 'Quality':
|
| 105 |
-
quality.append(idx)
|
| 106 |
-
# Unknownカテゴリは無視
|
| 107 |
-
|
| 108 |
-
label_data = LabelData(
|
| 109 |
-
names=names,
|
| 110 |
-
rating=np.array(rating, dtype=np.int64),
|
| 111 |
-
general=np.array(general, dtype=np.int64),
|
| 112 |
-
artist=np.array(artist, dtype=np.int64),
|
| 113 |
-
character=np.array(character, dtype=np.int64),
|
| 114 |
-
copyright=np.array(copyright, dtype=np.int64),
|
| 115 |
-
meta=np.array(meta, dtype=np.int64),
|
| 116 |
-
quality=np.array(quality, dtype=np.int64)
|
| 117 |
-
)
|
| 118 |
-
|
| 119 |
-
return label_data, idx_to_tag, tag_to_category
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
def preprocess_image(image: Image.Image, target_size=(448, 448)):
|
| 123 |
-
image = pil_ensure_rgb(image)
|
| 124 |
-
image = pil_pad_square(image)
|
| 125 |
-
image_resized = image.resize(target_size, Image.BICUBIC)
|
| 126 |
-
img_array = np.array(image_resized, dtype=np.float32) / 255.0
|
| 127 |
-
img_array = img_array.transpose(2, 0, 1) # HWC -> CHW
|
| 128 |
-
# RGB -> BGR (モデルがBGRを期待する場合 - WD Tagger v3はBGR)
|
| 129 |
-
# WD Tagger V2/V1はRGBなので注意
|
| 130 |
-
img_array = img_array[::-1, :, :]
|
| 131 |
-
mean = np.array([0.5, 0.5, 0.5], dtype=np.float32).reshape(3, 1, 1)
|
| 132 |
-
std = np.array([0.5, 0.5, 0.5], dtype=np.float32).reshape(3, 1, 1)
|
| 133 |
-
img_array = (img_array - mean) / std
|
| 134 |
-
img_array = np.expand_dims(img_array, axis=0) # Add batch dimension
|
| 135 |
-
return image, img_array # Return original PIL image and processed numpy array
|
| 136 |
-
|
| 137 |
-
def get_tags(probs, labels: LabelData, gen_threshold, char_threshold):
|
| 138 |
-
result = {
|
| 139 |
-
"rating": [], "general": [], "character": [],
|
| 140 |
-
"copyright": [], "artist": [], "meta": [], "quality": []
|
| 141 |
-
}
|
| 142 |
-
|
| 143 |
-
# Rating (select the max)
|
| 144 |
-
if labels.rating.size > 0:
|
| 145 |
-
rating_probs = probs[labels.rating]
|
| 146 |
-
if rating_probs.size > 0:
|
| 147 |
-
rating_idx = np.argmax(rating_probs)
|
| 148 |
-
# Check if the index is valid for names list
|
| 149 |
-
if labels.rating[rating_idx] < len(labels.names):
|
| 150 |
-
rating_name = labels.names[labels.rating[rating_idx]]
|
| 151 |
-
rating_conf = float(rating_probs[rating_idx])
|
| 152 |
-
result["rating"].append((rating_name, rating_conf))
|
| 153 |
-
else:
|
| 154 |
-
print(f"Warning: Rating index {labels.rating[rating_idx]} out of bounds for names list (size {len(labels.names)}).")
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
# Quality (select the max)
|
| 158 |
-
if labels.quality.size > 0:
|
| 159 |
-
quality_probs = probs[labels.quality]
|
| 160 |
-
if quality_probs.size > 0:
|
| 161 |
-
quality_idx = np.argmax(quality_probs)
|
| 162 |
-
if labels.quality[quality_idx] < len(labels.names):
|
| 163 |
-
quality_name = labels.names[labels.quality[quality_idx]]
|
| 164 |
-
quality_conf = float(quality_probs[quality_idx])
|
| 165 |
-
result["quality"].append((quality_name, quality_conf))
|
| 166 |
-
else:
|
| 167 |
-
print(f"Warning: Quality index {labels.quality[quality_idx]} out of bounds for names list (size {len(labels.names)}).")
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
category_map = {
|
| 171 |
-
"general": (labels.general, gen_threshold),
|
| 172 |
-
"character": (labels.character, char_threshold),
|
| 173 |
-
"copyright": (labels.copyright, char_threshold),
|
| 174 |
-
"artist": (labels.artist, char_threshold),
|
| 175 |
-
"meta": (labels.meta, gen_threshold)
|
| 176 |
-
}
|
| 177 |
-
|
| 178 |
-
for category, (indices, threshold) in category_map.items():
|
| 179 |
-
if indices.size > 0:
|
| 180 |
-
# Filter indices to be within the bounds of probs and labels.names
|
| 181 |
-
valid_indices = indices[(indices < len(probs)) & (indices < len(labels.names))]
|
| 182 |
-
if valid_indices.size > 0:
|
| 183 |
-
category_probs = probs[valid_indices]
|
| 184 |
-
mask = category_probs >= threshold
|
| 185 |
-
selected_indices = valid_indices[mask]
|
| 186 |
-
selected_probs = category_probs[mask]
|
| 187 |
-
for idx, prob in zip(selected_indices, selected_probs):
|
| 188 |
-
result[category].append((labels.names[idx], float(prob)))
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
# Sort by probability
|
| 192 |
-
for k in result:
|
| 193 |
-
result[k] = sorted(result[k], key=lambda x: x[1], reverse=True)
|
| 194 |
-
|
| 195 |
-
return result
|
| 196 |
-
|
| 197 |
-
def visualize_predictions(image: Image.Image, predictions, threshold=0.45):
|
| 198 |
-
# Filter out unwanted meta tags
|
| 199 |
-
filtered_meta = []
|
| 200 |
-
excluded_meta_patterns = ['id', 'commentary', 'request', 'mismatch']
|
| 201 |
-
for tag, prob in predictions["meta"]:
|
| 202 |
-
if not any(pattern in tag.lower() for pattern in excluded_meta_patterns):
|
| 203 |
-
filtered_meta.append((tag, prob))
|
| 204 |
-
predictions["meta"] = filtered_meta # Replace with filtered
|
| 205 |
-
|
| 206 |
-
# Create plot
|
| 207 |
-
fig = plt.figure(figsize=(20, 12), dpi=100)
|
| 208 |
-
gs = fig.add_gridspec(1, 2, width_ratios=[1.2, 1])
|
| 209 |
-
ax_img = fig.add_subplot(gs[0, 0])
|
| 210 |
-
ax_img.imshow(image)
|
| 211 |
-
ax_img.set_title("Original Image")
|
| 212 |
-
ax_img.axis('off')
|
| 213 |
-
ax_tags = fig.add_subplot(gs[0, 1])
|
| 214 |
-
|
| 215 |
-
all_tags = []
|
| 216 |
-
all_probs = []
|
| 217 |
-
all_colors = []
|
| 218 |
-
color_map = {'rating': 'red', 'character': 'blue', 'copyright': 'purple',
|
| 219 |
-
'artist': 'orange', 'general': 'green', 'meta': 'gray', 'quality': 'yellow'}
|
| 220 |
-
|
| 221 |
-
for cat, prefix, color in [('rating', 'R', 'red'), ('character', 'C', 'blue'),
|
| 222 |
-
('copyright', '©', 'purple'), ('artist', 'A', 'orange'),
|
| 223 |
-
('general', 'G', 'green'), ('meta', 'M', 'gray'), ('quality', 'Q', 'yellow')]:
|
| 224 |
-
for tag, prob in predictions[cat]:
|
| 225 |
-
all_tags.append(f"[{prefix}] {tag}")
|
| 226 |
-
all_probs.append(prob)
|
| 227 |
-
all_colors.append(color)
|
| 228 |
-
|
| 229 |
-
if not all_tags:
|
| 230 |
-
ax_tags.text(0.5, 0.5, "No tags found above threshold", ha='center', va='center')
|
| 231 |
-
ax_tags.set_title(f"Tags (threshold={threshold})")
|
| 232 |
-
ax_tags.axis('off')
|
| 233 |
-
plt.tight_layout()
|
| 234 |
-
# Save figure to a BytesIO object
|
| 235 |
-
buf = io.BytesIO()
|
| 236 |
-
plt.savefig(buf, format='png', dpi=100)
|
| 237 |
-
plt.close(fig)
|
| 238 |
-
buf.seek(0)
|
| 239 |
-
return Image.open(buf)
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
sorted_indices = sorted(range(len(all_probs)), key=lambda i: all_probs[i], reverse=True)
|
| 243 |
-
all_tags = [all_tags[i] for i in sorted_indices]
|
| 244 |
-
all_probs = [all_probs[i] for i in sorted_indices]
|
| 245 |
-
all_colors = [all_colors[i] for i in sorted_indices]
|
| 246 |
-
|
| 247 |
-
all_tags.reverse()
|
| 248 |
-
all_probs.reverse()
|
| 249 |
-
all_colors.reverse()
|
| 250 |
-
|
| 251 |
-
num_tags = len(all_tags)
|
| 252 |
-
bar_height = 0.8
|
| 253 |
-
if num_tags > 30: bar_height = 0.8 * (30 / num_tags)
|
| 254 |
-
y_positions = np.arange(num_tags)
|
| 255 |
-
|
| 256 |
-
bars = ax_tags.barh(y_positions, all_probs, height=bar_height, color=all_colors)
|
| 257 |
-
ax_tags.set_yticks(y_positions)
|
| 258 |
-
ax_tags.set_yticklabels(all_tags)
|
| 259 |
-
|
| 260 |
-
fontsize = 10
|
| 261 |
-
if num_tags > 40: fontsize = 8
|
| 262 |
-
elif num_tags > 60: fontsize = 6
|
| 263 |
-
for label in ax_tags.get_yticklabels(): label.set_fontsize(fontsize)
|
| 264 |
-
|
| 265 |
-
for i, (bar, prob) in enumerate(zip(bars, all_probs)):
|
| 266 |
-
ax_tags.text(min(prob + 0.02, 0.98), y_positions[i], f"{prob:.3f}",
|
| 267 |
-
va='center', fontsize=fontsize)
|
| 268 |
-
|
| 269 |
-
ax_tags.set_xlim(0, 1)
|
| 270 |
-
ax_tags.set_title(f"Tags (threshold={threshold})")
|
| 271 |
-
|
| 272 |
-
from matplotlib.patches import Patch
|
| 273 |
-
legend_elements = [Patch(facecolor=color, label=cat.capitalize()) for cat, color in color_map.items()]
|
| 274 |
-
ax_tags.legend(handles=legend_elements, loc='lower right', fontsize=8)
|
| 275 |
-
|
| 276 |
-
plt.tight_layout()
|
| 277 |
-
plt.subplots_adjust(bottom=0.05)
|
| 278 |
-
|
| 279 |
-
# Save figure to a BytesIO object
|
| 280 |
-
buf = io.BytesIO()
|
| 281 |
-
plt.savefig(buf, format='png', dpi=100)
|
| 282 |
-
plt.close(fig)
|
| 283 |
-
buf.seek(0)
|
| 284 |
-
return Image.open(buf)
|
| 285 |
-
|
| 286 |
-
# --- Gradio App Logic ---
|
| 287 |
-
|
| 288 |
-
# 定数
|
| 289 |
-
REPO_ID = "cella110n/cl_tagger"
|
| 290 |
-
# MODEL_FILENAME = "cl_eva02_tagger_v1_250426/model_optimized.onnx"
|
| 291 |
-
MODEL_FILENAME = "cl_eva02_tagger_v1_250426/model.onnx" # Use non-optimized if needed
|
| 292 |
-
TAG_MAPPING_FILENAME = "cl_eva02_tagger_v1_250426/tag_mapping.json"
|
| 293 |
-
CACHE_DIR = "./model_cache"
|
| 294 |
-
|
| 295 |
-
# グローバル変数(モデルとラベルをキャッシュ)
|
| 296 |
-
onnx_session = None
|
| 297 |
-
labels_data = None
|
| 298 |
-
tag_to_category_map = None
|
| 299 |
-
|
| 300 |
-
def download_model_files():
|
| 301 |
-
"""Hugging Face Hubからモデルとタグマッピングをダウンロード"""
|
| 302 |
-
print("Downloading model files...")
|
| 303 |
-
# 環境変数からHFトークンを取得 (プライベートリポジトリ用)
|
| 304 |
-
hf_token = os.environ.get("HF_TOKEN")
|
| 305 |
-
try:
|
| 306 |
-
model_path = hf_hub_download(repo_id=REPO_ID, filename=MODEL_FILENAME, cache_dir=CACHE_DIR, token=hf_token)
|
| 307 |
-
tag_mapping_path = hf_hub_download(repo_id=REPO_ID, filename=TAG_MAPPING_FILENAME, cache_dir=CACHE_DIR, token=hf_token)
|
| 308 |
-
print(f"Model downloaded to: {model_path}")
|
| 309 |
-
print(f"Tag mapping downloaded to: {tag_mapping_path}")
|
| 310 |
-
return model_path, tag_mapping_path
|
| 311 |
-
except Exception as e:
|
| 312 |
-
print(f"Error downloading files: {e}")
|
| 313 |
-
# トークンがない場合のエラーメッセージを改善
|
| 314 |
-
if "401 Client Error" in str(e) or "Repository not found" in str(e):
|
| 315 |
-
raise gr.Error(f"Could not download files from {REPO_ID}. "
|
| 316 |
-
f"If this is a private repository, make sure to set the HF_TOKEN secret in your Space settings.")
|
| 317 |
-
else:
|
| 318 |
-
raise gr.Error(f"Error downloading files: {e}")
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
def initialize_model():
|
| 322 |
-
"""モデルとラベルデータを初期化(キャッシュ)"""
|
| 323 |
-
global onnx_session, labels_data, tag_to_category_map
|
| 324 |
-
if onnx_session is None:
|
| 325 |
-
model_path, tag_mapping_path = download_model_files()
|
| 326 |
-
print("Loading model and labels...")
|
| 327 |
-
|
| 328 |
-
# --- Added Logging ---
|
| 329 |
-
print("--- Environment Check ---")
|
| 330 |
-
try:
|
| 331 |
-
import torch
|
| 332 |
-
print(f"PyTorch version: {torch.__version__}")
|
| 333 |
-
if torch.cuda.is_available():
|
| 334 |
-
print(f"PyTorch CUDA available: True")
|
| 335 |
-
print(f"PyTorch CUDA version: {torch.version.cuda}")
|
| 336 |
-
print(f"Detected GPU: {torch.cuda.get_device_name(0)}")
|
| 337 |
-
if torch.backends.cudnn.is_available():
|
| 338 |
-
print(f"PyTorch cuDNN available: True")
|
| 339 |
-
print(f"PyTorch cuDNN version: {torch.backends.cudnn.version()}")
|
| 340 |
-
else:
|
| 341 |
-
print("PyTorch cuDNN available: False")
|
| 342 |
-
else:
|
| 343 |
-
print("PyTorch CUDA available: False")
|
| 344 |
-
except ImportError:
|
| 345 |
-
print("PyTorch not found.")
|
| 346 |
-
except Exception as e:
|
| 347 |
-
print(f"Error during PyTorch check: {e}")
|
| 348 |
-
|
| 349 |
-
try:
|
| 350 |
-
print(f"ONNX Runtime build info: {ort.get_buildinfo()}")
|
| 351 |
-
except Exception as e:
|
| 352 |
-
print(f"Error getting ONNX Runtime build info: {e}")
|
| 353 |
-
print("-------------------------")
|
| 354 |
-
# --- End Added Logging ---
|
| 355 |
-
|
| 356 |
-
# ONNXセッションの初期化 (GPU優先)
|
| 357 |
-
available_providers = ort.get_available_providers()
|
| 358 |
-
print(f"Available ONNX Runtime providers: {available_providers}")
|
| 359 |
-
providers = []
|
| 360 |
-
if 'CUDAExecutionProvider' in available_providers:
|
| 361 |
-
providers.append('CUDAExecutionProvider')
|
| 362 |
-
# elif 'DmlExecutionProvider' in available_providers: # DirectML (Windows)
|
| 363 |
-
# providers.append('DmlExecutionProvider')
|
| 364 |
-
providers.append('CPUExecutionProvider') # Always include CPU as fallback
|
| 365 |
-
|
| 366 |
-
try:
|
| 367 |
-
onnx_session = ort.InferenceSession(model_path, providers=providers)
|
| 368 |
-
print(f"Using ONNX Runtime provider: {onnx_session.get_providers()[0]}")
|
| 369 |
-
except Exception as e:
|
| 370 |
-
print(f"Error initializing ONNX session with providers {providers}: {e}")
|
| 371 |
-
print("Falling back to CPUExecutionProvider only.")
|
| 372 |
-
onnx_session = ort.InferenceSession(model_path, providers=['CPUExecutionProvider'])
|
| 373 |
-
|
| 374 |
-
labels_data, _, tag_to_category_map = load_tag_mapping(tag_mapping_path)
|
| 375 |
-
print("Model and labels loaded.")
|
| 376 |
-
|
| 377 |
-
@spaces.GPU()
|
| 378 |
-
def predict(image_input, gen_threshold, char_threshold, output_mode):
|
| 379 |
-
"""Gradioインターフェース用の予測関数"""
|
| 380 |
-
initialize_model() # モデルがロードされていなければロード
|
| 381 |
-
|
| 382 |
-
if image_input is None:
|
| 383 |
-
return "Please upload an image.", None
|
| 384 |
-
|
| 385 |
-
print(f"Processing image with thresholds: gen={gen_threshold}, char={char_threshold}")
|
| 386 |
-
|
| 387 |
-
# PIL Imageオブジェクトであることを確認
|
| 388 |
-
if not isinstance(image_input, Image.Image):
|
| 389 |
-
try:
|
| 390 |
-
# URLの場合
|
| 391 |
-
if isinstance(image_input, str) and image_input.startswith("http"):
|
| 392 |
-
response = requests.get(image_input)
|
| 393 |
-
response.raise_for_status()
|
| 394 |
-
image = Image.open(io.BytesIO(response.content))
|
| 395 |
-
# ファイルパスの場合 (Gradioでは通常発生しないが念のため)
|
| 396 |
-
elif isinstance(image_input, str) and os.path.exists(image_input):
|
| 397 |
-
image = Image.open(image_input)
|
| 398 |
-
# Numpy配列の場合 (Gradio Imageコンポーネントからの入力)
|
| 399 |
-
elif isinstance(image_input, np.ndarray):
|
| 400 |
-
image = Image.fromarray(image_input)
|
| 401 |
-
else:
|
| 402 |
-
raise ValueError("Unsupported image input type")
|
| 403 |
-
except Exception as e:
|
| 404 |
-
print(f"Error loading image: {e}")
|
| 405 |
-
return f"Error loading image: {e}", None
|
| 406 |
-
else:
|
| 407 |
-
image = image_input
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
# 前処理
|
| 411 |
-
original_pil_image, input_data = preprocess_image(image)
|
| 412 |
-
|
| 413 |
-
# データ型をモデルの期待に合わせる (通常はfloat32)
|
| 414 |
-
input_name = onnx_session.get_inputs()[0].name
|
| 415 |
-
expected_type = onnx_session.get_inputs()[0].type
|
| 416 |
-
if expected_type == 'tensor(float16)':
|
| 417 |
-
input_data = input_data.astype(np.float16)
|
| 418 |
-
else:
|
| 419 |
-
input_data = input_data.astype(np.float32) # Default to float32
|
| 420 |
-
|
| 421 |
-
# 推論
|
| 422 |
-
start_time = time.time()
|
| 423 |
-
outputs = onnx_session.run(None, {input_name: input_data})[0]
|
| 424 |
-
inference_time = time.time() - start_time
|
| 425 |
-
print(f"Inference completed in {inference_time:.3f} seconds")
|
| 426 |
-
|
| 427 |
-
# シグモイド関数で確率に変換
|
| 428 |
-
probs = 1 / (1 + np.exp(-outputs[0])) # Apply sigmoid to the first batch item
|
| 429 |
-
|
| 430 |
-
# タグ取得
|
| 431 |
-
predictions = get_tags(probs, labels_data, gen_threshold, char_threshold)
|
| 432 |
-
|
| 433 |
-
# タグを整形
|
| 434 |
-
output_tags = []
|
| 435 |
-
# RatingとQualityを最初に追加
|
| 436 |
-
if predictions["rating"]:
|
| 437 |
-
output_tags.append(predictions["rating"][0][0].replace("_", " "))
|
| 438 |
-
if predictions["quality"]:
|
| 439 |
-
output_tags.append(predictions["quality"][0][0].replace("_", " "))
|
| 440 |
-
|
| 441 |
-
# 残りのカテゴリをアルファベット順に追加(オプション)
|
| 442 |
-
for category in ["artist", "character", "copyright", "general", "meta"]:
|
| 443 |
-
tags = [tag.replace("_", " ") for tag, prob in predictions[category]
|
| 444 |
-
if not (category == "meta" and any(p in tag.lower() for p in ['id', 'commentary','mismatch']))] # メタタグフィルタリング
|
| 445 |
-
output_tags.extend(tags)
|
| 446 |
-
|
| 447 |
-
output_text = ", ".join(output_tags)
|
| 448 |
-
|
| 449 |
-
if output_mode == "Tags Only":
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
.
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
"""
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
const
|
| 470 |
-
const
|
| 471 |
-
|
| 472 |
-
const
|
| 473 |
-
dt.
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
//
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
if (
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
//
|
| 500 |
-
|
| 501 |
-
//
|
| 502 |
-
|
| 503 |
-
//
|
| 504 |
-
|
| 505 |
-
//
|
| 506 |
-
|
| 507 |
-
//
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
//
|
| 526 |
-
|
| 527 |
-
//
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
gr.Markdown("
|
| 546 |
-
|
| 547 |
-
gr.
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
#
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
gr.
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
["https://pbs.twimg.com/media/
|
| 569 |
-
["https://pbs.twimg.com/media/
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
],
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
)
|
| 584 |
-
|
| 585 |
-
#
|
| 586 |
-
#
|
| 587 |
-
#
|
| 588 |
-
#
|
| 589 |
-
#
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
if
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
# Initialize model on startup to avoid delay on first prediction
|
| 599 |
-
# initialize_model() # Removed startup initialization
|
| 600 |
demo.launch()
|
|
|
|
| 1 |
+
import gradio as gr
|
| 2 |
+
import spaces
|
| 3 |
+
import onnxruntime as ort
|
| 4 |
+
import numpy as np
|
| 5 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 6 |
+
import json
|
| 7 |
+
import os
|
| 8 |
+
import io
|
| 9 |
+
import requests
|
| 10 |
+
import matplotlib.pyplot as plt
|
| 11 |
+
import matplotlib
|
| 12 |
+
from huggingface_hub import hf_hub_download
|
| 13 |
+
from dataclasses import dataclass
|
| 14 |
+
from typing import List, Dict, Optional, Tuple
|
| 15 |
+
|
| 16 |
+
# MatplotlibのバックエンドをAggに設定 (GUIなし環境用)
|
| 17 |
+
matplotlib.use('Agg')
|
| 18 |
+
|
| 19 |
+
# --- onnx_predict.pyからの移植 ---
|
| 20 |
+
|
| 21 |
+
@dataclass
|
| 22 |
+
class LabelData:
|
| 23 |
+
names: list[str]
|
| 24 |
+
rating: list[np.int64]
|
| 25 |
+
general: list[np.int64]
|
| 26 |
+
artist: list[np.int64]
|
| 27 |
+
character: list[np.int64]
|
| 28 |
+
copyright: list[np.int64]
|
| 29 |
+
meta: list[np.int64]
|
| 30 |
+
quality: list[np.int64]
|
| 31 |
+
|
| 32 |
+
def pil_ensure_rgb(image: Image.Image) -> Image.Image:
|
| 33 |
+
if image.mode not in ["RGB", "RGBA"]:
|
| 34 |
+
image = image.convert("RGBA") if "transparency" in image.info else image.convert("RGB")
|
| 35 |
+
if image.mode == "RGBA":
|
| 36 |
+
background = Image.new("RGB", image.size, (255, 255, 255))
|
| 37 |
+
background.paste(image, mask=image.split()[3])
|
| 38 |
+
image = background
|
| 39 |
+
return image
|
| 40 |
+
|
| 41 |
+
def pil_pad_square(image: Image.Image) -> Image.Image:
|
| 42 |
+
width, height = image.size
|
| 43 |
+
if width == height:
|
| 44 |
+
return image
|
| 45 |
+
new_size = max(width, height)
|
| 46 |
+
new_image = Image.new("RGB", (new_size, new_size), (255, 255, 255))
|
| 47 |
+
paste_position = ((new_size - width) // 2, (new_size - height) // 2)
|
| 48 |
+
new_image.paste(image, paste_position)
|
| 49 |
+
return new_image
|
| 50 |
+
|
| 51 |
+
def load_tag_mapping(mapping_path):
|
| 52 |
+
with open(mapping_path, 'r', encoding='utf-8') as f:
|
| 53 |
+
tag_mapping_data = json.load(f)
|
| 54 |
+
|
| 55 |
+
# 新旧フォーマット対応
|
| 56 |
+
if isinstance(tag_mapping_data, dict) and "idx_to_tag" in tag_mapping_data:
|
| 57 |
+
# 旧フォーマット (辞書の中にidx_to_tagとtag_to_categoryがある)
|
| 58 |
+
idx_to_tag_dict = tag_mapping_data["idx_to_tag"]
|
| 59 |
+
tag_to_category_dict = tag_mapping_data["tag_to_category"]
|
| 60 |
+
# tag_mapping_dataが文字列キーになっている可能性があるのでintに変換
|
| 61 |
+
idx_to_tag = {int(k): v for k, v in idx_to_tag_dict.items()}
|
| 62 |
+
tag_to_category = tag_to_category_dict
|
| 63 |
+
elif isinstance(tag_mapping_data, dict):
|
| 64 |
+
# 新フォーマット (キーがインデックスの辞書)
|
| 65 |
+
tag_mapping_data = {int(k): v for k, v in tag_mapping_data.items()}
|
| 66 |
+
idx_to_tag = {}
|
| 67 |
+
tag_to_category = {}
|
| 68 |
+
for idx, data in tag_mapping_data.items():
|
| 69 |
+
tag = data['tag']
|
| 70 |
+
category = data['category']
|
| 71 |
+
idx_to_tag[idx] = tag
|
| 72 |
+
tag_to_category[tag] = category
|
| 73 |
+
else:
|
| 74 |
+
raise ValueError("Unsupported tag mapping format")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
names = [None] * (max(idx_to_tag.keys()) + 1)
|
| 78 |
+
rating = []
|
| 79 |
+
general = []
|
| 80 |
+
artist = []
|
| 81 |
+
character = []
|
| 82 |
+
copyright = []
|
| 83 |
+
meta = []
|
| 84 |
+
quality = []
|
| 85 |
+
|
| 86 |
+
for idx, tag in idx_to_tag.items():
|
| 87 |
+
if idx >= len(names): # namesリストのサイズが足りない場合拡張
|
| 88 |
+
names.extend([None] * (idx - len(names) + 1))
|
| 89 |
+
names[idx] = tag
|
| 90 |
+
category = tag_to_category.get(tag, 'Unknown') # カテゴリが見つからない場合
|
| 91 |
+
|
| 92 |
+
if category == 'Rating':
|
| 93 |
+
rating.append(idx)
|
| 94 |
+
elif category == 'General':
|
| 95 |
+
general.append(idx)
|
| 96 |
+
elif category == 'Artist':
|
| 97 |
+
artist.append(idx)
|
| 98 |
+
elif category == 'Character':
|
| 99 |
+
character.append(idx)
|
| 100 |
+
elif category == 'Copyright':
|
| 101 |
+
copyright.append(idx)
|
| 102 |
+
elif category == 'Meta':
|
| 103 |
+
meta.append(idx)
|
| 104 |
+
elif category == 'Quality':
|
| 105 |
+
quality.append(idx)
|
| 106 |
+
# Unknownカテゴリは無視
|
| 107 |
+
|
| 108 |
+
label_data = LabelData(
|
| 109 |
+
names=names,
|
| 110 |
+
rating=np.array(rating, dtype=np.int64),
|
| 111 |
+
general=np.array(general, dtype=np.int64),
|
| 112 |
+
artist=np.array(artist, dtype=np.int64),
|
| 113 |
+
character=np.array(character, dtype=np.int64),
|
| 114 |
+
copyright=np.array(copyright, dtype=np.int64),
|
| 115 |
+
meta=np.array(meta, dtype=np.int64),
|
| 116 |
+
quality=np.array(quality, dtype=np.int64)
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
return label_data, idx_to_tag, tag_to_category
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def preprocess_image(image: Image.Image, target_size=(448, 448)):
|
| 123 |
+
image = pil_ensure_rgb(image)
|
| 124 |
+
image = pil_pad_square(image)
|
| 125 |
+
image_resized = image.resize(target_size, Image.BICUBIC)
|
| 126 |
+
img_array = np.array(image_resized, dtype=np.float32) / 255.0
|
| 127 |
+
img_array = img_array.transpose(2, 0, 1) # HWC -> CHW
|
| 128 |
+
# RGB -> BGR (モデルがBGRを期待する場合 - WD Tagger v3はBGR)
|
| 129 |
+
# WD Tagger V2/V1はRGBなので注意
|
| 130 |
+
img_array = img_array[::-1, :, :]
|
| 131 |
+
mean = np.array([0.5, 0.5, 0.5], dtype=np.float32).reshape(3, 1, 1)
|
| 132 |
+
std = np.array([0.5, 0.5, 0.5], dtype=np.float32).reshape(3, 1, 1)
|
| 133 |
+
img_array = (img_array - mean) / std
|
| 134 |
+
img_array = np.expand_dims(img_array, axis=0) # Add batch dimension
|
| 135 |
+
return image, img_array # Return original PIL image and processed numpy array
|
| 136 |
+
|
| 137 |
+
def get_tags(probs, labels: LabelData, gen_threshold, char_threshold):
|
| 138 |
+
result = {
|
| 139 |
+
"rating": [], "general": [], "character": [],
|
| 140 |
+
"copyright": [], "artist": [], "meta": [], "quality": []
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
# Rating (select the max)
|
| 144 |
+
if labels.rating.size > 0:
|
| 145 |
+
rating_probs = probs[labels.rating]
|
| 146 |
+
if rating_probs.size > 0:
|
| 147 |
+
rating_idx = np.argmax(rating_probs)
|
| 148 |
+
# Check if the index is valid for names list
|
| 149 |
+
if labels.rating[rating_idx] < len(labels.names):
|
| 150 |
+
rating_name = labels.names[labels.rating[rating_idx]]
|
| 151 |
+
rating_conf = float(rating_probs[rating_idx])
|
| 152 |
+
result["rating"].append((rating_name, rating_conf))
|
| 153 |
+
else:
|
| 154 |
+
print(f"Warning: Rating index {labels.rating[rating_idx]} out of bounds for names list (size {len(labels.names)}).")
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
# Quality (select the max)
|
| 158 |
+
if labels.quality.size > 0:
|
| 159 |
+
quality_probs = probs[labels.quality]
|
| 160 |
+
if quality_probs.size > 0:
|
| 161 |
+
quality_idx = np.argmax(quality_probs)
|
| 162 |
+
if labels.quality[quality_idx] < len(labels.names):
|
| 163 |
+
quality_name = labels.names[labels.quality[quality_idx]]
|
| 164 |
+
quality_conf = float(quality_probs[quality_idx])
|
| 165 |
+
result["quality"].append((quality_name, quality_conf))
|
| 166 |
+
else:
|
| 167 |
+
print(f"Warning: Quality index {labels.quality[quality_idx]} out of bounds for names list (size {len(labels.names)}).")
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
category_map = {
|
| 171 |
+
"general": (labels.general, gen_threshold),
|
| 172 |
+
"character": (labels.character, char_threshold),
|
| 173 |
+
"copyright": (labels.copyright, char_threshold),
|
| 174 |
+
"artist": (labels.artist, char_threshold),
|
| 175 |
+
"meta": (labels.meta, gen_threshold)
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
for category, (indices, threshold) in category_map.items():
|
| 179 |
+
if indices.size > 0:
|
| 180 |
+
# Filter indices to be within the bounds of probs and labels.names
|
| 181 |
+
valid_indices = indices[(indices < len(probs)) & (indices < len(labels.names))]
|
| 182 |
+
if valid_indices.size > 0:
|
| 183 |
+
category_probs = probs[valid_indices]
|
| 184 |
+
mask = category_probs >= threshold
|
| 185 |
+
selected_indices = valid_indices[mask]
|
| 186 |
+
selected_probs = category_probs[mask]
|
| 187 |
+
for idx, prob in zip(selected_indices, selected_probs):
|
| 188 |
+
result[category].append((labels.names[idx], float(prob)))
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
# Sort by probability
|
| 192 |
+
for k in result:
|
| 193 |
+
result[k] = sorted(result[k], key=lambda x: x[1], reverse=True)
|
| 194 |
+
|
| 195 |
+
return result
|
| 196 |
+
|
| 197 |
+
def visualize_predictions(image: Image.Image, predictions, threshold=0.45):
|
| 198 |
+
# Filter out unwanted meta tags
|
| 199 |
+
filtered_meta = []
|
| 200 |
+
excluded_meta_patterns = ['id', 'commentary', 'request', 'mismatch']
|
| 201 |
+
for tag, prob in predictions["meta"]:
|
| 202 |
+
if not any(pattern in tag.lower() for pattern in excluded_meta_patterns):
|
| 203 |
+
filtered_meta.append((tag, prob))
|
| 204 |
+
predictions["meta"] = filtered_meta # Replace with filtered
|
| 205 |
+
|
| 206 |
+
# Create plot
|
| 207 |
+
fig = plt.figure(figsize=(20, 12), dpi=100)
|
| 208 |
+
gs = fig.add_gridspec(1, 2, width_ratios=[1.2, 1])
|
| 209 |
+
ax_img = fig.add_subplot(gs[0, 0])
|
| 210 |
+
ax_img.imshow(image)
|
| 211 |
+
ax_img.set_title("Original Image")
|
| 212 |
+
ax_img.axis('off')
|
| 213 |
+
ax_tags = fig.add_subplot(gs[0, 1])
|
| 214 |
+
|
| 215 |
+
all_tags = []
|
| 216 |
+
all_probs = []
|
| 217 |
+
all_colors = []
|
| 218 |
+
color_map = {'rating': 'red', 'character': 'blue', 'copyright': 'purple',
|
| 219 |
+
'artist': 'orange', 'general': 'green', 'meta': 'gray', 'quality': 'yellow'}
|
| 220 |
+
|
| 221 |
+
for cat, prefix, color in [('rating', 'R', 'red'), ('character', 'C', 'blue'),
|
| 222 |
+
('copyright', '©', 'purple'), ('artist', 'A', 'orange'),
|
| 223 |
+
('general', 'G', 'green'), ('meta', 'M', 'gray'), ('quality', 'Q', 'yellow')]:
|
| 224 |
+
for tag, prob in predictions[cat]:
|
| 225 |
+
all_tags.append(f"[{prefix}] {tag}")
|
| 226 |
+
all_probs.append(prob)
|
| 227 |
+
all_colors.append(color)
|
| 228 |
+
|
| 229 |
+
if not all_tags:
|
| 230 |
+
ax_tags.text(0.5, 0.5, "No tags found above threshold", ha='center', va='center')
|
| 231 |
+
ax_tags.set_title(f"Tags (threshold={threshold})")
|
| 232 |
+
ax_tags.axis('off')
|
| 233 |
+
plt.tight_layout()
|
| 234 |
+
# Save figure to a BytesIO object
|
| 235 |
+
buf = io.BytesIO()
|
| 236 |
+
plt.savefig(buf, format='png', dpi=100)
|
| 237 |
+
plt.close(fig)
|
| 238 |
+
buf.seek(0)
|
| 239 |
+
return Image.open(buf)
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
sorted_indices = sorted(range(len(all_probs)), key=lambda i: all_probs[i], reverse=True)
|
| 243 |
+
all_tags = [all_tags[i] for i in sorted_indices]
|
| 244 |
+
all_probs = [all_probs[i] for i in sorted_indices]
|
| 245 |
+
all_colors = [all_colors[i] for i in sorted_indices]
|
| 246 |
+
|
| 247 |
+
all_tags.reverse()
|
| 248 |
+
all_probs.reverse()
|
| 249 |
+
all_colors.reverse()
|
| 250 |
+
|
| 251 |
+
num_tags = len(all_tags)
|
| 252 |
+
bar_height = 0.8
|
| 253 |
+
if num_tags > 30: bar_height = 0.8 * (30 / num_tags)
|
| 254 |
+
y_positions = np.arange(num_tags)
|
| 255 |
+
|
| 256 |
+
bars = ax_tags.barh(y_positions, all_probs, height=bar_height, color=all_colors)
|
| 257 |
+
ax_tags.set_yticks(y_positions)
|
| 258 |
+
ax_tags.set_yticklabels(all_tags)
|
| 259 |
+
|
| 260 |
+
fontsize = 10
|
| 261 |
+
if num_tags > 40: fontsize = 8
|
| 262 |
+
elif num_tags > 60: fontsize = 6
|
| 263 |
+
for label in ax_tags.get_yticklabels(): label.set_fontsize(fontsize)
|
| 264 |
+
|
| 265 |
+
for i, (bar, prob) in enumerate(zip(bars, all_probs)):
|
| 266 |
+
ax_tags.text(min(prob + 0.02, 0.98), y_positions[i], f"{prob:.3f}",
|
| 267 |
+
va='center', fontsize=fontsize)
|
| 268 |
+
|
| 269 |
+
ax_tags.set_xlim(0, 1)
|
| 270 |
+
ax_tags.set_title(f"Tags (threshold={threshold})")
|
| 271 |
+
|
| 272 |
+
from matplotlib.patches import Patch
|
| 273 |
+
legend_elements = [Patch(facecolor=color, label=cat.capitalize()) for cat, color in color_map.items()]
|
| 274 |
+
ax_tags.legend(handles=legend_elements, loc='lower right', fontsize=8)
|
| 275 |
+
|
| 276 |
+
plt.tight_layout()
|
| 277 |
+
plt.subplots_adjust(bottom=0.05)
|
| 278 |
+
|
| 279 |
+
# Save figure to a BytesIO object
|
| 280 |
+
buf = io.BytesIO()
|
| 281 |
+
plt.savefig(buf, format='png', dpi=100)
|
| 282 |
+
plt.close(fig)
|
| 283 |
+
buf.seek(0)
|
| 284 |
+
return Image.open(buf)
|
| 285 |
+
|
| 286 |
+
# --- Gradio App Logic ---
|
| 287 |
+
|
| 288 |
+
# 定数
|
| 289 |
+
REPO_ID = "cella110n/cl_tagger"
|
| 290 |
+
# MODEL_FILENAME = "cl_eva02_tagger_v1_250426/model_optimized.onnx"
|
| 291 |
+
MODEL_FILENAME = "cl_eva02_tagger_v1_250426/model.onnx" # Use non-optimized if needed
|
| 292 |
+
TAG_MAPPING_FILENAME = "cl_eva02_tagger_v1_250426/tag_mapping.json"
|
| 293 |
+
CACHE_DIR = "./model_cache"
|
| 294 |
+
|
| 295 |
+
# グローバル変数(モデルとラベルをキャッシュ)
|
| 296 |
+
onnx_session = None
|
| 297 |
+
labels_data = None
|
| 298 |
+
tag_to_category_map = None
|
| 299 |
+
|
| 300 |
+
def download_model_files():
|
| 301 |
+
"""Hugging Face Hubからモデルとタグマッピングをダウンロード"""
|
| 302 |
+
print("Downloading model files...")
|
| 303 |
+
# 環境変数からHFトークンを取得 (プライベートリポジトリ用)
|
| 304 |
+
hf_token = os.environ.get("HF_TOKEN")
|
| 305 |
+
try:
|
| 306 |
+
model_path = hf_hub_download(repo_id=REPO_ID, filename=MODEL_FILENAME, cache_dir=CACHE_DIR, token=hf_token)
|
| 307 |
+
tag_mapping_path = hf_hub_download(repo_id=REPO_ID, filename=TAG_MAPPING_FILENAME, cache_dir=CACHE_DIR, token=hf_token)
|
| 308 |
+
print(f"Model downloaded to: {model_path}")
|
| 309 |
+
print(f"Tag mapping downloaded to: {tag_mapping_path}")
|
| 310 |
+
return model_path, tag_mapping_path
|
| 311 |
+
except Exception as e:
|
| 312 |
+
print(f"Error downloading files: {e}")
|
| 313 |
+
# トークンがない場合のエラーメッセージを改善
|
| 314 |
+
if "401 Client Error" in str(e) or "Repository not found" in str(e):
|
| 315 |
+
raise gr.Error(f"Could not download files from {REPO_ID}. "
|
| 316 |
+
f"If this is a private repository, make sure to set the HF_TOKEN secret in your Space settings.")
|
| 317 |
+
else:
|
| 318 |
+
raise gr.Error(f"Error downloading files: {e}")
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def initialize_model():
|
| 322 |
+
"""モデルとラベルデータを初期化(キャッシュ)"""
|
| 323 |
+
global onnx_session, labels_data, tag_to_category_map
|
| 324 |
+
if onnx_session is None:
|
| 325 |
+
model_path, tag_mapping_path = download_model_files()
|
| 326 |
+
print("Loading model and labels...")
|
| 327 |
+
|
| 328 |
+
# --- Added Logging ---
|
| 329 |
+
print("--- Environment Check ---")
|
| 330 |
+
try:
|
| 331 |
+
import torch
|
| 332 |
+
print(f"PyTorch version: {torch.__version__}")
|
| 333 |
+
if torch.cuda.is_available():
|
| 334 |
+
print(f"PyTorch CUDA available: True")
|
| 335 |
+
print(f"PyTorch CUDA version: {torch.version.cuda}")
|
| 336 |
+
print(f"Detected GPU: {torch.cuda.get_device_name(0)}")
|
| 337 |
+
if torch.backends.cudnn.is_available():
|
| 338 |
+
print(f"PyTorch cuDNN available: True")
|
| 339 |
+
print(f"PyTorch cuDNN version: {torch.backends.cudnn.version()}")
|
| 340 |
+
else:
|
| 341 |
+
print("PyTorch cuDNN available: False")
|
| 342 |
+
else:
|
| 343 |
+
print("PyTorch CUDA available: False")
|
| 344 |
+
except ImportError:
|
| 345 |
+
print("PyTorch not found.")
|
| 346 |
+
except Exception as e:
|
| 347 |
+
print(f"Error during PyTorch check: {e}")
|
| 348 |
+
|
| 349 |
+
try:
|
| 350 |
+
print(f"ONNX Runtime build info: {ort.get_buildinfo()}")
|
| 351 |
+
except Exception as e:
|
| 352 |
+
print(f"Error getting ONNX Runtime build info: {e}")
|
| 353 |
+
print("-------------------------")
|
| 354 |
+
# --- End Added Logging ---
|
| 355 |
+
|
| 356 |
+
# ONNXセッションの初期化 (GPU優先)
|
| 357 |
+
available_providers = ort.get_available_providers()
|
| 358 |
+
print(f"Available ONNX Runtime providers: {available_providers}")
|
| 359 |
+
providers = []
|
| 360 |
+
if 'CUDAExecutionProvider' in available_providers:
|
| 361 |
+
providers.append('CUDAExecutionProvider')
|
| 362 |
+
# elif 'DmlExecutionProvider' in available_providers: # DirectML (Windows)
|
| 363 |
+
# providers.append('DmlExecutionProvider')
|
| 364 |
+
providers.append('CPUExecutionProvider') # Always include CPU as fallback
|
| 365 |
+
|
| 366 |
+
try:
|
| 367 |
+
onnx_session = ort.InferenceSession(model_path, providers=providers)
|
| 368 |
+
print(f"Using ONNX Runtime provider: {onnx_session.get_providers()[0]}")
|
| 369 |
+
except Exception as e:
|
| 370 |
+
print(f"Error initializing ONNX session with providers {providers}: {e}")
|
| 371 |
+
print("Falling back to CPUExecutionProvider only.")
|
| 372 |
+
onnx_session = ort.InferenceSession(model_path, providers=['CPUExecutionProvider'])
|
| 373 |
+
|
| 374 |
+
labels_data, _, tag_to_category_map = load_tag_mapping(tag_mapping_path)
|
| 375 |
+
print("Model and labels loaded.")
|
| 376 |
+
|
| 377 |
+
@spaces.GPU()
|
| 378 |
+
def predict(image_input, gen_threshold, char_threshold, output_mode):
|
| 379 |
+
"""Gradioインターフェース用の予測関数"""
|
| 380 |
+
initialize_model() # モデルがロードされていなければロード
|
| 381 |
+
|
| 382 |
+
if image_input is None:
|
| 383 |
+
return "Please upload an image.", None
|
| 384 |
+
|
| 385 |
+
print(f"Processing image with thresholds: gen={gen_threshold}, char={char_threshold}")
|
| 386 |
+
|
| 387 |
+
# PIL Imageオブジェクトであることを確認
|
| 388 |
+
if not isinstance(image_input, Image.Image):
|
| 389 |
+
try:
|
| 390 |
+
# URLの場合
|
| 391 |
+
if isinstance(image_input, str) and image_input.startswith("http"):
|
| 392 |
+
response = requests.get(image_input)
|
| 393 |
+
response.raise_for_status()
|
| 394 |
+
image = Image.open(io.BytesIO(response.content))
|
| 395 |
+
# ファイルパスの場合 (Gradioでは通常発生しないが念のため)
|
| 396 |
+
elif isinstance(image_input, str) and os.path.exists(image_input):
|
| 397 |
+
image = Image.open(image_input)
|
| 398 |
+
# Numpy配列の場合 (Gradio Imageコンポーネントからの入力)
|
| 399 |
+
elif isinstance(image_input, np.ndarray):
|
| 400 |
+
image = Image.fromarray(image_input)
|
| 401 |
+
else:
|
| 402 |
+
raise ValueError("Unsupported image input type")
|
| 403 |
+
except Exception as e:
|
| 404 |
+
print(f"Error loading image: {e}")
|
| 405 |
+
return f"Error loading image: {e}", None
|
| 406 |
+
else:
|
| 407 |
+
image = image_input
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
# 前処理
|
| 411 |
+
original_pil_image, input_data = preprocess_image(image)
|
| 412 |
+
|
| 413 |
+
# データ型をモデルの期待に合わせる (通常はfloat32)
|
| 414 |
+
input_name = onnx_session.get_inputs()[0].name
|
| 415 |
+
expected_type = onnx_session.get_inputs()[0].type
|
| 416 |
+
if expected_type == 'tensor(float16)':
|
| 417 |
+
input_data = input_data.astype(np.float16)
|
| 418 |
+
else:
|
| 419 |
+
input_data = input_data.astype(np.float32) # Default to float32
|
| 420 |
+
|
| 421 |
+
# 推論
|
| 422 |
+
start_time = time.time()
|
| 423 |
+
outputs = onnx_session.run(None, {input_name: input_data})[0]
|
| 424 |
+
inference_time = time.time() - start_time
|
| 425 |
+
print(f"Inference completed in {inference_time:.3f} seconds")
|
| 426 |
+
|
| 427 |
+
# シグモイド関数で確率に変換
|
| 428 |
+
probs = 1 / (1 + np.exp(-outputs[0])) # Apply sigmoid to the first batch item
|
| 429 |
+
|
| 430 |
+
# タグ取得
|
| 431 |
+
predictions = get_tags(probs, labels_data, gen_threshold, char_threshold)
|
| 432 |
+
|
| 433 |
+
# タグを整形
|
| 434 |
+
output_tags = []
|
| 435 |
+
# RatingとQualityを最初に追加
|
| 436 |
+
if predictions["rating"]:
|
| 437 |
+
output_tags.append(predictions["rating"][0][0].replace("_", " "))
|
| 438 |
+
if predictions["quality"]:
|
| 439 |
+
output_tags.append(predictions["quality"][0][0].replace("_", " "))
|
| 440 |
+
|
| 441 |
+
# 残りのカテゴリをアルファベット順に追加(オプション)
|
| 442 |
+
for category in ["artist", "character", "copyright", "general", "meta"]:
|
| 443 |
+
tags = [tag.replace("_", " ") for tag, prob in predictions[category]
|
| 444 |
+
if not (category == "meta" and any(p in tag.lower() for p in ['id', 'commentary','mismatch']))] # メタタグフィルタリング
|
| 445 |
+
output_tags.extend(tags)
|
| 446 |
+
|
| 447 |
+
output_text = ", ".join(output_tags)
|
| 448 |
+
|
| 449 |
+
if output_mode == "Tags Only":
|
| 450 |
+
return output_text, None
|
| 451 |
+
else: # Visualization
|
| 452 |
+
viz_image = visualize_predictions(original_pil_image, predictions, gen_threshold)
|
| 453 |
+
return output_text, viz_image
|
| 454 |
+
|
| 455 |
+
# --- Gradio Interface Definition ---
|
| 456 |
+
import time
|
| 457 |
+
|
| 458 |
+
# CSS for styling
|
| 459 |
+
css = """
|
| 460 |
+
.gradio-container { font-family: 'IBM Plex Sans', sans-serif; }
|
| 461 |
+
footer { display: none !important; }
|
| 462 |
+
.gr-prose { max-width: 100% !important; }
|
| 463 |
+
"""
|
| 464 |
+
# Custom JS for image pasting and URL handling
|
| 465 |
+
js = """
|
| 466 |
+
async function paste_image(blob, gen_thresh, char_thresh, out_mode) {
|
| 467 |
+
const data = await fetch(blob)
|
| 468 |
+
const image_data = await data.blob()
|
| 469 |
+
const file = new File([image_data], "pasted_image.png",{ type: image_data.type })
|
| 470 |
+
const dt = new DataTransfer()
|
| 471 |
+
dt.items.add(file)
|
| 472 |
+
const element = document.querySelector('#input-image input[type="file"]')
|
| 473 |
+
element.files = dt.files
|
| 474 |
+
// Trigger the change event manually
|
| 475 |
+
const event = new Event('change', { bubbles: true })
|
| 476 |
+
element.dispatchEvent(event)
|
| 477 |
+
// Wait a bit for Gradio to process the change, then trigger predict if needed
|
| 478 |
+
// await new Promise(resolve => setTimeout(resolve, 100)); // Optional delay
|
| 479 |
+
// You might need to manually trigger the prediction or rely on Gradio's auto-triggering
|
| 480 |
+
return [file, gen_thresh, char_thresh, out_mode]; // Return input for Gradio function
|
| 481 |
+
}
|
| 482 |
+
|
| 483 |
+
async function paste_update(evt){
|
| 484 |
+
if (!evt.clipboardData || !evt.clipboardData.items) return;
|
| 485 |
+
var url = evt.clipboardData.getData('text');
|
| 486 |
+
if (url) {
|
| 487 |
+
// Basic check for image URL (you might want a more robust check)
|
| 488 |
+
if (/\.(jpg|jpeg|png|webp|bmp)$/i.test(url)) {
|
| 489 |
+
// Create a button or link to load the URL
|
| 490 |
+
const url_container = document.getElementById('url-input-container');
|
| 491 |
+
url_container.innerHTML = `<p>Detected URL: <button id="load-url-btn" class="gr-button gr-button-sm gr-button-secondary">${url}</button></p>`;
|
| 492 |
+
|
| 493 |
+
document.getElementById('load-url-btn').onclick = async () => {
|
| 494 |
+
// Simulate file upload from URL - Gradio's Image component handles URLs directly
|
| 495 |
+
const element = document.querySelector('#input-image input[type="file"]');
|
| 496 |
+
// Can't directly set URL to file input, so we pass it to Gradio fn
|
| 497 |
+
// Or maybe update the image display src directly if possible?
|
| 498 |
+
|
| 499 |
+
// Let Gradio handle the URL - user needs to click predict
|
| 500 |
+
// We can pre-fill the image component if Gradio supports it via JS,
|
| 501 |
+
// but it's simpler to just let the user click predict after pasting URL.
|
| 502 |
+
alert("URL detected. Please ensure the image input is cleared and then press 'Predict' or re-upload the image.");
|
| 503 |
+
// Clear current image preview if possible?
|
| 504 |
+
|
| 505 |
+
// A workaround: display the URL and let the user manually trigger prediction
|
| 506 |
+
// Or, try to use Gradio's JS API if available to update the Image component value
|
| 507 |
+
// For now, just inform the user.
|
| 508 |
+
};
|
| 509 |
+
return; // Don't process as image paste if URL is found
|
| 510 |
+
}
|
| 511 |
+
}
|
| 512 |
+
|
| 513 |
+
var items = evt.clipboardData.items;
|
| 514 |
+
for (var i = 0; i < items.length; i++) {
|
| 515 |
+
if (items[i].type.indexOf("image") === 0) {
|
| 516 |
+
var blob = items[i].getAsFile();
|
| 517 |
+
var reader = new FileReader();
|
| 518 |
+
reader.onload = function(event){
|
| 519 |
+
// Update the Gradio Image component source directly
|
| 520 |
+
const imgElement = document.querySelector('#input-image img'); // Find the img tag inside the component
|
| 521 |
+
if (imgElement) {
|
| 522 |
+
imgElement.src = event.target.result;
|
| 523 |
+
// We still need to pass the blob to the Gradio function
|
| 524 |
+
// Use Gradio's JS API or hidden components if possible
|
| 525 |
+
// For now, let's use a simple alert and rely on manual trigger
|
| 526 |
+
alert("Image pasted. The preview should update. Please press 'Predict'.");
|
| 527 |
+
// Trigger paste_image function - requires Gradio JS interaction
|
| 528 |
+
// This part is tricky without official Gradio JS API for updates
|
| 529 |
+
}
|
| 530 |
+
};
|
| 531 |
+
reader.readAsDataURL(blob);
|
| 532 |
+
// Prevent default paste handling
|
| 533 |
+
evt.preventDefault();
|
| 534 |
+
break;
|
| 535 |
+
}
|
| 536 |
+
}
|
| 537 |
+
}
|
| 538 |
+
|
| 539 |
+
document.addEventListener('paste', paste_update);
|
| 540 |
+
"""
|
| 541 |
+
|
| 542 |
+
with gr.Blocks(css=css, js=js) as demo:
|
| 543 |
+
gr.Markdown("# WD EVA02 LoRA ONNX Tagger")
|
| 544 |
+
gr.Markdown("Upload an image or paste an image URL to predict tags using the fine-tuned WD EVA02 Tagger model (ONNX format).")
|
| 545 |
+
gr.Markdown(f"Model Repository: [{REPO_ID}](https://huggingface.co/{REPO_ID})")
|
| 546 |
+
|
| 547 |
+
with gr.Row():
|
| 548 |
+
with gr.Column(scale=1):
|
| 549 |
+
# Use elem_id for JS targeting
|
| 550 |
+
image_input = gr.Image(type="pil", label="Input Image", elem_id="input-image")
|
| 551 |
+
# Container for URL paste message
|
| 552 |
+
gr.HTML("<div id='url-input-container'></div>")
|
| 553 |
+
|
| 554 |
+
gen_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, value=0.55, label="General Tag Threshold")
|
| 555 |
+
char_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, value=0.60, label="Character/Copyright/Artist Tag Threshold")
|
| 556 |
+
output_mode = gr.Radio(choices=["Tags Only", "Tags + Visualization"], value="Tags + Visualization", label="Output Mode")
|
| 557 |
+
predict_button = gr.Button("Predict", variant="primary")
|
| 558 |
+
|
| 559 |
+
with gr.Column(scale=1):
|
| 560 |
+
output_tags = gr.Textbox(label="Predicted Tags", lines=10)
|
| 561 |
+
output_visualization = gr.Image(type="pil", label="Prediction Visualization")
|
| 562 |
+
|
| 563 |
+
# Examples
|
| 564 |
+
gr.Examples(
|
| 565 |
+
examples=[
|
| 566 |
+
["https://pbs.twimg.com/media/GXBXsRvbQAAg1kp.jpg", 0.55, 0.5, "Tags + Visualization"],
|
| 567 |
+
["https://pbs.twimg.com/media/GjlX0gibcAA4EJ4.jpg", 0.5, 0.5, "Tags Only"],
|
| 568 |
+
["https://pbs.twimg.com/media/Gj4nQbjbEAATeoH.jpg", 0.55, 0.5, "Tags + Visualization"],
|
| 569 |
+
["https://pbs.twimg.com/media/GkbtX0GaoAMlUZt.jpg", 0.45, 0.45, "Tags + Visualization"]
|
| 570 |
+
],
|
| 571 |
+
inputs=[image_input, gen_threshold, char_threshold, output_mode],
|
| 572 |
+
outputs=[output_tags, output_visualization],
|
| 573 |
+
fn=predict,
|
| 574 |
+
cache_examples=False # Slows down startup if True and large examples
|
| 575 |
+
)
|
| 576 |
+
|
| 577 |
+
predict_button.click(
|
| 578 |
+
fn=predict,
|
| 579 |
+
inputs=[image_input, gen_threshold, char_threshold, output_mode],
|
| 580 |
+
outputs=[output_tags, output_visualization]
|
| 581 |
+
)
|
| 582 |
+
|
| 583 |
+
# Add listener for image input changes (e.g., from pasting)
|
| 584 |
+
# This might trigger prediction automatically or require the button click
|
| 585 |
+
# image_input.change(
|
| 586 |
+
# fn=predict,
|
| 587 |
+
# inputs=[image_input, gen_threshold, char_threshold, output_mode],
|
| 588 |
+
# outputs=[output_tags, output_visualization]
|
| 589 |
+
# )
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
if __name__ == "__main__":
|
| 593 |
+
# 環境変数HF_TOKENがない場合に警告(プライベートリポジトリ用)
|
| 594 |
+
if not os.environ.get("HF_TOKEN"):
|
| 595 |
+
print("Warning: HF_TOKEN environment variable not set. Downloads from private repositories may fail.")
|
| 596 |
+
# Initialize model on startup to avoid delay on first prediction
|
| 597 |
+
# initialize_model() # Removed startup initialization
|
|
|
|
|
|
|
| 598 |
demo.launch()
|
requirements.txt
CHANGED
|
@@ -1,10 +1,11 @@
|
|
| 1 |
-
--extra-index-url https://download.pytorch.org/whl/cu118
|
| 2 |
-
torch
|
| 3 |
-
torchvision
|
| 4 |
-
torchaudio
|
| 5 |
-
onnxruntime-gpu==1.
|
| 6 |
-
numpy
|
| 7 |
-
Pillow
|
| 8 |
-
matplotlib
|
| 9 |
-
requests
|
|
|
|
| 10 |
huggingface_hub
|
|
|
|
| 1 |
+
--extra-index-url https://download.pytorch.org/whl/cu118
|
| 2 |
+
torch
|
| 3 |
+
torchvision
|
| 4 |
+
torchaudio
|
| 5 |
+
onnxruntime-gpu==1.18.0 # または onnxruntime>=1.16.0 (CPUのみの場合)
|
| 6 |
+
numpy
|
| 7 |
+
Pillow
|
| 8 |
+
matplotlib
|
| 9 |
+
requests
|
| 10 |
+
gradio>=4.44.0
|
| 11 |
huggingface_hub
|