VisionScout / scene_analysis_coordinator.py
DawnC's picture
Upload 6 files
ba55edb verified
import logging
import traceback
import numpy as np
from typing import Dict, List, Tuple, Optional, Any
from PIL import Image
class SceneAnalysisCoordinator:
"""
負責整個場景分析流程的協調和控制邏輯,包含主要的分析流程、
處理無檢測結果的回退邏輯,以及多源分析結果的整合。
"""
def __init__(self, component_initializer, scene_scoring_engine, landmark_processing_manager,
scene_confidence_threshold: float = 0.6):
"""
初始化場景分析協調器。
Args:
component_initializer: 組件初始化器實例
scene_scoring_engine: 場景評分引擎實例
landmark_processing_manager: 地標處理管理器實例
scene_confidence_threshold: 場景置信度閾值
"""
self.logger = logging.getLogger(__name__)
self.component_initializer = component_initializer
self.scene_scoring_engine = scene_scoring_engine
self.landmark_processing_manager = landmark_processing_manager
self.scene_confidence_threshold = scene_confidence_threshold
# 獲取必要的組件和數據
self.spatial_analyzer = component_initializer.get_component('spatial_analyzer')
self.descriptor = component_initializer.get_component('descriptor')
self.scene_describer = component_initializer.get_component('scene_describer')
self.clip_analyzer = component_initializer.get_component('clip_analyzer')
self.llm_enhancer = component_initializer.get_component('llm_enhancer')
self.scene_types = component_initializer.get_data_structure('SCENE_TYPES')
# 從組件初始化器獲取功能開關狀態
self.use_clip = component_initializer.use_clip
self.use_llm = component_initializer.use_llm
self.enable_landmark = component_initializer.enable_landmark
def analyze(self, detection_result: Any, lighting_info: Optional[Dict] = None,
class_confidence_threshold: float = 0.25, scene_confidence_threshold: float = 0.6,
enable_landmark: bool = True, places365_info: Optional[Dict] = None) -> Dict:
"""
分析檢測結果以確定場景類型並提供理解。
Args:
detection_result: 來自 YOLOv8 或類似系統的檢測結果
lighting_info: 可選的照明條件分析結果
class_confidence_threshold: 考慮物體的最小置信度
scene_confidence_threshold: 確定場景的最小置信度
enable_landmark: 是否為此次運行啟用地標檢測和識別
places365_info: 可選的 Places365 場景分類結果
Returns:
包含場景分析結果的字典
"""
current_run_enable_landmark = enable_landmark
self.logger.info(f"DIAGNOSTIC (SceneAnalyzer.analyze): Called with current_run_enable_landmark={current_run_enable_landmark}")
self.logger.debug(f"SceneAnalyzer received lighting_info type: {type(lighting_info)}")
self.logger.debug(f"SceneAnalyzer lighting_info source: {lighting_info.get('source', 'unknown') if isinstance(lighting_info, dict) else 'not_dict'}")
# 記錄 Places365 資訊
if places365_info:
self.logger.info(f"DIAGNOSTIC: Places365 info received - scene: {places365_info.get('scene_label', 'unknown')}, "
f"mapped: {places365_info.get('mapped_scene_type', 'unknown')}, "
f"confidence: {places365_info.get('confidence', 0.0):.3f}")
# 同步 enable_landmark 狀態到子組件(為此次分析運行)
self._sync_landmark_status_to_components(current_run_enable_landmark)
# 提取和處理原始圖像
original_image_pil, image_dims_val = self._extract_image_info(detection_result)
# 處理無 YOLO 檢測結果的情況
no_yolo_detections = self._check_no_yolo_detections(detection_result)
if no_yolo_detections:
return self._handle_no_yolo_detections(
original_image_pil, image_dims_val, current_run_enable_landmark,
lighting_info, places365_info
)
# 主處理流程(有 YOLO 檢測結果)
return self._handle_main_analysis_flow(
detection_result, original_image_pil, image_dims_val,
class_confidence_threshold, scene_confidence_threshold,
current_run_enable_landmark, lighting_info, places365_info
)
def _sync_landmark_status_to_components(self, current_run_enable_landmark: bool):
"""同步地標狀態到所有相關組件。"""
# 更新場景評分引擎
self.scene_scoring_engine.update_enable_landmark_status(current_run_enable_landmark)
# 更新地標處理管理器
self.landmark_processing_manager.update_enable_landmark_status(current_run_enable_landmark)
# 更新其他組件的地標狀態
for component_name in ['scene_describer', 'clip_analyzer', 'landmark_classifier']:
component = self.component_initializer.get_component(component_name)
if component and hasattr(component, 'enable_landmark'):
component.enable_landmark = current_run_enable_landmark
# 更新實例狀態
self.enable_landmark = current_run_enable_landmark
def _extract_image_info(self, detection_result) -> Tuple[Optional[Image.Image], Optional[Tuple[int, int]]]:
"""從檢測結果中提取圖像信息。"""
original_image_pil = None
image_dims_val = None # 將是 (width, height)
if (detection_result is not None and hasattr(detection_result, 'orig_img') and
detection_result.orig_img is not None):
if isinstance(detection_result.orig_img, np.ndarray):
try:
img_array = detection_result.orig_img
if img_array.ndim == 3 and img_array.shape[2] == 4: # RGBA
img_array = img_array[:, :, :3] # 轉換為 RGB
if img_array.ndim == 2: # 灰度
original_image_pil = Image.fromarray(img_array).convert("RGB")
else: # 假設 RGB 或 BGR(如果源是 cv2 BGR,PIL 在 fromarray 時會處理 BGR->RGB,但明確處理更好)
original_image_pil = Image.fromarray(img_array)
if hasattr(original_image_pil, 'mode') and original_image_pil.mode == 'BGR': # 明確將 OpenCV 的 BGR 轉換為 PIL 的 RGB
original_image_pil = original_image_pil.convert('RGB')
image_dims_val = (original_image_pil.width, original_image_pil.height)
except Exception as e:
self.logger.warning(f"Error converting NumPy orig_img to PIL: {e}")
elif hasattr(detection_result.orig_img, 'size') and callable(getattr(detection_result.orig_img, 'convert', None)):
original_image_pil = detection_result.orig_img.copy().convert("RGB") # 確保 RGB
image_dims_val = original_image_pil.size
else:
self.logger.warning(f"detection_result.orig_img (type: {type(detection_result.orig_img)}) is not a recognized NumPy array or PIL Image.")
else:
self.logger.warning("detection_result.orig_img not available. Image-based analysis will be limited.")
return original_image_pil, image_dims_val
def _check_no_yolo_detections(self, detection_result) -> bool:
"""檢查是否沒有 YOLO 檢測結果。"""
return (detection_result is None or
not hasattr(detection_result, 'boxes') or
not hasattr(detection_result.boxes, 'xyxy') or
len(detection_result.boxes.xyxy) == 0)
def _handle_no_yolo_detections(self, original_image_pil, image_dims_val,
current_run_enable_landmark, lighting_info, places365_info) -> Dict:
"""處理無 YOLO 檢測結果的情況。"""
tried_landmark_detection = False
landmark_detection_result = None
# 嘗試地標檢測
if original_image_pil and self.use_clip and current_run_enable_landmark:
landmark_detection_result = self._attempt_landmark_detection_no_yolo(
original_image_pil, image_dims_val, lighting_info
)
tried_landmark_detection = True
if landmark_detection_result:
return landmark_detection_result
# 如果地標檢測失敗或未嘗試,使用 CLIP 進行一般場景分析
if not landmark_detection_result and self.use_clip and original_image_pil:
clip_fallback_result = self._attempt_clip_fallback_analysis(
original_image_pil, image_dims_val, current_run_enable_landmark, lighting_info
)
if clip_fallback_result:
return clip_fallback_result
# 最終回退邏輯
return self._get_final_fallback_result(places365_info, lighting_info)
def _attempt_landmark_detection_no_yolo(self, original_image_pil, image_dims_val, lighting_info) -> Optional[Dict]:
"""在無 YOLO 檢測的情況下嘗試地標檢測。"""
try:
# 初始化地標分類器(如果需要)
landmark_classifier = self.component_initializer.get_component('landmark_classifier')
if not landmark_classifier and self.clip_analyzer:
if hasattr(self.clip_analyzer, 'get_clip_instance'):
try:
model, preprocess, device = self.clip_analyzer.get_clip_instance()
landmark_classifier = CLIPZeroShotClassifier(device=device)
self.landmark_processing_manager.set_landmark_classifier(landmark_classifier)
self.logger.info("Initialized landmark classifier with shared CLIP model")
except Exception as e:
self.logger.warning(f"Could not initialize landmark classifier: {e}")
return None
if landmark_classifier:
self.logger.info("Attempting landmark detection with no YOLO boxes")
landmark_results_no_yolo = landmark_classifier.intelligent_landmark_search(
original_image_pil, yolo_boxes=None, base_threshold=0.2 # 略微降低閾值,提高靈敏度
)
# 確保在無地標場景時返回有效結果
if landmark_results_no_yolo is None:
landmark_results_no_yolo = {"is_landmark_scene": False, "detected_landmarks": []}
if (landmark_results_no_yolo and landmark_results_no_yolo.get("is_landmark_scene", False)):
return self._process_landmark_detection_result(
landmark_results_no_yolo, image_dims_val, lighting_info
)
except Exception as e:
self.logger.error(f"Error in landmark-only detection path (analyze method): {e}")
traceback.print_exc()
return None
def _process_landmark_detection_result(self, landmark_results, image_dims_val, lighting_info) -> Dict:
"""處理地標檢測結果並生成最終輸出。"""
primary_landmark = landmark_results.get("primary_landmark")
# 放寬閾值條件,以便捕獲更多潛在地標
if not primary_landmark or primary_landmark.get("confidence", 0) <= 0.25:
return None
detected_objects_from_landmarks_list = []
w_img, h_img = image_dims_val if image_dims_val else (1, 1)
for lm_info_item in landmark_results.get("detected_landmarks", []):
if lm_info_item.get("confidence", 0) > 0.25: # 降低閾值與上面保持一致
# 安全獲取 box 值,避免索引錯誤
box = lm_info_item.get("box", [0, 0, w_img, h_img])
if len(box) < 4:
box = [0, 0, w_img, h_img]
# 計算中心點和標準化坐標
center_x, center_y = (box[0] + box[2]) / 2, (box[1] + box[3]) / 2
norm_cx = center_x / w_img if w_img > 0 else 0.5
norm_cy = center_y / h_img if h_img > 0 else 0.5
# 決定地標類型
landmark_type = "architectural" # 預設類型
landmark_id = lm_info_item.get("landmark_id", "")
landmark_classifier = self.component_initializer.get_component('landmark_classifier')
if (landmark_classifier and hasattr(landmark_classifier, '_determine_landmark_type') and landmark_id):
try:
landmark_type = landmark_classifier._determine_landmark_type(landmark_id)
except Exception as e:
self.logger.error(f"Error determining landmark type: {e}")
else:
# 使用簡單的基於 ID 的啟發式方法推斷類型
landmark_id_lower = landmark_id.lower() if isinstance(landmark_id, str) else ""
if "natural" in landmark_id_lower or any(term in landmark_id_lower for term in ["mountain", "waterfall", "canyon", "lake"]):
landmark_type = "natural"
elif "monument" in landmark_id_lower or "memorial" in landmark_id_lower or "historical" in landmark_id_lower:
landmark_type = "monument"
# 決定區域位置
region = "center" # 預設值
if self.spatial_analyzer and hasattr(self.spatial_analyzer, '_determine_region'):
try:
region = self.spatial_analyzer._determine_region(norm_cx, norm_cy)
except Exception as e:
self.logger.error(f"Error determining region: {e}")
# 取得並補 location
loc_lm = lm_info_item.get("location", "")
if not loc_lm and landmark_id in ALL_LANDMARKS:
loc_lm = ALL_LANDMARKS[landmark_id].get("location", "")
# 創建地標物體
landmark_obj = {
"class_id": lm_info_item.get("landmark_id", f"LM_{lm_info_item.get('landmark_name','unk')}")[:15],
"class_name": lm_info_item.get("landmark_name", "Unknown Landmark"),
"confidence": lm_info_item.get("confidence", 0.0),
"box": box,
"center": (center_x, center_y),
"normalized_center": (norm_cx, norm_cy),
"size": (box[2] - box[0], box[3] - box[1]),
"normalized_size": (
(box[2] - box[0])/(w_img if w_img>0 else 1),
(box[3] - box[1])/(h_img if h_img>0 else 1)
),
"area": (box[2] - box[0]) * (box[3] - box[1]),
"normalized_area": (
(box[2] - box[0]) * (box[3] - box[1])
) / ((w_img*h_img) if w_img*h_img >0 else 1),
"is_landmark": True,
"landmark_id": landmark_id,
"location": loc_lm or "Unknown Location",
"region": region,
"year_built": lm_info_item.get("year_built", ""),
"architectural_style": lm_info_item.get("architectural_style", ""),
"significance": lm_info_item.get("significance", ""),
"landmark_type": landmark_type
}
detected_objects_from_landmarks_list.append(landmark_obj)
if not detected_objects_from_landmarks_list:
return None
# 設定場景類型
best_scene_val = "tourist_landmark" # 預設
if primary_landmark:
try:
lm_type = primary_landmark.get("landmark_type", "architectural")
if lm_type and "natural" in lm_type.lower():
best_scene_val = "natural_landmark"
elif lm_type and ("historical" in lm_type.lower() or "monument" in lm_type.lower()):
best_scene_val = "historical_monument"
except Exception as e:
self.logger.error(f"Error determining scene type from landmark type: {e}")
# 確保場景類型有效
if best_scene_val not in self.scene_types:
best_scene_val = "tourist_landmark" # 預設場景類型
# 設定置信度
scene_confidence = primary_landmark.get("confidence", 0.0) if primary_landmark else 0.0
# 生成其他必要的分析結果
region_analysis = self._generate_region_analysis(detected_objects_from_landmarks_list)
functional_zones = self._generate_functional_zones(
detected_objects_from_landmarks_list,
best_scene_val
)
scene_description = self._generate_scene_description(
best_scene_val, detected_objects_from_landmarks_list, scene_confidence,
lighting_info, functional_zones, image_dims_val
)
enhanced_description = self._enhance_description_with_llm(
scene_description, best_scene_val, detected_objects_from_landmarks_list,
scene_confidence, lighting_info, functional_zones, landmark_results, image_dims_val
)
possible_activities = self._extract_possible_activities(detected_objects_from_landmarks_list, landmark_results)
safety_concerns = []
if self.descriptor and hasattr(self.descriptor, '_identify_safety_concerns'):
safety_concerns = self.descriptor._identify_safety_concerns(detected_objects_from_landmarks_list, best_scene_val)
# 準備最終結果
return {
"scene_type": best_scene_val,
"scene_name": self.scene_types.get(best_scene_val, {}).get("name", "Landmark"),
"confidence": round(float(scene_confidence), 4),
"description": scene_description,
"enhanced_description": enhanced_description,
"objects_present": detected_objects_from_landmarks_list,
"object_count": len(detected_objects_from_landmarks_list),
"regions": region_analysis,
"possible_activities": possible_activities,
"safety_concerns": safety_concerns,
"functional_zones": functional_zones,
"detected_landmarks": [lm for lm in detected_objects_from_landmarks_list if lm.get("is_landmark", False)],
"primary_landmark": primary_landmark,
"lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0}
}
def _attempt_clip_fallback_analysis(self, original_image_pil, image_dims_val,
current_run_enable_landmark, lighting_info) -> Optional[Dict]:
"""嘗試使用 CLIP 進行一般場景分析。"""
try:
clip_analysis_val = None
if self.clip_analyzer and hasattr(self.clip_analyzer, 'analyze_image'):
try:
clip_analysis_val = self.clip_analyzer.analyze_image(
original_image_pil,
enable_landmark=current_run_enable_landmark
)
except Exception as e:
self.logger.error(f"Error in CLIP analysis: {e}")
scene_type_llm = "llm_inferred_no_yolo"
confidence_llm = 0.0
if clip_analysis_val and isinstance(clip_analysis_val, dict):
top_scene = clip_analysis_val.get("top_scene")
if top_scene and isinstance(top_scene, tuple) and len(top_scene) >= 2:
confidence_llm = top_scene[1]
if isinstance(top_scene[0], str):
scene_type_llm = top_scene[0]
desc_llm = "Primary object detection did not yield results. This description is based on overall image context."
w_llm, h_llm = image_dims_val if image_dims_val else (1, 1)
enhanced_desc_llm = self._enhance_no_detection_description(
desc_llm, scene_type_llm, confidence_llm, lighting_info,
clip_analysis_val, current_run_enable_landmark, w_llm, h_llm
)
# 安全類型轉換
try:
confidence_float = float(confidence_llm)
except (ValueError, TypeError):
confidence_float = 0.0
# 確保增強描述不為空
if not enhanced_desc_llm or not isinstance(enhanced_desc_llm, str):
enhanced_desc_llm = desc_llm
# 返回結果
return {
"scene_type": scene_type_llm,
"confidence": round(confidence_float, 4),
"description": desc_llm,
"enhanced_description": enhanced_desc_llm,
"objects_present": [],
"object_count": 0,
"regions": {},
"possible_activities": [],
"safety_concerns": [],
"lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0}
}
except Exception as e:
self.logger.error(f"Error in CLIP no-detection fallback (analyze method): {e}")
traceback.print_exc()
return None
def _get_final_fallback_result(self, places365_info, lighting_info) -> Dict:
"""獲取最終的回退結果。"""
# 檢查 Places365 是否提供有用的場景信息(即使沒有 YOLO 檢測)
fallback_scene_type = "unknown"
fallback_confidence = 0.0
fallback_description = "No objects were detected in the image, and contextual analysis could not be performed or failed."
if places365_info and places365_info.get('confidence', 0) > 0.3:
fallback_scene_type = places365_info.get('mapped_scene_type', 'unknown')
fallback_confidence = places365_info.get('confidence', 0.0)
fallback_description = f"Scene appears to be {places365_info.get('scene_label', 'an unidentified location')} based on overall visual context."
return {
"scene_type": fallback_scene_type,
"confidence": fallback_confidence,
"description": fallback_description,
"enhanced_description": "The image analysis system could not detect any recognizable objects or landmarks in this image.",
"objects_present": [],
"object_count": 0,
"regions": {},
"possible_activities": [],
"safety_concerns": [],
"lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0}
}
def _handle_main_analysis_flow(self, detection_result, original_image_pil, image_dims_val,
class_confidence_threshold, scene_confidence_threshold,
current_run_enable_landmark, lighting_info, places365_info) -> Dict:
"""處理主要的分析流程(有 YOLO 檢測結果)。"""
# 更新類別名稱映射
if hasattr(detection_result, 'names'):
if hasattr(self.spatial_analyzer, 'class_names'):
self.spatial_analyzer.class_names = detection_result.names
# 提取檢測到的物體
detected_objects_main = self.spatial_analyzer._extract_detected_objects(
detection_result,
confidence_threshold=class_confidence_threshold
)
if not detected_objects_main:
return {
"scene_type": "unknown", "confidence": 0.0,
"description": "No objects detected with sufficient confidence by the primary vision system.",
"objects_present": [], "object_count": 0, "regions": {}, "possible_activities": [],
"safety_concerns": [], "lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0.0}
}
# 空間分析
region_analysis_val = self.spatial_analyzer._analyze_regions(detected_objects_main)
if current_run_enable_landmark:
self.logger.info("Using landmark detection logic for YOLO scene")
return self._handle_no_yolo_detections(
original_image_pil, image_dims_val, current_run_enable_landmark,
lighting_info, places365_info
)
# 地標處理和整合
landmark_objects_identified = []
landmark_specific_activities = []
final_landmark_info = {}
# 如果當前運行禁用地標檢測,清理地標物體
if not current_run_enable_landmark:
detected_objects_main = [obj for obj in detected_objects_main if not obj.get("is_landmark", False)]
final_landmark_info = {}
# 計算場景分數並進行融合
yolo_scene_scores = self.scene_scoring_engine.compute_scene_scores(
detected_objects_main, spatial_analysis_results=region_analysis_val
)
clip_scene_scores = {}
clip_analysis_results = None
if self.use_clip and original_image_pil is not None:
clip_analysis_results, clip_scene_scores = self._perform_clip_analysis(
original_image_pil, current_run_enable_landmark, lighting_info
)
# 融合場景分數
yolo_only_objects = [obj for obj in detected_objects_main if not obj.get("is_landmark")]
num_yolo_detections = len(yolo_only_objects)
avg_yolo_confidence = (sum(obj.get('confidence', 0.0) for obj in yolo_only_objects) / num_yolo_detections
if num_yolo_detections > 0 else 0.0)
scene_scores_fused = self.scene_scoring_engine.fuse_scene_scores(
yolo_scene_scores, clip_scene_scores,
num_yolo_detections=num_yolo_detections,
avg_yolo_confidence=avg_yolo_confidence,
lighting_info=lighting_info,
places365_info=places365_info
)
# 確定最終場景類型
final_best_scene, final_scene_confidence = self.scene_scoring_engine.determine_scene_type(scene_scores_fused)
# 處理禁用地標檢測時的替代場景類型
if (not current_run_enable_landmark and
final_best_scene in ["tourist_landmark", "natural_landmark", "historical_monument"]):
alt_scene_type = self.landmark_processing_manager.get_alternative_scene_type(
final_best_scene, detected_objects_main, scene_scores_fused
)
final_best_scene = alt_scene_type
final_scene_confidence = scene_scores_fused.get(alt_scene_type, 0.6)
# 生成最終的描述性內容
final_result = self._generate_final_result(
final_best_scene, final_scene_confidence, detected_objects_main,
landmark_specific_activities, landmark_objects_identified, final_landmark_info,
region_analysis_val, lighting_info, scene_scores_fused, current_run_enable_landmark,
clip_analysis_results, image_dims_val, scene_confidence_threshold
)
return final_result
def _perform_clip_analysis(self, original_image_pil, current_run_enable_landmark, lighting_info) -> Tuple[Optional[Dict], Dict]:
"""執行 CLIP 分析。"""
clip_analysis_results = None
clip_scene_scores = {}
try:
clip_analysis_results = self.clip_analyzer.analyze_image(
original_image_pil,
enable_landmark=current_run_enable_landmark,
exclude_categories=["landmark", "tourist", "monument", "tower", "attraction", "scenic", "historical", "famous"] if not current_run_enable_landmark else None
)
if isinstance(clip_analysis_results, dict):
clip_scene_scores = clip_analysis_results.get("scene_scores", {})
# 如果禁用地標檢測,再次過濾
if not current_run_enable_landmark:
clip_scene_scores = {k: v for k, v in clip_scene_scores.items()
if not any(kw in k.lower() for kw in ["landmark", "monument", "tourist"])}
if "cultural_analysis" in clip_analysis_results:
del clip_analysis_results["cultural_analysis"]
if ("top_scene" in clip_analysis_results and
any(term in clip_analysis_results.get("top_scene", ["unknown", 0.0])[0].lower()
for term in ["landmark", "monument", "tourist"])):
non_lm_cs = sorted([item for item in clip_scene_scores.items() if item[1] > 0],
key=lambda x: x[1], reverse=True)
clip_analysis_results["top_scene"] = non_lm_cs[0] if non_lm_cs else ("unknown", 0.0)
# 處理照明信息回退
if (not lighting_info and "lighting_condition" in clip_analysis_results):
lt, lc = clip_analysis_results.get("lighting_condition", ("unknown", 0.0))
lighting_info = {"time_of_day": lt, "confidence": lc, "source": "CLIP_fallback"}
except Exception as e:
self.logger.error(f"Error in main CLIP analysis for YOLO path (analyze method): {e}")
return clip_analysis_results, clip_scene_scores
def _generate_final_result(self, final_best_scene, final_scene_confidence, detected_objects_main,
landmark_specific_activities, landmark_objects_identified, final_landmark_info,
region_analysis_val, lighting_info, scene_scores_fused, current_run_enable_landmark,
clip_analysis_results, image_dims_val, scene_confidence_threshold) -> Dict:
"""生成最終的分析結果。"""
# 生成最終的描述性內容(活動、安全、區域)
final_activities = []
# 通用活動推斷
generic_activities = []
if self.descriptor and hasattr(self.descriptor, '_infer_possible_activities'):
generic_activities = self.descriptor._infer_possible_activities(
final_best_scene, detected_objects_main,
enable_landmark=current_run_enable_landmark, scene_scores=scene_scores_fused
)
# 優先處理策略:使用特定地標活動,不足時才從通用活動補充
if landmark_specific_activities:
# 如果有特定活動,優先保留,去除與特定活動重複的通用活動
unique_generic_activities = [act for act in generic_activities if act not in landmark_specific_activities]
# 如果特定活動少於3個,從通用活動中補充
if len(landmark_specific_activities) < 3:
# 補充通用活動但總數不超過7個
supplement_count = min(3 - len(landmark_specific_activities), len(unique_generic_activities))
if supplement_count > 0:
final_activities.extend(unique_generic_activities[:supplement_count])
else:
# 若無特定活動,則使用所有通用活動
final_activities.extend(generic_activities)
# 去重並排序,但確保特定地標活動保持在前面
final_activities_set = set(final_activities)
final_activities = []
# 先加入特定地標活動(按原順序)
for activity in landmark_specific_activities:
if activity in final_activities_set:
final_activities.append(activity)
final_activities_set.remove(activity)
# 再加入通用活動(按字母排序)
final_activities.extend(sorted(list(final_activities_set)))
# 安全問題識別
final_safety_concerns = []
if self.descriptor and hasattr(self.descriptor, '_identify_safety_concerns'):
final_safety_concerns = self.descriptor._identify_safety_concerns(detected_objects_main, final_best_scene)
# 功能區域識別
final_functional_zones = {}
if self.spatial_analyzer and hasattr(self.spatial_analyzer, '_identify_functional_zones'):
general_zones = self.spatial_analyzer._identify_functional_zones(detected_objects_main, final_best_scene)
final_functional_zones.update(general_zones)
# 地標相關的功能區域
if landmark_objects_identified and self.spatial_analyzer and hasattr(self.spatial_analyzer, '_identify_landmark_zones'):
landmark_zones = self.spatial_analyzer._identify_landmark_zones(landmark_objects_identified)
final_functional_zones.update(landmark_zones)
# 如果當前運行禁用地標檢測,過濾相關內容
if not current_run_enable_landmark:
final_functional_zones = {
str(k): v
for k, v in final_functional_zones.items()
if (not str(k).isdigit())
and (not any(kw in str(k).lower() for kw in ["landmark", "monument", "viewing", "tourist"]))
}
current_activities_temp = [act for act in final_activities
if not any(kw in act.lower() for kw in ["sightsee", "photograph", "tour", "histor", "landmark", "monument", "cultur"])]
final_activities = current_activities_temp
if not final_activities and self.descriptor and hasattr(self.descriptor, '_infer_possible_activities'):
final_activities = self.descriptor._infer_possible_activities("generic_street_view", detected_objects_main, enable_landmark=False)
# 創建淨化的光線資訊,避免不合理的時間描述
lighting_info_clean = None
if lighting_info:
lighting_info_clean = {
"is_indoor": lighting_info.get("is_indoor"),
"confidence": lighting_info.get("confidence", 0.0),
"time_of_day": lighting_info.get("time_of_day", "unknown")
}
# 生成場景描述
base_scene_description = self._generate_scene_description(
final_best_scene, detected_objects_main, final_scene_confidence,
lighting_info_clean, final_functional_zones, image_dims_val
)
# 清理地標引用(如果禁用地標檢測)
if not current_run_enable_landmark:
base_scene_description = self.landmark_processing_manager.remove_landmark_references(base_scene_description)
# LLM 增強
enhanced_final_description = self._enhance_final_description(
base_scene_description, final_best_scene, final_scene_confidence, detected_objects_main,
final_functional_zones, final_activities, final_safety_concerns, lighting_info,
clip_analysis_results, current_run_enable_landmark, image_dims_val, final_landmark_info
)
# 清理增強描述的地標引用
if not current_run_enable_landmark:
enhanced_final_description = self.landmark_processing_manager.remove_landmark_references(enhanced_final_description)
# 構建最終輸出字典
output_result = {
"scene_type": final_best_scene if final_scene_confidence >= scene_confidence_threshold else "unknown",
"scene_name": (self.scene_types.get(final_best_scene, {}).get("name", "Unknown Scene")
if final_scene_confidence >= scene_confidence_threshold else "Unknown Scene"),
"confidence": round(float(final_scene_confidence), 4),
"description": base_scene_description,
"enhanced_description": enhanced_final_description,
"objects_present": [{"class_id": obj.get("class_id", -1),
"class_name": obj.get("class_name", "unknown"),
"confidence": round(float(obj.get("confidence", 0.0)), 4)}
for obj in detected_objects_main],
"object_count": len(detected_objects_main),
"regions": region_analysis_val,
"possible_activities": final_activities,
"safety_concerns": final_safety_concerns,
"functional_zones": final_functional_zones,
"lighting_conditions": lighting_info if lighting_info else {"time_of_day": "unknown", "confidence": 0.0, "source": "default"}
}
# 添加替代場景
if self.descriptor and hasattr(self.descriptor, '_get_alternative_scenes'):
output_result["alternative_scenes"] = self.descriptor._get_alternative_scenes(
scene_scores_fused, scene_confidence_threshold, top_k=2
)
# 添加地標相關信息
if current_run_enable_landmark and final_landmark_info and final_landmark_info.get("detected_landmarks"):
output_result.update(final_landmark_info)
if final_best_scene in ["tourist_landmark", "natural_landmark", "historical_monument"]:
output_result["scene_source"] = "landmark_detection"
elif not current_run_enable_landmark:
for key_rm in ["detected_landmarks", "primary_landmark", "detailed_landmarks", "scene_source"]:
if key_rm in output_result:
del output_result[key_rm]
# 添加 CLIP 分析結果
if clip_analysis_results and isinstance(clip_analysis_results, dict) and "error" not in clip_analysis_results:
top_scene_clip = clip_analysis_results.get("top_scene", ("unknown", 0.0))
output_result["clip_analysis"] = {
"top_scene": (top_scene_clip[0], round(float(top_scene_clip[1]), 4)),
"cultural_analysis": clip_analysis_results.get("cultural_analysis", {}) if current_run_enable_landmark else {}
}
return output_result
# 輔助方法
def _generate_region_analysis(self, detected_objects):
"""生成區域分析結果。"""
if self.spatial_analyzer and hasattr(self.spatial_analyzer, '_analyze_regions'):
try:
return self.spatial_analyzer._analyze_regions(detected_objects)
except Exception as e:
self.logger.error(f"Error analyzing regions: {e}")
return {}
def _generate_functional_zones(self, detected_objects, scene_type):
"""
生成功能區域。
由於原本直接呼叫 _identify_landmark_zones,導致非地標場景必定回 {}。
這裡改為呼叫 _identify_functional_zones,並帶入 scene_type。
"""
try:
# 如果 spatial_analyzer 可以識別 functional zones,就調用它
if self.spatial_analyzer and hasattr(self.spatial_analyzer, '_identify_functional_zones'):
return self.spatial_analyzer._identify_functional_zones(detected_objects, scene_type)
except Exception as e:
self.logger.error(f"Error identifying functional zones: {e}")
self.logger.error(traceback.format_exc())
return {}
def _generate_scene_description(self, scene_type, detected_objects, confidence,
lighting_info, functional_zones, image_dims):
"""生成場景描述。"""
if self.scene_describer and hasattr(self.scene_describer, 'generate_description'):
try:
for obj in detected_objects:
if obj.get("is_landmark"):
loc_obj = obj.get("location", "")
lm_id_obj = obj.get("landmark_id")
if (not loc_obj) and lm_id_obj and lm_id_obj in ALL_LANDMARKS:
obj["location"] = ALL_LANDMARKS[lm_id_obj].get("location", "")
return self.scene_describer.generate_description(
scene_type=scene_type,
detected_objects=detected_objects,
confidence=confidence,
lighting_info=lighting_info,
functional_zones=list(functional_zones.keys()) if functional_zones else [],
enable_landmark=self.enable_landmark,
scene_scores={scene_type: confidence},
spatial_analysis={},
image_dimensions=image_dims
)
except Exception as e:
self.logger.error(f"Error generating scene description: {e}")
return f"A {scene_type} scene."
def _enhance_description_with_llm(self, scene_description, scene_type, detected_objects,
confidence, lighting_info, functional_zones, landmark_results, image_dims):
"""使用 LLM 增強描述。"""
if not self.use_llm or not self.llm_enhancer:
return scene_description
try:
prominent_objects_detail = ""
if self.scene_describer and hasattr(self.scene_describer, 'format_object_list_for_description'):
try:
prominent_objects_detail = self.scene_describer.format_object_list_for_description(
detected_objects[:min(1, len(detected_objects))]
)
except Exception as e:
self.logger.error(f"Error formatting object list: {e}")
w_img, h_img = image_dims if image_dims else (1, 1)
scene_data_llm = {
"original_description": scene_description,
"scene_type": scene_type,
"scene_name": self.scene_types.get(scene_type, {}).get("name", "Landmark"),
"detected_objects": detected_objects,
"object_list": "landmark",
"confidence": confidence,
"lighting_info": lighting_info,
"functional_zones": functional_zones,
"clip_analysis": landmark_results.get("clip_analysis_on_full_image", {}),
"enable_landmark": True,
"image_width": w_img,
"image_height": h_img,
"prominent_objects_detail": prominent_objects_detail
}
return self.llm_enhancer.enhance_description(scene_data_llm)
except Exception as e:
self.logger.error(f"Error enhancing description with LLM: {e}")
traceback.print_exc()
return scene_description
def _enhance_no_detection_description(self, desc, scene_type, confidence, lighting_info,
clip_analysis, enable_landmark, width, height):
"""增強無檢測結果的描述。"""
if not self.use_llm or not self.llm_enhancer:
return desc
try:
clip_analysis_safe = {}
if isinstance(clip_analysis, dict):
clip_analysis_safe = clip_analysis
scene_data_llm = {
"original_description": desc,
"scene_type": scene_type,
"scene_name": "Contextually Inferred (No Detections)",
"detected_objects": [],
"object_list": "general ambiance",
"confidence": confidence,
"lighting_info": lighting_info or {"time_of_day": "unknown", "confidence": 0.0},
"clip_analysis": clip_analysis_safe,
"enable_landmark": enable_landmark,
"image_width": width,
"image_height": height,
"prominent_objects_detail": "the overall visual context"
}
if hasattr(self.llm_enhancer, 'enhance_description'):
try:
enhanced = self.llm_enhancer.enhance_description(scene_data_llm)
if enhanced and len(enhanced.strip()) >= 20:
return enhanced
except Exception as e:
self.logger.error(f"Error in enhance_description: {e}")
if hasattr(self.llm_enhancer, 'handle_no_detection'):
try:
return self.llm_enhancer.handle_no_detection(clip_analysis_safe)
except Exception as e:
self.logger.error(f"Error in handle_no_detection: {e}")
except Exception as e:
self.logger.error(f"Error preparing data for LLM enhancement: {e}")
traceback.print_exc()
return desc
def _extract_possible_activities(self, detected_objects, landmark_results):
"""提取可能的活動。"""
possible_activities = ["Sightseeing"]
# 檢查是否有主要地標活動從 CLIP 分析結果中獲取
primary_landmark_activities = landmark_results.get("primary_landmark_activities", [])
if primary_landmark_activities:
self.logger.info(f"Using {len(primary_landmark_activities)} landmark-specific activities")
possible_activities = primary_landmark_activities
else:
# 從檢測到的地標中提取特定活動
landmark_specific_activities = self.landmark_processing_manager.extract_landmark_specific_activities(detected_objects)
if landmark_specific_activities:
possible_activities = list(set(landmark_specific_activities)) # 去重
self.logger.info(f"Extracted {len(possible_activities)} activities from landmark data")
else:
# 回退到通用活動推斷
if self.descriptor and hasattr(self.descriptor, '_infer_possible_activities'):
try:
possible_activities = self.descriptor._infer_possible_activities(
"tourist_landmark",
detected_objects,
enable_landmark=True,
scene_scores={"tourist_landmark": 0.8}
)
except Exception as e:
self.logger.error(f"Error inferring possible activities: {e}")
return possible_activities
def _enhance_final_description(self, base_description, scene_type, scene_confidence, detected_objects,
functional_zones, activities, safety_concerns, lighting_info,
clip_analysis_results, enable_landmark, image_dims, landmark_info):
"""增強最終描述。"""
if not self.use_llm or not self.llm_enhancer:
return base_description
try:
obj_list_for_llm = ", ".join(sorted(list(set(
obj["class_name"] for obj in detected_objects
if obj.get("confidence", 0) > 0.4 and not obj.get("is_landmark")
))))
if not obj_list_for_llm and enable_landmark and landmark_info.get("primary_landmark"):
obj_list_for_llm = landmark_info["primary_landmark"].get("class_name", "a prominent feature")
elif not obj_list_for_llm:
obj_list_for_llm = "various visual elements"
# 生成物體統計信息
object_statistics = {}
for obj in detected_objects:
class_name = obj.get("class_name", "unknown")
if class_name not in object_statistics:
object_statistics[class_name] = {
"count": 0,
"avg_confidence": 0.0,
"max_confidence": 0.0,
"instances": []
}
stats = object_statistics[class_name]
stats["count"] += 1
stats["instances"].append(obj)
stats["max_confidence"] = max(stats["max_confidence"], obj.get("confidence", 0.0))
# 計算平均信心度
for class_name, stats in object_statistics.items():
if stats["count"] > 0:
total_conf = sum(inst.get("confidence", 0.0) for inst in stats["instances"])
stats["avg_confidence"] = total_conf / stats["count"]
llm_scene_data = {
"original_description": base_description,
"scene_type": scene_type,
"scene_name": self.scene_types.get(scene_type, {}).get("name", "Unknown Scene"),
"detected_objects": detected_objects,
"object_list": obj_list_for_llm,
"object_statistics": object_statistics,
"confidence": scene_confidence,
"lighting_info": lighting_info,
"functional_zones": functional_zones,
"activities": activities,
"safety_concerns": safety_concerns,
"clip_analysis": clip_analysis_results if isinstance(clip_analysis_results, dict) else None,
"enable_landmark": enable_landmark,
"image_width": image_dims[0] if image_dims else None,
"image_height": image_dims[1] if image_dims else None,
"prominent_objects_detail": ""
}
# 添加顯著物體詳細信息
if self.scene_describer and hasattr(self.scene_describer, 'get_prominent_objects') and hasattr(self.scene_describer, 'format_object_list_for_description'):
try:
prominent_objects = self.scene_describer.get_prominent_objects(
detected_objects, min_prominence_score=0.1, max_categories_to_return=3, max_total_objects=7
)
llm_scene_data["prominent_objects_detail"] = self.scene_describer.format_object_list_for_description(prominent_objects)
except Exception as e:
self.logger.error(f"Error getting prominent objects: {e}")
if enable_landmark and landmark_info.get("primary_landmark"):
llm_scene_data["primary_landmark_info"] = landmark_info["primary_landmark"]
return self.llm_enhancer.enhance_description(llm_scene_data)
except Exception as e:
self.logger.error(f"Error in LLM Enhancement in main flow (analyze method): {e}")
return base_description