#!/usr/bin/env python3 """ ECMWF Real Wind Particle Visualization with Dual Layers Downloads current ECMWF 10m and 100m wind data and visualizes with particles """ import gradio as gr import folium from branca.element import Element import json import sys import requests import numpy as np import xarray as xr import tempfile import os from datetime import datetime, timedelta import warnings warnings.filterwarnings('ignore') # Import ECMWF OpenData client try: from ecmwf.opendata import Client as OpenDataClient OPENDATA_AVAILABLE = True except ImportError: OPENDATA_AVAILABLE = False def log_step(step, message): """Log each step with clear formatting""" print(f"šŸ”„ STEP {step}: {message}") sys.stdout.flush() class ECMWFWindDataProcessor: """Process real ECMWF wind data for particle visualization""" def __init__(self): self.temp_dir = tempfile.mkdtemp() self.client = None if OPENDATA_AVAILABLE: try: self.client = OpenDataClient() except: self.client = None # AWS S3 direct access URLs for ECMWF open data self.aws_base_url = "https://ecmwf-forecasts.s3.eu-central-1.amazonaws.com" def get_latest_forecast_info(self): """Get the latest available forecast run information""" try: # ECMWF runs at 00, 06, 12, 18 UTC now = datetime.utcnow() # Find the most recent model run (data available 7-9 hours after run time) for hours_back in range(4, 24, 6): # Check recent runs test_time = now - timedelta(hours=hours_back) # Round to nearest 6-hour cycle run_hour = (test_time.hour // 6) * 6 run_time = test_time.replace(hour=run_hour, minute=0, second=0, microsecond=0) date_str = run_time.strftime("%Y%m%d") time_str = f"{run_hour:02d}" return date_str, time_str, run_time # Fallback return now.strftime("%Y%m%d"), "12", now except Exception as e: # Emergency fallback now = datetime.utcnow() return now.strftime("%Y%m%d"), "12", now def download_wind_component(self, parameter="10u", step=0, max_retries=3): """Download ECMWF wind component data (10u, 10v, 100u, 100v)""" date_str, time_str, run_time = self.get_latest_forecast_info() # Method 1: Try ecmwf-opendata client (most reliable) if OPENDATA_AVAILABLE and self.client: try: filename = os.path.join(self.temp_dir, f'ecmwf_{parameter}_{step}h_{datetime.now().strftime("%Y%m%d_%H%M%S")}.grib') log_step("DOWNLOAD", f"Downloading {parameter} component via ECMWF client...") self.client.retrieve( type="fc", # forecast param=parameter, # 10u, 10v, 100u, 100v step=step, # forecast hour target=filename ) if os.path.exists(filename) and os.path.getsize(filename) > 1000: log_step("SUCCESS", f"Downloaded {parameter} component ({os.path.getsize(filename)} bytes)") return filename, f"āœ… ECMWF {parameter} data downloaded successfully!\\nRun: {date_str} {time_str}z, Step: +{step}h" except Exception as e: log_step("ERROR", f"Client method failed: {str(e)}") return None, f"āŒ Unable to download ECMWF {parameter} data" def extract_wind_data_from_grib(self, filename, parameter): """Extract wind data from GRIB file and return as array""" try: log_step("EXTRACT", f"Processing GRIB file for {parameter}...") # Open the GRIB file with xarray try: ds = xr.open_dataset(filename, engine='cfgrib', backend_kwargs={'indexpath': ''}) except: ds = xr.open_dataset(filename, engine='cfgrib') # Find the right variable data_vars = list(ds.data_vars.keys()) if not data_vars: return None, None, None, "No data variables found in file" data_var = data_vars[0] data = ds[data_var] # Handle coordinates if 'latitude' in ds.coords: lats = ds.latitude.values lons = ds.longitude.values elif 'lat' in ds.coords: lats = ds.lat.values lons = ds.lon.values else: return None, None, None, "Could not find latitude/longitude coordinates" # Get the data values (select first time step if multiple) if 'time' in data.dims and len(data.time) > 1: values = data.isel(time=0).values elif 'valid_time' in data.dims: values = data.isel(valid_time=0).values else: values = data.values # Handle 3D data (select first level if needed) if values.ndim > 2: values = values[0] log_step("SUCCESS", f"Extracted {parameter}: {values.shape} grid, lat range: {lats.min():.1f} to {lats.max():.1f}") ds.close() return lats, lons, values, "Success" except Exception as e: return None, None, None, f"Error extracting data: {str(e)}" def convert_to_wind_json(self, u_lats, u_lons, u_values, v_lats, v_lons, v_values, wind_level="10m"): """Convert ECMWF wind components to leaflet-velocity JSON format""" try: log_step("CONVERT", f"Converting ECMWF data to wind visualization format for {wind_level}...") # Ensure grids match if not (np.array_equal(u_lats, v_lats) and np.array_equal(u_lons, v_lons)): log_step("WARNING", "U and V grids don't match exactly, using U grid as reference") # Use U component grid as reference lats = u_lats lons = u_lons # Ensure lats are in descending order (North to South) for leaflet-velocity if lats[0] < lats[-1]: lats = lats[::-1] u_values = u_values[::-1, :] v_values = v_values[::-1, :] # Downsample to reduce data size for better performance # Skip every 4th point to reduce from 1M+ to ~65k points downsample_factor = 4 lats = lats[::downsample_factor] lons = lons[::downsample_factor] u_values = u_values[::downsample_factor, ::downsample_factor] v_values = v_values[::downsample_factor, ::downsample_factor] # Convert to lists and flatten in row-major order u_data = u_values.flatten().tolist() v_data = v_values.flatten().tolist() # Replace any NaN values with 0 u_data = [0.0 if np.isnan(x) else float(x) for x in u_data] v_data = [0.0 if np.isnan(x) else float(x) for x in v_data] # Create grid info ny, nx = u_values.shape lo1 = float(lons[0]) lo2 = float(lons[-1]) la1 = float(lats[0]) # North (highest) la2 = float(lats[-1]) # South (lowest) dx = float(lons[1] - lons[0]) dy = float(lats[0] - lats[1]) # Should be positive since lats are descending current_time = datetime.utcnow() ref_time = current_time.strftime("%Y-%m-%d %H:00:00") # Create leaflet-velocity compatible JSON structure wind_data = [ { "header": { "discipline": 0, "parameterCategory": 2, "parameterNumber": 2, "parameterName": "UGRD", "parameterNumberName": "eastward_wind", "nx": nx, "ny": ny, "lo1": lo1, "la1": la1, "lo2": lo2, "la2": la2, "dx": dx, "dy": dy, "refTime": ref_time }, "data": u_data }, { "header": { "discipline": 0, "parameterCategory": 2, "parameterNumber": 3, "parameterName": "VGRD", "parameterNumberName": "northward_wind", "nx": nx, "ny": ny, "lo1": lo1, "la1": la1, "lo2": lo2, "la2": la2, "dx": dx, "dy": dy, "refTime": ref_time }, "data": v_data } ] log_step("SUCCESS", f"Converted {wind_level} to wind JSON: {nx}x{ny} grid, {len(u_data)} points each") return wind_data, f"Successfully converted ECMWF {wind_level} data to wind visualization format" except Exception as e: return None, f"Error converting data: {str(e)}" def fetch_real_ecmwf_wind_data(): """Download and process real ECMWF 10m and 100m wind data""" log_step("WIND-1", "šŸŒ Fetching REAL ECMWF wind data (10m and 100m)...") processor = ECMWFWindDataProcessor() try: # Download 10m wind components log_step("WIND-2", "Downloading 10m U wind component...") u10_file, u10_msg = processor.download_wind_component("10u", step=0) log_step("WIND-3", "Downloading 10m V wind component...") v10_file, v10_msg = processor.download_wind_component("10v", step=0) # Download 100m wind components log_step("WIND-4", "Downloading 100m U wind component...") u100_file, u100_msg = processor.download_wind_component("100u", step=0) log_step("WIND-5", "Downloading 100m V wind component...") v100_file, v100_msg = processor.download_wind_component("100v", step=0) # Process 10m data wind_data_10m = None if u10_file and v10_file: log_step("WIND-6", "Processing 10m wind data...") u10_lats, u10_lons, u10_values, u10_status = processor.extract_wind_data_from_grib(u10_file, "10u") v10_lats, v10_lons, v10_values, v10_status = processor.extract_wind_data_from_grib(v10_file, "10v") if u10_values is not None and v10_values is not None: wind_data_10m, convert_msg = processor.convert_to_wind_json( u10_lats, u10_lons, u10_values, v10_lats, v10_lons, v10_values, "10m" ) # Process 100m data wind_data_100m = None if u100_file and v100_file: log_step("WIND-7", "Processing 100m wind data...") u100_lats, u100_lons, u100_values, u100_status = processor.extract_wind_data_from_grib(u100_file, "100u") v100_lats, v100_lons, v100_values, v100_status = processor.extract_wind_data_from_grib(v100_file, "100v") if u100_values is not None and v100_values is not None: wind_data_100m, convert_msg = processor.convert_to_wind_json( u100_lats, u100_lons, u100_values, v100_lats, v100_lons, v100_values, "100m" ) # Return real data if available, otherwise fallback if wind_data_10m is None: wind_data_10m = generate_synthetic_wind_data("10m") if wind_data_100m is None: wind_data_100m = generate_synthetic_wind_data("100m") log_step("WIND-8", f"āœ… SUCCESS: Wind data ready!") return wind_data_10m, wind_data_100m except Exception as e: log_step("WIND-ERROR", f"Failed to fetch real wind data: {str(e)}") log_step("WIND-FALLBACK", "Falling back to synthetic data...") # Fallback to synthetic data return generate_synthetic_wind_data("10m"), generate_synthetic_wind_data("100m") def fetch_ecmwf_forecast_data(forecast_hours=[0, 3, 6, 12, 18, 24]): """Download and process ECMWF forecast data for multiple hours""" log_step("FORECAST-1", f"šŸŒ Fetching ECMWF forecast data for hours: {forecast_hours}") processor = ECMWFWindDataProcessor() forecast_data = {} try: for hour in forecast_hours: log_step("FORECAST-2", f"Downloading forecast data for {hour}h...") # Download wind components for this forecast hour u10_file, u10_msg = processor.download_wind_component("10u", step=hour) v10_file, v10_msg = processor.download_wind_component("10v", step=hour) u100_file, u100_msg = processor.download_wind_component("100u", step=hour) v100_file, v100_msg = processor.download_wind_component("100v", step=hour) # Process 10m forecast data wind_data_10m = None if u10_file and v10_file: log_step("FORECAST-3", f"Processing {hour}h 10m forecast data...") u10_lats, u10_lons, u10_values, u10_status = processor.extract_wind_data_from_grib(u10_file, "10u") v10_lats, v10_lons, v10_values, v10_status = processor.extract_wind_data_from_grib(v10_file, "10v") if u10_values is not None and v10_values is not None: wind_data_10m, convert_msg = processor.convert_to_wind_json( u10_lats, u10_lons, u10_values, v10_lats, v10_lons, v10_values, "10m" ) # Process 100m forecast data wind_data_100m = None if u100_file and v100_file: log_step("FORECAST-4", f"Processing {hour}h 100m forecast data...") u100_lats, u100_lons, u100_values, u100_status = processor.extract_wind_data_from_grib(u100_file, "100u") v100_lats, v100_lons, v100_values, v100_status = processor.extract_wind_data_from_grib(v100_file, "100v") if u100_values is not None and v100_values is not None: wind_data_100m, convert_msg = processor.convert_to_wind_json( u100_lats, u100_lons, u100_values, v100_lats, v100_lons, v100_values, "100m" ) # Use fallback if no real data available if wind_data_10m is None: wind_data_10m = generate_synthetic_wind_data("10m") if wind_data_100m is None: wind_data_100m = generate_synthetic_wind_data("100m") # Store forecast data for this hour forecast_data[hour] = { "10m": wind_data_10m, "100m": wind_data_100m, "timestamp": hour } log_step("FORECAST-5", f"āœ… {hour}h forecast data ready!") log_step("FORECAST-6", f"āœ… SUCCESS: All forecast data ready for hours {list(forecast_data.keys())}") return forecast_data except Exception as e: log_step("FORECAST-ERROR", f"Failed to fetch forecast data: {str(e)}") log_step("FORECAST-FALLBACK", "Falling back to synthetic forecast data...") # Fallback: create synthetic forecast data for all hours fallback_data = {} for hour in forecast_hours: fallback_data[hour] = { "10m": generate_synthetic_wind_data("10m"), "100m": generate_synthetic_wind_data("100m"), "timestamp": hour } return fallback_data def generate_synthetic_wind_data(wind_level="10m"): """Generate synthetic wind data as fallback""" log_step("GEN-1", f"Generating synthetic {wind_level} wind data...") # Basic global grid nx, ny = 72, 36 lon_min, lon_max = -180, 175 lat_min, lat_max = -85, 85 lons = np.linspace(lon_min, lon_max, nx) lats = np.linspace(lat_max, lat_min, ny) u_data = [] v_data = [] # Adjust wind strength based on level strength_multiplier = 1.5 if wind_level == "100m" else 1.0 for j, lat in enumerate(lats): for i, lon in enumerate(lons): # Simple wind pattern with different strength for different levels u = (10 * np.sin(np.radians(lon/2)) + np.random.normal(0, 3)) * strength_multiplier v = (5 * np.cos(np.radians(lat)) + np.random.normal(0, 2)) * strength_multiplier u_data.append(round(u, 2)) v_data.append(round(v, 2)) current_time = datetime.utcnow() ref_time = current_time.strftime("%Y-%m-%d %H:00:00") wind_data = [ { "header": { "discipline": 0, "parameterCategory": 2, "parameterNumber": 2, "parameterName": "UGRD", "parameterNumberName": "eastward_wind", "nx": nx, "ny": ny, "lo1": lon_min, "la1": lat_max, "lo2": lon_max, "la2": lat_min, "dx": 5.0, "dy": 5.0, "refTime": ref_time }, "data": u_data }, { "header": { "discipline": 0, "parameterCategory": 2, "parameterNumber": 3, "parameterName": "VGRD", "parameterNumberName": "northward_wind", "nx": nx, "ny": ny, "lo1": lon_min, "la1": lat_max, "lo2": lon_max, "la2": lat_min, "dx": 5.0, "dy": 5.0, "refTime": ref_time }, "data": v_data } ] log_step("GEN-2", f"Generated synthetic {wind_level} wind data: {len(u_data)} points") return wind_data def create_wind_map(region="global", forecast_mode=False): """Create Leaflet-Velocity wind map with real ECMWF data""" # Set map parameters based on region if region == "global": center = [20, 0] zoom = 2 elif region == "north_america": center = [40, -100] zoom = 3 elif region == "europe": center = [50, 10] zoom = 4 else: center = [20, 0] zoom = 2 # Create map with dark theme as default m = folium.Map( location=center, tiles=None, # We'll add custom tiles without attribution zoom_start=zoom, control_scale=False, # Disable scale control width='100%', height='80vh' # Use viewport height for better mobile scaling ) # Add CartoDB dark matter tiles with minimal attribution (hidden by CSS) folium.TileLayer( tiles='https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png', attr='Ā© CartoDB', # Minimal required attribution name='Dark Matter', overlay=False, control=True, max_zoom=19 ).add_to(m) # Add light theme option (but don't make it active by default) folium.TileLayer( tiles='https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png', attr='Ā© CartoDB', # Minimal required attribution name="Light Matter", overlay=False, control=False, # We'll control this with our custom button max_zoom=19 ).add_to(m) # Fetch ECMWF wind data based on mode if forecast_mode: log_step(5, "Fetching ECMWF forecast data...") forecast_data = fetch_ecmwf_forecast_data() # Start with 0h forecast data wind_data_10m = forecast_data[0]["10m"] wind_data_100m = forecast_data[0]["100m"] log_step(6, f"Forecast data ready: {len(forecast_data)} hours, starting with 0h") else: log_step(5, "Fetching real ECMWF current data...") wind_data_10m, wind_data_100m = fetch_real_ecmwf_wind_data() forecast_data = None log_step(6, f"Current data ready: 10m={len(wind_data_10m)} components, 100m={len(wind_data_100m)} components") # Add Leaflet-Velocity from CDN with comprehensive mobile CSS velocity_css = """ """ m.get_root().html.add_child(Element(velocity_css)) velocity_js = """ """ m.get_root().html.add_child(Element(velocity_js)) # Get map variable name map_id = m.get_name() # Add wind visualization with embedded data js_code = f""" """ m.get_root().html.add_child(Element(js_code)) # Add layer control folium.LayerControl().add_to(m) return m._repr_html_() # Global state to prevent duplicate processing _processing_state = {"is_processing": False, "last_request": None} def update_visualization(region, forecast_mode=False): """Update wind visualization for selected region and mode""" global _processing_state # Create unique request identifier request_id = f"{region}_{forecast_mode}" # Prevent duplicate processing if _processing_state["is_processing"] and _processing_state["last_request"] == request_id: print(f"āš ļø Skipping duplicate request: {request_id}") return "ā³ Processing previous request...", "ā³ Processing..." _processing_state["is_processing"] = True _processing_state["last_request"] = request_id try: mode_str = "forecast" if forecast_mode else "current" print(f"šŸ”„ Creating {mode_str} wind visualization for {region}") map_html = create_wind_map(region, forecast_mode) if forecast_mode: success_msg = f"šŸ“… Forecast visualization loaded for {region.replace('_', ' ').title()} (0h-24h)" else: success_msg = f"āœ… Current wind data loaded for {region.replace('_', ' ').title()}" print(success_msg) return map_html, success_msg except Exception as e: error_msg = f"āŒ Error: {str(e)}" print(error_msg) return f"
Error: {str(e)}
", error_msg finally: # Reset processing state _processing_state["is_processing"] = False # Create Gradio interface with gr.Blocks(title="Wind Particle Visualization") as app: gr.Markdown(""" # šŸŒŖļø ECMWF Wind Visualization with Forecast - v4.0 LIVE **Real ECMWF wind data with forecast timeline** šŸŽ›ļø **TOP-RIGHT CORNER CONTROLS:** - šŸŒŖļø **10m Surface Winds** (blue particles) - 🚁 **100m Altitude Winds** (red particles) - šŸŒ™ **Dark/Light Theme Toggle** (switches map & particle colors) šŸ“… **FORECAST MODE:** Timeline controls at bottom for 0h-24h forecasts ⚔ **FEATURES:** Real ECMWF data • Forecast timeline • Instant particle clearing *Updated: August 15, 2025 - FORECAST VERSION* """) with gr.Row(): with gr.Column(scale=1): region = gr.Radio( choices=["global", "north_america", "europe"], value="global", label="šŸ—ŗļø Region" ) forecast_toggle = gr.Checkbox( label="šŸ“… Forecast Mode", value=False, info="Enable to see 0h-24h forecast timeline" ) update_btn = gr.Button("šŸŒŖļø Update Visualization", variant="primary") status = gr.Textbox( label="Status", lines=3, value="šŸ”„ Loading ECMWF dual wind visualization v3.0...\nšŸŽ›ļø Look for 3 buttons in top-right corner of map!\n⚔ All features: Theme toggle, particle clearing, dual winds" ) gr.Markdown(""" ### ✨ Features: - **Dual wind levels**: 10m (blue) and 100m (red) wind visualization - **Toggle controls**: Independent on/off buttons for each wind level - **Adaptive colors**: Lighter colors on dark theme, darker on light theme - **Dynamic tails**: Smaller tails and slower speed for low winds - **Theme switching**: Colors update instantly when changing themes ### šŸŒŖļø Wind Levels: - **10m winds** (surface): Blue particles, standard speed - **100m winds** (altitude): Red particles, stronger/faster ### šŸŽÆ Controls: - **Top-right corner**: Toggle buttons for wind layers - **Layer control**: Switch between dark/light map themes - **Wind displays**: Bottom corners show current wind data ### šŸŽÆ Troubleshooting: - **Wait 2-3 seconds** for particles to appear - **Use toggle buttons** to enable/disable wind layers - **Check browser console** (F12) for error messages - Particles appear as **flowing lines** with theme-appropriate colors """) with gr.Column(scale=3): wind_map = gr.HTML( label="Wind Particle Animation", value="
šŸ”„ Loading wind particle visualization...
" ) # Event handlers update_btn.click( update_visualization, inputs=[region, forecast_toggle], outputs=[wind_map, status] ) region.change( update_visualization, inputs=[region, forecast_toggle], outputs=[wind_map, status] ) forecast_toggle.change( update_visualization, inputs=[region, forecast_toggle], outputs=[wind_map, status] ) # Auto-load on startup app.load( lambda: update_visualization("global", False), outputs=[wind_map, status] ) if __name__ == "__main__": print("šŸš€ Starting ECMWF Wind Visualization with Forecast v4.0...") print("šŸŒ Real ECMWF wind data with forecast timeline (0h-24h)") print("šŸ“… Forecast mode available with interactive timeline controls") print("šŸŽ›ļø Features: Theme toggle, particle clearing, adaptive colors") print("ā° Build timestamp: August 15, 2025 - FORCE REBUILD") print("========================================") app.launch( server_name="0.0.0.0", server_port=7860, share=False )