Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -11,14 +11,19 @@ import io
|
|
11 |
import zipfile
|
12 |
import uuid
|
13 |
import traceback
|
14 |
-
from huggingface_hub import snapshot_download, hf_hub_download
|
15 |
from flask_cors import CORS
|
16 |
import numpy as np
|
17 |
import trimesh
|
18 |
from scipy.ndimage import gaussian_filter
|
19 |
import cv2
|
20 |
import torch.nn.functional as F
|
21 |
-
|
|
|
|
|
|
|
|
|
|
|
22 |
|
23 |
app = Flask(__name__)
|
24 |
CORS(app) # Enable CORS for all routes
|
@@ -38,6 +43,7 @@ os.makedirs(CACHE_DIR, exist_ok=True)
|
|
38 |
os.environ['HF_HOME'] = CACHE_DIR
|
39 |
os.environ['TRANSFORMERS_CACHE'] = os.path.join(CACHE_DIR, 'transformers')
|
40 |
os.environ['HF_DATASETS_CACHE'] = os.path.join(CACHE_DIR, 'datasets')
|
|
|
41 |
|
42 |
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
|
43 |
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024 # 16MB max
|
@@ -52,6 +58,9 @@ openlrm_model = None
|
|
52 |
model_loaded = False
|
53 |
model_loading = False
|
54 |
|
|
|
|
|
|
|
55 |
# Constants for processing
|
56 |
TIMEOUT_SECONDS = 240 # 4 minutes max for processing
|
57 |
MAX_DIMENSION = 512 # Max image dimension to process
|
@@ -90,6 +99,12 @@ def process_with_timeout(function, args, timeout):
|
|
90 |
|
91 |
return result[0], None
|
92 |
|
|
|
|
|
|
|
|
|
|
|
|
|
93 |
def allowed_file(filename):
|
94 |
return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
|
95 |
|
@@ -135,21 +150,49 @@ def preprocess_image(image_path):
|
|
135 |
|
136 |
return img
|
137 |
|
138 |
-
#
|
139 |
def remove_background(image):
|
|
|
140 |
try:
|
141 |
import rembg
|
142 |
return rembg.remove(image)
|
143 |
except ImportError:
|
144 |
print("Rembg not available, skipping background removal")
|
145 |
-
# Create a
|
|
|
|
|
|
|
146 |
return image
|
147 |
|
148 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
149 |
def load_openlrm_model():
|
150 |
global openlrm_model, model_loaded, model_loading
|
151 |
|
152 |
-
if model_loaded:
|
153 |
return openlrm_model
|
154 |
|
155 |
if model_loading:
|
@@ -160,103 +203,69 @@ def load_openlrm_model():
|
|
160 |
|
161 |
try:
|
162 |
model_loading = True
|
163 |
-
print("
|
164 |
-
|
165 |
-
# For Hugging Face free tier, use the smaller model
|
166 |
-
model_repo = "zxhezexin/openlrm-mix-small-1.1" # Smallest OpenLRM model that works well
|
167 |
-
model_file = "model.safetensors"
|
168 |
|
169 |
-
#
|
170 |
-
model_path = hf_hub_download(
|
171 |
-
repo_id=model_repo,
|
172 |
-
filename=model_file,
|
173 |
-
cache_dir=CACHE_DIR,
|
174 |
-
resume_download=True
|
175 |
-
)
|
176 |
-
|
177 |
-
# Download config file
|
178 |
-
config_path = hf_hub_download(
|
179 |
-
repo_id=model_repo,
|
180 |
-
filename="config.json",
|
181 |
-
cache_dir=CACHE_DIR,
|
182 |
-
resume_download=True
|
183 |
-
)
|
184 |
-
|
185 |
-
# Load OpenLRM for inference
|
186 |
-
# Simplified loading for memory efficiency
|
187 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
188 |
|
189 |
-
#
|
190 |
-
|
191 |
-
|
192 |
-
|
193 |
-
config = AutoConfig.from_pretrained(config_path)
|
194 |
-
|
195 |
-
# Initialize a lightweight model class
|
196 |
-
class OpenLRMWrapper:
|
197 |
-
def __init__(self, model_path, config, device):
|
198 |
-
self.model_path = model_path
|
199 |
-
self.config = config
|
200 |
self.device = device
|
201 |
-
|
202 |
-
self.model = None
|
203 |
|
204 |
def __call__(self, image):
|
205 |
-
|
206 |
-
|
207 |
-
|
208 |
-
from transformers import AutoModelForSeq2SeqLM
|
209 |
-
|
210 |
-
# Load model with minimal memory footprint for Hugging Face free tier
|
211 |
-
self.model = AutoModelForSeq2SeqLM.from_pretrained(
|
212 |
-
self.model_path,
|
213 |
-
config=self.config,
|
214 |
-
torch_dtype=torch.float16 if self.device == "cuda" else torch.float32,
|
215 |
-
device_map=self.device
|
216 |
-
)
|
217 |
|
218 |
-
#
|
219 |
-
|
220 |
-
|
221 |
-
|
222 |
-
|
223 |
-
|
224 |
-
|
225 |
-
|
226 |
-
|
227 |
-
|
228 |
-
|
229 |
-
|
230 |
-
transforms.ToTensor(),
|
231 |
-
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
|
232 |
-
])
|
233 |
-
tensor = transform(image).unsqueeze(0).to(self.device)
|
234 |
-
return tensor
|
235 |
|
236 |
-
|
237 |
-
|
238 |
-
|
239 |
-
|
240 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
241 |
|
242 |
-
# Create
|
243 |
-
openlrm_model =
|
244 |
|
245 |
model_loaded = True
|
246 |
-
print(f"
|
247 |
return openlrm_model
|
248 |
|
249 |
except Exception as e:
|
250 |
-
print(f"Error
|
251 |
print(traceback.format_exc())
|
252 |
-
|
253 |
-
# Fallback to depth estimation model if OpenLRM fails
|
254 |
-
load_depth_model()
|
255 |
return None
|
256 |
finally:
|
257 |
model_loading = False
|
258 |
|
259 |
-
#
|
260 |
def load_depth_model():
|
261 |
global depth_model, feature_extractor, model_loaded, model_loading
|
262 |
|
@@ -264,18 +273,37 @@ def load_depth_model():
|
|
264 |
return depth_model, feature_extractor
|
265 |
|
266 |
try:
|
267 |
-
print("Loading depth estimation model
|
268 |
|
269 |
-
#
|
270 |
-
model_name =
|
271 |
|
272 |
-
|
|
|
|
|
|
|
|
|
273 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
274 |
|
275 |
-
#
|
276 |
-
|
277 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
278 |
|
|
|
279 |
if device == "cuda":
|
280 |
depth_model = depth_model.to(device)
|
281 |
|
@@ -285,25 +313,73 @@ def load_depth_model():
|
|
285 |
except Exception as e:
|
286 |
print(f"Error loading depth model: {str(e)}")
|
287 |
print(traceback.format_exc())
|
288 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
289 |
|
290 |
-
# Process image to create 3D model using
|
291 |
def process_openlrm(image, job_id, detail_level='medium', output_format='obj'):
|
292 |
try:
|
293 |
-
# Load OpenLRM model
|
294 |
model = load_openlrm_model()
|
295 |
if model is None:
|
296 |
# Fallback to depth-based approach
|
297 |
return process_depth_based(image, job_id, detail_level, output_format)
|
298 |
|
299 |
# Preprocess image - remove background for better results
|
|
|
300 |
image_rgba = remove_background(image)
|
301 |
|
302 |
# Update progress
|
303 |
-
processing_jobs[job_id]['progress'] =
|
304 |
|
305 |
-
# Process with
|
306 |
-
# This is where the magic happens - OpenLRM will create a full 3D model
|
307 |
result = model(image_rgba)
|
308 |
|
309 |
# Update progress
|
@@ -326,61 +402,76 @@ def process_openlrm(image, job_id, detail_level='medium', output_format='obj'):
|
|
326 |
|
327 |
# Convert OpenLRM result to trimesh
|
328 |
def convert_to_trimesh(result, image):
|
329 |
-
#
|
330 |
-
|
331 |
-
|
332 |
-
|
333 |
-
#
|
334 |
-
|
335 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
336 |
|
337 |
-
# Create mesh with
|
338 |
mesh = trimesh.Trimesh(vertices=vertices, faces=faces)
|
339 |
|
340 |
# Add texture from the original image
|
341 |
if hasattr(image, 'convert'):
|
342 |
-
|
343 |
-
|
344 |
-
|
345 |
-
|
|
|
|
|
|
|
346 |
|
347 |
return mesh
|
348 |
|
349 |
# Sample helper functions for mesh creation
|
350 |
-
def generate_sample_vertices():
|
351 |
-
# Create a cube-like object for testing
|
352 |
-
x = np.linspace(-1, 1, 10)
|
353 |
-
y = np.linspace(-1, 1, 10)
|
354 |
-
z = np.linspace(-1, 1, 10)
|
355 |
-
x_grid, y_grid, z_grid = np.meshgrid(x, y, z)
|
356 |
-
vertices = np.vstack([x_grid.flatten(), y_grid.flatten(), z_grid.flatten()]).T
|
357 |
-
return vertices
|
358 |
-
|
359 |
-
def generate_sample_faces():
|
360 |
-
# Create simple faces connecting vertices
|
361 |
-
faces = []
|
362 |
-
n = 10 # Grid size from generate_sample_vertices
|
363 |
-
for i in range(n-1):
|
364 |
-
for j in range(n-1):
|
365 |
-
for k in range(n-1):
|
366 |
-
idx = i*n*n + j*n + k
|
367 |
-
faces.append([idx, idx+1, idx+n])
|
368 |
-
faces.append([idx+1, idx+n+1, idx+n])
|
369 |
-
return np.array(faces)
|
370 |
-
|
371 |
def sample_texture_from_image(image, vertices):
|
|
|
372 |
# Sample colors from image based on vertex positions
|
373 |
h, w = image.shape[:2]
|
374 |
colors = np.zeros((len(vertices), 4), dtype=np.uint8)
|
375 |
|
376 |
-
#
|
377 |
-
|
|
|
378 |
|
379 |
-
#
|
380 |
-
for i,
|
381 |
-
|
382 |
-
|
383 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
384 |
colors[i] = image[y, x]
|
385 |
else:
|
386 |
colors[i] = [200, 200, 200, 255] # Default color
|
@@ -390,40 +481,58 @@ def sample_texture_from_image(image, vertices):
|
|
390 |
# Process using depth-based approach as fallback
|
391 |
def process_depth_based(image, job_id, detail_level='medium', output_format='obj'):
|
392 |
try:
|
393 |
-
# Load depth model
|
394 |
-
|
395 |
-
if depth_model is None or feature_extractor is None:
|
396 |
-
depth_model, feature_extractor = load_depth_model()
|
397 |
|
398 |
# Update progress
|
399 |
processing_jobs[job_id]['progress'] = 30
|
400 |
|
401 |
-
#
|
402 |
-
|
403 |
-
#
|
404 |
-
|
405 |
-
|
406 |
-
|
407 |
-
|
408 |
-
|
409 |
-
outputs = depth_model(**inputs)
|
410 |
-
predicted_depth = outputs.predicted_depth
|
411 |
-
|
412 |
-
# Normalize and resize depth to original image size
|
413 |
-
depth_map = F.interpolate(
|
414 |
-
predicted_depth.unsqueeze(1),
|
415 |
-
size=(image.height, image.width),
|
416 |
-
mode="bicubic",
|
417 |
-
align_corners=False,
|
418 |
-
).squeeze().cpu().numpy()
|
419 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
420 |
# Update progress
|
421 |
processing_jobs[job_id]['progress'] = 60
|
422 |
|
423 |
-
#
|
424 |
-
|
425 |
-
|
426 |
-
|
|
|
|
|
|
|
427 |
|
428 |
# Create mesh from depth map
|
429 |
mesh = depth_to_mesh(depth_normalized, image,
|
@@ -434,12 +543,24 @@ def process_depth_based(image, job_id, detail_level='medium', output_format='obj
|
|
434 |
# Update progress
|
435 |
processing_jobs[job_id]['progress'] = 80
|
436 |
|
|
|
|
|
|
|
437 |
return mesh
|
438 |
|
439 |
except Exception as e:
|
440 |
print(f"Error in depth-based processing: {str(e)}")
|
441 |
print(traceback.format_exc())
|
442 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
443 |
|
444 |
# Enhanced depth map processing
|
445 |
def enhance_depth_map(depth_map, detail_level='medium'):
|
@@ -548,7 +669,7 @@ def depth_to_mesh(depth_map, image, resolution=100, detail_level='medium'):
|
|
548 |
mesh = trimesh.Trimesh(vertices=vertices, faces=faces)
|
549 |
|
550 |
# Apply texturing if image is provided
|
551 |
-
if image:
|
552 |
# Convert to numpy array if needed
|
553 |
if isinstance(image, Image.Image):
|
554 |
img_array = np.array(image)
|
@@ -556,153 +677,149 @@ def depth_to_mesh(depth_map, image, resolution=100, detail_level='medium'):
|
|
556 |
img_array = image
|
557 |
|
558 |
# Create vertex colors
|
559 |
-
if
|
560 |
# Create vertex colors by sampling the image
|
561 |
vertex_colors = np.zeros((vertices.shape[0], 4), dtype=np.uint8)
|
562 |
|
563 |
for i in range(resolution):
|
564 |
for j in range(resolution):
|
565 |
# Calculate image coordinates
|
566 |
-
img_x = j * (img_array.shape[1] - 1) / (resolution - 1)
|
567 |
-
img_y = i * (img_array.shape[0] - 1) / (resolution - 1)
|
568 |
-
|
569 |
-
# Bilinear interpolation
|
570 |
-
x0, y0 = int(img_x), int(img_y)
|
571 |
-
x1, y1 = min(x0 + 1, img_array.shape[1] - 1), min(y0 + 1, img_array.shape[0] - 1)
|
572 |
-
|
573 |
-
# Interpolation weights
|
574 |
-
wx = img_x - x0
|
575 |
-
wy = img_y - y0
|
576 |
|
577 |
vertex_idx = i * resolution + j
|
578 |
|
579 |
if len(img_array.shape) == 3 and img_array.shape[2] == 3: # RGB
|
580 |
-
|
581 |
-
|
582 |
-
(1-wx)*wy*img_array[y1, x0, 0] + wx*wy*img_array[y1, x1, 0])
|
583 |
-
g = int((1-wx)*(1-wy)*img_array[y0, x0, 1] + wx*(1-wy)*img_array[y0, x1, 1] +
|
584 |
-
(1-wx)*wy*img_array[y1, x0, 1] + wx*wy*img_array[y1, x1, 1])
|
585 |
-
b = int((1-wx)*(1-wy)*img_array[y0, x0, 2] + wx*(1-wy)*img_array[y0, x1, 2] +
|
586 |
-
(1-wx)*wy*img_array[y1, x0, 2] + wx*wy*img_array[y1, x1, 2])
|
587 |
-
|
588 |
-
vertex_colors[vertex_idx, :3] = [r, g, b]
|
589 |
-
vertex_colors[vertex_idx, 3] = 255 # Alpha
|
590 |
elif len(img_array.shape) == 3 and img_array.shape[2] == 4: # RGBA
|
591 |
-
|
592 |
-
vertex_colors[vertex_idx, c] = int((1-wx)*(1-wy)*img_array[y0, x0, c] +
|
593 |
-
wx*(1-wy)*img_array[y0, x1, c] +
|
594 |
-
(1-wx)*wy*img_array[y1, x0, c] +
|
595 |
-
wx*wy*img_array[y1, x1, c])
|
596 |
else:
|
597 |
# Handle grayscale
|
598 |
-
gray =
|
599 |
-
|
600 |
-
|
601 |
-
|
|
|
|
|
|
|
602 |
|
603 |
mesh.visual.vertex_colors = vertex_colors
|
604 |
|
605 |
# Apply smoothing to get rid of staircase artifacts
|
606 |
if detail_level != 'high':
|
607 |
-
|
|
|
|
|
|
|
|
|
608 |
|
609 |
# Fix normals for better rendering
|
610 |
-
|
611 |
-
|
612 |
-
|
613 |
-
|
614 |
-
|
615 |
-
|
616 |
-
|
617 |
-
|
618 |
-
|
619 |
-
|
620 |
-
|
621 |
-
|
622 |
-
|
623 |
-
|
624 |
-
|
625 |
-
|
626 |
-
|
627 |
-
|
|
|
|
|
|
|
|
|
|
|
628 |
|
629 |
return mesh
|
630 |
|
631 |
-
# Create a watertight model by adding side panels
|
632 |
def create_watertight_model(mesh):
|
633 |
-
|
634 |
-
|
635 |
-
|
636 |
-
|
637 |
-
|
638 |
-
# If no boundary edges, return the original mesh
|
639 |
-
if len(boundary_edges) == 0:
|
640 |
-
return mesh
|
641 |
-
|
642 |
-
# Create side panels along boundary edges
|
643 |
-
new_faces = []
|
644 |
-
|
645 |
-
# Sort boundary edges to form loops
|
646 |
-
edge_loops = []
|
647 |
-
current_loop = [boundary_edges[0][0], boundary_edges[0][1]]
|
648 |
-
boundary_edges = boundary_edges[1:]
|
649 |
-
|
650 |
-
# Try to create continuous edge loops
|
651 |
-
while len(boundary_edges) > 0:
|
652 |
-
found = False
|
653 |
-
for i, edge in enumerate(boundary_edges):
|
654 |
-
if edge[0] == current_loop[-1]:
|
655 |
-
current_loop.append(edge[1])
|
656 |
-
boundary_edges = np.delete(boundary_edges, i, axis=0)
|
657 |
-
found = True
|
658 |
-
break
|
659 |
-
elif edge[1] == current_loop[-1]:
|
660 |
-
current_loop.append(edge[0])
|
661 |
-
boundary_edges = np.delete(boundary_edges, i, axis=0)
|
662 |
-
found = True
|
663 |
-
break
|
664 |
|
665 |
-
|
666 |
-
|
667 |
-
|
668 |
-
|
669 |
-
|
670 |
-
|
671 |
-
|
672 |
-
|
673 |
-
|
674 |
-
|
675 |
-
|
676 |
-
|
677 |
-
|
678 |
-
|
679 |
-
|
680 |
-
|
|
|
681 |
|
682 |
-
# Create
|
683 |
-
|
684 |
-
|
685 |
-
|
686 |
-
# Add new faces to the mesh
|
687 |
-
if len(new_faces) > 0:
|
688 |
-
new_faces = np.array(new_faces)
|
689 |
-
combined_faces = np.vstack([mesh.faces, new_faces])
|
690 |
-
watertight_mesh = trimesh.Trimesh(vertices=mesh.vertices, faces=combined_faces)
|
691 |
|
692 |
-
#
|
693 |
-
|
694 |
-
|
695 |
|
696 |
-
|
697 |
-
|
698 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
699 |
|
700 |
@app.route('/health', methods=['GET'])
|
701 |
def health_check():
|
702 |
return jsonify({
|
703 |
"status": "healthy",
|
704 |
-
"model": "Enhanced 3D Model Generator
|
705 |
-
"device": "cuda" if torch.cuda.is_available() else "cpu"
|
|
|
706 |
}), 200
|
707 |
|
708 |
@app.route('/progress/<job_id>', methods=['GET'])
|
@@ -736,7 +853,7 @@ def progress(job_id):
|
|
736 |
break
|
737 |
check_count = 0
|
738 |
|
739 |
-
|
740 |
if job['status'] == 'completed':
|
741 |
yield f"data: {json.dumps({'status': 'completed', 'progress': 100, 'result_url': job['result_url'], 'preview_url': job['preview_url']})}\n\n"
|
742 |
else:
|
@@ -763,6 +880,12 @@ def convert_image_to_3d():
|
|
763 |
output_format = request.form.get('output_format', 'obj').lower()
|
764 |
detail_level = request.form.get('detail_level', 'medium').lower() # Parameter for detail level
|
765 |
model_type = request.form.get('model_type', 'openlrm').lower() # 'openlrm' or 'depth'
|
|
|
|
|
|
|
|
|
|
|
|
|
766 |
except ValueError:
|
767 |
return jsonify({"error": "Invalid parameter values"}), 400
|
768 |
|
@@ -869,9 +992,7 @@ def convert_image_to_3d():
|
|
869 |
os.remove(filepath)
|
870 |
|
871 |
# Force garbage collection to free memory
|
872 |
-
|
873 |
-
if torch.cuda.is_available():
|
874 |
-
torch.cuda.empty_cache()
|
875 |
|
876 |
except Exception as e:
|
877 |
# Handle errors
|
@@ -1014,7 +1135,7 @@ def model_info(job_id):
|
|
1014 |
@app.route('/', methods=['GET'])
|
1015 |
def index():
|
1016 |
return jsonify({
|
1017 |
-
"message": "Enhanced 3D Model Generator
|
1018 |
"endpoints": [
|
1019 |
"/convert",
|
1020 |
"/progress/<job_id>",
|
@@ -1028,164 +1149,49 @@ def index():
|
|
1028 |
"detail_level": "low, medium, or high - controls the level of detail in the final model",
|
1029 |
"model_type": "openlrm (default, full 3D) or depth (faster but simpler)"
|
1030 |
},
|
1031 |
-
"description": "This API creates high-quality 3D models from 2D images with full 3D structure and texturing"
|
|
|
1032 |
}), 200
|
1033 |
|
1034 |
-
#
|
1035 |
-
|
1036 |
-
|
1037 |
-
|
1038 |
-
if 'image' not in request.files:
|
1039 |
-
return jsonify({"error": "No image provided"}), 400
|
1040 |
-
|
1041 |
-
file = request.files['image']
|
1042 |
-
if file.filename == '':
|
1043 |
-
return jsonify({"error": "No image selected"}), 400
|
1044 |
-
|
1045 |
-
if not allowed_file(file.filename):
|
1046 |
-
return jsonify({"error": f"File type not allowed. Supported types: {', '.join(ALLOWED_EXTENSIONS)}"}), 400
|
1047 |
-
|
1048 |
-
# Create a job ID
|
1049 |
-
job_id = str(uuid.uuid4())
|
1050 |
-
output_dir = os.path.join(RESULTS_FOLDER, job_id)
|
1051 |
-
os.makedirs(output_dir, exist_ok=True)
|
1052 |
-
|
1053 |
-
# Save the uploaded file
|
1054 |
-
filename = secure_filename(file.filename)
|
1055 |
-
filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{filename}")
|
1056 |
-
file.save(filepath)
|
1057 |
|
1058 |
-
#
|
1059 |
-
|
1060 |
-
|
1061 |
-
|
1062 |
-
|
1063 |
-
|
1064 |
-
|
1065 |
-
|
1066 |
-
|
1067 |
|
1068 |
-
#
|
1069 |
-
|
1070 |
-
|
1071 |
-
|
1072 |
-
|
1073 |
try:
|
1074 |
-
|
1075 |
-
|
1076 |
-
processing_jobs[job_id]['progress'] = 10
|
1077 |
-
|
1078 |
-
# Dictionary to store results
|
1079 |
-
result_urls = {}
|
1080 |
-
|
1081 |
-
# Process with both models
|
1082 |
-
try:
|
1083 |
-
# First try with OpenLRM for full 3D
|
1084 |
-
processing_jobs[job_id]['progress'] = 30
|
1085 |
-
openlrm_mesh = process_openlrm(image, job_id, 'medium', 'glb')
|
1086 |
-
|
1087 |
-
# Export OpenLRM result
|
1088 |
-
openlrm_path = os.path.join(output_dir, "model_openlrm.glb")
|
1089 |
-
openlrm_mesh.export(openlrm_path, file_type='glb')
|
1090 |
-
result_urls['openlrm'] = f"/compare-download/{job_id}/openlrm"
|
1091 |
-
|
1092 |
-
processing_jobs[job_id]['progress'] = 60
|
1093 |
-
|
1094 |
-
# Then process with depth-based approach
|
1095 |
-
depth_mesh = process_depth_based(image, job_id, 'medium', 'glb')
|
1096 |
-
|
1097 |
-
# Export depth-based result
|
1098 |
-
depth_path = os.path.join(output_dir, "model_depth.glb")
|
1099 |
-
depth_mesh.export(depth_path, file_type='glb')
|
1100 |
-
result_urls['depth'] = f"/compare-download/{job_id}/depth"
|
1101 |
-
|
1102 |
-
processing_jobs[job_id]['progress'] = 90
|
1103 |
-
|
1104 |
-
except Exception as e:
|
1105 |
-
print(f"Error in comparison processing: {str(e)}")
|
1106 |
-
# If at least one model was successful, continue
|
1107 |
-
if not result_urls:
|
1108 |
-
raise
|
1109 |
-
|
1110 |
-
# Update job status
|
1111 |
-
processing_jobs[job_id]['status'] = 'completed'
|
1112 |
-
processing_jobs[job_id]['progress'] = 100
|
1113 |
-
processing_jobs[job_id]['result_urls'] = result_urls
|
1114 |
-
processing_jobs[job_id]['completed_at'] = time.time()
|
1115 |
-
|
1116 |
-
# Clean up temporary file
|
1117 |
-
if os.path.exists(filepath):
|
1118 |
-
os.remove(filepath)
|
1119 |
-
|
1120 |
-
# Force garbage collection
|
1121 |
-
gc.collect()
|
1122 |
-
if torch.cuda.is_available():
|
1123 |
-
torch.cuda.empty_cache()
|
1124 |
-
|
1125 |
except Exception as e:
|
1126 |
-
|
1127 |
-
|
1128 |
-
|
1129 |
-
|
1130 |
-
# Clean up on error
|
1131 |
-
if os.path.exists(filepath):
|
1132 |
-
os.remove(filepath)
|
1133 |
-
|
1134 |
-
# Start processing thread
|
1135 |
-
processing_thread = threading.Thread(target=process_comparison)
|
1136 |
-
processing_thread.daemon = True
|
1137 |
-
processing_thread.start()
|
1138 |
-
|
1139 |
-
# Return job ID immediately
|
1140 |
-
return jsonify({"job_id": job_id, "check_progress_at": f"/progress/{job_id}"}), 202
|
1141 |
-
|
1142 |
-
@app.route('/compare-download/<job_id>/<model_type>', methods=['GET'])
|
1143 |
-
def download_comparison_model(job_id, model_type):
|
1144 |
-
if job_id not in processing_jobs or processing_jobs[job_id]['status'] != 'completed':
|
1145 |
-
return jsonify({"error": "Model not found or processing not complete"}), 404
|
1146 |
-
|
1147 |
-
if 'comparison' not in processing_jobs[job_id] or not processing_jobs[job_id]['comparison']:
|
1148 |
-
return jsonify({"error": "This is not a comparison job"}), 400
|
1149 |
-
|
1150 |
-
if model_type not in ['openlrm', 'depth']:
|
1151 |
-
return jsonify({"error": "Invalid model type"}), 400
|
1152 |
-
|
1153 |
-
# Get the output directory for this job
|
1154 |
-
output_dir = os.path.join(RESULTS_FOLDER, job_id)
|
1155 |
-
model_path = os.path.join(output_dir, f"model_{model_type}.glb")
|
1156 |
-
|
1157 |
-
if os.path.exists(model_path):
|
1158 |
-
return send_file(model_path, as_attachment=True, download_name=f"model_{model_type}.glb")
|
1159 |
|
1160 |
-
|
1161 |
-
|
1162 |
-
|
1163 |
-
|
1164 |
-
|
1165 |
-
|
1166 |
-
|
1167 |
-
token = request.json.get('token')
|
1168 |
-
if token != 'admin_secure_token': # Replace with proper auth
|
1169 |
-
return jsonify({"error": "Unauthorized"}), 401
|
1170 |
-
|
1171 |
-
# Install dependencies
|
1172 |
-
import subprocess
|
1173 |
-
|
1174 |
-
# Install rembg for background removal
|
1175 |
-
subprocess.check_call([sys.executable, "-m", "pip", "install", "rembg"])
|
1176 |
-
|
1177 |
-
# Try to install torchmcubes with CUDA support
|
1178 |
-
try:
|
1179 |
-
subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "-y", "torchmcubes"])
|
1180 |
-
subprocess.check_call([sys.executable, "-m", "pip", "install", "git+https://github.com/tatsy/torchmcubes.git"])
|
1181 |
-
except:
|
1182 |
-
print("Could not install torchmcubes with CUDA support")
|
1183 |
-
|
1184 |
-
return jsonify({"message": "Dependencies installed successfully"}), 200
|
1185 |
-
except Exception as e:
|
1186 |
-
return jsonify({"error": f"Failed to install dependencies: {str(e)}"}), 500
|
1187 |
|
1188 |
if __name__ == '__main__':
|
|
|
|
|
|
|
1189 |
# Start the cleanup thread
|
1190 |
cleanup_old_jobs()
|
1191 |
|
|
|
11 |
import zipfile
|
12 |
import uuid
|
13 |
import traceback
|
14 |
+
from huggingface_hub import snapshot_download, hf_hub_download, login
|
15 |
from flask_cors import CORS
|
16 |
import numpy as np
|
17 |
import trimesh
|
18 |
from scipy.ndimage import gaussian_filter
|
19 |
import cv2
|
20 |
import torch.nn.functional as F
|
21 |
+
|
22 |
+
# Try to login with token if available
|
23 |
+
HF_TOKEN = os.environ.get("HF_TOKEN", None)
|
24 |
+
if HF_TOKEN:
|
25 |
+
print("Logging in with Hugging Face token")
|
26 |
+
login(token=HF_TOKEN)
|
27 |
|
28 |
app = Flask(__name__)
|
29 |
CORS(app) # Enable CORS for all routes
|
|
|
43 |
os.environ['HF_HOME'] = CACHE_DIR
|
44 |
os.environ['TRANSFORMERS_CACHE'] = os.path.join(CACHE_DIR, 'transformers')
|
45 |
os.environ['HF_DATASETS_CACHE'] = os.path.join(CACHE_DIR, 'datasets')
|
46 |
+
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:128' # Limit CUDA memory splits
|
47 |
|
48 |
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
|
49 |
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024 # 16MB max
|
|
|
58 |
model_loaded = False
|
59 |
model_loading = False
|
60 |
|
61 |
+
# Flag to control whether to use simplified mode (for Hugging Face Spaces)
|
62 |
+
USE_SIMPLIFIED_MODE = os.environ.get('USE_SIMPLIFIED_MODE', 'false').lower() == 'true'
|
63 |
+
|
64 |
# Constants for processing
|
65 |
TIMEOUT_SECONDS = 240 # 4 minutes max for processing
|
66 |
MAX_DIMENSION = 512 # Max image dimension to process
|
|
|
99 |
|
100 |
return result[0], None
|
101 |
|
102 |
+
def optimize_memory():
|
103 |
+
"""Free up memory to avoid OOM errors"""
|
104 |
+
gc.collect()
|
105 |
+
if torch.cuda.is_available():
|
106 |
+
torch.cuda.empty_cache()
|
107 |
+
|
108 |
def allowed_file(filename):
|
109 |
return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
|
110 |
|
|
|
150 |
|
151 |
return img
|
152 |
|
153 |
+
# Try to remove background - simplified version that won't fail if rembg is not available
|
154 |
def remove_background(image):
|
155 |
+
"""Remove background if rembg is available, otherwise return original image"""
|
156 |
try:
|
157 |
import rembg
|
158 |
return rembg.remove(image)
|
159 |
except ImportError:
|
160 |
print("Rembg not available, skipping background removal")
|
161 |
+
# Create a copy of the image with RGBA
|
162 |
+
if isinstance(image, Image.Image):
|
163 |
+
if image.mode != 'RGBA':
|
164 |
+
return image.convert('RGBA')
|
165 |
return image
|
166 |
|
167 |
+
# Function to select available models - checks which models are accessible
|
168 |
+
def select_available_model():
|
169 |
+
"""Try to find an available public model for depth estimation"""
|
170 |
+
public_models = [
|
171 |
+
"facebook/dpt-hybrid-midas", # Public DPT model
|
172 |
+
"Intel/dpt-large", # Intel's DPT model
|
173 |
+
"facebook/dinov2-base", # General vision model
|
174 |
+
]
|
175 |
+
|
176 |
+
# Try each model in turn
|
177 |
+
for model_name in public_models:
|
178 |
+
try:
|
179 |
+
print(f"Testing model availability: {model_name}")
|
180 |
+
# Just try to download the config to check if accessible
|
181 |
+
from transformers import AutoConfig
|
182 |
+
AutoConfig.from_pretrained(model_name, force_download=False)
|
183 |
+
print(f"Model {model_name} is available")
|
184 |
+
return model_name
|
185 |
+
except Exception as e:
|
186 |
+
print(f"Model {model_name} not available: {str(e)}")
|
187 |
+
|
188 |
+
print("No suitable models found. Using manual depth map generation.")
|
189 |
+
return None
|
190 |
+
|
191 |
+
# Updated OpenLRM loading with fallback to simplified model
|
192 |
def load_openlrm_model():
|
193 |
global openlrm_model, model_loaded, model_loading
|
194 |
|
195 |
+
if model_loaded and openlrm_model is not None:
|
196 |
return openlrm_model
|
197 |
|
198 |
if model_loading:
|
|
|
203 |
|
204 |
try:
|
205 |
model_loading = True
|
206 |
+
print("Initializing 3D model generator...")
|
|
|
|
|
|
|
|
|
207 |
|
208 |
+
# Device selection - prefer CUDA if available
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
209 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
210 |
|
211 |
+
# Instead of using OpenLRM which is problematic on Spaces, create a simpler wrapper
|
212 |
+
# This will generate basic 3D structure without requiring complex models
|
213 |
+
class Simple3DWrapper:
|
214 |
+
def __init__(self, device):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
215 |
self.device = device
|
216 |
+
print(f"Initialized simple 3D wrapper on {device}")
|
|
|
217 |
|
218 |
def __call__(self, image):
|
219 |
+
"""Create a 3D mesh representation from an image"""
|
220 |
+
# Generate a depth map without complex models
|
221 |
+
depth_map = create_simple_depth_map(image)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
222 |
|
223 |
+
# Convert depth map to vertices and faces
|
224 |
+
h, w = depth_map.shape
|
225 |
+
vertices = []
|
226 |
+
|
227 |
+
# Create vertices - scale to [-1, 1] range for x and y
|
228 |
+
scale_factor = 2.0
|
229 |
+
for i in range(h):
|
230 |
+
for j in range(w):
|
231 |
+
x = (j / w - 0.5) * scale_factor
|
232 |
+
y = (i / h - 0.5) * scale_factor
|
233 |
+
z = depth_map[i, j] * scale_factor * -1 # Negative to make closer objects "pop out"
|
234 |
+
vertices.append([x, y, z])
|
|
|
|
|
|
|
|
|
|
|
235 |
|
236 |
+
# Create faces - connect neighboring vertices
|
237 |
+
faces = []
|
238 |
+
for i in range(h-1):
|
239 |
+
for j in range(w-1):
|
240 |
+
v0 = i * w + j
|
241 |
+
v1 = i * w + (j + 1)
|
242 |
+
v2 = (i + 1) * w + j
|
243 |
+
v3 = (i + 1) * w + (j + 1)
|
244 |
+
|
245 |
+
# Two triangles per grid cell
|
246 |
+
faces.append([v0, v1, v3])
|
247 |
+
faces.append([v0, v3, v2])
|
248 |
+
|
249 |
+
return {
|
250 |
+
"vertices": np.array(vertices),
|
251 |
+
"faces": np.array(faces)
|
252 |
+
}
|
253 |
|
254 |
+
# Create the 3D model wrapper
|
255 |
+
openlrm_model = Simple3DWrapper(device)
|
256 |
|
257 |
model_loaded = True
|
258 |
+
print(f"Simple 3D model generator initialized on {device}")
|
259 |
return openlrm_model
|
260 |
|
261 |
except Exception as e:
|
262 |
+
print(f"Error initializing 3D model: {str(e)}")
|
263 |
print(traceback.format_exc())
|
|
|
|
|
|
|
264 |
return None
|
265 |
finally:
|
266 |
model_loading = False
|
267 |
|
268 |
+
# Updated depth model loading with public model support
|
269 |
def load_depth_model():
|
270 |
global depth_model, feature_extractor, model_loaded, model_loading
|
271 |
|
|
|
273 |
return depth_model, feature_extractor
|
274 |
|
275 |
try:
|
276 |
+
print("Loading depth estimation model...")
|
277 |
|
278 |
+
# Select an available public model
|
279 |
+
model_name = select_available_model()
|
280 |
|
281 |
+
if model_name is None:
|
282 |
+
print("No suitable depth model found. Using manual depth map generation.")
|
283 |
+
return None, None
|
284 |
+
|
285 |
+
# Device selection
|
286 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
287 |
|
288 |
+
# Import appropriate model class for the selected model
|
289 |
+
if "dpt" in model_name.lower():
|
290 |
+
from transformers import DPTForDepthEstimation, DPTFeatureExtractor
|
291 |
+
print(f"Loading DPT model: {model_name}")
|
292 |
+
feature_extractor = DPTFeatureExtractor.from_pretrained(model_name, token=HF_TOKEN)
|
293 |
+
depth_model = DPTForDepthEstimation.from_pretrained(model_name, token=HF_TOKEN)
|
294 |
+
elif "dinov2" in model_name.lower():
|
295 |
+
from transformers import AutoFeatureExtractor, AutoModel
|
296 |
+
print(f"Loading DINOv2 model: {model_name}")
|
297 |
+
feature_extractor = AutoFeatureExtractor.from_pretrained(model_name, token=HF_TOKEN)
|
298 |
+
depth_model = AutoModel.from_pretrained(model_name, token=HF_TOKEN)
|
299 |
+
else:
|
300 |
+
# Generic loading
|
301 |
+
from transformers import AutoFeatureExtractor, AutoModelForDepthEstimation
|
302 |
+
print(f"Loading Auto depth model: {model_name}")
|
303 |
+
feature_extractor = AutoFeatureExtractor.from_pretrained(model_name, token=HF_TOKEN)
|
304 |
+
depth_model = AutoModelForDepthEstimation.from_pretrained(model_name, token=HF_TOKEN)
|
305 |
|
306 |
+
# Move to appropriate device
|
307 |
if device == "cuda":
|
308 |
depth_model = depth_model.to(device)
|
309 |
|
|
|
313 |
except Exception as e:
|
314 |
print(f"Error loading depth model: {str(e)}")
|
315 |
print(traceback.format_exc())
|
316 |
+
print("Using manual depth map generation instead.")
|
317 |
+
return None, None
|
318 |
+
|
319 |
+
# Create a simple depth map without ML models
|
320 |
+
def create_simple_depth_map(image):
|
321 |
+
"""Create a simple depth map from image without ML models"""
|
322 |
+
# Convert to numpy array if needed
|
323 |
+
if isinstance(image, Image.Image):
|
324 |
+
img_array = np.array(image)
|
325 |
+
else:
|
326 |
+
img_array = image
|
327 |
+
|
328 |
+
# Convert to grayscale
|
329 |
+
if len(img_array.shape) == 3 and img_array.shape[2] >= 3:
|
330 |
+
gray = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY)
|
331 |
+
else:
|
332 |
+
gray = img_array.astype(np.uint8)
|
333 |
+
|
334 |
+
# Apply edge detection
|
335 |
+
edges = cv2.Canny(gray, 100, 200)
|
336 |
+
|
337 |
+
# Create depth map using blur and edges
|
338 |
+
depth_map = cv2.GaussianBlur(gray, (15, 15), 0)
|
339 |
+
|
340 |
+
# Combine with edges to preserve details
|
341 |
+
depth_map = depth_map.astype(float) / 255.0
|
342 |
+
edges = edges.astype(float) / 255.0
|
343 |
+
|
344 |
+
# Edges should be deeper in the depth map
|
345 |
+
depth_map = depth_map * (1.0 - 0.5 * edges)
|
346 |
+
|
347 |
+
# Center objects usually closer to viewer (create a radial gradient)
|
348 |
+
h, w = depth_map.shape
|
349 |
+
center_y, center_x = h // 2, w // 2
|
350 |
+
y, x = np.ogrid[:h, :w]
|
351 |
+
dist_from_center = np.sqrt((x - center_x)**2 + (y - center_y)**2)
|
352 |
+
max_dist = np.sqrt(center_x**2 + center_y**2)
|
353 |
+
dist_factor = dist_from_center / max_dist
|
354 |
+
|
355 |
+
# Apply center bias - center is closer (lower depth values)
|
356 |
+
depth_map = depth_map + 0.3 * dist_factor
|
357 |
+
|
358 |
+
# Normalize
|
359 |
+
depth_map = (depth_map - depth_map.min()) / (depth_map.max() - depth_map.min() + 1e-10)
|
360 |
+
|
361 |
+
# Smooth the depth map to avoid artifacts
|
362 |
+
depth_map = gaussian_filter(depth_map, sigma=1.0)
|
363 |
+
|
364 |
+
return depth_map
|
365 |
|
366 |
+
# Process image to create 3D model using simplified approach
|
367 |
def process_openlrm(image, job_id, detail_level='medium', output_format='obj'):
|
368 |
try:
|
369 |
+
# Load OpenLRM model - now returns simplified 3D generator
|
370 |
model = load_openlrm_model()
|
371 |
if model is None:
|
372 |
# Fallback to depth-based approach
|
373 |
return process_depth_based(image, job_id, detail_level, output_format)
|
374 |
|
375 |
# Preprocess image - remove background for better results
|
376 |
+
processing_jobs[job_id]['progress'] = 20
|
377 |
image_rgba = remove_background(image)
|
378 |
|
379 |
# Update progress
|
380 |
+
processing_jobs[job_id]['progress'] = 40
|
381 |
|
382 |
+
# Process with model to get 3D mesh
|
|
|
383 |
result = model(image_rgba)
|
384 |
|
385 |
# Update progress
|
|
|
402 |
|
403 |
# Convert OpenLRM result to trimesh
|
404 |
def convert_to_trimesh(result, image):
|
405 |
+
# Use the provided vertices and faces from the model result
|
406 |
+
vertices = np.array(result.get("vertices", []))
|
407 |
+
faces = np.array(result.get("faces", []))
|
408 |
+
|
409 |
+
# Create a default mesh if needed
|
410 |
+
if len(vertices) == 0 or len(faces) == 0:
|
411 |
+
# Generate sample vertices and faces
|
412 |
+
x = np.linspace(-1, 1, 20)
|
413 |
+
y = np.linspace(-1, 1, 20)
|
414 |
+
z = np.linspace(-1, 1, 10)
|
415 |
+
|
416 |
+
# Create grid points
|
417 |
+
xx, yy = np.meshgrid(x, y)
|
418 |
+
zz = np.zeros_like(xx)
|
419 |
+
|
420 |
+
# Create a simple height field
|
421 |
+
vertices = np.vstack([xx.flatten(), yy.flatten(), zz.flatten()]).T
|
422 |
+
|
423 |
+
# Create faces
|
424 |
+
faces = []
|
425 |
+
n = 20 # Grid size
|
426 |
+
for i in range(n-1):
|
427 |
+
for j in range(n-1):
|
428 |
+
idx = i*n + j
|
429 |
+
faces.append([idx, idx+1, idx+n])
|
430 |
+
faces.append([idx+1, idx+n+1, idx+n])
|
431 |
+
faces = np.array(faces)
|
432 |
|
433 |
+
# Create mesh with provided data
|
434 |
mesh = trimesh.Trimesh(vertices=vertices, faces=faces)
|
435 |
|
436 |
# Add texture from the original image
|
437 |
if hasattr(image, 'convert'):
|
438 |
+
try:
|
439 |
+
img_array = np.array(image.convert("RGBA"))
|
440 |
+
if img_array.shape[2] == 4: # RGBA
|
441 |
+
vertex_colors = sample_texture_from_image(img_array, vertices)
|
442 |
+
mesh.visual.vertex_colors = vertex_colors
|
443 |
+
except Exception as e:
|
444 |
+
print(f"Error applying texture: {e}")
|
445 |
|
446 |
return mesh
|
447 |
|
448 |
# Sample helper functions for mesh creation
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
449 |
def sample_texture_from_image(image, vertices):
|
450 |
+
"""Sample colors from image based on vertex positions"""
|
451 |
# Sample colors from image based on vertex positions
|
452 |
h, w = image.shape[:2]
|
453 |
colors = np.zeros((len(vertices), 4), dtype=np.uint8)
|
454 |
|
455 |
+
# Find the range of vertex positions
|
456 |
+
min_x, min_y = vertices[:, 0].min(), vertices[:, 1].min()
|
457 |
+
max_x, max_y = vertices[:, 0].max(), vertices[:, 1].max()
|
458 |
|
459 |
+
# Normalize vertex positions to [0,1] for sampling
|
460 |
+
for i, v in enumerate(vertices):
|
461 |
+
# Map from vertex coordinates to image coordinates
|
462 |
+
x_norm = (v[0] - min_x) / (max_x - min_x) if max_x > min_x else 0.5
|
463 |
+
y_norm = (v[1] - min_y) / (max_y - min_y) if max_y > min_y else 0.5
|
464 |
+
|
465 |
+
# Clamp to valid range
|
466 |
+
x_norm = max(0, min(1, x_norm))
|
467 |
+
y_norm = max(0, min(1, y_norm))
|
468 |
+
|
469 |
+
# Convert to image coordinates
|
470 |
+
x = int(x_norm * (w-1))
|
471 |
+
y = int(y_norm * (h-1))
|
472 |
+
|
473 |
+
# Sample color
|
474 |
+
if 0 <= x < w and 0 <= y < h:
|
475 |
colors[i] = image[y, x]
|
476 |
else:
|
477 |
colors[i] = [200, 200, 200, 255] # Default color
|
|
|
481 |
# Process using depth-based approach as fallback
|
482 |
def process_depth_based(image, job_id, detail_level='medium', output_format='obj'):
|
483 |
try:
|
484 |
+
# Load depth model
|
485 |
+
depth_model_result = load_depth_model()
|
|
|
|
|
486 |
|
487 |
# Update progress
|
488 |
processing_jobs[job_id]['progress'] = 30
|
489 |
|
490 |
+
# Check if model loading was successful
|
491 |
+
if depth_model_result[0] is None:
|
492 |
+
# Use manual depth map generation
|
493 |
+
print("Using manual depth map generation")
|
494 |
+
depth_map = create_simple_depth_map(image)
|
495 |
+
else:
|
496 |
+
# Extract model and feature extractor
|
497 |
+
depth_model, feature_extractor = depth_model_result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
498 |
|
499 |
+
# Get depth map from model
|
500 |
+
with torch.no_grad():
|
501 |
+
# Prepare image for the model
|
502 |
+
inputs = feature_extractor(images=image, return_tensors="pt")
|
503 |
+
if torch.cuda.is_available():
|
504 |
+
inputs = {k: v.cuda() for k, v in inputs.items()}
|
505 |
+
|
506 |
+
# Forward pass
|
507 |
+
outputs = depth_model(**inputs)
|
508 |
+
|
509 |
+
# Different models have different output formats
|
510 |
+
if hasattr(outputs, "predicted_depth"):
|
511 |
+
predicted_depth = outputs.predicted_depth
|
512 |
+
elif hasattr(outputs, "logits"): # For some models
|
513 |
+
predicted_depth = outputs.logits
|
514 |
+
else:
|
515 |
+
# Generic handling - take the first output tensor
|
516 |
+
predicted_depth = list(outputs.values())[0]
|
517 |
+
|
518 |
+
# Resize depth to original image size
|
519 |
+
depth_map = F.interpolate(
|
520 |
+
predicted_depth.unsqueeze(1),
|
521 |
+
size=(image.height, image.width),
|
522 |
+
mode="bicubic",
|
523 |
+
align_corners=False,
|
524 |
+
).squeeze().cpu().numpy()
|
525 |
+
|
526 |
# Update progress
|
527 |
processing_jobs[job_id]['progress'] = 60
|
528 |
|
529 |
+
# Normalize depth map if from model
|
530 |
+
if 'depth_map' not in locals():
|
531 |
+
depth_min = depth_map.min()
|
532 |
+
depth_max = depth_map.max()
|
533 |
+
depth_normalized = (depth_map - depth_min) / (depth_max - depth_min + 1e-10)
|
534 |
+
else:
|
535 |
+
depth_normalized = depth_map
|
536 |
|
537 |
# Create mesh from depth map
|
538 |
mesh = depth_to_mesh(depth_normalized, image,
|
|
|
543 |
# Update progress
|
544 |
processing_jobs[job_id]['progress'] = 80
|
545 |
|
546 |
+
# Clean up to free memory
|
547 |
+
optimize_memory()
|
548 |
+
|
549 |
return mesh
|
550 |
|
551 |
except Exception as e:
|
552 |
print(f"Error in depth-based processing: {str(e)}")
|
553 |
print(traceback.format_exc())
|
554 |
+
|
555 |
+
# Ultimate fallback - create a simple mesh from the image
|
556 |
+
try:
|
557 |
+
print("Using emergency fallback mesh generation")
|
558 |
+
depth_map = create_simple_depth_map(image)
|
559 |
+
mesh = depth_to_mesh(depth_map, image, resolution=50, detail_level='low')
|
560 |
+
return mesh
|
561 |
+
except Exception as fallback_error:
|
562 |
+
print(f"Fallback mesh generation failed: {fallback_error}")
|
563 |
+
raise
|
564 |
|
565 |
# Enhanced depth map processing
|
566 |
def enhance_depth_map(depth_map, detail_level='medium'):
|
|
|
669 |
mesh = trimesh.Trimesh(vertices=vertices, faces=faces)
|
670 |
|
671 |
# Apply texturing if image is provided
|
672 |
+
if image is not None:
|
673 |
# Convert to numpy array if needed
|
674 |
if isinstance(image, Image.Image):
|
675 |
img_array = np.array(image)
|
|
|
677 |
img_array = image
|
678 |
|
679 |
# Create vertex colors
|
680 |
+
if len(img_array.shape) >= 2:
|
681 |
# Create vertex colors by sampling the image
|
682 |
vertex_colors = np.zeros((vertices.shape[0], 4), dtype=np.uint8)
|
683 |
|
684 |
for i in range(resolution):
|
685 |
for j in range(resolution):
|
686 |
# Calculate image coordinates
|
687 |
+
img_x = min(max(0, int(j * (img_array.shape[1] - 1) / (resolution - 1))), img_array.shape[1] - 1)
|
688 |
+
img_y = min(max(0, int(i * (img_array.shape[0] - 1) / (resolution - 1))), img_array.shape[0] - 1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
689 |
|
690 |
vertex_idx = i * resolution + j
|
691 |
|
692 |
if len(img_array.shape) == 3 and img_array.shape[2] == 3: # RGB
|
693 |
+
r, g, b = img_array[img_y, img_x]
|
694 |
+
vertex_colors[vertex_idx] = [r, g, b, 255]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
695 |
elif len(img_array.shape) == 3 and img_array.shape[2] == 4: # RGBA
|
696 |
+
vertex_colors[vertex_idx] = img_array[img_y, img_x]
|
|
|
|
|
|
|
|
|
697 |
else:
|
698 |
# Handle grayscale
|
699 |
+
gray = img_array[img_y, img_x]
|
700 |
+
if np.isscalar(gray):
|
701 |
+
vertex_colors[vertex_idx] = [gray, gray, gray, 255]
|
702 |
+
else:
|
703 |
+
# Just in case gray is some kind of array
|
704 |
+
gray_val = np.mean(gray)
|
705 |
+
vertex_colors[vertex_idx] = [gray_val, gray_val, gray_val, 255]
|
706 |
|
707 |
mesh.visual.vertex_colors = vertex_colors
|
708 |
|
709 |
# Apply smoothing to get rid of staircase artifacts
|
710 |
if detail_level != 'high':
|
711 |
+
try:
|
712 |
+
# Use laplacian smoothing if available
|
713 |
+
mesh = mesh.smoothed(method='laplacian', iterations=1)
|
714 |
+
except Exception as e:
|
715 |
+
print(f"Smoothing error (non-critical): {e}")
|
716 |
|
717 |
# Fix normals for better rendering
|
718 |
+
try:
|
719 |
+
mesh.fix_normals()
|
720 |
+
except Exception as e:
|
721 |
+
print(f"Normal fixing error (non-critical): {e}")
|
722 |
+
|
723 |
+
# Simulate full 3D by duplicating and flipping the mesh only if detail level is higher
|
724 |
+
if detail_level == 'high' and not USE_SIMPLIFIED_MODE:
|
725 |
+
try:
|
726 |
+
# Create a complete 3D object by duplicating and flipping the mesh
|
727 |
+
back_mesh = mesh.copy()
|
728 |
+
# Flip to create the back side
|
729 |
+
back_mesh.vertices[:, 2] = -back_mesh.vertices[:, 2] - 0.1 # Add small offset to avoid z-fighting
|
730 |
+
# Fix normals after flipping
|
731 |
+
back_mesh.fix_normals()
|
732 |
+
|
733 |
+
# Combine front and back meshes
|
734 |
+
combined_mesh = trimesh.util.concatenate([mesh, back_mesh])
|
735 |
+
|
736 |
+
# Add side panels to create a watertight model
|
737 |
+
combined_mesh = create_watertight_model(combined_mesh)
|
738 |
+
return combined_mesh
|
739 |
+
except Exception as e:
|
740 |
+
print(f"3D completion error (non-critical): {e}")
|
741 |
|
742 |
return mesh
|
743 |
|
744 |
+
# Create a watertight model by adding side panels
|
745 |
def create_watertight_model(mesh):
|
746 |
+
try:
|
747 |
+
# Extract boundary edges - simplified approach to avoid errors
|
748 |
+
edges = mesh.edges_unique
|
749 |
+
edge_faces = mesh.edges_face
|
750 |
+
boundary_edges = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
751 |
|
752 |
+
# Find edges that are only part of one face (boundaries)
|
753 |
+
edge_face_counts = np.bincount(edge_faces.flatten(), minlength=len(mesh.faces))
|
754 |
+
boundary_face_indices = np.where(edge_face_counts == 1)[0]
|
755 |
+
|
756 |
+
# Get boundary edges
|
757 |
+
for i, edge in enumerate(edges):
|
758 |
+
faces = edge_faces[i]
|
759 |
+
if -1 in faces or len(np.unique(faces)) == 1:
|
760 |
+
boundary_edges.append(edge)
|
761 |
+
|
762 |
+
# If no boundary edges, return the original mesh
|
763 |
+
if len(boundary_edges) == 0:
|
764 |
+
return mesh
|
765 |
+
|
766 |
+
# Simplify for Hugging Face Space - just return original mesh
|
767 |
+
if USE_SIMPLIFIED_MODE:
|
768 |
+
return mesh
|
769 |
|
770 |
+
# Create side panels along boundary edges - simplified version
|
771 |
+
new_faces = []
|
772 |
+
new_vertices = mesh.vertices.copy()
|
|
|
|
|
|
|
|
|
|
|
|
|
773 |
|
774 |
+
# Just add a base and close the model
|
775 |
+
min_z = np.min(mesh.vertices[:, 2])
|
776 |
+
max_z = np.max(mesh.vertices[:, 2])
|
777 |
|
778 |
+
# Find vertices near the minimum z height
|
779 |
+
bottom_vertices = np.where(np.isclose(mesh.vertices[:, 2], min_z, atol=0.1))[0]
|
780 |
+
|
781 |
+
if len(bottom_vertices) > 3:
|
782 |
+
# Create a simple bottom face - simplified approach
|
783 |
+
center = np.mean(mesh.vertices[bottom_vertices], axis=0)
|
784 |
+
center_idx = len(new_vertices)
|
785 |
+
new_vertices = np.vstack([new_vertices, center])
|
786 |
+
|
787 |
+
# Add triangles connecting the boundary vertices to the center
|
788 |
+
for i in range(len(bottom_vertices)-1):
|
789 |
+
new_faces.append([bottom_vertices[i], bottom_vertices[i+1], center_idx])
|
790 |
+
|
791 |
+
# Close the loop
|
792 |
+
new_faces.append([bottom_vertices[-1], bottom_vertices[0], center_idx])
|
793 |
+
|
794 |
+
# Create new mesh with added faces
|
795 |
+
if len(new_faces) > 0:
|
796 |
+
new_faces = np.array(new_faces)
|
797 |
+
combined_faces = np.vstack([mesh.faces, new_faces])
|
798 |
+
watertight_mesh = trimesh.Trimesh(vertices=new_vertices, faces=combined_faces)
|
799 |
+
|
800 |
+
# Copy vertex colors if they exist
|
801 |
+
if hasattr(mesh.visual, 'vertex_colors') and mesh.visual.vertex_colors is not None:
|
802 |
+
# Extend vertex colors array for new vertices
|
803 |
+
extended_colors = np.vstack([
|
804 |
+
mesh.visual.vertex_colors,
|
805 |
+
np.full((len(new_vertices) - len(mesh.vertices), 4), [200, 200, 200, 255], dtype=np.uint8)
|
806 |
+
])
|
807 |
+
watertight_mesh.visual.vertex_colors = extended_colors
|
808 |
+
|
809 |
+
return watertight_mesh
|
810 |
+
|
811 |
+
return mesh
|
812 |
+
except Exception as e:
|
813 |
+
print(f"Watertight model creation failed (non-critical): {e}")
|
814 |
+
return mesh
|
815 |
|
816 |
@app.route('/health', methods=['GET'])
|
817 |
def health_check():
|
818 |
return jsonify({
|
819 |
"status": "healthy",
|
820 |
+
"model": "Enhanced 3D Model Generator",
|
821 |
+
"device": "cuda" if torch.cuda.is_available() else "cpu",
|
822 |
+
"simplified_mode": USE_SIMPLIFIED_MODE
|
823 |
}), 200
|
824 |
|
825 |
@app.route('/progress/<job_id>', methods=['GET'])
|
|
|
853 |
break
|
854 |
check_count = 0
|
855 |
|
856 |
+
# Send final status
|
857 |
if job['status'] == 'completed':
|
858 |
yield f"data: {json.dumps({'status': 'completed', 'progress': 100, 'result_url': job['result_url'], 'preview_url': job['preview_url']})}\n\n"
|
859 |
else:
|
|
|
880 |
output_format = request.form.get('output_format', 'obj').lower()
|
881 |
detail_level = request.form.get('detail_level', 'medium').lower() # Parameter for detail level
|
882 |
model_type = request.form.get('model_type', 'openlrm').lower() # 'openlrm' or 'depth'
|
883 |
+
|
884 |
+
# Adjust parameters for simplified mode
|
885 |
+
if USE_SIMPLIFIED_MODE:
|
886 |
+
mesh_resolution = min(mesh_resolution, 100) # Lower resolution for simplified mode
|
887 |
+
if detail_level == 'high':
|
888 |
+
detail_level = 'medium' # Downgrade detail level in simplified mode
|
889 |
except ValueError:
|
890 |
return jsonify({"error": "Invalid parameter values"}), 400
|
891 |
|
|
|
992 |
os.remove(filepath)
|
993 |
|
994 |
# Force garbage collection to free memory
|
995 |
+
optimize_memory()
|
|
|
|
|
996 |
|
997 |
except Exception as e:
|
998 |
# Handle errors
|
|
|
1135 |
@app.route('/', methods=['GET'])
|
1136 |
def index():
|
1137 |
return jsonify({
|
1138 |
+
"message": "Enhanced 3D Model Generator",
|
1139 |
"endpoints": [
|
1140 |
"/convert",
|
1141 |
"/progress/<job_id>",
|
|
|
1149 |
"detail_level": "low, medium, or high - controls the level of detail in the final model",
|
1150 |
"model_type": "openlrm (default, full 3D) or depth (faster but simpler)"
|
1151 |
},
|
1152 |
+
"description": "This API creates high-quality 3D models from 2D images with full 3D structure and texturing",
|
1153 |
+
"simplified_mode": USE_SIMPLIFIED_MODE
|
1154 |
}), 200
|
1155 |
|
1156 |
+
# System compatibility check function
|
1157 |
+
def check_system_compatibility():
|
1158 |
+
"""Check if the system can run the full model or needs simplified mode"""
|
1159 |
+
print("Checking system compatibility...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1160 |
|
1161 |
+
# Check available memory
|
1162 |
+
try:
|
1163 |
+
import psutil
|
1164 |
+
mem = psutil.virtual_memory()
|
1165 |
+
free_mem_gb = mem.available / (1024 ** 3)
|
1166 |
+
print(f"Available memory: {free_mem_gb:.2f} GB")
|
1167 |
+
except ImportError:
|
1168 |
+
print("psutil not available, cannot check memory")
|
1169 |
+
free_mem_gb = 1.0 # Assume low memory
|
1170 |
|
1171 |
+
# Check GPU
|
1172 |
+
gpu_available = torch.cuda.is_available()
|
1173 |
+
gpu_mem_gb = 0
|
1174 |
+
if gpu_available:
|
|
|
1175 |
try:
|
1176 |
+
gpu_mem_gb = torch.cuda.get_device_properties(0).total_memory / (1024 ** 3)
|
1177 |
+
print(f"GPU available: {gpu_available}, Memory: {gpu_mem_gb:.2f} GB")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1178 |
except Exception as e:
|
1179 |
+
print(f"Error checking GPU memory: {e}")
|
1180 |
+
else:
|
1181 |
+
print("No GPU available")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1182 |
|
1183 |
+
# Set simplified mode if limited resources
|
1184 |
+
global USE_SIMPLIFIED_MODE
|
1185 |
+
if free_mem_gb < 4.0 or (gpu_available and gpu_mem_gb < 2.0):
|
1186 |
+
print("Limited resources detected, using simplified mode")
|
1187 |
+
USE_SIMPLIFIED_MODE = True
|
1188 |
+
else:
|
1189 |
+
print("Sufficient resources detected")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1190 |
|
1191 |
if __name__ == '__main__':
|
1192 |
+
# Check system compatibility
|
1193 |
+
check_system_compatibility()
|
1194 |
+
|
1195 |
# Start the cleanup thread
|
1196 |
cleanup_old_jobs()
|
1197 |
|