First attempt at recalculating the correct world spaces back to local spaces.
Browse files- fbx_handler.py +405 -56
fbx_handler.py
CHANGED
@@ -1,4 +1,6 @@
|
|
1 |
# Import core libs.
|
|
|
|
|
2 |
import pandas as pd
|
3 |
import numpy as np
|
4 |
from pathlib import Path
|
@@ -7,6 +9,7 @@ from typing import List, Union, Tuple
|
|
7 |
# Import util libs.
|
8 |
import contextlib
|
9 |
import fbx
|
|
|
10 |
|
11 |
# Import custom data.
|
12 |
import globals
|
@@ -46,6 +49,162 @@ def make_ghost_markers(missing: int) -> np.array:
|
|
46 |
])
|
47 |
|
48 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
49 |
class FBXContainer:
|
50 |
# TODO: Model is currently built for training. Add testing mode.
|
51 |
def __init__(self, fbx_file: Path,
|
@@ -80,14 +239,15 @@ class FBXContainer:
|
|
80 |
self.vol_x = volume_dims[0]
|
81 |
self.vol_y = volume_dims[1]
|
82 |
self.vol_z = volume_dims[2]
|
83 |
-
|
84 |
self.scale = scale
|
85 |
|
86 |
self.max_actors = max_actors
|
87 |
# Maximum point cloud size = 73 * max_actors + unlabeled markers.
|
88 |
self.pc_size = pc_size
|
89 |
|
90 |
-
self.
|
|
|
91 |
self.valid_frames = []
|
92 |
|
93 |
self.__init_scene()
|
@@ -102,15 +262,16 @@ class FBXContainer:
|
|
102 |
Destroys the importer to remove the reference to the loaded file.
|
103 |
"""
|
104 |
# Create an FBX manager and importer.
|
105 |
-
manager = fbx.FbxManager.Create()
|
106 |
-
importer = fbx.FbxImporter.Create(manager, '')
|
107 |
|
108 |
# Import the FBX file.
|
109 |
-
importer.Initialize(str(self.
|
110 |
-
self.scene = fbx.FbxScene.Create(manager, '')
|
111 |
importer.Import(self.scene)
|
112 |
self.root = self.scene.GetRootNode()
|
113 |
self.time_mode = self.scene.GetGlobalSettings().GetTimeMode()
|
|
|
114 |
|
115 |
# Destroy importer to remove reference to imported file.
|
116 |
# This will allow us to delete the uploaded file.
|
@@ -126,9 +287,9 @@ class FBXContainer:
|
|
126 |
|
127 |
# Find the total number of frames to expect from the local time span.
|
128 |
local_time_span = anim_stack.GetLocalTimeSpan()
|
129 |
-
self.num_frames = int(local_time_span.GetDuration().GetFrameCount(
|
130 |
-
self.start_frame = local_time_span.GetStart().GetFrameCount(
|
131 |
-
self.end_frame = local_time_span.GetStop().GetFrameCount(
|
132 |
|
133 |
def __init_actors(self):
|
134 |
"""
|
@@ -161,9 +322,8 @@ class FBXContainer:
|
|
161 |
for actor_idx in range(actor_node.GetChildCount()):
|
162 |
child = actor_node.GetChild(actor_idx)
|
163 |
# Child name might have namespaces in it like this: Vera:ARIEL
|
164 |
-
# We want to match only on the actual name, so
|
165 |
-
|
166 |
-
if child_name == marker_name:
|
167 |
actor_markers[marker_name] = child
|
168 |
|
169 |
assert len(actor_markers) == len(self.marker_names), f'{actor_node.GetName()} does not have all markers.'
|
@@ -177,7 +337,7 @@ class FBXContainer:
|
|
177 |
# Find the Unlabeled_Markers parent node.
|
178 |
for i in range(self.root.GetChildCount()):
|
179 |
gen1_node = self.root.GetChild(i)
|
180 |
-
if gen1_node
|
181 |
self.unlabeled_markers_parent = gen1_node
|
182 |
self.unlabeled_markers = [gen1_node.GetChild(um) for um in range(gen1_node.GetChildCount())]
|
183 |
return
|
@@ -214,7 +374,7 @@ class FBXContainer:
|
|
214 |
return
|
215 |
|
216 |
# Get all keyframes on the animation curve and store their frame numbers.
|
217 |
-
keys = [t_curve.KeyGet(i).GetTime().GetFrameCount(
|
218 |
# Check for each frame in frames if it is present in the list of keyframed frames.
|
219 |
for frame in frames:
|
220 |
if frame not in keys:
|
@@ -334,6 +494,43 @@ class FBXContainer:
|
|
334 |
self._check_actor(actor)
|
335 |
return self.markers[actor][name]
|
336 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
337 |
def print_valid_frames_stats_for_actor(self, actor: int = 0):
|
338 |
"""
|
339 |
Prints: actor name, total amount of frames in the animation, amount of valid frames for the given actor,
|
@@ -395,18 +592,24 @@ class FBXContainer:
|
|
395 |
|
396 |
return pd.DataFrame(all_poses, columns=columns)
|
397 |
|
398 |
-
def
|
399 |
"""
|
400 |
Evaluates the world translation of the given marker at the given time,
|
401 |
scales it down by scale and turns it into a vector list.
|
402 |
:param m: `fbx.FbxNode` marker to evaluate the world translation of.
|
403 |
:param time: `fbx.FbxTime` time to evaluate at.
|
|
|
404 |
:return: Vector in the form: [tx, ty, tz].
|
405 |
"""
|
406 |
t = m.EvaluateGlobalTransform(time).GetT()
|
407 |
-
|
408 |
-
|
409 |
-
|
|
|
|
|
|
|
|
|
|
|
410 |
|
411 |
return [x, y, z]
|
412 |
|
@@ -421,21 +624,32 @@ class FBXContainer:
|
|
421 |
curve = marker.LclTranslation.GetCurve(self.anim_layer, 'X')
|
422 |
return False if curve is None else curve.KeyFind(time) != -1
|
423 |
|
424 |
-
def
|
425 |
"""
|
426 |
-
For each actor,
|
427 |
-
:param time
|
428 |
-
:
|
|
|
429 |
"""
|
430 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
431 |
# Iterate through all actors to get their markers' world translations and add them to the cloud list.
|
432 |
for actor_idx in range(self.actor_count):
|
433 |
-
|
434 |
cloud.extend(
|
435 |
# This actor's point cloud is made up of all markers that have a keyframe at the given time.
|
436 |
# For each marker, we create this row: [actor class (index+1), marker class (index+1), tx, ty, tz].
|
437 |
# We use index+1 because the unlabeled markers will use index 0 for both classes.
|
438 |
-
[actor_idx + 1, marker_class, *self.
|
439 |
for marker_class, (marker_name, m) in enumerate(
|
440 |
self.markers[actor_idx].items(), start=1
|
441 |
)
|
@@ -445,56 +659,58 @@ class FBXContainer:
|
|
445 |
if self.is_kf_present(m, time)
|
446 |
)
|
447 |
|
448 |
-
# Unlabeled markers are their own 'actor', so we only need one loop here.
|
449 |
-
for m in self.unlabeled_markers:
|
450 |
-
if self.is_kf_present(m, time):
|
451 |
-
# Unlabeled markers use actor class 0 and marker class 0.
|
452 |
-
cloud.extend([[0, 0, *self.get_transformed_worldspace(m, time)]])
|
453 |
-
|
454 |
# If the data is extremely noisy, it might have only a few labeled markers and a lot of unlabeled markers.
|
455 |
# The returned point cloud is not allowed to be bigger than the maximum size (self.pc_size),
|
456 |
# so return the cloud as a np array that cuts off any excessive markers.
|
457 |
return np.array(cloud)[:self.pc_size]
|
458 |
|
459 |
-
def
|
460 |
"""
|
461 |
Convenience method that calls self.get_sparse_cloud() for all frames in the frame range
|
462 |
and returns the combined result.
|
|
|
463 |
:return: `np.array` that contains a sparse cloud for each frame in the frame range.
|
464 |
"""
|
465 |
-
|
466 |
-
# The SetFrame() method is a void/in-place method, so we can't use list comprehension
|
467 |
-
# to create a list of fbx.FbxTime()s.
|
468 |
-
time = fbx.FbxTime()
|
469 |
-
times = []
|
470 |
-
for f in self.get_frame_range():
|
471 |
-
# Use in-place function to update the time. This returns None, so we can't append this directly.
|
472 |
-
time.SetFrame(f)
|
473 |
-
times.append(time)
|
474 |
|
475 |
-
|
476 |
-
|
477 |
-
|
478 |
"""
|
479 |
For each frame in the frame range, collects the point cloud that is present in the file.
|
480 |
Then it creates a ghost cloud of random markers that are treated as unlabeled markers,
|
481 |
and adds them together to create a dense cloud whose shape is always (self.pc_size, 5).
|
482 |
Optionally shuffles this dense cloud before adding it to the final list.
|
483 |
-
:param shuffle: If `True`, shuffles the dense point cloud
|
|
|
|
|
|
|
484 |
:return: `np.array` that contains a dense point cloud for each frame,
|
485 |
with a shape of (self.num_frames, self.pc_size, 5).
|
486 |
"""
|
487 |
-
|
488 |
clouds = []
|
489 |
-
|
490 |
-
|
491 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
492 |
missing = self.pc_size - cloud.shape[0]
|
493 |
|
494 |
# Only bother creating ghost markers if there are any missing rows.
|
|
|
|
|
495 |
if missing > 0:
|
496 |
ghost_cloud = make_ghost_markers(missing)
|
497 |
-
cloud = np.vstack([
|
498 |
|
499 |
# Shuffle the rows if needed. Because each row contains all dependent and independent variables,
|
500 |
# shuffling won't mess up the labels.
|
@@ -505,7 +721,7 @@ class FBXContainer:
|
|
505 |
|
506 |
return np.array(clouds)
|
507 |
|
508 |
-
def
|
509 |
-> Tuple[np.array, np.array, np.array]:
|
510 |
"""
|
511 |
Splits a timeline dense cloud with shape (self.num_frames, self.pc_size, 5) into 3 different
|
@@ -516,10 +732,11 @@ class FBXContainer:
|
|
516 |
:param cloud: `np.array` of shape (self.num_frames, self.pc_size, 5) that contains a dense point cloud
|
517 |
(self.pc_size, 5) per frame in the frame range.
|
518 |
:param shuffle: `bool` whether to shuffle the generated cloud if no cloud was given.
|
|
|
519 |
:return: Return tuple of `np.array` as (actor classes, marker classes, translation vectors).
|
520 |
"""
|
521 |
if cloud is None:
|
522 |
-
cloud = self.
|
523 |
|
524 |
assert cloud.shape[1] == 1000, f"Dense cloud doesn't have enough points. {cloud.shape[1]}/1000."
|
525 |
assert cloud.shape[2] == 5, f"Dense cloud is missing columns: {cloud.shape[2]}/5."
|
@@ -533,7 +750,7 @@ class FBXContainer:
|
|
533 |
:param c: `float` actor class index.
|
534 |
:return: `str` actor name.
|
535 |
"""
|
536 |
-
return 'UNLABELED' if c == 0
|
537 |
|
538 |
def convert_class_to_marker(self, c: float = 0):
|
539 |
"""
|
@@ -541,7 +758,7 @@ class FBXContainer:
|
|
541 |
:param c: `float` marker class index.
|
542 |
:return: `str` marker name.
|
543 |
"""
|
544 |
-
return 'UNLABELED' if c == 0
|
545 |
|
546 |
def export(self, t: str = 'csv', output_file: Path = None) -> Union[bytes, Path]:
|
547 |
# Get the dataframe with all animation data.
|
@@ -551,7 +768,7 @@ class FBXContainer:
|
|
551 |
return df.to_csv(index=False).encode('utf-8')
|
552 |
|
553 |
if output_file is None:
|
554 |
-
output_file = self.
|
555 |
|
556 |
if output_file.suffix != '.csv':
|
557 |
raise ValueError(f'{output_file} needs to be a .csv file.')
|
@@ -559,6 +776,138 @@ class FBXContainer:
|
|
559 |
df.to_csv(output_file, index=False)
|
560 |
return output_file
|
561 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
562 |
|
563 |
# d = FBXContainer(Path('G:/Firestorm/mocap-ai/data/fbx/dowg/TAKE_01+1_ALL_001.fbx'))
|
564 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
# Import core libs.
|
2 |
+
from pprint import pprint
|
3 |
+
|
4 |
import pandas as pd
|
5 |
import numpy as np
|
6 |
from pathlib import Path
|
|
|
9 |
# Import util libs.
|
10 |
import contextlib
|
11 |
import fbx
|
12 |
+
import itertools
|
13 |
|
14 |
# Import custom data.
|
15 |
import globals
|
|
|
49 |
])
|
50 |
|
51 |
|
52 |
+
def append_suffix(file_path: Path, suffix: str = '_INF'):
|
53 |
+
"""
|
54 |
+
Adds a suffix to the given file path.
|
55 |
+
:param file_path: `Path` object to the original file.
|
56 |
+
:param suffix: `str` suffix to add to the end of the original file name.
|
57 |
+
:return: Updated `Path`.
|
58 |
+
"""
|
59 |
+
new_file_name = file_path.stem + suffix + file_path.suffix
|
60 |
+
return file_path.with_name(new_file_name)
|
61 |
+
|
62 |
+
|
63 |
+
def merge_tdc(actor_classes: np.array,
|
64 |
+
marker_classes: np.array,
|
65 |
+
translation_vectors: np.array,
|
66 |
+
ordered: bool = True) -> np.array:
|
67 |
+
# Actor and marker classes enter as shape (x, 1000), so use np.expand_dims to create an extra dimension at the end.
|
68 |
+
# Return the concatenated array of shape (x, 1000, 5), which matches the original timeline dense cloud before
|
69 |
+
# splitting it into sub arrays.
|
70 |
+
tdc = np.concatenate((np.expand_dims(actor_classes, -1), np.expand_dims(marker_classes, -1),
|
71 |
+
translation_vectors), axis=2)
|
72 |
+
if ordered:
|
73 |
+
tdc = sort_cloud(tdc)
|
74 |
+
|
75 |
+
return tdc
|
76 |
+
|
77 |
+
|
78 |
+
def sort_cloud(cloud: np.array) -> np.array:
|
79 |
+
"""
|
80 |
+
Convenience function to sort a timeline dense cloud by actor and marker classes.
|
81 |
+
Not required.
|
82 |
+
:param cloud: `np.array` point cloud to sort.
|
83 |
+
:return: sorted `np.array` point cloud.
|
84 |
+
"""
|
85 |
+
# Extract the first two elements of the third dimension
|
86 |
+
actor_classes = cloud[:, :, 0]
|
87 |
+
marker_classes = cloud[:, :, 1]
|
88 |
+
|
89 |
+
# Create an empty array with the same shape as the input array
|
90 |
+
sorted_tdc = np.empty_like(cloud)
|
91 |
+
|
92 |
+
# Sort the input array row by row using the first two elements of the third dimension
|
93 |
+
for i in range(cloud.shape[0]):
|
94 |
+
# Get the sorting indices for the current row
|
95 |
+
sorted_indices = np.lexsort((marker_classes[i], actor_classes[i]))
|
96 |
+
|
97 |
+
# Sort the current row using the sorted indices
|
98 |
+
sorted_tdc[i] = cloud[i, sorted_indices]
|
99 |
+
|
100 |
+
return sorted_tdc
|
101 |
+
|
102 |
+
|
103 |
+
def isolate_labeled_markers_from_tdc(tdc: np.array) -> np.array:
|
104 |
+
return np.stack([tdc[i, tdc[i, :, 0] > 0.] for i in range(tdc.shape[0])], axis=0)
|
105 |
+
|
106 |
+
|
107 |
+
def create_keyframe(anim_curve: fbx.FbxAnimCurve, frame: int, value: float):
|
108 |
+
# Create an FbxTime object with the given frame number
|
109 |
+
t = fbx.FbxTime()
|
110 |
+
t.SetFrame(frame)
|
111 |
+
|
112 |
+
# Create a new keyframe with the specified value
|
113 |
+
key_index = anim_curve.KeyAdd(t)[0]
|
114 |
+
anim_curve.KeySetValue(key_index, value)
|
115 |
+
return True
|
116 |
+
|
117 |
+
|
118 |
+
def replace_keyframes_on_curve(anim_curve: fbx.FbxAnimCurve, frames: List[int], values: np.array):
|
119 |
+
# Check if the anim_curve is of type FbxAnimCurve
|
120 |
+
if not isinstance(anim_curve, fbx.FbxAnimCurve):
|
121 |
+
print("Input is not an FbxAnimCurve instance.")
|
122 |
+
return False
|
123 |
+
|
124 |
+
# Check if the frames and values lists have the same length
|
125 |
+
if len(frames) != len(values):
|
126 |
+
print("Frames and values lists have different lengths.")
|
127 |
+
return False
|
128 |
+
|
129 |
+
# Remove all existing keyframes
|
130 |
+
anim_curve.KeyClear()
|
131 |
+
|
132 |
+
# Create new keyframes with the given frames and values
|
133 |
+
for frame, value in zip(frames, values):
|
134 |
+
create_keyframe(anim_curve, frame, value)
|
135 |
+
|
136 |
+
return True
|
137 |
+
|
138 |
+
|
139 |
+
def get_child_node_by_name(parent_node: fbx.FbxNode, name: str, ignore_namespace: bool = False) \
|
140 |
+
-> Union[fbx.FbxNode, None]:
|
141 |
+
for c in range(parent_node.GetChildCount()):
|
142 |
+
child = parent_node.GetChild(c)
|
143 |
+
if match_name(child, name, ignore_namespace):
|
144 |
+
return child
|
145 |
+
return None
|
146 |
+
|
147 |
+
|
148 |
+
def match_name(node: fbx.FbxNode, name: str, ignore_namespace: bool = True) -> bool:
|
149 |
+
node_name = node.GetName()
|
150 |
+
if ignore_namespace:
|
151 |
+
node_name = node_name.split(':')[-1]
|
152 |
+
return node_name == name
|
153 |
+
|
154 |
+
|
155 |
+
def timeline_cloud_to_dict(data: np.array, start_frame: int = 0) -> dict:
|
156 |
+
# Initialize an empty dictionary.
|
157 |
+
result = {}
|
158 |
+
|
159 |
+
# Iterate over the first dimension (frames) and second dimension (markers).
|
160 |
+
for frame, node in itertools.product(range(data.shape[0]), range(data.shape[1])):
|
161 |
+
|
162 |
+
# Extract the actor class, node class, and translation vector.
|
163 |
+
actor_class = int(data[frame, node, 0])
|
164 |
+
marker_class = int(data[frame, node, 1])
|
165 |
+
# If actor or marker class is predicted to be 0 (unlabeled marker), then skip adding it to the dict,
|
166 |
+
# because we only want to keyframe labeled markers.
|
167 |
+
if actor_class == 0 or marker_class == 0:
|
168 |
+
continue
|
169 |
+
|
170 |
+
translation_vector = data[frame, node, 2:]
|
171 |
+
|
172 |
+
# Create the actor dictionary if it doesn't exist.
|
173 |
+
if actor_class not in result:
|
174 |
+
result[actor_class] = {}
|
175 |
+
|
176 |
+
# Create the node dictionary if it doesn't exist.
|
177 |
+
if marker_class not in result[actor_class]:
|
178 |
+
result[actor_class][marker_class] = {}
|
179 |
+
|
180 |
+
# Add the frame number and translation vector to the node dictionary.
|
181 |
+
result[actor_class][marker_class][frame + start_frame] = translation_vector
|
182 |
+
|
183 |
+
return result
|
184 |
+
|
185 |
+
|
186 |
+
def world_to_local_translation(node, parent_node, f):
|
187 |
+
t = fbx.FbxTime()
|
188 |
+
t.SetFrame(f)
|
189 |
+
child_world_matrix = node.EvaluateGlobalTransform(t)
|
190 |
+
child_world_matrix = np.array([
|
191 |
+
[child_world_matrix.Get(i, j) for j in range(4)] for i in range(4)
|
192 |
+
])
|
193 |
+
# Convert the world_translation vector to a homogeneous 4D vector by appending a 1
|
194 |
+
# world_translation_homogeneous = np.append(world_translation, 1)
|
195 |
+
|
196 |
+
# Get the parent's world transformation matrix as a numpy array
|
197 |
+
parent_world_matrix = parent_node.EvaluateGlobalTransform(t)
|
198 |
+
parent_world_matrix = np.array([
|
199 |
+
[parent_world_matrix.Get(i, j) for j in range(4)] for i in range(4)
|
200 |
+
])
|
201 |
+
|
202 |
+
# Compute the inverse of the parent's world transformation matrix
|
203 |
+
parent_world_matrix_inv = np.linalg.inv(parent_world_matrix)
|
204 |
+
|
205 |
+
return np.dot(parent_world_matrix_inv, child_world_matrix)
|
206 |
+
|
207 |
+
|
208 |
class FBXContainer:
|
209 |
# TODO: Model is currently built for training. Add testing mode.
|
210 |
def __init__(self, fbx_file: Path,
|
|
|
239 |
self.vol_x = volume_dims[0]
|
240 |
self.vol_y = volume_dims[1]
|
241 |
self.vol_z = volume_dims[2]
|
242 |
+
|
243 |
self.scale = scale
|
244 |
|
245 |
self.max_actors = max_actors
|
246 |
# Maximum point cloud size = 73 * max_actors + unlabeled markers.
|
247 |
self.pc_size = pc_size
|
248 |
|
249 |
+
self.input_fbx = fbx_file
|
250 |
+
self.output_fbx = append_suffix(fbx_file, '_INF')
|
251 |
self.valid_frames = []
|
252 |
|
253 |
self.__init_scene()
|
|
|
262 |
Destroys the importer to remove the reference to the loaded file.
|
263 |
"""
|
264 |
# Create an FBX manager and importer.
|
265 |
+
self.manager = fbx.FbxManager.Create()
|
266 |
+
importer = fbx.FbxImporter.Create(self.manager, '')
|
267 |
|
268 |
# Import the FBX file.
|
269 |
+
importer.Initialize(str(self.input_fbx))
|
270 |
+
self.scene = fbx.FbxScene.Create(self.manager, '')
|
271 |
importer.Import(self.scene)
|
272 |
self.root = self.scene.GetRootNode()
|
273 |
self.time_mode = self.scene.GetGlobalSettings().GetTimeMode()
|
274 |
+
fbx.FbxTime.SetGlobalTimeMode(self.time_mode)
|
275 |
|
276 |
# Destroy importer to remove reference to imported file.
|
277 |
# This will allow us to delete the uploaded file.
|
|
|
287 |
|
288 |
# Find the total number of frames to expect from the local time span.
|
289 |
local_time_span = anim_stack.GetLocalTimeSpan()
|
290 |
+
self.num_frames = int(local_time_span.GetDuration().GetFrameCount())
|
291 |
+
self.start_frame = local_time_span.GetStart().GetFrameCount()
|
292 |
+
self.end_frame = local_time_span.GetStop().GetFrameCount()
|
293 |
|
294 |
def __init_actors(self):
|
295 |
"""
|
|
|
322 |
for actor_idx in range(actor_node.GetChildCount()):
|
323 |
child = actor_node.GetChild(actor_idx)
|
324 |
# Child name might have namespaces in it like this: Vera:ARIEL
|
325 |
+
# We want to match only on the actual name, so ignore namespaces.
|
326 |
+
if match_name(child, marker_name, ignore_namespace=True):
|
|
|
327 |
actor_markers[marker_name] = child
|
328 |
|
329 |
assert len(actor_markers) == len(self.marker_names), f'{actor_node.GetName()} does not have all markers.'
|
|
|
337 |
# Find the Unlabeled_Markers parent node.
|
338 |
for i in range(self.root.GetChildCount()):
|
339 |
gen1_node = self.root.GetChild(i)
|
340 |
+
if match_name(gen1_node, 'Unlabeled_Markers'):
|
341 |
self.unlabeled_markers_parent = gen1_node
|
342 |
self.unlabeled_markers = [gen1_node.GetChild(um) for um in range(gen1_node.GetChildCount())]
|
343 |
return
|
|
|
374 |
return
|
375 |
|
376 |
# Get all keyframes on the animation curve and store their frame numbers.
|
377 |
+
keys = [t_curve.KeyGet(i).GetTime().GetFrameCount() for i in range(t_curve.KeyGetCount())]
|
378 |
# Check for each frame in frames if it is present in the list of keyframed frames.
|
379 |
for frame in frames:
|
380 |
if frame not in keys:
|
|
|
494 |
self._check_actor(actor)
|
495 |
return self.markers[actor][name]
|
496 |
|
497 |
+
def get_parent_node_by_name(self, parent_name: str, ignore_namespace: bool = True) -> Union[fbx.FbxNode, None]:
|
498 |
+
"""
|
499 |
+
Utility function to get a parent node reference by name.
|
500 |
+
:param parent_name: `str` name that will be looked for in the node name.
|
501 |
+
:param ignore_namespace: `bool` Whether to ignore namespaces in a node's name.
|
502 |
+
:return:
|
503 |
+
"""
|
504 |
+
# Find all parent nodes (/System, /Unlabeled_Markers, /Actor1, etc).
|
505 |
+
parent_nodes = [self.root.GetChild(i) for i in range(self.root.GetChildCount())]
|
506 |
+
|
507 |
+
return next(
|
508 |
+
(
|
509 |
+
parent_node
|
510 |
+
for parent_node in parent_nodes
|
511 |
+
if match_name(parent_node, parent_name, ignore_namespace=ignore_namespace)
|
512 |
+
),
|
513 |
+
None,
|
514 |
+
)
|
515 |
+
|
516 |
+
def get_node_by_path(self, path: str) -> fbx.FbxNode:
|
517 |
+
"""
|
518 |
+
Utility function to retrieve a node reference from a path like /Actor1/Hips/UpperLeg_l.
|
519 |
+
:param path: `str` path with forward slashes to follow.
|
520 |
+
:return: `fbx.FbxNode` reference to that node.
|
521 |
+
"""
|
522 |
+
# Split the path into node names.
|
523 |
+
node_names = [x for x in path.split('/') if x]
|
524 |
+
# Start the list of node references with the parent node that has its own function.
|
525 |
+
nodes = [self.get_parent_node_by_name(node_names[0], False)]
|
526 |
+
# Extend the list with each following child node of the previous parent.
|
527 |
+
nodes.extend(
|
528 |
+
get_child_node_by_name(nodes[idx], node_name)
|
529 |
+
for idx, node_name in enumerate(node_names[1:])
|
530 |
+
)
|
531 |
+
# Return the last node in the chain, which will be the node we were looking for.
|
532 |
+
return nodes[-1]
|
533 |
+
|
534 |
def print_valid_frames_stats_for_actor(self, actor: int = 0):
|
535 |
"""
|
536 |
Prints: actor name, total amount of frames in the animation, amount of valid frames for the given actor,
|
|
|
592 |
|
593 |
return pd.DataFrame(all_poses, columns=columns)
|
594 |
|
595 |
+
def get_worldspace(self, m: fbx.FbxNode, time: fbx.FbxTime, apply_transform: bool = True) -> List[float]:
|
596 |
"""
|
597 |
Evaluates the world translation of the given marker at the given time,
|
598 |
scales it down by scale and turns it into a vector list.
|
599 |
:param m: `fbx.FbxNode` marker to evaluate the world translation of.
|
600 |
:param time: `fbx.FbxTime` time to evaluate at.
|
601 |
+
:param apply_transform: `bool` Whether to transform the translation or not.
|
602 |
:return: Vector in the form: [tx, ty, tz].
|
603 |
"""
|
604 |
t = m.EvaluateGlobalTransform(time).GetT()
|
605 |
+
if not apply_transform:
|
606 |
+
return [t[i] for i in range(3)]
|
607 |
+
|
608 |
+
# First multiply by self.scale, which turns meters to centimeters.
|
609 |
+
# Then divide by volume dimensions, to normalize to the total area of the capture volume.
|
610 |
+
x = np.clip(t[0], -(self.vol_x * 0.5), self.vol_x * 0.5) * self.scale / self.vol_x
|
611 |
+
y = np.clip(t[1], -(self.vol_y * 0.5), self.vol_y * 0.5) * self.scale / self.vol_y
|
612 |
+
z = np.clip(t[2], -(self.vol_z * 0.5), self.vol_z * 0.5) * self.scale / self.vol_z
|
613 |
|
614 |
return [x, y, z]
|
615 |
|
|
|
624 |
curve = marker.LclTranslation.GetCurve(self.anim_layer, 'X')
|
625 |
return False if curve is None else curve.KeyFind(time) != -1
|
626 |
|
627 |
+
def get_sc(self, frame: int, apply_transform: bool = True) -> np.array:
|
628 |
"""
|
629 |
+
For each actor at the given time, find all markers with keyframes and add their values to a point cloud.
|
630 |
+
:param frame: `fbx.FbxTime` time at which to evaluate the marker.
|
631 |
+
:param apply_transform: `bool` Whether to transform the translation or not.
|
632 |
+
:return: sparse point cloud as `np.array`.
|
633 |
"""
|
634 |
+
time = fbx.FbxTime()
|
635 |
+
time.SetFrame(frame)
|
636 |
+
# Start with a cloud of unlabeled markers, which will use actor and marker class 0.
|
637 |
+
# It is important to start with these before the labeled markers,
|
638 |
+
# because by adding the labeled markers after (which use classes 1-74),
|
639 |
+
# we eventually return an array that doesn't need to be sorted anymore.
|
640 |
+
cloud = [
|
641 |
+
[0, 0, *self.get_worldspace(m, time, apply_transform)]
|
642 |
+
for m in self.unlabeled_markers
|
643 |
+
if self.is_kf_present(m, time)
|
644 |
+
]
|
645 |
+
|
646 |
# Iterate through all actors to get their markers' world translations and add them to the cloud list.
|
647 |
for actor_idx in range(self.actor_count):
|
|
|
648 |
cloud.extend(
|
649 |
# This actor's point cloud is made up of all markers that have a keyframe at the given time.
|
650 |
# For each marker, we create this row: [actor class (index+1), marker class (index+1), tx, ty, tz].
|
651 |
# We use index+1 because the unlabeled markers will use index 0 for both classes.
|
652 |
+
[actor_idx + 1, marker_class, *self.get_worldspace(m, time, apply_transform)]
|
653 |
for marker_class, (marker_name, m) in enumerate(
|
654 |
self.markers[actor_idx].items(), start=1
|
655 |
)
|
|
|
659 |
if self.is_kf_present(m, time)
|
660 |
)
|
661 |
|
|
|
|
|
|
|
|
|
|
|
|
|
662 |
# If the data is extremely noisy, it might have only a few labeled markers and a lot of unlabeled markers.
|
663 |
# The returned point cloud is not allowed to be bigger than the maximum size (self.pc_size),
|
664 |
# so return the cloud as a np array that cuts off any excessive markers.
|
665 |
return np.array(cloud)[:self.pc_size]
|
666 |
|
667 |
+
def get_tsc(self, apply_transform: bool = True) -> np.array:
|
668 |
"""
|
669 |
Convenience method that calls self.get_sparse_cloud() for all frames in the frame range
|
670 |
and returns the combined result.
|
671 |
+
:param apply_transform: `bool` Whether to transform the translation or not.
|
672 |
:return: `np.array` that contains a sparse cloud for each frame in the frame range.
|
673 |
"""
|
674 |
+
return np.array([self.get_sc(f, apply_transform) for f in self.get_frame_range()])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
675 |
|
676 |
+
def get_tdc(self, shuffle: bool = False,
|
677 |
+
r: Union[int, Tuple[int, int]] = None,
|
678 |
+
apply_transform: bool = True) -> np.array:
|
679 |
"""
|
680 |
For each frame in the frame range, collects the point cloud that is present in the file.
|
681 |
Then it creates a ghost cloud of random markers that are treated as unlabeled markers,
|
682 |
and adds them together to create a dense cloud whose shape is always (self.pc_size, 5).
|
683 |
Optionally shuffles this dense cloud before adding it to the final list.
|
684 |
+
:param shuffle: If `True`, shuffles the dense point cloud of each frame.
|
685 |
+
:param r: tuple of `int` that indicates the frame range to get. Default is None,
|
686 |
+
resulting in the animation frame range.
|
687 |
+
:param apply_transform: `bool` Whether to transform the translation or not.
|
688 |
:return: `np.array` that contains a dense point cloud for each frame,
|
689 |
with a shape of (self.num_frames, self.pc_size, 5).
|
690 |
"""
|
691 |
+
|
692 |
clouds = []
|
693 |
+
|
694 |
+
# If r is one int, use 0 as start frame.
|
695 |
+
if isinstance(r, int):
|
696 |
+
r = list(range(r))
|
697 |
+
# If r is two ints, use that as specific frame range.
|
698 |
+
elif isinstance(r, tuple) and len(r) >= 2:
|
699 |
+
r = list(range(r[0], r[1]))
|
700 |
+
# If r is empty, use the animation frame range.
|
701 |
+
else:
|
702 |
+
r = self.get_frame_range()
|
703 |
+
|
704 |
+
for frame in r:
|
705 |
+
cloud = self.get_sc(frame, apply_transform)
|
706 |
missing = self.pc_size - cloud.shape[0]
|
707 |
|
708 |
# Only bother creating ghost markers if there are any missing rows.
|
709 |
+
# If we need to add ghost markers, add them before the existing cloud,
|
710 |
+
# so that the cloud will remain a sorted array regarding the actor and marker classes.
|
711 |
if missing > 0:
|
712 |
ghost_cloud = make_ghost_markers(missing)
|
713 |
+
cloud = np.vstack([ghost_cloud, cloud])
|
714 |
|
715 |
# Shuffle the rows if needed. Because each row contains all dependent and independent variables,
|
716 |
# shuffling won't mess up the labels.
|
|
|
721 |
|
722 |
return np.array(clouds)
|
723 |
|
724 |
+
def split_tdc(self, cloud: np.array = None, shuffle: bool = False, apply_transform: bool = True) \
|
725 |
-> Tuple[np.array, np.array, np.array]:
|
726 |
"""
|
727 |
Splits a timeline dense cloud with shape (self.num_frames, self.pc_size, 5) into 3 different
|
|
|
732 |
:param cloud: `np.array` of shape (self.num_frames, self.pc_size, 5) that contains a dense point cloud
|
733 |
(self.pc_size, 5) per frame in the frame range.
|
734 |
:param shuffle: `bool` whether to shuffle the generated cloud if no cloud was given.
|
735 |
+
:param apply_transform: `bool` Whether to transform the translation or not.
|
736 |
:return: Return tuple of `np.array` as (actor classes, marker classes, translation vectors).
|
737 |
"""
|
738 |
if cloud is None:
|
739 |
+
cloud = self.get_tdc(shuffle, apply_transform=apply_transform)
|
740 |
|
741 |
assert cloud.shape[1] == 1000, f"Dense cloud doesn't have enough points. {cloud.shape[1]}/1000."
|
742 |
assert cloud.shape[2] == 5, f"Dense cloud is missing columns: {cloud.shape[2]}/5."
|
|
|
750 |
:param c: `float` actor class index.
|
751 |
:return: `str` actor name.
|
752 |
"""
|
753 |
+
return 'UNLABELED' if int(c) == 0 else self.actor_names[int(c) - 1]
|
754 |
|
755 |
def convert_class_to_marker(self, c: float = 0):
|
756 |
"""
|
|
|
758 |
:param c: `float` marker class index.
|
759 |
:return: `str` marker name.
|
760 |
"""
|
761 |
+
return 'UNLABELED' if int(c) == 0 else self.marker_names[int(c) - 1]
|
762 |
|
763 |
def export(self, t: str = 'csv', output_file: Path = None) -> Union[bytes, Path]:
|
764 |
# Get the dataframe with all animation data.
|
|
|
768 |
return df.to_csv(index=False).encode('utf-8')
|
769 |
|
770 |
if output_file is None:
|
771 |
+
output_file = self.input_fbx.with_suffix('.csv')
|
772 |
|
773 |
if output_file.suffix != '.csv':
|
774 |
raise ValueError(f'{output_file} needs to be a .csv file.')
|
|
|
776 |
df.to_csv(output_file, index=False)
|
777 |
return output_file
|
778 |
|
779 |
+
def export_fbx(self, output_file: Path = None) -> bool:
|
780 |
+
"""
|
781 |
+
Exports the entire scene to the output file. If output_file is None, it uses the automatic output_fbx property.
|
782 |
+
:param output_file: `Path` path to an FBX file to export the contents to.
|
783 |
+
:return: `True` if successful, otherwise `False`.
|
784 |
+
"""
|
785 |
+
if output_file is None:
|
786 |
+
output_file = self.output_fbx
|
787 |
+
|
788 |
+
# Create an exporter using the manager
|
789 |
+
exporter = fbx.FbxExporter.Create(self.manager, "")
|
790 |
+
|
791 |
+
# Initialize the exporter with the output file path
|
792 |
+
result = exporter.Initialize(str(output_file))
|
793 |
+
if not result:
|
794 |
+
print(f"Failed to initialize the exporter for file '{output_file}'.")
|
795 |
+
return False
|
796 |
+
|
797 |
+
# Export the scene
|
798 |
+
result = exporter.Export(self.scene)
|
799 |
+
if not result:
|
800 |
+
print(f"Failed to export the scene to file '{output_file}'.")
|
801 |
+
return False
|
802 |
+
|
803 |
+
# Clean up the manager and exporter
|
804 |
+
exporter.Destroy()
|
805 |
+
|
806 |
+
return True
|
807 |
+
|
808 |
+
def remove_node(self, node: fbx.FbxNode, recursive: bool = False) -> bool:
|
809 |
+
"""
|
810 |
+
Removes a node by reference from the scene.
|
811 |
+
:param node: `fbx.FbxNode` to remove.
|
812 |
+
:param recursive: `bool` Apply deletion recursively.
|
813 |
+
:return: True if success.
|
814 |
+
"""
|
815 |
+
if recursive:
|
816 |
+
children = [node.GetChild(c) for c in range(node.GetChildCount())]
|
817 |
+
for child in children:
|
818 |
+
self.remove_node(child, True)
|
819 |
+
# Disconnect the marker node from its parent
|
820 |
+
node.GetParent().RemoveChild(node)
|
821 |
+
|
822 |
+
# Remove the marker node from the scene
|
823 |
+
self.scene.RemoveNode(node)
|
824 |
+
|
825 |
+
return True
|
826 |
+
|
827 |
+
def remove_unlabeled_markers(self) -> None:
|
828 |
+
"""
|
829 |
+
Uses self.remove_node() to delete all unlabeled markers from the scene.
|
830 |
+
"""
|
831 |
+
for m in self.unlabeled_markers:
|
832 |
+
self.remove_node(m)
|
833 |
+
|
834 |
+
self.remove_node(self.unlabeled_markers_parent)
|
835 |
+
|
836 |
+
def remove_system(self) -> None:
|
837 |
+
system_node = self.get_parent_node_by_name('System')
|
838 |
+
self.remove_node(system_node, recursive=True)
|
839 |
+
|
840 |
+
def cleanup(self) -> None:
|
841 |
+
self.remove_unlabeled_markers()
|
842 |
+
self.remove_system()
|
843 |
+
|
844 |
+
def replace_keyframes_per_marker(self, marker: fbx.FbxNode, marker_keys: dict) -> None:
|
845 |
+
parent = marker.GetParent()
|
846 |
+
for axis in ['X', 'Y', 'Z']:
|
847 |
+
curve = marker.LclTranslation.GetCurve(self.anim_layer, axis)
|
848 |
+
curve.KeyModifyBegin()
|
849 |
+
if curve is not None:
|
850 |
+
curve.KeyClear()
|
851 |
+
|
852 |
+
for frame, world_translation in marker_keys.items():
|
853 |
+
local_translation = world_to_local_translation(world_translation, parent)
|
854 |
+
|
855 |
+
for axis_idx in range(3):
|
856 |
+
create_keyframe(curve, frame, local_translation[axis_idx])
|
857 |
+
curve.KeyModifyEnd()
|
858 |
+
|
859 |
+
def replace_keyframes_per_actor(self, actor: int, actor_keys: dict) -> None:
|
860 |
+
for marker_class, (marker_name, marker) in enumerate(self.markers[actor].items(), start=1):
|
861 |
+
self.replace_keyframes_per_marker(marker, actor_keys[marker_class])
|
862 |
+
|
863 |
+
def replace_keyframes_for_all_actors(self, key_dict: dict) -> None:
|
864 |
+
for actor_idx in range(self.actor_count):
|
865 |
+
self.replace_keyframes_per_actor(actor_idx, key_dict[actor_idx + 1])
|
866 |
+
|
867 |
|
868 |
# d = FBXContainer(Path('G:/Firestorm/mocap-ai/data/fbx/dowg/TAKE_01+1_ALL_001.fbx'))
|
869 |
+
# t = fbx.FbxTime()
|
870 |
+
# t.SetFrame(0)
|
871 |
+
#
|
872 |
+
# one = list(d.unlabeled_markers[0].EvaluateGlobalTransform(t).GetT())
|
873 |
+
# # one = np.array([
|
874 |
+
# # [one.Get(i, j) for j in range(4)] for i in range(4)
|
875 |
+
# # ])
|
876 |
+
# two = world_to_local_translation(d.unlabeled_markers[0], d.unlabeled_markers_parent, 0)[3, :3]
|
877 |
+
# print(one)
|
878 |
+
# print(two)
|
879 |
+
# for e in zip(one, two):
|
880 |
+
# print(e)
|
881 |
+
# f = d.get_sc(0, False)
|
882 |
+
# f = f[f[:, 1] == 1.]
|
883 |
+
# print(f)
|
884 |
+
# print(d.convert_class_to_actor(f[0]))
|
885 |
+
# print(d.convert_class_to_marker(f[1]))
|
886 |
+
# print(f[2:5])
|
887 |
+
# lt = world_to_local_translation(f[2:5], d.unlabeled_markers_parent)
|
888 |
+
# print(lt)
|
889 |
+
|
890 |
+
# for u in sc:
|
891 |
+
# print(d.convert_class_to_actor(u[0]), d.convert_class_to_marker(u[1]))
|
892 |
+
# train_cloud = d.get_tdc()
|
893 |
+
# train_actors, train_markers, train_X = d.split_tdc(train_cloud)
|
894 |
+
# test_cloud = d.get_tdc(r=1, apply_transform=False)
|
895 |
+
# for row in test_cloud:
|
896 |
+
# for m in row[:5]:
|
897 |
+
# t = m[2:5]
|
898 |
+
# print(t)
|
899 |
+
# lt = world_to_local_translation()
|
900 |
+
# print()
|
901 |
+
# print(row[:5, 2:5])
|
902 |
+
# print(test_cloud.shape)
|
903 |
+
# print(test_cloud[0, :, :2])
|
904 |
+
# print(d.convert_class_to_actor(test_cloud[0, 0, 0]))
|
905 |
+
# print(d.convert_class_to_marker(test_cloud[0, 0, 1]))
|
906 |
+
# test_actors, test_markers, test_X = d.split_tdc(test_cloud, apply_transform=False)
|
907 |
+
# Predict...
|
908 |
+
# merged_preds = merge_tdc(train_actors, train_markers, test_X, ordered=False)
|
909 |
+
# di = timeline_cloud_to_dict(test_cloud)
|
910 |
+
|
911 |
+
# d.replace_keyframes_for_all_actors(di)
|
912 |
+
# d.cleanup()
|
913 |
+
# d.export_fbx(Path('G:/Firestorm/mocap-ai/data/fbx/export/TAKE_01+1_ALL_001.fbx'))
|