import xml.etree.ElementTree as ET
import re
import numpy as np
from PIL import Image, ImageDraw, ImageFont
from io import BytesIO
import base64
# =========================
# ===== Grid related ======
# =========================
def create_grid_image(res=50, cell_size=12, header_size=12):
# Define the size of the grid
rows = res
cols = res
img_width = (cols + 1) * cell_size
img_height = (rows + 1) * cell_size
# Create a new image with a white background
img = Image.new('RGB', (img_width, img_height), 'white')
draw = ImageDraw.Draw(img)
# Load a font
try:
font = ImageFont.truetype("arial.ttf", header_size*0.85)
except IOError:
font = ImageFont.load_default()
# Draw the headers
for j in range(cols):
# Draw column header (letters)
text = str(j +1)
text_bbox = draw.textbbox((0, 0), text, font=font)
text_width = text_bbox[2] - text_bbox[0]
text_height = text_bbox[3] - text_bbox[1]
text_x = (j + 1) * cell_size + (cell_size - text_width) / 2
text_y = img_height - cell_size # - (cell_size - text_height) / 2
draw.text((text_x, text_y), text, fill="black", font=font)
for i in range(rows):
# Draw row header (numbers)
text = str(rows - i)
text_bbox = draw.textbbox((0, 0), text, font=font)
text_width = text_bbox[2] - text_bbox[0]
text_height = text_bbox[3] - text_bbox[1]
text_x = (cell_size - text_width) / 2
text_y = i * cell_size + (cell_size - text_height) / 2 - 0.2*text_height
draw.text((text_x, text_y), text, fill="black", font=font)
# Draw the grid
i = 1
draw.line([(i * cell_size, 0), (i * cell_size, img_height)], fill="black")
# Horizontal lines
draw.line([(0, img_height - cell_size), (img_width, img_height - cell_size)], fill="black")
positions={}
# Draw the grid
for i in range(rows)[::-1]:
for j in range(cols):
# Draw cell border
if j == 0:
draw.rectangle([(j + 0) * cell_size, (i + 0) * cell_size, (j + 1) * cell_size, (i + 1) * cell_size], outline="black")
if i == rows - 1:
draw.rectangle([(j + 0) * cell_size, (i + 1) * cell_size, (j + 1) * cell_size, (i + 2) * cell_size], outline="black")
# Calculate the position of the text
text = f"x{j + 1}y{i + 1}"
text_bbox = draw.textbbox((0, 0), text, font=font)
text_width = text_bbox[2] - text_bbox[0]
text_height = text_bbox[3] - text_bbox[1]
text_x = (j + 1) * cell_size + (cell_size - text_width) / 2
text_y = (i + 1) * cell_size + (cell_size - text_height) / 2
center_y = int(img_height - cell_size - (i * cell_size) - cell_size / 2)
center_x = int(j * cell_size + cell_size / 2 + cell_size)
positions[text] = (center_x, center_y)
return img, positions
def cells_to_pixels(res=50, cell_size=12, header_size=12):
# Define the size of the grid
rows = res
cols = res
img_width = (cols + 1) * cell_size
img_height = (rows + 1) * cell_size
positions={}
# Draw the grid
for i in range(rows)[::-1]:
for j in range(cols):
# Calculate the position of the text
text = f"x{j + 1}y{i + 1}"
center_y = int(img_height - cell_size - (i * cell_size) - cell_size / 2)
center_x = int(j * cell_size + cell_size / 2 + cell_size)
positions[text] = (center_x, center_y)
return positions
# =========================
# ===== LLM related =======
# =========================
def image_to_str(image: Image):
buffer = BytesIO()
image.save(buffer, format="JPEG")
buffer.seek(0)
image = base64.b64encode(buffer.read()).decode('utf-8')
return image
# =================================
# ===== SVG process related =======
# =================================
def bezier_point(P, t):
"""Calculate a point on the Bézier curve for a given t."""
return (1-t)**3 * P[0] + 3*(1-t)**2 * t * P[1] + 3*(1-t) * t**2 * P[2] + t**3 * P[3]
def estimate_bezier_control_points_helper(sampled_points, t_values):
n = len(sampled_points)
if n == 1:
# Linear interpolation: the control points are simply the two points
P0 = np.array(sampled_points[0])
P1 = np.array(sampled_points[0]).astype(np.float64) + 0.0001
return np.array([P0, P1])
if n == 2:
# Linear interpolation: the control points are simply the two points
P0 = np.array(sampled_points[0])
P1 = np.array(sampled_points[1])
return np.array([P0, P1])
if n > len(t_values):
t_values = np.linespace(0,1,n)
elif n == 3:
# Quadratic Bézier curve: we need to solve for three control points
A = np.zeros((n, 3))
for i in range(n):
t = t_values[i]
A[i, 0] = (1-t)**2
A[i, 1] = 2*(1-t)*t
A[i, 2] = t**2
# Points (flattened)
B = np.array(sampled_points).reshape(-1, 2) # Assuming 2D points
# Solve the system (least squares)
P = np.linalg.lstsq(A, B, rcond=None)[0]
return P
# Matrix A
A = np.zeros((n, 4))
for i in range(n):
t = t_values[i]
A[i, 0] = (1-t)**3
A[i, 1] = 3*(1-t)**2 * t
A[i, 2] = 3*(1-t) * t**2
A[i, 3] = t**3
# Points (flattened)
B = np.array(sampled_points).reshape(-1, 2) # Assuming 2D points
# Solve the system (least squares)
P = np.linalg.lstsq(A, B, rcond=None)[0]
return P
def estimate_bezier_control_points( sampled_points, t_values):
if len(sampled_points) != len(t_values):
t_values = np.linspace(0,1, len(sampled_points))
P = estimate_bezier_control_points_helper(sampled_points, t_values)
if len(sampled_points) > 4:
# Calculate the mean squared error between sampled points and the fitted Bézier curve.
errors = []
for i, t in enumerate(t_values):
B_t = bezier_point(P, t)
error = np.linalg.norm(B_t - sampled_points[i])
errors.append(error)
error = np.mean(errors)
if error > 5 and len(sampled_points) >= 7:
mid = len(sampled_points) // 2
left_sampled_points = sampled_points[:mid+1]
right_sampled_points = sampled_points[mid:]
left_t_values = np.array(t_values[:mid+1])
right_t_values = np.array(t_values[mid:])
if len(left_sampled_points) == 3: # this applies in case we have 7 points
left_sampled_points.append(right_sampled_points[0])
left_t_values.append(right_t_values[0])
# Normalize t_values for each segment
left_t_values = (left_t_values - left_t_values[0]) / (left_t_values[-1] - left_t_values[0])
right_t_values = (right_t_values - right_t_values[0]) / (right_t_values[-1] - right_t_values[0])
# Recursively fit curves to each segment
P_left = estimate_bezier_control_points_helper(left_sampled_points, left_t_values)
P_right = estimate_bezier_control_points_helper(right_sampled_points, right_t_values)
P_right[0] = P_left[-1] # I added this to have the long strokes look more connected
return [P_left, P_right]
return [P]
def get_control_points(strokes_all, t_values_all, cells_to_pixels_map):
net_points = []
for j in range(len(strokes_all)):
sampled_cells = strokes_all[j]
t_values = t_values_all[j]
sampled_points = []
for cell in sampled_cells:
y,x = cells_to_pixels_map[cell]
sampled_points.append([y,x])
points_lst = estimate_bezier_control_points(sampled_points, t_values)
net_points.append(points_lst)
return net_points
def get_control_points_single_stroke(strokes_all, t_values_all, cells_to_pixels_map):
sampled_points = []
for cell in strokes_all:
y,x = cells_to_pixels_map[cell]
sampled_points.append([y,x])
points_lst = estimate_bezier_control_points(sampled_points, t_values_all)
return points_lst
def create_svg_path_data(control_points):
# Start the path with 'M' for the first point
# print("control_points", control_points[0])
path_data = 'M ' + np.array2string(np.array(control_points[0]), formatter={'float_kind':lambda x: "%.2f" % x}, separator=' ')[1:-1]
# Add 'L' for each subsequent point
# check if point
if len(control_points) == 1:
path_data += ' '
# check if line
elif len(control_points) == 2:
path_data += ' L '
# check if quadratic
elif len(control_points) == 3:
path_data += ' Q '
# check if cubic
elif len(control_points) == 4:
path_data += ' C '
# path_data += ' C '
for point in control_points[1:]:
# print("pt", point[0], point[1])
path_data += str(point[0]) + " " + str(point[1]) + " "
# Return the complete 'd' attribute string
return path_data
def format_svg(all_control_points, dim, stroke_width):
svg_width, svg_height = dim
sketch_text_svg = f""""
return sketch_text_svg
def format_svg_single_stroke(group, dim, stroke_width, stroke_counter, stroke_color="black"):
sketch_text_svg = ""
gropu_text = f"""\n"""
for sub_path_cp in group:
path_data = create_svg_path_data(sub_path_cp)
gropu_text += f"""\n"""
gropu_text += "\n"
sketch_text_svg += gropu_text
return sketch_text_svg
# Note that this parse only the *first* part in the text in which you have the tags.
def parse_xml_string(llm_output, res):
strokes_start_marker = ""
strokes_end_marker = ""
# Find the start and end indices of the JSON string
start_index = llm_output.find(strokes_start_marker)
if start_index != -1:
# start_index += len(strokes_start_marker) # Move past the marker
end_index = llm_output.find(strokes_end_marker, start_index)
else:
return None # XML markers not found
if end_index == -1:
return None # End marker not found
# Extract the JSON string
strokes_str = llm_output[start_index:end_index + len(strokes_end_marker)].strip()#[:-1]
xml_str = f"{strokes_str}"
# Parse the XML string
root = ET.fromstring(xml_str)
# Initialize lists to hold strokes and t_values
strokes_list = "[\n"
t_values_list = "[\n"
# Iterate over all the strokes
for stroke in root.find('strokes'):
# Extract points and clean them up
points_text = stroke.find('points').text
# Extract t_values and convert them to float
t_values_text = stroke.find('t_values').text
# Append to the lists
strokes_list += f"[{points_text}],\n"
t_values_list += f"[{t_values_text}],\n"
strokes_list = re.sub(r'\d+', lambda x: str(min(int(x.group()), res)), strokes_list)
strokes_list = re.sub(r'\d+', lambda x: str(max(int(x.group()), 1)), strokes_list)
strokes_list += "]"
t_values_list += "]"
return strokes_list, t_values_list
def parse_xml_string_single_stroke(llm_output, res, stroke_counter):
strokes_start_marker = f""
strokes_end_marker = f""
# Find the start and end indices of the JSON string
start_index = llm_output.find(strokes_start_marker)
if start_index != -1:
# start_index += len(strokes_start_marker) # Move past the marker
end_index = llm_output.find(strokes_end_marker, start_index)
else:
return None # XML markers not found
if end_index == -1:
return None # End marker not found
# Extract the JSON string
strokes_str = llm_output[start_index:end_index + len(strokes_end_marker)].strip()#[:-1]
xml_str = f"{strokes_str}"
# Parse the XML string
root = ET.fromstring(xml_str)
# Iterate over all the strokes
stroke = root.find(f"s{stroke_counter}")
points_text = stroke.find('points').text
# Extract t_values and convert them to float
t_values_text = stroke.find('t_values').text
# Append to the lists
strokes_list = f"[{points_text}]"
t_values_list = f"[{t_values_text}]"
strokes_list = re.sub(r'\d+', lambda x: str(min(int(x.group()), res)), strokes_list)
strokes_list = re.sub(r'\d+', lambda x: str(max(int(x.group()), 1)), strokes_list)
return strokes_list, t_values_list
# =====================================
# ===== Collaborative Sketching =======
# =====================================
def get_cur_stroke_text(stroke_counter, llm_output):
start_marker = f""
end_marker = f""
# Find the start and end indices of the JSON string
start_index = llm_output.find(start_marker)
if start_index != -1:
# start_index += len(strokes_start_marker) # Move past the marker
end_index = llm_output.find(end_marker, start_index)
else:
return "" # XML markers not found
if end_index == -1:
return "" # End marker not found
# Extract the JSON string
strokes_str = llm_output[start_index:end_index + len(end_marker)].strip()#[:-1]
return strokes_str