Spaces:
Sleeping
Sleeping
File size: 1,804 Bytes
31af2b2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
import requests
from smolagents import tool
import openai
import base64
@tool
def analyse_image(image_url: str) -> str:
"""
analyse the provided image, and return a description or transcription of the contents.
Args:
image_url (str): The URL of the image to be analysed. Usually with an image extension like png, jpg, etc.
Returns:
str: description or transcription of the contents of the provided image
"""
# some security:
if "https://agents-course-unit4-scoring.hf.space" not in image_url:
return "the requested URL is not whitelisted, refusing to fetch data"
resp = requests.get(image_url)
if resp.status_code != 200:
return f"failed to fetch the requested image: (status={resp.status_code})\n{resp.text}"
mime = resp.headers.get("content-type")
# todo filer mimetypes for security and correctness
image_bytes = base64.b64encode(resp.content).decode("utf-8")
# Create the message to GPT-4o (vision)
response = openai.chat.completions.create(
model="gpt-4o",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "Please analyze the contents of this image. Provide a short (two sentence) description of the contents, and then output your analysis. The analysis should be in the most appropriate format. e.g. if the image is a document, maybe transcription is best. if it's a picture, describe the contents in detail. if it's a chessboard, outline the situation in chess notation. etc. "},
{"type": "image_url", "image_url": {"url": f"data:{mime};base64," + image_bytes}}
]
}
],
max_tokens=500,
)
return response.choices[0].message.content
|