Async calls to model

app.py

@@ -1,10 +1,19 @@
 import gradio as gr
 from transformers import pipeline
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+
+executor = ThreadPoolExecutor()
 
 model_pipeline = pipeline("text2text-generation", model="tribler/dsi-search-on-toy-dataset")
 
-def process_query(query):
-    results = model_pipeline(query, max_length=60)
+async def async_model_pipeline(query):
+    loop = asyncio.get_running_loop()
+    results = await loop.run_in_executor(executor, model_pipeline, query, {"max_length": 60})
+    return results
+
+async def process_query(query):
+    results = await async_model_pipeline(query)
     result_text = results[0]['generated_text'].strip()
     if result_text.startswith("http"):
         youtube_id = result_text.split('watch?v=')[-1]
@@ -13,8 +22,8 @@ def process_query(query):
     elif result_text.startswith("magnet"):
         return gr.HTML(f'<a href="{result_text}" target="_blank">{result_text}</a>')
     else:
-      bitcoin_logo_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/4/46/Bitcoin.svg/800px-Bitcoin.svg.png"
-      return gr.Textbox(f'<div style="display:flex;align-items:center;"><img src="{bitcoin_logo_url}" alt="Bitcoin Logo" style="width:20px;height:20px;margin-right:5px;"><span>{result_text}</span></div>')
+        bitcoin_logo_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/4/46/Bitcoin.svg/800px-Bitcoin.svg.png"
+        return gr.Textbox(f'<div style="display:flex;align-items:center;"><img src="{bitcoin_logo_url}" alt="Bitcoin Logo" style="width:20px;height:20px;margin-right:5px;"><span>{result_text}</span></div>')
 
 interface = gr.Interface(fn=process_query,
                           inputs=gr.Textbox(label="Query"),
@@ -22,9 +31,9 @@ interface = gr.Interface(fn=process_query,
                           title="Search Interface",
                           submit_btn="Find",
                           description="Search for movie trailers, music torrents, and bitcoin wallet addresses.",
-                          article="This interface searches a toy dataset and returns a YouTube URL, magnet link, or Bitcoin wallet address. Generative AI is slowly starting to become capable of content discovery, relevance ranking, movie creation, financial transactions, market trading, and market matchmaking. The recent advent of end-to-end generative architectures might have disruptive effect on various industries. Social media platforms, movie industry, search engines, and financial intermediaries could *in principle* be replaced with fully decentralised alternatives (think Bitcoin and Bittorrent level decentralisation). This means a shift of power towards ordinary Internet citizens. See our De-DSI paper at EuroMLSys 2024 for scientific details. Please consider contributing and joining our community around the International Workshop on Distributed Infrastructure for Common Good. ",
-                          examples=[["spider man"], ["oceans 13"], ["sister starlight"], ["bitcoin address of xileno"]])
+                          article="This interface searches a toy dataset and returns a YouTube URL, magnet link, or Bitcoin wallet address. Generative AI is slowly starting to become capable of content discovery, relevance ranking, movie creation, financial transactions, market trading, and market matchmaking. The recent advent of end-to-end generative architectures might have disruptive effect on various industries. Social media platforms, movie industry, search engines, and financial intermediaries could *in principle* be replaced with fully decentralised alternatives (think Bitcoin and Bittorrent level decentralisation). This means a shift of power towards ordinary Internet citizens. See our De-DSI paper at EuroMLSys 2024 for scientific details. Please consider contributing and joining our community around the International Workshop on Distributed Infrastructure for Common Good.",
+                          examples=[["spider man"], ["oceans 13"], ["sister starlight"], ["bitcoin address of xileno"]],
+                          enable_queue=True)
 
 if __name__ == "__main__":
     interface.launch()
-