app.py

import gradio as gr
from llama_cpp import Llama
from huggingface_hub import hf_hub_download

hf_hub_download(repo_id="mmnga/rinna-llama-3-youko-8b-gguf", filename="rinna-llama-3-youko-8b-Q4_K_M.gguf", repo_type="model", local_dir=".")
llm = Llama(model_path="./rinna-llama-3-youko-8b-Q4_K_M.gguf", n_ctx=128, n_threads=2, n_threads_batch=2, verbose=False)

def execute(name):
    output = llm(
        prompt,max_tokens=128,echo=True,
    )
    return output

demo = gr.Interface(fn=execute, inputs="text", outputs="text")
demo.launch()