nock2 commited on
Commit
9369d0f
Β·
verified Β·
1 Parent(s): 3a62741

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +45 -15
app.py CHANGED
@@ -6,6 +6,9 @@ from einops import rearrange
6
  import gradio as gr
7
  from stable_audio_tools import get_pretrained_model
8
  from stable_audio_tools.inference.generation import generate_diffusion_cond
 
 
 
9
 
10
  # Authenticate
11
  token = os.getenv("HUGGINGFACE_TOKEN")
@@ -13,19 +16,25 @@ if not token:
13
  raise RuntimeError("HUGGINGFACE_TOKEN not set")
14
  login(token=token, add_to_git_credential=False)
15
 
16
- # Load model
17
  device = "cuda" if torch.cuda.is_available() else "cpu"
18
- model, config = get_pretrained_model("stabilityai/stable-audio-open-small")
19
- model = model.to(device)
20
- sample_rate = config["sample_rate"]
21
- sample_size = config["sample_size"]
22
 
23
- # Inference function
 
 
 
 
 
 
24
  def generate_audio(prompt):
25
  conditioning = [{"prompt": prompt, "seconds_total": 11}]
26
  with torch.no_grad():
27
  output = generate_diffusion_cond(
28
- model,
29
  steps=8,
30
  conditioning=conditioning,
31
  sample_size=sample_size,
@@ -37,24 +46,45 @@ def generate_audio(prompt):
37
  torchaudio.save(path, output, sample_rate)
38
  return path
39
 
40
- # πŸŒ€ Hot Prompt Club UI
41
- gr.Interface(
42
- fn=generate_audio,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  inputs=gr.Textbox(
44
  label="🎀 Prompt your sonic art here",
45
  placeholder="e.g. 'drunk driving with mario and yung lean'"
46
  ),
47
- outputs=gr.Audio(
48
- type="filepath",
49
- label="🧠 Generated Audio"
50
  ),
51
  title='🌐 Hot Prompts in Your Area: "My Husband Is Dead"',
52
- description="Enter a fun sound idea for music art.",
53
  examples=[
54
  "ghosts peeing in a server room",
55
  "tech startup boss villain entrance music",
56
  "AI doing acid in a technofeudalist dystopia"
57
  ],
58
  css="style.css"
59
- ).launch()
60
 
 
 
6
  import gradio as gr
7
  from stable_audio_tools import get_pretrained_model
8
  from stable_audio_tools.inference.generation import generate_diffusion_cond
9
+ from diffusers import DiffusionPipeline
10
+ from PIL import Image
11
+ from moviepy.editor import AudioFileClip, ImageClip
12
 
13
  # Authenticate
14
  token = os.getenv("HUGGINGFACE_TOKEN")
 
16
  raise RuntimeError("HUGGINGFACE_TOKEN not set")
17
  login(token=token, add_to_git_credential=False)
18
 
19
+ # Load audio model
20
  device = "cuda" if torch.cuda.is_available() else "cpu"
21
+ audio_model, audio_config = get_pretrained_model("stabilityai/stable-audio-open-small")
22
+ audio_model = audio_model.to(device)
23
+ sample_rate = audio_config["sample_rate"]
24
+ sample_size = audio_config["sample_size"]
25
 
26
+ # Load image model (Kandinsky)
27
+ image_pipe = DiffusionPipeline.from_pretrained(
28
+ "kandinsky-community/kandinsky-3",
29
+ torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
30
+ ).to(device)
31
+
32
+ # Generate audio
33
  def generate_audio(prompt):
34
  conditioning = [{"prompt": prompt, "seconds_total": 11}]
35
  with torch.no_grad():
36
  output = generate_diffusion_cond(
37
+ audio_model,
38
  steps=8,
39
  conditioning=conditioning,
40
  sample_size=sample_size,
 
46
  torchaudio.save(path, output, sample_rate)
47
  return path
48
 
49
+ # Generate image
50
+ def generate_image(prompt):
51
+ image = image_pipe(prompt=prompt, height=500, width=500).images[0]
52
+ image_path = "output.png"
53
+ image.save(image_path)
54
+ return image_path
55
+
56
+ # Combine audio + image into mp4
57
+ def combine_to_video(image_path, audio_path, output_path="output.mp4"):
58
+ clip = ImageClip(image_path).set_duration(12).set_audio(AudioFileClip(audio_path))
59
+ clip = clip.set_fps(1)
60
+ clip.write_videofile(output_path, codec="libx264", audio_codec="aac", fps=1)
61
+ return output_path
62
+
63
+ # Unified generation
64
+ def generate_av(prompt):
65
+ audio_path = generate_audio(prompt)
66
+ image_path = generate_image(prompt)
67
+ video_path = combine_to_video(image_path, audio_path)
68
+ return video_path
69
+
70
+ # UI
71
+ interface = gr.Interface(
72
+ fn=generate_av,
73
  inputs=gr.Textbox(
74
  label="🎀 Prompt your sonic art here",
75
  placeholder="e.g. 'drunk driving with mario and yung lean'"
76
  ),
77
+ outputs=gr.Video(
78
+ label="🧠 Generated Audiovisual Clip"
 
79
  ),
80
  title='🌐 Hot Prompts in Your Area: "My Husband Is Dead"',
81
+ description="Enter a fun sound idea for music art. Returns a synced image + audio mp4.",
82
  examples=[
83
  "ghosts peeing in a server room",
84
  "tech startup boss villain entrance music",
85
  "AI doing acid in a technofeudalist dystopia"
86
  ],
87
  css="style.css"
88
+ )
89
 
90
+ interface.launch()