Upload 6 files

README.md

@@ -1,12 +1,38 @@
 ---
-title: Dermoscopic Image Generation
-emoji: 📈
-colorFrom: blue
-colorTo: blue
+title: Glide Text2im
+emoji: 📊
+colorFrom: purple
+colorTo: gray
 sdk: gradio
-sdk_version: 3.15.0
 app_file: app.py
 pinned: false
+duplicated_from: valhalla/glide-text2im
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Configuration
+
+`title`: _string_  
+Display title for the Space
+
+`emoji`: _string_  
+Space emoji (emoji-only character allowed)
+
+`colorFrom`: _string_  
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`colorTo`: _string_  
+Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
+
+`sdk`: _string_  
+Can be either `gradio` or `streamlit`
+
+`sdk_version` : _string_  
+Only applicable for `streamlit` SDK.  
+See [doc](https://hf.co/docs/hub/spaces) for more info on supported versions.
+
+`app_file`: _string_  
+Path to your main application file (which contains either `gradio` or `streamlit` Python code).  
+Path is relative to the root of the repository.
+
+`pinned`: _boolean_  
+Whether the Space stays on top of your list.

app.py

@@ -0,0 +1,255 @@
+
+import os
+os.system('pip install -e .')
+import gradio as gr
+
+import base64
+from io import BytesIO
+# from fastapi import FastAPI
+
+from PIL import Image
+import torch as th
+
+from glide_text2im.download import load_checkpoint
+from glide_text2im.model_creation import (
+    create_model_and_diffusion,
+    model_and_diffusion_defaults,
+    model_and_diffusion_defaults_upsampler
+)
+
+# print("Loading models...")
+# app = FastAPI()
+
+# This notebook supports both CPU and GPU.
+# On CPU, generating one sample may take on the order of 20 minutes.
+# On a GPU, it should be under a minute.
+
+has_cuda = th.cuda.is_available()
+device = th.device('cpu' if not has_cuda else 'cuda')
+
+# # Create base model.
+# options = model_and_diffusion_defaults()
+# options['use_fp16'] = has_cuda
+# options['timestep_respacing'] = '40' # use 100 diffusion steps for fast sampling (Previous it was 100)
+# model, diffusion = create_model_and_diffusion(**options)
+# model.eval()
+# if has_cuda:
+#     model.convert_to_fp16()
+# model.to(device)
+# # model.load_state_dict(load_checkpoint('base', device))
+# model.load_state_dict(th.load("base.pt", map_location=device))
+# print('total base parameters', sum(x.numel() for x in model.parameters()))
+
+# # Create upsampler model.
+# options_up = model_and_diffusion_defaults_upsampler()
+# options_up['use_fp16'] = has_cuda
+# options_up['timestep_respacing'] = 'fast27' # use 27 diffusion steps for very fast sampling
+# model_up, diffusion_up = create_model_and_diffusion(**options_up)
+# model_up.eval()
+# if has_cuda:
+#     model_up.convert_to_fp16()
+# model_up.to(device)
+# # model_up.load_state_dict(load_checkpoint('upsample', device))
+# model.load_state_dict(th.load("upsample.pt", map_location=device))
+# print('total upsampler parameters', sum(x.numel() for x in model_up.parameters()))
+
+base_timestep_respacing = '40' #@param {type:"string"}
+
+sr_timestep_respacing = 'fast27'
+
+#@title Create base model.
+glide_path = 'base.pt' #@param {type:"string"}
+import os
+options = model_and_diffusion_defaults()
+options['use_fp16'] = has_cuda
+options['timestep_respacing'] = base_timestep_respacing # use 100 diffusion steps for fast sampling
+model, diffusion = create_model_and_diffusion(**options)
+
+if len(glide_path) > 0:
+    assert os.path.exists(
+        glide_path
+    ), f"Failed to resume from {glide_path}, file does not exist."
+    weights = th.load(glide_path, map_location="cpu")
+    model, diffusion = create_model_and_diffusion(**options)
+    model.load_state_dict(weights)
+    print(f"Resumed from {glide_path} successfully.")
+else:
+    model, diffusion = create_model_and_diffusion(**options)
+    model.load_state_dict(load_checkpoint("base", device))
+model.eval()
+if has_cuda:
+    model.convert_to_fp16()
+model.to(device)
+print('total base parameters', sum(x.numel() for x in model.parameters()))
+
+
+#@title Create upsampler model.
+sr_glide_path = "upsample.pt" #@param {type:"string"}
+
+
+options_up = model_and_diffusion_defaults_upsampler()
+options_up['use_fp16'] = has_cuda
+options_up['timestep_respacing'] = sr_timestep_respacing # use 27 diffusion steps for very fast sampling
+
+if len(sr_glide_path) > 0:
+    assert os.path.exists(
+        sr_glide_path
+    ), f"Failed to resume from {sr_glide_path}, file does not exist."
+    weights = th.load(sr_glide_path, map_location="cpu")
+    model_up, diffusion_up = create_model_and_diffusion(**options_up)
+    model_up.load_state_dict(weights)
+    print(f"Resumed from {sr_glide_path} successfully.")
+else:
+    model_up, diffusion_up = create_model_and_diffusion(**options)
+    model_up.load_state_dict(load_checkpoint("upsample", device))
+
+if has_cuda:
+    model_up.convert_to_fp16()
+model_up.to(device)
+print('total upsampler parameters', sum(x.numel() for x in model_up.parameters()))
+
+
+
+
+def get_images(batch: th.Tensor):
+    """ Display a batch of images inline. """
+    scaled = ((batch + 1)*127.5).round().clamp(0,255).to(th.uint8).cpu()
+    reshaped = scaled.permute(2, 0, 3, 1).reshape([batch.shape[2], -1, 3])
+    return Image.fromarray(reshaped.numpy())
+
+
+# Create a classifier-free guidance sampling function
+guidance_scale = 3.0
+
+def model_fn(x_t, ts, **kwargs):
+    half = x_t[: len(x_t) // 2]
+    combined = th.cat([half, half], dim=0)
+    model_out = model(combined, ts, **kwargs)
+    eps, rest = model_out[:, :3], model_out[:, 3:]
+    cond_eps, uncond_eps = th.split(eps, len(eps) // 2, dim=0)
+    half_eps = uncond_eps + guidance_scale * (cond_eps - uncond_eps)
+    eps = th.cat([half_eps, half_eps], dim=0)
+    return th.cat([eps, rest], dim=1)
+
+
+# @app.get("/")
+def read_root():
+    return {"glide!"}
+
+# @app.get("/{generate}")
+def sample(prompt):
+    # Sampling parameters
+    batch_size = 1
+
+    # Tune this parameter to control the sharpness of 256x256 images.
+    # A value of 1.0 is sharper, but sometimes results in grainy artifacts.
+    upsample_temp = 0.997
+
+    ##############################
+    # Sample from the base model #
+    ##############################
+
+    # Create the text tokens to feed to the model.
+    tokens = model.tokenizer.encode(prompt)
+    tokens, mask = model.tokenizer.padded_tokens_and_mask(
+        tokens, options['text_ctx']
+    )
+
+    # Create the classifier-free guidance tokens (empty)
+    full_batch_size = batch_size * 2
+    uncond_tokens, uncond_mask = model.tokenizer.padded_tokens_and_mask(
+        [], options['text_ctx']
+    )
+
+    # Pack the tokens together into model kwargs.
+    model_kwargs = dict(
+        tokens=th.tensor(
+            [tokens] * batch_size + [uncond_tokens] * batch_size, device=device
+        ),
+        mask=th.tensor(
+            [mask] * batch_size + [uncond_mask] * batch_size,
+            dtype=th.bool,
+            device=device,
+        ),
+    )
+
+    # Sample from the base model.
+    model.del_cache()
+    samples = diffusion.p_sample_loop(
+        model_fn,
+        (full_batch_size, 3, options["image_size"], options["image_size"]),
+        device=device,
+        clip_denoised=True,
+        progress=True,
+        model_kwargs=model_kwargs,
+        cond_fn=None,
+    )[:batch_size]
+    model.del_cache()
+
+
+    ##############################
+    # Upsample the 64x64 samples #
+    ##############################
+
+    tokens = model_up.tokenizer.encode(prompt)
+    tokens, mask = model_up.tokenizer.padded_tokens_and_mask(
+        tokens, options_up['text_ctx']
+    )
+
+    # Create the model conditioning dict.
+    model_kwargs = dict(
+        # Low-res image to upsample.
+        low_res=((samples+1)*127.5).round()/127.5 - 1,
+
+        # Text tokens
+        tokens=th.tensor(
+            [tokens] * batch_size, device=device
+        ),
+        mask=th.tensor(
+            [mask] * batch_size,
+            dtype=th.bool,
+            device=device,
+        ),
+    )
+
+    # Sample from the base model.
+    model_up.del_cache()
+    up_shape = (batch_size, 3, options_up["image_size"], options_up["image_size"])
+    up_samples = diffusion_up.ddim_sample_loop(
+        model_up,
+        up_shape,
+        noise=th.randn(up_shape, device=device) * upsample_temp,
+        device=device,
+        clip_denoised=True,
+        progress=True,
+        model_kwargs=model_kwargs,
+        cond_fn=None,
+    )[:batch_size]
+    model_up.del_cache()
+
+    # Show the output
+    image = get_images(up_samples)
+    # image = to_base64(image)
+    # return {"image": image}
+    return image
+
+
+def to_base64(pil_image):
+    buffered = BytesIO()
+    pil_image.save(buffered, format="JPEG")
+    return base64.b64encode(buffered.getvalue())
+
+title = "Interactive demo: glide-text2im"
+description = "Demo for OpenAI's GLIDE: Towards Photorealistic Image Generation and Editing with Text-Guided Diffusion Models."
+article = "<p style='text-align: center'><a href='https://arxiv.org/abs/2112.10741'>GLIDE: Towards Photorealistic Image Generation and Editing with Text-Guided Diffusion Models</a> | <a href='https://github.com/openai/glide-text2im/'>Official Repo</a></p>"
+examples =["an oil painting of a corgi"]
+
+iface = gr.Interface(fn=sample, 
+                     inputs=gr.inputs.Textbox(label='What would you like to see?'), 
+                     outputs=gr.outputs.Image(type="pil", label="Model input + completions"),
+                     title=title,
+                     description=description,
+                     article=article,
+                     examples=examples,
+                     enable_queue=True)
+iface.launch(debug=True)

model-card.md

@@ -0,0 +1,50 @@
+# Overview
+
+This card describes the diffusion model GLIDE (filtered) and noised CLIP model described in the paper [GLIDE: Towards
+Photorealistic Image Generation and Editing with Text-Guided Diffusion Models](https://arxiv.org/abs/2112.10741)
+
+# Datasets
+
+GLIDE (filtered) was trained on a filtered version of a dataset comprised of several hundred million text-image pairs
+collected from the internet. We constructed a set of filters intended to remove all images of people, violent objects, and some
+and hate symbols (see Appendix F of the paper for details). The size of the dataset after filtering was approximately
+67M text-image pairs.
+
+Our noised CLIP model which was trained on the dataset described above, augmented with a filtered version of the dataset used
+to train the [original CLIP models](https://github.com/openai/clip). The total size of this augmented dataset is approximately 137M pairs.
+
+# Performance
+
+Qualitatively, we find that the generated images from GLIDE (filtered) often look semi-realistic, but the small size of the model hinders
+its ability to bind attributes to objects and perform compositional tasks. Because the dataset used to train GLIDE
+(filtered) has been preprocessed to remove images of people, this also limits its world knowledge, especially in regard
+to concepts that involve people.
+Finally, due to the dataset used to train GLIDE (filtered), the model has reduced capabilities to compose multiple objects in complex ways compared to models of a similar size trained on our internal dataset.
+
+We do not directly measure quantitative metrics for GLIDE (filtered). In particular, most of the evaluations we report for our other models are biased against GLIDE (filtered), since they use prompts that often require generations of people. Evaluating people-free models remains an open area of research.
+
+# Intended Use
+
+We release these models to help advance research in generative modeling. Due to the limitations and biases of GLIDE (filtered), we do not currently recommend it for commercial use.
+
+Functionally, these models are intended to be able to perform the following tasks for research purposes:
+ * Generate images from natural language prompts
+ * Iteratively edit and refine images using inpainting
+
+These models are explicitly not intended to generate images of people or other subjects we filtered for (see Appendix F of the paper for details).
+
+# Limitations
+
+Despite the dataset filtering applied before training, GLIDE (filtered) continues to exhibit biases that extend beyond those found in images of people.
+We explore some of these biases in our paper. For example:
+
+  * It produces different outputs when asked to generate toys for boys and toys for girls.
+  * It gravitates toward generating images of churches when asked to generate "a religious place",
+    and this bias is amplified by classifier-free guidance.
+  * It may have a greater propensity for generating hate symbols other than swastikas and confederate flags. Our filter
+    for hate symbols focused specifically on these two cases, as we found few relevant images of hate symbols in our
+    dataset. However, we also found that the model has diminished capabilities across a wider set of symbols.
+
+GLIDE (filtered) can fail to produce realistic outputs for complex prompts or for prompts that involve concepts that are
+not well-represented in its training data. While the data for the model was filtered to remove certain types of images,
+the data still exhibits biases toward Western-centric concepts.

requirements.txt

@@ -0,0 +1,5 @@
+git+https://github.com/openai/glide-text2im.git
+fastapi
+uvicorn
+regex
+git-lfs

server.py

@@ -0,0 +1,175 @@
+import base64
+from io import BytesIO
+from fastapi import FastAPI
+
+from PIL import Image
+import torch as th
+
+from glide_text2im.download import load_checkpoint
+from glide_text2im.model_creation import (
+    create_model_and_diffusion,
+    model_and_diffusion_defaults,
+    model_and_diffusion_defaults_upsampler
+)
+
+print("Loading models...")
+app = FastAPI()
+
+# This notebook supports both CPU and GPU.
+# On CPU, generating one sample may take on the order of 20 minutes.
+# On a GPU, it should be under a minute.
+
+has_cuda = th.cuda.is_available()
+device = th.device('cpu' if not has_cuda else 'cuda')
+
+# Create base model.
+options = model_and_diffusion_defaults()
+options['use_fp16'] = has_cuda
+options['timestep_respacing'] = '100' # use 100 diffusion steps for fast sampling
+model, diffusion = create_model_and_diffusion(**options)
+model.eval()
+if has_cuda:
+    model.convert_to_fp16()
+model.to(device)
+model.load_state_dict(load_checkpoint('base', device))
+print('total base parameters', sum(x.numel() for x in model.parameters()))
+
+# Create upsampler model.
+options_up = model_and_diffusion_defaults_upsampler()
+options_up['use_fp16'] = has_cuda
+options_up['timestep_respacing'] = 'fast27' # use 27 diffusion steps for very fast sampling
+model_up, diffusion_up = create_model_and_diffusion(**options_up)
+model_up.eval()
+if has_cuda:
+    model_up.convert_to_fp16()
+model_up.to(device)
+model_up.load_state_dict(load_checkpoint('upsample', device))
+print('total upsampler parameters', sum(x.numel() for x in model_up.parameters()))
+
+
+def get_images(batch: th.Tensor):
+    """ Display a batch of images inline. """
+    scaled = ((batch + 1)*127.5).round().clamp(0,255).to(th.uint8).cpu()
+    reshaped = scaled.permute(2, 0, 3, 1).reshape([batch.shape[2], -1, 3])
+    Image.fromarray(reshaped.numpy())
+
+
+# Create a classifier-free guidance sampling function
+guidance_scale = 3.0
+
+def model_fn(x_t, ts, **kwargs):
+    half = x_t[: len(x_t) // 2]
+    combined = th.cat([half, half], dim=0)
+    model_out = model(combined, ts, **kwargs)
+    eps, rest = model_out[:, :3], model_out[:, 3:]
+    cond_eps, uncond_eps = th.split(eps, len(eps) // 2, dim=0)
+    half_eps = uncond_eps + guidance_scale * (cond_eps - uncond_eps)
+    eps = th.cat([half_eps, half_eps], dim=0)
+    return th.cat([eps, rest], dim=1)
+
+
+@app.get("/")
+def read_root():
+    return {"glide!"}
+
+@app.get("/{generate}")
+def sample(prompt):
+    # Sampling parameters
+    batch_size = 1
+
+    # Tune this parameter to control the sharpness of 256x256 images.
+    # A value of 1.0 is sharper, but sometimes results in grainy artifacts.
+    upsample_temp = 0.997
+
+    ##############################
+    # Sample from the base model #
+    ##############################
+
+    # Create the text tokens to feed to the model.
+    tokens = model.tokenizer.encode(prompt)
+    tokens, mask = model.tokenizer.padded_tokens_and_mask(
+        tokens, options['text_ctx']
+    )
+
+    # Create the classifier-free guidance tokens (empty)
+    full_batch_size = batch_size * 2
+    uncond_tokens, uncond_mask = model.tokenizer.padded_tokens_and_mask(
+        [], options['text_ctx']
+    )
+
+    # Pack the tokens together into model kwargs.
+    model_kwargs = dict(
+        tokens=th.tensor(
+            [tokens] * batch_size + [uncond_tokens] * batch_size, device=device
+        ),
+        mask=th.tensor(
+            [mask] * batch_size + [uncond_mask] * batch_size,
+            dtype=th.bool,
+            device=device,
+        ),
+    )
+
+    # Sample from the base model.
+    model.del_cache()
+    samples = diffusion.p_sample_loop(
+        model_fn,
+        (full_batch_size, 3, options["image_size"], options["image_size"]),
+        device=device,
+        clip_denoised=True,
+        progress=True,
+        model_kwargs=model_kwargs,
+        cond_fn=None,
+    )[:batch_size]
+    model.del_cache()
+
+
+    ##############################
+    # Upsample the 64x64 samples #
+    ##############################
+
+    tokens = model_up.tokenizer.encode(prompt)
+    tokens, mask = model_up.tokenizer.padded_tokens_and_mask(
+        tokens, options_up['text_ctx']
+    )
+
+    # Create the model conditioning dict.
+    model_kwargs = dict(
+        # Low-res image to upsample.
+        low_res=((samples+1)*127.5).round()/127.5 - 1,
+
+        # Text tokens
+        tokens=th.tensor(
+            [tokens] * batch_size, device=device
+        ),
+        mask=th.tensor(
+            [mask] * batch_size,
+            dtype=th.bool,
+            device=device,
+        ),
+    )
+
+    # Sample from the base model.
+    model_up.del_cache()
+    up_shape = (batch_size, 3, options_up["image_size"], options_up["image_size"])
+    up_samples = diffusion_up.ddim_sample_loop(
+        model_up,
+        up_shape,
+        noise=th.randn(up_shape, device=device) * upsample_temp,
+        device=device,
+        clip_denoised=True,
+        progress=True,
+        model_kwargs=model_kwargs,
+        cond_fn=None,
+    )[:batch_size]
+    model_up.del_cache()
+
+    # Show the output
+    image = get_images(up_samples)
+    image = to_base64(image)
+    return {"image": image}
+
+
+def to_base64(pil_image):
+    buffered = BytesIO()
+    pil_image.save(buffered, format="JPEG")
+    return base64.b64encode(buffered.getvalue())

setup.py

@@ -0,0 +1,15 @@
+from setuptools import setup
+
+setup(
+    name="glide-text2im",
+    packages=["glide_text2im"],
+    install_requires=[
+        "Pillow",
+        "attrs",
+        "torch",
+        "filelock",
+        "requests",
+        "tqdm",
+    ],
+    author="OpenAI",
+)