| import pathlib | |
| import gradio as gr | |
| import open_clip | |
| import torch | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| model, _, transform = open_clip.create_model_and_transforms( | |
| "coca_ViT-L-14", | |
| pretrained="mscoco_finetuned_laion2B-s13B-b90k" | |
| ) | |
| model.to(device) | |
| def output_generate(image): | |
| im = transform(image).unsqueeze(0).to(device) | |
| with torch.no_grad(), torch.cuda.amp.autocast(): | |
| generated = model.generate(im, seq_len=20) | |
| return open_clip.decode(generated[0].detach()).split("<end_of_text>")[0].replace("<start_of_text>", "") | |
| paths = sorted(pathlib.Path("images").glob("*.jpg")) | |
| iface = gr.Interface( | |
| fn=output_generate, | |
| inputs=gr.Image(label="Input image", type="pil"), | |
| outputs=gr.Text(label="Caption output"), | |
| title="CoCa: Contrastive Captioners", | |
| description=( | |
| "An open source implementation of **Contrastive Captioners** https://arxiv.org/abs/2205.01917. " | |
| "Built using [open_clip](https://github.com/mlfoundations/open_clip) with an effort from [LAION](https://laion.ai/)" | |
| " and the support of [Stability AI](https://stability.ai/)" | |
| ), | |
| examples=[path.as_posix() for path in paths], | |
| ) | |
| iface.launch() | |