Initial Demo.

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.faiss filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+artifacts/ filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,163 @@
+# Custom 
+_*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

README.md

@@ -0,0 +1,11 @@
+---
+title: CondViT LRVSF Demo
+emoji: 🔎
+colorFrom: red
+colorTo: yellow
+sdk: gradio
+sdk_version: 3.32.0
+app_file: app.py
+pinned: false
+license: mit
+---

app.py

@@ -0,0 +1,89 @@
+import pandas as pd
+import torch
+import faiss
+import gradio as gr
+
+import base64
+
+from PIL import Image
+from io import BytesIO
+
+from src.model import ConditionalViT, B16_Params, categories
+from src.transform import valid_tf
+from src.process_images import process_img, make_img_html
+from src.examples import ExamplesHandler
+from src.js_loader import JavaScriptLoader
+
+# Load Model
+m = ConditionalViT(**B16_Params, n_categories=len(categories))
+m.load_state_dict(torch.load("./artifacts/cat_condvit_b16.pth", map_location="cpu"))
+m.eval()
+
+# Load data
+index = faiss.read_index("./artifacts/gallery_index.faiss")
+gal_imgs = pd.read_parquet("./artifacts/gallery_imgs.parquet")
+tfs = valid_tf((224, 224))
+
+K = 5
+
+examples = [
+    ["examples/3.jpg", "Outwear"],
+    ["examples/3.jpg", "Lower Body"],
+    ["examples/3.jpg", "Feet"],
+    ["examples/757.jpg", "Bags"],
+    ["examples/757.jpg", "Upper Body"],
+    ["examples/769.jpg", "Upper Body"],
+    ["examples/1811.jpg", "Lower Body"],
+    ["examples/1811.jpg", "Bags"],
+]
+
+@torch.inference_mode()
+def retrieval(image, category):
+    if image is None or category is None: return
+
+    q_emb = m(tfs(image).unsqueeze(0), torch.tensor([category]))
+
+    r = index.search(q_emb, K)
+
+    imgs = [process_img(idx, gal_imgs) for idx in r[1][0]]
+
+    html = [make_img_html(i) for i in imgs]
+    html += ["<p></p>"] # Avoid Gradio's last-child{margin-bottom:0!important;}
+
+    return "\n".join(html)
+
+
+
+JavaScriptLoader("src/custom_functions.js")
+with gr.Blocks(css="src/style.css") as demo:
+    with gr.Column():
+        gr.Markdown("""
+        # Conditional ViT Demo 
+        [[`Paper`](https://arxiv.org/abs/2306.02928)] 
+        [[`Code`](https://github.com/Simon-Lepage/CondViT-LRVSF)] 
+        [[`Dataset`](https://huggingface.co/datasets/Slep/LAION-RVS-Fashion)] 
+
+        *Running on 2 vCPU, 16Go RAM.*
+
+        - **Model :** Categorical CondViT-B/16
+        - **Gallery :** 93K images.
+        """)
+
+        # Input section
+        with gr.Row():
+            img = gr.Image(label="Query Image", type="pil", elem_id="query_img")
+            with gr.Column():
+                cat = gr.Dropdown(choices = categories, label="Category", value="Upper Body", type='index', elem_id="dropdown")
+                submit = gr.Button("Submit")
+        
+        # Examples
+        gr.Examples(examples, inputs=[img, cat], fn=retrieval, elem_id = "preset_examples", examples_per_page=100)
+        gr.HTML(value=ExamplesHandler(examples).to_html(), label = "examples", elem_id = "html_examples")
+        
+        # Outputs
+        gr.Markdown("# Retrieved Items")
+        out = gr.HTML(label="Results", elem_id = "html_output")
+
+        submit.click(fn=retrieval, inputs=[img, cat], outputs=out)
+
+demo.launch()

artifacts/cat_condvit_b16.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:badeeedf937ce955946ebbbbe7a1d531b171374e8fc58697436e7dfca31aa694
+size 344860235

artifacts/gallery_imgs.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f633c261f353bd64498c267a7d6744c424471a4f09f69f2ac8f9df5537a259e
+size 1487433539

artifacts/gallery_index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ed82e5e12fb2c8d6aaa89336623fc5948f7e29805f961f0e07ce92b0cebd8c2
+size 192118829

requirements.txt

@@ -0,0 +1,7 @@
+torch
+torchvision
+pillow
+faiss-cpu
+pandas
+pyarrow
+gradio

src/custom_functions.js

@@ -0,0 +1,5 @@
+remap_click = (i) => {
+    let examples = document.getElementById("preset_examples");    
+    let rows = examples.getElementsByTagName("tbody")[0].getElementsByTagName("tr");
+    rows[i].click();
+}

src/custom_js.py

@@ -0,0 +1,27 @@
+import gradio
+
+# Adapted from https://github.com/gradio-app/gradio/discussions/2932
+
+class JavaScriptLoader:
+    def __init__(self, target):
+        #Copy the template response
+        self.original_template = gradio.routes.templates.TemplateResponse
+        #Prep the js files
+        self.load_js(target)
+        #reassign the template response to your method, so gradio calls your method instead
+        gradio.routes.templates.TemplateResponse = self.template_response
+
+    def load_js(self, target):
+        with open(target, 'r', encoding="utf-8") as file:
+                self.loaded_script = f"<script>\n{file.read()}\n</script>"
+
+    def template_response(self, *args, **kwargs):
+        """Once gradio calls your method, you call the original, you modify it to include
+        your scripts and you return the modified version
+        """
+        response = self.original_template(*args, **kwargs)
+        response.body = response.body.replace(
+            '</head>'.encode('utf-8'), self.loaded_script + "\n</head>".encode("utf-8")
+        )
+        response.init_headers()
+        return response

src/examples.py

@@ -0,0 +1,21 @@
+from PIL import Image
+from .process_images import make_img_html
+
+
+class ExamplesHandler:
+    def __init__(self, examples):
+        self.examples = examples 
+
+    def to_html(self):
+
+        ret = ""
+        for i, (img_path, category) in enumerate(self.examples): 
+            ret += f"<figure id='example_{i}' onclick='remap_click({i})'>"
+            img = Image.open(img_path).convert("RGB")
+            ret += make_img_html(img)
+            ret += f"<figcaption>{category}</figcaption>"
+            ret += "</figure>"
+
+        ret += "<p></p>"
+
+        return ret

src/js_loader.py

@@ -0,0 +1,25 @@
+import gradio
+
+class JavaScriptLoader:
+    def __init__(self, target):
+        #Copy the template response
+        self.original_template = gradio.routes.templates.TemplateResponse
+        #Prep the js files
+        self.load_js(target)
+        #reassign the template response to your method, so gradio calls your method instead
+        gradio.routes.templates.TemplateResponse = self.template_response
+
+    def load_js(self, target):
+        with open(target, 'r', encoding="utf-8") as file:
+                self.loaded_script = f"<script>\n{file.read()}\n</script>"
+
+    def template_response(self, *args, **kwargs):
+        """Once gradio calls your method, you call the original, you modify it to include
+        your scripts and you return the modified version
+        """
+        response = self.original_template(*args, **kwargs)
+        response.body = response.body.replace(
+            '</head>'.encode('utf-8'), (self.loaded_script + "\n</head>").encode("utf-8")
+        )
+        response.init_headers()
+        return response

src/model.py

@@ -0,0 +1,213 @@
+categories = [
+    "Bags",
+    "Feet",
+    "Hands",
+    "Head",
+    "Lower Body",
+    "Neck",
+    "Outwear",
+    "Upper Body",
+    "Waist",
+    "Whole Body",
+]
+
+import torch
+from torch import nn
+
+from collections import OrderedDict
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class LayerNorm(nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+
+    def forward(self, x: torch.Tensor):
+        if self.weight.dtype != x.dtype:
+            orig_type = x.dtype
+            ret = super().forward(x.type(self.weight.dtype))
+            return ret.type(orig_type)
+        else:
+            return super().forward(x)
+
+
+class QuickGELU(nn.Module):
+    def forward(self, x: torch.Tensor):
+        return x * torch.sigmoid(1.702 * x)
+
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(
+        self,
+        d_model: int,
+        n_head: int,
+        attn_mask: torch.Tensor = None,
+    ):
+        super().__init__()
+
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = LayerNorm(d_model)
+        self.mlp = nn.Sequential(
+            OrderedDict(
+                [
+                    (
+                        "c_fc",
+                        nn.Linear(d_model, d_model * 4),
+                    ),
+                    ("gelu", QuickGELU()),
+                    (
+                        "c_proj",
+                        nn.Linear(d_model * 4, d_model),
+                    ),
+                ]
+            )
+        )
+        self.ln_2 = LayerNorm(d_model)
+        self.attn_mask = attn_mask
+
+    def attention(self, x: torch.Tensor):
+        self.attn_mask = (
+            self.attn_mask.to(dtype=x.dtype, device=x.device)
+            if self.attn_mask is not None
+            else None
+        )
+        return self.attn(
+            x,
+            x,
+            x,
+            need_weights=False,
+            attn_mask=self.attn_mask,
+        )[0]
+
+    def forward(self, x: torch.Tensor):
+        x = x + self.attention(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+
+class Transformer(nn.Module):
+    def __init__(
+        self,
+        width: int,
+        layers: int,
+        heads: int,
+        attn_mask: torch.Tensor = None,
+    ):
+        super().__init__()
+        self.width = width
+        self.layers = layers
+        self.resblocks = nn.Sequential(
+            *[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)]
+        )
+
+    def forward(self, x: torch.Tensor):
+        return self.resblocks(x)
+
+
+class ConditionalViT(nn.Module):
+    def __init__(
+        self,
+        input_resolution: int,
+        patch_size: int,
+        width: int,
+        layers: int,
+        heads: int,
+        output_dim: int,
+        n_categories: int = None,
+        **kwargs,
+    ):
+        if kwargs:
+            logger.warning(f"Got unused kwargs : {kwargs}")
+
+        super().__init__()
+        self.input_resolution = input_resolution
+        self.output_dim = output_dim
+        self.conv1 = nn.Conv2d(
+            in_channels=3,
+            out_channels=width,
+            kernel_size=patch_size,
+            stride=patch_size,
+            bias=False,
+        )
+
+        scale = width**-0.5
+
+        self.class_embedding = nn.Parameter(scale * torch.randn(width))
+
+        self.n_categories = n_categories
+        if self.n_categories:
+            self.c_embedding = nn.Embedding(self.n_categories, width)
+            self.c_pos_embedding = nn.Parameter(scale * torch.randn(1, width))
+
+        self.positional_embedding = nn.Parameter(
+            scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width)
+        )
+        self.ln_pre = LayerNorm(width)
+
+        self.transformer = Transformer(width, layers, heads)
+        self.ln_post = LayerNorm(width)
+        self.logit_scale = torch.nn.Parameter(torch.ones([]) * 4.6052)
+
+        self.proj = nn.Parameter(scale * torch.randn(width, output_dim))
+
+    def forward(self, imgs: torch.Tensor, c: torch.Tensor = None):
+        """
+        imgs : Batch of images
+        c : category indices. 0 = "No given category".
+        """
+
+        x = self.conv1(imgs)  # shape = [*, width, grid, grid]
+        # shape = [*, width, grid ** 2]
+        x = x.reshape(x.shape[0], x.shape[1], -1)
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+
+        # [CLS, grid] + maybe Categories.
+        tokens = [self.class_embedding.tile(x.shape[0], 1, 1), x]  # NLD
+        pos_embed = [self.positional_embedding]  # LD
+
+        if self.n_categories and c is not None:  # If c is None, we don't add the token
+            tokens += [self.c_embedding(c).unsqueeze(1)]  # ND -> N1D
+            pos_embed += [self.c_pos_embedding]  # 1D
+
+        x = torch.cat(
+            tokens,
+            dim=1,
+        )  # shape = [*, grid ** 2 + 1|2, width] = N(L|L+1)D
+        pos_embed = torch.cat(pos_embed, dim=0).unsqueeze(0)  # 1(L|L+1)D
+
+        x = x + pos_embed
+        x = self.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        x = self.ln_post(x[:, 0, :])
+
+        x = x @ self.proj
+
+        return x
+
+
+# SIZES
+B32_Params = {
+    "input_resolution": 224,
+    "patch_size": 32,
+    "width": 768,
+    "layers": 12,
+    "heads": 12,
+    "output_dim": 512,
+}
+
+B16_Params = {
+    "input_resolution": 224,
+    "patch_size": 16,
+    "width": 768,
+    "layers": 12,
+    "heads": 12,
+    "output_dim": 512,
+}
+
+params = {"B32": B32_Params, "B16": B16_Params}

src/process_images.py

@@ -0,0 +1,15 @@
+from PIL import Image 
+from io import BytesIO
+import base64
+
+# Index to PIL
+def process_img(idx, ds):
+    img = Image.open(BytesIO(ds.iloc[idx].jpg)).convert("RGB")
+    return img
+
+def make_img_html(img):
+    b = BytesIO()
+    img.save(b, format='PNG')
+    buffer = b.getvalue()
+
+    return f'<img height=200px src="data:image/png;base64,{base64.b64encode(buffer).decode()}"></img>'

src/style.css

@@ -0,0 +1,42 @@
+/* OUTPUT */
+#html_output, #html_examples {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    flex-wrap: wrap;
+}
+
+#html_output > img {
+    align-self: center;
+    height:200px;
+    border: 2px solid;
+    border-color: var(--block-border-color);
+    border-radius: var(--block-radius);
+    margin:1.5em;
+}
+
+/* EXAMPLE */
+#html_examples > figure > img {
+    align-self: center;
+    height: 100px;
+    border: 2px solid;
+    border-color: var(--block-border-color);
+    border-radius: var(--block-radius);
+    margin:.7em;
+}
+
+#html_examples > figure {
+    transition-duration: 0.2s;
+}
+
+#html_examples > figure:hover{
+    transform: scale(1.2);
+}
+
+#html_examples > figure > figcaption {
+    text-align: center;    
+}
+
+#preset_examples {
+    display: none;
+}

src/transform.py

@@ -0,0 +1,29 @@
+from torchvision.transforms import transforms as tf
+import torchvision.transforms.functional as F
+
+
+class SquarePad:
+    def __init__(self, color):
+        self.col = color
+
+    def __call__(self, image):
+        max_wh = max(image.size)
+        p_left, p_top = [(max_wh - s) // 2 for s in image.size]
+        p_right, p_bottom = [
+            max_wh - (s + pad) for s, pad in zip(image.size, [p_left, p_top])
+        ]
+        padding = (p_left, p_top, p_right, p_bottom)
+        return F.pad(image, padding, self.col, "constant")
+
+def valid_tf(size):
+    return tf.Compose(
+        [
+            SquarePad(255),
+            tf.Resize(size),
+            tf.ToTensor(),
+            tf.Normalize(
+                mean=(0.48145466, 0.4578275, 0.40821073),
+                std=(0.26862954, 0.26130258, 0.27577711),
+            ),
+        ]
+    )