initial commit

app.py

@@ -0,0 +1,114 @@
+import pandas as pd
+import numpy as np
+from PIL import Image
+import torch
+import torchvision
+import clip
+import matplotlib.pyplot as plt
+import seaborn as sns
+import gradio as gr
+
+
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+model_name = 'ViT-B/16' #@param  ['RN50', 'RN101', 'RN50x4', 'RN50x16', 'ViT-B/32', 'ViT-B/16']
+model, preprocess = clip.load(model_name)
+
+model.to(DEVICE).eval()
+resolution = model.visual.input_resolution
+resizer = torchvision.transforms.Resize(size=(resolution, resolution))
+
+
+def create_rgb_tensor(color):
+  """color is e.g. [1,0,0]"""
+  return torch.tensor(color, device=DEVICE).reshape((1, 3, 1, 1))
+
+def encode_color(color):
+  """color is e.g. [1,0,0]"""
+  rgb = create_rgb_tensor(color)
+  return model.encode_image( resizer(rgb) )
+
+def encode_text(text):
+  tokenized_text = clip.tokenize(text).cuda()
+  return model.encode_text(tokenized_text)
+
+def lerp(x, y, steps=11):
+  """Linear interpolation between two tensors """
+
+  weights = torch.tensor(np.linspace(0,1,steps), device=DEVICE).reshape([-1, 1, 1, 1])
+
+  interpolated = x * (1 - weights) + y * weights
+
+  return interpolated
+
+def get_interpolated_scores(x, y, encoded_text, steps=11):
+  interpolated = lerp(x, y, steps)
+  interpolated_encodings = model.encode_image(resizer(interpolated))
+
+  scores = torch.cosine_similarity(interpolated_encodings, encoded_text).detach().cpu().numpy()
+
+  rgb = interpolated.detach().cpu().numpy().reshape(-1,  3)
+  interpolated_hex = [rgb2hex(x) for x in rgb]  
+
+  data = pd.DataFrame({
+      'similarity': scores,
+      'color': interpolated_hex
+  }).reset_index().rename(columns={'index':'step'})
+
+  return data
+
+def rgb2hex(rgb):
+    rgb = (rgb * 255).astype(int)
+    r,g,b = rgb
+    return "#{:02x}{:02x}{:02x}".format(r,g,b)
+
+
+def similarity_plot(data, text_prompt):
+  title = f'CLIP Cosine Similarity Prompt="{text_prompt}"'
+  
+  fig, ax = plt.subplots()
+  plot = data['similarity'].plot(kind='bar',
+                                 ax=ax,
+                                 stacked=True,
+                                 title=title,
+                                 color=data['color'],
+                                 width=1.0,
+                                 xlim=(0, 2),
+                                 grid=False)
+
+  
+  plot.get_xaxis().set_visible(False) ; 
+  return fig
+
+
+
+def interpolation_experiment(rgb_start, rgb_end, text_prompt, steps=11):
+
+  start = create_rgb_tensor(rgb_start)
+  end = create_rgb_tensor(rgb_end)
+  encoded_text = encode_text(text_prompt)
+
+  data = get_interpolated_scores(start, end, encoded_text, steps)
+  return similarity_plot(data, text_prompt)
+
+
+
+
+start_input = gr.inputs.Textbox(lines=1, default="1, 0, 0", label="Start RGB")
+end_input = gr.inputs.Textbox(lines=1, default="0, 1, 0", label="End RGB")
+' (Comma separated numbers between 0 and 1)'
+
+text_input = gr.inputs.Textbox(lines=1, label="Text Prompt", default='A solid red square')
+
+steps_input = gr.inputs.Slider(minimum=1, maximum=100, step=1, default=11, label="Interpolation Steps")
+
+def gradio_fn(rgb_start, rgb_end, text_prompt, steps=11):
+
+  rgb_start = [float(x.strip()) for x in rgb_start.split(',')]
+  rgb_end =  [float(x.strip()) for x in rgb_end.split(',')]
+  out = interpolation_experiment(rgb_start, rgb_end, text_prompt, steps)
+
+  return out
+
+iface = gr.Interface( fn=gradio_fn, inputs=[start_input, end_input, text_input, steps_input], outputs="plot")
+iface.launch(debug=True, share=False)

requirements.txt

@@ -0,0 +1,11 @@
+comet_ml
+ftfy
+regex
+git+https://github.com/openai/CLIP.git
+pandas
+Pillow
+tqdm
+torch
+torchvision
+matplotlib
+seaborn